diff --git a/.bazelignore b/.bazelignore deleted file mode 100644 index ece29c3646955..0000000000000 --- a/.bazelignore +++ /dev/null @@ -1 +0,0 @@ -common/.kunit diff --git a/.gitignore b/.gitignore index d1a8ab3f98aaf..23883fd8f7696 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,7 @@ sphinx_*/ # Rust analyzer configuration /rust-project.json +drivers/soc/oplus/ +kernel/locking/locking_main.h +kernel/locking/oplus_locking.c +kernel/locking/sa_common_struct.h diff --git a/BUILD.bazel b/BUILD.bazel old mode 100644 new mode 100755 index 783df99093a3d..0b317cdea2360 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -141,7 +141,8 @@ filegroup( "android/abi_gki_aarch64_virtual_device", "android/abi_gki_aarch64_vivo", "android/abi_gki_aarch64_xiaomi", - "android/abi_gki_aarch64_xiaomi2", + "android/abi_gki_aarch64_xiaomi_wear", + "android/abi_gki_aarch64_xiaomi_xring", ], visibility = ["//visibility:public"], ) @@ -174,7 +175,7 @@ define_common_kernels(target_configs = { "ddk_headers_archive": ":kernel_aarch64_ddk_headers_archive", "extra_dist": [ ":test_mappings_zip", - ":tests_zip_arm64", + ":tests_zip_arm64_16k", ], }, "kernel_aarch64_autofdo": { @@ -889,6 +890,16 @@ pkg_files( visibility = ["//visibility:private"], ) +pkg_files( + name = "kunit_modules_arm64_16k", + srcs = [ + ":kernel_aarch64_16k/" + e + for e in get_kunit_modules_list("arm64") + ], + prefix = _KUNIT_DIR + "/arm64", + visibility = ["//visibility:private"], +) + pkg_files( name = "kunit_modules_x86_64", srcs = [ @@ -908,6 +919,15 @@ pkg_filegroup( visibility = ["//visibility:private"], ) +pkg_filegroup( + name = "kunit_tests_arm64_16k_pkg_files", + srcs = [ + ":kunit_modules_arm64_16k", + ":kunit_tests_config_arm64", + ], + visibility = ["//visibility:private"], +) + pkg_filegroup( name = "kunit_tests_x86_64_pkg_files", srcs = [ @@ -927,6 +947,14 @@ pkg_install( visibility = ["//visibility:private"], ) +pkg_install( + name = "kunit_tests_arm64_16k", + srcs = [ + ":kunit_tests_arm64_16k_pkg_files", + ], + visibility = ["//visibility:private"], +) + pkg_install( name = "kunit_tests_x86_64", srcs = [ @@ -1025,6 +1053,7 @@ ddk_headers( name = "all_headers_allowlist_aarch64", hdrs = [ "drivers/dma-buf/heaps/deferred-free-helper.h", + "drivers/dma/dmaengine.h", "drivers/extcon/extcon.h", "drivers/pci/controller/dwc/pcie-designware.h", "drivers/thermal/thermal_core.h", @@ -1046,6 +1075,7 @@ ddk_headers( "arch/arm64/include", "arch/arm64/include/uapi", "drivers/dma-buf", + "drivers/dma", "drivers/extcon", "drivers/pci/controller/dwc", "drivers/thermal", @@ -2478,6 +2508,16 @@ pkg_zip( visibility = ["//visibility:public"], ) +pkg_zip( + name = "tests_zip_arm64_16k", + srcs = [ + ":kselftest_tests_arm64_pkg_files", + ":kunit_tests_arm64_16k_pkg_files", + ], + out = "arm64_16k/tests.zip", + visibility = ["//visibility:public"], +) + copy_to_dist_dir( name = "tests_zip_x86_64_dist", data = [":tests_zip_x86_64"], @@ -2492,6 +2532,14 @@ copy_to_dist_dir( flat = True, ) +load(":consolidate.bzl", "define_consolidate") + +define_consolidate() + +load(":consolidate_16k.bzl", "define_consolidate_16k") + +define_consolidate_16k() + _TEST_MAPPINGS = glob(["**/TEST_MAPPING"]) pkg_files( diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index 825e619250bf2..f2ec42949a54d 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -270,6 +270,12 @@ Description: Shows the operation capability bits displayed in bitmap format correlates to the operations allowed. It's visible only on platforms that support the capability. +What: /sys/bus/dsa/devices/wq./driver_name +Date: Sept 8, 2023 +KernelVersion: 6.7.0 +Contact: dmaengine@vger.kernel.org +Description: Name of driver to be bounded to the wq. + What: /sys/bus/dsa/devices/engine./group_id Date: Oct 25, 2019 KernelVersion: 5.6.0 diff --git a/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd b/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd index 5a775b8f65435..fc82aa4e54b00 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd +++ b/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd @@ -75,3 +75,13 @@ Description: The default value is 1 (GNU Remote Debug command). Other permissible value is 0 which is for vendor defined debug target. + +What: /sys/bus/pci/drivers/xhci_hcd/.../dbc_poll_interval_ms +Date: February 2024 +Contact: Mathias Nyman +Description: + This attribute adjust the polling interval used to check for + DbC events. Unit is milliseconds. Accepted values range from 0 + up to 5000. The default value is 64 ms. + This polling interval is used while DbC is enabled but has no + active data transfers. diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 657bdee28d845..cfa393c51f4d6 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -514,6 +514,7 @@ Description: information about CPUs heterogeneity. What: /sys/devices/system/cpu/vulnerabilities /sys/devices/system/cpu/vulnerabilities/gather_data_sampling + /sys/devices/system/cpu/vulnerabilities/indirect_target_selection /sys/devices/system/cpu/vulnerabilities/itlb_multihit /sys/devices/system/cpu/vulnerabilities/l1tf /sys/devices/system/cpu/vulnerabilities/mds @@ -525,7 +526,9 @@ What: /sys/devices/system/cpu/vulnerabilities /sys/devices/system/cpu/vulnerabilities/spectre_v1 /sys/devices/system/cpu/vulnerabilities/spectre_v2 /sys/devices/system/cpu/vulnerabilities/srbds + /sys/devices/system/cpu/vulnerabilities/tsa /sys/devices/system/cpu/vulnerabilities/tsx_async_abort + /sys/devices/system/cpu/vulnerabilities/vmscape Date: January 2018 Contact: Linux kernel mailing list Description: Information about CPU vulnerabilities diff --git a/Documentation/ABI/testing/sysfs-driver-typec-displayport b/Documentation/ABI/testing/sysfs-driver-typec-displayport index 256c87c5219a6..314acd54e13e4 100644 --- a/Documentation/ABI/testing/sysfs-driver-typec-displayport +++ b/Documentation/ABI/testing/sysfs-driver-typec-displayport @@ -62,3 +62,13 @@ Description: by VESA DisplayPort Alt Mode on USB Type-C Standard. - 0 when HPD’s logical state is low (HPD_Low) as defined by VESA DisplayPort Alt Mode on USB Type-C Standard. + +What: /sys/bus/typec/devices/.../displayport/irq_hpd +Date: June 2025 +Contact: RD Babiera +Description: + IRQ_HPD events are sent over the USB PD protocol in Status Update and + Attention messages. IRQ_HPD can only be asserted when HPD is high, + and is asserted when an IRQ_HPD has been issued since the last Status + Update. This is a read only node that returns the number of IRQ events + raised in the driver's lifetime. diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs index 29538f9d352f4..bbbad1969df5c 100644 --- a/Documentation/ABI/testing/sysfs-driver-ufs +++ b/Documentation/ABI/testing/sysfs-driver-ufs @@ -711,7 +711,7 @@ Description: This file shows the thin provisioning type. This is one of The file is read only. -What: /sys/class/scsi_device/*/device/unit_descriptor/physical_memory_resourse_count +What: /sys/class/scsi_device/*/device/unit_descriptor/physical_memory_resource_count Date: February 2018 Contact: Stanislav Nijnikov Description: This file shows the total physical memory resources. This is diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index eec506c44d979..3134bfd05b089 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -867,3 +867,59 @@ Description: This threshold is used to control triggering garbage collection whi reserved section before preallocating on pinned file. By default, the value is ovp_sections, especially, for zoned ufs, the value is 1. + +What: /sys/fs/f2fs//effective_lookup_mode +Date: August 2025 +Contact: "Daniel Lee" +Description: + This is a read-only entry to show the effective directory lookup mode + F2FS is currently using for casefolded directories. + This considers both the "lookup_mode" mount option and the on-disk + encoding flag, SB_ENC_NO_COMPAT_FALLBACK_FL. + + Possible values are: + - "perf": Hash-only lookup. + - "compat": Hash-based lookup with a linear search fallback enabled + - "auto:perf": lookup_mode is auto and fallback is disabled on-disk + - "auto:compat": lookup_mode is auto and fallback is enabled on-disk + +What: /sys/fs/f2fs//max_lock_elapsed_time +Date: December 2025 +Contact: "Chao Yu" +Description: This is a threshold, once a thread enters critical region that lock covers, total + elapsed time exceeds this threshold, f2fs will print tracepoint to dump information + of related context. This sysfs entry can be used to control the value of threshold, + by default, the value is 500 ms. + +What: /sys/fs/f2fs//adjust_lock_priority +Date: January 2026 +Contact: "Chao Yu" +Description: This sysfs entry can be used to enable/disable to adjust priority for task + which is in critical region covered by lock. + ========== ================== + Flag_Value Flag_Description + ========== ================== + 0x00000000 Disabled (default) + 0x00000001 cp_rwsem + 0x00000002 node_change + 0x00000004 node_write + 0x00000008 gc_lock + 0x00000010 cp_global + 0x00000020 io_rwsem + ========== ================== + +What: /sys/fs/f2fs//lock_duration_priority +Date: January 2026 +Contact: "Chao Yu" +Description: f2fs can tune priority of thread which has entered into critical region covered by + f2fs rwsemphore lock. This sysfs entry can be used to control priority value, the + range is [100,139], by default the value is 120. + +What: /sys/fs/f2fs//critical_task_priority +Date: February 2026 +Contact: "Chao Yu" +Description: It can be used to tune priority of f2fs critical task, e.g. f2fs_ckpt, f2fs_gc + threads, limitation as below: + - it requires user has CAP_SYS_NICE capability. + - the range is [100, 139], by default the value is 100. + diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index b26b5274eaaf1..95148b75c8022 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -938,6 +938,24 @@ All cgroup core files are prefixed with "cgroup." A dying cgroup can consume system resources not exceeding limits, which were active at the moment of cgroup deletion. + cgroup.stat.local + A read-only flat-keyed file which exists in non-root cgroups. + The following entry is defined: + + frozen_usec + Cumulative time that this cgroup has spent between freezing and + thawing, regardless of whether by self or ancestor groups. + NB: (not) reaching "frozen" state is not accounted here. + + Using the following ASCII representation of a cgroup's freezer + state, :: + + 1 _____ + frozen 0 __/ \__ + ab cd + + the duration being measured is the span between a and c. + cgroup.freeze A read-write single value file which exists on non-root cgroups. Allowed values are "0" and "1". The default is "0". @@ -1246,17 +1264,10 @@ PAGE_SIZE multiple when read back. This is a simple interface to trigger memory reclaim in the target cgroup. - This file accepts a single key, the number of bytes to reclaim. - No nested keys are currently supported. - Example:: echo "1G" > memory.reclaim - The interface can be later extended with nested keys to - configure the reclaim behavior. For example, specify the - type of memory to reclaim from (anon, file, ..). - Please note that the kernel can over or under reclaim from the target cgroup. If less bytes are reclaimed than the specified amount, -EAGAIN is returned. @@ -1268,6 +1279,17 @@ PAGE_SIZE multiple when read back. This means that the networking layer will not adapt based on reclaim induced by memory.reclaim. +The following nested keys are defined. + + ========== ================================ + swappiness Swappiness value to reclaim with + ========== ================================ + + Specifying a swappiness value instructs the kernel to perform + the reclaim with that swappiness value. Note that this has the + same semantics as vm.swappiness applied to memcg reclaim with + all the existing limitations and potential future extensions. + memory.peak A read-only single value file which exists on non-root cgroups. @@ -1532,6 +1554,15 @@ PAGE_SIZE multiple when read back. collapsing an existing range of pages. This counter is not present when CONFIG_TRANSPARENT_HUGEPAGE is not set. + thp_swpout (npn) + Number of transparent hugepages which are swapout in one piece + without splitting. + + thp_swpout_fallback (npn) + Number of transparent hugepages which were split before swapout. + Usually because failed to allocate some continuous swap space + for the huge page. + memory.numa_stat A read-only nested-keyed file which exists on non-root cgroups. diff --git a/Documentation/admin-guide/hw-vuln/index.rst b/Documentation/admin-guide/hw-vuln/index.rst index ff0b440ef2dc9..5d6c001b8a988 100644 --- a/Documentation/admin-guide/hw-vuln/index.rst +++ b/Documentation/admin-guide/hw-vuln/index.rst @@ -22,3 +22,5 @@ are configurable at compile, boot or run time. srso gather_data_sampling reg-file-data-sampling + indirect-target-selection + vmscape diff --git a/Documentation/admin-guide/hw-vuln/indirect-target-selection.rst b/Documentation/admin-guide/hw-vuln/indirect-target-selection.rst new file mode 100644 index 0000000000000..d9ca64108d233 --- /dev/null +++ b/Documentation/admin-guide/hw-vuln/indirect-target-selection.rst @@ -0,0 +1,168 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Indirect Target Selection (ITS) +=============================== + +ITS is a vulnerability in some Intel CPUs that support Enhanced IBRS and were +released before Alder Lake. ITS may allow an attacker to control the prediction +of indirect branches and RETs located in the lower half of a cacheline. + +ITS is assigned CVE-2024-28956 with a CVSS score of 4.7 (Medium). + +Scope of Impact +--------------- +- **eIBRS Guest/Host Isolation**: Indirect branches in KVM/kernel may still be + predicted with unintended target corresponding to a branch in the guest. + +- **Intra-Mode BTI**: In-kernel training such as through cBPF or other native + gadgets. + +- **Indirect Branch Prediction Barrier (IBPB)**: After an IBPB, indirect + branches may still be predicted with targets corresponding to direct branches + executed prior to the IBPB. This is fixed by the IPU 2025.1 microcode, which + should be available via distro updates. Alternatively microcode can be + obtained from Intel's github repository [#f1]_. + +Affected CPUs +------------- +Below is the list of ITS affected CPUs [#f2]_ [#f3]_: + + ======================== ============ ==================== =============== + Common name Family_Model eIBRS Intra-mode BTI + Guest/Host Isolation + ======================== ============ ==================== =============== + SKYLAKE_X (step >= 6) 06_55H Affected Affected + ICELAKE_X 06_6AH Not affected Affected + ICELAKE_D 06_6CH Not affected Affected + ICELAKE_L 06_7EH Not affected Affected + TIGERLAKE_L 06_8CH Not affected Affected + TIGERLAKE 06_8DH Not affected Affected + KABYLAKE_L (step >= 12) 06_8EH Affected Affected + KABYLAKE (step >= 13) 06_9EH Affected Affected + COMETLAKE 06_A5H Affected Affected + COMETLAKE_L 06_A6H Affected Affected + ROCKETLAKE 06_A7H Not affected Affected + ======================== ============ ==================== =============== + +- All affected CPUs enumerate Enhanced IBRS feature. +- IBPB isolation is affected on all ITS affected CPUs, and need a microcode + update for mitigation. +- None of the affected CPUs enumerate BHI_CTRL which was introduced in Golden + Cove (Alder Lake and Sapphire Rapids). This can help guests to determine the + host's affected status. +- Intel Atom CPUs are not affected by ITS. + +Mitigation +---------- +As only the indirect branches and RETs that have their last byte of instruction +in the lower half of the cacheline are vulnerable to ITS, the basic idea behind +the mitigation is to not allow indirect branches in the lower half. + +This is achieved by relying on existing retpoline support in the kernel, and in +compilers. ITS-vulnerable retpoline sites are runtime patched to point to newly +added ITS-safe thunks. These safe thunks consists of indirect branch in the +second half of the cacheline. Not all retpoline sites are patched to thunks, if +a retpoline site is evaluated to be ITS-safe, it is replaced with an inline +indirect branch. + +Dynamic thunks +~~~~~~~~~~~~~~ +From a dynamically allocated pool of safe-thunks, each vulnerable site is +replaced with a new thunk, such that they get a unique address. This could +improve the branch prediction accuracy. Also, it is a defense-in-depth measure +against aliasing. + +Note, for simplicity, indirect branches in eBPF programs are always replaced +with a jump to a static thunk in __x86_indirect_its_thunk_array. If required, +in future this can be changed to use dynamic thunks. + +All vulnerable RETs are replaced with a static thunk, they do not use dynamic +thunks. This is because RETs get their prediction from RSB mostly that does not +depend on source address. RETs that underflow RSB may benefit from dynamic +thunks. But, RETs significantly outnumber indirect branches, and any benefit +from a unique source address could be outweighed by the increased icache +footprint and iTLB pressure. + +Retpoline +~~~~~~~~~ +Retpoline sequence also mitigates ITS-unsafe indirect branches. For this +reason, when retpoline is enabled, ITS mitigation only relocates the RETs to +safe thunks. Unless user requested the RSB-stuffing mitigation. + +RSB Stuffing +~~~~~~~~~~~~ +RSB-stuffing via Call Depth Tracking is a mitigation for Retbleed RSB-underflow +attacks. And it also mitigates RETs that are vulnerable to ITS. + +Mitigation in guests +^^^^^^^^^^^^^^^^^^^^ +All guests deploy ITS mitigation by default, irrespective of eIBRS enumeration +and Family/Model of the guest. This is because eIBRS feature could be hidden +from a guest. One exception to this is when a guest enumerates BHI_DIS_S, which +indicates that the guest is running on an unaffected host. + +To prevent guests from unnecessarily deploying the mitigation on unaffected +platforms, Intel has defined ITS_NO bit(62) in MSR IA32_ARCH_CAPABILITIES. When +a guest sees this bit set, it should not enumerate the ITS bug. Note, this bit +is not set by any hardware, but is **intended for VMMs to synthesize** it for +guests as per the host's affected status. + +Mitigation options +^^^^^^^^^^^^^^^^^^ +The ITS mitigation can be controlled using the "indirect_target_selection" +kernel parameter. The available options are: + + ======== =================================================================== + on (default) Deploy the "Aligned branch/return thunks" mitigation. + If spectre_v2 mitigation enables retpoline, aligned-thunks are only + deployed for the affected RET instructions. Retpoline mitigates + indirect branches. + + off Disable ITS mitigation. + + vmexit Equivalent to "=on" if the CPU is affected by guest/host isolation + part of ITS. Otherwise, mitigation is not deployed. This option is + useful when host userspace is not in the threat model, and only + attacks from guest to host are considered. + + stuff Deploy RSB-fill mitigation when retpoline is also deployed. + Otherwise, deploy the default mitigation. When retpoline mitigation + is enabled, RSB-stuffing via Call-Depth-Tracking also mitigates + ITS. + + force Force the ITS bug and deploy the default mitigation. + ======== =================================================================== + +Sysfs reporting +--------------- + +The sysfs file showing ITS mitigation status is: + + /sys/devices/system/cpu/vulnerabilities/indirect_target_selection + +Note, microcode mitigation status is not reported in this file. + +The possible values in this file are: + +.. list-table:: + + * - Not affected + - The processor is not vulnerable. + * - Vulnerable + - System is vulnerable and no mitigation has been applied. + * - Vulnerable, KVM: Not affected + - System is vulnerable to intra-mode BTI, but not affected by eIBRS + guest/host isolation. + * - Mitigation: Aligned branch/return thunks + - The mitigation is enabled, affected indirect branches and RETs are + relocated to safe thunks. + * - Mitigation: Retpolines, Stuffing RSB + - The mitigation is enabled using retpoline and RSB stuffing. + +References +---------- +.. [#f1] Microcode repository - https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files + +.. [#f2] Affected Processors list - https://www.intel.com/content/www/us/en/developer/topic-technology/software-security-guidance/processors-affected-consolidated-product-cpu-model.html + +.. [#f3] Affected Processors list (machine readable) - https://github.com/intel/Intel-affected-processor-list diff --git a/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst b/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst index c98fd11907cc8..e916dc232b0f0 100644 --- a/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst +++ b/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst @@ -157,9 +157,7 @@ This is achieved by using the otherwise unused and obsolete VERW instruction in combination with a microcode update. The microcode clears the affected CPU buffers when the VERW instruction is executed. -Kernel reuses the MDS function to invoke the buffer clearing: - - mds_clear_cpu_buffers() +Kernel does the buffer clearing with x86_clear_cpu_buffers(). On MDS affected CPUs, the kernel already invokes CPU buffer clear on kernel/userspace, hypervisor/guest and C-state (idle) transitions. No diff --git a/Documentation/admin-guide/hw-vuln/vmscape.rst b/Documentation/admin-guide/hw-vuln/vmscape.rst new file mode 100644 index 0000000000000..d9b9a2b6c114c --- /dev/null +++ b/Documentation/admin-guide/hw-vuln/vmscape.rst @@ -0,0 +1,110 @@ +.. SPDX-License-Identifier: GPL-2.0 + +VMSCAPE +======= + +VMSCAPE is a vulnerability that may allow a guest to influence the branch +prediction in host userspace. It particularly affects hypervisors like QEMU. + +Even if a hypervisor may not have any sensitive data like disk encryption keys, +guest-userspace may be able to attack the guest-kernel using the hypervisor as +a confused deputy. + +Affected processors +------------------- + +The following CPU families are affected by VMSCAPE: + +**Intel processors:** + - Skylake generation (Parts without Enhanced-IBRS) + - Cascade Lake generation - (Parts affected by ITS guest/host separation) + - Alder Lake and newer (Parts affected by BHI) + +Note that, BHI affected parts that use BHB clearing software mitigation e.g. +Icelake are not vulnerable to VMSCAPE. + +**AMD processors:** + - Zen series (families 0x17, 0x19, 0x1a) + +** Hygon processors:** + - Family 0x18 + +Mitigation +---------- + +Conditional IBPB +---------------- + +Kernel tracks when a CPU has run a potentially malicious guest and issues an +IBPB before the first exit to userspace after VM-exit. If userspace did not run +between VM-exit and the next VM-entry, no IBPB is issued. + +Note that the existing userspace mitigation against Spectre-v2 is effective in +protecting the userspace. They are insufficient to protect the userspace VMMs +from a malicious guest. This is because Spectre-v2 mitigations are applied at +context switch time, while the userspace VMM can run after a VM-exit without a +context switch. + +Vulnerability enumeration and mitigation is not applied inside a guest. This is +because nested hypervisors should already be deploying IBPB to isolate +themselves from nested guests. + +SMT considerations +------------------ + +When Simultaneous Multi-Threading (SMT) is enabled, hypervisors can be +vulnerable to cross-thread attacks. For complete protection against VMSCAPE +attacks in SMT environments, STIBP should be enabled. + +The kernel will issue a warning if SMT is enabled without adequate STIBP +protection. Warning is not issued when: + +- SMT is disabled +- STIBP is enabled system-wide +- Intel eIBRS is enabled (which implies STIBP protection) + +System information and options +------------------------------ + +The sysfs file showing VMSCAPE mitigation status is: + + /sys/devices/system/cpu/vulnerabilities/vmscape + +The possible values in this file are: + + * 'Not affected': + + The processor is not vulnerable to VMSCAPE attacks. + + * 'Vulnerable': + + The processor is vulnerable and no mitigation has been applied. + + * 'Mitigation: IBPB before exit to userspace': + + Conditional IBPB mitigation is enabled. The kernel tracks when a CPU has + run a potentially malicious guest and issues an IBPB before the first + exit to userspace after VM-exit. + + * 'Mitigation: IBPB on VMEXIT': + + IBPB is issued on every VM-exit. This occurs when other mitigations like + RETBLEED or SRSO are already issuing IBPB on VM-exit. + +Mitigation control on the kernel command line +---------------------------------------------- + +The mitigation can be controlled via the ``vmscape=`` command line parameter: + + * ``vmscape=off``: + + Disable the VMSCAPE mitigation. + + * ``vmscape=ibpb``: + + Enable conditional IBPB mitigation (default when CONFIG_MITIGATION_VMSCAPE=y). + + * ``vmscape=force``: + + Force vulnerability detection and mitigation even on processors that are + not known to be affected. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6b3e3c0b7f177..f6153c3b603e7 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2084,6 +2084,23 @@ different crypto accelerators. This option can be used to achieve best performance for particular HW. + indirect_target_selection= [X86,Intel] Mitigation control for Indirect + Target Selection(ITS) bug in Intel CPUs. Updated + microcode is also required for a fix in IBPB. + + on: Enable mitigation (default). + off: Disable mitigation. + force: Force the ITS bug and deploy default + mitigation. + vmexit: Only deploy mitigation if CPU is affected by + guest/host isolation part of ITS. + stuff: Deploy RSB-fill mitigation when retpoline is + also deployed. Otherwise, deploy the default + mitigation. + + For details see: + Documentation/admin-guide/hw-vuln/indirect-target-selection.rst + init= [KNL] Format: Run specified binary instead of /sbin/init as init @@ -2638,6 +2655,11 @@ (enabled). Disable by KVM if hardware lacks support for NPT. + kvm-arm.hyp_lm_size_mb= + [KVM,ARM,EARLY] Maximum amount of contiguous memory mappable in + the pKVM hypervisor linear map, in MB. Any attempt to map more + memory than this into pKVM stage-1 at run-time may be fatal. + kvm-arm.mode= [KVM,ARM] Select one of KVM/arm64's modes of operation. @@ -3367,6 +3389,7 @@ expose users to several CPU vulnerabilities. Equivalent to: if nokaslr then kpti=0 [ARM64] gather_data_sampling=off [X86] + indirect_target_selection=off [X86] kvm.nx_huge_pages=off [X86] l1tf=off [X86] mds=off [X86] @@ -3386,6 +3409,7 @@ srbds=off [X86,INTEL] ssbd=force-off [ARM64] tsx_async_abort=off [X86] + vmscape=off [X86] Exceptions: This does not have any effect on @@ -5685,6 +5709,9 @@ rootflags= [KNL] Set root filesystem mount option string + initramfs_options= [KNL] + Specify mount options for for the initramfs mount. + rootfstype= [KNL] Set root filesystem type rootwait [KNL] Wait (indefinitely) for root device to show up. @@ -6702,6 +6729,19 @@ If not specified, "default" is used. In this case, the RNG's choice is left to each individual trust source. + tsa= [X86] Control mitigation for Transient Scheduler + Attacks on AMD CPUs. Search the following in your + favourite search engine for more details: + + "Technical guidance for mitigating transient scheduler + attacks". + + off - disable the mitigation + on - enable the mitigation (default) + user - mitigate only user/kernel transitions + vm - mitigate only guest/host transitions + + tsc= Disable clocksource stability checks for TSC. Format: [x86] reliable: mark tsc clocksource as reliable, this @@ -7118,6 +7158,16 @@ vmpoff= [KNL,S390] Perform z/VM CP command after power off. Format: + vmscape= [X86] Controls mitigation for VMscape attacks. + VMscape attacks can leak information from a userspace + hypervisor to a guest via speculative side-channels. + + off - disable the mitigation + ibpb - use Indirect Branch Prediction Barrier + (IBPB) mitigation (default) + force - force vulnerability detection even on + unaffected processors + vsyscall= [X86-64] Controls the behavior of vsyscalls (i.e. calls to fixed addresses of 0xffffffffff600x00 from legacy diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 8209c7a7c3970..fbc833841bef5 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -187,6 +187,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-600 | #1076982,1209401| N/A | diff --git a/Documentation/arch/x86/mds.rst b/Documentation/arch/x86/mds.rst index c58c72362911c..43106f349cc35 100644 --- a/Documentation/arch/x86/mds.rst +++ b/Documentation/arch/x86/mds.rst @@ -93,7 +93,7 @@ enters a C-state. The kernel provides a function to invoke the buffer clearing: - mds_clear_cpu_buffers() + x86_clear_cpu_buffers() Also macro CLEAR_CPU_BUFFERS can be used in ASM late in exit-to-user path. Other than CFLAGS.ZF, this macro doesn't clobber any registers. @@ -185,9 +185,9 @@ Mitigation points idle clearing would be a window dressing exercise and is therefore not activated. - The invocation is controlled by the static key mds_idle_clear which is - switched depending on the chosen mitigation mode and the SMT state of - the system. + The invocation is controlled by the static key cpu_buf_idle_clear which is + switched depending on the chosen mitigation mode and the SMT state of the + system. The buffer clear is only invoked before entering the C-State to prevent that stale data from the idling CPU from spilling to the Hyper-Thread diff --git a/Documentation/bpf/map_hash.rst b/Documentation/bpf/map_hash.rst index d2343952f2cbd..8606bf958a8cf 100644 --- a/Documentation/bpf/map_hash.rst +++ b/Documentation/bpf/map_hash.rst @@ -233,10 +233,16 @@ attempts in order to enforce the LRU property which have increasing impacts on other CPUs involved in the following operation attempts: - Attempt to use CPU-local state to batch operations -- Attempt to fetch free nodes from global lists +- Attempt to fetch ``target_free`` free nodes from global lists - Attempt to pull any node from a global list and remove it from the hashmap - Attempt to pull any node from any CPU's list and remove it from the hashmap +The number of nodes to borrow from the global list in a batch, ``target_free``, +depends on the size of the map. Larger batch size reduces lock contention, but +may also exhaust the global structure. The value is computed at map init to +avoid exhaustion, by limiting aggregate reservation by all CPUs to half the map +size. With a minimum of a single element and maximum budget of 128 at a time. + This algorithm is described visually in the following diagram. See the description in commit 3a08c2fd7634 ("bpf: LRU List") for a full explanation of the corresponding operations: diff --git a/Documentation/bpf/map_lru_hash_update.dot b/Documentation/bpf/map_lru_hash_update.dot index a0fee349d29c2..ab10058f5b79f 100644 --- a/Documentation/bpf/map_lru_hash_update.dot +++ b/Documentation/bpf/map_lru_hash_update.dot @@ -35,18 +35,18 @@ digraph { fn_bpf_lru_list_pop_free_to_local [shape=rectangle,fillcolor=2, label="Flush local pending, Rotate Global list, move - LOCAL_FREE_TARGET + target_free from global -> local"] // Also corresponds to: // fn__local_list_flush() // fn_bpf_lru_list_rotate() fn___bpf_lru_node_move_to_free[shape=diamond,fillcolor=2, - label="Able to free\nLOCAL_FREE_TARGET\nnodes?"] + label="Able to free\ntarget_free\nnodes?"] fn___bpf_lru_list_shrink_inactive [shape=rectangle,fillcolor=3, label="Shrink inactive list up to remaining - LOCAL_FREE_TARGET + target_free (global LRU -> local)"] fn___bpf_lru_list_shrink [shape=diamond,fillcolor=2, label="> 0 entries in\nlocal free list?"] diff --git a/Documentation/core-api/symbol-namespaces.rst b/Documentation/core-api/symbol-namespaces.rst index 12e4aecdae945..29875e25e376f 100644 --- a/Documentation/core-api/symbol-namespaces.rst +++ b/Documentation/core-api/symbol-namespaces.rst @@ -28,6 +28,9 @@ kernel. As of today, modules that make use of symbols exported into namespaces, are required to import the namespace. Otherwise the kernel will, depending on its configuration, reject loading the module or warn about a missing import. +Additionally, it is possible to put symbols into a module namespace, strictly +limiting which modules are allowed to use these symbols. + 2. How to define Symbol Namespaces ================================== @@ -84,6 +87,22 @@ unit as preprocessor statement. The above example would then read:: within the corresponding compilation unit before any EXPORT_SYMBOL macro is used. +2.3 Using the EXPORT_SYMBOL_GPL_FOR_MODULES() macro +=================================================== + +Symbols exported using this macro are put into a module namespace. This +namespace cannot be imported. + +The macro takes a comma separated list of module names, allowing only those +modules to access this symbol. Simple tail-globs are supported. + +For example: + + EXPORT_SYMBOL_GPL_FOR_MODULES(preempt_notifier_inc, "kvm,kvm-*") + +will limit usage of this symbol to modules whoes name matches the given +patterns. + 3. How to use Symbols exported in Namespaces ============================================ @@ -155,3 +174,6 @@ in-tree modules:: You can also run nsdeps for external module builds. A typical usage is:: $ make -C M=$PWD nsdeps + +Note: it will happily generate an import statement for the module namespace; +which will not work and generates build and runtime failures. diff --git a/Documentation/devicetree/bindings/display/msm/qcom,mdp5.yaml b/Documentation/devicetree/bindings/display/msm/qcom,mdp5.yaml index 91c774f106ceb..ab1196d1ec2dd 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,mdp5.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,mdp5.yaml @@ -59,7 +59,6 @@ properties: - const: bus - const: core - const: vsync - - const: lut - const: tbu - const: tbu_rt # MSM8996 has additional iommu clock diff --git a/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dpu.yaml b/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dpu.yaml index 4ebea60b8c5ba..8c52fa0ea5f8e 100644 --- a/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dpu.yaml +++ b/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dpu.yaml @@ -25,7 +25,7 @@ properties: maxItems: 1 clocks: - minItems: 2 + maxItems: 2 clock-names: items: diff --git a/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dsi-host.yaml b/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dsi-host.yaml index bc5594d186430..300bf2252c3e8 100644 --- a/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dsi-host.yaml +++ b/Documentation/devicetree/bindings/display/sprd/sprd,sharkl3-dsi-host.yaml @@ -20,7 +20,7 @@ properties: maxItems: 2 clocks: - minItems: 1 + maxItems: 1 clock-names: items: diff --git a/Documentation/devicetree/bindings/i2c/nvidia,tegra20-i2c.yaml b/Documentation/devicetree/bindings/i2c/nvidia,tegra20-i2c.yaml index 424a4fc218b66..2cb311ddd912a 100644 --- a/Documentation/devicetree/bindings/i2c/nvidia,tegra20-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/nvidia,tegra20-i2c.yaml @@ -103,7 +103,10 @@ properties: resets: items: - - description: module reset + - description: + Module reset. This property is optional for controllers in Tegra194, + Tegra234 etc where an internal software reset is available as an + alternative. reset-names: items: @@ -119,6 +122,13 @@ properties: - const: rx - const: tx +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + allOf: - $ref: /schemas/i2c/i2c-controller.yaml - if: @@ -172,6 +182,18 @@ allOf: items: - description: phandle to the VENC power domain + - if: + not: + properties: + compatible: + contains: + enum: + - nvidia,tegra194-i2c + then: + required: + - resets + - reset-names + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.yaml b/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.yaml index dc3a3f709feaa..bac4d0b51d8a1 100644 --- a/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.yaml @@ -58,8 +58,7 @@ properties: fsl,phy-tx-vboost-level-microvolt: description: Adjust the boosted transmit launch pk-pk differential amplitude - minimum: 880 - maximum: 1120 + enum: [844, 1008, 1156] fsl,phy-comp-dis-tune-percent: description: diff --git a/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml b/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml index 5ac994b3c0aa1..b304bc5a08c40 100644 --- a/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml +++ b/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml @@ -57,11 +57,24 @@ required: - clocks - clock-names - '#phy-cells' - - power-domains - resets - reset-names - rockchip,grf +allOf: + - if: + properties: + compatible: + contains: + enum: + - rockchip,px30-csi-dphy + - rockchip,rk1808-csi-dphy + - rockchip,rk3326-csi-dphy + - rockchip,rk3368-csi-dphy + then: + required: + - power-domains + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml index 19d47fd414bc0..ce04d2eadec9d 100644 --- a/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml @@ -50,18 +50,20 @@ patternProperties: groups: description: Name of the pin group to use for the functions. - $ref: /schemas/types.yaml#/definitions/string - enum: [i2c0_grp, i2c1_grp, i2c2_grp, i2c3_grp, i2c4_grp, - i2c5_grp, i2c6_grp, i2c7_grp, i2c8_grp, - spi0_grp, spi0_cs0_grp, spi0_cs1_grp, spi0_cs2_grp, - spi1_grp, spi2_grp, spi3_grp, spi4_grp, spi5_grp, spi6_grp, - uart0_grp, uart1_grp, uart2_grp, uart3_grp, - pwm0_gpio4_grp, pwm0_gpio8_grp, pwm0_gpio12_grp, - pwm0_gpio16_grp, pwm1_gpio5_grp, pwm1_gpio9_grp, - pwm1_gpio13_grp, pwm1_gpio17_grp, pwm2_gpio6_grp, - pwm2_gpio10_grp, pwm2_gpio14_grp, pwm2_gpio18_grp, - pwm3_gpio7_grp, pwm3_gpio11_grp, pwm3_gpio15_grp, - pwm3_gpio19_grp, pcmif_out_grp, pcmif_in_grp] + items: + enum: [i2c0_grp, i2c1_grp, i2c2_grp, i2c3_grp, i2c4_grp, + i2c5_grp, i2c6_grp, i2c7_grp, i2c8_grp, + spi0_grp, spi0_cs0_grp, spi0_cs1_grp, spi0_cs2_grp, + spi1_grp, spi2_grp, spi3_grp, spi4_grp, spi5_grp, spi6_grp, + uart0_grp, uart1_grp, uart2_grp, uart3_grp, + pwm0_gpio4_grp, pwm0_gpio8_grp, pwm0_gpio12_grp, + pwm0_gpio16_grp, pwm1_gpio5_grp, pwm1_gpio9_grp, + pwm1_gpio13_grp, pwm1_gpio17_grp, pwm2_gpio6_grp, + pwm2_gpio10_grp, pwm2_gpio14_grp, pwm2_gpio18_grp, + pwm3_gpio7_grp, pwm3_gpio11_grp, pwm3_gpio15_grp, + pwm3_gpio19_grp, pcmif_out_grp, pcmif_in_grp] + minItems: 1 + maxItems: 8 drive-strength: enum: [2, 4, 6, 8, 16, 24, 32] diff --git a/Documentation/devicetree/bindings/regulator/mediatek,mt6357-regulator.yaml b/Documentation/devicetree/bindings/regulator/mediatek,mt6357-regulator.yaml index 6327bb2f6ee08..698266c09e253 100644 --- a/Documentation/devicetree/bindings/regulator/mediatek,mt6357-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/mediatek,mt6357-regulator.yaml @@ -33,7 +33,7 @@ patternProperties: "^ldo-v(camio18|aud28|aux18|io18|io28|rf12|rf18|cn18|cn28|fe28)$": type: object - $ref: fixed-regulator.yaml# + $ref: regulator.yaml# unevaluatedProperties: false description: Properties for single fixed LDO regulator. @@ -112,7 +112,6 @@ examples: regulator-enable-ramp-delay = <220>; }; mt6357_vfe28_reg: ldo-vfe28 { - compatible = "regulator-fixed"; regulator-name = "vfe28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; @@ -125,14 +124,12 @@ examples: regulator-enable-ramp-delay = <110>; }; mt6357_vrf18_reg: ldo-vrf18 { - compatible = "regulator-fixed"; regulator-name = "vrf18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-enable-ramp-delay = <110>; }; mt6357_vrf12_reg: ldo-vrf12 { - compatible = "regulator-fixed"; regulator-name = "vrf12"; regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; @@ -157,14 +154,12 @@ examples: regulator-enable-ramp-delay = <264>; }; mt6357_vcn28_reg: ldo-vcn28 { - compatible = "regulator-fixed"; regulator-name = "vcn28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; regulator-enable-ramp-delay = <264>; }; mt6357_vcn18_reg: ldo-vcn18 { - compatible = "regulator-fixed"; regulator-name = "vcn18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; @@ -183,7 +178,6 @@ examples: regulator-enable-ramp-delay = <264>; }; mt6357_vcamio_reg: ldo-vcamio18 { - compatible = "regulator-fixed"; regulator-name = "vcamio"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; @@ -212,28 +206,24 @@ examples: regulator-always-on; }; mt6357_vaux18_reg: ldo-vaux18 { - compatible = "regulator-fixed"; regulator-name = "vaux18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-enable-ramp-delay = <264>; }; mt6357_vaud28_reg: ldo-vaud28 { - compatible = "regulator-fixed"; regulator-name = "vaud28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; regulator-enable-ramp-delay = <264>; }; mt6357_vio28_reg: ldo-vio28 { - compatible = "regulator-fixed"; regulator-name = "vio28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; regulator-enable-ramp-delay = <264>; }; mt6357_vio18_reg: ldo-vio18 { - compatible = "regulator-fixed"; regulator-name = "vio18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; diff --git a/Documentation/devicetree/bindings/serial/8250.yaml b/Documentation/devicetree/bindings/serial/8250.yaml index 692aa05500fd5..6ba0325039be2 100644 --- a/Documentation/devicetree/bindings/serial/8250.yaml +++ b/Documentation/devicetree/bindings/serial/8250.yaml @@ -45,7 +45,7 @@ allOf: - ns16550 - ns16550a then: - anyOf: + oneOf: - required: [ clock-frequency ] - required: [ clocks ] diff --git a/Documentation/devicetree/bindings/serial/brcm,bcm7271-uart.yaml b/Documentation/devicetree/bindings/serial/brcm,bcm7271-uart.yaml index 89c462653e2d3..8cc848ae11cb7 100644 --- a/Documentation/devicetree/bindings/serial/brcm,bcm7271-uart.yaml +++ b/Documentation/devicetree/bindings/serial/brcm,bcm7271-uart.yaml @@ -41,7 +41,7 @@ properties: - const: dma_intr2 clocks: - minItems: 1 + maxItems: 1 clock-names: const: sw_baud diff --git a/Documentation/devicetree/bindings/usb/cypress,hx3.yaml b/Documentation/devicetree/bindings/usb/cypress,hx3.yaml index 47add0d85fb89..02349bbb0d46f 100644 --- a/Documentation/devicetree/bindings/usb/cypress,hx3.yaml +++ b/Documentation/devicetree/bindings/usb/cypress,hx3.yaml @@ -14,9 +14,22 @@ allOf: properties: compatible: - enum: - - usb4b4,6504 - - usb4b4,6506 + oneOf: + - enum: + - usb4b4,6504 + - usb4b4,6506 + - items: + - enum: + - usb4b4,6500 + - usb4b4,6508 + - const: usb4b4,6504 + - items: + - enum: + - usb4b4,6502 + - usb4b4,6503 + - usb4b4,6507 + - usb4b4,650a + - const: usb4b4,6506 reg: true diff --git a/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml b/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml index 9ea1e4cd0709c..69be6affa9b53 100644 --- a/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml @@ -85,13 +85,21 @@ required: - reg - "#address-cells" - "#size-cells" - - dma-ranges - ranges - clocks - clock-names - interrupts - power-domains +allOf: + - if: + properties: + compatible: + const: fsl,imx8mp-dwc3 + then: + required: + - dma-ranges + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index dc275ab60e534..7376a924e9aca 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -773,6 +773,8 @@ patternProperties: description: Linux-specific binding "^linx,.*": description: Linx Technologies + "^liontron,.*": + description: Shenzhen Liontron Technology Co., Ltd "^liteon,.*": description: LITE-ON Technology Corp. "^litex,.*": diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index fb7d2ee022bc0..c0fefb96db8fa 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -173,9 +173,12 @@ data_flush Enable data flushing before checkpoint in order to persist data of regular and symlink. reserve_root=%d Support configuring reserved space which is used for allocation from a privileged user with specified uid or - gid, unit: 4KB, the default limit is 0.2% of user blocks. -resuid=%d The user ID which may use the reserved blocks. -resgid=%d The group ID which may use the reserved blocks. + gid, unit: 4KB, the default limit is 12.5% of user blocks. +reserve_node=%d Support configuring reserved nodes which are used for + allocation from a privileged user with specified uid or + gid, the default limit is 12.5% of all nodes. +resuid=%d The user ID which may use the reserved blocks and nodes. +resgid=%d The group ID which may use the reserved blocks and nodes. fault_injection=%d Enable fault injection in all supported types with specified injection rate. fault_type=%d Support configuring fault injection type, should be @@ -235,9 +238,9 @@ usrjquota= Appoint specified file and type during mount, so that quota grpjquota= information can be properly updated during recovery flow, prjjquota= : must be in root directory; jqfmt= : [vfsold,vfsv0,vfsv1]. -offusrjquota Turn off user journalled quota. -offgrpjquota Turn off group journalled quota. -offprjjquota Turn off project journalled quota. +usrjquota= Turn off user journalled quota. +grpjquota= Turn off group journalled quota. +prjjquota= Turn off project journalled quota. quota Enable plain user disk quota accounting. noquota Disable all plain disk quota option. alloc_mode=%s Adjust block allocation policy, which supports "reuse" @@ -365,6 +368,25 @@ errors=%s Specify f2fs behavior on critical errors. This supports modes: pending node write drop keep N/A pending meta write keep keep N/A ====================== =============== =============== ======== +lookup_mode=%s Control the directory lookup behavior for casefolded + directories. This option has no effect on directories + that do not have the casefold feature enabled. + + ================== ======================================== + Value Description + ================== ======================================== + perf (Default) Enforces a hash-only lookup. + The linear search fallback is always + disabled, ignoring the on-disk flag. + compat Enables the linear search fallback for + compatibility with directory entries + created by older kernel that used a + different case-folding algorithm. + This mode ignores the on-disk flag. + auto F2FS determines the mode based on the + on-disk `SB_ENC_NO_COMPAT_FALLBACK_FL` + flag. + ================== ======================================== ======================== ============================================================ Debugfs Entries diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst index 1b84f818e574e..be11eee30e5b1 100644 --- a/Documentation/filesystems/fscrypt.rst +++ b/Documentation/filesystems/fscrypt.rst @@ -141,9 +141,8 @@ However, these ioctls have some limitations: CONFIG_PAGE_POISONING=y in your kernel config and add page_poison=1 to your kernel command line. However, this has a performance cost. -- Secret keys might still exist in CPU registers, in crypto - accelerator hardware (if used by the crypto API to implement any of - the algorithms), or in other places not explicitly considered here. +- Secret keys might still exist in CPU registers or in other places + not explicitly considered here. Limitations of v1 policies ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -375,9 +374,12 @@ the work is done by XChaCha12, which is much faster than AES when AES acceleration is unavailable. For more information about Adiantum, see `the Adiantum paper `_. -The (AES-128-CBC-ESSIV, AES-128-CTS-CBC) pair exists only to support -systems whose only form of AES acceleration is an off-CPU crypto -accelerator such as CAAM or CESA that does not support XTS. +The (AES-128-CBC-ESSIV, AES-128-CTS-CBC) pair was added to try to +provide a more efficient option for systems that lack AES instructions +in the CPU but do have a non-inline crypto engine such as CAAM or CESA +that supports AES-CBC (and not AES-XTS). This is deprecated. It has +been shown that just doing AES on the CPU is actually faster. +Moreover, Adiantum is faster still and is recommended on such systems. The remaining mode pairs are the "national pride ciphers": @@ -1286,22 +1288,13 @@ this by validating all top-level encryption policies prior to access. Inline encryption support ========================= -By default, fscrypt uses the kernel crypto API for all cryptographic -operations (other than HKDF, which fscrypt partially implements -itself). The kernel crypto API supports hardware crypto accelerators, -but only ones that work in the traditional way where all inputs and -outputs (e.g. plaintexts and ciphertexts) are in memory. fscrypt can -take advantage of such hardware, but the traditional acceleration -model isn't particularly efficient and fscrypt hasn't been optimized -for it. - -Instead, many newer systems (especially mobile SoCs) have *inline -encryption hardware* that can encrypt/decrypt data while it is on its -way to/from the storage device. Linux supports inline encryption -through a set of extensions to the block layer called *blk-crypto*. -blk-crypto allows filesystems to attach encryption contexts to bios -(I/O requests) to specify how the data will be encrypted or decrypted -in-line. For more information about blk-crypto, see +Many newer systems (especially mobile SoCs) have *inline encryption +hardware* that can encrypt/decrypt data while it is on its way to/from +the storage device. Linux supports inline encryption through a set of +extensions to the block layer called *blk-crypto*. blk-crypto allows +filesystems to attach encryption contexts to bios (I/O requests) to +specify how the data will be encrypted or decrypted in-line. For more +information about blk-crypto, see :ref:`Documentation/block/inline-encryption.rst `. On supported filesystems (currently ext4 and f2fs), fscrypt can use diff --git a/Documentation/firmware-guide/acpi/i2c-muxes.rst b/Documentation/firmware-guide/acpi/i2c-muxes.rst index 3a8997ccd7c4b..f366539acd792 100644 --- a/Documentation/firmware-guide/acpi/i2c-muxes.rst +++ b/Documentation/firmware-guide/acpi/i2c-muxes.rst @@ -14,7 +14,7 @@ Consider this topology:: | | | 0x70 |--CH01--> i2c client B (0x50) +------+ +------+ -which corresponds to the following ASL:: +which corresponds to the following ASL (in the scope of \_SB):: Device (SMB1) { @@ -24,7 +24,7 @@ which corresponds to the following ASL:: Name (_HID, ...) Name (_CRS, ResourceTemplate () { I2cSerialBus (0x70, ControllerInitiated, I2C_SPEED, - AddressingMode7Bit, "^SMB1", 0x00, + AddressingMode7Bit, "\\_SB.SMB1", 0x00, ResourceConsumer,,) } @@ -37,7 +37,7 @@ which corresponds to the following ASL:: Name (_HID, ...) Name (_CRS, ResourceTemplate () { I2cSerialBus (0x50, ControllerInitiated, I2C_SPEED, - AddressingMode7Bit, "^CH00", 0x00, + AddressingMode7Bit, "\\_SB.SMB1.CH00", 0x00, ResourceConsumer,,) } } @@ -52,7 +52,7 @@ which corresponds to the following ASL:: Name (_HID, ...) Name (_CRS, ResourceTemplate () { I2cSerialBus (0x50, ControllerInitiated, I2C_SPEED, - AddressingMode7Bit, "^CH01", 0x00, + AddressingMode7Bit, "\\_SB.SMB1.CH01", 0x00, ResourceConsumer,,) } } diff --git a/Documentation/hwmon/dell-smm-hwmon.rst b/Documentation/hwmon/dell-smm-hwmon.rst index d8f1d6859b964..1c12fbba440bc 100644 --- a/Documentation/hwmon/dell-smm-hwmon.rst +++ b/Documentation/hwmon/dell-smm-hwmon.rst @@ -32,12 +32,12 @@ Temperature sensors and fans can be queried and set via the standard =============================== ======= ======================================= Name Perm Description =============================== ======= ======================================= -fan[1-3]_input RO Fan speed in RPM. -fan[1-3]_label RO Fan label. -fan[1-3]_min RO Minimal Fan speed in RPM -fan[1-3]_max RO Maximal Fan speed in RPM -fan[1-3]_target RO Expected Fan speed in RPM -pwm[1-3] RW Control the fan PWM duty-cycle. +fan[1-4]_input RO Fan speed in RPM. +fan[1-4]_label RO Fan label. +fan[1-4]_min RO Minimal Fan speed in RPM +fan[1-4]_max RO Maximal Fan speed in RPM +fan[1-4]_target RO Expected Fan speed in RPM +pwm[1-4] RW Control the fan PWM duty-cycle. pwm1_enable WO Enable or disable automatic BIOS fan control (not supported on all laptops, see below for details). @@ -93,7 +93,7 @@ Again, when you find new codes, we'd be happy to have your patches! --------------------------- The driver also exports the fans as thermal cooling devices with -``type`` set to ``dell-smm-fan[1-3]``. This allows for easy fan control +``type`` set to ``dell-smm-fan[1-4]``. This allows for easy fan control using one of the thermal governors. Module parameters diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml index 3e38f69567939..b463949736c56 100644 --- a/Documentation/netlink/specs/ethtool.yaml +++ b/Documentation/netlink/specs/ethtool.yaml @@ -1489,9 +1489,6 @@ operations: do: &module-eeprom-get-op request: - attributes: - - header - reply: attributes: - header - offset @@ -1499,6 +1496,9 @@ operations: - page - bank - i2c-address + reply: + attributes: + - header - data dump: *module-eeprom-get-op - diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst index f7a73421eb76a..e774b48de9f51 100644 --- a/Documentation/networking/bonding.rst +++ b/Documentation/networking/bonding.rst @@ -444,6 +444,18 @@ arp_missed_max The default value is 2, and the allowable range is 1 - 255. +coupled_control + + Specifies whether the LACP state machine's MUX in the 802.3ad mode + should have separate Collecting and Distributing states. + + This is by implementing the independent control state machine per + IEEE 802.1AX-2008 5.4.15 in addition to the existing coupled control + state machine. + + The default value is 1. This setting does not separate the Collecting + and Distributing states, maintaining the bond in coupled control. + downdelay Specifies the time, in milliseconds, to wait before disabling diff --git a/Documentation/networking/can.rst b/Documentation/networking/can.rst index d7e1ada905b2d..3bdd155838105 100644 --- a/Documentation/networking/can.rst +++ b/Documentation/networking/can.rst @@ -740,7 +740,7 @@ The broadcast manager sends responses to user space in the same form: struct timeval ival1, ival2; /* count and subsequent interval */ canid_t can_id; /* unique can_id for task */ __u32 nframes; /* number of can_frames following */ - struct can_frame frames[0]; + struct can_frame frames[]; }; The aligned payload 'frames' uses the same basic CAN frame structure defined diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index a66054d0763a6..66d657d53b6d3 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -2304,6 +2304,20 @@ accept_ra_pinfo - BOOLEAN - enabled if accept_ra is enabled. - disabled if accept_ra is disabled. +ra_honor_pio_pflag - BOOLEAN + The Prefix Information Option P-flag indicates the network can + allocate a unique IPv6 prefix per client using DHCPv6-PD. + This sysctl can be enabled when a userspace DHCPv6-PD client + is running to cause the P-flag to take effect: i.e. the + P-flag suppresses any effects of the A-flag within the same + PIO. For a given PIO, P=1 and A=1 is treated as A=0. + + - If disabled, the P-flag is ignored. + - If enabled, the P-flag will disable SLAAC autoconfiguration + for the given Prefix Information Option. + + Default: 0 (disabled) + accept_ra_rt_info_min_plen - INTEGER Minimum prefix length of Route Information in RA. diff --git a/Documentation/networking/mptcp-sysctl.rst b/Documentation/networking/mptcp-sysctl.rst index 15f1919d640c0..b797f3dd4b692 100644 --- a/Documentation/networking/mptcp-sysctl.rst +++ b/Documentation/networking/mptcp-sysctl.rst @@ -20,6 +20,8 @@ add_addr_timeout - INTEGER (seconds) resent to an MPTCP peer that has not acknowledged a previous ADD_ADDR message. + Do not retransmit if set to 0. + The default value matches TCP_RTO_MAX. This is a per-namespace sysctl. diff --git a/Documentation/networking/seg6-sysctl.rst b/Documentation/networking/seg6-sysctl.rst index 07c20e470bafe..1b6af4779be11 100644 --- a/Documentation/networking/seg6-sysctl.rst +++ b/Documentation/networking/seg6-sysctl.rst @@ -25,6 +25,9 @@ seg6_require_hmac - INTEGER Default is 0. +/proc/sys/net/ipv6/seg6_* variables: +==================================== + seg6_flowlabel - INTEGER Controls the behaviour of computing the flowlabel of outer IPv6 header in case of SR T.encaps diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst index 3170747226f6d..0b650bb550e64 100644 --- a/Documentation/scheduler/index.rst +++ b/Documentation/scheduler/index.rst @@ -19,6 +19,7 @@ Scheduler sched-nice-design sched-rt-group sched-stats + sched-ext sched-debug text_files diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst new file mode 100644 index 0000000000000..84c30b44f104c --- /dev/null +++ b/Documentation/scheduler/sched-ext.rst @@ -0,0 +1,230 @@ +========================== +Extensible Scheduler Class +========================== + +sched_ext is a scheduler class whose behavior can be defined by a set of BPF +programs - the BPF scheduler. + +* sched_ext exports a full scheduling interface so that any scheduling + algorithm can be implemented on top. + +* The BPF scheduler can group CPUs however it sees fit and schedule them + together, as tasks aren't tied to specific CPUs at the time of wakeup. + +* The BPF scheduler can be turned on and off dynamically anytime. + +* The system integrity is maintained no matter what the BPF scheduler does. + The default scheduling behavior is restored anytime an error is detected, + a runnable task stalls, or on invoking the SysRq key sequence + :kbd:`SysRq-S`. + +Switching to and from sched_ext +=============================== + +``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and +``tools/sched_ext`` contains the example schedulers. + +sched_ext is used only when the BPF scheduler is loaded and running. + +If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be +treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is +loaded. On load, such tasks will be switched to and scheduled by sched_ext. + +The BPF scheduler can choose to schedule all normal and lower class tasks by +calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this +case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and +``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers, +this mode can be selected with the ``-a`` option. + +Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or +detection of any internal error including stalled runnable tasks aborts the +BPF scheduler and reverts all tasks back to CFS. + +.. code-block:: none + + # make -j16 -C tools/sched_ext + # tools/sched_ext/scx_example_simple + local=0 global=3 + local=5 global=24 + local=9 global=44 + local=13 global=56 + local=17 global=72 + ^CEXIT: BPF scheduler unregistered + +If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler +and whether a given task is on sched_ext can be determined as follows: + +.. code-block:: none + + # cat /sys/kernel/debug/sched/ext + ops : simple + enabled : 1 + switching_all : 1 + switched_all : 1 + enable_state : enabled + + # grep ext /proc/self/sched + ext.enabled : 1 + +The Basics +========== + +Userspace can implement an arbitrary BPF scheduler by loading a set of BPF +programs that implement ``struct sched_ext_ops``. The only mandatory field +is ``ops.name`` which must be a valid BPF object name. All operations are +optional. The following modified excerpt is from +``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO +scheduler. + +.. code-block:: c + + s32 BPF_STRUCT_OPS(simple_init) + { + if (!switch_partial) + scx_bpf_switch_all(); + return 0; + } + + void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) + { + if (enq_flags & SCX_ENQ_LOCAL) + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags); + else + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags); + } + + void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) + { + exit_type = ei->type; + } + + SEC(".struct_ops") + struct sched_ext_ops simple_ops = { + .enqueue = (void *)simple_enqueue, + .init = (void *)simple_init, + .exit = (void *)simple_exit, + .name = "simple", + }; + +Dispatch Queues +--------------- + +To match the impedance between the scheduler core and the BPF scheduler, +sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a +priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``), +and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage +an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and +``scx_bpf_destroy_dsq()``. + +A CPU always executes a task from its local DSQ. A task is "dispatched" to a +DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's +local DSQ. + +When a CPU is looking for the next task to run, if the local DSQ is not +empty, the first task is picked. Otherwise, the CPU tries to consume the +global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()`` +is invoked. + +Scheduling Cycle +---------------- + +The following briefly shows how a waking task is scheduled and executed. + +1. When a task is waking up, ``ops.select_cpu()`` is the first operation + invoked. This serves two purposes. First, CPU selection optimization + hint. Second, waking up the selected CPU if idle. + + The CPU selected by ``ops.select_cpu()`` is an optimization hint and not + binding. The actual decision is made at the last step of scheduling. + However, there is a small performance gain if the CPU + ``ops.select_cpu()`` returns matches the CPU the task eventually runs on. + + A side-effect of selecting a CPU is waking it up from idle. While a BPF + scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper, + using ``ops.select_cpu()`` judiciously can be simpler and more efficient. + + Note that the scheduler core will ignore an invalid CPU selection, for + example, if it's outside the allowed cpumask of the task. + +2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can + make one of the following decisions: + + * Immediately dispatch the task to either the global or local DSQ by + calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or + ``SCX_DSQ_LOCAL``, respectively. + + * Immediately dispatch the task to a custom DSQ by calling + ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63. + + * Queue the task on the BPF side. + +3. When a CPU is ready to schedule, it first looks at its local DSQ. If + empty, it then looks at the global DSQ. If there still isn't a task to + run, ``ops.dispatch()`` is invoked which can use the following two + functions to populate the local DSQ. + + * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can + be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``, + ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()`` + currently can't be called with BPF locks held, this is being worked on + and will be supported. ``scx_bpf_dispatch()`` schedules dispatching + rather than performing them immediately. There can be up to + ``ops.dispatch_max_batch`` pending tasks. + + * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ + to the dispatching DSQ. This function cannot be called with any BPF + locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks + before trying to consume the specified DSQ. + +4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ, + the CPU runs the first one. If empty, the following steps are taken: + + * Try to consume the global DSQ. If successful, run the task. + + * If ``ops.dispatch()`` has dispatched any tasks, retry #3. + + * If the previous task is an SCX task and still runnable, keep executing + it (see ``SCX_OPS_ENQ_LAST``). + + * Go idle. + +Note that the BPF scheduler can always choose to dispatch tasks immediately +in ``ops.enqueue()`` as illustrated in the above simple example. If only the +built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as +a task is never queued on the BPF scheduler and both the local and global +DSQs are consumed automatically. + +``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use +``scx_bpf_dispatch_vtime()`` for the priority queue. See the function +documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for +more information. + +Where to Look +============= + +* ``include/linux/sched/ext.h`` defines the core data structures, ops table + and constants. + +* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers. + The functions prefixed with ``scx_bpf_`` can be called from the BPF + scheduler. + +* ``tools/sched_ext/`` hosts example BPF scheduler implementations. + + * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example + using a custom DSQ. + + * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting + five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. + +ABI Instability +=============== + +The APIs provided by sched_ext to BPF schedulers programs have no stability +guarantees. This includes the ops table callbacks and constants defined in +``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in +``kernel/sched/ext.c``. + +While we will attempt to provide a relatively stable API surface when +possible, they are subject to change without warning between kernel +versions. diff --git a/Documentation/trace/histogram-design.rst b/Documentation/trace/histogram-design.rst index 5765eb3e9efa7..a30f4bed11b4e 100644 --- a/Documentation/trace/histogram-design.rst +++ b/Documentation/trace/histogram-design.rst @@ -380,7 +380,9 @@ entry, ts0, corresponding to the ts0 variable in the sched_waking trigger above. sched_waking histogram -----------------------:: +---------------------- + +.. code-block:: +------------------+ | hist_data |<-------------------------------------------------------+ diff --git a/Documentation/userspace-api/netlink/specs.rst b/Documentation/userspace-api/netlink/specs.rst index cc4e2430997ef..a8218284e67a4 100644 --- a/Documentation/userspace-api/netlink/specs.rst +++ b/Documentation/userspace-api/netlink/specs.rst @@ -408,10 +408,21 @@ This section describes the attribute types supported by the ``genetlink`` compatibility level. Refer to documentation of different levels for additional attribute types. -Scalar integer types +Common integer types -------------------- -Fixed-width integer types: +``sint`` and ``uint`` represent signed and unsigned 64 bit integers. +If the value can fit on 32 bits only 32 bits are carried in netlink +messages, otherwise full 64 bits are carried. Note that the payload +is only aligned to 4B, so the full 64 bit value may be unaligned! + +Common integer types should be preferred over fix-width types in majority +of cases. + +Fix-width integer types +----------------------- + +Fixed-width integer types include: ``u8``, ``u16``, ``u32``, ``u64``, ``s8``, ``s16``, ``s32``, ``s64``. Note that types smaller than 32 bit should be avoided as using them @@ -421,6 +432,9 @@ See :ref:`pad_type` for padding of 64 bit attributes. The payload of the attribute is the integer in host order unless ``byte-order`` specifies otherwise. +64 bit values are usually aligned by the kernel but it is recommended +that the user space is able to deal with unaligned values. + .. _pad_type: pad diff --git a/MAINTAINERS b/MAINTAINERS index 2384b55682ee5..77ff4cfecd442 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14768,6 +14768,7 @@ NETDEVSIM M: Jakub Kicinski S: Maintained F: drivers/net/netdevsim/* +F: tools/testing/selftests/drivers/net/netdevsim/* NETEM NETWORK EMULATOR M: Stephen Hemminger @@ -19132,6 +19133,8 @@ R: Ben Segall (CONFIG_CFS_BANDWIDTH) R: Mel Gorman (CONFIG_NUMA_BALANCING) R: Daniel Bristot de Oliveira (SCHED_DEADLINE) R: Valentin Schneider (TOPOLOGY) +R: Tejun Heo (SCHED_EXT) +R: David Vernet (SCHED_EXT) L: linux-kernel@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core @@ -19140,6 +19143,7 @@ F: include/linux/sched.h F: include/linux/wait.h F: include/uapi/linux/sched.h F: kernel/sched/ +F: tools/sched_ext/ SCSI LIBSAS SUBSYSTEM R: John Garry diff --git a/Makefile b/Makefile index dd5996b823b62..83447398a568a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 PATCHLEVEL = 6 -SUBLEVEL = 89 +SUBLEVEL = 118 EXTRAVERSION = NAME = Pinguïn Aangedreven @@ -1091,7 +1091,7 @@ KBUILD_USERCFLAGS += $(filter -m32 -m64 --target=%, $(KBUILD_CPPFLAGS) $(KBUILD KBUILD_USERLDFLAGS += $(filter -m32 -m64 --target=%, $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS)) # userspace programs are linked via the compiler, use the correct linker -ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_LD_IS_LLD),yy) +ifdef CONFIG_CC_IS_CLANG KBUILD_USERLDFLAGS += $(call cc-option, --ld-path=$(LD)) endif @@ -1398,11 +1398,11 @@ endif tools/: FORCE $(Q)mkdir -p $(objtree)/tools - $(Q)$(MAKE) LDFLAGS= O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ + $(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ tools/%: FORCE $(Q)mkdir -p $(objtree)/tools - $(Q)$(MAKE) LDFLAGS= O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ $* + $(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ $* # --------------------------------------------------------------------------- # Kernel selftest diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg old mode 100644 new mode 100755 index ad1e794551ecd..8e5e7b09d8e10 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -1038,6 +1038,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x24d0f0fb } +pointer_reference { + id: 0x03b1760d + kind: POINTER + pointee_type_id: 0x24853ea8 +} pointer_reference { id: 0x03b207f4 kind: POINTER @@ -1933,6 +1938,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x30ae0fb9 } +pointer_reference { + id: 0x06cbb7fd + kind: POINTER + pointee_type_id: 0x316e396a +} pointer_reference { id: 0x06dcdc5a kind: POINTER @@ -2393,6 +2403,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x0b7b35bf } +pointer_reference { + id: 0x0850608e + kind: POINTER + pointee_type_id: 0x0b0164a7 +} pointer_reference { id: 0x0858434c kind: POINTER @@ -2603,6 +2618,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x0e39f770 } +pointer_reference { + id: 0x091f52f7 + kind: POINTER + pointee_type_id: 0x0e3dad42 +} pointer_reference { id: 0x092632cb kind: POINTER @@ -3468,6 +3488,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x1a5de19e } +pointer_reference { + id: 0x0c080ab3 + kind: POINTER + pointee_type_id: 0x1a60cc51 +} pointer_reference { id: 0x0c08e6c9 kind: POINTER @@ -4858,6 +4883,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x1e840f82 } +pointer_reference { + id: 0x0d3144be + kind: POINTER + pointee_type_id: 0x1e85f467 +} pointer_reference { id: 0x0d32da14 kind: POINTER @@ -5728,6 +5758,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x1d657c6d } +pointer_reference { + id: 0x0dc9e98b + kind: POINTER + pointee_type_id: 0x1d6740b1 +} pointer_reference { id: 0x0dca21d4 kind: POINTER @@ -6323,6 +6358,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x136d0952 } +pointer_reference { + id: 0x0e4bb5c4 + kind: POINTER + pointee_type_id: 0x136e318e +} pointer_reference { id: 0x0e4d1ea4 kind: POINTER @@ -6483,6 +6523,16 @@ pointer_reference { kind: POINTER pointee_type_id: 0x13e38c46 } +pointer_reference { + id: 0x0e6ab35b + kind: POINTER + pointee_type_id: 0x13ea2bf1 +} +pointer_reference { + id: 0x0e6b837a + kind: POINTER + pointee_type_id: 0x13eeeb77 +} pointer_reference { id: 0x0e6ba6f7 kind: POINTER @@ -6528,6 +6578,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x139004c0 } +pointer_reference { + id: 0x0e747c33 + kind: POINTER + pointee_type_id: 0x13911651 +} pointer_reference { id: 0x0e74cc56 kind: POINTER @@ -7493,6 +7548,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x16a2cb2b } +pointer_reference { + id: 0x0f3912c5 + kind: POINTER + pointee_type_id: 0x16a4ad88 +} pointer_reference { id: 0x0f392d09 kind: POINTER @@ -8423,6 +8483,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x6bc04187 } +pointer_reference { + id: 0x10617c73 + kind: POINTER + pointee_type_id: 0x6bc51750 +} pointer_reference { id: 0x10617cac kind: POINTER @@ -8918,6 +8983,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x61859821 } +pointer_reference { + id: 0x12f234f7 + kind: POINTER + pointee_type_id: 0x61883543 +} pointer_reference { id: 0x13012dce kind: POINTER @@ -10578,6 +10648,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x43d7722e } +pointer_reference { + id: 0x1a6a7d94 + kind: POINTER + pointee_type_id: 0x43e910ce +} pointer_reference { id: 0x1a6ea392 kind: POINTER @@ -12768,6 +12843,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xa0ac1862 } +pointer_reference { + id: 0x22bd991e + kind: POINTER + pointee_type_id: 0xa0b682e4 +} pointer_reference { id: 0x22c8bf71 kind: POINTER @@ -16888,6 +16968,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9946f014 } +pointer_reference { + id: 0x2cc1c3e8 + kind: POINTER + pointee_type_id: 0x9947e93f +} pointer_reference { id: 0x2cc33558 kind: POINTER @@ -17553,6 +17638,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9e732029 } +pointer_reference { + id: 0x2d0d996b + kind: POINTER + pointee_type_id: 0x9e768332 +} pointer_reference { id: 0x2d0e067a kind: POINTER @@ -17878,6 +17968,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9ee7e85b } +pointer_reference { + id: 0x2d29d09d + kind: POINTER + pointee_type_id: 0x9ee7a4e9 +} pointer_reference { id: 0x2d2b1c2c kind: POINTER @@ -17908,6 +18003,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9ef2dc18 } +pointer_reference { + id: 0x2d2c9ff4 + kind: POINTER + pointee_type_id: 0x9ef2994d +} pointer_reference { id: 0x2d2d19a5 kind: POINTER @@ -18258,6 +18358,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9f6e36d6 } +pointer_reference { + id: 0x2d4bbf27 + kind: POINTER + pointee_type_id: 0x9f6e1a01 +} pointer_reference { id: 0x2d4bcb46 kind: POINTER @@ -22728,6 +22833,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x977bcf07 } +pointer_reference { + id: 0x2f4f676a + kind: POINTER + pointee_type_id: 0x977d7b37 +} pointer_reference { id: 0x2f4f9f95 kind: POINTER @@ -24473,6 +24583,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xeff3c532 } +pointer_reference { + id: 0x316e396a + kind: POINTER + pointee_type_id: 0xeff80337 +} pointer_reference { id: 0x3176a085 kind: POINTER @@ -24618,6 +24733,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xed6c3fa4 } +pointer_reference { + id: 0x31d242e7 + kind: POINTER + pointee_type_id: 0xed09ed00 +} pointer_reference { id: 0x31d250f7 kind: POINTER @@ -25528,6 +25648,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xe4c72dc2 } +pointer_reference { + id: 0x33a6c210 + kind: POINTER + pointee_type_id: 0xe4dbeedf +} pointer_reference { id: 0x33a82f3d kind: POINTER @@ -26528,6 +26653,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xfff9ca03 } +pointer_reference { + id: 0x3571c173 + kind: POINTER + pointee_type_id: 0xff87e351 +} pointer_reference { id: 0x35761627 kind: POINTER @@ -27678,6 +27808,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xf435685e } +pointer_reference { + id: 0x379f2ae1 + kind: POINTER + pointee_type_id: 0xf43c4d1b +} pointer_reference { id: 0x379f8c10 kind: POINTER @@ -27993,6 +28128,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xcae7eb7d } +pointer_reference { + id: 0x382bc397 + kind: POINTER + pointee_type_id: 0xcaefe8c1 +} pointer_reference { id: 0x383098d7 kind: POINTER @@ -28713,6 +28853,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xcce651c2 } +pointer_reference { + id: 0x39aad3c3 + kind: POINTER + pointee_type_id: 0xcceba991 +} pointer_reference { id: 0x39ab3cd0 kind: POINTER @@ -29908,6 +30053,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xdb5e7b14 } +pointer_reference { + id: 0x3c497d0a + kind: POINTER + pointee_type_id: 0xdb6512b4 +} pointer_reference { id: 0x3c4ed50c kind: POINTER @@ -30898,6 +31048,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xd37ad96d } +pointer_reference { + id: 0x3e521881 + kind: POINTER + pointee_type_id: 0xd308849a +} pointer_reference { id: 0x3e59d1e6 kind: POINTER @@ -31648,6 +31803,36 @@ typedef { name: "HUF_repeat" referred_type_id: 0x668736f3 } +typedef { + id: 0x2324d4ee + name: "LZ4HC_CCtx_internal" + referred_type_id: 0x0ec2f9ed +} +typedef { + id: 0xe4dbeedf + name: "LZ4_streamDecode_t" + referred_type_id: 0x7b7c4473 +} +typedef { + id: 0x52eae58e + name: "LZ4_streamDecode_t_internal" + referred_type_id: 0x303af64b +} +typedef { + id: 0x16a4ad88 + name: "LZ4_streamHC_t" + referred_type_id: 0x7b572b19 +} +typedef { + id: 0x24853ea8 + name: "LZ4_stream_t" + referred_type_id: 0x7b6a1bfa +} +typedef { + id: 0x8827e777 + name: "LZ4_stream_t_internal" + referred_type_id: 0x0ec065d6 +} typedef { id: 0x9362c2f7 name: "SeqCollector" @@ -34448,16 +34633,31 @@ typedef { name: "zstd_cctx" referred_type_id: 0x91c5a92d } +typedef { + id: 0xcaefe8c1 + name: "zstd_cdict" + referred_type_id: 0xca006561 +} typedef { id: 0x760c4a75 name: "zstd_compression_parameters" referred_type_id: 0xeeb6e146 } +typedef { + id: 0x87250506 + name: "zstd_custom_mem" + referred_type_id: 0x57313a72 +} typedef { id: 0xb12e6800 name: "zstd_dctx" referred_type_id: 0xd36c0f70 } +typedef { + id: 0xa0b682e4 + name: "zstd_ddict" + referred_type_id: 0xdf046314 +} typedef { id: 0xa7b4c582 name: "zstd_parameters" @@ -35233,6 +35433,11 @@ qualified { qualifier: CONST qualified_type_id: 0x43c73e81 } +qualified { + id: 0xcceba991 + qualifier: CONST + qualified_type_id: 0x43e910ce +} qualified { id: 0xcd0444ad qualifier: CONST @@ -36123,6 +36328,11 @@ qualified { qualifier: CONST qualified_type_id: 0x1d3e5adb } +qualified { + id: 0xdb6512b4 + qualifier: CONST + qualified_type_id: 0x1dd3fc58 +} qualified { id: 0xdb91d338 qualifier: CONST @@ -37223,6 +37433,11 @@ qualified { qualifier: CONST qualified_type_id: 0xc3a94ecd } +qualified { + id: 0xed09ed00 + qualifier: CONST + qualified_type_id: 0xc460028a +} qualified { id: 0xed2778f4 qualifier: CONST @@ -37558,6 +37773,11 @@ qualified { qualifier: CONST qualified_type_id: 0xa0e8fc65 } +qualified { + id: 0xf43c4d1b + qualifier: CONST + qualified_type_id: 0xa0b682e4 +} qualified { id: 0xf43ed6df qualifier: CONST @@ -38258,6 +38478,11 @@ qualified { qualifier: CONST qualified_type_id: 0x8d9a7429 } +qualified { + id: 0xff87e351 + qualifier: CONST + qualified_type_id: 0x8e583bcd +} qualified { id: 0xff98be03 qualifier: CONST @@ -38616,6 +38841,11 @@ array { number_of_elements: 241 element_type_id: 0x0baa70a7 } +array { + id: 0x14f6aed9 + number_of_elements: 32774 + element_type_id: 0xf435685e +} array { id: 0x1537c376 number_of_elements: 20 @@ -39896,6 +40126,11 @@ array { number_of_elements: 2 element_type_id: 0x1a065450 } +array { + id: 0x6a7ab4b2 + number_of_elements: 65536 + element_type_id: 0xc93e017b +} array { id: 0x6a8619af number_of_elements: 2 @@ -40366,6 +40601,11 @@ array { number_of_elements: 9 element_type_id: 0xb02b353a } +array { + id: 0x87673b5a + number_of_elements: 4096 + element_type_id: 0x1c3dbe5a +} array { id: 0x87a3edc2 number_of_elements: 1 @@ -40391,6 +40631,11 @@ array { number_of_elements: 12 element_type_id: 0x11c404ba } +array { + id: 0x8879416f + number_of_elements: 32768 + element_type_id: 0x4585663f +} array { id: 0x8a2ac6fa number_of_elements: 6 @@ -41011,6 +41256,11 @@ array { number_of_elements: 1 element_type_id: 0x33756485 } +array { + id: 0xb93acad6 + number_of_elements: 4 + element_type_id: 0xd5df6730 +} array { id: 0xb993d8a0 number_of_elements: 6 @@ -41617,6 +41867,11 @@ array { id: 0xe09ad666 element_type_id: 0x928d625a } +array { + id: 0xe09e2127 + number_of_elements: 2052 + element_type_id: 0xd5df6730 +} array { id: 0xe0b14214 element_type_id: 0x92233392 @@ -42180,6 +42435,10 @@ member { id: 0x22e86cdc type_id: 0x048fc360 } +member { + id: 0x22fde5dd + type_id: 0x04d9e766 +} member { id: 0x230f36dc type_id: 0x0312ab60 @@ -42188,10 +42447,6 @@ member { id: 0x233f54d1 type_id: 0x03d32356 } -member { - id: 0x23531e28 - type_id: 0x026208b2 -} member { id: 0x2380a48f type_id: 0x012ce22f @@ -42467,6 +42722,10 @@ member { id: 0x2a99a59c type_id: 0x2548e662 } +member { + id: 0x2aac4c8c + type_id: 0x259f4220 +} member { id: 0x2ad28b61 type_id: 0x24645d96 @@ -43296,8 +43555,8 @@ member { offset: 96 } member { - id: 0x34aaae1a - type_id: 0x5d84d1ff + id: 0x34aab4f5 + type_id: 0x5d84ba41 offset: 24 } member { @@ -43349,6 +43608,11 @@ member { type_id: 0x5c5fa7e4 offset: 96 } +member { + id: 0x34dd58d6 + type_id: 0x5c5b3484 + offset: 2240 +} member { id: 0x34df0c4f type_id: 0x5c52412d @@ -43425,11 +43689,21 @@ member { type_id: 0x5a41364f offset: 768 } +member { + id: 0x355c1a4d + type_id: 0x5a5e007a + offset: 28608 +} member { id: 0x355c619d type_id: 0x5a5fd068 offset: 96 } +member { + id: 0x355c8e01 + type_id: 0x5a5c7df4 + offset: 9344 +} member { id: 0x3568359b type_id: 0x5a8e8177 @@ -43511,6 +43785,11 @@ member { type_id: 0x56037e9c offset: 1408 } +member { + id: 0x366394f0 + type_id: 0x56a024aa + offset: 1920 +} member { id: 0x36707abc type_id: 0x56efb9c6 @@ -43778,6 +44057,11 @@ member { id: 0x3a3e9211 type_id: 0x67d43857 } +member { + id: 0x3a6f0eb6 + type_id: 0x66926050 + offset: 896 +} member { id: 0x3a7ed482 type_id: 0x66d53bfd @@ -44082,6 +44366,11 @@ member { type_id: 0x7b23ef96 offset: 448 } +member { + id: 0x3d06fe1e + type_id: 0x7b3589ce + offset: 14976 +} member { id: 0x3d1a48d1 type_id: 0x7b4751fa @@ -48842,6 +49131,12 @@ member { type_id: 0x6720d32f offset: 32 } +member { + id: 0x9f97e4f3 + name: "advertising" + type_id: 0xe62ebf07 + offset: 64 +} member { id: 0x69f9ba62 name: "advertising_eee" @@ -50216,24 +50511,12 @@ member { type_id: 0x92233392 offset: 18176 } -member { - id: 0xb26b83b7 - name: "android_backport_reserved1" - type_id: 0x92233392 - offset: 9344 -} member { id: 0xb26b8cac name: "android_backport_reserved1" type_id: 0x92233392 offset: 23104 } -member { - id: 0xb26b8ea6 - name: "android_backport_reserved1" - type_id: 0x92233392 - offset: 14976 -} member { id: 0xb26b8ecf name: "android_backport_reserved1" @@ -50417,12 +50700,6 @@ member { type_id: 0x92233392 offset: 1280 } -member { - id: 0x2d081365 - name: "android_kabi_reserved1" - type_id: 0x92233392 - offset: 28608 -} member { id: 0x2d081368 name: "android_kabi_reserved1" @@ -55198,6 +55475,12 @@ member { type_id: 0xe62ebf07 offset: 32 } +member { + id: 0xbed54502 + name: "autoneg" + type_id: 0xb3e7bac9 + offset: 144 +} member { id: 0xbed548e9 name: "autoneg" @@ -56671,6 +56954,12 @@ member { name: "base" type_id: 0x3b0d08ae } +member { + id: 0x85e2a01d + name: "base" + type_id: 0x384c5795 + offset: 2097216 +} member { id: 0x85e2db57 name: "base" @@ -57605,6 +57894,12 @@ member { type_id: 0xc8b17aa7 offset: 2112 } +member { + id: 0x5e2b8157 + name: "bh" + type_id: 0xa5960bec + offset: 4544 +} member { id: 0x5e2b8380 name: "bh" @@ -57904,6 +58199,12 @@ member { type_id: 0x2d3f9ded offset: 576 } +member { + id: 0x6d132c4c + name: "bind" + type_id: 0x2d0d996b + offset: 128 +} member { id: 0x6d137dd8 name: "bind" @@ -60460,6 +60761,12 @@ member { type_id: 0x244b9b37 offset: 896 } +member { + id: 0xae892c09 + name: "bufferStart" + type_id: 0x1a6a7d94 + offset: 131200 +} member { id: 0x790ede18 name: "buffer_accessing" @@ -62756,6 +63063,13 @@ member { offset: 8680 bitsize: 1 } +member { + id: 0x372f6ae9 + name: "can_dma_sg" + type_id: 0x4585663f + offset: 960 + bitsize: 1 +} member { id: 0xdf02dfb1 name: "can_do_streams" @@ -64490,6 +64804,12 @@ member { type_id: 0x4585663f offset: 32 } +member { + id: 0x4b989f52 + name: "chainTable" + type_id: 0x6a7ab4b2 + offset: 1048576 +} member { id: 0x4bc073a3 name: "chainTable" @@ -65092,6 +65412,12 @@ member { type_id: 0x2fa652dc offset: 1024 } +member { + id: 0x2afab2be + name: "check_connect" + type_id: 0x2d2c9ff4 + offset: 384 +} member { id: 0xca18cac1 name: "check_contaminant" @@ -69098,6 +69424,12 @@ member { type_id: 0x6720d32f offset: 352 } +member { + id: 0x73e57430 + name: "compressionLevel" + type_id: 0x4585663f + offset: 2097440 +} member { id: 0xd5a82d64 name: "compression_level" @@ -73888,6 +74220,12 @@ member { type_id: 0x6fe0e5a6 offset: 32 } +member { + id: 0x4e7d7717 + name: "currentOffset" + type_id: 0x1c3dbe5a + offset: 131072 +} member { id: 0xfa702ad6 name: "current_bss" @@ -75025,6 +75363,12 @@ member { name: "data" type_id: 0xe62ebf07 } +member { + id: 0xff770e06 + name: "data" + type_id: 0xe5217790 + offset: 1856 +} member { id: 0xff7ae59e name: "data" @@ -75244,6 +75588,12 @@ member { type_id: 0x33756485 offset: 256 } +member { + id: 0xffa15c7e + name: "data" + type_id: 0x33756485 + offset: 1088 +} member { id: 0xffab319e name: "data" @@ -77318,6 +77668,12 @@ member { type_id: 0x33756485 offset: 1920 } +member { + id: 0xf04d7467 + name: "deferred" + type_id: 0xfb1df068 + offset: 4096 +} member { id: 0xf065af4e name: "deferred" @@ -77547,6 +77903,12 @@ member { type_id: 0xe8034002 offset: 16 } +member { + id: 0x84e83fe7 + name: "delay" + type_id: 0xd298e888 + offset: 1216 +} member { id: 0x84f3a6f8 name: "delay" @@ -78392,6 +78754,11 @@ member { type_id: 0x0483e6f8 offset: 64 } +member { + id: 0x317e664b + name: "description" + type_id: 0x0483e6f8 +} member { id: 0x317e66e0 name: "description" @@ -81213,6 +81580,12 @@ member { name: "dict" type_id: 0x391f15ea } +member { + id: 0x12d37402 + name: "dictBase" + type_id: 0x384c5795 + offset: 2097280 +} member { id: 0x12dc8916 name: "dictBase" @@ -81266,6 +81639,12 @@ member { type_id: 0xe03de1c1 offset: 4352 } +member { + id: 0xcc1cb688 + name: "dictLimit" + type_id: 0x4585663f + offset: 2097344 +} member { id: 0xccb90f8f name: "dictLimit" @@ -81278,6 +81657,12 @@ member { type_id: 0x3c534f6c offset: 1856 } +member { + id: 0x5e1f400b + name: "dictSize" + type_id: 0x1c3dbe5a + offset: 131264 +} member { id: 0x5ef7409d name: "dictSize" @@ -81296,6 +81681,12 @@ member { type_id: 0xea9852f4 offset: 241472 } +member { + id: 0x8d0ea612 + name: "dictionary" + type_id: 0x39aad3c3 + offset: 131136 +} member { id: 0x74ab0220 name: "die_revision" @@ -83222,12 +83613,35 @@ member { type_id: 0x00c83ba6 offset: 768 } +member { + id: 0x082d878d + name: "dmabuf_count" + type_id: 0x4585663f + offset: 320 +} +member { + id: 0x32b44094 + name: "dmabuf_info" + type_id: 0x12f234f7 +} +member { + id: 0x32b4483a + name: "dmabuf_info" + type_id: 0x12f234f7 + offset: 768 +} member { id: 0x810e204c name: "dmabufs" type_id: 0xeb923a9b offset: 384 } +member { + id: 0x8136732c + name: "dmabufs" + type_id: 0xd3c80119 + offset: 192 +} member { id: 0x065bb89a name: "dmac" @@ -83497,6 +83911,12 @@ member { name: "done" type_id: 0x4585663f } +member { + id: 0x1d18f5cb + name: "done" + type_id: 0x578f9c2b + offset: 3200 +} member { id: 0x1d400b5c name: "done" @@ -84405,6 +84825,12 @@ member { type_id: 0x42af4f5c offset: 1536 } +member { + id: 0x5e589542 + name: "driver_info" + type_id: 0x31d242e7 + offset: 128 +} member { id: 0x3f6b1d8c name: "driver_input_lock" @@ -84507,6 +84933,12 @@ member { type_id: 0x18bd6530 offset: 448 } +member { + id: 0xe49f6571 + name: "driver_priv" + type_id: 0x18bd6530 + offset: 256 +} member { id: 0xe49f671e name: "driver_priv" @@ -85597,6 +86029,12 @@ member { name: "dup_xol_work" type_id: 0xe3222f5b } +member { + id: 0x703907cc + name: "duplex" + type_id: 0xb3e7bac9 + offset: 112 +} member { id: 0x70390fe0 name: "duplex" @@ -86089,6 +86527,12 @@ member { type_id: 0x4585663f offset: 192 } +member { + id: 0x61d268eb + name: "early_init" + type_id: 0x2d2c9ff4 + offset: 832 +} member { id: 0xdaa08a17 name: "early_init" @@ -87779,6 +88223,12 @@ member { type_id: 0x18bd6530 offset: 64 } +member { + id: 0x82371a25 + name: "end" + type_id: 0x384c5795 + offset: 2097152 +} member { id: 0x823c27da name: "end" @@ -89331,12 +89781,24 @@ member { name: "eth" type_id: 0xe0ea9a24 } +member { + id: 0xca94b2d4 + name: "eth_tp_mdix" + type_id: 0xb3e7bac9 + offset: 240 +} member { id: 0xca94b363 name: "eth_tp_mdix" type_id: 0xb3e7bac9 offset: 104 } +member { + id: 0x8530641a + name: "eth_tp_mdix_ctrl" + type_id: 0xb3e7bac9 + offset: 248 +} member { id: 0x8530689b name: "eth_tp_mdix_ctrl" @@ -90915,6 +91377,12 @@ member { type_id: 0xc6b28f4f offset: 512 } +member { + id: 0xd709b6a7 + name: "extDictSize" + type_id: 0xf435685e + offset: 64 +} member { id: 0x997db32f name: "ext_cap" @@ -91182,6 +91650,11 @@ member { offset: 64 bitsize: 1 } +member { + id: 0xa18dddfc + name: "externalDict" + type_id: 0x39aad3c3 +} member { id: 0x15ff4764 name: "external_debug_state" @@ -95676,6 +96149,12 @@ member { type_id: 0x33756485 offset: 832 } +member { + id: 0x2d5bf2a1 + name: "flags" + type_id: 0x33756485 + offset: 5248 +} member { id: 0x2d5bf2da name: "flags" @@ -99011,6 +99490,17 @@ member { type_id: 0x2dd58efa offset: 1024 } +member { + id: 0x33536f6a + name: "freeze_seq" + type_id: 0xd0e93144 +} +member { + id: 0x4cd1d9b8 + name: "freeze_start_nsec" + type_id: 0x92233392 + offset: 64 +} member { id: 0xebc2b6b0 name: "freeze_super" @@ -99029,6 +99519,11 @@ member { type_id: 0x8eaee9d1 offset: 14752 } +member { + id: 0x14d41b24 + name: "freezer" + type_id: 0x14ab8135 +} member { id: 0xd5e4618a name: "freq" @@ -99308,6 +99803,12 @@ member { type_id: 0x6d7f5ff6 offset: 384 } +member { + id: 0xa90a46d9 + name: "frozen_nsec" + type_id: 0x92233392 + offset: 128 +} member { id: 0x696cf202 name: "frozen_offsets" @@ -104587,6 +105088,12 @@ member { type_id: 0xc93e017b offset: 1840 } +member { + id: 0xb7653122 + name: "hard_mtu" + type_id: 0xc9082b19 + offset: 2208 +} member { id: 0x195dbd54 name: "hard_packet_limit" @@ -104704,6 +105211,12 @@ member { offset: 5604 bitsize: 1 } +member { + id: 0x3d367244 + name: "has_compat_metadata" + type_id: 0x6d7f5ff6 + offset: 368 +} member { id: 0x0f2de2c4 name: "has_conns" @@ -105065,6 +105578,16 @@ member { type_id: 0x329f41d7 offset: 768 } +member { + id: 0x5af1eef6 + name: "hashTable" + type_id: 0x8879416f +} +member { + id: 0x5afef08c + name: "hashTable" + type_id: 0x87673b5a +} member { id: 0xad919854 name: "hashTable3" @@ -112017,6 +112540,18 @@ member { name: "in" type_id: 0x18bd6530 } +member { + id: 0x5ccc4e6f + name: "in" + type_id: 0x6720d32f + offset: 1024 +} +member { + id: 0x5ceeec17 + name: "in" + type_id: 0x4585663f + offset: 992 +} member { id: 0x5ceeecc5 name: "in" @@ -112725,6 +113260,12 @@ member { offset: 1153 bitsize: 1 } +member { + id: 0x9f5f61bb + name: "indication" + type_id: 0x0e747c33 + offset: 896 +} member { id: 0x3d384dd8 name: "indio_dev" @@ -113394,6 +113935,12 @@ member { type_id: 0x1cb0211a offset: 96 } +member { + id: 0x094028d1 + name: "initCheck" + type_id: 0x1c3dbe5a + offset: 131104 +} member { id: 0x231328dc name: "init_ack_masked" @@ -114646,6 +115193,21 @@ member { type_id: 0x18bd6530 offset: 320 } +member { + id: 0x481d5dfa + name: "internal_donotuse" + type_id: 0x2324d4ee +} +member { + id: 0x486c93cb + name: "internal_donotuse" + type_id: 0x52eae58e +} +member { + id: 0x48b65ec9 + name: "internal_donotuse" + type_id: 0x8827e777 +} member { id: 0x0837e836 name: "internal_flags" @@ -114664,6 +115226,12 @@ member { type_id: 0x295c7202 offset: 296 } +member { + id: 0x085b3bc9 + name: "internal_flags" + type_id: 0x4585663f + offset: 1184 +} member { id: 0x589ca94a name: "internal_idx" @@ -114744,12 +115312,24 @@ member { offset: 1216 bitsize: 1 } +member { + id: 0xdd499a75 + name: "interrupt" + type_id: 0x0130219f + offset: 3584 +} member { id: 0xdd4afb18 name: "interrupt" type_id: 0x02578e46 offset: 256 } +member { + id: 0x72db081c + name: "interrupt_count" + type_id: 0x4585663f + offset: 3648 +} member { id: 0xd6a7ee2b name: "interrupt_enable" @@ -114780,6 +115360,12 @@ member { type_id: 0x0130219f offset: 3840 } +member { + id: 0x19761760 + name: "interrupt_mutex" + type_id: 0xa7c362b0 + offset: 3712 +} member { id: 0x88026c22 name: "interrupt_out" @@ -114957,6 +115543,12 @@ member { type_id: 0xada50e68 offset: 2080 } +member { + id: 0x8a1b2331 + name: "intf" + type_id: 0x21069feb + offset: 64 +} member { id: 0x8a1d1999 name: "intf" @@ -116128,6 +116720,11 @@ member { type_id: 0x0c55d62d offset: 512 } +member { + id: 0xc185c03f + name: "iotlb_sync_map" + type_id: 0x2f4f676a +} member { id: 0xc1a6eb7e name: "iotlb_sync_map" @@ -119389,6 +119986,12 @@ member { type_id: 0x28b9ec9a offset: 1792 } +member { + id: 0x7de43299 + name: "kevent" + type_id: 0x1f3c8679 + offset: 4864 +} member { id: 0x20068778 name: "key" @@ -120031,6 +120634,11 @@ member { type_id: 0x6720d32f offset: 14144 } +member { + id: 0xb713907c + name: "kmi_ext_info" + type_id: 0x3e521881 +} member { id: 0x5bb2a4ff name: "kn" @@ -123778,6 +124386,12 @@ member { name: "link_rate" type_id: 0x4585663f } +member { + id: 0x027dc99d + name: "link_reset" + type_id: 0x2d2c9ff4 + offset: 576 +} member { id: 0x8a9ef2e8 name: "link_setup" @@ -126434,6 +127048,12 @@ member { type_id: 0x0d364e53 offset: 64 } +member { + id: 0xce70f3f0 + name: "lowLimit" + type_id: 0x4585663f + offset: 2097376 +} member { id: 0xced54d54 name: "lowLimit" @@ -126600,6 +127220,12 @@ member { type_id: 0xe62ebf07 offset: 96 } +member { + id: 0x6279847e + name: "lp_advertising" + type_id: 0xe62ebf07 + offset: 256 +} member { id: 0x62ffd7b1 name: "lp_advertising" @@ -127456,6 +128082,12 @@ member { type_id: 0xec7c6db2 offset: 1152 } +member { + id: 0x42f7cb5a + name: "manage_power" + type_id: 0x2d4bbf27 + offset: 448 +} member { id: 0x9ea77b6e name: "manage_runtime_start_stop" @@ -131953,6 +132585,12 @@ member { offset: 368 bitsize: 16 } +member { + id: 0x63fb0e8e + name: "maxpacket" + type_id: 0x4585663f + offset: 1152 +} member { id: 0x0806dc1f name: "maxpacket_limit" @@ -131978,6 +132616,18 @@ member { type_id: 0x33756485 offset: 4800 } +member { + id: 0x51858165 + name: "maxrxpkt" + type_id: 0xe62ebf07 + offset: 192 +} +member { + id: 0x349eb14e + name: "maxtxpkt" + type_id: 0xe62ebf07 + offset: 160 +} member { id: 0x32c64e99 name: "maxtype" @@ -132545,6 +133195,12 @@ member { type_id: 0x2ebb318f offset: 256 } +member { + id: 0xe2b8933a + name: "mdio_support" + type_id: 0xb3e7bac9 + offset: 152 +} member { id: 0xe2b8944e name: "mdio_support" @@ -133753,6 +134409,12 @@ member { type_id: 0x18bd6530 offset: 9536 } +member { + id: 0x3ebe7eb6 + name: "mii" + type_id: 0x62249e35 + offset: 2304 +} member { id: 0xbf23beb4 name: "mii_ts" @@ -137203,6 +137865,12 @@ member { type_id: 0x11f20613 offset: 576 } +member { + id: 0x8935554d + name: "msg_enable" + type_id: 0x6720d32f + offset: 1792 +} member { id: 0xc31fa462 name: "msg_flags" @@ -140595,6 +141263,12 @@ member { type_id: 0x0ca27481 offset: 704 } +member { + id: 0x723dee2d + name: "net" + type_id: 0x32a623d7 + offset: 1728 +} member { id: 0x72bc7afa name: "net" @@ -141383,6 +142057,12 @@ member { type_id: 0xe03de1c1 offset: 352 } +member { + id: 0x15d50624 + name: "nextToUpdate" + type_id: 0x4585663f + offset: 2097408 +} member { id: 0x7b6ac82c name: "next_addr" @@ -144563,6 +145243,11 @@ member { type_id: 0x6720d32f offset: 1056 } +member { + id: 0x014cd3d7 + name: "nr_task_refs" + type_id: 0x1f4573ef +} member { id: 0xe321d910 name: "nr_tasks" @@ -150576,6 +151261,18 @@ member { type_id: 0xc440583a offset: 448 } +member { + id: 0xb396f982 + name: "out" + type_id: 0x6720d32f + offset: 1056 +} +member { + id: 0xb3b45209 + name: "out" + type_id: 0x4585663f + offset: 1024 +} member { id: 0xb3b4582d name: "out" @@ -152200,6 +152897,18 @@ member { name: "padding2" type_id: 0x23cbe491 } +member { + id: 0x16657abd + name: "padding4" + type_id: 0x44377683 + offset: 8 +} +member { + id: 0xbedde9a2 + name: "padding_pkt" + type_id: 0x3e10b518 + offset: 1664 +} member { id: 0xe861beb9 name: "paddr" @@ -155848,6 +156557,12 @@ member { type_id: 0x0bfc9031 offset: 64 } +member { + id: 0xcc9165e4 + name: "phy_address" + type_id: 0xb3e7bac9 + offset: 128 +} member { id: 0xcc9168ba name: "phy_address" @@ -155930,6 +156645,12 @@ member { type_id: 0xc9082b19 offset: 19264 } +member { + id: 0x1173939d + name: "phy_mutex" + type_id: 0xa7c362b0 + offset: 512 +} member { id: 0xf21c3d8a name: "phy_num_led_triggers" @@ -157150,12 +157871,24 @@ member { type_id: 0x6d7f5ff6 offset: 72 } +member { + id: 0x1619739e + name: "pkt_cnt" + type_id: 0x5d8155a5 + offset: 904 +} member { id: 0xb6534d07 name: "pkt_ctr" type_id: 0xc9082b19 offset: 64 } +member { + id: 0xe531626b + name: "pkt_err" + type_id: 0x5d8155a5 + offset: 912 +} member { id: 0xa971f923 name: "pkt_offset" @@ -158652,6 +159385,12 @@ member { type_id: 0xb3e7bac9 offset: 72 } +member { + id: 0x4848fa31 + name: "port" + type_id: 0xb3e7bac9 + offset: 120 +} member { id: 0x488e9071 name: "port" @@ -160457,6 +161196,13 @@ member { type_id: 0xe276adef offset: 64 } +member { + id: 0x52bd4ccc + name: "preferpd" + type_id: 0xb3e7bac9 + offset: 4 + bitsize: 1 +} member { id: 0x9f058ff3 name: "preferred_bpp" @@ -160510,6 +161256,18 @@ member { type_id: 0x0365de47 offset: 28288 } +member { + id: 0x1ed5fa06 + name: "prefixEnd" + type_id: 0x39aad3c3 + offset: 128 +} +member { + id: 0xdf18a77a + name: "prefixSize" + type_id: 0xf435685e + offset: 192 +} member { id: 0x5e21364d name: "prefixStart" @@ -166119,6 +166877,11 @@ member { type_id: 0xe62ebf07 offset: 672 } +member { + id: 0xf7da7e97 + name: "ra_honor_pio_pflag" + type_id: 0xb3e7bac9 +} member { id: 0x66aa8ae5 name: "ra_mtu" @@ -169029,6 +169792,12 @@ member { type_id: 0x2d31618f offset: 64 } +member { + id: 0x8a6b6b4e + name: "recover" + type_id: 0x0e6ab35b + offset: 768 +} member { id: 0x8a6bb60c name: "recover" @@ -169592,6 +170361,12 @@ member { name: "refcnt" type_id: 0x1f4573ef } +member { + id: 0xb7dcf16e + name: "refcnt" + type_id: 0xa722c13e + offset: 128 +} member { id: 0xb7dcf1d9 name: "refcnt" @@ -172965,10 +173740,10 @@ member { bitsize: 1 } member { - id: 0x688b9626 + id: 0x72849e28 name: "reserved" type_id: 0xb3e7bac9 - bitsize: 6 + bitsize: 4 } member { id: 0xc5c2fe1d @@ -173773,6 +174548,12 @@ member { type_id: 0x3c14a620 offset: 7232 } +member { + id: 0x6290827d + name: "reset" + type_id: 0x2d2c9ff4 + offset: 256 +} member { id: 0x62908bb1 name: "reset" @@ -175086,6 +175867,12 @@ member { offset: 2824 bitsize: 1 } +member { + id: 0xa64626ae + name: "retry_llist" + type_id: 0x0bf3e543 + offset: 384 +} member { id: 0x0e577c51 name: "retry_long" @@ -176097,6 +176884,13 @@ member { type_id: 0x5d8155a5 offset: 104 } +member { + id: 0xad209f54 + name: "routeraddr" + type_id: 0xb3e7bac9 + offset: 5 + bitsize: 1 +} member { id: 0x74fcbeff name: "routes" @@ -176565,6 +177359,11 @@ member { type_id: 0x914dbfdc offset: 160 } +member { + id: 0x10789eb1 + name: "rss" + type_id: 0x33756485 +} member { id: 0x10ad0965 name: "rss" @@ -176581,6 +177380,12 @@ member { name: "rss_context" type_id: 0xe62ebf07 } +member { + id: 0x027688eb + name: "rss_hwm" + type_id: 0x33756485 + offset: 64 +} member { id: 0x9625f4bd name: "rss_query_en" @@ -177903,6 +178708,12 @@ member { type_id: 0xe62ebf07 offset: 224 } +member { + id: 0x92de6fb0 + name: "rx_fixup" + type_id: 0x2d29d09d + offset: 640 +} member { id: 0xe733e49b name: "rx_frame_errors" @@ -178229,6 +179040,12 @@ member { type_id: 0x578f9c2b offset: 25664 } +member { + id: 0x671dc45f + name: "rx_qlen" + type_id: 0xc93e017b + offset: 928 +} member { id: 0x1fbb54e5 name: "rx_queue" @@ -178320,6 +179137,12 @@ member { type_id: 0x0baa70a7 offset: 352 } +member { + id: 0x0b67960e + name: "rx_speed" + type_id: 0xfc0e1dbd + offset: 2688 +} member { id: 0x15ec0ef9 name: "rx_start_bit_low_too_short_cnt" @@ -178442,6 +179265,12 @@ member { type_id: 0x295c7202 offset: 8 } +member { + id: 0x415177a2 + name: "rx_urb_size" + type_id: 0xf435685e + offset: 2240 +} member { id: 0x6eb6ee7e name: "rx_uses_md_dst" @@ -178527,12 +179356,24 @@ member { type_id: 0x054f691a offset: 1024 } +member { + id: 0x94aefae8 + name: "rxq" + type_id: 0x578f9c2b + offset: 2816 +} member { id: 0x94d9a882 name: "rxq" type_id: 0x20d23755 offset: 256 } +member { + id: 0x6e10a38c + name: "rxq_pause" + type_id: 0x578f9c2b + offset: 3392 +} member { id: 0xd6108810 name: "rxqs" @@ -183825,6 +184666,12 @@ member { type_id: 0x0f392d09 offset: 320 } +member { + id: 0x098f8c42 + name: "set_rx_mode" + type_id: 0x0e6ab35b + offset: 960 +} member { id: 0x142af89e name: "set_rxfh" @@ -189331,6 +190178,12 @@ member { type_id: 0x6d7f5ff6 offset: 832 } +member { + id: 0xa002871a + name: "speed" + type_id: 0xe8034002 + offset: 96 +} member { id: 0xa00cae5d name: "speed" @@ -189407,6 +190260,12 @@ member { name: "speed" type_id: 0x16ee9b34 } +member { + id: 0x93cedaae + name: "speed_hi" + type_id: 0xe8034002 + offset: 224 +} member { id: 0x5ba71926 name: "speed_hz" @@ -192865,11 +193724,23 @@ member { type_id: 0x067c4b9a offset: 896 } +member { + id: 0x2064b31e + name: "status" + type_id: 0x0399ec3e + offset: 1088 +} member { id: 0x2065db16 name: "status" type_id: 0x02fc92e9 } +member { + id: 0x2069471f + name: "status" + type_id: 0x0e6b837a + offset: 512 +} member { id: 0x206a163d name: "status" @@ -193348,6 +194219,12 @@ member { type_id: 0x2c8534d8 offset: 576 } +member { + id: 0x69a55104 + name: "stop" + type_id: 0x2d2c9ff4 + offset: 320 +} member { id: 0x69a5ab33 name: "stop" @@ -195228,6 +196105,12 @@ member { type_id: 0x0c5653bc offset: 128 } +member { + id: 0xdccb64ac + name: "suspend_count" + type_id: 0x5d8155a5 + offset: 896 +} member { id: 0xdcd36a92 name: "suspend_count" @@ -197488,6 +198371,11 @@ member { type_id: 0x4585663f offset: 64 } +member { + id: 0xb60baeb1 + name: "table" + type_id: 0xe09e2127 +} member { id: 0xb6223810 name: "table" @@ -197500,6 +198388,11 @@ member { type_id: 0xc9082b19 offset: 288 } +member { + id: 0xb6520a5a + name: "table" + type_id: 0xb93acad6 +} member { id: 0xb658da11 name: "table" @@ -197548,6 +198441,11 @@ member { name: "table" type_id: 0x1d33485a } +member { + id: 0xb6ffc63e + name: "table" + type_id: 0x14f6aed9 +} member { id: 0x5cce5f02 name: "tableEnd" @@ -202590,6 +203488,12 @@ member { type_id: 0xb3e7bac9 offset: 128 } +member { + id: 0x555e8335 + name: "transceiver" + type_id: 0xb3e7bac9 + offset: 136 +} member { id: 0x625eae55 name: "transfer" @@ -204816,6 +205720,12 @@ member { type_id: 0x295c7202 offset: 9792 } +member { + id: 0xbcd1df4e + name: "tx_fixup" + type_id: 0x10617c73 + offset: 704 +} member { id: 0x34444209 name: "tx_flags" @@ -205186,6 +206096,12 @@ member { type_id: 0xc9082b19 offset: 224 } +member { + id: 0x327a5eed + name: "tx_qlen" + type_id: 0xc93e017b + offset: 944 +} member { id: 0x580681a6 name: "tx_queue_len" @@ -205285,6 +206201,12 @@ member { type_id: 0x0baa70a7 offset: 320 } +member { + id: 0x71b830b2 + name: "tx_speed" + type_id: 0xfc0e1dbd + offset: 2752 +} member { id: 0x4a0d72b2 name: "tx_state" @@ -205453,6 +206375,12 @@ member { name: "txpower" type_id: 0xe8d14bc6 } +member { + id: 0x420ec279 + name: "txq" + type_id: 0x578f9c2b + offset: 3008 +} member { id: 0x427514a8 name: "txq" @@ -207906,6 +208834,12 @@ member { type_id: 0x0e59d199 offset: 192 } +member { + id: 0x5994b26d + name: "unbind" + type_id: 0x0e4bb5c4 + offset: 192 +} member { id: 0x5994ea90 name: "unbind" @@ -218530,6 +219464,12 @@ member { type_id: 0x5d75997e offset: 3264 } +member { + id: 0x4ddbf650 + name: "xid" + type_id: 0xc9082b19 + offset: 2176 +} member { id: 0x3025a984 name: "xlate" @@ -219481,16 +220421,6 @@ struct_union { member_id: 0x2d8a4e32 } } -struct_union { - id: 0x026208b2 - kind: STRUCT - definition { - bytesize: 1 - member_id: 0x688b9626 - member_id: 0xb2dd5b41 - member_id: 0x7adb5caf - } -} struct_union { id: 0x02c70092 kind: STRUCT @@ -219653,6 +220583,18 @@ struct_union { member_id: 0x72d76384 } } +struct_union { + id: 0x04d9e766 + kind: STRUCT + definition { + bytesize: 1 + member_id: 0x72849e28 + member_id: 0x52bd4ccc + member_id: 0xad209f54 + member_id: 0xb2dd5b41 + member_id: 0x7adb5caf + } +} struct_union { id: 0x04ecbf8c kind: STRUCT @@ -220585,6 +221527,35 @@ struct_union { member_id: 0xee98e652 } } +struct_union { + id: 0x0ec065d6 + kind: STRUCT + definition { + bytesize: 16416 + member_id: 0x5afef08c + member_id: 0x4e7d7717 + member_id: 0x094028d1 + member_id: 0x8d0ea612 + member_id: 0xae892c09 + member_id: 0x5e1f400b + } +} +struct_union { + id: 0x0ec2f9ed + kind: STRUCT + definition { + bytesize: 262184 + member_id: 0x5af1eef6 + member_id: 0x4b989f52 + member_id: 0x82371a25 + member_id: 0x85e2a01d + member_id: 0x12d37402 + member_id: 0xcc1cb688 + member_id: 0xce70f3f0 + member_id: 0x15d50624 + member_id: 0x73e57430 + } +} struct_union { id: 0x0efc611f kind: STRUCT @@ -221253,7 +222224,7 @@ struct_union { member_id: 0x090040f7 member_id: 0xca4d1cc3 member_id: 0x2d0816df - member_id: 0xb26b83b7 + member_id: 0x355c8e01 } } struct_union { @@ -221384,6 +222355,16 @@ struct_union { member_id: 0x33e89b57 } } +struct_union { + id: 0x14ab8135 + kind: STRUCT + definition { + bytesize: 24 + member_id: 0x33536f6a + member_id: 0x4cd1d9b8 + member_id: 0xa90a46d9 + } +} struct_union { id: 0x14b5803c kind: STRUCT @@ -222633,6 +223614,15 @@ struct_union { member_id: 0x0193fe4b } } +struct_union { + id: 0x259f4220 + kind: STRUCT + definition { + bytesize: 8 + member_id: 0xf7da7e97 + member_id: 0x16657abd + } +} struct_union { id: 0x260445a5 kind: STRUCT @@ -223372,6 +224362,17 @@ struct_union { member_id: 0xe21dd6a2 } } +struct_union { + id: 0x303af64b + kind: STRUCT + definition { + bytesize: 32 + member_id: 0xa18dddfc + member_id: 0xd709b6a7 + member_id: 0x1ed5fa06 + member_id: 0xdf18a77a + } +} struct_union { id: 0x3050ef8f kind: STRUCT @@ -223553,6 +224554,7 @@ struct_union { member_id: 0x89cec87f member_id: 0x3a0676ee member_id: 0x4987e167 + member_id: 0xa64626ae } } struct_union { @@ -225858,6 +226860,16 @@ struct_union { member_id: 0x0f4b9ba4 } } +struct_union { + id: 0x56a024aa + kind: UNION + definition { + bytesize: 8 + member_id: 0x014cd3d7 + member_id: 0x27000c61 + member_id: 0x36752b74 + } +} struct_union { id: 0x56efb9c6 kind: UNION @@ -225983,6 +226995,26 @@ struct_union { member_id: 0x36752b74 } } +struct_union { + id: 0x5a5c7df4 + kind: UNION + definition { + bytesize: 8 + member_id: 0x32b44094 + member_id: 0x2ef635de + member_id: 0x36752b74 + } +} +struct_union { + id: 0x5a5e007a + kind: UNION + definition { + bytesize: 8 + member_id: 0x32b44094 + member_id: 0x27000c61 + member_id: 0x36752b74 + } +} struct_union { id: 0x5a5fd068 kind: UNION @@ -226139,6 +227171,16 @@ struct_union { member_id: 0x75f74db3 } } +struct_union { + id: 0x5c5b3484 + kind: UNION + definition { + bytesize: 8 + member_id: 0x2aac4c8c + member_id: 0x2bdfee76 + member_id: 0x36752b74 + } +} struct_union { id: 0x5c5fa7e4 kind: UNION @@ -226269,12 +227311,12 @@ struct_union { } } struct_union { - id: 0x5d84d1ff + id: 0x5d84ba41 kind: UNION definition { bytesize: 1 member_id: 0x2ddb63e4 - member_id: 0x23531e28 + member_id: 0x22fde5dd } } struct_union { @@ -227152,6 +228194,16 @@ struct_union { member_id: 0x082b1975 } } +struct_union { + id: 0x66926050 + kind: UNION + definition { + bytesize: 8 + member_id: 0xc185c03f + member_id: 0x27000c61 + member_id: 0x36752b74 + } +} struct_union { id: 0x66d53bfd kind: UNION @@ -228239,6 +229291,16 @@ struct_union { member_id: 0x312d72de } } +struct_union { + id: 0x7b3589ce + kind: UNION + definition { + bytesize: 8 + member_id: 0xb713907c + member_id: 0x2ef635de + member_id: 0x36752b74 + } +} struct_union { id: 0x7b4751fa kind: UNION @@ -228247,6 +229309,24 @@ struct_union { member_id: 0xb6f630ef } } +struct_union { + id: 0x7b572b19 + kind: UNION + definition { + bytesize: 262192 + member_id: 0xb6ffc63e + member_id: 0x481d5dfa + } +} +struct_union { + id: 0x7b6a1bfa + kind: UNION + definition { + bytesize: 16416 + member_id: 0xb60baeb1 + member_id: 0x48b65ec9 + } +} struct_union { id: 0x7b73e7a1 kind: UNION @@ -228259,6 +229339,15 @@ struct_union { member_id: 0x3a8b8086 } } +struct_union { + id: 0x7b7c4473 + kind: UNION + definition { + bytesize: 32 + member_id: 0xb6520a5a + member_id: 0x486c93cb + } +} struct_union { id: 0x7ba2ddfd kind: UNION @@ -233412,7 +234501,7 @@ struct_union { member_id: 0x0d45cc23 member_id: 0x144e1098 member_id: 0x1084e7e7 - member_id: 0xb26b8ea6 + member_id: 0x3d06fe1e member_id: 0xd29089f7 } } @@ -233463,6 +234552,15 @@ struct_union { member_id: 0xea724160 } } +struct_union { + id: 0xd308849a + kind: STRUCT + name: "cgroup_kmi_ext_info" + definition { + bytesize: 24 + member_id: 0x14d41b24 + } +} struct_union { id: 0x74e43e85 kind: STRUCT @@ -233643,6 +234741,11 @@ struct_union { member_id: 0xacd55a4c } } +struct_union { + id: 0x0e3dad42 + kind: STRUCT + name: "class_compat" +} struct_union { id: 0x48a13124 kind: STRUCT @@ -237174,7 +238277,7 @@ struct_union { member_id: 0xb265164c member_id: 0xaf33fcbc member_id: 0x5f2a8271 - member_id: 0x2d0814ec + member_id: 0x366394f0 member_id: 0x63760b26 } } @@ -237787,6 +238890,35 @@ struct_union { kind: STRUCT name: "driver_data" } +struct_union { + id: 0xc460028a + kind: STRUCT + name: "driver_info" + definition { + bytesize: 160 + member_id: 0x317e664b + member_id: 0x2d0fa06c + member_id: 0x6d132c4c + member_id: 0x5994b26d + member_id: 0x6290827d + member_id: 0x69a55104 + member_id: 0x2afab2be + member_id: 0x42f7cb5a + member_id: 0x2069471f + member_id: 0x027dc99d + member_id: 0x92de6fb0 + member_id: 0xbcd1df4e + member_id: 0x8a6b6b4e + member_id: 0x61d268eb + member_id: 0x9f5f61bb + member_id: 0x098f8c42 + member_id: 0x5ccc4e6f + member_id: 0xb396f982 + member_id: 0xffa15c7e + member_id: 0x2d081bee + member_id: 0x63760b66 + } +} struct_union { id: 0x3812b02e kind: STRUCT @@ -238875,6 +240007,7 @@ struct_union { member_id: 0x2d0fa9fb member_id: 0x25732f5e member_id: 0x6d30506c + member_id: 0x085b3bc9 member_id: 0x1b7ec983 member_id: 0x75d72bf4 } @@ -241172,6 +242305,31 @@ struct_union { member_id: 0x26f5c78d } } +struct_union { + id: 0x1a60cc51 + kind: STRUCT + name: "ethtool_cmd" + definition { + bytesize: 44 + member_id: 0x1af6f030 + member_id: 0x878d4469 + member_id: 0x9f97e4f3 + member_id: 0xa002871a + member_id: 0x703907cc + member_id: 0x4848fa31 + member_id: 0xcc9165e4 + member_id: 0x555e8335 + member_id: 0xbed54502 + member_id: 0xe2b8933a + member_id: 0x349eb14e + member_id: 0x51858165 + member_id: 0x93cedaae + member_id: 0xca94b2d4 + member_id: 0x8530641a + member_id: 0x6279847e + member_id: 0xd4368a4c + } +} struct_union { id: 0xe87b6f2e kind: STRUCT @@ -243052,6 +244210,7 @@ struct_union { member_id: 0x3b07b402 member_id: 0xf0dd114d member_id: 0x654043c9 + member_id: 0x32b4483a member_id: 0xfca11463 member_id: 0xa92fa8fd member_id: 0x1aac0c74 @@ -251403,7 +252562,7 @@ struct_union { member_id: 0x2d0812b0 member_id: 0x637607e0 member_id: 0xac894cc9 - member_id: 0xe0f63db8 + member_id: 0x34dd58d6 } } struct_union { @@ -253656,7 +254815,7 @@ struct_union { member_id: 0xa60fc2c2 member_id: 0xd12a0a5e member_id: 0x80f5a8ec - member_id: 0x2d081f94 + member_id: 0x3a6f0eb6 member_id: 0x63760151 member_id: 0xac894e49 member_id: 0xe0f6393d @@ -262893,7 +264052,7 @@ struct_union { member_id: 0x5ce532c4 member_id: 0xb5de8e04 member_id: 0x2165da89 - member_id: 0x34aaae1a + member_id: 0x34aab4f5 member_id: 0xe91a1d71 member_id: 0xbada6e7d member_id: 0x08cabd3c @@ -273188,6 +274347,20 @@ struct_union { member_id: 0x528c4588 } } +struct_union { + id: 0x61883543 + kind: STRUCT + name: "task_dma_buf_info" + definition { + bytesize: 48 + member_id: 0x10789eb1 + member_id: 0x027688eb + member_id: 0xb7dcf16e + member_id: 0x2d1fe165 + member_id: 0x8136732c + member_id: 0x082d878d + } +} struct_union { id: 0x84d533ac kind: STRUCT @@ -273447,7 +274620,7 @@ struct_union { member_id: 0xedf50eb4 member_id: 0xd666c042 member_id: 0x9a3ad2fc - member_id: 0x2d081365 + member_id: 0x355c1a4d member_id: 0x63760904 member_id: 0xac894e01 member_id: 0xe0f6361e @@ -279273,6 +280446,57 @@ struct_union { member_id: 0x4b47d823 } } +struct_union { + id: 0x1d6740b1 + kind: STRUCT + name: "usbnet" + definition { + bytesize: 696 + member_id: 0xbb0eb3a5 + member_id: 0x8a1b2331 + member_id: 0x5e589542 + member_id: 0xf415f74d + member_id: 0xe49f6571 + member_id: 0x202d9055 + member_id: 0x1173939d + member_id: 0xdccb64ac + member_id: 0x1619739e + member_id: 0xe531626b + member_id: 0x671dc45f + member_id: 0x327a5eed + member_id: 0x372f6ae9 + member_id: 0x5ceeec17 + member_id: 0xb3b45209 + member_id: 0x2064b31e + member_id: 0x63fb0e8e + member_id: 0x84e83fe7 + member_id: 0xbedde9a2 + member_id: 0x723dee2d + member_id: 0x8935554d + member_id: 0xff770e06 + member_id: 0x4ddbf650 + member_id: 0xb7653122 + member_id: 0x415177a2 + member_id: 0x3ebe7eb6 + member_id: 0x0b67960e + member_id: 0x71b830b2 + member_id: 0x94aefae8 + member_id: 0x420ec279 + member_id: 0x1d18f5cb + member_id: 0x6e10a38c + member_id: 0xdd499a75 + member_id: 0x72db081c + member_id: 0x19761760 + member_id: 0xf04d7467 + member_id: 0x5e2b8157 + member_id: 0x7de43299 + member_id: 0x2d5bf2a1 + member_id: 0x2d0819a8 + member_id: 0x63760f99 + member_id: 0xac894d81 + member_id: 0xe0f63ef4 + } +} struct_union { id: 0x05ebf2f3 kind: STRUCT @@ -285591,6 +286815,7 @@ struct_union { member_id: 0x7c00ef52 member_id: 0x0dfb76d6 member_id: 0x94a26ac4 + member_id: 0x3d367244 member_id: 0xe615f34c member_id: 0x67e96e72 member_id: 0xa4fb8ecb @@ -285640,6 +286865,7 @@ struct_union { member_id: 0x7c00ef52 member_id: 0x0dfb76d6 member_id: 0x94a26ac4 + member_id: 0x3d367244 member_id: 0xb39b94b9 member_id: 0x67e9cc00 member_id: 0xa4fb3edf @@ -287787,7 +289013,7 @@ enumeration { } enumerator { name: "__BPF_ARG_TYPE_LIMIT" - value: 33554431 + value: 67108863 } } } @@ -289459,7 +290685,7 @@ enumeration { } enumerator { name: "__BPF_REG_TYPE_LIMIT" - value: 33554431 + value: 67108863 } } } @@ -289541,7 +290767,7 @@ enumeration { } enumerator { name: "__BPF_RET_TYPE_LIMIT" - value: 33554431 + value: 67108863 } } } @@ -305139,6 +306365,14 @@ function { return_type_id: 0x181ee594 parameter_id: 0xe90b32b7 } +function { + id: 0x043c4ad0 + return_type_id: 0x382bc397 + parameter_id: 0x391f15ea + parameter_id: 0xf435685e + parameter_id: 0x760c4a75 + parameter_id: 0x87250506 +} function { id: 0x044f79a4 return_type_id: 0xbdd18903 @@ -305362,6 +306596,12 @@ function { parameter_id: 0x21d7b2e4 parameter_id: 0x0c3d2d88 } +function { + id: 0x09df0d20 + return_type_id: 0x48b5725f + parameter_id: 0x6720d32f + parameter_id: 0x23da1e1b +} function { id: 0x09fe2e07 return_type_id: 0x33f8b54b @@ -305615,6 +306855,11 @@ function { parameter_id: 0x397d00ab parameter_id: 0x6720d32f } +function { + id: 0x0ef7ccf3 + return_type_id: 0x48b5725f + parameter_id: 0x79be7582 +} function { id: 0x0f2f7206 return_type_id: 0x17fa285b @@ -306743,6 +307988,12 @@ function { parameter_id: 0x6720d32f parameter_id: 0x6d7f5ff6 } +function { + id: 0x10ff6b9a + return_type_id: 0x48b5725f + parameter_id: 0x0258f96e + parameter_id: 0x3c4114bd +} function { id: 0x10ffd8df return_type_id: 0x48b5725f @@ -307617,6 +308868,12 @@ function { parameter_id: 0x3ea31487 parameter_id: 0x4585663f } +function { + id: 0x1201f8b1 + return_type_id: 0x48b5725f + parameter_id: 0x0a2e9ae5 + parameter_id: 0x0483e6f8 +} function { id: 0x12051b11 return_type_id: 0x48b5725f @@ -307665,6 +308922,11 @@ function { parameter_id: 0x0c56e1ee parameter_id: 0x6720d32f } +function { + id: 0x1213f72a + return_type_id: 0x48b5725f + parameter_id: 0x0a2e9ae5 +} function { id: 0x121493f8 return_type_id: 0x48b5725f @@ -307931,6 +309193,12 @@ function { return_type_id: 0x48b5725f parameter_id: 0x0bb0c019 } +function { + id: 0x1276a8bd + return_type_id: 0x48b5725f + parameter_id: 0x0dc9e98b + parameter_id: 0x6720d32f +} function { id: 0x1277c151 return_type_id: 0x48b5725f @@ -308159,6 +309427,12 @@ function { return_type_id: 0x48b5725f parameter_id: 0x09427c40 } +function { + id: 0x12ca966e + return_type_id: 0x48b5725f + parameter_id: 0x0f3912c5 + parameter_id: 0x6720d32f +} function { id: 0x12cacb36 return_type_id: 0x48b5725f @@ -308625,6 +309899,12 @@ function { parameter_id: 0x0e5eb556 parameter_id: 0x18bd6530 } +function { + id: 0x136e318e + return_type_id: 0x48b5725f + parameter_id: 0x0dc9e98b + parameter_id: 0x21069feb +} function { id: 0x136e393b return_type_id: 0x48b5725f @@ -308747,6 +310027,13 @@ function { parameter_id: 0x3399c453 parameter_id: 0x384c5795 } +function { + id: 0x13911651 + return_type_id: 0x48b5725f + parameter_id: 0x0dc9e98b + parameter_id: 0x18bd6530 + parameter_id: 0x6720d32f +} function { id: 0x1393d7c4 return_type_id: 0x48b5725f @@ -308979,6 +310266,17 @@ function { parameter_id: 0x0e6ba6f7 parameter_id: 0x3a9fa744 } +function { + id: 0x13ea2bf1 + return_type_id: 0x48b5725f + parameter_id: 0x0dc9e98b +} +function { + id: 0x13eeeb77 + return_type_id: 0x48b5725f + parameter_id: 0x0dc9e98b + parameter_id: 0x0130219f +} function { id: 0x13f12250 return_type_id: 0x48b5725f @@ -309034,6 +310332,12 @@ function { return_type_id: 0x48b5725f parameter_id: 0x0d9a04b9 } +function { + id: 0x13ff1655 + return_type_id: 0x48b5725f + parameter_id: 0x0dc9e98b + parameter_id: 0x054f691a +} function { id: 0x1402b86b return_type_id: 0x48b5725f @@ -309172,6 +310476,12 @@ function { parameter_id: 0x019dcb45 parameter_id: 0x228954ca } +function { + id: 0x142e3633 + return_type_id: 0x48b5725f + parameter_id: 0x12191e2a + parameter_id: 0x0c080ab3 +} function { id: 0x142e61fd return_type_id: 0x48b5725f @@ -309368,6 +310678,12 @@ function { parameter_id: 0x12191e2a parameter_id: 0x19357319 } +function { + id: 0x147b2718 + return_type_id: 0x48b5725f + parameter_id: 0x119c80ef + parameter_id: 0x2115ac15 +} function { id: 0x147c9fd8 return_type_id: 0x48b5725f @@ -309753,6 +311069,13 @@ function { parameter_id: 0x3e10b518 parameter_id: 0x1b44744f } +function { + id: 0x150da89f + return_type_id: 0x48b5725f + parameter_id: 0x1042c9d1 + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f +} function { id: 0x150f2137 return_type_id: 0x48b5725f @@ -310829,6 +312152,11 @@ function { parameter_id: 0x3e10b518 parameter_id: 0xf435685e } +function { + id: 0x169466ca + return_type_id: 0x48b5725f + parameter_id: 0x1830dd64 +} function { id: 0x169718bc return_type_id: 0x48b5725f @@ -311317,6 +312645,12 @@ function { parameter_id: 0x3bfbd7cf parameter_id: 0x0f626ee5 } +function { + id: 0x17267b6f + return_type_id: 0xd5cc9c9a + parameter_id: 0x0cbf60eb + parameter_id: 0x3e10b518 +} function { id: 0x1726cb6c return_type_id: 0x48b5725f @@ -314043,6 +315377,12 @@ function { return_type_id: 0x48b5725f parameter_id: 0x2c518f27 } +function { + id: 0x1b8fc015 + return_type_id: 0x48b5725f + parameter_id: 0x2e94a1e1 + parameter_id: 0x2cae7fa9 +} function { id: 0x1b90a8d8 return_type_id: 0x48b5725f @@ -314676,6 +316016,12 @@ function { parameter_id: 0x274194ac parameter_id: 0x2e18f543 } +function { + id: 0x1c56d411 + return_type_id: 0x48b5725f + parameter_id: 0x33756485 + parameter_id: 0x04f728c1 +} function { id: 0x1c5815c3 return_type_id: 0x48b5725f @@ -314875,6 +316221,13 @@ function { parameter_id: 0x3760766d parameter_id: 0x7194a543 } +function { + id: 0x1c86b5f0 + return_type_id: 0x48b5725f + parameter_id: 0x3176a085 + parameter_id: 0x11cfee5a + parameter_id: 0x11cfee5a +} function { id: 0x1c88728f return_type_id: 0x48b5725f @@ -317482,6 +318835,12 @@ function { parameter_id: 0x3df8f8c1 parameter_id: 0x2e8d5f05 } +function { + id: 0x1f5cc15e + return_type_id: 0x48b5725f + parameter_id: 0x3ead5537 + parameter_id: 0x1bf16028 +} function { id: 0x1f5d7673 return_type_id: 0xd5cc9c9a @@ -317573,6 +318932,12 @@ function { parameter_id: 0x5d8155a5 parameter_id: 0xc93e017b } +function { + id: 0x1f841b2f + return_type_id: 0x48b5725f + parameter_id: 0x3e6239e1 + parameter_id: 0x2131312a +} function { id: 0x1f849f7d return_type_id: 0x48b5725f @@ -317728,6 +319093,12 @@ function { parameter_id: 0x4585663f parameter_id: 0x4585663f } +function { + id: 0x1fadab22 + return_type_id: 0x48b5725f + parameter_id: 0x3f84ee3c + parameter_id: 0x35304fb1 +} function { id: 0x1fae5edf return_type_id: 0x48b5725f @@ -317758,6 +319129,12 @@ function { return_type_id: 0x48b5725f parameter_id: 0x3ca2533c } +function { + id: 0x1fb4bf8e + return_type_id: 0x48b5725f + parameter_id: 0x3f84ee3c + parameter_id: 0x33756485 +} function { id: 0x1fb5d3a2 return_type_id: 0x48b5725f @@ -318308,6 +319685,14 @@ function { parameter_id: 0x25ffeea5 parameter_id: 0x0258f96e } +function { + id: 0x299442f2 + return_type_id: 0x0490bb4a + parameter_id: 0x4585663f + parameter_id: 0x6720d32f + parameter_id: 0x3e10b518 + parameter_id: 0x25e45be7 +} function { id: 0x29d15eb9 return_type_id: 0x33756485 @@ -318963,6 +320348,14 @@ function { return_type_id: 0x33756485 parameter_id: 0x15a30023 } +function { + id: 0x34f12b6b + return_type_id: 0xe62ebf07 + parameter_id: 0x3c497d0a + parameter_id: 0x00c72527 + parameter_id: 0x4585663f + parameter_id: 0x4585663f +} function { id: 0x34f176e3 return_type_id: 0x33756485 @@ -319057,6 +320450,11 @@ function { parameter_id: 0x347303b4 parameter_id: 0x3e10b518 } +function { + id: 0x356e90af + return_type_id: 0x23f6f3f6 + parameter_id: 0x27162aac +} function { id: 0x35757b7f return_type_id: 0xe62ebf07 @@ -319077,6 +320475,13 @@ function { parameter_id: 0x0258f96e parameter_id: 0x0ae4a2c7 } +function { + id: 0x35cdf327 + return_type_id: 0x045d9eb9 + parameter_id: 0x0258f96e + parameter_id: 0x0ae4a2c7 + parameter_id: 0x3e10b518 +} function { id: 0x35de1544 return_type_id: 0x48b5725f @@ -320200,6 +321605,14 @@ function { return_type_id: 0x18bd6530 parameter_id: 0x4585663f } +function { + id: 0x4812e6a9 + return_type_id: 0x35304fb1 + parameter_id: 0x3f84ee3c + parameter_id: 0x33756485 + parameter_id: 0x33756485 + parameter_id: 0x6d7f5ff6 +} function { id: 0x48135354 return_type_id: 0x35304fb1 @@ -320458,6 +321871,11 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x6720d32f } +function { + id: 0x4f10276c + return_type_id: 0x213700a8 + parameter_id: 0x0258f96e +} function { id: 0x4f33ba92 return_type_id: 0x21003da7 @@ -321196,6 +322614,13 @@ function { parameter_id: 0xf435685e parameter_id: 0xf435685e } +function { + id: 0x56fc975d + return_type_id: 0x92233392 + parameter_id: 0x0665e6b6 + parameter_id: 0x26ad5dc1 + parameter_id: 0x92233392 +} function { id: 0x56fd367d return_type_id: 0x18bd6530 @@ -322274,6 +323699,12 @@ function { parameter_id: 0x295c7202 parameter_id: 0xc9082b19 } +function { + id: 0x62ea57b4 + return_type_id: 0x1d19a9d5 + parameter_id: 0x6720d32f + parameter_id: 0x1bf16028 +} function { id: 0x633d6ee0 return_type_id: 0x054f691a @@ -322290,6 +323721,11 @@ function { return_type_id: 0x2cae7fa9 parameter_id: 0x0258f96e } +function { + id: 0x636e6c75 + return_type_id: 0x38b136d3 + parameter_id: 0x2309ad3e +} function { id: 0x637004ab return_type_id: 0x1582ab06 @@ -322316,6 +323752,11 @@ function { parameter_id: 0x257d12af parameter_id: 0x0bb4e60f } +function { + id: 0x6424be43 + return_type_id: 0x091f52f7 + parameter_id: 0x3e10b518 +} function { id: 0x643d13df return_type_id: 0x054f691a @@ -322638,6 +324079,13 @@ function { id: 0x6bb7aee3 return_type_id: 0x7b6559c5 } +function { + id: 0x6bc51750 + return_type_id: 0x054f691a + parameter_id: 0x0dc9e98b + parameter_id: 0x054f691a + parameter_id: 0xf1a6dfed +} function { id: 0x6c0528d9 return_type_id: 0xa256f9e0 @@ -322856,6 +324304,13 @@ function { parameter_id: 0x32caaf24 parameter_id: 0x1bf16028 } +function { + id: 0x6f9910f4 + return_type_id: 0x1d44326e + parameter_id: 0x0ca27481 + parameter_id: 0x3e6396e0 + parameter_id: 0x3399c453 +} function { id: 0x6f9cf068 return_type_id: 0x1d44326e @@ -323653,6 +325108,12 @@ function { parameter_id: 0x0da985ba parameter_id: 0x3e10b518 } +function { + id: 0x8265b46c + return_type_id: 0x120540d1 + parameter_id: 0x3bb11c92 + parameter_id: 0x3e001c39 +} function { id: 0x826967e0 return_type_id: 0x2e2c982d @@ -324859,6 +326320,14 @@ function { parameter_id: 0x3ae3ff84 parameter_id: 0xc9082b19 } +function { + id: 0x90203fcd + return_type_id: 0x6720d32f + parameter_id: 0x3571c173 + parameter_id: 0x3e001c39 + parameter_id: 0x1df06cce + parameter_id: 0xf435685e +} function { id: 0x90208300 return_type_id: 0x6720d32f @@ -326981,6 +328450,13 @@ function { parameter_id: 0xc9082b19 parameter_id: 0x6720d32f } +function { + id: 0x9188d94b + return_type_id: 0x6720d32f + parameter_id: 0x33a6c210 + parameter_id: 0x3e10b518 + parameter_id: 0x6720d32f +} function { id: 0x9189b7a1 return_type_id: 0x6720d32f @@ -327046,6 +328522,15 @@ function { parameter_id: 0x33756485 parameter_id: 0x04025b72 } +function { + id: 0x9191b4cd + return_type_id: 0x6720d32f + parameter_id: 0x33a6c210 + parameter_id: 0x3e10b518 + parameter_id: 0x0483e6f8 + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f +} function { id: 0x91935bd4 return_type_id: 0x6720d32f @@ -327376,6 +328861,12 @@ function { parameter_id: 0x4585663f parameter_id: 0xf1a6dfed } +function { + id: 0x91bef2c3 + return_type_id: 0x6720d32f + parameter_id: 0x33756485 + parameter_id: 0x38d23361 +} function { id: 0x91bf7954 return_type_id: 0x6720d32f @@ -328146,6 +329637,11 @@ function { parameter_id: 0xc9082b19 parameter_id: 0x308ee26c } +function { + id: 0x9203828a + return_type_id: 0x6720d32f + parameter_id: 0x3e0d8696 +} function { id: 0x920433df return_type_id: 0x6720d32f @@ -328215,6 +329711,16 @@ function { parameter_id: 0x6720d32f parameter_id: 0x6720d32f } +function { + id: 0x920e8cef + return_type_id: 0x6720d32f + parameter_id: 0x3e10b518 + parameter_id: 0x0483e6f8 + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f + parameter_id: 0x18bd6530 +} function { id: 0x920e9545 return_type_id: 0x6720d32f @@ -329005,6 +330511,12 @@ function { parameter_id: 0x391f15ea parameter_id: 0xf435685e } +function { + id: 0x9294d8c1 + return_type_id: 0x6720d32f + parameter_id: 0x3c01aef6 + parameter_id: 0x051414e1 +} function { id: 0x92956fd0 return_type_id: 0x6720d32f @@ -330493,6 +332005,14 @@ function { parameter_id: 0x18ea6ae3 parameter_id: 0x4585663f } +function { + id: 0x93b7b77f + return_type_id: 0x6720d32f + parameter_id: 0x3ba261b0 + parameter_id: 0x384c5795 + parameter_id: 0xf435685e + parameter_id: 0xfc0e1dbd +} function { id: 0x93b941f7 return_type_id: 0x6720d32f @@ -332325,6 +333845,14 @@ function { parameter_id: 0x2e94a1e1 parameter_id: 0xe02e14d6 } +function { + id: 0x95a5d553 + return_type_id: 0x6720d32f + parameter_id: 0x2cc1c3e8 + parameter_id: 0xc9082b19 + parameter_id: 0xc9082b19 + parameter_id: 0xc9082b19 +} function { id: 0x95a77977 return_type_id: 0x6720d32f @@ -333710,6 +335238,13 @@ function { parameter_id: 0x2bc93f35 parameter_id: 0x0258f96e } +function { + id: 0x977d7b37 + return_type_id: 0x6720d32f + parameter_id: 0x28350343 + parameter_id: 0x33756485 + parameter_id: 0xf435685e +} function { id: 0x977e98cb return_type_id: 0x6720d32f @@ -334459,6 +335994,16 @@ function { parameter_id: 0x4585663f parameter_id: 0x4585663f } +function { + id: 0x986040b6 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0xfc0e1dbd + parameter_id: 0xfc0e1dbd + parameter_id: 0x33756485 + parameter_id: 0x33756485 + parameter_id: 0x0b30ee00 +} function { id: 0x9860482f return_type_id: 0x6720d32f @@ -334689,6 +336234,11 @@ function { parameter_id: 0x27a7c613 parameter_id: 0x2cacacc0 } +function { + id: 0x98764c5c + return_type_id: 0x6720d32f + parameter_id: 0x17dabdcd +} function { id: 0x98773f07 return_type_id: 0x6720d32f @@ -334696,6 +336246,17 @@ function { parameter_id: 0x27a7c613 parameter_id: 0x27a7c613 } +function { + id: 0x98787515 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0xf1a6dfed + parameter_id: 0x4585663f + parameter_id: 0x0d3144be + parameter_id: 0x06cbb7fd + parameter_id: 0x0850608e + parameter_id: 0x1bf16028 +} function { id: 0x98788d90 return_type_id: 0x6720d32f @@ -335586,6 +337147,11 @@ function { parameter_id: 0x11e6864c parameter_id: 0x33756485 } +function { + id: 0x993539b7 + return_type_id: 0xf435685e + parameter_id: 0x382bc397 +} function { id: 0x99361691 return_type_id: 0x6720d32f @@ -336904,6 +338470,15 @@ function { parameter_id: 0x1d19a9d5 parameter_id: 0x13580d6c } +function { + id: 0x9a2ab257 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f + parameter_id: 0x11cfee5a + parameter_id: 0x11cfee5a +} function { id: 0x9a2ab624 return_type_id: 0x6720d32f @@ -337637,6 +339212,12 @@ function { parameter_id: 0x1fba7d3d parameter_id: 0x3e5d9ebb } +function { + id: 0x9a98740b + return_type_id: 0x6720d32f + parameter_id: 0x1a1051a0 + parameter_id: 0x6720d32f +} function { id: 0x9aa0b156 return_type_id: 0x6720d32f @@ -337644,6 +339225,15 @@ function { parameter_id: 0x4585663f parameter_id: 0x33756485 } +function { + id: 0x9aa1ff3f + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x4585663f + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f + parameter_id: 0x11cfee5a +} function { id: 0x9aa2d024 return_type_id: 0x6720d32f @@ -338219,6 +339809,20 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x2b16c036 } +function { + id: 0x9b047f13 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x29082583 + parameter_id: 0x3f979b84 +} +function { + id: 0x9b04a709 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x2b16c036 + parameter_id: 0x1d19a9d5 +} function { id: 0x9b04b788 return_type_id: 0x6720d32f @@ -338226,6 +339830,11 @@ function { parameter_id: 0x2b16c036 parameter_id: 0x1d5bae2a } +function { + id: 0x9b04f747 + return_type_id: 0x6720d32f + parameter_id: 0x1a1051a0 +} function { id: 0x9b072131 return_type_id: 0x6720d32f @@ -338338,6 +339947,14 @@ function { parameter_id: 0xf435685e parameter_id: 0x15afc3b7 } +function { + id: 0x9b1007fb + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x2e029f76 + parameter_id: 0x1d19a9d5 + parameter_id: 0x3c63caea +} function { id: 0x9b10b4a6 return_type_id: 0x6720d32f @@ -338408,6 +340025,13 @@ function { parameter_id: 0x2e18f543 parameter_id: 0x11cfee5a } +function { + id: 0x9b13c3e5 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x2e029f76 + parameter_id: 0x11cfee5a +} function { id: 0x9b144a97 return_type_id: 0x6720d32f @@ -338415,6 +340039,16 @@ function { parameter_id: 0x0483e6f8 parameter_id: 0xf435685e } +function { + id: 0x9b14e056 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x2e029f76 + parameter_id: 0x0dd653d2 + parameter_id: 0x089ac080 + parameter_id: 0x1d19a9d5 + parameter_id: 0x3c63caea +} function { id: 0x9b14e220 return_type_id: 0x6720d32f @@ -338554,6 +340188,14 @@ function { parameter_id: 0x1b8590a8 parameter_id: 0x107606b0 } +function { + id: 0x9b207e3e + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x2208f89a + parameter_id: 0x1e820193 + parameter_id: 0x11cfee5a +} function { id: 0x9b222516 return_type_id: 0x6720d32f @@ -338669,6 +340311,13 @@ function { parameter_id: 0x2360e10b parameter_id: 0x13580d6c } +function { + id: 0x9b27064b + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x21df69d0 + parameter_id: 0x3f063da2 +} function { id: 0x9b273c8b return_type_id: 0x6720d32f @@ -338939,6 +340588,12 @@ function { parameter_id: 0x6720d32f parameter_id: 0x11cfee5a } +function { + id: 0x9b3403c3 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x26ee682a +} function { id: 0x9b35e16b return_type_id: 0x6720d32f @@ -338967,6 +340622,16 @@ function { parameter_id: 0x4585663f parameter_id: 0x1bf16028 } +function { + id: 0x9b37ef14 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x275ab027 + parameter_id: 0x12bb35ff + parameter_id: 0x6720d32f + parameter_id: 0x36f98e5c + parameter_id: 0x11cfee5a +} function { id: 0x9b37fa13 return_type_id: 0x6720d32f @@ -339140,6 +340805,15 @@ function { parameter_id: 0x00c72527 parameter_id: 0x13580d6c } +function { + id: 0x9b430741 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x39182992 + parameter_id: 0x2170d06d + parameter_id: 0x2170d06d + parameter_id: 0x11cfee5a +} function { id: 0x9b43a9b0 return_type_id: 0x6720d32f @@ -339266,6 +340940,12 @@ function { parameter_id: 0x38fa32ef parameter_id: 0x13580d6c } +function { + id: 0x9b48ef76 + return_type_id: 0x6720d32f + parameter_id: 0x1830dd64 + parameter_id: 0x310ec01d +} function { id: 0x9b49a977 return_type_id: 0x6720d32f @@ -339701,6 +341381,13 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x3df9bbb6 } +function { + id: 0x9b587c9a + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x3e10b518 + parameter_id: 0x3e10b518 +} function { id: 0x9b5888dd return_type_id: 0x6720d32f @@ -340156,6 +341843,19 @@ function { parameter_id: 0x3e10b518 parameter_id: 0xf435685e } +function { + id: 0x9b6c13bd + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x33756485 + parameter_id: 0x39182992 + parameter_id: 0x0d164218 + parameter_id: 0x3e6239e1 + parameter_id: 0x6720d32f + parameter_id: 0x6d7f5ff6 + parameter_id: 0x064d6086 + parameter_id: 0x064d6086 +} function { id: 0x9b6cf0a4 return_type_id: 0x6720d32f @@ -340614,6 +342314,21 @@ function { parameter_id: 0x3caf1899 parameter_id: 0x0a2e9ae5 } +function { + id: 0x9b82ade6 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x0a63398f + parameter_id: 0x13580d6c + parameter_id: 0x13580d6c +} +function { + id: 0x9b82e086 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x0a63398f + parameter_id: 0x13580d6c +} function { id: 0x9b83293d return_type_id: 0x6720d32f @@ -340742,6 +342457,17 @@ function { parameter_id: 0xf435685e parameter_id: 0x17a93606 } +function { + id: 0x9b87e976 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x0a63398f + parameter_id: 0x064d6086 + parameter_id: 0x1321a5b2 + parameter_id: 0x06835e9c + parameter_id: 0x6720d32f + parameter_id: 0x4585663f +} function { id: 0x9b87f766 return_type_id: 0x6720d32f @@ -340810,6 +342536,12 @@ function { parameter_id: 0x36311c57 parameter_id: 0x18bd6530 } +function { + id: 0x9b8cc3ad + return_type_id: 0x6720d32f + parameter_id: 0x18150d9f + parameter_id: 0x0258f96e +} function { id: 0x9b8e1adc return_type_id: 0x6720d32f @@ -340934,6 +342666,13 @@ function { parameter_id: 0x92233392 parameter_id: 0x92233392 } +function { + id: 0x9b93638b + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x0dd653d2 + parameter_id: 0x2e029f76 +} function { id: 0x9b938987 return_type_id: 0x6720d32f @@ -340986,6 +342725,15 @@ function { parameter_id: 0x0ed82db2 parameter_id: 0x082be49e } +function { + id: 0x9b973ccf + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x0a63398f + parameter_id: 0x4585663f + parameter_id: 0x1bf16028 + parameter_id: 0x13580d6c +} function { id: 0x9b974729 return_type_id: 0x6720d32f @@ -341620,6 +343368,13 @@ function { parameter_id: 0x0d5f1839 parameter_id: 0xa2114faf } +function { + id: 0x9bb259e2 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x064d6086 + parameter_id: 0x13580d6c +} function { id: 0x9bb2f2c9 return_type_id: 0x6720d32f @@ -341895,6 +343650,17 @@ function { parameter_id: 0x11cffa09 parameter_id: 0x92233392 } +function { + id: 0x9bbbc132 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x0a63398f + parameter_id: 0xf1a6dfed + parameter_id: 0x6720d32f + parameter_id: 0x0b0164a7 + parameter_id: 0x11cfee5a + parameter_id: 0x11cfee5a +} function { id: 0x9bbc66c0 return_type_id: 0x6720d32f @@ -342024,6 +343790,14 @@ function { parameter_id: 0x1b36c7a2 parameter_id: 0x0d956bfc } +function { + id: 0x9bc00937 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1b8590a8 + parameter_id: 0x02cc0dbc + parameter_id: 0x4585663f +} function { id: 0x9bc01a31 return_type_id: 0x6720d32f @@ -342323,6 +344097,14 @@ function { parameter_id: 0x31e58fe0 parameter_id: 0x2efe8065 } +function { + id: 0x9bc93dc0 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1b8590a8 + parameter_id: 0x2170d06d + parameter_id: 0x33756485 +} function { id: 0x9bc999f4 return_type_id: 0x6720d32f @@ -342381,6 +344163,14 @@ function { parameter_id: 0x1d19a9d5 parameter_id: 0x4585663f } +function { + id: 0x9bcafaa2 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1d19a9d5 + parameter_id: 0x4585663f + parameter_id: 0x11cfee5a +} function { id: 0x9bcb2775 return_type_id: 0x6720d32f @@ -342527,6 +344317,13 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x1f8dbf97 } +function { + id: 0x9bd1c19e + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1e820193 + parameter_id: 0x11cfee5a +} function { id: 0x9bd1c436 return_type_id: 0x6720d32f @@ -342655,6 +344452,14 @@ function { parameter_id: 0x6720d32f parameter_id: 0x13580d6c } +function { + id: 0x9bd481b1 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1d19a9d5 + parameter_id: 0x3c692b7e + parameter_id: 0x11cfee5a +} function { id: 0x9bd49403 return_type_id: 0x6720d32f @@ -342858,6 +344663,18 @@ function { parameter_id: 0x1d19a9d5 parameter_id: 0x310ec01d } +function { + id: 0x9bd7c10d + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1d19a9d5 + parameter_id: 0x33756485 + parameter_id: 0x33756485 + parameter_id: 0x33756485 + parameter_id: 0x6720d32f + parameter_id: 0x11cfee5a + parameter_id: 0x11cfee5a +} function { id: 0x9bd7cd8d return_type_id: 0x6720d32f @@ -343324,6 +345141,16 @@ function { parameter_id: 0x1b8590a8 parameter_id: 0x20007040 } +function { + id: 0x9be1991a + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1321a5b2 + parameter_id: 0x0a63398f + parameter_id: 0x0b036186 + parameter_id: 0x4585663f + parameter_id: 0x13580d6c +} function { id: 0x9be1ca34 return_type_id: 0x6720d32f @@ -343506,6 +345333,14 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x1013df15 } +function { + id: 0x9bf05906 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1582ab06 + parameter_id: 0x2170d06d + parameter_id: 0x6d7f5ff6 +} function { id: 0x9bf1212e return_type_id: 0x6720d32f @@ -343514,6 +345349,14 @@ function { parameter_id: 0x2170d06d parameter_id: 0x33756485 } +function { + id: 0x9bf15498 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1582ab06 + parameter_id: 0x2170d06d + parameter_id: 0x2e18f543 +} function { id: 0x9bf18f52 return_type_id: 0x6720d32f @@ -343527,6 +345370,13 @@ function { return_type_id: 0x6720d32f parameter_id: 0x19c40476 } +function { + id: 0x9bf1ecfb + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1582ab06 + parameter_id: 0x2170d06d +} function { id: 0x9bf23070 return_type_id: 0x6720d32f @@ -343534,6 +345384,13 @@ function { parameter_id: 0x11b57133 parameter_id: 0x6d7f5ff6 } +function { + id: 0x9bf236f2 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1582ab06 + parameter_id: 0x2e18f543 +} function { id: 0x9bf26c94 return_type_id: 0x6720d32f @@ -343702,6 +345559,13 @@ function { return_type_id: 0x3e10b518 parameter_id: 0x26e55184 } +function { + id: 0x9bfd4598 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x15a30023 + parameter_id: 0x11cfee5a +} function { id: 0x9bfdc334 return_type_id: 0x6720d32f @@ -346381,6 +348245,13 @@ function { parameter_id: 0x1bb6a471 parameter_id: 0x4585663f } +function { + id: 0x9d8d344c + return_type_id: 0x6720d32f + parameter_id: 0x03b1760d + parameter_id: 0x3e10b518 + parameter_id: 0x6720d32f +} function { id: 0x9d8e51d7 return_type_id: 0x6720d32f @@ -346438,6 +348309,16 @@ function { parameter_id: 0x2131312a parameter_id: 0x06433852 } +function { + id: 0x9d945856 + return_type_id: 0x6720d32f + parameter_id: 0x03b1760d + parameter_id: 0x3e10b518 + parameter_id: 0x0483e6f8 + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f +} function { id: 0x9d94842e return_type_id: 0x6720d32f @@ -346503,6 +348384,11 @@ function { parameter_id: 0x09626b7f parameter_id: 0x914dbfdc } +function { + id: 0x9d9dd01b + return_type_id: 0xf435685e + parameter_id: 0x2a886524 +} function { id: 0x9d9e97bc return_type_id: 0x6720d32f @@ -346844,6 +348730,16 @@ function { parameter_id: 0x11a649c9 parameter_id: 0x4585663f } +function { + id: 0x9dc2f160 + return_type_id: 0xf435685e + parameter_id: 0x2a886524 + parameter_id: 0x18bd6530 + parameter_id: 0xf435685e + parameter_id: 0x391f15ea + parameter_id: 0xf435685e + parameter_id: 0x31344499 +} function { id: 0x9dc2f17b return_type_id: 0xf435685e @@ -347114,6 +349010,12 @@ function { parameter_id: 0x0258f96e parameter_id: 0x334c07d5 } +function { + id: 0x9ddc45fb + return_type_id: 0x6720d32f + parameter_id: 0x0258f96e + parameter_id: 0x32a623d7 +} function { id: 0x9ddc5532 return_type_id: 0x6720d32f @@ -348010,6 +349912,28 @@ function { return_type_id: 0x6720d32f parameter_id: 0x0f78474f } +function { + id: 0x9e5fde3c + return_type_id: 0x6720d32f + parameter_id: 0x0dc9e98b + parameter_id: 0x295c7202 + parameter_id: 0x295c7202 + parameter_id: 0x914dbfdc + parameter_id: 0x914dbfdc + parameter_id: 0x391f15ea + parameter_id: 0x914dbfdc +} +function { + id: 0x9e5fdeba + return_type_id: 0x6720d32f + parameter_id: 0x0dc9e98b + parameter_id: 0x295c7202 + parameter_id: 0x295c7202 + parameter_id: 0x914dbfdc + parameter_id: 0x914dbfdc + parameter_id: 0x18bd6530 + parameter_id: 0x914dbfdc +} function { id: 0x9e609ac2 return_type_id: 0x6720d32f @@ -348164,6 +350088,12 @@ function { parameter_id: 0x13689f5b parameter_id: 0x18bd6530 } +function { + id: 0x9e768332 + return_type_id: 0x6720d32f + parameter_id: 0x0dc9e98b + parameter_id: 0x21069feb +} function { id: 0x9e768b87 return_type_id: 0x6720d32f @@ -348284,6 +350214,11 @@ function { parameter_id: 0x0b756bd6 parameter_id: 0x0c715d8d } +function { + id: 0x9e8921bb + return_type_id: 0xf435685e + parameter_id: 0x26dba3a7 +} function { id: 0x9e8b4d60 return_type_id: 0x6720d32f @@ -348518,6 +350453,13 @@ function { parameter_id: 0x0e04e400 parameter_id: 0x02aa06b1 } +function { + id: 0x9eaf2d7e + return_type_id: 0x6720d32f + parameter_id: 0x0f3912c5 + parameter_id: 0x3e10b518 + parameter_id: 0x6720d32f +} function { id: 0x9eb16478 return_type_id: 0x6720d32f @@ -348559,6 +350501,15 @@ function { parameter_id: 0x0fabaf3b parameter_id: 0x370a73b0 } +function { + id: 0x9eb640f8 + return_type_id: 0x6720d32f + parameter_id: 0x0f3912c5 + parameter_id: 0x3e10b518 + parameter_id: 0x0483e6f8 + parameter_id: 0x6720d32f + parameter_id: 0x6720d32f +} function { id: 0x9eb724d7 return_type_id: 0x6720d32f @@ -348762,6 +350713,16 @@ function { parameter_id: 0x391f15ea parameter_id: 0xf435685e } +function { + id: 0x9ed600da + return_type_id: 0xf435685e + parameter_id: 0x26dba3a7 + parameter_id: 0x18bd6530 + parameter_id: 0xf435685e + parameter_id: 0x391f15ea + parameter_id: 0xf435685e + parameter_id: 0x379f2ae1 +} function { id: 0x9ed68372 return_type_id: 0x6720d32f @@ -348836,6 +350797,12 @@ function { parameter_id: 0x0258f96e parameter_id: 0xfc0e1dbd } +function { + id: 0x9ee7a4e9 + return_type_id: 0x6720d32f + parameter_id: 0x0dc9e98b + parameter_id: 0x054f691a +} function { id: 0x9ee7e85b return_type_id: 0x6720d32f @@ -348909,6 +350876,11 @@ function { parameter_id: 0x6720d32f parameter_id: 0x6720d32f } +function { + id: 0x9ef2994d + return_type_id: 0x6720d32f + parameter_id: 0x0dc9e98b +} function { id: 0x9ef2dc18 return_type_id: 0x6720d32f @@ -349452,6 +351424,12 @@ function { parameter_id: 0xc9082b19 parameter_id: 0x6720d32f } +function { + id: 0x9f6e1a01 + return_type_id: 0x6720d32f + parameter_id: 0x0dc9e98b + parameter_id: 0x6720d32f +} function { id: 0x9f6e36d6 return_type_id: 0x6720d32f @@ -349700,6 +351678,11 @@ function { parameter_id: 0x914dbfdc parameter_id: 0x3247ae94 } +function { + id: 0x9f90af15 + return_type_id: 0xf435685e + parameter_id: 0x22bd991e +} function { id: 0x9f958461 return_type_id: 0x6720d32f @@ -351022,6 +353005,16 @@ function { parameter_id: 0x4585663f parameter_id: 0x0258f96e } +function { + id: 0xab7ddc65 + return_type_id: 0x0258f96e + parameter_id: 0x1a1051a0 + parameter_id: 0x31e58fe0 + parameter_id: 0x4585663f + parameter_id: 0x0258f96e + parameter_id: 0x18bd6530 + parameter_id: 0x06433852 +} function { id: 0xab955662 return_type_id: 0x25782362 @@ -351293,6 +353286,11 @@ function { parameter_id: 0xcd84e3cd parameter_id: 0x33756485 } +function { + id: 0xb0abc898 + return_type_id: 0x3e10b518 + parameter_id: 0x8bbf7ced +} function { id: 0xb0ad4fdd return_type_id: 0x1a2c7f0e @@ -352572,6 +354570,13 @@ function { parameter_id: 0x126add1c parameter_id: 0x4585663f } +function { + id: 0xcb5a4db0 + return_type_id: 0x4585663f + parameter_id: 0x12191e2a + parameter_id: 0x4585663f + parameter_id: 0x4585663f +} function { id: 0xcb7f2e4a return_type_id: 0x2314b69f @@ -352777,6 +354782,11 @@ function { parameter_id: 0xcd84e3cd parameter_id: 0x33756485 } +function { + id: 0xce1fb5db + return_type_id: 0x2a886524 + parameter_id: 0x87250506 +} function { id: 0xce25f646 return_type_id: 0x65d74a06 @@ -353007,6 +355017,13 @@ function { parameter_id: 0x3e10b518 parameter_id: 0x13580d6c } +function { + id: 0xd596d12a + return_type_id: 0x760c4a75 + parameter_id: 0x6720d32f + parameter_id: 0xd5df6730 + parameter_id: 0xf435685e +} function { id: 0xd597088b return_type_id: 0x334927f6 @@ -353343,6 +355360,11 @@ function { parameter_id: 0x3104c07e parameter_id: 0x3654c061 } +function { + id: 0xde51e0f8 + return_type_id: 0x26dba3a7 + parameter_id: 0x87250506 +} function { id: 0xdeac32ec return_type_id: 0x4585663f @@ -353434,6 +355456,23 @@ function { parameter_id: 0x18a2fb63 parameter_id: 0x4585663f } +function { + id: 0xe10f8963 + return_type_id: 0x2eab5b8a + parameter_id: 0x0de4c6b1 + parameter_id: 0xfa5f37bb + parameter_id: 0x4585663f + parameter_id: 0x4585663f +} +function { + id: 0xe10f9802 + return_type_id: 0x2eab5b8a + parameter_id: 0x0de4c6b1 + parameter_id: 0xfa5f37bb + parameter_id: 0x4585663f + parameter_id: 0x4585663f + parameter_id: 0x4585663f +} function { id: 0xe12a6d4e return_type_id: 0x07450c2a @@ -354165,6 +356204,11 @@ function { parameter_id: 0x3fa672fd parameter_id: 0x3e10b518 } +function { + id: 0xf1b60446 + return_type_id: 0x6d7f5ff6 + parameter_id: 0x3c497d0a +} function { id: 0xf1c60201 return_type_id: 0x6d7f5ff6 @@ -354601,6 +356645,11 @@ function { parameter_id: 0x3399c453 parameter_id: 0x6720d32f } +function { + id: 0xf3ed77aa + return_type_id: 0x0756289d + parameter_id: 0x12e6ffae +} function { id: 0xf3ef3eef return_type_id: 0x0b7f62fc @@ -355433,6 +357482,12 @@ function { parameter_id: 0x06835e9c parameter_id: 0xfe028135 } +function { + id: 0xfd016131 + return_type_id: 0x6d7f5ff6 + parameter_id: 0x0d09fa5f + parameter_id: 0x39d1288c +} function { id: 0xfd07a22d return_type_id: 0x6d7f5ff6 @@ -355708,6 +357763,13 @@ function { return_type_id: 0x6d7f5ff6 parameter_id: 0x045d9eb9 } +function { + id: 0xffb39db8 + return_type_id: 0x22bd991e + parameter_id: 0x391f15ea + parameter_id: 0xf435685e + parameter_id: 0x87250506 +} function { id: 0xffb4ff33 return_type_id: 0x32a623d7 @@ -355806,6 +357868,42 @@ elf_symbol { type_id: 0x2dfa71ba full_name: "I_BDEV" } +elf_symbol { + id: 0x26c56443 + name: "LZ4_compress_HC" + is_defined: true + symbol_type: FUNCTION + crc: 0xddf86133 + type_id: 0x920e8cef + full_name: "LZ4_compress_HC" +} +elf_symbol { + id: 0xf4f8ddc2 + name: "LZ4_compress_HC_continue" + is_defined: true + symbol_type: FUNCTION + crc: 0x38f7b6e0 + type_id: 0x9eb640f8 + full_name: "LZ4_compress_HC_continue" +} +elf_symbol { + id: 0x7c322a62 + name: "LZ4_compress_fast" + is_defined: true + symbol_type: FUNCTION + crc: 0x6004858d + type_id: 0x920e8cef + full_name: "LZ4_compress_fast" +} +elf_symbol { + id: 0x38116da8 + name: "LZ4_compress_fast_continue" + is_defined: true + symbol_type: FUNCTION + crc: 0xf9eced44 + type_id: 0x9d945856 + full_name: "LZ4_compress_fast_continue" +} elf_symbol { id: 0x31a154b4 name: "LZ4_decompress_safe" @@ -355815,6 +357913,15 @@ elf_symbol { type_id: 0x920e9545 full_name: "LZ4_decompress_safe" } +elf_symbol { + id: 0xc12163b9 + name: "LZ4_decompress_safe_continue" + is_defined: true + symbol_type: FUNCTION + crc: 0x8a47043d + type_id: 0x9191b4cd + full_name: "LZ4_decompress_safe_continue" +} elf_symbol { id: 0x740d2b27 name: "LZ4_decompress_safe_partial" @@ -355824,6 +357931,42 @@ elf_symbol { type_id: 0x920e8c8d full_name: "LZ4_decompress_safe_partial" } +elf_symbol { + id: 0x758b3708 + name: "LZ4_loadDict" + is_defined: true + symbol_type: FUNCTION + crc: 0x749849d8 + type_id: 0x9d8d344c + full_name: "LZ4_loadDict" +} +elf_symbol { + id: 0xef9094cc + name: "LZ4_loadDictHC" + is_defined: true + symbol_type: FUNCTION + crc: 0x93ff008c + type_id: 0x9eaf2d7e + full_name: "LZ4_loadDictHC" +} +elf_symbol { + id: 0x35928a4a + name: "LZ4_resetStreamHC" + is_defined: true + symbol_type: FUNCTION + crc: 0xd25422cd + type_id: 0x12ca966e + full_name: "LZ4_resetStreamHC" +} +elf_symbol { + id: 0xab09bd5e + name: "LZ4_setStreamDecode" + is_defined: true + symbol_type: FUNCTION + crc: 0x3b321462 + type_id: 0x9188d94b + full_name: "LZ4_setStreamDecode" +} elf_symbol { id: 0xf56ca4c7 name: "__ClearPageMovable" @@ -356209,6 +358352,15 @@ elf_symbol { type_id: 0x56d6c623 full_name: "__blkg_prfill_rwstat" } +elf_symbol { + id: 0xbf1d0365 + name: "__blkg_prfill_u64" + is_defined: true + symbol_type: FUNCTION + crc: 0x9bec4b1b + type_id: 0x56fc975d + full_name: "__blkg_prfill_u64" +} elf_symbol { id: 0x2d1c1f2d name: "__blockdev_direct_IO" @@ -357266,6 +359418,15 @@ elf_symbol { type_id: 0xf74d9260 full_name: "__folio_start_writeback" } +elf_symbol { + id: 0xcd2cca92 + name: "__free_iova" + is_defined: true + symbol_type: FUNCTION + crc: 0xb3a0080d + type_id: 0x1fadab22 + full_name: "__free_iova" +} elf_symbol { id: 0x5b1ea047 name: "__free_pages" @@ -357656,6 +359817,15 @@ elf_symbol { type_id: 0x5f133b10 full_name: "__irq_domain_add" } +elf_symbol { + id: 0xc37fd40e + name: "__irq_domain_alloc_fwnode" + is_defined: true + symbol_type: FUNCTION + crc: 0x51594136 + type_id: 0x299442f2 + full_name: "__irq_domain_alloc_fwnode" +} elf_symbol { id: 0x64bb2ece name: "__irq_domain_alloc_irqs" @@ -359565,6 +361735,15 @@ elf_symbol { type_id: 0x9b96d02c full_name: "__traceiter_android_rvh_attach_entity_load_avg" } +elf_symbol { + id: 0x965ea17d + name: "__traceiter_android_rvh_balance_fair" + is_defined: true + symbol_type: FUNCTION + crc: 0x9401da37 + type_id: 0x9b1007fb + full_name: "__traceiter_android_rvh_balance_fair" +} elf_symbol { id: 0x144db0a1 name: "__traceiter_android_rvh_before_do_sched_yield" @@ -359574,6 +361753,24 @@ elf_symbol { type_id: 0x9b79f498 full_name: "__traceiter_android_rvh_before_do_sched_yield" } +elf_symbol { + id: 0x166cd967 + name: "__traceiter_android_rvh_before_pick_task_fair" + is_defined: true + symbol_type: FUNCTION + crc: 0x2772b75b + type_id: 0x9b14e056 + full_name: "__traceiter_android_rvh_before_pick_task_fair" +} +elf_symbol { + id: 0xb2eed0b6 + name: "__traceiter_android_rvh_blk_mq_has_request" + is_defined: true + symbol_type: FUNCTION + crc: 0xb438f426 + type_id: 0x9b207e3e + full_name: "__traceiter_android_rvh_blk_mq_has_request" +} elf_symbol { id: 0xe193199f name: "__traceiter_android_rvh_bpf_int_jit_compile_ro" @@ -359700,6 +361897,15 @@ elf_symbol { type_id: 0x9a32aa77 full_name: "__traceiter_android_rvh_cpu_util_cfs_boost" } +elf_symbol { + id: 0xb3da5d93 + name: "__traceiter_android_rvh_cpufreq_create_policy" + is_defined: true + symbol_type: FUNCTION + crc: 0x47e97219 + type_id: 0x9ba9c2f7 + full_name: "__traceiter_android_rvh_cpufreq_create_policy" +} elf_symbol { id: 0x1baec33d name: "__traceiter_android_rvh_cpufreq_transition" @@ -360024,6 +362230,15 @@ elf_symbol { type_id: 0x9b8eabb4 full_name: "__traceiter_android_rvh_gic_v3_set_affinity" } +elf_symbol { + id: 0x97095cae + name: "__traceiter_android_rvh_gup_longterm_locked" + is_defined: true + symbol_type: FUNCTION + crc: 0x82bcda65 + type_id: 0x986040b6 + full_name: "__traceiter_android_rvh_gup_longterm_locked" +} elf_symbol { id: 0xe443e259 name: "__traceiter_android_rvh_handle_bad_stack" @@ -360663,6 +362878,15 @@ elf_symbol { type_id: 0x9bdc22a4 full_name: "__traceiter_android_rvh_schedule" } +elf_symbol { + id: 0x40bb3f78 + name: "__traceiter_android_rvh_schedule_bug" + is_defined: true + symbol_type: FUNCTION + crc: 0xfa5d1e9c + type_id: 0x9bcd4ff7 + full_name: "__traceiter_android_rvh_schedule_bug" +} elf_symbol { id: 0x0f807f10 name: "__traceiter_android_rvh_select_fallback_rq" @@ -360870,6 +363094,15 @@ elf_symbol { type_id: 0x9baf3eaf full_name: "__traceiter_android_rvh_show_max_freq" } +elf_symbol { + id: 0x4ae52717 + name: "__traceiter_android_rvh_swap_bio_charge" + is_defined: true + symbol_type: FUNCTION + crc: 0x3f478e7d + type_id: 0x9bf93663 + full_name: "__traceiter_android_rvh_swap_bio_charge" +} elf_symbol { id: 0xb80ecc98 name: "__traceiter_android_rvh_swap_readpage_bdev_sync" @@ -360879,6 +363112,15 @@ elf_symbol { type_id: 0x9bab3090 full_name: "__traceiter_android_rvh_swap_readpage_bdev_sync" } +elf_symbol { + id: 0x20ccf43a + name: "__traceiter_android_rvh_task_fits_cpu" + is_defined: true + symbol_type: FUNCTION + crc: 0xc68a9de3 + type_id: 0x9bd7c10d + full_name: "__traceiter_android_rvh_task_fits_cpu" +} elf_symbol { id: 0x3b650ee3 name: "__traceiter_android_rvh_tcp_rcv_spurious_retrans" @@ -361662,6 +363904,15 @@ elf_symbol { type_id: 0x9a1334e6 full_name: "__traceiter_android_vh_binder_wait_for_work" } +elf_symbol { + id: 0x3e149573 + name: "__traceiter_android_vh_bio_add_page_merge_bypass" + is_defined: true + symbol_type: FUNCTION + crc: 0xd346e199 + type_id: 0x9bfd4598 + full_name: "__traceiter_android_vh_bio_add_page_merge_bypass" +} elf_symbol { id: 0x6f2a9ca9 name: "__traceiter_android_vh_blk_fill_rwbs" @@ -361824,6 +364075,15 @@ elf_symbol { type_id: 0x9bcd4ff7 full_name: "__traceiter_android_vh_check_uninterruptible_tasks_dn" } +elf_symbol { + id: 0x253229ea + name: "__traceiter_android_vh_chk_task" + is_defined: true + symbol_type: FUNCTION + crc: 0xa543dc37 + type_id: 0x9b93638b + full_name: "__traceiter_android_vh_chk_task" +} elf_symbol { id: 0xef7737f8 name: "__traceiter_android_vh_cleanup_old_buffers_bypass" @@ -361833,6 +364093,15 @@ elf_symbol { type_id: 0x9b63bb96 full_name: "__traceiter_android_vh_cleanup_old_buffers_bypass" } +elf_symbol { + id: 0x124b6497 + name: "__traceiter_android_vh_clear_curr_lazy" + is_defined: true + symbol_type: FUNCTION + crc: 0xe9dcec24 + type_id: 0x9bdbdcc4 + full_name: "__traceiter_android_vh_clear_curr_lazy" +} elf_symbol { id: 0xe50d4e50 name: "__traceiter_android_vh_clear_reclaimed_folio" @@ -362067,6 +364336,24 @@ elf_symbol { type_id: 0x9b73e68e full_name: "__traceiter_android_vh_cpuidle_psci_exit" } +elf_symbol { + id: 0x9004b9db + name: "__traceiter_android_vh_cpuset_attach_task" + is_defined: true + symbol_type: FUNCTION + crc: 0x7ecb2d63 + type_id: 0x9b04a709 + full_name: "__traceiter_android_vh_cpuset_attach_task" +} +elf_symbol { + id: 0x3d81eceb + name: "__traceiter_android_vh_cpuset_css_online" + is_defined: true + symbol_type: FUNCTION + crc: 0xd81bb314 + type_id: 0x9b03e163 + full_name: "__traceiter_android_vh_cpuset_css_online" +} elf_symbol { id: 0x96cfd964 name: "__traceiter_android_vh_customize_alloc_gfp" @@ -362508,6 +364795,15 @@ elf_symbol { type_id: 0x9bc214f0 full_name: "__traceiter_android_vh_f2fs_restore_priority" } +elf_symbol { + id: 0x150ea575 + name: "__traceiter_android_vh_fas_gpu_qos_update_tracer" + is_defined: true + symbol_type: FUNCTION + crc: 0xfff672c9 + type_id: 0x9b047f13 + full_name: "__traceiter_android_vh_fas_gpu_qos_update_tracer" +} elf_symbol { id: 0x93a4717b name: "__traceiter_android_vh_file_is_tiny_bypass" @@ -362535,6 +364831,15 @@ elf_symbol { type_id: 0x9bc0ce91 full_name: "__traceiter_android_vh_filemap_fault_end" } +elf_symbol { + id: 0xbd855995 + name: "__traceiter_android_vh_filemap_fault_folio_locked" + is_defined: true + symbol_type: FUNCTION + crc: 0xa9cc4143 + type_id: 0x9bc93dc0 + full_name: "__traceiter_android_vh_filemap_fault_folio_locked" +} elf_symbol { id: 0x980c4b2d name: "__traceiter_android_vh_filemap_fault_pre_folio_locked" @@ -362571,6 +364876,15 @@ elf_symbol { type_id: 0x9bf4cdf9 full_name: "__traceiter_android_vh_filemap_get_folio" } +elf_symbol { + id: 0x2c39dada + name: "__traceiter_android_vh_filemap_get_folio_end" + is_defined: true + symbol_type: FUNCTION + crc: 0x87b09c16 + type_id: 0x9bf1ecfb + full_name: "__traceiter_android_vh_filemap_get_folio_end" +} elf_symbol { id: 0xb7d91f76 name: "__traceiter_android_vh_filemap_map_pages" @@ -362607,6 +364921,15 @@ elf_symbol { type_id: 0x9bc62aec full_name: "__traceiter_android_vh_filemap_read" } +elf_symbol { + id: 0x48ad4ccb + name: "__traceiter_android_vh_filemap_read_end" + is_defined: true + symbol_type: FUNCTION + crc: 0x2291fcc5 + type_id: 0x9bc00937 + full_name: "__traceiter_android_vh_filemap_read_end" +} elf_symbol { id: 0x0e217a5e name: "__traceiter_android_vh_filemap_update_page" @@ -362652,6 +364975,15 @@ elf_symbol { type_id: 0x9bccd612 full_name: "__traceiter_android_vh_flush_wq_wait_start" } +elf_symbol { + id: 0xfaa99ddb + name: "__traceiter_android_vh_folio_add_lru" + is_defined: true + symbol_type: FUNCTION + crc: 0x2de2460e + type_id: 0x9b2a7922 + full_name: "__traceiter_android_vh_folio_add_lru" +} elf_symbol { id: 0x5dd57eff name: "__traceiter_android_vh_folio_add_lru_folio_activate" @@ -362688,6 +365020,15 @@ elf_symbol { type_id: 0x9b2a7922 full_name: "__traceiter_android_vh_folio_trylock_set" } +elf_symbol { + id: 0xb10c10b1 + name: "__traceiter_android_vh_force_page_cache_ra" + is_defined: true + symbol_type: FUNCTION + crc: 0x8ebc54b3 + type_id: 0x9bf236f2 + full_name: "__traceiter_android_vh_force_page_cache_ra" +} elf_symbol { id: 0xa51a0f0c name: "__traceiter_android_vh_free_mod_mem" @@ -363183,6 +365524,15 @@ elf_symbol { type_id: 0x9872969e full_name: "__traceiter_android_vh_kvmalloc_node_use_vmalloc" } +elf_symbol { + id: 0xedae015e + name: "__traceiter_android_vh_lock_delay_schedule" + is_defined: true + symbol_type: FUNCTION + crc: 0xf0740155 + type_id: 0x9bcafaa2 + full_name: "__traceiter_android_vh_lock_delay_schedule" +} elf_symbol { id: 0xf56758ea name: "__traceiter_android_vh_lock_folio_drop_mmap_end" @@ -363201,6 +365551,24 @@ elf_symbol { type_id: 0x9b94b8e2 full_name: "__traceiter_android_vh_lock_folio_drop_mmap_start" } +elf_symbol { + id: 0xc955d513 + name: "__traceiter_android_vh_lock_task_exit" + is_defined: true + symbol_type: FUNCTION + crc: 0x17116f86 + type_id: 0x9bdbdcc4 + full_name: "__traceiter_android_vh_lock_task_exit" +} +elf_symbol { + id: 0xfc3decf6 + name: "__traceiter_android_vh_lock_task_fork" + is_defined: true + symbol_type: FUNCTION + crc: 0xc9d186e0 + type_id: 0x9bdbdcc4 + full_name: "__traceiter_android_vh_lock_task_fork" +} elf_symbol { id: 0x66cefe64 name: "__traceiter_android_vh_logbuf" @@ -363237,6 +365605,15 @@ elf_symbol { type_id: 0x9b222516 full_name: "__traceiter_android_vh_look_around_migrate_folio" } +elf_symbol { + id: 0xaf9604df + name: "__traceiter_android_vh_loop_skip_queue_work" + is_defined: true + symbol_type: FUNCTION + crc: 0xccc3120d + type_id: 0x9bd1c19e + full_name: "__traceiter_android_vh_loop_skip_queue_work" +} elf_symbol { id: 0xa6d58835 name: "__traceiter_android_vh_lru_gen_add_folio_skip" @@ -363426,6 +365803,15 @@ elf_symbol { type_id: 0x9bb62df9 full_name: "__traceiter_android_vh_meminfo_proc_show" } +elf_symbol { + id: 0xe02fe9e3 + name: "__traceiter_android_vh_mempool_alloc_skip_wait" + is_defined: true + symbol_type: FUNCTION + crc: 0xabe64979 + type_id: 0x9b702fa1 + full_name: "__traceiter_android_vh_mempool_alloc_skip_wait" +} elf_symbol { id: 0x105350cb name: "__traceiter_android_vh_mglru_should_abort_scan" @@ -363471,6 +365857,78 @@ elf_symbol { type_id: 0x9b883042 full_name: "__traceiter_android_vh_mm_compaction_end" } +elf_symbol { + id: 0x28fd3694 + name: "__traceiter_android_vh_mm_customize_ac" + is_defined: true + symbol_type: FUNCTION + crc: 0xc217cd07 + type_id: 0x98787515 + full_name: "__traceiter_android_vh_mm_customize_ac" +} +elf_symbol { + id: 0x7a520c7e + name: "__traceiter_android_vh_mm_customize_file_is_tiny" + is_defined: true + symbol_type: FUNCTION + crc: 0x2a03eb45 + type_id: 0x9aa1ff3f + full_name: "__traceiter_android_vh_mm_customize_file_is_tiny" +} +elf_symbol { + id: 0x4f807371 + name: "__traceiter_android_vh_mm_customize_lru_add_dst" + is_defined: true + symbol_type: FUNCTION + crc: 0x973264a8 + type_id: 0x9b430741 + full_name: "__traceiter_android_vh_mm_customize_lru_add_dst" +} +elf_symbol { + id: 0x7637ee8f + name: "__traceiter_android_vh_mm_customize_pgdat_balanced" + is_defined: true + symbol_type: FUNCTION + crc: 0x86a9060f + type_id: 0x9a2ab257 + full_name: "__traceiter_android_vh_mm_customize_pgdat_balanced" +} +elf_symbol { + id: 0x46918874 + name: "__traceiter_android_vh_mm_customize_rmqueue" + is_defined: true + symbol_type: FUNCTION + crc: 0x18e2077a + type_id: 0x9b973ccf + full_name: "__traceiter_android_vh_mm_customize_rmqueue" +} +elf_symbol { + id: 0x0b6a1d80 + name: "__traceiter_android_vh_mm_customize_suitable_zone" + is_defined: true + symbol_type: FUNCTION + crc: 0xc42fa581 + type_id: 0x9bbbc132 + full_name: "__traceiter_android_vh_mm_customize_suitable_zone" +} +elf_symbol { + id: 0x2be8352c + name: "__traceiter_android_vh_mm_customize_zone_max_order" + is_defined: true + symbol_type: FUNCTION + crc: 0xf1dc287d + type_id: 0x9b82e086 + full_name: "__traceiter_android_vh_mm_customize_zone_max_order" +} +elf_symbol { + id: 0xa792d107 + name: "__traceiter_android_vh_mm_customize_zone_pageset" + is_defined: true + symbol_type: FUNCTION + crc: 0xbdbf230e + type_id: 0x9b82ade6 + full_name: "__traceiter_android_vh_mm_customize_zone_pageset" +} elf_symbol { id: 0x9f58159a name: "__traceiter_android_vh_mm_direct_reclaim_enter" @@ -363498,6 +365956,15 @@ elf_symbol { type_id: 0x9bb5b719 full_name: "__traceiter_android_vh_mm_free_page" } +elf_symbol { + id: 0x945dd126 + name: "__traceiter_android_vh_mm_isolate_priv_lru" + is_defined: true + symbol_type: FUNCTION + crc: 0x8449665c + type_id: 0x9b6c13bd + full_name: "__traceiter_android_vh_mm_isolate_priv_lru" +} elf_symbol { id: 0x6f5c8275 name: "__traceiter_android_vh_mm_kcompactd_cpu_online" @@ -363651,6 +366118,15 @@ elf_symbol { type_id: 0x9b2239e7 full_name: "__traceiter_android_vh_mutex_wait_start" } +elf_symbol { + id: 0xdb0a9300 + name: "__traceiter_android_vh_nr_pcp_alloc" + is_defined: true + symbol_type: FUNCTION + crc: 0xa19e6593 + type_id: 0x9be1991a + full_name: "__traceiter_android_vh_nr_pcp_alloc" +} elf_symbol { id: 0x1c5ab7ec name: "__traceiter_android_vh_ogki_async_psi_bypass" @@ -363849,6 +366325,24 @@ elf_symbol { type_id: 0x9bbd796e full_name: "__traceiter_android_vh_ogki_ufs_dsm" } +elf_symbol { + id: 0x1ff6ff8b + name: "__traceiter_android_vh_oom_evaluate_task_bypass" + is_defined: true + symbol_type: FUNCTION + crc: 0x72894f8f + type_id: 0x9bd481b1 + full_name: "__traceiter_android_vh_oom_evaluate_task_bypass" +} +elf_symbol { + id: 0x2884ad94 + name: "__traceiter_android_vh_oom_reaper_delay_bypass" + is_defined: true + symbol_type: FUNCTION + crc: 0xd9f66562 + type_id: 0x9bdfaf3f + full_name: "__traceiter_android_vh_oom_reaper_delay_bypass" +} elf_symbol { id: 0x724577f8 name: "__traceiter_android_vh_oom_swapmem_gather_finish" @@ -363876,6 +366370,24 @@ elf_symbol { type_id: 0x9bb7fe1c full_name: "__traceiter_android_vh_page_add_new_anon_rmap" } +elf_symbol { + id: 0xbf28fbd5 + name: "__traceiter_android_vh_page_cache_ra_order_bypass" + is_defined: true + symbol_type: FUNCTION + crc: 0x4d1e561b + type_id: 0x9b37ef14 + full_name: "__traceiter_android_vh_page_cache_ra_order_bypass" +} +elf_symbol { + id: 0x7f95d934 + name: "__traceiter_android_vh_page_cache_ra_unbounded" + is_defined: true + symbol_type: FUNCTION + crc: 0x6da13386 + type_id: 0x9bf15498 + full_name: "__traceiter_android_vh_page_cache_ra_unbounded" +} elf_symbol { id: 0xd0065122 name: "__traceiter_android_vh_page_cache_readahead_end" @@ -363939,6 +366451,15 @@ elf_symbol { type_id: 0x9bb62df9 full_name: "__traceiter_android_vh_pagetypeinfo_show" } +elf_symbol { + id: 0xc5066ceb + name: "__traceiter_android_vh_pcp_alloc_factor_adjust" + is_defined: true + symbol_type: FUNCTION + crc: 0xebad8b8c + type_id: 0x9b87e976 + full_name: "__traceiter_android_vh_pcp_alloc_factor_adjust" +} elf_symbol { id: 0x13b0736e name: "__traceiter_android_vh_percpu_rwsem_down_read" @@ -364128,6 +366649,24 @@ elf_symbol { type_id: 0x9b49a977 full_name: "__traceiter_android_vh_ptype_head" } +elf_symbol { + id: 0x358b7cff + name: "__traceiter_android_vh_put_super" + is_defined: true + symbol_type: FUNCTION + crc: 0x30b41dae + type_id: 0x9b3403c3 + full_name: "__traceiter_android_vh_put_super" +} +elf_symbol { + id: 0x1adac14e + name: "__traceiter_android_vh_put_task" + is_defined: true + symbol_type: FUNCTION + crc: 0x591f8208 + type_id: 0x9bdbdcc4 + full_name: "__traceiter_android_vh_put_task" +} elf_symbol { id: 0x2c963d28 name: "__traceiter_android_vh_queue_request_and_unlock" @@ -364236,6 +366775,24 @@ elf_symbol { type_id: 0x9b5a0fe0 full_name: "__traceiter_android_vh_regmap_update" } +elf_symbol { + id: 0x4ac41e54 + name: "__traceiter_android_vh_remove_mapping" + is_defined: true + symbol_type: FUNCTION + crc: 0x10e37e0a + type_id: 0x9bf05906 + full_name: "__traceiter_android_vh_remove_mapping" +} +elf_symbol { + id: 0xda7f66bc + name: "__traceiter_android_vh_remove_mapping_failed" + is_defined: true + symbol_type: FUNCTION + crc: 0x85189340 + type_id: 0x9bf05906 + full_name: "__traceiter_android_vh_remove_mapping_failed" +} elf_symbol { id: 0x55f0aa22 name: "__traceiter_android_vh_replace_anon_vma_name" @@ -364245,6 +366802,15 @@ elf_symbol { type_id: 0x9b84923c full_name: "__traceiter_android_vh_replace_anon_vma_name" } +elf_symbol { + id: 0x799188c0 + name: "__traceiter_android_vh_resched_curr_lazy" + is_defined: true + symbol_type: FUNCTION + crc: 0x9c46e73e + type_id: 0x9b13c3e5 + full_name: "__traceiter_android_vh_resched_curr_lazy" +} elf_symbol { id: 0xd0707ed1 name: "__traceiter_android_vh_reserve_highatomic_bypass" @@ -364254,6 +366820,15 @@ elf_symbol { type_id: 0x9bb1c4e2 full_name: "__traceiter_android_vh_reserve_highatomic_bypass" } +elf_symbol { + id: 0xfb6f79f6 + name: "__traceiter_android_vh_restore_curr_resched" + is_defined: true + symbol_type: FUNCTION + crc: 0xe6e9c33d + type_id: 0x9bb259e2 + full_name: "__traceiter_android_vh_restore_curr_resched" +} elf_symbol { id: 0x2f6e3ceb name: "__traceiter_android_vh_restore_mm_flags" @@ -365199,6 +367774,15 @@ elf_symbol { type_id: 0x9bb5b719 full_name: "__traceiter_android_vh_test_clear_look_around_ref" } +elf_symbol { + id: 0x6d485f59 + name: "__traceiter_android_vh_thaw_killed_process" + is_defined: true + symbol_type: FUNCTION + crc: 0x17e70a68 + type_id: 0x9be885da + full_name: "__traceiter_android_vh_thaw_killed_process" +} elf_symbol { id: 0x40d428a7 name: "__traceiter_android_vh_thermal_pm_notify_suspend" @@ -365271,6 +367855,15 @@ elf_symbol { type_id: 0x9aa97b5f full_name: "__traceiter_android_vh_try_to_freeze_todo" } +elf_symbol { + id: 0xd8abda6b + name: "__traceiter_android_vh_try_to_freeze_todo_logging" + is_defined: true + symbol_type: FUNCTION + crc: 0x7632c0b1 + type_id: 0x9be885da + full_name: "__traceiter_android_vh_try_to_freeze_todo_logging" +} elf_symbol { id: 0x0119fc41 name: "__traceiter_android_vh_try_to_freeze_todo_unfrozen" @@ -366270,6 +368863,15 @@ elf_symbol { type_id: 0x9b8f15f1 full_name: "__traceiter_pelt_se_tp" } +elf_symbol { + id: 0x46a383e5 + name: "__traceiter_rcu_stall_warning" + is_defined: true + symbol_type: FUNCTION + crc: 0xfe9171ac + type_id: 0x9b587c9a + full_name: "__traceiter_rcu_stall_warning" +} elf_symbol { id: 0x147cc848 name: "__traceiter_rpm_idle" @@ -366495,6 +369097,15 @@ elf_symbol { type_id: 0x9ab9affb full_name: "__traceiter_softirq_exit" } +elf_symbol { + id: 0x536acf4e + name: "__traceiter_spi_transfer_stop" + is_defined: true + symbol_type: FUNCTION + crc: 0x6f4a804b + type_id: 0x9b27064b + full_name: "__traceiter_spi_transfer_stop" +} elf_symbol { id: 0x863777a0 name: "__traceiter_suspend_resume" @@ -366675,6 +369286,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_attach_entity_load_avg" } +elf_symbol { + id: 0x06c757e7 + name: "__tracepoint_android_rvh_balance_fair" + is_defined: true + symbol_type: OBJECT + crc: 0x4801efe0 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_balance_fair" +} elf_symbol { id: 0xd7757253 name: "__tracepoint_android_rvh_before_do_sched_yield" @@ -366684,6 +369304,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_before_do_sched_yield" } +elf_symbol { + id: 0xd4910655 + name: "__tracepoint_android_rvh_before_pick_task_fair" + is_defined: true + symbol_type: OBJECT + crc: 0x90890009 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_before_pick_task_fair" +} +elf_symbol { + id: 0xc601df2c + name: "__tracepoint_android_rvh_blk_mq_has_request" + is_defined: true + symbol_type: OBJECT + crc: 0x98e4cce9 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_blk_mq_has_request" +} elf_symbol { id: 0xe64720ed name: "__tracepoint_android_rvh_bpf_int_jit_compile_ro" @@ -366810,6 +369448,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_cpu_util_cfs_boost" } +elf_symbol { + id: 0xe8729b05 + name: "__tracepoint_android_rvh_cpufreq_create_policy" + is_defined: true + symbol_type: OBJECT + crc: 0xe0c069f8 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_cpufreq_create_policy" +} elf_symbol { id: 0x9f779ef7 name: "__tracepoint_android_rvh_cpufreq_transition" @@ -367134,6 +369781,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_gic_v3_set_affinity" } +elf_symbol { + id: 0x9906fd54 + name: "__tracepoint_android_rvh_gup_longterm_locked" + is_defined: true + symbol_type: OBJECT + crc: 0x2b7a85c3 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_gup_longterm_locked" +} elf_symbol { id: 0x7fd0bf2f name: "__tracepoint_android_rvh_handle_bad_stack" @@ -367773,6 +370429,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_schedule" } +elf_symbol { + id: 0xa5be18ba + name: "__tracepoint_android_rvh_schedule_bug" + is_defined: true + symbol_type: OBJECT + crc: 0xf717c7eb + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_schedule_bug" +} elf_symbol { id: 0x0daed96e name: "__tracepoint_android_rvh_select_fallback_rq" @@ -367980,6 +370645,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_show_max_freq" } +elf_symbol { + id: 0x3e08e111 + name: "__tracepoint_android_rvh_swap_bio_charge" + is_defined: true + symbol_type: OBJECT + crc: 0x9c887a20 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_swap_bio_charge" +} elf_symbol { id: 0x64ce7cd6 name: "__tracepoint_android_rvh_swap_readpage_bdev_sync" @@ -367989,6 +370663,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_swap_readpage_bdev_sync" } +elf_symbol { + id: 0x2d1d2e40 + name: "__tracepoint_android_rvh_task_fits_cpu" + is_defined: true + symbol_type: OBJECT + crc: 0xfa31adb5 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_task_fits_cpu" +} elf_symbol { id: 0x5380a8d5 name: "__tracepoint_android_rvh_tcp_rcv_spurious_retrans" @@ -368772,6 +371455,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_binder_wait_for_work" } +elf_symbol { + id: 0xf2df2fc9 + name: "__tracepoint_android_vh_bio_add_page_merge_bypass" + is_defined: true + symbol_type: OBJECT + crc: 0xfdae8b60 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_bio_add_page_merge_bypass" +} elf_symbol { id: 0x0211d69f name: "__tracepoint_android_vh_blk_fill_rwbs" @@ -368934,6 +371626,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_check_uninterruptible_tasks_dn" } +elf_symbol { + id: 0xc974433c + name: "__tracepoint_android_vh_chk_task" + is_defined: true + symbol_type: OBJECT + crc: 0x32ced20a + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_chk_task" +} elf_symbol { id: 0xca10f06e name: "__tracepoint_android_vh_cleanup_old_buffers_bypass" @@ -368943,6 +371644,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_cleanup_old_buffers_bypass" } +elf_symbol { + id: 0x00405915 + name: "__tracepoint_android_vh_clear_curr_lazy" + is_defined: true + symbol_type: OBJECT + crc: 0xdbeec7db + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_clear_curr_lazy" +} elf_symbol { id: 0xe8cdcd02 name: "__tracepoint_android_vh_clear_reclaimed_folio" @@ -369177,6 +371887,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_cpuidle_psci_exit" } +elf_symbol { + id: 0xa1b79b01 + name: "__tracepoint_android_vh_cpuset_attach_task" + is_defined: true + symbol_type: OBJECT + crc: 0x9560838f + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_cpuset_attach_task" +} +elf_symbol { + id: 0xa16eb965 + name: "__tracepoint_android_vh_cpuset_css_online" + is_defined: true + symbol_type: OBJECT + crc: 0x6391c5db + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_cpuset_css_online" +} elf_symbol { id: 0x611b30be name: "__tracepoint_android_vh_customize_alloc_gfp" @@ -369618,6 +372346,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_f2fs_restore_priority" } +elf_symbol { + id: 0x2b59c1a7 + name: "__tracepoint_android_vh_fas_gpu_qos_update_tracer" + is_defined: true + symbol_type: OBJECT + crc: 0x3ee1cd20 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_fas_gpu_qos_update_tracer" +} elf_symbol { id: 0x50a83025 name: "__tracepoint_android_vh_file_is_tiny_bypass" @@ -369645,6 +372382,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_filemap_fault_end" } +elf_symbol { + id: 0xee4c2077 + name: "__tracepoint_android_vh_filemap_fault_folio_locked" + is_defined: true + symbol_type: OBJECT + crc: 0x41cc713d + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_filemap_fault_folio_locked" +} elf_symbol { id: 0xf045a23f name: "__tracepoint_android_vh_filemap_fault_pre_folio_locked" @@ -369681,6 +372427,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_filemap_get_folio" } +elf_symbol { + id: 0x7b27aae8 + name: "__tracepoint_android_vh_filemap_get_folio_end" + is_defined: true + symbol_type: OBJECT + crc: 0x3818b56b + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_filemap_get_folio_end" +} elf_symbol { id: 0xb34d9200 name: "__tracepoint_android_vh_filemap_map_pages" @@ -369717,6 +372472,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_filemap_read" } +elf_symbol { + id: 0x25deef51 + name: "__tracepoint_android_vh_filemap_read_end" + is_defined: true + symbol_type: OBJECT + crc: 0xb4a7e928 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_filemap_read_end" +} elf_symbol { id: 0xa8c30e54 name: "__tracepoint_android_vh_filemap_update_page" @@ -369762,6 +372526,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_flush_wq_wait_start" } +elf_symbol { + id: 0xc09285c1 + name: "__tracepoint_android_vh_folio_add_lru" + is_defined: true + symbol_type: OBJECT + crc: 0x2a23f777 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_folio_add_lru" +} elf_symbol { id: 0x5027d955 name: "__tracepoint_android_vh_folio_add_lru_folio_activate" @@ -369798,6 +372571,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_folio_trylock_set" } +elf_symbol { + id: 0xd808cc6f + name: "__tracepoint_android_vh_force_page_cache_ra" + is_defined: true + symbol_type: OBJECT + crc: 0x78c2948d + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_force_page_cache_ra" +} elf_symbol { id: 0x4f752d52 name: "__tracepoint_android_vh_free_mod_mem" @@ -370293,6 +373075,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_kvmalloc_node_use_vmalloc" } +elf_symbol { + id: 0x247a5590 + name: "__tracepoint_android_vh_lock_delay_schedule" + is_defined: true + symbol_type: OBJECT + crc: 0xb64c92dd + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_lock_delay_schedule" +} elf_symbol { id: 0xb1ce9338 name: "__tracepoint_android_vh_lock_folio_drop_mmap_end" @@ -370311,6 +373102,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_lock_folio_drop_mmap_start" } +elf_symbol { + id: 0x911d1d91 + name: "__tracepoint_android_vh_lock_task_exit" + is_defined: true + symbol_type: OBJECT + crc: 0x395925d7 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_lock_task_exit" +} +elf_symbol { + id: 0x86a7c204 + name: "__tracepoint_android_vh_lock_task_fork" + is_defined: true + symbol_type: OBJECT + crc: 0xc5b1123b + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_lock_task_fork" +} elf_symbol { id: 0x39d42312 name: "__tracepoint_android_vh_logbuf" @@ -370347,6 +373156,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_look_around_migrate_folio" } +elf_symbol { + id: 0xb6464279 + name: "__tracepoint_android_vh_loop_skip_queue_work" + is_defined: true + symbol_type: OBJECT + crc: 0x113a3a0b + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_loop_skip_queue_work" +} elf_symbol { id: 0x72b9b667 name: "__tracepoint_android_vh_lru_gen_add_folio_skip" @@ -370536,6 +373354,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_meminfo_proc_show" } +elf_symbol { + id: 0x5e49798d + name: "__tracepoint_android_vh_mempool_alloc_skip_wait" + is_defined: true + symbol_type: OBJECT + crc: 0x2317dfbe + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mempool_alloc_skip_wait" +} elf_symbol { id: 0xca4f3601 name: "__tracepoint_android_vh_mglru_should_abort_scan" @@ -370581,6 +373408,78 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_mm_compaction_end" } +elf_symbol { + id: 0xd947b2e6 + name: "__tracepoint_android_vh_mm_customize_ac" + is_defined: true + symbol_type: OBJECT + crc: 0x3f179df5 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_ac" +} +elf_symbol { + id: 0x2e4d22dc + name: "__tracepoint_android_vh_mm_customize_file_is_tiny" + is_defined: true + symbol_type: OBJECT + crc: 0xe3958935 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_file_is_tiny" +} +elf_symbol { + id: 0x95745257 + name: "__tracepoint_android_vh_mm_customize_lru_add_dst" + is_defined: true + symbol_type: OBJECT + crc: 0x3aecb3cb + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_lru_add_dst" +} +elf_symbol { + id: 0x9ba36d51 + name: "__tracepoint_android_vh_mm_customize_pgdat_balanced" + is_defined: true + symbol_type: OBJECT + crc: 0x9ff913d9 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_pgdat_balanced" +} +elf_symbol { + id: 0x19f24652 + name: "__tracepoint_android_vh_mm_customize_rmqueue" + is_defined: true + symbol_type: OBJECT + crc: 0xa845e3ff + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_rmqueue" +} +elf_symbol { + id: 0x94940652 + name: "__tracepoint_android_vh_mm_customize_suitable_zone" + is_defined: true + symbol_type: OBJECT + crc: 0x2252b177 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_suitable_zone" +} +elf_symbol { + id: 0x4c4316c6 + name: "__tracepoint_android_vh_mm_customize_zone_max_order" + is_defined: true + symbol_type: OBJECT + crc: 0xfb3906ea + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_zone_max_order" +} +elf_symbol { + id: 0xc25513c9 + name: "__tracepoint_android_vh_mm_customize_zone_pageset" + is_defined: true + symbol_type: OBJECT + crc: 0x94a19f6a + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_customize_zone_pageset" +} elf_symbol { id: 0xd333a65c name: "__tracepoint_android_vh_mm_direct_reclaim_enter" @@ -370608,6 +373507,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_mm_free_page" } +elf_symbol { + id: 0xba9e92f8 + name: "__tracepoint_android_vh_mm_isolate_priv_lru" + is_defined: true + symbol_type: OBJECT + crc: 0x3e8c5eb2 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_mm_isolate_priv_lru" +} elf_symbol { id: 0x0f593caf name: "__tracepoint_android_vh_mm_kcompactd_cpu_online" @@ -370761,6 +373669,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_mutex_wait_start" } +elf_symbol { + id: 0xaf0e1b2e + name: "__tracepoint_android_vh_nr_pcp_alloc" + is_defined: true + symbol_type: OBJECT + crc: 0x5786a94f + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_nr_pcp_alloc" +} elf_symbol { id: 0x830fa786 name: "__tracepoint_android_vh_ogki_async_psi_bypass" @@ -370959,6 +373876,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_ogki_ufs_dsm" } +elf_symbol { + id: 0x3a3a83c9 + name: "__tracepoint_android_vh_oom_evaluate_task_bypass" + is_defined: true + symbol_type: OBJECT + crc: 0xe3144ac3 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_oom_evaluate_task_bypass" +} +elf_symbol { + id: 0x05b8726a + name: "__tracepoint_android_vh_oom_reaper_delay_bypass" + is_defined: true + symbol_type: OBJECT + crc: 0xddfa5818 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_oom_reaper_delay_bypass" +} elf_symbol { id: 0xdfe6a3ee name: "__tracepoint_android_vh_oom_swapmem_gather_finish" @@ -370986,6 +373921,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_page_add_new_anon_rmap" } +elf_symbol { + id: 0x6e9a5b3f + name: "__tracepoint_android_vh_page_cache_ra_order_bypass" + is_defined: true + symbol_type: OBJECT + crc: 0x98a590ee + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_page_cache_ra_order_bypass" +} +elf_symbol { + id: 0x62445736 + name: "__tracepoint_android_vh_page_cache_ra_unbounded" + is_defined: true + symbol_type: OBJECT + crc: 0x4d013baf + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_page_cache_ra_unbounded" +} elf_symbol { id: 0xa5409038 name: "__tracepoint_android_vh_page_cache_readahead_end" @@ -371049,6 +374002,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_pagetypeinfo_show" } +elf_symbol { + id: 0x7c21ed99 + name: "__tracepoint_android_vh_pcp_alloc_factor_adjust" + is_defined: true + symbol_type: OBJECT + crc: 0x96e57aef + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_pcp_alloc_factor_adjust" +} elf_symbol { id: 0xa4c454d8 name: "__tracepoint_android_vh_percpu_rwsem_down_read" @@ -371238,6 +374200,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_ptype_head" } +elf_symbol { + id: 0x428834b1 + name: "__tracepoint_android_vh_put_super" + is_defined: true + symbol_type: OBJECT + crc: 0xb23edd99 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_put_super" +} +elf_symbol { + id: 0x74120954 + name: "__tracepoint_android_vh_put_task" + is_defined: true + symbol_type: OBJECT + crc: 0xd1ab6463 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_put_task" +} elf_symbol { id: 0xf04b1c62 name: "__tracepoint_android_vh_queue_request_and_unlock" @@ -371346,6 +374326,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_regmap_update" } +elf_symbol { + id: 0x8a5eba26 + name: "__tracepoint_android_vh_remove_mapping" + is_defined: true + symbol_type: OBJECT + crc: 0x3102043e + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_remove_mapping" +} +elf_symbol { + id: 0xa6c678f2 + name: "__tracepoint_android_vh_remove_mapping_failed" + is_defined: true + symbol_type: OBJECT + crc: 0x88a2f87a + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_remove_mapping_failed" +} elf_symbol { id: 0x42ed2724 name: "__tracepoint_android_vh_replace_anon_vma_name" @@ -371355,6 +374353,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_replace_anon_vma_name" } +elf_symbol { + id: 0x2732d436 + name: "__tracepoint_android_vh_resched_curr_lazy" + is_defined: true + symbol_type: OBJECT + crc: 0x7589574b + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_resched_curr_lazy" +} elf_symbol { id: 0xa8bc195b name: "__tracepoint_android_vh_reserve_highatomic_bypass" @@ -371364,6 +374371,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_reserve_highatomic_bypass" } +elf_symbol { + id: 0x7ca6fd40 + name: "__tracepoint_android_vh_restore_curr_resched" + is_defined: true + symbol_type: OBJECT + crc: 0x77ad0841 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_restore_curr_resched" +} elf_symbol { id: 0x9490325d name: "__tracepoint_android_vh_restore_mm_flags" @@ -372309,6 +375325,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_test_clear_look_around_ref" } +elf_symbol { + id: 0x4776310b + name: "__tracepoint_android_vh_thaw_killed_process" + is_defined: true + symbol_type: OBJECT + crc: 0xd3564467 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_thaw_killed_process" +} elf_symbol { id: 0xafd4e4b1 name: "__tracepoint_android_vh_thermal_pm_notify_suspend" @@ -372381,6 +375406,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_try_to_freeze_todo" } +elf_symbol { + id: 0x8d5e4915 + name: "__tracepoint_android_vh_try_to_freeze_todo_logging" + is_defined: true + symbol_type: OBJECT + crc: 0xada315ae + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_try_to_freeze_todo_logging" +} elf_symbol { id: 0x09ba106b name: "__tracepoint_android_vh_try_to_freeze_todo_unfrozen" @@ -373380,6 +376414,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_pelt_se_tp" } +elf_symbol { + id: 0xd12d0b3f + name: "__tracepoint_rcu_stall_warning" + is_defined: true + symbol_type: OBJECT + crc: 0x062987a8 + type_id: 0x18ccbd2c + full_name: "__tracepoint_rcu_stall_warning" +} elf_symbol { id: 0x36579996 name: "__tracepoint_rpm_idle" @@ -373605,6 +376648,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_softirq_exit" } +elf_symbol { + id: 0x676093e0 + name: "__tracepoint_spi_transfer_stop" + is_defined: true + symbol_type: OBJECT + crc: 0xc81eeaeb + type_id: 0x18ccbd2c + full_name: "__tracepoint_spi_transfer_stop" +} elf_symbol { id: 0x5b327cd6 name: "__tracepoint_suspend_resume" @@ -374028,6 +377080,15 @@ elf_symbol { type_id: 0x97ce5a08 full_name: "__xa_alloc_cyclic" } +elf_symbol { + id: 0x017f1776 + name: "__xa_clear_mark" + is_defined: true + symbol_type: FUNCTION + crc: 0x8391b043 + type_id: 0x1ae27633 + full_name: "__xa_clear_mark" +} elf_symbol { id: 0x1820daaf name: "__xa_cmpxchg" @@ -374595,6 +377656,15 @@ elf_symbol { type_id: 0x1b7558dd full_name: "activate_task" } +elf_symbol { + id: 0xbe1d1ccc + name: "active_load_balance_cpu_stop" + is_defined: true + symbol_type: FUNCTION + crc: 0xe3ada447 + type_id: 0x9bafba63 + full_name: "active_load_balance_cpu_stop" +} elf_symbol { id: 0xd20e014a name: "add_cpu" @@ -374982,6 +378052,15 @@ elf_symbol { type_id: 0xca4b711f full_name: "alloc_io_pgtable_ops" } +elf_symbol { + id: 0x82a5f160 + name: "alloc_iova" + is_defined: true + symbol_type: FUNCTION + crc: 0xfd3617ee + type_id: 0x4812e6a9 + full_name: "alloc_iova" +} elf_symbol { id: 0x9cff6ebf name: "alloc_iova_fast" @@ -375191,6 +378270,15 @@ elf_symbol { type_id: 0x33756485 full_name: "arch_freq_scale" } +elf_symbol { + id: 0x97ee5cc4 + name: "arch_invalidate_pmem" + is_defined: true + symbol_type: FUNCTION + crc: 0x2c635527 + type_id: 0x1567dd7e + full_name: "arch_invalidate_pmem" +} elf_symbol { id: 0x0c805860 name: "arch_stack_walk" @@ -375332,6 +378420,15 @@ elf_symbol { type_id: 0xd473b4c2 full_name: "async_schedule_node_domain" } +elf_symbol { + id: 0xa09651b4 + name: "async_synchronize_cookie" + is_defined: true + symbol_type: FUNCTION + crc: 0x7a81541b + type_id: 0x0ef7ccf3 + full_name: "async_synchronize_cookie" +} elf_symbol { id: 0x67de90c8 name: "async_synchronize_full_domain" @@ -376457,6 +379554,15 @@ elf_symbol { type_id: 0x1ba86ccd full_name: "blk_mq_wait_quiesce_done" } +elf_symbol { + id: 0x1fa93c73 + name: "blk_op_str" + is_defined: true + symbol_type: FUNCTION + crc: 0x19c42957 + type_id: 0xb0abc898 + full_name: "blk_op_str" +} elf_symbol { id: 0x76783663 name: "blk_pm_runtime_init" @@ -376925,6 +380031,24 @@ elf_symbol { type_id: 0x13f12250 full_name: "blkdev_put" } +elf_symbol { + id: 0x3a816bcd + name: "blkg_conf_exit" + is_defined: true + symbol_type: FUNCTION + crc: 0x5ae3d8f9 + type_id: 0x1213f72a + full_name: "blkg_conf_exit" +} +elf_symbol { + id: 0xb0f09bc0 + name: "blkg_conf_init" + is_defined: true + symbol_type: FUNCTION + crc: 0x256303a5 + type_id: 0x1201f8b1 + full_name: "blkg_conf_init" +} elf_symbol { id: 0x35eb35db name: "blkg_conf_prep" @@ -378076,6 +381200,15 @@ elf_symbol { type_id: 0x1e99dfd1 full_name: "cec_unregister_adapter" } +elf_symbol { + id: 0x4d06ba53 + name: "cgroup_add_dfl_cftypes" + is_defined: true + symbol_type: FUNCTION + crc: 0x4a841d8b + type_id: 0x9f4f3809 + full_name: "cgroup_add_dfl_cftypes" +} elf_symbol { id: 0x3c85cae0 name: "cgroup_add_legacy_cftypes" @@ -378157,6 +381290,15 @@ elf_symbol { type_id: 0x9017f3f4 full_name: "check_zeroed_user" } +elf_symbol { + id: 0x61d03c8a + name: "class_compat_register" + is_defined: true + symbol_type: FUNCTION + crc: 0x32c3cb4e + type_id: 0x6424be43 + full_name: "class_compat_register" +} elf_symbol { id: 0xb5759473 name: "class_create" @@ -380273,6 +383415,15 @@ elf_symbol { type_id: 0x10985193 full_name: "cpuidle_pause_and_lock" } +elf_symbol { + id: 0x3a3b1479 + name: "cpuidle_register" + is_defined: true + symbol_type: FUNCTION + crc: 0xd93d8695 + type_id: 0x9b48ef76 + full_name: "cpuidle_register" +} elf_symbol { id: 0x9b056e94 name: "cpuidle_register_governor" @@ -380291,6 +383442,15 @@ elf_symbol { type_id: 0x10985193 full_name: "cpuidle_resume_and_unlock" } +elf_symbol { + id: 0x92dc6c49 + name: "cpuidle_unregister" + is_defined: true + symbol_type: FUNCTION + crc: 0x39e6940e + type_id: 0x169466ca + full_name: "cpuidle_unregister" +} elf_symbol { id: 0xd878ab56 name: "cpumask_any_and_distribute" @@ -381480,6 +384640,15 @@ elf_symbol { type_id: 0x141901a7 full_name: "d_invalidate" } +elf_symbol { + id: 0xd9d103aa + name: "d_lookup" + is_defined: true + symbol_type: FUNCTION + crc: 0x3b6f6226 + type_id: 0x8265b46c + full_name: "d_lookup" +} elf_symbol { id: 0x52864a7e name: "d_make_root" @@ -382404,6 +385573,15 @@ elf_symbol { type_id: 0x1c76bf87 full_name: "dev_pm_domain_detach_list" } +elf_symbol { + id: 0x5647272b + name: "dev_pm_domain_set" + is_defined: true + symbol_type: FUNCTION + crc: 0x8e3b0cf7 + type_id: 0x10ff6b9a + full_name: "dev_pm_domain_set" +} elf_symbol { id: 0x6ef38ba9 name: "dev_pm_genpd_add_notifier" @@ -382539,6 +385717,15 @@ elf_symbol { type_id: 0x81de8179 full_name: "dev_pm_opp_find_freq_ceil" } +elf_symbol { + id: 0xb411a3b0 + name: "dev_pm_opp_find_freq_ceil_indexed" + is_defined: true + symbol_type: FUNCTION + crc: 0xdf032798 + type_id: 0x81ecc373 + full_name: "dev_pm_opp_find_freq_ceil_indexed" +} elf_symbol { id: 0x56a85a70 name: "dev_pm_opp_find_freq_exact" @@ -382557,6 +385744,15 @@ elf_symbol { type_id: 0x81de8179 full_name: "dev_pm_opp_find_freq_floor" } +elf_symbol { + id: 0xfb909691 + name: "dev_pm_opp_find_freq_floor_indexed" + is_defined: true + symbol_type: FUNCTION + crc: 0x1bbba20c + type_id: 0x81ecc373 + full_name: "dev_pm_opp_find_freq_floor_indexed" +} elf_symbol { id: 0xdb13d287 name: "dev_pm_opp_find_level_exact" @@ -383286,6 +386482,15 @@ elf_symbol { type_id: 0x9497b363 full_name: "devfreq_update_status" } +elf_symbol { + id: 0x6ff644b0 + name: "devfreq_update_target" + is_defined: true + symbol_type: FUNCTION + crc: 0x94c2ede5 + type_id: 0x9497b363 + full_name: "devfreq_update_target" +} elf_symbol { id: 0x2ad163ea name: "device_add" @@ -384141,6 +387346,15 @@ elf_symbol { type_id: 0x3511a755 full_name: "devm_clk_get_optional_enabled" } +elf_symbol { + id: 0xcd6f8e13 + name: "devm_clk_hw_get_clk" + is_defined: true + symbol_type: FUNCTION + crc: 0x9051435e + type_id: 0x35cdf327 + full_name: "devm_clk_hw_get_clk" +} elf_symbol { id: 0x0d9ec99a name: "devm_clk_hw_register" @@ -385131,6 +388345,15 @@ elf_symbol { type_id: 0x3bb8868a full_name: "devm_power_supply_register" } +elf_symbol { + id: 0xd4544d3c + name: "devm_power_supply_register_no_ws" + is_defined: true + symbol_type: FUNCTION + crc: 0xfc1c846e + type_id: 0x3bb8868a + full_name: "devm_power_supply_register_no_ws" +} elf_symbol { id: 0x36be1a91 name: "devm_pwm_get" @@ -385627,6 +388850,15 @@ elf_symbol { type_id: 0x88f0a30c full_name: "dget_parent" } +elf_symbol { + id: 0x30dd6796 + name: "disable_hardirq" + is_defined: true + symbol_type: FUNCTION + crc: 0xbd3fe1e3 + type_id: 0xefc5028b + full_name: "disable_hardirq" +} elf_symbol { id: 0x573e2956 name: "disable_irq" @@ -387125,6 +390357,15 @@ elf_symbol { type_id: 0x84ad5b06 full_name: "do_send_sig_info" } +elf_symbol { + id: 0xfa43ef2b + name: "do_settimeofday64" + is_defined: true + symbol_type: FUNCTION + crc: 0x7a95e5ae + type_id: 0x9203828a + full_name: "do_settimeofday64" +} elf_symbol { id: 0xb50c088c name: "do_trace_netlink_extack" @@ -389087,6 +392328,15 @@ elf_symbol { type_id: 0xf06a3ce3 full_name: "drm_edid_is_valid" } +elf_symbol { + id: 0xd1f29cd4 + name: "drm_edid_override_connector_update" + is_defined: true + symbol_type: FUNCTION + crc: 0x3e20bf8a + type_id: 0x91ed8ab4 + full_name: "drm_edid_override_connector_update" +} elf_symbol { id: 0x2396487e name: "drm_edid_raw" @@ -389159,6 +392409,15 @@ elf_symbol { type_id: 0x930bf787 full_name: "drm_event_reserve_init_locked" } +elf_symbol { + id: 0x2027d999 + name: "drm_file_get_master" + is_defined: true + symbol_type: FUNCTION + crc: 0x2765be64 + type_id: 0xf3ed77aa + full_name: "drm_file_get_master" +} elf_symbol { id: 0x9d83d0f8 name: "drm_flip_work_cleanup" @@ -390185,6 +393444,15 @@ elf_symbol { type_id: 0x92c9aa18 full_name: "drm_mode_create_dp_colorspace_property" } +elf_symbol { + id: 0x465f41c2 + name: "drm_mode_create_dvi_i_properties" + is_defined: true + symbol_type: FUNCTION + crc: 0x57e7bcfa + type_id: 0x9341cc84 + full_name: "drm_mode_create_dvi_i_properties" +} elf_symbol { id: 0xbe3f1757 name: "drm_mode_create_hdmi_colorspace_property" @@ -391554,6 +394822,15 @@ elf_symbol { type_id: 0x140e294e full_name: "dst_release" } +elf_symbol { + id: 0xe09fd784 + name: "dummy_irq_chip" + is_defined: true + symbol_type: OBJECT + crc: 0x06ae18cc + type_id: 0xadd7ccb6 + full_name: "dummy_irq_chip" +} elf_symbol { id: 0xded28924 name: "dump_backtrace" @@ -393159,6 +396436,15 @@ elf_symbol { type_id: 0xf6f86f1f full_name: "folio_clear_dirty_for_io" } +elf_symbol { + id: 0x1ac8aa52 + name: "folio_deactivate" + is_defined: true + symbol_type: FUNCTION + crc: 0x7abc9b3a + type_id: 0x18c46588 + full_name: "folio_deactivate" +} elf_symbol { id: 0xf83588d6 name: "folio_end_private_2" @@ -393186,6 +396472,15 @@ elf_symbol { type_id: 0x637004ab full_name: "folio_mapping" } +elf_symbol { + id: 0xd2e101fd + name: "folio_mark_accessed" + is_defined: true + symbol_type: FUNCTION + crc: 0x74311ee4 + type_id: 0x18c46588 + full_name: "folio_mark_accessed" +} elf_symbol { id: 0xcef0ca54 name: "folio_mark_dirty" @@ -393276,6 +396571,15 @@ elf_symbol { type_id: 0x1b29bd73 full_name: "folio_zero_new_buffers" } +elf_symbol { + id: 0xf20cee49 + name: "folios_put_refs" + is_defined: true + symbol_type: FUNCTION + crc: 0xe3e054ae + type_id: 0x1f5cc15e + full_name: "folios_put_refs" +} elf_symbol { id: 0xcb14b5cc name: "follow_pfn" @@ -393457,6 +396761,15 @@ elf_symbol { type_id: 0x165a4073 full_name: "free_io_pgtable_ops" } +elf_symbol { + id: 0x25649749 + name: "free_iova" + is_defined: true + symbol_type: FUNCTION + crc: 0x42e921d3 + type_id: 0x1fb4bf8e + full_name: "free_iova" +} elf_symbol { id: 0x998ad938 name: "free_iova_fast" @@ -393538,6 +396851,15 @@ elf_symbol { type_id: 0x16a2dac9 full_name: "free_workqueue_attrs" } +elf_symbol { + id: 0x2da69553 + name: "freelist_task" + is_defined: true + symbol_type: OBJECT + crc: 0x454f148d + type_id: 0x1d19a9d5 + full_name: "freelist_task" +} elf_symbol { id: 0x2ed5588c name: "freezer_active" @@ -395314,6 +398636,15 @@ elf_symbol { type_id: 0x1bc52dc4 full_name: "get_slabinfo" } +elf_symbol { + id: 0xea4eb631 + name: "get_swap_device" + is_defined: true + symbol_type: FUNCTION + crc: 0x375dff50 + type_id: 0x356e90af + full_name: "get_swap_device" +} elf_symbol { id: 0x35d05a7d name: "get_task_cred" @@ -396853,6 +400184,15 @@ elf_symbol { type_id: 0x9d398c85 full_name: "hid_driver_suspend" } +elf_symbol { + id: 0x74041f0e + name: "hid_field_extract" + is_defined: true + symbol_type: FUNCTION + crc: 0x59849d89 + type_id: 0x34f12b6b + full_name: "hid_field_extract" +} elf_symbol { id: 0x8717f26f name: "hid_hw_close" @@ -396934,6 +400274,15 @@ elf_symbol { type_id: 0x9d13a27c full_name: "hid_input_report" } +elf_symbol { + id: 0xbfab1179 + name: "hid_is_usb" + is_defined: true + symbol_type: FUNCTION + crc: 0xe0a3d0c1 + type_id: 0xf1b60446 + full_name: "hid_is_usb" +} elf_symbol { id: 0x0eb11e95 name: "hid_open_report" @@ -396952,6 +400301,24 @@ elf_symbol { type_id: 0x9e711486 full_name: "hid_parse_report" } +elf_symbol { + id: 0x88241dc1 + name: "hid_register_report" + is_defined: true + symbol_type: FUNCTION + crc: 0x665f4981 + type_id: 0xe10f8963 + full_name: "hid_register_report" +} +elf_symbol { + id: 0x741d5af7 + name: "hid_report_raw_event" + is_defined: true + symbol_type: FUNCTION + crc: 0xee30c096 + type_id: 0x9d13a27c + full_name: "hid_report_raw_event" +} elf_symbol { id: 0x6e37f09d name: "hid_unregister_driver" @@ -396961,6 +400328,15 @@ elf_symbol { type_id: 0x104da524 full_name: "hid_unregister_driver" } +elf_symbol { + id: 0x29d6d842 + name: "hid_validate_values" + is_defined: true + symbol_type: FUNCTION + crc: 0xc1730de5 + type_id: 0xe10f9802 + full_name: "hid_validate_values" +} elf_symbol { id: 0x334ff68f name: "high_memory" @@ -396970,6 +400346,15 @@ elf_symbol { type_id: 0x18bd6530 full_name: "high_memory" } +elf_symbol { + id: 0xd12f6730 + name: "host1x_context_device_bus_type" + is_defined: true + symbol_type: OBJECT + crc: 0x19002790 + type_id: 0x257935aa + full_name: "host1x_context_device_bus_type" +} elf_symbol { id: 0xd7ecf501 name: "housekeeping_cpumask" @@ -399063,6 +402448,24 @@ elf_symbol { type_id: 0x167935b9 full_name: "inode_nohighmem" } +elf_symbol { + id: 0xb2c3f64e + name: "inode_owner_or_capable" + is_defined: true + symbol_type: FUNCTION + crc: 0x103c7097 + type_id: 0xfd016131 + full_name: "inode_owner_or_capable" +} +elf_symbol { + id: 0x29a1f6c6 + name: "inode_permission" + is_defined: true + symbol_type: FUNCTION + crc: 0x73b676fb + type_id: 0x9eb543ce + full_name: "inode_permission" +} elf_symbol { id: 0xab642653 name: "inode_query_iversion" @@ -399145,6 +402548,15 @@ elf_symbol { type_id: 0xfab4e970 full_name: "input_device_enabled" } +elf_symbol { + id: 0x594eb27d + name: "input_enable_softrepeat" + is_defined: true + symbol_type: FUNCTION + crc: 0x85a146c6 + type_id: 0x150da89f + full_name: "input_enable_softrepeat" +} elf_symbol { id: 0x1cfb5d03 name: "input_event" @@ -399649,6 +403061,15 @@ elf_symbol { type_id: 0x1b338a63 full_name: "iommu_detach_device_pasid" } +elf_symbol { + id: 0xc578c7af + name: "iommu_detach_group" + is_defined: true + symbol_type: FUNCTION + crc: 0xf1ed8a55 + type_id: 0x1b8fc015 + full_name: "iommu_detach_group" +} elf_symbol { id: 0x4c05b91e name: "iommu_dev_disable_feature" @@ -400108,6 +403529,24 @@ elf_symbol { type_id: 0xaa8f5c2d full_name: "iov_iter_zero" } +elf_symbol { + id: 0x20af6f53 + name: "iova_cache_get" + is_defined: true + symbol_type: FUNCTION + crc: 0x438d8df2 + type_id: 0x9d80e32f + full_name: "iova_cache_get" +} +elf_symbol { + id: 0x3f2fa978 + name: "iova_cache_put" + is_defined: true + symbol_type: FUNCTION + crc: 0xc7061ef3 + type_id: 0x10985193 + full_name: "iova_cache_put" +} elf_symbol { id: 0x6c6beb07 name: "iova_domain_init_rcaches" @@ -400585,6 +404024,15 @@ elf_symbol { type_id: 0x9abe486f full_name: "irq_domain_disconnect_hierarchy" } +elf_symbol { + id: 0x3fa49947 + name: "irq_domain_free_fwnode" + is_defined: true + symbol_type: FUNCTION + crc: 0xddedcc5c + type_id: 0x11bc7f41 + full_name: "irq_domain_free_fwnode" +} elf_symbol { id: 0x327b1cbb name: "irq_domain_free_irqs_common" @@ -401219,6 +404667,15 @@ elf_symbol { type_id: 0xc20627da full_name: "jiffies_to_msecs" } +elf_symbol { + id: 0xaccdc826 + name: "jiffies_to_timespec64" + is_defined: true + symbol_type: FUNCTION + crc: 0x188ea314 + type_id: 0x1c56d411 + full_name: "jiffies_to_timespec64" +} elf_symbol { id: 0x81dadb36 name: "jiffies_to_usecs" @@ -401319,6 +404776,15 @@ elf_symbol { type_id: 0x3643daf2 full_name: "kernel_cpustat" } +elf_symbol { + id: 0x27bb5e77 + name: "kernel_getpeername" + is_defined: true + symbol_type: FUNCTION + crc: 0x2efc4c91 + type_id: 0x994ba444 + full_name: "kernel_getpeername" +} elf_symbol { id: 0x1b2f5f11 name: "kernel_getsockname" @@ -403591,6 +407057,15 @@ elf_symbol { type_id: 0xc59b3a62 full_name: "lookup_user_key" } +elf_symbol { + id: 0x1128cd26 + name: "loop_process_cmd_list" + is_defined: true + symbol_type: FUNCTION + crc: 0x3f5d3947 + type_id: 0x1f841b2f + full_name: "loop_process_cmd_list" +} elf_symbol { id: 0x493ce9fc name: "loops_per_jiffy" @@ -403636,6 +407111,15 @@ elf_symbol { type_id: 0x90448105 full_name: "lzo1x_decompress_safe" } +elf_symbol { + id: 0xdda6583e + name: "lzorle1x_1_compress" + is_defined: true + symbol_type: FUNCTION + crc: 0x79f697e4 + type_id: 0x9044872a + full_name: "lzorle1x_1_compress" +} elf_symbol { id: 0x7bd17841 name: "mac_pton" @@ -404815,6 +408299,15 @@ elf_symbol { type_id: 0x9aaabb4a full_name: "migrate_swap" } +elf_symbol { + id: 0x68c3a63b + name: "mii_check_media" + is_defined: true + symbol_type: FUNCTION + crc: 0xe310a431 + type_id: 0xcb5a4db0 + full_name: "mii_check_media" +} elf_symbol { id: 0x399512e2 name: "mii_ethtool_get_link_ksettings" @@ -404824,6 +408317,15 @@ elf_symbol { type_id: 0x147ac3d5 full_name: "mii_ethtool_get_link_ksettings" } +elf_symbol { + id: 0xbc90165b + name: "mii_ethtool_gset" + is_defined: true + symbol_type: FUNCTION + crc: 0xda85aa4d + type_id: 0x142e3633 + full_name: "mii_ethtool_gset" +} elf_symbol { id: 0x69f7fc4b name: "mii_ethtool_set_link_ksettings" @@ -405688,6 +409190,15 @@ elf_symbol { type_id: 0x9d19aa55 full_name: "mmc_select_timing" } +elf_symbol { + id: 0xfd26acf1 + name: "mmc_send_abort_tuning" + is_defined: true + symbol_type: FUNCTION + crc: 0x40d95903 + type_id: 0x9322918f + full_name: "mmc_send_abort_tuning" +} elf_symbol { id: 0xfb31b7b7 name: "mmc_send_status" @@ -406972,6 +410483,24 @@ elf_symbol { type_id: 0x9e768b87 full_name: "nf_register_net_hooks" } +elf_symbol { + id: 0x7d136425 + name: "nf_sk_lookup_slow_v4" + is_defined: true + symbol_type: FUNCTION + crc: 0xd62cd0ac + type_id: 0x6f9910f4 + full_name: "nf_sk_lookup_slow_v4" +} +elf_symbol { + id: 0xfe90bb40 + name: "nf_sk_lookup_slow_v6" + is_defined: true + symbol_type: FUNCTION + crc: 0x514b71c5 + type_id: 0x6f9910f4 + full_name: "nf_sk_lookup_slow_v6" +} elf_symbol { id: 0x840dad92 name: "nf_unregister_net_hook" @@ -407270,6 +410799,15 @@ elf_symbol { type_id: 0x152401b7 full_name: "nsecs_to_jiffies" } +elf_symbol { + id: 0x03adeb35 + name: "nsecs_to_jiffies64" + is_defined: true + symbol_type: FUNCTION + crc: 0x75d0deb9 + type_id: 0x73531faf + full_name: "nsecs_to_jiffies64" +} elf_symbol { id: 0x77722cfe name: "nvdimm_bus_register" @@ -409875,6 +413413,15 @@ elf_symbol { type_id: 0x04ce0b6f full_name: "pci_find_ext_capability" } +elf_symbol { + id: 0x780d1110 + name: "pci_find_host_bridge" + is_defined: true + symbol_type: FUNCTION + crc: 0x2e22e561 + type_id: 0x636e6c75 + full_name: "pci_find_host_bridge" +} elf_symbol { id: 0xbbb674cc name: "pci_find_next_bus" @@ -409983,6 +413530,15 @@ elf_symbol { type_id: 0x93acae9b full_name: "pci_host_probe" } +elf_symbol { + id: 0x64fb0865 + name: "pci_ignore_hotplug" + is_defined: true + symbol_type: FUNCTION + crc: 0x8723eb68 + type_id: 0x14e1f000 + full_name: "pci_ignore_hotplug" +} elf_symbol { id: 0x9c6c58ea name: "pci_iomap" @@ -410595,6 +414151,15 @@ elf_symbol { type_id: 0x7eefaca3 full_name: "pcie_link_speed" } +elf_symbol { + id: 0x912e005a + name: "pcie_reset_flr" + is_defined: true + symbol_type: FUNCTION + crc: 0x3f585be2 + type_id: 0x984cbfc3 + full_name: "pcie_reset_flr" +} elf_symbol { id: 0xffa3ecd1 name: "pcie_set_mps" @@ -411252,6 +414817,15 @@ elf_symbol { type_id: 0x8d09c85c full_name: "phy_get_internal_delay" } +elf_symbol { + id: 0x7de90fab + name: "phy_get_pause" + is_defined: true + symbol_type: FUNCTION + crc: 0x92770062 + type_id: 0x1c86b5f0 + full_name: "phy_get_pause" +} elf_symbol { id: 0x30e87de6 name: "phy_init" @@ -412053,6 +415627,15 @@ elf_symbol { type_id: 0x3945ae7e full_name: "pidfd_get_pid" } +elf_symbol { + id: 0x13474ea8 + name: "pidfd_get_task" + is_defined: true + symbol_type: FUNCTION + crc: 0xc9e4687d + type_id: 0x62ea57b4 + full_name: "pidfd_get_task" +} elf_symbol { id: 0x2f08ffc6 name: "pin_get_name" @@ -412656,6 +416239,15 @@ elf_symbol { type_id: 0xad77a8f0 full_name: "platform_find_device_by_driver" } +elf_symbol { + id: 0x072cf683 + name: "platform_get_ethdev_address" + is_defined: true + symbol_type: FUNCTION + crc: 0xad01dec9 + type_id: 0x9ddc45fb + full_name: "platform_get_ethdev_address" +} elf_symbol { id: 0xb11c92aa name: "platform_get_irq" @@ -412764,6 +416356,24 @@ elf_symbol { type_id: 0x1e879adb full_name: "platform_unregister_drivers" } +elf_symbol { + id: 0x2c4cd39b + name: "plist_add" + is_defined: true + symbol_type: FUNCTION + crc: 0x267f26d1 + type_id: 0x147b2718 + full_name: "plist_add" +} +elf_symbol { + id: 0xd734aca0 + name: "plist_del" + is_defined: true + symbol_type: FUNCTION + crc: 0x1a5a9583 + type_id: 0x147b2718 + full_name: "plist_del" +} elf_symbol { id: 0xb634bd54 name: "pm_clk_add" @@ -412818,6 +416428,33 @@ elf_symbol { type_id: 0x9d16dd74 full_name: "pm_clk_suspend" } +elf_symbol { + id: 0xfa3ae6ff + name: "pm_generic_freeze" + is_defined: true + symbol_type: FUNCTION + crc: 0xaa59b3a8 + type_id: 0x9d16dd74 + full_name: "pm_generic_freeze" +} +elf_symbol { + id: 0xd43b4f66 + name: "pm_generic_poweroff" + is_defined: true + symbol_type: FUNCTION + crc: 0x6e5c45fc + type_id: 0x9d16dd74 + full_name: "pm_generic_poweroff" +} +elf_symbol { + id: 0xfb61e52a + name: "pm_generic_restore" + is_defined: true + symbol_type: FUNCTION + crc: 0x758be1ec + type_id: 0x9d16dd74 + full_name: "pm_generic_restore" +} elf_symbol { id: 0xe243fac9 name: "pm_generic_resume" @@ -412854,6 +416491,15 @@ elf_symbol { type_id: 0x9d16dd74 full_name: "pm_generic_suspend" } +elf_symbol { + id: 0xcae30dd3 + name: "pm_generic_thaw" + is_defined: true + symbol_type: FUNCTION + crc: 0xc6da423b + type_id: 0x9d16dd74 + full_name: "pm_generic_thaw" +} elf_symbol { id: 0x92ae3cd2 name: "pm_genpd_add_device" @@ -413340,6 +416986,15 @@ elf_symbol { type_id: 0x9d80e32f full_name: "power_supply_is_system_supplied" } +elf_symbol { + id: 0x778ba508 + name: "power_supply_powers" + is_defined: true + symbol_type: FUNCTION + crc: 0x974bc8a3 + type_id: 0x9b8cc3ad + full_name: "power_supply_powers" +} elf_symbol { id: 0x26c5a1b9 name: "power_supply_put" @@ -414916,6 +418571,15 @@ elf_symbol { type_id: 0xb809e664 full_name: "read_cache_page_gfp" } +elf_symbol { + id: 0x94f761df + name: "read_comp_algo_dictionary" + is_defined: true + symbol_type: FUNCTION + crc: 0xf3d1ca9b + type_id: 0x17267b6f + full_name: "read_comp_algo_dictionary" +} elf_symbol { id: 0xfdbe6bc0 name: "read_sanitised_ftr_reg" @@ -416473,6 +420137,15 @@ elf_symbol { type_id: 0x9285caa7 full_name: "reset_control_assert" } +elf_symbol { + id: 0x3fb5c5d4 + name: "reset_control_bulk_acquire" + is_defined: true + symbol_type: FUNCTION + crc: 0x0207a6c6 + type_id: 0x84c7bf9c + full_name: "reset_control_bulk_acquire" +} elf_symbol { id: 0xace4bcdf name: "reset_control_bulk_assert" @@ -416491,6 +420164,15 @@ elf_symbol { type_id: 0x84c7bf9c full_name: "reset_control_bulk_deassert" } +elf_symbol { + id: 0x1a32d2b3 + name: "reset_control_bulk_release" + is_defined: true + symbol_type: FUNCTION + crc: 0xf4cd9f8f + type_id: 0x09df0d20 + full_name: "reset_control_bulk_release" +} elf_symbol { id: 0xd76b82b2 name: "reset_control_deassert" @@ -417752,6 +421434,15 @@ elf_symbol { type_id: 0x16d1bcdb full_name: "sched_clock_register" } +elf_symbol { + id: 0x36d6d8fb + name: "sched_energy_present" + is_defined: true + symbol_type: OBJECT + crc: 0xe69d902d + type_id: 0x8e47c273 + full_name: "sched_energy_present" +} elf_symbol { id: 0xd2090791 name: "sched_feat_keys" @@ -418166,6 +421857,15 @@ elf_symbol { type_id: 0x0ff8d13d full_name: "scsi_host_alloc" } +elf_symbol { + id: 0xfaba855b + name: "scsi_host_busy" + is_defined: true + symbol_type: FUNCTION + crc: 0x9665b881 + type_id: 0x98764c5c + full_name: "scsi_host_busy" +} elf_symbol { id: 0x022517f0 name: "scsi_host_lookup" @@ -419291,6 +422991,15 @@ elf_symbol { type_id: 0x1d80f189 full_name: "serdev_device_wait_until_sent" } +elf_symbol { + id: 0x3a629e14 + name: "serdev_device_write" + is_defined: true + symbol_type: FUNCTION + crc: 0x76f68b17 + type_id: 0x93b7b77f + full_name: "serdev_device_write" +} elf_symbol { id: 0x6e10178a name: "serdev_device_write_buf" @@ -419309,6 +423018,15 @@ elf_symbol { type_id: 0x1e70c9ff full_name: "serdev_device_write_flush" } +elf_symbol { + id: 0x826ffe72 + name: "serdev_device_write_wakeup" + is_defined: true + symbol_type: FUNCTION + crc: 0x626c1a0d + type_id: 0x1e70c9ff + full_name: "serdev_device_write_wakeup" +} elf_symbol { id: 0x0746da0a name: "serial8250_clear_and_reinit_fifos" @@ -420221,6 +423939,15 @@ elf_symbol { type_id: 0xf10425d9 full_name: "simple_dir_operations" } +elf_symbol { + id: 0xb8a763fb + name: "simple_empty" + is_defined: true + symbol_type: FUNCTION + crc: 0xf90f2bcc + type_id: 0x9901b31b + full_name: "simple_empty" +} elf_symbol { id: 0x1b5d526c name: "simple_get_link" @@ -422705,6 +426432,15 @@ elf_symbol { type_id: 0x909c23c2 full_name: "snd_soc_get_dai_id" } +elf_symbol { + id: 0x4086fab0 + name: "snd_soc_get_dai_name" + is_defined: true + symbol_type: FUNCTION + crc: 0x347721f4 + type_id: 0x9294d8c1 + full_name: "snd_soc_get_dai_name" +} elf_symbol { id: 0xa64c7fe5 name: "snd_soc_get_dai_via_args" @@ -426113,6 +429849,15 @@ elf_symbol { type_id: 0x19c5ab78 full_name: "tegra_bpmp_free_mrq" } +elf_symbol { + id: 0x4a1dceb2 + name: "tegra_bpmp_get" + is_defined: true + symbol_type: FUNCTION + crc: 0x066facf2 + type_id: 0x4f10276c + full_name: "tegra_bpmp_get" +} elf_symbol { id: 0xa2565005 name: "tegra_bpmp_mrq_is_supported" @@ -426158,6 +429903,15 @@ elf_symbol { type_id: 0x9548acb9 full_name: "tegra_bpmp_transfer_atomic" } +elf_symbol { + id: 0x3c42a066 + name: "tegra_fuse_readl" + is_defined: true + symbol_type: FUNCTION + crc: 0xa9ed62d2 + type_id: 0x91bef2c3 + full_name: "tegra_fuse_readl" +} elf_symbol { id: 0x07f159e7 name: "tegra_ivc_init" @@ -426329,6 +430083,15 @@ elf_symbol { type_id: 0x95a4c6b8 full_name: "thermal_zone_bind_cooling_device" } +elf_symbol { + id: 0x6bef1b0f + name: "thermal_zone_device_critical" + is_defined: true + symbol_type: FUNCTION + crc: 0x9fe77c40 + type_id: 0x192105f2 + full_name: "thermal_zone_device_critical" +} elf_symbol { id: 0x96242237 name: "thermal_zone_device_disable" @@ -426736,6 +430499,15 @@ elf_symbol { type_id: 0x57dbd34b full_name: "trace_clock_local" } +elf_symbol { + id: 0x476977df + name: "trace_dump_stack" + is_defined: true + symbol_type: FUNCTION + crc: 0x9430b198 + type_id: 0x09506558 + full_name: "trace_dump_stack" +} elf_symbol { id: 0x0e209244 name: "trace_event_buffer_commit" @@ -427547,6 +431319,15 @@ elf_symbol { type_id: 0x1210f89b full_name: "ttm_tt_fini" } +elf_symbol { + id: 0x3ffae0cf + name: "tty_buffer_set_limit" + is_defined: true + symbol_type: FUNCTION + crc: 0xd72e1b4c + type_id: 0x9a98740b + full_name: "tty_buffer_set_limit" +} elf_symbol { id: 0x0b4dd20d name: "tty_chars_in_buffer" @@ -427691,6 +431472,15 @@ elf_symbol { type_id: 0x97258bf7 full_name: "tty_mode_ioctl" } +elf_symbol { + id: 0xb626741e + name: "tty_port_alloc_xmit_buf" + is_defined: true + symbol_type: FUNCTION + crc: 0x878fe5ac + type_id: 0x9b04f747 + full_name: "tty_port_alloc_xmit_buf" +} elf_symbol { id: 0x2a899db0 name: "tty_port_close" @@ -427709,6 +431499,15 @@ elf_symbol { type_id: 0x161c45fb full_name: "tty_port_destroy" } +elf_symbol { + id: 0x568d2ee4 + name: "tty_port_free_xmit_buf" + is_defined: true + symbol_type: FUNCTION + crc: 0x341c1990 + type_id: 0x161c45fb + full_name: "tty_port_free_xmit_buf" +} elf_symbol { id: 0x604f0f0b name: "tty_port_hangup" @@ -427772,6 +431571,15 @@ elf_symbol { type_id: 0xab7dda53 full_name: "tty_port_register_device" } +elf_symbol { + id: 0xd4a176ac + name: "tty_port_register_device_attr" + is_defined: true + symbol_type: FUNCTION + crc: 0x115d6426 + type_id: 0xab7ddc65 + full_name: "tty_port_register_device_attr" +} elf_symbol { id: 0xdcd44915 name: "tty_port_tty_get" @@ -431456,6 +435264,240 @@ elf_symbol { type_id: 0x12d17df8 full_name: "usb_wakeup_notification" } +elf_symbol { + id: 0x8189acd0 + name: "usbnet_cdc_zte_rx_fixup" + is_defined: true + symbol_type: FUNCTION + crc: 0xa983761b + type_id: 0x9ee7a4e9 + full_name: "usbnet_cdc_zte_rx_fixup" +} +elf_symbol { + id: 0xbbf007f4 + name: "usbnet_change_mtu" + is_defined: true + symbol_type: FUNCTION + crc: 0x435f7c8e + type_id: 0x90b5e896 + full_name: "usbnet_change_mtu" +} +elf_symbol { + id: 0xd28057f3 + name: "usbnet_defer_kevent" + is_defined: true + symbol_type: FUNCTION + crc: 0x1f61e2be + type_id: 0x1276a8bd + full_name: "usbnet_defer_kevent" +} +elf_symbol { + id: 0x580c56b8 + name: "usbnet_disconnect" + is_defined: true + symbol_type: FUNCTION + crc: 0x9e97a5b8 + type_id: 0x18d9f669 + full_name: "usbnet_disconnect" +} +elf_symbol { + id: 0xe395160a + name: "usbnet_generic_cdc_bind" + is_defined: true + symbol_type: FUNCTION + crc: 0xf86d1255 + type_id: 0x9e768332 + full_name: "usbnet_generic_cdc_bind" +} +elf_symbol { + id: 0x8f0c866d + name: "usbnet_get_drvinfo" + is_defined: true + symbol_type: FUNCTION + crc: 0xff0c468e + type_id: 0x1cc68e0f + full_name: "usbnet_get_drvinfo" +} +elf_symbol { + id: 0x52816b1c + name: "usbnet_get_endpoints" + is_defined: true + symbol_type: FUNCTION + crc: 0x2b3d70b6 + type_id: 0x9e768332 + full_name: "usbnet_get_endpoints" +} +elf_symbol { + id: 0x4f4e9353 + name: "usbnet_get_link" + is_defined: true + symbol_type: FUNCTION + crc: 0x29647872 + type_id: 0x4951a397 + full_name: "usbnet_get_link" +} +elf_symbol { + id: 0x3f51a906 + name: "usbnet_get_link_ksettings_mii" + is_defined: true + symbol_type: FUNCTION + crc: 0x542790f2 + type_id: 0x914dbe16 + full_name: "usbnet_get_link_ksettings_mii" +} +elf_symbol { + id: 0x6515c416 + name: "usbnet_get_msglevel" + is_defined: true + symbol_type: FUNCTION + crc: 0x6756e0bd + type_id: 0x4951a397 + full_name: "usbnet_get_msglevel" +} +elf_symbol { + id: 0x6c854cb3 + name: "usbnet_nway_reset" + is_defined: true + symbol_type: FUNCTION + crc: 0x00a6ee95 + type_id: 0x91296bda + full_name: "usbnet_nway_reset" +} +elf_symbol { + id: 0xcd021e80 + name: "usbnet_open" + is_defined: true + symbol_type: FUNCTION + crc: 0x05022b70 + type_id: 0x91296bda + full_name: "usbnet_open" +} +elf_symbol { + id: 0x147e4ad2 + name: "usbnet_probe" + is_defined: true + symbol_type: FUNCTION + crc: 0x34bbff65 + type_id: 0x952154fc + full_name: "usbnet_probe" +} +elf_symbol { + id: 0x167360fc + name: "usbnet_read_cmd" + is_defined: true + symbol_type: FUNCTION + crc: 0xfef19b91 + type_id: 0x9e5fdeba + full_name: "usbnet_read_cmd" +} +elf_symbol { + id: 0x69b1070f + name: "usbnet_read_cmd_nopm" + is_defined: true + symbol_type: FUNCTION + crc: 0xf0c01aa9 + type_id: 0x9e5fdeba + full_name: "usbnet_read_cmd_nopm" +} +elf_symbol { + id: 0x3be16591 + name: "usbnet_resume" + is_defined: true + symbol_type: FUNCTION + crc: 0x93351aba + type_id: 0x95c144d5 + full_name: "usbnet_resume" +} +elf_symbol { + id: 0x6e3c0f91 + name: "usbnet_set_link_ksettings_mii" + is_defined: true + symbol_type: FUNCTION + crc: 0x3548ee6f + type_id: 0x91cc9f9b + full_name: "usbnet_set_link_ksettings_mii" +} +elf_symbol { + id: 0x99cb2edf + name: "usbnet_set_msglevel" + is_defined: true + symbol_type: FUNCTION + crc: 0x5ad0cec8 + type_id: 0x1f15f9ca + full_name: "usbnet_set_msglevel" +} +elf_symbol { + id: 0x2325cd65 + name: "usbnet_skb_return" + is_defined: true + symbol_type: FUNCTION + crc: 0x4d91baa1 + type_id: 0x13ff1655 + full_name: "usbnet_skb_return" +} +elf_symbol { + id: 0x9cf79f8d + name: "usbnet_start_xmit" + is_defined: true + symbol_type: FUNCTION + crc: 0xba157904 + type_id: 0x5515f11d + full_name: "usbnet_start_xmit" +} +elf_symbol { + id: 0x2f365f93 + name: "usbnet_stop" + is_defined: true + symbol_type: FUNCTION + crc: 0xda116f94 + type_id: 0x91296bda + full_name: "usbnet_stop" +} +elf_symbol { + id: 0x6229bd38 + name: "usbnet_suspend" + is_defined: true + symbol_type: FUNCTION + crc: 0xa4480689 + type_id: 0x96011ad3 + full_name: "usbnet_suspend" +} +elf_symbol { + id: 0x90b81315 + name: "usbnet_tx_timeout" + is_defined: true + symbol_type: FUNCTION + crc: 0xc85d991d + type_id: 0x1d27ccfe + full_name: "usbnet_tx_timeout" +} +elf_symbol { + id: 0xb011e423 + name: "usbnet_write_cmd" + is_defined: true + symbol_type: FUNCTION + crc: 0x2c929a73 + type_id: 0x9e5fde3c + full_name: "usbnet_write_cmd" +} +elf_symbol { + id: 0x27363bb6 + name: "usbnet_write_cmd_async" + is_defined: true + symbol_type: FUNCTION + crc: 0x67b83014 + type_id: 0x9e5fde3c + full_name: "usbnet_write_cmd_async" +} +elf_symbol { + id: 0x10932377 + name: "usbnet_write_cmd_nopm" + is_defined: true + symbol_type: FUNCTION + crc: 0xc0069a92 + type_id: 0x9e5fde3c + full_name: "usbnet_write_cmd_nopm" +} elf_symbol { id: 0x37ac0936 name: "user_describe" @@ -431528,6 +435570,15 @@ elf_symbol { type_id: 0x45396185 full_name: "utf8_data_table_new" } +elf_symbol { + id: 0x97509378 + name: "utf8_normalize" + is_defined: true + symbol_type: FUNCTION + crc: 0x57db8fd6 + type_id: 0x90203fcd + full_name: "utf8_normalize" +} elf_symbol { id: 0x4b0dcba7 name: "utf8_to_utf32" @@ -432104,6 +436155,15 @@ elf_symbol { type_id: 0x9bba7997 full_name: "v4l2_fh_release" } +elf_symbol { + id: 0xc28778a5 + name: "v4l2_fill_pixfmt" + is_defined: true + symbol_type: FUNCTION + crc: 0xcc501597 + type_id: 0x95a5d553 + full_name: "v4l2_fill_pixfmt" +} elf_symbol { id: 0x538ad5cc name: "v4l2_find_dv_timings_cap" @@ -436307,6 +440367,15 @@ elf_symbol { type_id: 0x80d705f2 full_name: "zlib_inflate" } +elf_symbol { + id: 0x84c3c252 + name: "zlib_inflateEnd" + is_defined: true + symbol_type: FUNCTION + crc: 0x107e5878 + type_id: 0x814b86be + full_name: "zlib_inflateEnd" +} elf_symbol { id: 0xf6eda681 name: "zlib_inflateIncomp" @@ -436460,6 +440529,51 @@ elf_symbol { type_id: 0x9dc2f17b full_name: "zstd_compress_cctx" } +elf_symbol { + id: 0xd045d6af + name: "zstd_compress_using_cdict" + is_defined: true + symbol_type: FUNCTION + crc: 0xe210316b + type_id: 0x9dc2f160 + full_name: "zstd_compress_using_cdict" +} +elf_symbol { + id: 0xe1a54f6b + name: "zstd_create_cctx_advanced" + is_defined: true + symbol_type: FUNCTION + crc: 0x556d59f5 + type_id: 0xce1fb5db + full_name: "zstd_create_cctx_advanced" +} +elf_symbol { + id: 0x04058f3f + name: "zstd_create_cdict_byreference" + is_defined: true + symbol_type: FUNCTION + crc: 0x6eb92bfb + type_id: 0x043c4ad0 + full_name: "zstd_create_cdict_byreference" +} +elf_symbol { + id: 0x0576d81a + name: "zstd_create_dctx_advanced" + is_defined: true + symbol_type: FUNCTION + crc: 0x77856e63 + type_id: 0xde51e0f8 + full_name: "zstd_create_dctx_advanced" +} +elf_symbol { + id: 0xb729831c + name: "zstd_create_ddict_byreference" + is_defined: true + symbol_type: FUNCTION + crc: 0x33cf7944 + type_id: 0xffb39db8 + full_name: "zstd_create_ddict_byreference" +} elf_symbol { id: 0xb2b54222 name: "zstd_dctx_workspace_bound" @@ -436478,6 +440592,69 @@ elf_symbol { type_id: 0x9ed60004 full_name: "zstd_decompress_dctx" } +elf_symbol { + id: 0x6f74cced + name: "zstd_decompress_using_ddict" + is_defined: true + symbol_type: FUNCTION + crc: 0x273b1906 + type_id: 0x9ed600da + full_name: "zstd_decompress_using_ddict" +} +elf_symbol { + id: 0xc282f08e + name: "zstd_default_clevel" + is_defined: true + symbol_type: FUNCTION + crc: 0xbc630e8b + type_id: 0x9d80e32f + full_name: "zstd_default_clevel" +} +elf_symbol { + id: 0x27a93b2c + name: "zstd_free_cctx" + is_defined: true + symbol_type: FUNCTION + crc: 0xeafdafcc + type_id: 0x9d9dd01b + full_name: "zstd_free_cctx" +} +elf_symbol { + id: 0x0fea3e22 + name: "zstd_free_cdict" + is_defined: true + symbol_type: FUNCTION + crc: 0xcb3d63da + type_id: 0x993539b7 + full_name: "zstd_free_cdict" +} +elf_symbol { + id: 0xde450a6f + name: "zstd_free_dctx" + is_defined: true + symbol_type: FUNCTION + crc: 0x5752e7d6 + type_id: 0x9e8921bb + full_name: "zstd_free_dctx" +} +elf_symbol { + id: 0x4df06dae + name: "zstd_free_ddict" + is_defined: true + symbol_type: FUNCTION + crc: 0xf7bd965b + type_id: 0x9f90af15 + full_name: "zstd_free_ddict" +} +elf_symbol { + id: 0xadaec705 + name: "zstd_get_cparams" + is_defined: true + symbol_type: FUNCTION + crc: 0x5c931650 + type_id: 0xd596d12a + full_name: "zstd_get_cparams" +} elf_symbol { id: 0x3542c796 name: "zstd_get_params" @@ -436521,8 +440698,17 @@ interface { symbol_id: 0xd5cb4002 symbol_id: 0x17a6dc14 symbol_id: 0x2dddbe52 + symbol_id: 0x26c56443 + symbol_id: 0xf4f8ddc2 + symbol_id: 0x7c322a62 + symbol_id: 0x38116da8 symbol_id: 0x31a154b4 + symbol_id: 0xc12163b9 symbol_id: 0x740d2b27 + symbol_id: 0x758b3708 + symbol_id: 0xef9094cc + symbol_id: 0x35928a4a + symbol_id: 0xab09bd5e symbol_id: 0xf56ca4c7 symbol_id: 0x59348d38 symbol_id: 0x5b94e7ac @@ -436567,6 +440753,7 @@ interface { symbol_id: 0x5a6afeee symbol_id: 0xc8eb813d symbol_id: 0x7bf707d4 + symbol_id: 0xbf1d0365 symbol_id: 0x2d1c1f2d symbol_id: 0x9bed4545 symbol_id: 0xefdcf4c1 @@ -436684,6 +440871,7 @@ interface { symbol_id: 0x69ff7fd9 symbol_id: 0xebf4b11f symbol_id: 0xb8e29ef0 + symbol_id: 0xcd2cca92 symbol_id: 0x5b1ea047 symbol_id: 0x07f88ce8 symbol_id: 0xe458ae39 @@ -436727,6 +440915,7 @@ interface { symbol_id: 0x1c57a05b symbol_id: 0xdf2d2827 symbol_id: 0x70f1b87e + symbol_id: 0xc37fd40e symbol_id: 0x64bb2ece symbol_id: 0x16e2bd0b symbol_id: 0xe064ac65 @@ -436939,7 +441128,10 @@ interface { symbol_id: 0x0b48afa1 symbol_id: 0xa927338c symbol_id: 0x6d4cc1a2 + symbol_id: 0x965ea17d symbol_id: 0x144db0a1 + symbol_id: 0x166cd967 + symbol_id: 0xb2eed0b6 symbol_id: 0xe193199f symbol_id: 0x7b2cd91d symbol_id: 0x192bbbd5 @@ -436954,6 +441146,7 @@ interface { symbol_id: 0x6c4603a3 symbol_id: 0x4053abff symbol_id: 0xcd9f9741 + symbol_id: 0xb3da5d93 symbol_id: 0x1baec33d symbol_id: 0xf5806276 symbol_id: 0x1a7f4cf6 @@ -436990,6 +441183,7 @@ interface { symbol_id: 0xede4c5f2 symbol_id: 0xa23fdad4 symbol_id: 0x41d29e07 + symbol_id: 0x97095cae symbol_id: 0xe443e259 symbol_id: 0xf24e6a40 symbol_id: 0xe199ac0d @@ -437061,6 +441255,7 @@ interface { symbol_id: 0x415dd5e7 symbol_id: 0xd7ca6cc8 symbol_id: 0x2bb177b4 + symbol_id: 0x40bb3f78 symbol_id: 0x0f807f10 symbol_id: 0xe48752ef symbol_id: 0x4c24a4be @@ -437084,7 +441279,9 @@ interface { symbol_id: 0x1228e7e9 symbol_id: 0x73c83ef4 symbol_id: 0x46515de8 + symbol_id: 0x4ae52717 symbol_id: 0xb80ecc98 + symbol_id: 0x20ccf43a symbol_id: 0x3b650ee3 symbol_id: 0xcf016f05 symbol_id: 0x79480d0a @@ -437172,6 +441369,7 @@ interface { symbol_id: 0x5cf60b10 symbol_id: 0xcc2edef9 symbol_id: 0x61c82fb5 + symbol_id: 0x3e149573 symbol_id: 0x6f2a9ca9 symbol_id: 0x47b414fe symbol_id: 0x4ae900a4 @@ -437190,7 +441388,9 @@ interface { symbol_id: 0xeebf3d23 symbol_id: 0xa2224fa6 symbol_id: 0xd37cc550 + symbol_id: 0x253229ea symbol_id: 0xef7737f8 + symbol_id: 0x124b6497 symbol_id: 0xe50d4e50 symbol_id: 0xaed0a325 symbol_id: 0xc308efa9 @@ -437217,6 +441417,8 @@ interface { symbol_id: 0xca50936b symbol_id: 0x431c5bf5 symbol_id: 0x9f47ed94 + symbol_id: 0x9004b9db + symbol_id: 0x3d81eceb symbol_id: 0x96cfd964 symbol_id: 0xecf16451 symbol_id: 0x3444cc6c @@ -437266,26 +441468,32 @@ interface { symbol_id: 0xcb0a2c01 symbol_id: 0x377d134d symbol_id: 0x905272e6 + symbol_id: 0x150ea575 symbol_id: 0x93a4717b symbol_id: 0x6091a763 symbol_id: 0x19fc04e0 + symbol_id: 0xbd855995 symbol_id: 0x980c4b2d symbol_id: 0x734973d4 symbol_id: 0xddad3a11 symbol_id: 0x49c69e22 + symbol_id: 0x2c39dada symbol_id: 0xb7d91f76 symbol_id: 0x4eda1196 symbol_id: 0x97914e0c symbol_id: 0x158bf9d3 + symbol_id: 0x48ad4ccb symbol_id: 0x0e217a5e symbol_id: 0x1b65e021 symbol_id: 0xd44631ef symbol_id: 0xf110e9f6 symbol_id: 0x5f626e29 + symbol_id: 0xfaa99ddb symbol_id: 0x5dd57eff symbol_id: 0xef70c295 symbol_id: 0x1fca37bf symbol_id: 0xe6702595 + symbol_id: 0xb10c10b1 symbol_id: 0xa51a0f0c symbol_id: 0x9e91661b symbol_id: 0xc189c2a7 @@ -437341,12 +441549,16 @@ interface { symbol_id: 0x19426a78 symbol_id: 0x04e61b81 symbol_id: 0x0992491b + symbol_id: 0xedae015e symbol_id: 0xf56758ea symbol_id: 0x95b7ecea + symbol_id: 0xc955d513 + symbol_id: 0xfc3decf6 symbol_id: 0x66cefe64 symbol_id: 0x57087e9e symbol_id: 0xe19d2bf8 symbol_id: 0x993f42ff + symbol_id: 0xaf9604df symbol_id: 0xa6d58835 symbol_id: 0x3a741247 symbol_id: 0xf6964618 @@ -437368,14 +441580,24 @@ interface { symbol_id: 0x3431d426 symbol_id: 0x8c174a7d symbol_id: 0xea8ce2b0 + symbol_id: 0xe02fe9e3 symbol_id: 0x105350cb symbol_id: 0x83742db6 symbol_id: 0xf8413699 symbol_id: 0x22de652b symbol_id: 0xf928bf8a + symbol_id: 0x28fd3694 + symbol_id: 0x7a520c7e + symbol_id: 0x4f807371 + symbol_id: 0x7637ee8f + symbol_id: 0x46918874 + symbol_id: 0x0b6a1d80 + symbol_id: 0x2be8352c + symbol_id: 0xa792d107 symbol_id: 0x9f58159a symbol_id: 0x29c67d40 symbol_id: 0x3fe16974 + symbol_id: 0x945dd126 symbol_id: 0x6f5c8275 symbol_id: 0xf182fb15 symbol_id: 0xe44dacb1 @@ -437393,6 +441615,7 @@ interface { symbol_id: 0x34502396 symbol_id: 0xe17999f4 symbol_id: 0x721b87b7 + symbol_id: 0xdb0a9300 symbol_id: 0x1c5ab7ec symbol_id: 0xb514c5cc symbol_id: 0x056c5936 @@ -437415,9 +441638,13 @@ interface { symbol_id: 0xf7d13588 symbol_id: 0xd82b5082 symbol_id: 0x084f4da7 + symbol_id: 0x1ff6ff8b + symbol_id: 0x2884ad94 symbol_id: 0x724577f8 symbol_id: 0xac55c769 symbol_id: 0xacaadcc9 + symbol_id: 0xbf28fbd5 + symbol_id: 0x7f95d934 symbol_id: 0xd0065122 symbol_id: 0xed7b87e3 symbol_id: 0x03508ed9 @@ -437425,6 +441652,7 @@ interface { symbol_id: 0xb4d5ffdc symbol_id: 0x1fc96009 symbol_id: 0x3c5e668a + symbol_id: 0xc5066ceb symbol_id: 0x13b0736e symbol_id: 0xc72f2012 symbol_id: 0xd14f3adb @@ -437446,6 +441674,8 @@ interface { symbol_id: 0xf2c39651 symbol_id: 0x3c69cece symbol_id: 0x93303c51 + symbol_id: 0x358b7cff + symbol_id: 0x1adac14e symbol_id: 0x2c963d28 symbol_id: 0x3a545b61 symbol_id: 0x96662dde @@ -437458,8 +441688,12 @@ interface { symbol_id: 0xf2957eae symbol_id: 0x7bd6f0e3 symbol_id: 0xe2d75052 + symbol_id: 0x4ac41e54 + symbol_id: 0xda7f66bc symbol_id: 0x55f0aa22 + symbol_id: 0x799188c0 symbol_id: 0xd0707ed1 + symbol_id: 0xfb6f79f6 symbol_id: 0x2f6e3ceb symbol_id: 0xf10ce16f symbol_id: 0x34b157e6 @@ -437565,6 +441799,7 @@ interface { symbol_id: 0xfcc3558a symbol_id: 0x9c4fb9ab symbol_id: 0x48f0cf25 + symbol_id: 0x6d485f59 symbol_id: 0x40d428a7 symbol_id: 0x3d1d57b0 symbol_id: 0xa00355c9 @@ -437573,6 +441808,7 @@ interface { symbol_id: 0x641d703d symbol_id: 0xeac5f4dd symbol_id: 0x2bc25325 + symbol_id: 0xd8abda6b symbol_id: 0x0119fc41 symbol_id: 0xd9f43028 symbol_id: 0xc0482ac3 @@ -437684,6 +441920,7 @@ interface { symbol_id: 0xbdceccc2 symbol_id: 0x3e69b303 symbol_id: 0x7f1a363f + symbol_id: 0x46a383e5 symbol_id: 0x147cc848 symbol_id: 0x189202e9 symbol_id: 0x04c5b642 @@ -437709,6 +441946,7 @@ interface { symbol_id: 0x8ce8e50d symbol_id: 0x62a42e92 symbol_id: 0xa7ccbbf5 + symbol_id: 0x536acf4e symbol_id: 0x863777a0 symbol_id: 0x313b4348 symbol_id: 0xcac5a7d5 @@ -437729,7 +441967,10 @@ interface { symbol_id: 0x748c1fd7 symbol_id: 0xcb42202e symbol_id: 0xc9400cec + symbol_id: 0x06c757e7 symbol_id: 0xd7757253 + symbol_id: 0xd4910655 + symbol_id: 0xc601df2c symbol_id: 0xe64720ed symbol_id: 0x1bc75987 symbol_id: 0x1e8a7e23 @@ -437744,6 +441985,7 @@ interface { symbol_id: 0xb2f08341 symbol_id: 0x05576f29 symbol_id: 0xa4ed84bf + symbol_id: 0xe8729b05 symbol_id: 0x9f779ef7 symbol_id: 0x331f5cf0 symbol_id: 0xc53e6388 @@ -437780,6 +442022,7 @@ interface { symbol_id: 0xb894fcfc symbol_id: 0x555d2f3e symbol_id: 0x47c27595 + symbol_id: 0x9906fd54 symbol_id: 0x7fd0bf2f symbol_id: 0x1bf33aee symbol_id: 0xe06d5cfb @@ -437851,6 +442094,7 @@ interface { symbol_id: 0x7babd1c5 symbol_id: 0x43527efa symbol_id: 0x51af6212 + symbol_id: 0xa5be18ba symbol_id: 0x0daed96e symbol_id: 0x3bb904a5 symbol_id: 0xec4d6758 @@ -437874,7 +442118,9 @@ interface { symbol_id: 0x8a4070f7 symbol_id: 0x00b7ed82 symbol_id: 0xe8cacf26 + symbol_id: 0x3e08e111 symbol_id: 0x64ce7cd6 + symbol_id: 0x2d1d2e40 symbol_id: 0x5380a8d5 symbol_id: 0x1f12a317 symbol_id: 0x454d16cc @@ -437962,6 +442208,7 @@ interface { symbol_id: 0xa9d55136 symbol_id: 0xabc729f7 symbol_id: 0x75b174bb + symbol_id: 0xf2df2fc9 symbol_id: 0x0211d69f symbol_id: 0xf458bc84 symbol_id: 0x002cab5a @@ -437980,7 +442227,9 @@ interface { symbol_id: 0x22653fb1 symbol_id: 0x2cf5984c symbol_id: 0x62b6878e + symbol_id: 0xc974433c symbol_id: 0xca10f06e + symbol_id: 0x00405915 symbol_id: 0xe8cdcd02 symbol_id: 0xbbfbc9db symbol_id: 0x36fc8313 @@ -438007,6 +442256,8 @@ interface { symbol_id: 0x750e1421 symbol_id: 0x66a766d7 symbol_id: 0xc44d61e6 + symbol_id: 0xa1b79b01 + symbol_id: 0xa16eb965 symbol_id: 0x611b30be symbol_id: 0xc2fde553 symbol_id: 0x66823d32 @@ -438056,26 +442307,32 @@ interface { symbol_id: 0x34b6590b symbol_id: 0x26d15e93 symbol_id: 0xbf5d713c + symbol_id: 0x2b59c1a7 symbol_id: 0x50a83025 symbol_id: 0x0c03d499 symbol_id: 0x197d6be2 + symbol_id: 0xee4c2077 symbol_id: 0xf045a23f symbol_id: 0xdac9a862 symbol_id: 0x9eaf9eab symbol_id: 0x6d970e8c + symbol_id: 0x7b27aae8 symbol_id: 0xb34d9200 symbol_id: 0x223c9b64 symbol_id: 0xcef3bcd6 symbol_id: 0xf61927fd + symbol_id: 0x25deef51 symbol_id: 0xa8c30e54 symbol_id: 0x843cb07b symbol_id: 0xb548aa95 symbol_id: 0xbb05ffe8 symbol_id: 0x0a50cb7b + symbol_id: 0xc09285c1 symbol_id: 0x5027d955 symbol_id: 0xb2684007 symbol_id: 0xa3ede5d5 symbol_id: 0x539bf337 + symbol_id: 0xd808cc6f symbol_id: 0x4f752d52 symbol_id: 0x901d0e89 symbol_id: 0x04a824b5 @@ -438131,12 +442388,16 @@ interface { symbol_id: 0xb1f4423e symbol_id: 0x918a6f4b symbol_id: 0x8bd577fd + symbol_id: 0x247a5590 symbol_id: 0xb1ce9338 symbol_id: 0xeb8159ac + symbol_id: 0x911d1d91 + symbol_id: 0x86a7c204 symbol_id: 0x39d42312 symbol_id: 0x94ddd62c symbol_id: 0xda2d53f2 symbol_id: 0x50a5a949 + symbol_id: 0xb6464279 symbol_id: 0x72b9b667 symbol_id: 0xc6d82ff1 symbol_id: 0x6e96d5d2 @@ -438158,14 +442419,24 @@ interface { symbol_id: 0xac62c748 symbol_id: 0xa2bd1edf symbol_id: 0xa5b4e5b2 + symbol_id: 0x5e49798d symbol_id: 0xca4f3601 symbol_id: 0x420ef2d0 symbol_id: 0xb32b3b17 symbol_id: 0xa3d5f70d symbol_id: 0x72c79d80 + symbol_id: 0xd947b2e6 + symbol_id: 0x2e4d22dc + symbol_id: 0x95745257 + symbol_id: 0x9ba36d51 + symbol_id: 0x19f24652 + symbol_id: 0x94940652 + symbol_id: 0x4c4316c6 + symbol_id: 0xc25513c9 symbol_id: 0xd333a65c symbol_id: 0xddcff44a symbol_id: 0x533ca98e + symbol_id: 0xba9e92f8 symbol_id: 0x0f593caf symbol_id: 0x47bcd15f symbol_id: 0xb6da564f @@ -438183,6 +442454,7 @@ interface { symbol_id: 0x2d21cb40 symbol_id: 0x76e879b6 symbol_id: 0xae5bbde9 + symbol_id: 0xaf0e1b2e symbol_id: 0x830fa786 symbol_id: 0x36cbe626 symbol_id: 0xcda04300 @@ -438205,9 +442477,13 @@ interface { symbol_id: 0x32321776 symbol_id: 0xd716ec80 symbol_id: 0x62ffe4fd + symbol_id: 0x3a3a83c9 + symbol_id: 0x05b8726a symbol_id: 0xdfe6a3ee symbol_id: 0x6bc613d3 symbol_id: 0x20d2ceb3 + symbol_id: 0x6e9a5b3f + symbol_id: 0x62445736 symbol_id: 0xa5409038 symbol_id: 0xeeab2985 symbol_id: 0x988bfcb7 @@ -438215,6 +442491,7 @@ interface { symbol_id: 0x352038ba symbol_id: 0x94cb1cab symbol_id: 0x2f61d5a4 + symbol_id: 0x7c21ed99 symbol_id: 0xa4c454d8 symbol_id: 0x7d42b7c8 symbol_id: 0x3d63616d @@ -438236,6 +442513,8 @@ interface { symbol_id: 0x0e92ee53 symbol_id: 0xe7b3cb34 symbol_id: 0xb0c197a3 + symbol_id: 0x428834b1 + symbol_id: 0x74120954 symbol_id: 0xf04b1c62 symbol_id: 0x811d5fab symbol_id: 0x2d9a331c @@ -438248,8 +442527,12 @@ interface { symbol_id: 0x57c5897c symbol_id: 0x03334941 symbol_id: 0x13b2fb38 + symbol_id: 0x8a5eba26 + symbol_id: 0xa6c678f2 symbol_id: 0x42ed2724 + symbol_id: 0x2732d436 symbol_id: 0xa8bc195b + symbol_id: 0x7ca6fd40 symbol_id: 0x9490325d symbol_id: 0xd6904c6d symbol_id: 0x8dee298c @@ -438355,6 +442638,7 @@ interface { symbol_id: 0xbf50cdb8 symbol_id: 0x1a59144d symbol_id: 0x4ef2c337 + symbol_id: 0x4776310b symbol_id: 0xafd4e4b1 symbol_id: 0xba33290e symbol_id: 0xd292bd77 @@ -438363,6 +442647,7 @@ interface { symbol_id: 0x2df766e3 symbol_id: 0x0896a337 symbol_id: 0xd9d2bcff + symbol_id: 0x8d5e4915 symbol_id: 0x09ba106b symbol_id: 0xf9580976 symbol_id: 0x0f6579e9 @@ -438474,6 +442759,7 @@ interface { symbol_id: 0x67e2af24 symbol_id: 0xf3cb5921 symbol_id: 0xcb194ec9 + symbol_id: 0xd12d0b3f symbol_id: 0x36579996 symbol_id: 0xfbd9814b symbol_id: 0x7c591834 @@ -438499,6 +442785,7 @@ interface { symbol_id: 0xf68d6127 symbol_id: 0x9a65f074 symbol_id: 0x158cc323 + symbol_id: 0x676093e0 symbol_id: 0x5b327cd6 symbol_id: 0x9deae602 symbol_id: 0xe91cdbb3 @@ -438546,6 +442833,7 @@ interface { symbol_id: 0xcf1808d5 symbol_id: 0x01b711f3 symbol_id: 0xb0e141a3 + symbol_id: 0x017f1776 symbol_id: 0x1820daaf symbol_id: 0x4b52e164 symbol_id: 0xd6e3f912 @@ -438609,6 +442897,7 @@ interface { symbol_id: 0xfbb6649f symbol_id: 0xdf7862e2 symbol_id: 0xa3514d5f + symbol_id: 0xbe1d1ccc symbol_id: 0xd20e014a symbol_id: 0xf09e1ca4 symbol_id: 0x4c40d07d @@ -438652,6 +442941,7 @@ interface { symbol_id: 0x962c959a symbol_id: 0xdeecf7a4 symbol_id: 0xeba70df6 + symbol_id: 0x82a5f160 symbol_id: 0x9cff6ebf symbol_id: 0xcf6df527 symbol_id: 0x00374dfe @@ -438675,6 +442965,7 @@ interface { symbol_id: 0x520c1997 symbol_id: 0x2d3715da symbol_id: 0x7ff45ec2 + symbol_id: 0x97ee5cc4 symbol_id: 0x0c805860 symbol_id: 0xc447be59 symbol_id: 0x1198e2f6 @@ -438691,6 +442982,7 @@ interface { symbol_id: 0xac417394 symbol_id: 0xd9184490 symbol_id: 0xe6ba30b7 + symbol_id: 0xa09651b4 symbol_id: 0x67de90c8 symbol_id: 0x5f6a1554 symbol_id: 0x3beebbde @@ -438816,6 +443108,7 @@ interface { symbol_id: 0x5e358f2a symbol_id: 0xa36916c0 symbol_id: 0xd2d09a22 + symbol_id: 0x1fa93c73 symbol_id: 0x76783663 symbol_id: 0xeae629ca symbol_id: 0x012046d8 @@ -438868,6 +443161,8 @@ interface { symbol_id: 0x2a322266 symbol_id: 0x8c705d8e symbol_id: 0x57e1cd01 + symbol_id: 0x3a816bcd + symbol_id: 0xb0f09bc0 symbol_id: 0x35eb35db symbol_id: 0x120ecf6c symbol_id: 0xabd39b51 @@ -438996,6 +443291,7 @@ interface { symbol_id: 0xc4e53e95 symbol_id: 0xba1a33c8 symbol_id: 0xe2dc88d9 + symbol_id: 0x4d06ba53 symbol_id: 0x3c85cae0 symbol_id: 0x4ce62869 symbol_id: 0xd7e100b7 @@ -439005,6 +443301,7 @@ interface { symbol_id: 0xb86aeda2 symbol_id: 0x91718d34 symbol_id: 0x65e5fa26 + symbol_id: 0x61d03c8a symbol_id: 0xb5759473 symbol_id: 0x2a71a0e1 symbol_id: 0x4a8d54b6 @@ -439240,8 +443537,10 @@ interface { symbol_id: 0x28b04407 symbol_id: 0x474c5b56 symbol_id: 0xb1a9cc86 + symbol_id: 0x3a3b1479 symbol_id: 0x9b056e94 symbol_id: 0x2061f0ca + symbol_id: 0x92dc6c49 symbol_id: 0xd878ab56 symbol_id: 0x4c5ac0a5 symbol_id: 0xd53a5822 @@ -439374,6 +443673,7 @@ interface { symbol_id: 0x66e2e005 symbol_id: 0x04f5a6e5 symbol_id: 0x53946e0e + symbol_id: 0xd9d103aa symbol_id: 0x52864a7e symbol_id: 0x03fd2deb symbol_id: 0x8d5ab7b4 @@ -439476,6 +443776,7 @@ interface { symbol_id: 0xdfeb23ad symbol_id: 0xe5fe5ba3 symbol_id: 0x924ff6bb + symbol_id: 0x5647272b symbol_id: 0x6ef38ba9 symbol_id: 0xbd3b590c symbol_id: 0xa7986ac9 @@ -439491,8 +443792,10 @@ interface { symbol_id: 0x489dfd09 symbol_id: 0x4059d2bb symbol_id: 0x22bf9346 + symbol_id: 0xb411a3b0 symbol_id: 0x56a85a70 symbol_id: 0x15de010f + symbol_id: 0xfb909691 symbol_id: 0xdb13d287 symbol_id: 0xd301890a symbol_id: 0x87f90d23 @@ -439574,6 +443877,7 @@ interface { symbol_id: 0x799c3ec1 symbol_id: 0x34ba5507 symbol_id: 0x13972c3d + symbol_id: 0x6ff644b0 symbol_id: 0x2ad163ea symbol_id: 0x02599b70 symbol_id: 0x21c2b40a @@ -439669,6 +443973,7 @@ interface { symbol_id: 0x4cc1057f symbol_id: 0xf569682d symbol_id: 0x8837a38c + symbol_id: 0xcd6f8e13 symbol_id: 0x0d9ec99a symbol_id: 0x159517e6 symbol_id: 0x08be3ee4 @@ -439779,6 +444084,7 @@ interface { symbol_id: 0xce4b7f5f symbol_id: 0x3600a769 symbol_id: 0x7880f04e + symbol_id: 0xd4544d3c symbol_id: 0x36be1a91 symbol_id: 0x46d775bd symbol_id: 0xeeb338c3 @@ -439834,6 +444140,7 @@ interface { symbol_id: 0x7ae5eeb4 symbol_id: 0x317870a4 symbol_id: 0xf34ac6e1 + symbol_id: 0x30dd6796 symbol_id: 0x573e2956 symbol_id: 0xe5bfa8c8 symbol_id: 0xd48a6769 @@ -439998,6 +444305,7 @@ interface { symbol_id: 0xac16424d symbol_id: 0x6bebb14a symbol_id: 0x9c1c9dfa + symbol_id: 0xfa43ef2b symbol_id: 0xb50c088c symbol_id: 0x28bb5738 symbol_id: 0x718fec30 @@ -440216,6 +444524,7 @@ interface { symbol_id: 0x3d89aa92 symbol_id: 0xd5cd8e9b symbol_id: 0x903baf1e + symbol_id: 0xd1f29cd4 symbol_id: 0x2396487e symbol_id: 0xf7be00e0 symbol_id: 0x9ff39d6a @@ -440224,6 +444533,7 @@ interface { symbol_id: 0x99beb7c8 symbol_id: 0x0ba86fe5 symbol_id: 0x90dbe313 + symbol_id: 0x2027d999 symbol_id: 0x9d83d0f8 symbol_id: 0x11744540 symbol_id: 0x4b6be7d2 @@ -440338,6 +444648,7 @@ interface { symbol_id: 0xd4a7d3d3 symbol_id: 0x51184dc0 symbol_id: 0xf5b01d8e + symbol_id: 0x465f41c2 symbol_id: 0xbe3f1757 symbol_id: 0xb2afee20 symbol_id: 0x58b810bd @@ -440490,6 +444801,7 @@ interface { symbol_id: 0x383fe893 symbol_id: 0xa9bf10bc symbol_id: 0xc70188be + symbol_id: 0xe09fd784 symbol_id: 0xded28924 symbol_id: 0xcb93fb69 symbol_id: 0xe3421d56 @@ -440668,9 +444980,11 @@ interface { symbol_id: 0x3c7c2553 symbol_id: 0x06c58be7 symbol_id: 0xab55569c + symbol_id: 0x1ac8aa52 symbol_id: 0xf83588d6 symbol_id: 0xa1c5bd8d symbol_id: 0x159a69a3 + symbol_id: 0xd2e101fd symbol_id: 0xcef0ca54 symbol_id: 0x39840ab2 symbol_id: 0xc05a6c7d @@ -440681,6 +444995,7 @@ interface { symbol_id: 0xa8ec4a05 symbol_id: 0x51f2e056 symbol_id: 0xc857b1c3 + symbol_id: 0xf20cee49 symbol_id: 0xcb14b5cc symbol_id: 0xa5d0148f symbol_id: 0x1721c938 @@ -440701,6 +445016,7 @@ interface { symbol_id: 0xb511ce50 symbol_id: 0xef2d8e08 symbol_id: 0x9b815cdc + symbol_id: 0x25649749 symbol_id: 0x998ad938 symbol_id: 0x27e49ce0 symbol_id: 0xa0e729a5 @@ -440710,6 +445026,7 @@ interface { symbol_id: 0x31dfc6e3 symbol_id: 0x0f402ef5 symbol_id: 0x7a974ed0 + symbol_id: 0x2da69553 symbol_id: 0x2ed5588c symbol_id: 0x3918f832 symbol_id: 0x238d82fe @@ -440906,6 +445223,7 @@ interface { symbol_id: 0x7aac3cd4 symbol_id: 0x3a06dd48 symbol_id: 0x75d8e345 + symbol_id: 0xea4eb631 symbol_id: 0x35d05a7d symbol_id: 0x02327717 symbol_id: 0x6d155c9e @@ -441077,6 +445395,7 @@ interface { symbol_id: 0x2ffc7c7e symbol_id: 0x1706be22 symbol_id: 0x4c3911f0 + symbol_id: 0x74041f0e symbol_id: 0x8717f26f symbol_id: 0x361004c8 symbol_id: 0xcf5ea9a2 @@ -441086,10 +445405,15 @@ interface { symbol_id: 0x52d444b1 symbol_id: 0x7d0e44ca symbol_id: 0x10de460f + symbol_id: 0xbfab1179 symbol_id: 0x0eb11e95 symbol_id: 0x6ca4d0ed + symbol_id: 0x88241dc1 + symbol_id: 0x741d5af7 symbol_id: 0x6e37f09d + symbol_id: 0x29d6d842 symbol_id: 0x334ff68f + symbol_id: 0xd12f6730 symbol_id: 0xd7ecf501 symbol_id: 0xa5f8942d symbol_id: 0x552bcd06 @@ -441321,6 +445645,8 @@ interface { symbol_id: 0x68666cb6 symbol_id: 0xf4ae7c22 symbol_id: 0x4ff0f410 + symbol_id: 0xb2c3f64e + symbol_id: 0x29a1f6c6 symbol_id: 0xab642653 symbol_id: 0x1bb1212e symbol_id: 0x90d2e1dd @@ -441330,6 +445656,7 @@ interface { symbol_id: 0x2336c16f symbol_id: 0x5b69d05f symbol_id: 0x5a86c89e + symbol_id: 0x594eb27d symbol_id: 0x1cfb5d03 symbol_id: 0x60efc3a4 symbol_id: 0xc802de31 @@ -441386,6 +445713,7 @@ interface { symbol_id: 0xe80863b1 symbol_id: 0xd81a7d03 symbol_id: 0x0566bca1 + symbol_id: 0xc578c7af symbol_id: 0x4c05b91e symbol_id: 0xa5459730 symbol_id: 0xe77a1ae2 @@ -441437,6 +445765,8 @@ interface { symbol_id: 0x1c2c7614 symbol_id: 0xc559647d symbol_id: 0x38ac7923 + symbol_id: 0x20af6f53 + symbol_id: 0x3f2fa978 symbol_id: 0x6c6beb07 symbol_id: 0x1be0f35f symbol_id: 0xd9fe9b1d @@ -441490,6 +445820,7 @@ interface { symbol_id: 0x857cda02 symbol_id: 0x135db1d2 symbol_id: 0xba41ecd4 + symbol_id: 0x3fa49947 symbol_id: 0x327b1cbb symbol_id: 0x61a826d2 symbol_id: 0x3cc69c51 @@ -441560,6 +445891,7 @@ interface { symbol_id: 0x6bcfa61d symbol_id: 0x01438401 symbol_id: 0x9175ce1c + symbol_id: 0xaccdc826 symbol_id: 0x81dadb36 symbol_id: 0x9bfc3a5e symbol_id: 0xc750a072 @@ -441571,6 +445903,7 @@ interface { symbol_id: 0x8cd9004f symbol_id: 0x21d6616d symbol_id: 0xd9f8da7b + symbol_id: 0x27bb5e77 symbol_id: 0x1b2f5f11 symbol_id: 0x6746106c symbol_id: 0xc5f22a59 @@ -441823,11 +446156,13 @@ interface { symbol_id: 0x06d93a2c symbol_id: 0x3c2ed3a0 symbol_id: 0x7d6628ba + symbol_id: 0x1128cd26 symbol_id: 0x493ce9fc symbol_id: 0x786bc3fa symbol_id: 0x9a75fb33 symbol_id: 0xb78a4a87 symbol_id: 0xc84a133c + symbol_id: 0xdda6583e symbol_id: 0x7bd17841 symbol_id: 0xf4530070 symbol_id: 0xaea3dd01 @@ -441959,7 +446294,9 @@ interface { symbol_id: 0x1c9ded63 symbol_id: 0x89c6398f symbol_id: 0xbad16ab3 + symbol_id: 0x68c3a63b symbol_id: 0x399512e2 + symbol_id: 0xbc90165b symbol_id: 0x69f7fc4b symbol_id: 0x1579ecd2 symbol_id: 0xe4572a81 @@ -442056,6 +446393,7 @@ interface { symbol_id: 0x0399dc23 symbol_id: 0x10fd9f55 symbol_id: 0xd0f0d35f + symbol_id: 0xfd26acf1 symbol_id: 0xfb31b7b7 symbol_id: 0xfe0604fe symbol_id: 0x0654cb12 @@ -442198,6 +446536,8 @@ interface { symbol_id: 0x82e37620 symbol_id: 0x8b1dfb41 symbol_id: 0x43078d96 + symbol_id: 0x7d136425 + symbol_id: 0xfe90bb40 symbol_id: 0x840dad92 symbol_id: 0x619db28f symbol_id: 0x6aeef72a @@ -442231,6 +446571,7 @@ interface { symbol_id: 0xfab02ca8 symbol_id: 0xd7668767 symbol_id: 0x50d8c489 + symbol_id: 0x03adeb35 symbol_id: 0x77722cfe symbol_id: 0x58765106 symbol_id: 0x14f0cfc8 @@ -442520,6 +446861,7 @@ interface { symbol_id: 0x3852868c symbol_id: 0x27f20808 symbol_id: 0x63876663 + symbol_id: 0x780d1110 symbol_id: 0xbbb674cc symbol_id: 0xdea420f5 symbol_id: 0x08190210 @@ -442532,6 +446874,7 @@ interface { symbol_id: 0x9ac8ef20 symbol_id: 0x35c96922 symbol_id: 0xbe6406c3 + symbol_id: 0x64fb0865 symbol_id: 0x9c6c58ea symbol_id: 0x2fefe933 symbol_id: 0x1c994923 @@ -442600,6 +446943,7 @@ interface { symbol_id: 0x8897c24a symbol_id: 0xb69b7010 symbol_id: 0xc0468e7f + symbol_id: 0x912e005a symbol_id: 0xffa3ecd1 symbol_id: 0x42595f98 symbol_id: 0xd085753f @@ -442673,6 +447017,7 @@ interface { symbol_id: 0x0dfe21ad symbol_id: 0xa1f2194b symbol_id: 0x8c404a01 + symbol_id: 0x7de90fab symbol_id: 0x30e87de6 symbol_id: 0x6fc1e83f symbol_id: 0x0e1df21c @@ -442762,6 +447107,7 @@ interface { symbol_id: 0x0c0b5c37 symbol_id: 0x67b50d72 symbol_id: 0xe1f384d2 + symbol_id: 0x13474ea8 symbol_id: 0x2f08ffc6 symbol_id: 0xbe1a0b7f symbol_id: 0x6fa67e06 @@ -442829,6 +447175,7 @@ interface { symbol_id: 0xe8ed13a6 symbol_id: 0x2c91b5be symbol_id: 0xf8b12136 + symbol_id: 0x072cf683 symbol_id: 0xb11c92aa symbol_id: 0xaa322a74 symbol_id: 0x0d9f5d07 @@ -442841,16 +447188,22 @@ interface { symbol_id: 0xdea8ab93 symbol_id: 0x55fd0d76 symbol_id: 0x25e7db1a + symbol_id: 0x2c4cd39b + symbol_id: 0xd734aca0 symbol_id: 0xb634bd54 symbol_id: 0x5b13a53d symbol_id: 0xaba73bf2 symbol_id: 0xa2a45fe2 symbol_id: 0xe8ce7aa9 symbol_id: 0xe1b4680c + symbol_id: 0xfa3ae6ff + symbol_id: 0xd43b4f66 + symbol_id: 0xfb61e52a symbol_id: 0xe243fac9 symbol_id: 0xf140b7ff symbol_id: 0x2e17e257 symbol_id: 0xb89cc5d0 + symbol_id: 0xcae30dd3 symbol_id: 0x92ae3cd2 symbol_id: 0x8dbf40f1 symbol_id: 0x80c5af01 @@ -442905,6 +447258,7 @@ interface { symbol_id: 0x4804f611 symbol_id: 0x842fc505 symbol_id: 0xd28bb85a + symbol_id: 0x778ba508 symbol_id: 0x26c5a1b9 symbol_id: 0xdb34fc88 symbol_id: 0xb0eab20a @@ -443080,6 +447434,7 @@ interface { symbol_id: 0x42f9d663 symbol_id: 0x5b227eda symbol_id: 0x5c3d4ca3 + symbol_id: 0x94f761df symbol_id: 0xfdbe6bc0 symbol_id: 0x04f125b0 symbol_id: 0x68b1776b @@ -443253,8 +447608,10 @@ interface { symbol_id: 0xb15014ad symbol_id: 0x089f5fe6 symbol_id: 0x0c73acc8 + symbol_id: 0x3fb5c5d4 symbol_id: 0xace4bcdf symbol_id: 0x57ee69c1 + symbol_id: 0x1a32d2b3 symbol_id: 0xd76b82b2 symbol_id: 0x30c7b7f4 symbol_id: 0x9c7a2d6c @@ -443395,6 +447752,7 @@ interface { symbol_id: 0x12fb2fd6 symbol_id: 0xac6ce0b0 symbol_id: 0xe224f61e + symbol_id: 0x36d6d8fb symbol_id: 0xd2090791 symbol_id: 0xedd0d1ed symbol_id: 0xda5d60be @@ -443441,6 +447799,7 @@ interface { symbol_id: 0xb77321e1 symbol_id: 0x8ef5c221 symbol_id: 0x32b196e0 + symbol_id: 0xfaba855b symbol_id: 0x022517f0 symbol_id: 0x4d8a452e symbol_id: 0x7509c737 @@ -443566,8 +447925,10 @@ interface { symbol_id: 0x39477e15 symbol_id: 0x05a66e98 symbol_id: 0x3c71822d + symbol_id: 0x3a629e14 symbol_id: 0x6e10178a symbol_id: 0x5d6a93ff + symbol_id: 0x826ffe72 symbol_id: 0x0746da0a symbol_id: 0x98b882af symbol_id: 0x45ffec2b @@ -443669,6 +448030,7 @@ interface { symbol_id: 0xa07fb91d symbol_id: 0x40c45272 symbol_id: 0xbdcb1d7d + symbol_id: 0xb8a763fb symbol_id: 0x1b5d526c symbol_id: 0x30b604c4 symbol_id: 0x341fb250 @@ -443945,6 +448307,7 @@ interface { symbol_id: 0x7918ef41 symbol_id: 0x97843792 symbol_id: 0x54622a57 + symbol_id: 0x4086fab0 symbol_id: 0xa64c7fe5 symbol_id: 0x5eb2e502 symbol_id: 0x33a917a0 @@ -444324,11 +448687,13 @@ interface { symbol_id: 0x591431f1 symbol_id: 0x6869c83d symbol_id: 0xe54ea1f3 + symbol_id: 0x4a1dceb2 symbol_id: 0xa2565005 symbol_id: 0xfc2dbec4 symbol_id: 0x08907db0 symbol_id: 0x81a07067 symbol_id: 0xb47ee8bd + symbol_id: 0x3c42a066 symbol_id: 0x07f159e7 symbol_id: 0x94af2b02 symbol_id: 0xa7d44351 @@ -444348,6 +448713,7 @@ interface { symbol_id: 0x8c0c76a2 symbol_id: 0xc3d9d164 symbol_id: 0x91df6de8 + symbol_id: 0x6bef1b0f symbol_id: 0x96242237 symbol_id: 0x571b278c symbol_id: 0x54b08d37 @@ -444393,6 +448759,7 @@ interface { symbol_id: 0x86b2ecdf symbol_id: 0xa274099f symbol_id: 0x00fb9c6d + symbol_id: 0x476977df symbol_id: 0x0e209244 symbol_id: 0xe8ad61b2 symbol_id: 0x4f11a926 @@ -444483,6 +448850,7 @@ interface { symbol_id: 0x677985f3 symbol_id: 0x6c2259cd symbol_id: 0xacf009d6 + symbol_id: 0x3ffae0cf symbol_id: 0x0b4dd20d symbol_id: 0xae3ac3f6 symbol_id: 0xa7c71d5a @@ -444499,8 +448867,10 @@ interface { symbol_id: 0xbf8f4498 symbol_id: 0x0b050f75 symbol_id: 0x8a712bc6 + symbol_id: 0xb626741e symbol_id: 0x2a899db0 symbol_id: 0x954d57e8 + symbol_id: 0x568d2ee4 symbol_id: 0x604f0f0b symbol_id: 0x5b997ef3 symbol_id: 0x6a405f9b @@ -444508,6 +448878,7 @@ interface { symbol_id: 0x4e9dfcab symbol_id: 0x4a92dfd1 symbol_id: 0xb958a87a + symbol_id: 0xd4a176ac symbol_id: 0xdcd44915 symbol_id: 0xca491b5d symbol_id: 0x1f87b736 @@ -444917,6 +449288,32 @@ interface { symbol_id: 0x3bf4fac5 symbol_id: 0xcac8f190 symbol_id: 0x140d9164 + symbol_id: 0x8189acd0 + symbol_id: 0xbbf007f4 + symbol_id: 0xd28057f3 + symbol_id: 0x580c56b8 + symbol_id: 0xe395160a + symbol_id: 0x8f0c866d + symbol_id: 0x52816b1c + symbol_id: 0x4f4e9353 + symbol_id: 0x3f51a906 + symbol_id: 0x6515c416 + symbol_id: 0x6c854cb3 + symbol_id: 0xcd021e80 + symbol_id: 0x147e4ad2 + symbol_id: 0x167360fc + symbol_id: 0x69b1070f + symbol_id: 0x3be16591 + symbol_id: 0x6e3c0f91 + symbol_id: 0x99cb2edf + symbol_id: 0x2325cd65 + symbol_id: 0x9cf79f8d + symbol_id: 0x2f365f93 + symbol_id: 0x6229bd38 + symbol_id: 0x90b81315 + symbol_id: 0xb011e423 + symbol_id: 0x27363bb6 + symbol_id: 0x10932377 symbol_id: 0x37ac0936 symbol_id: 0xa3900874 symbol_id: 0x4cfd694f @@ -444925,6 +449322,7 @@ interface { symbol_id: 0x1473720a symbol_id: 0xa459c02b symbol_id: 0xd2d0fdc4 + symbol_id: 0x97509378 symbol_id: 0x4b0dcba7 symbol_id: 0xa040d5a9 symbol_id: 0xb0c1eaf9 @@ -444989,6 +449387,7 @@ interface { symbol_id: 0x592a4678 symbol_id: 0xff77e1ca symbol_id: 0xe66642fe + symbol_id: 0xc28778a5 symbol_id: 0x538ad5cc symbol_id: 0x2244c8f0 symbol_id: 0xfb876b5c @@ -445455,6 +449854,7 @@ interface { symbol_id: 0xf158bd25 symbol_id: 0xee403ebb symbol_id: 0x52ed8956 + symbol_id: 0x84c3c252 symbol_id: 0xf6eda681 symbol_id: 0xd7745fba symbol_id: 0xd9789392 @@ -445472,8 +449872,20 @@ interface { symbol_id: 0x5303ad4a symbol_id: 0x9c1e0c55 symbol_id: 0x4ba0be41 + symbol_id: 0xd045d6af + symbol_id: 0xe1a54f6b + symbol_id: 0x04058f3f + symbol_id: 0x0576d81a + symbol_id: 0xb729831c symbol_id: 0xb2b54222 symbol_id: 0xb8699f23 + symbol_id: 0x6f74cced + symbol_id: 0xc282f08e + symbol_id: 0x27a93b2c + symbol_id: 0x0fea3e22 + symbol_id: 0xde450a6f + symbol_id: 0x4df06dae + symbol_id: 0xadaec705 symbol_id: 0x3542c796 symbol_id: 0xae02eaa9 symbol_id: 0x0d8da27c diff --git a/android/abi_gki_aarch64.stg.allowed_breaks b/android/abi_gki_aarch64.stg.allowed_breaks index 8f5f52dbe65af..95d2ab13dc3de 100644 --- a/android/abi_gki_aarch64.stg.allowed_breaks +++ b/android/abi_gki_aarch64.stg.allowed_breaks @@ -213,3 +213,65 @@ type 'struct scm_stat' changed type 'struct scm_fp_list' changed member 'bool dead' was added +type 'struct kvm_iommu_ops' changed + member 'u64 android_kabi_reserved1' was removed + member 'union { int(* iotlb_sync_map)(struct kvm_hyp_iommu_domain*, unsigned long, size_t); struct { u64 android_kabi_reserved1; }; union { }; }' was added + +type 'struct prefix_info' changed + member changed from 'union { __u8 flags; struct { __u8 reserved:6; __u8 autoconf:1; __u8 onlink:1; }; }' to 'union { __u8 flags; struct { __u8 reserved:4; __u8 preferpd:1; __u8 routeraddr:1; __u8 autoconf:1; __u8 onlink:1; }; }' + type changed from 'union { __u8 flags; struct { __u8 reserved:6; __u8 autoconf:1; __u8 onlink:1; }; }' to 'union { __u8 flags; struct { __u8 reserved:4; __u8 preferpd:1; __u8 routeraddr:1; __u8 autoconf:1; __u8 onlink:1; }; }' + member changed from 'struct { __u8 reserved:6; __u8 autoconf:1; __u8 onlink:1; }' to 'struct { __u8 reserved:4; __u8 preferpd:1; __u8 routeraddr:1; __u8 autoconf:1; __u8 onlink:1; }' + type changed from 'struct { __u8 reserved:6; __u8 autoconf:1; __u8 onlink:1; }' to 'struct { __u8 reserved:4; __u8 preferpd:1; __u8 routeraddr:1; __u8 autoconf:1; __u8 onlink:1; }' + member changed from '__u8 reserved:6' to '__u8 reserved:4' + bit-field size changed from 6 to 4 + member '__u8 preferpd:1' was added + member '__u8 routeraddr:1' was added + +type 'struct ipv6_devconf' changed + member 'u64 android_kabi_reserved4' was removed + member 'union { struct { __u8 ra_honor_pio_pflag; __u8 padding4[7]; }; struct { u64 android_kabi_reserved4; }; union { }; }' was added + +type 'struct task_struct' changed + member 'u64 android_kabi_reserved1' was removed + member 'union { struct task_dma_buf_info* dmabuf_info; struct { u64 android_kabi_reserved1; }; union { }; }' was added + +type 'struct dma_buf' changed + member 'u64 android_kabi_reserved1' was removed + member 'union { atomic64_t nr_task_refs; struct { u64 android_kabi_reserved1; }; union { }; }' was added + +type 'struct xt_match' changed + member 'bool has_compat_metadata' was added + +type 'struct xt_target' changed + member 'bool has_compat_metadata' was added + +type 'struct drm_framebuffer' changed + member 'unsigned int internal_flags' was added + +type 'struct cgroup' changed + member 'u64 android_backport_reserved1' was removed + member 'union { struct cgroup_kmi_ext_info* kmi_ext_info; struct { u64 android_backport_reserved1; }; union { }; }' was added + +type 'struct io_ring_ctx' changed + member changed from 'struct { struct llist_head work_llist; unsigned long check_cq; atomic_t cq_wait_nr; atomic_t cq_timeouts; struct wait_queue_head cq_wait; }' to 'struct { struct llist_head work_llist; unsigned long check_cq; atomic_t cq_wait_nr; atomic_t cq_timeouts; struct wait_queue_head cq_wait; struct llist_head retry_llist; }' + type changed from 'struct { struct llist_head work_llist; unsigned long check_cq; atomic_t cq_wait_nr; atomic_t cq_timeouts; struct wait_queue_head cq_wait; }' to 'struct { struct llist_head work_llist; unsigned long check_cq; atomic_t cq_wait_nr; atomic_t cq_timeouts; struct wait_queue_head cq_wait; struct llist_head retry_llist; }' + member 'struct llist_head retry_llist' was added + +type 'enum bpf_reg_type' changed + enumerator '__BPF_REG_TYPE_LIMIT' value changed from 33554431 to 67108863 + +type 'enum bpf_return_type' changed + enumerator '__BPF_RET_TYPE_LIMIT' value changed from 33554431 to 67108863 + +type 'enum bpf_arg_type' changed + enumerator '__BPF_ARG_TYPE_LIMIT' value changed from 33554431 to 67108863 + +type 'struct mm_struct' changed + member changed from 'struct { struct { atomic_t mm_count; }; struct maple_tree mm_mt; unsigned long(* get_unmapped_area)(struct file*, unsigned long, unsigned long, unsigned long, unsigned long); unsigned long mmap_base; unsigned long mmap_legacy_base; unsigned long task_size; pgd_t* pgd; atomic_t membarrier_state; atomic_t mm_users; atomic_long_t pgtables_bytes; int map_count; spinlock_t page_table_lock; struct rw_semaphore mmap_lock; struct list_head mmlist; int mm_lock_seq; unsigned long hiwater_rss; unsigned long hiwater_vm; unsigned long total_vm; unsigned long locked_vm; atomic64_t pinned_vm; unsigned long data_vm; unsigned long exec_vm; unsigned long stack_vm; unsigned long def_flags; seqcount_t write_protect_seq; spinlock_t arg_lock; unsigned long start_code; unsigned long end_code; unsigned long start_data; unsigned long end_data; unsigned long start_brk; unsigned long brk; unsigned long start_stack; unsigned long arg_start; unsigned long arg_end; unsigned long env_start; unsigned long env_end; unsigned long saved_auxv[50]; struct percpu_counter rss_stat[4]; struct linux_binfmt* binfmt; mm_context_t context; unsigned long flags; spinlock_t ioctx_lock; struct kioctx_table* ioctx_table; struct task_struct* owner; struct user_namespace* user_ns; struct file* exe_file; struct mmu_notifier_subscriptions* notifier_subscriptions; atomic_t tlb_flush_pending; atomic_t tlb_flush_batched; struct uprobes_state uprobes_state; struct work_struct async_put_work; struct { struct list_head list; unsigned long bitmap; struct mem_cgroup* memcg; } lru_gen; u64 android_kabi_reserved1; u64 android_backport_reserved1; }' to 'struct { struct { atomic_t mm_count; }; struct maple_tree mm_mt; unsigned long(* get_unmapped_area)(struct file*, unsigned long, unsigned long, unsigned long, unsigned long); unsigned long mmap_base; unsigned long mmap_legacy_base; unsigned long task_size; pgd_t* pgd; atomic_t membarrier_state; atomic_t mm_users; atomic_long_t pgtables_bytes; int map_count; spinlock_t page_table_lock; struct rw_semaphore mmap_lock; struct list_head mmlist; int mm_lock_seq; unsigned long hiwater_rss; unsigned long hiwater_vm; unsigned long total_vm; unsigned long locked_vm; atomic64_t pinned_vm; unsigned long data_vm; unsigned long exec_vm; unsigned long stack_vm; unsigned long def_flags; seqcount_t write_protect_seq; spinlock_t arg_lock; unsigned long start_code; unsigned long end_code; unsigned long start_data; unsigned long end_data; unsigned long start_brk; unsigned long brk; unsigned long start_stack; unsigned long arg_start; unsigned long arg_end; unsigned long env_start; unsigned long env_end; unsigned long saved_auxv[50]; struct percpu_counter rss_stat[4]; struct linux_binfmt* binfmt; mm_context_t context; unsigned long flags; spinlock_t ioctx_lock; struct kioctx_table* ioctx_table; struct task_struct* owner; struct user_namespace* user_ns; struct file* exe_file; struct mmu_notifier_subscriptions* notifier_subscriptions; atomic_t tlb_flush_pending; atomic_t tlb_flush_batched; struct uprobes_state uprobes_state; struct work_struct async_put_work; struct { struct list_head list; unsigned long bitmap; struct mem_cgroup* memcg; } lru_gen; u64 android_kabi_reserved1; union { struct task_dma_buf_info* dmabuf_info; struct { u64 android_backport_reserved1; }; union { }; }; }' + type changed from 'struct { struct { atomic_t mm_count; }; struct maple_tree mm_mt; unsigned long(* get_unmapped_area)(struct file*, unsigned long, unsigned long, unsigned long, unsigned long); unsigned long mmap_base; unsigned long mmap_legacy_base; unsigned long task_size; pgd_t* pgd; atomic_t membarrier_state; atomic_t mm_users; atomic_long_t pgtables_bytes; int map_count; spinlock_t page_table_lock; struct rw_semaphore mmap_lock; struct list_head mmlist; int mm_lock_seq; unsigned long hiwater_rss; unsigned long hiwater_vm; unsigned long total_vm; unsigned long locked_vm; atomic64_t pinned_vm; unsigned long data_vm; unsigned long exec_vm; unsigned long stack_vm; unsigned long def_flags; seqcount_t write_protect_seq; spinlock_t arg_lock; unsigned long start_code; unsigned long end_code; unsigned long start_data; unsigned long end_data; unsigned long start_brk; unsigned long brk; unsigned long start_stack; unsigned long arg_start; unsigned long arg_end; unsigned long env_start; unsigned long env_end; unsigned long saved_auxv[50]; struct percpu_counter rss_stat[4]; struct linux_binfmt* binfmt; mm_context_t context; unsigned long flags; spinlock_t ioctx_lock; struct kioctx_table* ioctx_table; struct task_struct* owner; struct user_namespace* user_ns; struct file* exe_file; struct mmu_notifier_subscriptions* notifier_subscriptions; atomic_t tlb_flush_pending; atomic_t tlb_flush_batched; struct uprobes_state uprobes_state; struct work_struct async_put_work; struct { struct list_head list; unsigned long bitmap; struct mem_cgroup* memcg; } lru_gen; u64 android_kabi_reserved1; u64 android_backport_reserved1; }' to 'struct { struct { atomic_t mm_count; }; struct maple_tree mm_mt; unsigned long(* get_unmapped_area)(struct file*, unsigned long, unsigned long, unsigned long, unsigned long); unsigned long mmap_base; unsigned long mmap_legacy_base; unsigned long task_size; pgd_t* pgd; atomic_t membarrier_state; atomic_t mm_users; atomic_long_t pgtables_bytes; int map_count; spinlock_t page_table_lock; struct rw_semaphore mmap_lock; struct list_head mmlist; int mm_lock_seq; unsigned long hiwater_rss; unsigned long hiwater_vm; unsigned long total_vm; unsigned long locked_vm; atomic64_t pinned_vm; unsigned long data_vm; unsigned long exec_vm; unsigned long stack_vm; unsigned long def_flags; seqcount_t write_protect_seq; spinlock_t arg_lock; unsigned long start_code; unsigned long end_code; unsigned long start_data; unsigned long end_data; unsigned long start_brk; unsigned long brk; unsigned long start_stack; unsigned long arg_start; unsigned long arg_end; unsigned long env_start; unsigned long env_end; unsigned long saved_auxv[50]; struct percpu_counter rss_stat[4]; struct linux_binfmt* binfmt; mm_context_t context; unsigned long flags; spinlock_t ioctx_lock; struct kioctx_table* ioctx_table; struct task_struct* owner; struct user_namespace* user_ns; struct file* exe_file; struct mmu_notifier_subscriptions* notifier_subscriptions; atomic_t tlb_flush_pending; atomic_t tlb_flush_batched; struct uprobes_state uprobes_state; struct work_struct async_put_work; struct { struct list_head list; unsigned long bitmap; struct mem_cgroup* memcg; } lru_gen; u64 android_kabi_reserved1; union { struct task_dma_buf_info* dmabuf_info; struct { u64 android_backport_reserved1; }; union { }; }; }' + member 'u64 android_backport_reserved1' was removed + member 'union { struct task_dma_buf_info* dmabuf_info; struct { u64 android_backport_reserved1; }; union { }; }' was added + +type 'struct files_struct' changed + member 'struct task_dma_buf_info* dmabuf_info' was added + diff --git a/android/abi_gki_aarch64_amlogic b/android/abi_gki_aarch64_amlogic index ae6e6cb73da12..37206f356f11e 100644 --- a/android/abi_gki_aarch64_amlogic +++ b/android/abi_gki_aarch64_amlogic @@ -131,6 +131,7 @@ cec_notifier_set_phys_addr cec_notifier_set_phys_addr_from_edid __check_object_size + class_compat_register class_create class_create_file_ns class_destroy @@ -204,15 +205,19 @@ component_master_del component_match_add_release component_unbind_all + config_group_find_item + config_item_init_type_name console_printk __const_udelay consume_skb contig_page_data __contpte_try_unfold + _copy_from_iter copy_from_kernel_nofault __copy_overflow copy_page_from_iter_atomic copy_splice_read + _copy_to_iter cpu_all_bits cpu_bit_bitmap cpufreq_boost_enabled @@ -245,10 +250,13 @@ crypto_aead_setauthsize crypto_aead_setkey crypto_ahash_digest + crypto_ahash_final + crypto_ahash_finup crypto_ahash_setkey crypto_alloc_aead crypto_alloc_ahash crypto_alloc_base + crypto_alloc_rng crypto_alloc_shash crypto_alloc_skcipher crypto_cipher_encrypt_one @@ -258,13 +266,17 @@ crypto_dequeue_request crypto_destroy_tfm crypto_enqueue_request + crypto_get_default_null_skcipher crypto_has_alg crypto_init_queue __crypto_memneq + crypto_put_default_null_skcipher crypto_register_ahash crypto_register_alg crypto_register_shash crypto_register_skcipher + crypto_req_done + crypto_rng_reset crypto_sha1_finup crypto_sha1_update crypto_shash_digest @@ -334,6 +346,7 @@ device_del device_destroy device_for_each_child + device_for_each_child_reverse device_get_child_node_count device_get_match_data device_get_next_child_node @@ -358,6 +371,7 @@ dev_kfree_skb_any_reason __devm_add_action devm_alloc_etherdev_mqs + devm_backlight_device_register devm_blk_crypto_profile_init devm_clk_bulk_get devm_clk_bulk_get_all @@ -381,6 +395,7 @@ devm_hwrng_register devm_iio_channel_get devm_iio_device_alloc + devm_iio_trigger_register devm_input_allocate_device devm_ioremap devm_ioremap_resource @@ -435,6 +450,7 @@ _dev_notice dev_open dev_pm_clear_wake_irq + dev_pm_domain_attach dev_pm_domain_attach_by_id dev_pm_domain_attach_by_name dev_pm_domain_detach @@ -564,6 +580,7 @@ dput dql_completed dql_reset + driver_register driver_unregister drm_add_modes_noedid drm_atomic_add_affected_connectors @@ -573,6 +590,7 @@ drm_atomic_get_new_bridge_state drm_atomic_get_new_connector_for_encoder drm_atomic_get_new_private_obj_state + drm_atomic_get_old_private_obj_state drm_atomic_get_plane_state drm_atomic_get_private_obj_state drm_atomic_helper_async_commit @@ -615,6 +633,7 @@ drm_atomic_helper_wait_for_dependencies drm_atomic_helper_wait_for_fences drm_atomic_helper_wait_for_vblanks + drm_atomic_private_obj_fini drm_atomic_private_obj_init drm_atomic_set_crtc_for_connector drm_atomic_set_crtc_for_plane @@ -623,6 +642,7 @@ drm_atomic_set_mode_prop_for_crtc drm_atomic_state_alloc drm_atomic_state_clear + drm_atomic_state_default_release __drm_atomic_state_free drm_compat_ioctl drm_connector_attach_content_type_property @@ -634,6 +654,7 @@ drm_connector_list_iter_begin drm_connector_list_iter_end drm_connector_list_iter_next + drm_connector_register drm_connector_set_vrr_capable_property drm_connector_unregister drm_connector_update_edid_property @@ -642,6 +663,7 @@ drm_crtc_arm_vblank_event drm_crtc_cleanup __drm_crtc_commit_free + drm_crtc_commit_wait drm_crtc_enable_color_mgmt drm_crtc_from_index drm_crtc_handle_vblank @@ -653,13 +675,23 @@ drm_crtc_vblank_off drm_crtc_vblank_on drm_crtc_vblank_put + drm_crtc_wait_one_vblank ___drm_dbg + __drm_debug drm_debugfs_create_files drm_dev_alloc __drm_dev_dbg + drm_dev_printk drm_dev_put drm_dev_register drm_dev_unregister + drm_display_mode_from_cea_vic + drm_edid_dup + drm_edid_duplicate + drm_edid_free + drm_edid_get_monitor_name + drm_edid_raw + drm_edid_read_ddc drm_encoder_cleanup drm_encoder_init __drm_err @@ -707,6 +739,7 @@ drm_mode_duplicate drm_mode_get_hv_timing drm_mode_object_find + drm_mode_object_get drm_mode_object_put drm_mode_probed_add drm_modeset_acquire_fini @@ -717,11 +750,16 @@ drm_modeset_lock drm_modeset_lock_all drm_modeset_lock_all_ctx + drm_modeset_lock_single_interruptible drm_modeset_unlock drm_modeset_unlock_all drm_mode_vrefresh drm_object_attach_property + drm_object_property_set_value drm_open + drm_panel_add + drm_panel_init + drm_panel_of_backlight drm_plane_cleanup drm_plane_create_alpha_property drm_plane_create_blend_mode_property @@ -793,6 +831,7 @@ extcon_set_state extcon_set_state_sync extcon_unregister_notifier + extract_iter_to_sg fasync_helper fault_in_iov_iter_readable __fdget @@ -903,6 +942,7 @@ __genphy_config_aneg genphy_handle_interrupt_no_ack genphy_read_abilities + genphy_read_lpa genphy_read_mmd_unsupported genphy_read_status genphy_restart_aneg @@ -980,6 +1020,8 @@ gpio_request gpio_to_desc grab_cache_page_write_begin + guid_gen + guid_null handle_fasteoi_irq handle_simple_irq handle_sysrq @@ -1082,6 +1124,7 @@ input_event input_free_device input_register_device + input_set_abs_params input_set_capability input_unregister_device insert_inode_locked @@ -1102,8 +1145,10 @@ ioremap_prot io_schedule iounmap + iov_iter_advance iov_iter_alignment iov_iter_init + iov_iter_npages iov_iter_revert iov_iter_zero iput @@ -1269,12 +1314,14 @@ __local_bh_enable_ip __lock_buffer lockref_get + lock_sock_nested logfc log_post_read_mmio log_post_write_mmio log_read_mmio log_write_mmio lookup_bdev + lookup_user_key loops_per_jiffy LZ4_decompress_safe LZ4_decompress_safe_partial @@ -1352,6 +1399,7 @@ mmc_release_host mmc_remove_host mmc_request_done + mmc_send_abort_tuning mmc_send_tuning mmc_set_data_timeout mmc_switch @@ -1458,6 +1506,8 @@ of_device_get_match_data of_device_is_available of_device_is_compatible + of_device_modalias + of_device_uevent_modalias of_dma_configure_id of_dma_is_coherent of_find_compatible_node @@ -1512,6 +1562,7 @@ of_prop_next_string of_prop_next_u32 of_pwm_xlate_with_flags + of_remove_property of_reserved_mem_device_init_by_idx of_reserved_mem_device_init_by_name of_reserved_mem_device_release @@ -1726,6 +1777,8 @@ proc_mkdir proc_mkdir_data proc_remove + proto_register + proto_unregister __pskb_copy_fclone pskb_expand_head __pskb_pull_tail @@ -1829,6 +1882,7 @@ regmap_multi_reg_write regmap_raw_write regmap_read + regmap_test_bits regmap_update_bits_base regmap_write regulator_bulk_disable @@ -1845,6 +1899,7 @@ release_firmware __release_region release_resource + release_sock remap_pfn_range remap_vmalloc_range remove_cpu @@ -1884,6 +1939,7 @@ rps_cpu_mask rps_sock_flow_table rtc_add_group + rtc_read_alarm rtc_time64_to_tm rtc_tm_to_time64 rtc_update_irq @@ -1940,6 +1996,8 @@ sdio_writel sdio_writesb sdio_writew + security_sk_clone + security_sock_graft send_sig seq_list_next seq_list_start @@ -2000,6 +2058,7 @@ single_open_size single_release si_swapinfo + sk_alloc skb_add_rx_frag skb_checksum_help skb_clone @@ -2026,6 +2085,7 @@ skb_scrub_packet skb_trim skb_tstamp_tx + sk_free skip_spaces smpboot_register_percpu_thread smp_call_function @@ -2046,6 +2106,7 @@ snd_pcm_lib_preallocate_pages snd_pcm_period_elapsed snd_pcm_rate_to_rate_bit + snd_pcm_set_managed_buffer_all snd_pcm_stop snd_pcm_stop_xrun _snd_pcm_stream_lock_irqsave @@ -2068,6 +2129,7 @@ snd_soc_dai_set_tdm_slot snd_soc_dapm_get_enum_double snd_soc_dapm_put_enum_double + snd_soc_get_dai_name snd_soc_get_volsw snd_soc_get_volsw_range snd_soc_info_enum_double @@ -2082,6 +2144,7 @@ snd_soc_of_parse_audio_simple_widgets snd_soc_of_parse_card_name snd_soc_of_parse_tdm_slot + snd_soc_of_put_dai_link_codecs snd_soc_pm_ops snd_soc_put_volsw snd_soc_put_volsw_range @@ -2090,7 +2153,25 @@ snd_soc_unregister_component snprintf __sock_create + sock_init_data + sock_kfree_s + sock_kmalloc + sock_kzfree_s + sock_no_accept + sock_no_bind + sock_no_connect + sock_no_getname + sock_no_ioctl + sock_no_listen + sock_no_mmap + sock_no_recvmsg + sock_no_sendmsg + sock_no_shutdown + sock_no_socketpair + sock_register sock_release + sock_unregister + sock_wake_async sock_wfree sort spi_add_device @@ -2172,6 +2253,7 @@ sysfs_create_file_ns sysfs_create_files sysfs_create_group + sysfs_create_groups sysfs_create_link sysfs_emit __sysfs_match_string @@ -2200,6 +2282,7 @@ thermal_cooling_device_unregister thermal_of_cooling_device_register thermal_zone_bind_cooling_device + thermal_zone_device_critical thermal_zone_device_disable thermal_zone_device_enable thermal_zone_device_register_with_trips @@ -2218,6 +2301,7 @@ _totalram_pages touch_softlockup_watchdog trace_clock_local + trace_dump_stack trace_event_buffer_commit trace_event_buffer_reserve trace_event_ignore_this_pid @@ -2246,6 +2330,8 @@ __traceiter_android_vh_printk_caller __traceiter_android_vh_printk_caller_id __traceiter_android_vh_sched_show_task + __traceiter_android_vh_watchdog_timer_softlockup + __traceiter_android_vh_wq_lockup_pool __traceiter_gpu_mem_total __traceiter_irq_handler_entry __traceiter_irq_handler_exit @@ -2254,6 +2340,7 @@ __traceiter_mmap_lock_start_locking __traceiter_mm_page_alloc __traceiter_mm_page_free + __traceiter_rcu_stall_warning __traceiter_rwmmio_post_read __traceiter_rwmmio_post_write __traceiter_rwmmio_read @@ -2282,6 +2369,8 @@ __tracepoint_android_vh_printk_caller __tracepoint_android_vh_printk_caller_id __tracepoint_android_vh_sched_show_task + __tracepoint_android_vh_watchdog_timer_softlockup + __tracepoint_android_vh_wq_lockup_pool __tracepoint_gpu_mem_total __tracepoint_irq_handler_entry __tracepoint_irq_handler_exit @@ -2292,6 +2381,7 @@ __tracepoint_mm_page_free tracepoint_probe_register tracepoint_probe_unregister + __tracepoint_rcu_stall_warning __tracepoint_rwmmio_post_read __tracepoint_rwmmio_post_write __tracepoint_rwmmio_read @@ -2392,6 +2482,9 @@ usb_ep_type_string usb_find_interface usb_free_urb + usb_function_activate + usb_function_deactivate + usb_gadget_frame_number usb_gadget_giveback_request usb_gadget_map_request_by_dev usb_gadget_register_driver_owner @@ -2426,6 +2519,8 @@ usb_kill_anchored_urbs usb_kill_urb usb_lock_device_for_reset + usbnet_cdc_zte_rx_fixup + usbnet_generic_cdc_bind usb_phy_set_charger_current usb_put_dev usb_put_hcd @@ -2458,6 +2553,7 @@ utf16s_to_utf8s utf8_to_utf32 uuid_null + uvc_format_by_guid v4l2_ctrl_add_handler v4l2_ctrl_find v4l2_ctrl_handler_free @@ -2482,6 +2578,7 @@ v4l2_fh_exit v4l2_fh_init v4l2_fh_open + v4l2_fill_pixfmt __v4l2_find_nearest_size v4l2_i2c_subdev_init v4l2_m2m_buf_queue @@ -2504,6 +2601,7 @@ v4l2_m2m_qbuf v4l2_m2m_release v4l2_m2m_try_schedule + v4l2_simplify_fraction v4l2_src_change_event_subscribe v4l2_subdev_link_validate v4l_bound_align_image @@ -2574,10 +2672,12 @@ wakeup_source_register wakeup_source_unregister __wake_up_sync + __wake_up_sync_key __warn_flushing_systemwide_wq __warn_printk wireless_nlevent_flush wireless_send_event + woken_wake_function work_busy write_cache_pages write_inode_now diff --git a/android/abi_gki_aarch64_db845c b/android/abi_gki_aarch64_db845c index 1d4ad7ca8a899..8fbd86d5bafc1 100644 --- a/android/abi_gki_aarch64_db845c +++ b/android/abi_gki_aarch64_db845c @@ -153,6 +153,7 @@ __devm_regmap_init_i2c __devm_regmap_init_mmio_clk devm_regulator_bulk_get + devm_regulator_bulk_get_enable devm_regulator_get devm_regulator_get_optional devm_regulator_register @@ -269,6 +270,7 @@ flush_work __flush_workqueue fortify_panic + fput free_io_pgtable_ops free_irq fwnode_handle_put @@ -1608,6 +1610,7 @@ vscnprintf ww_mutex_lock ww_mutex_lock_interruptible + ww_mutex_trylock ww_mutex_unlock # required by msm_serial.ko @@ -1710,6 +1713,7 @@ # required by pinctrl-msm.ko device_property_read_u16_array + devm_register_sys_off_handler gpiochip_line_is_valid gpiochip_lock_as_irq gpiochip_unlock_as_irq @@ -2003,6 +2007,7 @@ # required by snd-soc-lpass-va-macro.ko clk_hw_get_clk dapm_regulator_event + devm_clk_hw_get_clk regcache_sync_region # required by snd-soc-lpass-wsa-macro.ko diff --git a/android/abi_gki_aarch64_fips140 b/android/abi_gki_aarch64_fips140 index c028bb857211a..e0fb93015f6e0 100644 --- a/android/abi_gki_aarch64_fips140 +++ b/android/abi_gki_aarch64_fips140 @@ -94,6 +94,8 @@ kmalloc_trace kmemdup ktime_get + kvfree + kvmalloc_node __list_add_valid_or_report __list_del_entry_valid_or_report memcpy diff --git a/android/abi_gki_aarch64_galaxy b/android/abi_gki_aarch64_galaxy index b39e1faec87dd..9392b5c07eb73 100644 --- a/android/abi_gki_aarch64_galaxy +++ b/android/abi_gki_aarch64_galaxy @@ -44,6 +44,8 @@ mas_walk mas_prev mas_next + mii_check_media + mii_ethtool_gset mm_access netlink_ack param_get_uint @@ -246,6 +248,10 @@ __tracepoint_vm_unmapped_area __tracepoint_workqueue_execute_start __tracepoint_kfree_skb + usbnet_get_link + usbnet_get_link_ksettings_mii + usbnet_nway_reset + usbnet_set_link_ksettings_mii usb_set_device_state yield regulator_get_current_limit diff --git a/android/abi_gki_aarch64_honor b/android/abi_gki_aarch64_honor index 33decf01d4493..73d66820bd3dd 100644 --- a/android/abi_gki_aarch64_honor +++ b/android/abi_gki_aarch64_honor @@ -95,11 +95,18 @@ zero_fill_bio_iter percpu_ref_is_zero vm_normal_folio_pmd + folios_put_refs __trace_bputs + __traceiter_android_vh_folio_add_lru + __tracepoint_android_vh_folio_add_lru __traceiter_android_vh_proactive_compact_wmark_high __tracepoint_android_vh_proactive_compact_wmark_high __traceiter_android_vh_killed_process __tracepoint_android_vh_killed_process + __traceiter_android_vh_oom_reaper_delay_bypass + __tracepoint_android_vh_oom_reaper_delay_bypass + __traceiter_android_vh_thaw_killed_process + __tracepoint_android_vh_thaw_killed_process __traceiter_android_vh_rwsem_write_wait_finish __tracepoint_android_vh_rwsem_write_wait_finish __tracepoint_android_rvh_cpuinfo_c_show diff --git a/android/abi_gki_aarch64_imx b/android/abi_gki_aarch64_imx index d2e8e2b9b6bfd..b0fea5c239a14 100644 --- a/android/abi_gki_aarch64_imx +++ b/android/abi_gki_aarch64_imx @@ -519,6 +519,7 @@ devres_remove_group dev_set_name _dev_warn + disable_hardirq disable_irq disable_irq_nosync disable_percpu_irq @@ -852,6 +853,7 @@ drm_vblank_init drop_reasons_register_subsys drop_reasons_unregister_subsys + dummy_irq_chip dump_stack dw_pcie_ep_init dw_pcie_ep_raise_legacy_irq @@ -1190,6 +1192,8 @@ irq_dispose_mapping __irq_domain_add irq_domain_add_legacy + __irq_domain_alloc_fwnode + irq_domain_free_fwnode irq_domain_get_irq_data irq_domain_remove irq_domain_set_info @@ -1723,6 +1727,7 @@ phy_ethtool_set_link_ksettings phy_ethtool_set_wol phy_exit + phy_get_pause phy_init phy_init_eee phy_init_hw @@ -1833,6 +1838,7 @@ __platform_driver_probe __platform_driver_register platform_driver_unregister + platform_get_ethdev_address platform_get_irq platform_get_irq_byname platform_get_irq_byname_optional @@ -2610,6 +2616,26 @@ usb_hcd_map_urb_for_dma usb_hcd_unmap_urb_for_dma usb_initialize_gadget + usbnet_change_mtu + usbnet_defer_kevent + usbnet_disconnect + usbnet_get_drvinfo + usbnet_get_endpoints + usbnet_get_msglevel + usbnet_open + usbnet_probe + usbnet_read_cmd + usbnet_read_cmd_nopm + usbnet_resume + usbnet_set_msglevel + usbnet_skb_return + usbnet_start_xmit + usbnet_stop + usbnet_suspend + usbnet_tx_timeout + usbnet_write_cmd + usbnet_write_cmd_async + usbnet_write_cmd_nopm usb_phy_set_charger_current usb_phy_set_charger_state usb_phy_set_event diff --git a/android/abi_gki_aarch64_lenovo b/android/abi_gki_aarch64_lenovo index b432bf922c101..cf5575fe5abb3 100644 --- a/android/abi_gki_aarch64_lenovo +++ b/android/abi_gki_aarch64_lenovo @@ -189,6 +189,7 @@ cpus_read_lock cpus_read_unlock cpu_topology + cpuset_cpus_allowed crc32_le css_next_child css_task_iter_end @@ -1260,6 +1261,7 @@ __traceiter_android_rvh_wake_up_new_task __traceiter_android_vh_dup_task_struct __traceiter_android_vh_cpufreq_online + __traceiter_android_rvh_cpufreq_create_policy __traceiter_android_vh_update_topology_flags_workfn __traceiter_binder_transaction_received __traceiter_android_vh_alloc_swap_slot_cache @@ -1287,6 +1289,7 @@ __tracepoint_android_rvh_wake_up_new_task __tracepoint_android_vh_dup_task_struct __tracepoint_android_vh_cpufreq_online + __tracepoint_android_rvh_cpufreq_create_policy __tracepoint_android_vh_update_topology_flags_workfn __tracepoint_binder_transaction_received __tracepoint_cpu_frequency_limits diff --git a/android/abi_gki_aarch64_mtk b/android/abi_gki_aarch64_mtk index ebecd7f8c5dde..2a33194968c8f 100644 --- a/android/abi_gki_aarch64_mtk +++ b/android/abi_gki_aarch64_mtk @@ -1196,9 +1196,14 @@ hid_add_device hid_allocate_device hid_destroy_device + hid_field_extract hid_ignore hid_input_report + hid_is_usb hid_parse_report + hid_register_report + hid_report_raw_event + hid_validate_values housekeeping_cpumask housekeeping_overridden housekeeping_test_cpu @@ -1322,6 +1327,7 @@ input_alloc_absinfo input_allocate_device input_close_device + input_enable_softrepeat input_event input_free_device input_mt_destroy_slots @@ -1778,6 +1784,8 @@ net_selftest_get_strings nf_conntrack_destroy nf_register_net_hooks + nf_sk_lookup_slow_v4 + nf_sk_lookup_slow_v6 nf_unregister_net_hooks nla_find nla_memcpy @@ -2215,6 +2223,7 @@ power_supply_get_drvdata power_supply_get_property power_supply_is_system_supplied + power_supply_powers power_supply_put power_supply_register power_supply_reg_notifier @@ -3801,6 +3810,7 @@ zlib_deflateReset zlib_deflate_workspacesize zlib_inflate + zlib_inflateEnd zlib_inflateIncomp zlib_inflateInit2 zlib_inflateReset diff --git a/android/abi_gki_aarch64_nvidia b/android/abi_gki_aarch64_nvidia index 2497126b83726..ef2587aba9fa0 100644 --- a/android/abi_gki_aarch64_nvidia +++ b/android/abi_gki_aarch64_nvidia @@ -1,175 +1,416 @@ [abi_symbol_list] # commonly used symbols alloc_chrdev_region + __alloc_pages + __alloc_skb + alloc_workqueue alt_cb_patch_nops + anon_inode_getfile __arch_copy_from_user __arch_copy_to_user + arm64_use_ng_mappings + __bitmap_clear + bitmap_find_next_zero_area_off + __bitmap_set + bpf_trace_run1 + bpf_trace_run2 + bpf_trace_run3 + bpf_trace_run4 + bpf_trace_run5 + bpf_trace_run6 + bpf_trace_run8 + cancel_delayed_work + cancel_delayed_work_sync + capable cdev_add cdev_del cdev_init __check_object_size class_create class_destroy + clk_disable + clk_enable + clk_get_rate + clk_prepare + clk_set_rate + clk_unprepare complete + __const_udelay + cpu_number + __cpu_online_mask + __cpu_possible_mask + debugfs_create_bool + debugfs_create_dir + debugfs_create_file + debugfs_create_u32 + debugfs_create_u64 + debugfs_remove + delayed_work_timer_fn + destroy_workqueue dev_driver_string _dev_err + dev_get_by_name device_create + device_create_file + device_del device_destroy + device_unregister _dev_info + devm_clk_get + devm_clk_put + devm_free_irq + devm_gpio_request_one + devm_ioremap + devm_ioremap_resource devm_kfree devm_kmalloc devm_memremap + devm_of_platform_populate devm_request_threaded_irq + __devm_reset_control_get + dev_set_name _dev_warn + disable_irq + disable_irq_nosync + dma_alloc_attrs + dma_buf_attach + dma_buf_detach + dma_buf_export + dma_buf_get + dma_buf_map_attachment + dma_buf_put + dma_buf_unmap_attachment + dma_fence_add_callback + dma_fence_array_ops + dma_fence_context_alloc + dma_fence_init + dma_fence_release + dma_fence_remove_callback + dma_fence_signal + dma_fence_signal_timestamp_locked + dma_fence_wait_timeout + dma_free_attrs + dma_map_page_attrs + dma_map_resource + dma_map_sg_attrs + dma_set_coherent_mask + dma_set_mask + dma_sync_single_for_cpu + dma_unmap_resource + dma_unmap_sg_attrs + down_read + down_write + down_write_trylock + enable_irq + __fdget + fd_install + fget + _find_first_zero_bit + _find_next_bit + finish_wait + flush_delayed_work fortify_panic + fput free_irq + __free_pages + generic_file_llseek + __get_free_pages + __get_task_comm + get_unused_fd_flags + gpiod_get_raw_value + gpiod_set_raw_value + gpiod_to_irq + gpio_to_desc + hrtimer_cancel + hrtimer_init + hrtimer_start_range_ns + init_net + __init_rwsem __init_swait_queue_head init_timer_key + init_wait_entry __init_waitqueue_head + iommu_get_domain_for_dev + ioremap_prot + iounmap + is_vmalloc_addr + jiffies + jiffies_to_timespec64 jiffies_to_usecs + kasan_flag_enabled + kasprintf kfree + kimage_voffset __kmalloc kmalloc_caches + kmalloc_large kmalloc_trace + kmem_cache_alloc + kmem_cache_create + kmem_cache_destroy + kmem_cache_free + krealloc + kstrdup kstrtouint + kthread_create_on_node + kthread_should_stop + kthread_stop + ktime_get + ktime_get_mono_fast_ns + ktime_get_real_ts64 + ktime_get_with_offset + kvfree_call_rcu + __list_add_valid_or_report + __list_del_entry_valid_or_report log_post_read_mmio + log_post_write_mmio log_read_mmio + log_write_mmio + memcmp memcpy __memcpy_fromio memset + memstart_addr + mod_timer module_layout + __msecs_to_jiffies + msleep __mutex_init mutex_lock + mutex_trylock mutex_unlock + netlink_unicast + noop_llseek + __num_online_cpus + of_device_get_match_data + of_device_is_available + of_device_is_compatible + of_dma_configure_id + of_find_compatible_node + of_find_device_by_node + of_find_matching_node_and_match + of_find_node_by_name + of_find_node_opts_by_path of_find_property + of_get_named_gpio + of_get_next_child + of_get_property + of_machine_compatible_match + of_match_device + __of_parse_phandle_with_args + of_property_count_elems_of_size + of_property_match_string + of_property_read_string + of_property_read_string_helper of_property_read_u32_index of_property_read_variable_u32_array + of_property_read_variable_u8_array panic + param_ops_bool + perf_trace_buf_alloc + perf_trace_run_bpf_submit + pfn_is_map_memory pid_task + platform_device_unregister __platform_driver_register platform_driver_unregister + platform_get_irq + platform_get_irq_byname + platform_get_resource + platform_get_resource_byname + __platform_register_drivers + platform_unregister_drivers + __pm_runtime_disable + pm_runtime_enable + pm_runtime_force_suspend + __pm_runtime_idle + __pm_runtime_resume + pm_runtime_set_autosuspend_delay + __pm_runtime_suspend + __pm_runtime_use_autosuspend + preempt_schedule + preempt_schedule_notrace + prepare_to_wait_event _printk + put_device __put_task_struct + put_unused_fd + queue_delayed_work_on + queue_work_on + ___ratelimit _raw_spin_lock + _raw_spin_lock_irqsave _raw_spin_unlock + _raw_spin_unlock_irqrestore + refcount_dec_and_mutex_lock + refcount_warn_saturate + register_chrdev_region + register_pm_notifier + release_firmware + remap_pfn_range request_threaded_irq + reset_control_assert + reset_control_reset + schedule schedule_timeout + scnprintf + seq_lseek + seq_printf + seq_puts + seq_read + seq_write + sg_alloc_table + sg_alloc_table_from_pages_segment + sg_free_table + sg_init_table + sg_next + simple_attr_open + simple_attr_read + simple_attr_release + simple_attr_write + single_open + single_release snprintf + soc_device_match + split_page + sprintf + sscanf __stack_chk_fail + strchr + strcmp + strcpy strlen strncmp + strncpy strnlen strscpy + __sw_hweight64 + sync_file_create + sync_file_get_fence sysfs_create_group sysfs_remove_group system_cpucaps system_wq + tegra_bpmp_transfer tegra_ivc_notified tegra_ivc_read_advance tegra_ivc_read_get_next_frame tegra_ivc_reset tegra_ivc_write_advance tegra_ivc_write_get_next_frame + tegra_sku_info + trace_event_buffer_commit + trace_event_buffer_reserve + trace_event_printf + trace_event_raw_init + trace_event_reg + trace_handle_return __traceiter_rwmmio_post_read + __traceiter_rwmmio_post_write __traceiter_rwmmio_read + __traceiter_rwmmio_write __tracepoint_rwmmio_post_read + __tracepoint_rwmmio_post_write __tracepoint_rwmmio_read + __tracepoint_rwmmio_write + trace_print_hex_seq + trace_raw_output_prep + __trace_trigger_soft_disabled + __udelay unregister_chrdev_region + up + up_read + up_write + usleep_range_state + vfree + vmalloc + vmap + vsnprintf + vunmap + vzalloc + wait_for_completion + wait_for_completion_interruptible + wait_for_completion_timeout __wake_up + wake_up_process __warn_printk + xa_destroy + xa_erase + xa_load -# required by ivc-cdev.ko - device_del - devm_free_irq - noop_llseek - remap_pfn_range +# required by cpuidle-tegra-auto.ko + cpuidle_register + cpuidle_unregister + +# required by host1x-emu.ko + nsecs_to_jiffies64 + +# required by host1x.ko + alloc_iova + __free_iova + free_iova + host1x_context_device_bus_type + iommu_detach_group + iova_cache_get + iova_cache_put + pm_generic_freeze + pm_generic_poweroff + pm_generic_restore + pm_generic_thaw + reset_control_bulk_acquire + reset_control_bulk_release # required by ivc_ext.ko - dma_sync_single_for_cpu __memcpy_toio +# required by nvgpu.ko + async_synchronize_cookie + dev_pm_domain_set + pcie_reset_flr + pci_find_host_bridge + pci_ignore_hotplug + tegra_bpmp_get + tegra_fuse_readl + +# required by nvmap.ko + arch_invalidate_pmem + # required by nvsciipc.ko _dev_notice - __fdget find_get_pid - fput platform_device_register_full - platform_device_unregister - sprintf # required by tegra_bpmp.ko clk_hw_determine_rate_no_reparent clk_hw_get_name clk_hw_unregister - debugfs_create_dir - debugfs_create_file - debugfs_remove dentry_path_raw devm_clk_hw_register devm_reset_controller_register - dma_alloc_attrs - dma_free_attrs - _find_next_bit - kmalloc_large - kstrdup - ktime_get of_clk_add_hw_provider - of_device_get_match_data of_genpd_add_provider_onecell - __of_parse_phandle_with_args of_platform_default_populate pm_genpd_init pm_genpd_remove - seq_lseek - seq_read - seq_write single_open_size - single_release - strncpy tegra_bpmp_free_mrq tegra_bpmp_mrq_is_supported tegra_bpmp_mrq_return tegra_bpmp_request_mrq - tegra_bpmp_transfer tegra_bpmp_transfer_atomic - tegra_sku_info # required by tegra_hv.ko - arm64_use_ng_mappings class_create_file_ns - ioremap_prot - iounmap irq_get_irq_data - memstart_addr of_add_property of_chosen - of_find_compatible_node of_irq_get - pfn_is_map_memory tegra_ivc_init # required by tegra_hv_pm_ctl.ko - __alloc_skb find_vpid - finish_wait - init_net - init_wait_entry - msleep __netlink_kernel_create - netlink_unicast __nlmsg_put - prepare_to_wait_event - register_pm_notifier - schedule - strcmp - wait_for_completion_timeout - -# required by tegra_hv_vblk_oops.ko - delayed_work_timer_fn - dma_map_page_attrs - __get_free_pages - is_vmalloc_addr - queue_delayed_work_on # required by tegra_vblk.ko blk_execute_rq @@ -192,41 +433,20 @@ blk_queue_physical_block_size blk_queue_write_cache __blk_rq_map_sg - capable - __cpu_possible_mask del_gendisk device_add_disk - device_create_file - disable_irq disk_check_media_change - dma_map_sg_attrs - dma_unmap_sg_attrs - enable_irq - _find_first_zero_bit - jiffies - kasan_flag_enabled kthread_create_on_cpu - kthread_create_on_node - __list_add_valid_or_report - __list_del_entry_valid_or_report - mod_timer - __num_online_cpus - of_find_node_by_name put_disk - queue_work_on - _raw_spin_lock_irqsave - _raw_spin_unlock_irqrestore __register_blkdev sched_setattr_nocheck set_capacity set_disk_ro - sg_init_table sg_nents - __sw_hweight64 timer_delete unregister_blkdev - vfree - vzalloc - wait_for_completion - wait_for_completion_interruptible - wake_up_process + +# required by tegradisp-drm.ko + drm_edid_override_connector_update + drm_file_get_master + drm_mode_create_dvi_i_properties diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus old mode 100644 new mode 100755 index 219baa63c2ff5..63af968790ce4 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -87,6 +87,7 @@ ktime_get_real_ts64 lock_rename lookup_one_len_unlocked + loop_process_cmd_list mem_cgroup_from_id mem_cgroup_update_lru_size memcg_page_state @@ -271,6 +272,7 @@ __traceiter_android_vh_kvmalloc_node_use_vmalloc __traceiter_android_vh_look_around __traceiter_android_vh_look_around_migrate_folio + __traceiter_android_vh_loop_skip_queue_work __traceiter_android_vh_map_util_freq __traceiter_android_vh_mark_page_accessed __traceiter_android_vh_mem_cgroup_alloc @@ -283,6 +285,7 @@ __traceiter_android_vh_mutex_opt_spin_finish __traceiter_android_vh_mutex_opt_spin_start __traceiter_android_vh_mutex_unlock_slowpath + __traceiter_android_vh_oom_evaluate_task_bypass __traceiter_android_vh_page_add_new_anon_rmap __traceiter_android_vh_page_referenced_check_bypass __traceiter_android_vh_page_should_be_protected @@ -439,6 +442,7 @@ __tracepoint_android_vh_kvmalloc_node_use_vmalloc __tracepoint_android_vh_look_around __tracepoint_android_vh_look_around_migrate_folio + __tracepoint_android_vh_loop_skip_queue_work __tracepoint_android_vh_map_util_freq __tracepoint_android_vh_mark_page_accessed __tracepoint_android_vh_mem_cgroup_alloc @@ -451,6 +455,7 @@ __tracepoint_android_vh_mutex_opt_spin_finish __tracepoint_android_vh_mutex_opt_spin_start __tracepoint_android_vh_mutex_unlock_slowpath + __tracepoint_android_vh_oom_evaluate_task_bypass __tracepoint_android_vh_page_add_new_anon_rmap __tracepoint_android_vh_page_referenced_check_bypass __tracepoint_android_vh_page_should_be_protected @@ -547,6 +552,7 @@ unregister_net_sysctl_table unregister_sysctl_table unregister_tcf_proto_ops + utf8_normalize v4l2_i2c_subdev_init vfs_create vfs_getattr @@ -584,3 +590,15 @@ contpte_ptep_get contpte_ptep_set_access_flags __ptep_set_access_flags + __traceiter_android_vh_resched_curr_lazy + __traceiter_android_vh_restore_curr_resched + __traceiter_android_vh_clear_curr_lazy + __tracepoint_android_vh_resched_curr_lazy + __tracepoint_android_vh_restore_curr_resched + __tracepoint_android_vh_clear_curr_lazy + __traceiter_android_vh_lock_delay_schedule + __traceiter_android_vh_lock_task_fork + __traceiter_android_vh_lock_task_exit + __tracepoint_android_vh_lock_delay_schedule + __tracepoint_android_vh_lock_task_fork + __tracepoint_android_vh_lock_task_exit diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index d17b443f7c9e6..611085dc9e9cc 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -1,5 +1,6 @@ [abi_symbol_list] activate_task + active_load_balance_cpu_stop add_cpu add_timer add_timer_on @@ -100,6 +101,7 @@ blk_mq_quiesce_tagset blk_mq_unfreeze_queue blk_mq_unquiesce_tagset + blk_op_str blk_put_queue blk_queue_flag_clear blk_queue_flag_set @@ -112,6 +114,7 @@ blk_queue_update_dma_alignment blk_rq_map_user_io blk_rq_unmap_user + blk_status_to_errno blocking_notifier_call_chain blocking_notifier_chain_register blocking_notifier_chain_unregister @@ -225,8 +228,10 @@ __const_udelay consume_skb contig_page_data + contpte_ptep_get _copy_from_iter __copy_overflow + copy_page _copy_to_iter __cpu_active_mask cpu_all_bits @@ -378,8 +383,12 @@ devfreq_recommended_opp devfreq_register_opp_notifier devfreq_remove_device + devfreq_remove_governor + devfreq_resume_device + devfreq_suspend_device devfreq_unregister_opp_notifier devfreq_update_interval + devfreq_update_target __dev_fwnode __dev_get_by_index dev_get_by_index @@ -434,6 +443,7 @@ devm_clk_bulk_get_optional devm_clk_get devm_clk_get_optional + devm_clk_get_optional_enabled devm_clk_hw_register devm_clk_put devm_devfreq_add_device @@ -537,6 +547,7 @@ devm_usb_get_phy_by_phandle devm_watchdog_register_device _dev_notice + dev_open dev_pm_clear_wake_irq dev_pm_domain_attach dev_pm_domain_attach_by_id @@ -578,6 +589,7 @@ devres_add __devres_alloc_node devres_free + dev_set_mtu dev_set_name dev_vprintk_emit _dev_warn @@ -624,6 +636,7 @@ dma_fence_context_alloc dma_fence_default_wait dma_fence_enable_sw_signaling + dma_fence_free dma_fence_get_status dma_fence_init dma_fence_release @@ -675,6 +688,7 @@ do_SAK do_send_sig_info do_trace_netlink_extack + double_rq_lock do_wait_intr_irq down downgrade_write @@ -712,6 +726,7 @@ drm_atomic_get_plane_state drm_atomic_get_private_obj_state drm_atomic_helper_async_check + drm_atomic_helper_async_commit drm_atomic_helper_bridge_destroy_state __drm_atomic_helper_bridge_duplicate_state drm_atomic_helper_bridge_duplicate_state @@ -728,6 +743,7 @@ drm_atomic_helper_commit_cleanup_done drm_atomic_helper_commit_duplicated_state drm_atomic_helper_commit_hw_done + drm_atomic_helper_commit_modeset_disables drm_atomic_helper_commit_modeset_enables drm_atomic_helper_commit_planes drm_atomic_helper_commit_tail @@ -763,7 +779,9 @@ drm_atomic_helper_update_legacy_modeset_state drm_atomic_helper_update_plane drm_atomic_helper_wait_for_dependencies + drm_atomic_helper_wait_for_fences drm_atomic_helper_wait_for_flip_done + drm_atomic_helper_wait_for_vblanks drm_atomic_nonblocking_commit drm_atomic_normalize_zpos drm_atomic_private_obj_fini @@ -924,6 +942,7 @@ drm_prime_pages_to_sg drm_prime_sg_to_page_array drm_printf + __drm_printfn_coredump __drm_printfn_debug __drm_printfn_info __drm_printfn_seq_file @@ -939,6 +958,7 @@ drm_property_create_signed_range drm_property_lookup_blob drm_property_replace_blob + __drm_puts_coredump __drm_puts_seq_file drm_read drm_rect_clip_scaled @@ -949,6 +969,7 @@ drm_send_event drm_send_event_locked drm_simple_encoder_init + drm_state_dump drm_sysfs_connector_property_event drm_universal_plane_init drm_vblank_init @@ -958,6 +979,7 @@ drop_reasons_register_subsys drop_reasons_unregister_subsys dump_backtrace + dump_page dump_stack dump_tasks dw_handle_msi_irq @@ -1036,6 +1058,7 @@ free_hyp_memcache free_iova_fast free_irq + freelist_task free_netdev __free_pages free_pages @@ -1098,6 +1121,7 @@ get_random_u8 get_reclaim_params get_sg_io_hdr + get_swap_device __get_task_comm get_task_cred get_task_mm @@ -1178,6 +1202,7 @@ i2c_find_adapter_by_fwnode i2c_find_device_by_fwnode i2c_for_each_dev + i2c_freq_mode_string i2c_generic_scl_recovery i2c_get_adapter i2c_match_id @@ -1215,6 +1240,7 @@ i3c_master_register i3c_master_set_info i3c_master_unregister + I_BDEV icc_link_create icc_node_add icc_node_create @@ -1274,6 +1300,8 @@ init_wait_entry __init_waitqueue_head init_wait_var_entry + inode_owner_or_capable + inode_permission input_alloc_absinfo input_allocate_device input_close_device @@ -1407,6 +1435,7 @@ __kfifo_init __kfifo_out __kfifo_out_peek + __kfifo_to_user kfree kfree_sensitive kfree_skb_list_reason @@ -1449,6 +1478,7 @@ ksize ksoftirqd kstat + kstat_irqs_usr kstrdup kstrndup kstrtobool @@ -1523,6 +1553,8 @@ __list_add_valid_or_report __list_del_entry_valid_or_report list_sort + llist_add_batch + llist_reverse_order __local_bh_enable_ip lock_sock_nested log_abnormal_wakeup_reason @@ -1534,9 +1566,23 @@ loops_per_jiffy lru_cache_disable lru_disable_count + LZ4_compress_fast + LZ4_compress_fast_continue + LZ4_compress_HC + LZ4_compress_HC_continue + LZ4_decompress_safe + LZ4_decompress_safe_continue + LZ4_loadDict + LZ4_loadDictHC + LZ4_resetStreamHC + LZ4_setStreamDecode + lzo1x_1_compress + lzo1x_decompress_safe + lzorle1x_1_compress mac_pton mas_empty_area_rev mas_find + mas_prev max_load_balance_interval mbox_chan_received_data mbox_chan_txdone @@ -1628,9 +1674,12 @@ napi_schedule_prep __ndelay __netdev_alloc_skb + netdev_core_stats_alloc netdev_err netdev_info netdev_refcnt_read + netdev_rx_handler_register + netdev_rx_handler_unregister netdev_set_default_ethtool_ops netdev_state_change netdev_update_features @@ -1653,6 +1702,7 @@ netlink_unregister_notifier net_ns_type_operations net_ratelimit + next_arg nf_register_net_hooks nf_unregister_net_hooks nla_find @@ -1667,8 +1717,10 @@ __nlmsg_put nonseekable_open noop_llseek + nop_mnt_idmap nr_cpu_ids nr_irqs + nr_swap_pages ns_capable ns_capable_noaudit nsecs_to_jiffies @@ -1894,6 +1946,7 @@ phy_power_on phy_set_mode_ext phy_set_speed + pidfd_get_task pid_task pinconf_generic_dt_free_map pinconf_generic_dt_node_to_map @@ -1946,6 +1999,8 @@ platform_msi_domain_free_irqs __platform_register_drivers platform_unregister_drivers + plist_add + plist_del pm_genpd_add_subdomain pm_genpd_init pm_genpd_remove @@ -2015,6 +2070,7 @@ pskb_expand_head __pskb_pull_tail ___pskb_trim + __pte_offset_map_lock push_cpu_stop __put_cred put_device @@ -2083,6 +2139,7 @@ rdev_get_id rdev_get_regmap readahead_expand + read_comp_algo_dictionary reboot_mode refcount_dec_not_one refcount_warn_saturate @@ -2224,6 +2281,7 @@ runqueues sbitmap_weight sched_clock + sched_energy_present sched_feat_keys sched_setattr_nocheck sched_set_fifo @@ -2546,6 +2604,7 @@ sysfs_notify sysfs_remove_file_from_group sysfs_remove_file_ns + sysfs_remove_files sysfs_remove_group sysfs_remove_groups sysfs_remove_link @@ -2630,7 +2689,9 @@ trace_event_raw_init trace_event_reg trace_handle_return + __traceiter_android_rvh_after_enqueue_task __traceiter_android_rvh_attach_entity_load_avg + __traceiter_android_rvh_blk_mq_has_request __traceiter_android_rvh_can_migrate_task __traceiter_android_rvh_cgroup_force_kthread_migration __traceiter_android_rvh_check_preempt_wakeup @@ -2643,6 +2704,7 @@ __traceiter_android_rvh_enqueue_task __traceiter_android_rvh_enqueue_task_fair __traceiter_android_rvh_find_lowest_rq + __traceiter_android_rvh_gup_longterm_locked __traceiter_android_rvh_hw_protection_shutdown __traceiter_android_rvh_iommu_alloc_insert_iova __traceiter_android_rvh_iommu_iovad_init_alloc_algo @@ -2664,12 +2726,14 @@ __traceiter_android_rvh_select_task_rq_fair __traceiter_android_rvh_select_task_rq_rt __traceiter_android_rvh_set_cpus_allowed_by_task + __traceiter_android_rvh_set_cpus_allowed_ptr __traceiter_android_rvh_set_iowait __traceiter_android_rvh_setscheduler __traceiter_android_rvh_setscheduler_prio __traceiter_android_rvh_set_task_cpu __traceiter_android_rvh_set_user_nice_locked __traceiter_android_rvh_swap_readpage_bdev_sync + __traceiter_android_rvh_task_fits_cpu __traceiter_android_rvh_tick_entry __traceiter_android_rvh_try_to_wake_up_success __traceiter_android_rvh_uclamp_eff_get @@ -2691,33 +2755,40 @@ __traceiter_android_vh_binder_set_priority __traceiter_android_vh_calculate_totalreserve_pages __traceiter_android_vh_check_new_page + __traceiter_android_vh_chk_task __traceiter_android_vh_cpu_idle_enter __traceiter_android_vh_cpu_idle_exit + __traceiter_android_vh_do_async_mmap_readahead __traceiter_android_vh_dump_throttled_rt_tasks __traceiter_android_vh_dup_task_struct __traceiter_android_vh_early_resume_begin __traceiter_android_vh_enable_thermal_genl_check __traceiter_android_vh_ep_create_wakeup_source + __traceiter_android_vh_exit_check __traceiter_android_vh_filemap_get_folio __traceiter_android_vh_free_one_page_flag_check __traceiter_android_vh_free_pages_prepare_init __traceiter_android_vh_ipi_stop + __traceiter_android_vh_kvmalloc_node_use_vmalloc __traceiter_android_vh_mm_compaction_begin __traceiter_android_vh_mm_compaction_end __traceiter_android_vh_mm_kcompactd_cpu_online __traceiter_android_vh_post_alloc_hook __traceiter_android_vh_prio_inheritance __traceiter_android_vh_prio_restore + __traceiter_android_vh_put_task __traceiter_android_vh_resume_end __traceiter_android_vh_rmqueue __traceiter_android_vh_scheduler_tick __traceiter_android_vh_setscheduler_uclamp __traceiter_android_vh_set_task_comm + __traceiter_android_vh_show_mem __traceiter_android_vh_si_meminfo_adjust __traceiter_android_vh_swap_readpage_bdev_sync __traceiter_android_vh_swap_writepage __traceiter_android_vh_sysrq_crash __traceiter_android_vh_timerfd_create + __traceiter_android_vh_try_to_freeze_todo_logging __traceiter_android_vh_tune_swappiness __traceiter_android_vh_typec_store_partner_src_caps __traceiter_android_vh_typec_tcpm_log @@ -2734,6 +2805,7 @@ __traceiter_android_vh_ufs_update_sysfs __traceiter_android_vh_usb_dev_resume __traceiter_android_vh_use_amu_fie + __traceiter_android_vh_vmscan_kswapd_done __traceiter_android_vh_xhci_full_reset_on_remove __traceiter_clock_set_rate __traceiter_cma_alloc_finish @@ -2749,6 +2821,7 @@ __traceiter_mmap_lock_start_locking __traceiter_mm_vmscan_direct_reclaim_begin __traceiter_mm_vmscan_direct_reclaim_end + __traceiter_mm_vmscan_kswapd_wake __traceiter_pelt_cfs_tp __traceiter_pelt_dl_tp __traceiter_pelt_irq_tp @@ -2767,10 +2840,13 @@ __traceiter_softirq_entry __traceiter_softirq_exit __traceiter_suspend_resume + __traceiter_task_newtask __traceiter_workqueue_execute_end __traceiter_workqueue_execute_start trace_output_call + __tracepoint_android_rvh_after_enqueue_task __tracepoint_android_rvh_attach_entity_load_avg + __tracepoint_android_rvh_blk_mq_has_request __tracepoint_android_rvh_can_migrate_task __tracepoint_android_rvh_cgroup_force_kthread_migration __tracepoint_android_rvh_check_preempt_wakeup @@ -2783,6 +2859,7 @@ __tracepoint_android_rvh_enqueue_task __tracepoint_android_rvh_enqueue_task_fair __tracepoint_android_rvh_find_lowest_rq + __tracepoint_android_rvh_gup_longterm_locked __tracepoint_android_rvh_hw_protection_shutdown __tracepoint_android_rvh_iommu_alloc_insert_iova __tracepoint_android_rvh_iommu_iovad_init_alloc_algo @@ -2804,12 +2881,14 @@ __tracepoint_android_rvh_select_task_rq_fair __tracepoint_android_rvh_select_task_rq_rt __tracepoint_android_rvh_set_cpus_allowed_by_task + __tracepoint_android_rvh_set_cpus_allowed_ptr __tracepoint_android_rvh_set_iowait __tracepoint_android_rvh_setscheduler __tracepoint_android_rvh_setscheduler_prio __tracepoint_android_rvh_set_task_cpu __tracepoint_android_rvh_set_user_nice_locked __tracepoint_android_rvh_swap_readpage_bdev_sync + __tracepoint_android_rvh_task_fits_cpu __tracepoint_android_rvh_tick_entry __tracepoint_android_rvh_try_to_wake_up_success __tracepoint_android_rvh_uclamp_eff_get @@ -2831,33 +2910,40 @@ __tracepoint_android_vh_binder_set_priority __tracepoint_android_vh_calculate_totalreserve_pages __tracepoint_android_vh_check_new_page + __tracepoint_android_vh_chk_task __tracepoint_android_vh_cpu_idle_enter __tracepoint_android_vh_cpu_idle_exit + __tracepoint_android_vh_do_async_mmap_readahead __tracepoint_android_vh_dump_throttled_rt_tasks __tracepoint_android_vh_dup_task_struct __tracepoint_android_vh_early_resume_begin __tracepoint_android_vh_enable_thermal_genl_check __tracepoint_android_vh_ep_create_wakeup_source + __tracepoint_android_vh_exit_check __tracepoint_android_vh_filemap_get_folio __tracepoint_android_vh_free_one_page_flag_check __tracepoint_android_vh_free_pages_prepare_init __tracepoint_android_vh_ipi_stop + __tracepoint_android_vh_kvmalloc_node_use_vmalloc __tracepoint_android_vh_mm_compaction_begin __tracepoint_android_vh_mm_compaction_end __tracepoint_android_vh_mm_kcompactd_cpu_online __tracepoint_android_vh_post_alloc_hook __tracepoint_android_vh_prio_inheritance __tracepoint_android_vh_prio_restore + __tracepoint_android_vh_put_task __tracepoint_android_vh_resume_end __tracepoint_android_vh_rmqueue __tracepoint_android_vh_scheduler_tick __tracepoint_android_vh_setscheduler_uclamp __tracepoint_android_vh_set_task_comm + __tracepoint_android_vh_show_mem __tracepoint_android_vh_si_meminfo_adjust __tracepoint_android_vh_swap_readpage_bdev_sync __tracepoint_android_vh_swap_writepage __tracepoint_android_vh_sysrq_crash __tracepoint_android_vh_timerfd_create + __tracepoint_android_vh_try_to_freeze_todo_logging __tracepoint_android_vh_tune_swappiness __tracepoint_android_vh_typec_store_partner_src_caps __tracepoint_android_vh_typec_tcpm_log @@ -2874,6 +2960,7 @@ __tracepoint_android_vh_ufs_update_sysfs __tracepoint_android_vh_usb_dev_resume __tracepoint_android_vh_use_amu_fie + __tracepoint_android_vh_vmscan_kswapd_done __tracepoint_android_vh_xhci_full_reset_on_remove __tracepoint_clock_set_rate __tracepoint_cma_alloc_finish @@ -2889,6 +2976,7 @@ __tracepoint_mmap_lock_start_locking __tracepoint_mm_vmscan_direct_reclaim_begin __tracepoint_mm_vmscan_direct_reclaim_end + __tracepoint_mm_vmscan_kswapd_wake __tracepoint_pelt_cfs_tp __tracepoint_pelt_dl_tp __tracepoint_pelt_irq_tp @@ -2909,6 +2997,7 @@ __tracepoint_softirq_entry __tracepoint_softirq_exit __tracepoint_suspend_resume + __tracepoint_task_newtask __tracepoint_workqueue_execute_end __tracepoint_workqueue_execute_start trace_print_array_seq @@ -2929,6 +3018,7 @@ tty_termios_baud_rate tty_termios_encode_baud_rate typec_mux_get_drvdata + typec_mux_put typec_mux_register typec_mux_set typec_mux_unregister @@ -3004,6 +3094,7 @@ unregister_pernet_subsys unregister_pm_notifier unregister_reboot_notifier + unregister_restart_handler unregister_shrinker unregister_syscore_ops unregister_sysctl_table @@ -3181,6 +3272,7 @@ wakeup_source_unregister __wake_up_sync wake_up_var + walk_page_range __warn_printk watchdog_init_timeout watchdog_register_device @@ -3194,6 +3286,7 @@ ww_mutex_unlock x509_load_certificate_list __xa_alloc + __xa_clear_mark xa_destroy __xa_erase xa_erase @@ -3201,6 +3294,7 @@ xa_find_after __xa_insert xa_load + xa_set_mark xas_find xas_pause __xa_store @@ -3221,13 +3315,44 @@ xhci_set_interrupter_moderation xhci_stop_endpoint_sync xhci_suspend + zlib_deflate + zlib_deflateEnd + zlib_deflateInit2 + zlib_deflateReset + zlib_deflate_workspacesize + zlib_inflate + zlib_inflateEnd + zlib_inflateInit2 + zlib_inflateReset + zlib_inflate_workspacesize zs_compact zs_create_pool zs_destroy_pool zs_free zs_get_total_pages zs_huge_class_size + zs_lookup_class_index zs_malloc zs_map_object zs_pool_stats + zstd_cctx_workspace_bound + zstd_compress_cctx + zstd_compress_using_cdict + zstd_create_cctx_advanced + zstd_create_cdict_byreference + zstd_create_dctx_advanced + zstd_create_ddict_byreference + zstd_dctx_workspace_bound + zstd_decompress_dctx + zstd_decompress_using_ddict + zstd_default_clevel + zstd_free_cctx + zstd_free_cdict + zstd_free_dctx + zstd_free_ddict + zstd_get_cparams + zstd_get_params + zstd_init_cctx + zstd_init_dctx + zstd_is_error zs_unmap_object diff --git a/android/abi_gki_aarch64_pixel_watch b/android/abi_gki_aarch64_pixel_watch index db61c65ca0ff8..a3a0c1e66554d 100644 --- a/android/abi_gki_aarch64_pixel_watch +++ b/android/abi_gki_aarch64_pixel_watch @@ -28,6 +28,7 @@ amba_driver_register amba_driver_unregister android_rvh_probe_register + anon_inode_getfd anon_inode_getfile __arch_copy_from_user __arch_copy_to_user @@ -286,8 +287,10 @@ default_llseek deferred_free delayed_work_timer_fn + desc_to_gpio destroy_workqueue dev_addr_mod + _dev_alert dev_alloc_name __dev_change_net_namespace dev_close @@ -406,6 +409,7 @@ devm_power_supply_register devm_pwm_get devm_qcom_smem_state_get + devm_register_sys_off_handler devm_regmap_add_irq_chip devm_regmap_del_irq_chip devm_regmap_field_alloc @@ -486,6 +490,7 @@ divider_ro_round_rate_parent divider_round_rate_parent dma_alloc_attrs + dma_alloc_noncontiguous dma_alloc_pages dma_async_device_register dma_async_device_unregister @@ -521,6 +526,7 @@ dma_fence_signal_timestamp_locked dma_fence_wait_timeout dma_free_attrs + dma_free_noncontiguous dma_free_pages dma_get_sgtable_attrs dma_get_slave_channel @@ -552,6 +558,8 @@ dma_sync_single_for_device dma_unmap_page_attrs dma_unmap_sg_attrs + dma_vmap_noncontiguous + dma_vunmap_noncontiguous do_trace_netlink_extack double_rq_lock do_wait_intr @@ -631,8 +639,10 @@ drm_crtc_init_with_planes drm_crtc_send_vblank_event drm_crtc_set_max_vblank_count + drm_crtc_vblank_get drm_crtc_vblank_off drm_crtc_vblank_on + drm_crtc_vblank_put drm_crtc_vblank_reset drm_crtc_wait_one_vblank ___drm_dbg @@ -864,11 +874,14 @@ gpiod_direction_input gpiod_direction_output gpiod_direction_output_raw + gpiod_get gpiod_get_array_value gpiod_get_optional gpiod_get_raw_value gpiod_get_raw_value_cansleep gpiod_get_value + gpiod_is_active_low + gpiod_put gpiod_set_raw_value gpiod_set_value gpiod_set_value_cansleep @@ -1702,6 +1715,7 @@ remap_pfn_range remove_cpu remove_proc_entry + remove_proc_subtree remove_wait_queue report_iommu_fault request_firmware @@ -2091,6 +2105,7 @@ tick_nohz_get_sleep_length timer_delete timer_delete_sync + timer_shutdown_sync topology_clear_scale_freq_source topology_update_done topology_update_thermal_pressure @@ -2171,6 +2186,10 @@ __traceiter_mmap_lock_acquire_returned __traceiter_mmap_lock_released __traceiter_mmap_lock_start_locking + __traceiter_rwmmio_post_read + __traceiter_rwmmio_post_write + __traceiter_rwmmio_read + __traceiter_rwmmio_write __traceiter_sched_overutilized_tp __traceiter_sched_switch __traceiter_sk_data_ready @@ -2246,6 +2265,10 @@ tracepoint_probe_register tracepoint_probe_register_prio tracepoint_probe_unregister + __tracepoint_rwmmio_post_read + __tracepoint_rwmmio_post_write + __tracepoint_rwmmio_read + __tracepoint_rwmmio_write __tracepoint_sched_overutilized_tp __tracepoint_sched_switch __tracepoint_sk_data_ready @@ -2429,6 +2452,7 @@ __xa_alloc __xa_alloc_cyclic xa_destroy + __xa_erase xa_erase xa_find xa_find_after diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom old mode 100644 new mode 100755 index 9eff65a1b51a3..e56b4012f67d8 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -226,6 +226,7 @@ cpu_irqtime cpu_latency_qos_add_request cpu_latency_qos_remove_request + cpu_latency_qos_request_active cpu_latency_qos_update_request cpumask_any_and_distribute cpu_number @@ -341,6 +342,7 @@ device_match_fwnode device_match_name device_match_of_node + device_node_to_regmap device_property_present device_property_read_string device_property_read_u16_array @@ -2052,6 +2054,7 @@ __traceiter_android_rvh_dequeue_task __traceiter_android_rvh_do_sched_yield __traceiter_android_rvh_enqueue_task + __traceiter_android_rvh_entity_tick __traceiter_android_rvh_find_busiest_queue __traceiter_android_rvh_find_lowest_rq __traceiter_android_rvh_find_new_ilb @@ -2079,6 +2082,7 @@ __traceiter_android_rvh_sched_newidle_balance __traceiter_android_rvh_sched_nohz_balancer_kick __traceiter_android_rvh_schedule + __traceiter_android_rvh_schedule_bug __traceiter_android_rvh_select_task_rq_fair __traceiter_android_rvh_select_task_rq_rt __traceiter_android_rvh_set_cpus_allowed_by_task @@ -2146,6 +2150,12 @@ __traceiter_sched_switch __traceiter_sk_data_ready __traceiter_suspend_resume + __traceiter_android_rvh_enqueue_entity + __traceiter_android_rvh_dequeue_entity + __traceiter_android_vh_rtmutex_wait_start + __tracepoint_android_vh_rtmutex_wait_start + __tracepoint_android_rvh_enqueue_entity + __tracepoint_android_rvh_dequeue_entity __tracepoint_android_rvh_account_irq __tracepoint_android_rvh_before_do_sched_yield __tracepoint_android_rvh_build_perf_domains @@ -2158,6 +2168,7 @@ __tracepoint_android_rvh_dequeue_task __tracepoint_android_rvh_do_sched_yield __tracepoint_android_rvh_enqueue_task + __tracepoint_android_rvh_entity_tick __tracepoint_android_rvh_find_busiest_queue __tracepoint_android_rvh_find_lowest_rq __tracepoint_android_rvh_find_new_ilb @@ -2185,6 +2196,7 @@ __tracepoint_android_rvh_sched_newidle_balance __tracepoint_android_rvh_sched_nohz_balancer_kick __tracepoint_android_rvh_schedule + __tracepoint_android_rvh_schedule_bug __tracepoint_android_rvh_select_task_rq_fair __tracepoint_android_rvh_select_task_rq_rt __tracepoint_android_rvh_set_cpus_allowed_by_task diff --git a/android/abi_gki_aarch64_sunxi b/android/abi_gki_aarch64_sunxi index 27b308dd7254c..a76134ad0a561 100644 --- a/android/abi_gki_aarch64_sunxi +++ b/android/abi_gki_aarch64_sunxi @@ -96,3 +96,7 @@ drm_panel_remove_follower hid_driver_reset_resume hid_driver_suspend + devm_power_supply_register_no_ws + __tracepoint_android_vh_bio_add_page_merge_bypass + __traceiter_android_vh_bio_add_page_merge_bypass + diff --git a/android/abi_gki_aarch64_transsion b/android/abi_gki_aarch64_transsion index 999370ab2ca81..b4f7c10636238 100644 --- a/android/abi_gki_aarch64_transsion +++ b/android/abi_gki_aarch64_transsion @@ -1,6 +1,7 @@ [abi_symbol_list] # commonly used symbols register_tracepoint_module_notifier + cgroup_add_dfl_cftypes # required by tran block module __percpu_counter_init_many diff --git a/android/abi_gki_aarch64_virtual_device b/android/abi_gki_aarch64_virtual_device index 2231cd6ffb610..f01ce16d9d947 100644 --- a/android/abi_gki_aarch64_virtual_device +++ b/android/abi_gki_aarch64_virtual_device @@ -51,6 +51,7 @@ debugfs_create_file debugfs_create_u32 debugfs_create_u8 + debugfs_lookup_and_remove debugfs_remove default_llseek delayed_work_timer_fn diff --git a/android/abi_gki_aarch64_vivo b/android/abi_gki_aarch64_vivo index 0bb21b7195163..f6f0299ca129a 100644 --- a/android/abi_gki_aarch64_vivo +++ b/android/abi_gki_aarch64_vivo @@ -74,6 +74,7 @@ seq_put_decimal_ull seq_read_iter set_blocksize + set_task_ioprio __set_task_comm set_task_ioprio skb_orphan_partial @@ -85,6 +86,8 @@ timer_reduce __traceiter_android_rvh_alloc_and_link_pwqs __traceiter_android_rvh_bpf_skb_load_bytes + __traceiter_android_rvh_balance_fair + __traceiter_android_rvh_before_pick_task_fair __traceiter_android_rvh_check_preempt_wakeup __traceiter_android_rvh_cpufreq_transition __traceiter_android_rvh_create_worker @@ -125,6 +128,8 @@ __traceiter_android_vh_blk_mq_kick_requeue_list __traceiter_android_vh_build_skb_around __traceiter_android_vh_cgroup_attach + __traceiter_android_vh_cpuset_attach_task + __traceiter_android_vh_cpuset_css_online __traceiter_android_vh_check_folio_look_around_ref __traceiter_android_vh_check_nanosleep_syscall __traceiter_android_vh_clear_rwsem_reader_owned @@ -140,7 +145,11 @@ __traceiter_android_vh_dup_task_struct __traceiter_android_vh_f2fs_file_open __traceiter_android_vh_f2fs_ra_op_flags + __traceiter_android_vh_fas_gpu_qos_update_tracer + __traceiter_android_vh_filemap_fault_folio_locked + __traceiter_android_vh_filemap_read_end __traceiter_android_vh_filemap_update_page + __traceiter_android_vh_force_page_cache_ra __traceiter_android_vh_free_unref_page_list_bypass __traceiter_android_vh_free_task __traceiter_android_vh_free_unref_page_bypass @@ -156,7 +165,17 @@ __traceiter_android_vh_look_around_migrate_folio __traceiter_android_vh_lruvec_add_folio __traceiter_android_vh_lruvec_del_folio + __traceiter_android_vh_mempool_alloc_skip_wait + __traceiter_android_vh_mm_customize_ac + __traceiter_android_vh_mm_customize_file_is_tiny + __traceiter_android_vh_mm_customize_lru_add_dst + __traceiter_android_vh_mm_customize_pgdat_balanced + __traceiter_android_vh_mm_customize_rmqueue + __traceiter_android_vh_mm_customize_suitable_zone + __traceiter_android_vh_mm_customize_zone_pageset + __traceiter_android_vh_mm_customize_zone_max_order __traceiter_android_vh_mm_free_page + __traceiter_android_vh_mm_isolate_priv_lru __traceiter_android_vh_mmap_region __traceiter_android_vh_mutex_init __traceiter_android_vh_mutex_unlock_slowpath @@ -165,6 +184,7 @@ __traceiter_android_vh_mutex_wait_start __traceiter_android_vh_oom_swapmem_gather_finish __traceiter_android_vh_oom_swapmem_gather_init + __traceiter_android_vh_page_cache_ra_unbounded __traceiter_android_vh_percpu_rwsem_down_read __traceiter_android_vh_percpu_rwsem_up_write __traceiter_android_vh_percpu_rwsem_wq_add @@ -219,6 +239,8 @@ __traceiter_workqueue_execute_start __tracepoint_android_rvh_alloc_and_link_pwqs __tracepoint_android_rvh_bpf_skb_load_bytes + __tracepoint_android_rvh_balance_fair + __tracepoint_android_rvh_before_pick_task_fair __tracepoint_android_rvh_check_preempt_wakeup __tracepoint_android_rvh_cpufreq_transition __tracepoint_android_rvh_create_worker @@ -259,6 +281,8 @@ __tracepoint_android_vh_blk_mq_kick_requeue_list __tracepoint_android_vh_build_skb_around __tracepoint_android_vh_cgroup_attach + __tracepoint_android_vh_cpuset_attach_task + __tracepoint_android_vh_cpuset_css_online __tracepoint_android_vh_check_folio_look_around_ref __tracepoint_android_vh_check_nanosleep_syscall __tracepoint_android_vh_clear_rwsem_reader_owned @@ -274,7 +298,12 @@ __tracepoint_android_vh_dup_task_struct __tracepoint_android_vh_f2fs_file_open __tracepoint_android_vh_f2fs_ra_op_flags + + __tracepoint_android_vh_fas_gpu_qos_update_tracer + __tracepoint_android_vh_filemap_fault_folio_locked + __tracepoint_android_vh_filemap_read_end __tracepoint_android_vh_filemap_update_page + __tracepoint_android_vh_force_page_cache_ra __tracepoint_android_vh_free_task __tracepoint_android_vh_free_unref_page_bypass __tracepoint_android_vh_fuse_request_end @@ -289,7 +318,17 @@ __tracepoint_android_vh_look_around_migrate_folio __tracepoint_android_vh_lruvec_add_folio __tracepoint_android_vh_lruvec_del_folio + __tracepoint_android_vh_mempool_alloc_skip_wait + __tracepoint_android_vh_mm_customize_ac + __tracepoint_android_vh_mm_customize_file_is_tiny + __tracepoint_android_vh_mm_customize_lru_add_dst + __tracepoint_android_vh_mm_customize_pgdat_balanced + __tracepoint_android_vh_mm_customize_rmqueue + __tracepoint_android_vh_mm_customize_suitable_zone + __tracepoint_android_vh_mm_customize_zone_pageset + __tracepoint_android_vh_mm_customize_zone_max_order __tracepoint_android_vh_mm_free_page + __tracepoint_android_vh_mm_isolate_priv_lru __tracepoint_android_vh_mmap_region __tracepoint_android_vh_mutex_init __tracepoint_android_vh_mutex_unlock_slowpath @@ -298,6 +337,7 @@ __tracepoint_android_vh_mutex_wait_start __tracepoint_android_vh_oom_swapmem_gather_finish __tracepoint_android_vh_oom_swapmem_gather_init + __tracepoint_android_vh_page_cache_ra_unbounded __tracepoint_android_vh_percpu_rwsem_down_read __tracepoint_android_vh_percpu_rwsem_up_write __tracepoint_android_vh_percpu_rwsem_wq_add diff --git a/android/abi_gki_aarch64_xiaomi b/android/abi_gki_aarch64_xiaomi index 66172fb2a8486..861c5bd19829f 100644 --- a/android/abi_gki_aarch64_xiaomi +++ b/android/abi_gki_aarch64_xiaomi @@ -23,6 +23,8 @@ __tracepoint_android_vh_tune_swappiness __traceiter_android_vh_do_shrink_slab_ex __tracepoint_android_vh_do_shrink_slab_ex + __traceiter_android_vh_migration_target_bypass + __tracepoint_android_vh_migration_target_bypass # required by lz4 decompress module __tracepoint_android_vh_lz4_decompress_bypass @@ -77,6 +79,9 @@ scsi_device_set_state blk_mq_quiesce_tagset blk_mq_unquiesce_tagset + dev_pm_opp_find_freq_floor_indexed + scsi_host_busy + dev_pm_opp_find_freq_ceil_indexed memblock_free __mem_cgroup_charge __mem_cgroup_uncharge @@ -183,6 +188,10 @@ free_uid find_user +#required by dynamic_readahead.ko + __traceiter_android_vh_page_cache_ra_order_bypass + __tracepoint_android_vh_page_cache_ra_order_bypass + #required by mi_freqwdg.ko __traceiter_android_vh_freq_qos_remove_request __traceiter_android_vh_freq_qos_update_request @@ -195,6 +204,10 @@ __tracepoint_android_rvh_dequeue_task_fair __tracepoint_android_rvh_entity_tick +# required by mi_damon.ko + folio_deactivate + folio_mark_accessed + #required by cpq.ko elv_rb_former_request elv_rb_latter_request @@ -210,6 +223,14 @@ elv_rqhash_add blk_req_needs_zone_write_lock __blk_req_zone_write_lock + blkcg_print_blkgs + blkg_conf_init + blkg_conf_prep + blkg_conf_exit + __blkg_prfill_u64 + blkcg_root + blkcg_activate_policy + blkcg_deactivate_policy #required by cifs.ko add_swap_extent @@ -461,6 +482,8 @@ bio_init __bio_add_page blkdev_get_by_dev + __traceiter_android_rvh_swap_bio_charge + __tracepoint_android_rvh_swap_bio_charge #required by mem_reclaim_ctl.ko __traceiter_android_vh_page_should_be_protected @@ -498,6 +521,17 @@ __traceiter_android_vh_alloc_flags_cma_adjust __tracepoint_android_vh_futex_wait_queue_start __traceiter_android_vh_futex_wait_queue_start +__traceiter_android_vh_remove_mapping +__traceiter_android_vh_remove_mapping_failed +__traceiter_android_vh_filemap_get_folio_end +__traceiter_android_vh_do_new_mount_fc +__traceiter_android_vh_put_super +__tracepoint_android_vh_remove_mapping +__tracepoint_android_vh_remove_mapping_failed +__tracepoint_android_vh_filemap_get_folio_end +__tracepoint_android_vh_do_new_mount_fc +__tracepoint_android_vh_put_super + # required by zsmalloc.ko __traceiter_android_vh_zs_shrinker_adjust __traceiter_android_vh_zs_shrinker_bypass @@ -540,3 +574,7 @@ __tracepoint_android_vh_filemap_map_pages_range __traceiter_android_vh_pageset_update __tracepoint_android_vh_pageset_update +__traceiter_android_vh_nr_pcp_alloc +__tracepoint_android_vh_nr_pcp_alloc +__traceiter_android_vh_pcp_alloc_factor_adjust +__tracepoint_android_vh_pcp_alloc_factor_adjust diff --git a/android/abi_gki_aarch64_xiaomi_wear b/android/abi_gki_aarch64_xiaomi_wear new file mode 100644 index 0000000000000..c1d890dd771e2 --- /dev/null +++ b/android/abi_gki_aarch64_xiaomi_wear @@ -0,0 +1,461 @@ +[abi_symbol_list] +# commonly used symbols + add_wait_queue + alloc_chrdev_region + alt_cb_patch_nops + __arch_copy_from_user + __arch_copy_to_user + bus_find_device + bus_register + bus_register_notifier + bus_unregister + bus_unregister_notifier + cancel_delayed_work_sync + cancel_work_sync + cdev_add + cdev_del + cdev_device_add + cdev_device_del + cdev_init + __check_object_size + class_create + class_destroy + compat_ptr_ioctl + complete + complete_all + completion_done + _copy_from_iter + _copy_to_iter + delayed_work_timer_fn + _dev_err + device_add + device_create + device_destroy + device_for_each_child + device_initialize + device_unregister + _dev_info + devm_kmalloc + dev_set_name + _dev_warn + disable_irq + driver_register + driver_unregister + enable_irq + finish_wait + fortify_panic + get_device + ida_alloc_range + ida_free + idr_alloc + idr_alloc_cyclic + idr_destroy + idr_find + idr_remove + init_net + __init_swait_queue_head + init_timer_key + init_wait_entry + __init_waitqueue_head + iov_iter_revert + jiffies + kfree + __kmalloc + kmalloc_caches + kmalloc_trace + kthread_create_on_node + kthread_should_stop + kthread_stop + ktime_get_with_offset + __list_add_valid_or_report + __list_del_entry_valid_or_report + memcpy + memmove + memset + misc_deregister + misc_register + module_layout + __msecs_to_jiffies + msleep + __mutex_init + mutex_lock + mutex_unlock + of_find_property + of_property_read_string + of_property_read_variable_u32_array + __pm_relax + __pm_stay_awake + prepare_to_wait_event + _printk + put_device + __put_task_struct + queue_delayed_work_on + queue_work_on + _raw_spin_lock_irqsave + _raw_spin_unlock_irqrestore + refcount_warn_saturate + remove_wait_queue + skb_dequeue + skb_put + skb_queue_purge_reason + skb_queue_tail + snprintf + sprintf + __stack_chk_fail + strcmp + strcspn + strlen + strncmp + strncpy + strnlen + strscpy + strstr + system_cpucaps + system_wq + unregister_chrdev_region + usleep_range_state + utf8_data_table + wait_for_completion + wait_for_completion_interruptible + wait_for_completion_interruptible_timeout + wait_for_completion_timeout + __wake_up + wake_up_process + wakeup_source_register + wakeup_source_unregister + +# required by ivshmem_rproc.ko + alloc_workqueue + destroy_workqueue + dma_set_coherent_mask + dma_set_mask + jiffies_to_usecs + memremap + memunmap + rproc_add + rproc_add_carveout + rproc_alloc + rproc_free + rproc_mem_entry_init + rproc_vq_interrupt + +# required by net_rpmsgdrv.ko + alloc_netdev_mqs + dev_get_by_name + dev_kfree_skb_any_reason + down + ether_setup + eth_type_trans + free_netdev + get_random_bytes + __netdev_alloc_skb + netdev_err + netif_carrier_off + netif_carrier_on + netif_rx + netif_tx_wake_queue + register_netdev + skb_copy_bits + skb_tstamp_tx + up + +# required by pci-ivshmem.ko + free_irq + iounmap + log_post_read_mmio + log_post_write_mmio + log_read_mmio + log_write_mmio + pci_alloc_irq_vectors + pci_clear_master + pci_disable_device + pci_enable_device + pci_free_irq_vectors + pci_ioremap_bar + pci_irq_vector + __pci_register_driver + pci_release_regions + pci_request_regions + pci_set_master + pci_unregister_driver + request_threaded_irq + __traceiter_rwmmio_post_read + __traceiter_rwmmio_post_write + __traceiter_rwmmio_read + __traceiter_rwmmio_write + __tracepoint_rwmmio_post_read + __tracepoint_rwmmio_post_write + __tracepoint_rwmmio_read + __tracepoint_rwmmio_write + +# required by rpmsg_battery.ko + power_supply_changed + power_supply_get_by_name + power_supply_register + power_supply_unregister + strrchr + +# required by rpmsg_char.ko + __alloc_skb + kfree_skb_reason + mutex_lock_interruptible + _raw_spin_lock + _raw_spin_unlock + schedule + +# required by rpmsg_core.ko + add_uevent_var + device_find_child + dev_pm_domain_attach + dev_pm_domain_detach + driver_set_override + kstrndup + of_device_modalias + of_device_uevent_modalias + of_match_device + +# required by rpmsg_fs.ko + current_time + d_add + d_alloc_name + d_drop + deactivate_locked_super + d_lookup + d_make_root + d_move + dput + full_name_hash + get_next_ino + iput + kill_anon_super + kmalloc_large + kstrdup + make_vfsgid + make_vfsuid + mount_nodev + new_inode + nop_mnt_idmap + pm_relax + pm_stay_awake + register_filesystem + simple_empty + simple_strtol + strsep + unregister_filesystem + vfree + vfs_setpos + vmalloc + +# required by rpmsg_input.ko + input_allocate_device + input_event + input_free_device + input_mt_init_slots + input_register_device + input_set_abs_params + input_set_capability + input_unregister_device + +# required by rpmsg_ping.ko + _ctype + _find_first_zero_bit + get_random_u32 + kstrtoll + ktime_get + strchr + stream_open + +# required by rpmsg_port.ko + ___ratelimit + +# required by rpmsg_port_spi.ko + crc16 + device_set_wakeup_capable + device_wakeup_disable + device_wakeup_enable + devm_gpiod_get + devm_request_threaded_irq + disable_irq_nosync + gpiod_get_value + gpiod_set_value + gpiod_to_irq + gpio_to_desc + irq_set_irq_wake + of_get_named_gpio + of_property_read_variable_u16_array + of_property_read_variable_u8_array + pm_wakeup_dev_event + spi_async + __spi_register_driver + __traceiter_spi_transfer_stop + tracepoint_probe_register + tracepoint_probe_unregister + __tracepoint_spi_transfer_stop + +# required by rpmsg_port_uart.ko + print_hex_dump + serdev_device_close + __serdev_device_driver_register + serdev_device_get_tiocm + serdev_device_open + serdev_device_set_baudrate + serdev_device_set_flow_control + serdev_device_set_tiocm + serdev_device_wait_until_sent + serdev_device_write + serdev_device_write_wakeup + stack_trace_print + stack_trace_save_tsk + +# required by rpmsg_rtc.ko + devm_rtc_allocate_device + __devm_rtc_register_device + do_settimeofday64 + ktime_get_real_ts64 + rtc_time64_to_tm + rtc_tm_to_time64 + rtc_update_irq + +# required by rpmsg_socket.ko + crc32_le + __kfifo_alloc + __kfifo_free + __kfifo_in + __kfifo_to_user + lock_sock_nested + proto_register + proto_unregister + _raw_write_lock_bh + _raw_write_unlock_bh + release_sock + security_sock_graft + sk_alloc + sk_free + sock_init_data + sock_no_socketpair + sock_register + sock_unregister + +# required by rpmsg_syslog.ko + krealloc + strnchr + +# required by rpmsg_tty.ko + __tty_alloc_driver + tty_buffer_set_limit + tty_driver_kref_put + tty_flip_buffer_push + __tty_insert_flip_string_flags + tty_kref_put + tty_port_alloc_xmit_buf + tty_port_close + tty_port_free_xmit_buf + tty_port_hangup + tty_port_init + tty_port_open + tty_port_put + tty_port_register_device_attr + tty_port_tty_get + tty_port_tty_wakeup + tty_register_driver + tty_std_termios + tty_unregister_device + tty_unregister_driver + +# required by rpmsg_usrsock.ko + idr_get_next + kernel_accept + kernel_bind + kernel_connect + kernel_getpeername + kernel_getsockname + kernel_listen + kernel_recvmsg + kernel_sendmsg + kernel_sock_shutdown + kmem_cache_alloc + kmem_cache_create + kmem_cache_destroy + kmem_cache_free + of_get_parent + sock_create_kern + sock_release + +# required by rpmsg_wakelock.ko + kobject_init_and_add + kobject_put + kobj_sysfs_ops + _raw_spin_lock_irq + _raw_spin_unlock_irq + rb_erase + rb_insert_color + +# required by trace_irqoff.ko + add_timer_on + __alloc_percpu + cpu_number + __cpu_online_mask + free_percpu + hrtimer_cancel + hrtimer_forward + hrtimer_init + hrtimer_start_range_ns + __irq_regs + kstrtobool_from_user + kstrtoul_from_user + mod_timer + __per_cpu_offset + proc_create + proc_mkdir + remove_proc_subtree + sched_clock + seq_lseek + seq_printf + seq_putc + seq_puts + seq_read + single_open + single_release + smp_call_function_single + stack_trace_save + timer_delete_sync + +# required by uio_ivshmem.ko + __uio_register_device + uio_unregister_device + +# required by uorb.ko + cancel_delayed_work + down_read + down_write + init_special_inode + iov_iter_kvec + kern_path + ns_to_timespec64 + path_put + up_read + up_write + __usecs_to_jiffies + +# required by virtio_rpmsg_bus.ko + dma_alloc_attrs + dma_free_attrs + is_vmalloc_addr + memstart_addr + pfn_is_map_memory + register_virtio_driver + schedule_timeout + sg_init_one + sg_init_table + unregister_virtio_driver + virtio_check_driver_offered_feature + virtqueue_add_inbuf + virtqueue_add_outbuf + virtqueue_disable_cb + virtqueue_enable_cb + virtqueue_get_buf + virtqueue_get_vring_size + virtqueue_kick + virtqueue_kick_prepare + virtqueue_notify + vmalloc_to_page diff --git a/android/abi_gki_aarch64_xiaomi2 b/android/abi_gki_aarch64_xiaomi_xring similarity index 99% rename from android/abi_gki_aarch64_xiaomi2 rename to android/abi_gki_aarch64_xiaomi_xring index c27a275ed9370..bbaae6d45165b 100644 --- a/android/abi_gki_aarch64_xiaomi2 +++ b/android/abi_gki_aarch64_xiaomi_xring @@ -1911,6 +1911,7 @@ scsi_report_bus_reset scsi_scan_host scsi_unblock_requests + scsi_host_busy sdev_prefix_printk security_file_ioctl select_fallback_rq diff --git a/android/abi_gki_protected_exports_aarch64 b/android/abi_gki_protected_exports_aarch64 index 9e4b937ac9023..e45a05f3898fe 100644 --- a/android/abi_gki_protected_exports_aarch64 +++ b/android/abi_gki_protected_exports_aarch64 @@ -200,6 +200,7 @@ l2cap_chan_close l2cap_chan_connect l2cap_chan_create l2cap_chan_del +l2cap_chan_hold l2cap_chan_list l2cap_chan_put l2cap_chan_send diff --git a/android/gki/aarch64/afdo/README.md b/android/gki/aarch64/afdo/README.md index 8c7781fd62ecc..dbb3686f32a48 100644 --- a/android/gki/aarch64/afdo/README.md +++ b/android/gki/aarch64/afdo/README.md @@ -1,25 +1,36 @@ # AutoFDO profiles for Android common kernels -This directory contains AutoFDO profiles for Android common kernels. These profiles can be used to -optimize kernel builds for specific architectures and kernel versions. +This directory contains AutoFDO profiles for Android common kernels. These profiles are used to +optimize kernel builds, improving performance for specific architectures and kernel versions. ## kernel.afdo -kernel.afdo is an AArch64 kernel profile collected on kernel version 6.6.82 ( -SHA b62ea68f41a901d5f07f48bd6f1d3a117d801411, build server ID 13287877) using Pixel 6. +The AutoFDO profile (kernel.afdo) for vmlinux is collected on kernel version 6.6.111 ( +SHA f75074e8249d905be0ac5a997aadfc68393e8a15, build server ID 14413421) using Pixel 10. -### Performance improvements +## Performance improvements + +The following benchmarks demonstrate the performance gains from the latest AutoFDO profile. +Please note that results can fluctuate slightly between updates due to inherent testing noise. +A profile is considered successful when it yields positive improvements across the majority of +benchmarks. Tests were performed on Pixel 6 and Pixel 10. | Benchmark | Improvement | | --------------------- | ----------- | -| Boot time | 2.2% | -| Cold App launch time | 2.7% | -| Binder-rpc | 4.4% | -| Binder-addints | 14.1% | -| Hwbinder | 17.0% | -| Bionic (syscall_mmap) | 1.6% | +| Boot time | 2.1% | +| Cold App launch time | 3.7% | +| Binder-rpc | 6.7% | +| Binder-addints | 14.5% | +| Hwbinder | 7.8% | +| Bionic (syscall_mmap) | 3.8% | + +Other improvements tested on past profiles: -Benchmark results were tested on Pixel 6. +| Benchmark | Improvement | +| ---------------- | ----------- | +| Bionic (pthread) | 1.9% | +| Bionic (stdio) | 5.4% | +| Bionic (all) | 2.9% | To test a kernel prebuilt with the AutoFDO profile, navigate to [Android build server]( https://ci.android.com/builds/branches/aosp_kernel-common-android15-6.6/grid) and download @@ -31,11 +42,11 @@ A kernel profile is generated by running app crawling and app launching for top Play Store. While running, we collect ETM data for the kernel, which records executed instruction stream. Finally, we merge and convert ETM data to one AutoFDO profile. -1. Build a kernel image and flash it on an Android device +### 1. Build a kernel image and flash it on an Android device * The source code and test device used to generate each profile are described above. * We use a Pixel device. But using other real devices should get a similar profile. -2. Run app crawling and app launching for top 100 apps +### 2. Run app crawling and app launching for top 100 apps * Add a gmail account on the test device. Because app crawler can use the account to automatically login some of the apps. * We run [App Crawler](https://developer.android.com/studio/test/other-testing-tools/app-crawler) @@ -43,5 +54,36 @@ stream. Finally, we merge and convert ETM data to one AutoFDO profile. * We run app launching for one app for 3 seconds, and run it 15 times. After each running, the app is killed and cache is cleared. So we get profile for cold app startups. -3. Record ETM data while running app crawling and app launching. - * We use cmdline `simpleperf record -e cs-etm:k -a` to [record ETM data for the kernel](https://android.googlesource.com/platform/system/extras/+/master/simpleperf/doc/collect_etm_data_for_autofdo.md). +### 3. Generate the AutoFDO profile + +First, collect performance data using simpleperf with Coresight ETM or ARM ETE while running app +crawling and app launching scenarios. Following is an example recording kernel activity for 180 +seconds. For a complete guide, refer to [Record ETM data for the kernel](https://android.googlesource.com/platform/system/extras/+/refs/heads/android16-release/simpleperf/doc/collect_etm_data_for_autofdo.md#A-complete-example_kernel). + +```sh +(device) / $ simpleperf record -e cs-etm:k -a --duration 180 -z -o perf.data +``` + +Next, convert the ETM data to an AutoFDO profile using `simpleperf inject` and `create_llvm_prof` +on the host machine. For a complete guide, refer to [Convert ETM data to AutoFDO Profile on Host](https://android.googlesource.com/platform/system/extras/+/refs/heads/android16-release/simpleperf/doc/collect_etm_data_for_autofdo.md#A-complete-example_kernel). + +Note that `create_llvm_prof` recently enabled the symbol list by default, which can cause Clang +to de-optimize any kernel function not listed in the profile. This is usually not we want in the +kernel, unless the profile cover all critical paths. So we recommend adding `--prof_sym_list=false` +to avoid de-optimization. This also prevents the following build error from section mismatch. + +``` +WARNING: modpost: vmlinux: section mismatch in reference: list_add+0x0 (section: .text.hot.list_add) -> dir_list (section: .init.data) +``` + +We use the following command to perform the conversion. + +```sh +# Convert the AutoFDO profile to the LLVM profile format: +# The --prof_sym_list=false flag is important for kernel profiles. Without it, clang +# assumes any function not listed in the profile is cold. This can lead to unwanted +# deoptimizations, even when -fprofile-sample-accurate is not enabled. +(host) $ create_llvm_prof --profiler=text --binary={vmlinux_dir}/vmlinux \ + --profile=kernel.autofdo --format=extbinary --use_fs_discriminator \ + --out=kernel.afdo --prof_sym_list=false +``` diff --git a/android/gki/aarch64/afdo/kernel.afdo b/android/gki/aarch64/afdo/kernel.afdo index 3ce73ba9446c1..45fd7bce1e2c0 100644 Binary files a/android/gki/aarch64/afdo/kernel.afdo and b/android/gki/aarch64/afdo/kernel.afdo differ diff --git a/arch/alpha/kernel/asm-offsets.c b/arch/alpha/kernel/asm-offsets.c index 11c35cf45b461..cb205f22096d7 100644 --- a/arch/alpha/kernel/asm-offsets.c +++ b/arch/alpha/kernel/asm-offsets.c @@ -4,6 +4,7 @@ * This code generates raw asm output which is post-processed to extract * and format the required data. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/arc/include/asm/bitops.h b/arch/arc/include/asm/bitops.h index f5a936496f060..24981bba974d3 100644 --- a/arch/arc/include/asm/bitops.h +++ b/arch/arc/include/asm/bitops.h @@ -133,6 +133,8 @@ static inline __attribute__ ((const)) int fls(unsigned int x) */ static inline __attribute__ ((const)) unsigned long __fls(unsigned long x) { + if (__builtin_constant_p(x)) + return x ? BITS_PER_LONG - 1 - __builtin_clzl(x) : 0; /* FLS insn has exactly same semantics as the API */ return __builtin_arc_fls(x); } diff --git a/arch/arc/kernel/asm-offsets.c b/arch/arc/kernel/asm-offsets.c index f77deb7991757..2978da85fcb65 100644 --- a/arch/arc/kernel/asm-offsets.c +++ b/arch/arc/kernel/asm-offsets.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 5ba42f69f8ce0..351d0b039b5ca 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -142,7 +142,7 @@ endif # Need -Uarm for gcc < 3.x KBUILD_CPPFLAGS +=$(cpp-y) KBUILD_CFLAGS +=$(CFLAGS_ABI) $(CFLAGS_ISA) $(arch-y) $(tune-y) $(call cc-option,-mshort-load-bytes,$(call cc-option,-malignment-traps,)) -msoft-float -Uarm -KBUILD_AFLAGS +=$(CFLAGS_ABI) $(AFLAGS_ISA) -Wa,$(arch-y) $(tune-y) -include asm/unified.h -msoft-float +KBUILD_AFLAGS +=$(CFLAGS_ABI) $(AFLAGS_ISA) -Wa,$(arch-y) $(tune-y) -include $(srctree)/arch/arm/include/asm/unified.h -msoft-float CHECKFLAGS += -D__arm__ diff --git a/arch/arm/boot/dts/broadcom/bcm47189-luxul-xap-1440.dts b/arch/arm/boot/dts/broadcom/bcm47189-luxul-xap-1440.dts index ac44c745bdf8e..a39a021a39107 100644 --- a/arch/arm/boot/dts/broadcom/bcm47189-luxul-xap-1440.dts +++ b/arch/arm/boot/dts/broadcom/bcm47189-luxul-xap-1440.dts @@ -55,8 +55,8 @@ mdio { /delete-node/ switch@1e; - bcm54210e: ethernet-phy@0 { - reg = <0>; + bcm54210e: ethernet-phy@25 { + reg = <25>; }; }; }; diff --git a/arch/arm/boot/dts/intel/socfpga/socfpga_cyclone5_sodia.dts b/arch/arm/boot/dts/intel/socfpga/socfpga_cyclone5_sodia.dts index 2564671fc1c67..59745ba9c8dab 100644 --- a/arch/arm/boot/dts/intel/socfpga/socfpga_cyclone5_sodia.dts +++ b/arch/arm/boot/dts/intel/socfpga/socfpga_cyclone5_sodia.dts @@ -66,8 +66,10 @@ mdio0 { #address-cells = <1>; #size-cells = <0>; - phy0: ethernet-phy@0 { - reg = <0>; + compatible = "snps,dwmac-mdio"; + + phy0: ethernet-phy@4 { + reg = <4>; rxd0-skew-ps = <0>; rxd1-skew-ps = <0>; rxd2-skew-ps = <0>; diff --git a/arch/arm/boot/dts/marvell/kirkwood-openrd-client.dts b/arch/arm/boot/dts/marvell/kirkwood-openrd-client.dts index d4e0b8150a84c..cf26e2ceaaa07 100644 --- a/arch/arm/boot/dts/marvell/kirkwood-openrd-client.dts +++ b/arch/arm/boot/dts/marvell/kirkwood-openrd-client.dts @@ -38,7 +38,7 @@ simple-audio-card,mclk-fs = <256>; simple-audio-card,cpu { - sound-dai = <&audio0 0>; + sound-dai = <&audio0>; }; simple-audio-card,codec { diff --git a/arch/arm/boot/dts/microchip/at91sam9263ek.dts b/arch/arm/boot/dts/microchip/at91sam9263ek.dts index ce8baff6a9f4e..e42e1a75a715d 100644 --- a/arch/arm/boot/dts/microchip/at91sam9263ek.dts +++ b/arch/arm/boot/dts/microchip/at91sam9263ek.dts @@ -152,7 +152,7 @@ nand@3 { reg = <0x3 0x0 0x800000>; rb-gpios = <&pioA 22 GPIO_ACTIVE_HIGH>; - cs-gpios = <&pioA 15 GPIO_ACTIVE_HIGH>; + cs-gpios = <&pioD 15 GPIO_ACTIVE_HIGH>; nand-bus-width = <8>; nand-ecc-mode = "soft"; nand-on-flash-bbt; diff --git a/arch/arm/boot/dts/microchip/tny_a9263.dts b/arch/arm/boot/dts/microchip/tny_a9263.dts index 62b7d9f9a926c..c8b6318aaa838 100644 --- a/arch/arm/boot/dts/microchip/tny_a9263.dts +++ b/arch/arm/boot/dts/microchip/tny_a9263.dts @@ -64,7 +64,7 @@ nand@3 { reg = <0x3 0x0 0x800000>; rb-gpios = <&pioA 22 GPIO_ACTIVE_HIGH>; - cs-gpios = <&pioA 15 GPIO_ACTIVE_HIGH>; + cs-gpios = <&pioD 15 GPIO_ACTIVE_HIGH>; nand-bus-width = <8>; nand-ecc-mode = "soft"; nand-on-flash-bbt; diff --git a/arch/arm/boot/dts/microchip/usb_a9263.dts b/arch/arm/boot/dts/microchip/usb_a9263.dts index 45745915b2e16..454176ce6d3ff 100644 --- a/arch/arm/boot/dts/microchip/usb_a9263.dts +++ b/arch/arm/boot/dts/microchip/usb_a9263.dts @@ -58,7 +58,7 @@ }; spi0: spi@fffa4000 { - cs-gpios = <&pioB 15 GPIO_ACTIVE_HIGH>; + cs-gpios = <&pioA 5 GPIO_ACTIVE_LOW>; status = "okay"; flash@0 { compatible = "atmel,at45", "atmel,dataflash"; @@ -84,7 +84,7 @@ nand@3 { reg = <0x3 0x0 0x800000>; rb-gpios = <&pioA 22 GPIO_ACTIVE_HIGH>; - cs-gpios = <&pioA 15 GPIO_ACTIVE_HIGH>; + cs-gpios = <&pioD 15 GPIO_ACTIVE_HIGH>; nand-bus-width = <8>; nand-ecc-mode = "soft"; nand-on-flash-bbt; diff --git a/arch/arm/boot/dts/nvidia/tegra114.dtsi b/arch/arm/boot/dts/nvidia/tegra114.dtsi index 86f14e2fd29f3..6c057b5069514 100644 --- a/arch/arm/boot/dts/nvidia/tegra114.dtsi +++ b/arch/arm/boot/dts/nvidia/tegra114.dtsi @@ -139,7 +139,7 @@ reg = <0x54400000 0x00040000>; clocks = <&tegra_car TEGRA114_CLK_DSIB>, <&tegra_car TEGRA114_CLK_DSIBLP>, - <&tegra_car TEGRA114_CLK_PLL_D2_OUT0>; + <&tegra_car TEGRA114_CLK_PLL_D_OUT0>; clock-names = "dsi", "lp", "parent"; resets = <&tegra_car 82>; reset-names = "dsi"; diff --git a/arch/arm/boot/dts/nvidia/tegra20-asus-tf101.dts b/arch/arm/boot/dts/nvidia/tegra20-asus-tf101.dts index a3757b7daeda4..071a31191ec27 100644 --- a/arch/arm/boot/dts/nvidia/tegra20-asus-tf101.dts +++ b/arch/arm/boot/dts/nvidia/tegra20-asus-tf101.dts @@ -502,6 +502,9 @@ compatible = "asahi-kasei,ak8974"; reg = <0xe>; + interrupt-parent = <&gpio>; + interrupts = ; + avdd-supply = <&vdd_3v3_sys>; dvdd-supply = <&vdd_1v8_sys>; @@ -515,7 +518,7 @@ reg = <0x1a>; interrupt-parent = <&gpio>; - interrupts = ; + interrupts = ; gpio-controller; #gpio-cells = <2>; diff --git a/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts b/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts index 5d4b29d765853..6cc4c2f08b15d 100644 --- a/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts +++ b/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts @@ -259,7 +259,7 @@ pinctrl-0 = <&pinctrl_audmux>; status = "okay"; - ssi2 { + mux-ssi2 { fsl,audmux-port = <1>; fsl,port-config = < (IMX_AUDMUX_V2_PTCR_SYN | @@ -271,7 +271,7 @@ >; }; - aud3 { + mux-aud3 { fsl,audmux-port = <2>; fsl,port-config = < IMX_AUDMUX_V2_PTCR_SYN diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-imx6ull-opos6ul.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-imx6ull-opos6ul.dtsi index f2386dcb9ff2c..dda4fa91b2f2c 100644 --- a/arch/arm/boot/dts/nxp/imx/imx6ul-imx6ull-opos6ul.dtsi +++ b/arch/arm/boot/dts/nxp/imx/imx6ul-imx6ull-opos6ul.dtsi @@ -40,6 +40,9 @@ reg = <1>; interrupt-parent = <&gpio4>; interrupts = <16 IRQ_TYPE_LEVEL_LOW>; + micrel,led-mode = <1>; + clocks = <&clks IMX6UL_CLK_ENET_REF>; + clock-names = "rmii-ref"; status = "okay"; }; }; diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi index 33d5f27285a47..9ece280d163af 100644 --- a/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi +++ b/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi @@ -169,7 +169,6 @@ pinctrl-0 = <&pinctrl_uart2>; linux,rs485-enabled-at-boot-time; rs485-rx-during-tx; - rs485-rts-active-low; uart-has-rtscts; status = "okay"; }; diff --git a/arch/arm/boot/dts/nxp/vf/vfxxx.dtsi b/arch/arm/boot/dts/nxp/vf/vfxxx.dtsi index d1095b700c565..d96148abf5bb6 100644 --- a/arch/arm/boot/dts/nxp/vf/vfxxx.dtsi +++ b/arch/arm/boot/dts/nxp/vf/vfxxx.dtsi @@ -612,7 +612,7 @@ ftm: ftm@400b8000 { compatible = "fsl,ftm-timer"; - reg = <0x400b8000 0x1000 0x400b9000 0x1000>; + reg = <0x400b8000 0x1000>, <0x400b9000 0x1000>; interrupts = <44 IRQ_TYPE_LEVEL_HIGH>; clock-names = "ftm-evt", "ftm-src", "ftm-evt-counter-en", "ftm-src-counter-en"; diff --git a/arch/arm/boot/dts/qcom/qcom-apq8064.dtsi b/arch/arm/boot/dts/qcom/qcom-apq8064.dtsi index 950adb63af701..b8f160cfe8e15 100644 --- a/arch/arm/boot/dts/qcom/qcom-apq8064.dtsi +++ b/arch/arm/boot/dts/qcom/qcom-apq8064.dtsi @@ -213,12 +213,6 @@ }; }; - sfpb_mutex: hwmutex { - compatible = "qcom,sfpb-mutex"; - syscon = <&sfpb_wrapper_mutex 0x604 0x4>; - #hwlock-cells = <1>; - }; - smem { compatible = "qcom,smem"; memory-region = <&smem_region>; @@ -322,9 +316,10 @@ pinctrl-0 = <&ps_hold>; }; - sfpb_wrapper_mutex: syscon@1200000 { - compatible = "syscon"; - reg = <0x01200000 0x8000>; + sfpb_mutex: hwmutex@1200600 { + compatible = "qcom,sfpb-mutex"; + reg = <0x01200600 0x100>; + #hwlock-cells = <1>; }; intc: interrupt-controller@2000000 { @@ -343,6 +338,8 @@ <1 3 0x301>; reg = <0x0200a000 0x100>; clock-frequency = <27000000>; + clocks = <&sleep_clk>; + clock-names = "sleep"; cpu-offset = <0x80000>; }; diff --git a/arch/arm/boot/dts/renesas/r8a7791-porter.dts b/arch/arm/boot/dts/renesas/r8a7791-porter.dts index fcc9a2313e1df..0cd08f7b8d8e0 100644 --- a/arch/arm/boot/dts/renesas/r8a7791-porter.dts +++ b/arch/arm/boot/dts/renesas/r8a7791-porter.dts @@ -266,7 +266,7 @@ }; can0_pins: can0 { - groups = "can0_data"; + groups = "can0_data_b"; function = "can0"; }; diff --git a/arch/arm/boot/dts/ti/omap/am335x-baltos.dtsi b/arch/arm/boot/dts/ti/omap/am335x-baltos.dtsi index c14d5b70c72f6..56f704082f94a 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-baltos.dtsi +++ b/arch/arm/boot/dts/ti/omap/am335x-baltos.dtsi @@ -270,7 +270,7 @@ vcc7-supply = <&vbat>; vccio-supply = <&vbat>; - ti,en-ck32k-xtal = <1>; + ti,en-ck32k-xtal; regulators { vrtc_reg: regulator@0 { diff --git a/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi b/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi index 96451c8a815cc..27e73e745e25f 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi +++ b/arch/arm/boot/dts/ti/omap/am335x-bone-common.dtsi @@ -385,7 +385,7 @@ /* Support GPIO reset on revision C3 boards */ reset-gpios = <&gpio1 8 GPIO_ACTIVE_LOW>; reset-assert-us = <300>; - reset-deassert-us = <6500>; + reset-deassert-us = <50000>; }; }; diff --git a/arch/arm/boot/dts/ti/omap/am335x-boneblack.dts b/arch/arm/boot/dts/ti/omap/am335x-boneblack.dts index 16b567e3cb472..b4fdcf9c02b50 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-boneblack.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-boneblack.dts @@ -35,7 +35,7 @@ "P9_18 [spi0_d1]", "P9_17 [spi0_cs0]", "[mmc0_cd]", - "P8_42A [ecappwm0]", + "P9_42A [ecappwm0]", "P8_35 [lcd d12]", "P8_33 [lcd d13]", "P8_31 [lcd d14]", diff --git a/arch/arm/boot/dts/ti/omap/am335x-cm-t335.dts b/arch/arm/boot/dts/ti/omap/am335x-cm-t335.dts index 72990e7ffe10e..44718a4cec474 100644 --- a/arch/arm/boot/dts/ti/omap/am335x-cm-t335.dts +++ b/arch/arm/boot/dts/ti/omap/am335x-cm-t335.dts @@ -483,8 +483,6 @@ status = "okay"; op-mode = <0>; /* MCASP_IIS_MODE */ tdm-slots = <2>; - /* 16 serializers */ - num-serializer = <16>; serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ 0 0 2 1 0 0 0 0 0 0 0 0 0 0 0 0 >; diff --git a/arch/arm/boot/dts/ti/omap/omap3-devkit8000-lcd-common.dtsi b/arch/arm/boot/dts/ti/omap/omap3-devkit8000-lcd-common.dtsi index a7f99ae0c1fe9..78c657429f641 100644 --- a/arch/arm/boot/dts/ti/omap/omap3-devkit8000-lcd-common.dtsi +++ b/arch/arm/boot/dts/ti/omap/omap3-devkit8000-lcd-common.dtsi @@ -65,7 +65,7 @@ ti,debounce-max = /bits/ 16 <10>; ti,debounce-tol = /bits/ 16 <5>; ti,debounce-rep = /bits/ 16 <1>; - ti,keep-vref-on = <1>; + ti,keep-vref-on; ti,settle-delay-usec = /bits/ 16 <150>; wakeup-source; diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 847b7a0033569..1f684e29cff2e 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -4,7 +4,7 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm)" config CRYPTO_CURVE25519_NEON tristate "Public key crypto: Curve25519 (NEON)" - depends on KERNEL_MODE_NEON + depends on KERNEL_MODE_NEON && !CPU_BIG_ENDIAN select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_ARCH_HAVE_LIB_CURVE25519 help diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index 0ca94b90bc4ec..ba98daeb119cd 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -245,7 +245,7 @@ static int ctr_encrypt(struct skcipher_request *req) while (walk.nbytes > 0) { const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; - int bytes = walk.nbytes; + unsigned int bytes = walk.nbytes; if (unlikely(bytes < AES_BLOCK_SIZE)) src = dst = memcpy(buf + sizeof(buf) - bytes, diff --git a/arch/arm/include/asm/ptrace.h b/arch/arm/include/asm/ptrace.h index 7f44e88d1f25b..14a38cc67e0bc 100644 --- a/arch/arm/include/asm/ptrace.h +++ b/arch/arm/include/asm/ptrace.h @@ -10,6 +10,7 @@ #include #ifndef __ASSEMBLY__ +#include #include struct pt_regs { @@ -35,8 +36,8 @@ struct svc_pt_regs { #ifndef CONFIG_CPU_V7M #define isa_mode(regs) \ - ((((regs)->ARM_cpsr & PSR_J_BIT) >> (__ffs(PSR_J_BIT) - 1)) | \ - (((regs)->ARM_cpsr & PSR_T_BIT) >> (__ffs(PSR_T_BIT)))) + (FIELD_GET(PSR_J_BIT, (regs)->ARM_cpsr) << 1 | \ + FIELD_GET(PSR_T_BIT, (regs)->ARM_cpsr)) #else #define isa_mode(regs) 1 /* Thumb */ #endif diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index c7d2510e5a786..853c4f81ba4a5 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -13,6 +13,7 @@ #define arch_set_freq_scale topology_set_freq_scale #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant +#define arch_scale_freq_ref topology_get_freq_ref #endif /* Replace task scheduler's default cpu-invariant accounting */ diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 219cbc7e5d134..3840e1e22b751 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -7,6 +7,8 @@ * This code generates raw asm output which is post-processed to extract * and format the required data. */ +#define COMPILE_OFFSETS + #include #include #include diff --git a/arch/arm/mach-aspeed/Kconfig b/arch/arm/mach-aspeed/Kconfig index 080019aa6fcd8..fcf287edd0e5e 100644 --- a/arch/arm/mach-aspeed/Kconfig +++ b/arch/arm/mach-aspeed/Kconfig @@ -2,7 +2,6 @@ menuconfig ARCH_ASPEED bool "Aspeed BMC architectures" depends on (CPU_LITTLE_ENDIAN && ARCH_MULTI_V5) || ARCH_MULTI_V6 || ARCH_MULTI_V7 - select SRAM select WATCHDOG select ASPEED_WATCHDOG select MFD_SYSCON diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c index 22ecaf09d00f9..f635ad29511f6 100644 --- a/arch/arm/mach-at91/pm.c +++ b/arch/arm/mach-at91/pm.c @@ -538,11 +538,12 @@ extern u32 at91_pm_suspend_in_sram_sz; static int at91_suspend_finish(unsigned long val) { - unsigned char modified_gray_code[] = { - 0x00, 0x01, 0x02, 0x03, 0x06, 0x07, 0x04, 0x05, 0x0c, 0x0d, - 0x0e, 0x0f, 0x0a, 0x0b, 0x08, 0x09, 0x18, 0x19, 0x1a, 0x1b, - 0x1e, 0x1f, 0x1c, 0x1d, 0x14, 0x15, 0x16, 0x17, 0x12, 0x13, - 0x10, 0x11, + /* SYNOPSYS workaround to fix a bug in the calibration logic */ + unsigned char modified_fix_code[] = { + 0x00, 0x01, 0x01, 0x06, 0x07, 0x0c, 0x06, 0x07, 0x0b, 0x18, + 0x0a, 0x0b, 0x0c, 0x0d, 0x0d, 0x0a, 0x13, 0x13, 0x12, 0x13, + 0x14, 0x15, 0x15, 0x12, 0x18, 0x19, 0x19, 0x1e, 0x1f, 0x14, + 0x1e, 0x1f, }; unsigned int tmp, index; int i; @@ -553,25 +554,25 @@ static int at91_suspend_finish(unsigned long val) * restore the ZQ0SR0 with the value saved here. But the * calibration is buggy and restoring some values from ZQ0SR0 * is forbidden and risky thus we need to provide processed - * values for these (modified gray code values). + * values for these. */ tmp = readl(soc_pm.data.ramc_phy + DDR3PHY_ZQ0SR0); /* Store pull-down output impedance select. */ index = (tmp >> DDR3PHY_ZQ0SR0_PDO_OFF) & 0x1f; - soc_pm.bu->ddr_phy_calibration[0] = modified_gray_code[index]; + soc_pm.bu->ddr_phy_calibration[0] = modified_fix_code[index] << DDR3PHY_ZQ0SR0_PDO_OFF; /* Store pull-up output impedance select. */ index = (tmp >> DDR3PHY_ZQ0SR0_PUO_OFF) & 0x1f; - soc_pm.bu->ddr_phy_calibration[0] |= modified_gray_code[index]; + soc_pm.bu->ddr_phy_calibration[0] |= modified_fix_code[index] << DDR3PHY_ZQ0SR0_PUO_OFF; /* Store pull-down on-die termination impedance select. */ index = (tmp >> DDR3PHY_ZQ0SR0_PDODT_OFF) & 0x1f; - soc_pm.bu->ddr_phy_calibration[0] |= modified_gray_code[index]; + soc_pm.bu->ddr_phy_calibration[0] |= modified_fix_code[index] << DDR3PHY_ZQ0SR0_PDODT_OFF; /* Store pull-up on-die termination impedance select. */ index = (tmp >> DDR3PHY_ZQ0SRO_PUODT_OFF) & 0x1f; - soc_pm.bu->ddr_phy_calibration[0] |= modified_gray_code[index]; + soc_pm.bu->ddr_phy_calibration[0] |= modified_fix_code[index] << DDR3PHY_ZQ0SRO_PUODT_OFF; /* * The 1st 8 words of memory might get corrupted in the process diff --git a/arch/arm/mach-at91/pm_suspend.S b/arch/arm/mach-at91/pm_suspend.S index e5869cca5e791..99aaf5cf89696 100644 --- a/arch/arm/mach-at91/pm_suspend.S +++ b/arch/arm/mach-at91/pm_suspend.S @@ -689,6 +689,10 @@ sr_dis_exit: bic tmp2, tmp2, #AT91_PMC_PLL_UPDT_ID str tmp2, [pmc, #AT91_PMC_PLL_UPDT] + /* save acr */ + ldr tmp2, [pmc, #AT91_PMC_PLL_ACR] + str tmp2, .saved_acr + /* save div. */ mov tmp1, #0 ldr tmp2, [pmc, #AT91_PMC_PLL_CTRL0] @@ -758,7 +762,7 @@ sr_dis_exit: str tmp1, [pmc, #AT91_PMC_PLL_UPDT] /* step 2. */ - ldr tmp1, =AT91_PMC_PLL_ACR_DEFAULT_PLLA + ldr tmp1, .saved_acr str tmp1, [pmc, #AT91_PMC_PLL_ACR] /* step 3. */ @@ -872,7 +876,7 @@ e_done: /** * at91_mckx_ps_restore: restore MCK1..4 settings * - * Side effects: overwrites tmp1, tmp2 + * Side effects: overwrites tmp1, tmp2 and tmp3 */ .macro at91_mckx_ps_restore #ifdef CONFIG_SOC_SAMA7 @@ -916,7 +920,7 @@ r_ps: bic tmp3, tmp3, #AT91_PMC_MCR_V2_ID_MSK orr tmp3, tmp3, tmp1 orr tmp3, tmp3, #AT91_PMC_MCR_V2_CMD - str tmp2, [pmc, #AT91_PMC_MCR_V2] + str tmp3, [pmc, #AT91_PMC_MCR_V2] wait_mckrdy tmp1 @@ -1134,6 +1138,8 @@ ENDPROC(at91_pm_suspend_in_sram) .word 0 .saved_mckr: .word 0 +.saved_acr: + .word 0 .saved_pllar: .word 0 .saved_sam9_lpr: diff --git a/arch/arm/mach-bcm/Kconfig b/arch/arm/mach-bcm/Kconfig index 8789d93a7c04b..c705bec1410c5 100644 --- a/arch/arm/mach-bcm/Kconfig +++ b/arch/arm/mach-bcm/Kconfig @@ -186,6 +186,7 @@ config ARCH_BRCMSTB select ARCH_HAS_RESET_CONTROLLER select ARM_AMBA select ARM_GIC + select ARM_GIC_V3 select ARM_ERRATA_798181 if SMP select HAVE_ARM_ARCH_TIMER select ZONE_DMA if ARM_LPAE diff --git a/arch/arm/mach-omap2/clockdomain.h b/arch/arm/mach-omap2/clockdomain.h index c36fb27212615..86a2f9e5d0ef9 100644 --- a/arch/arm/mach-omap2/clockdomain.h +++ b/arch/arm/mach-omap2/clockdomain.h @@ -48,6 +48,7 @@ #define CLKDM_NO_AUTODEPS (1 << 4) #define CLKDM_ACTIVE_WITH_MPU (1 << 5) #define CLKDM_MISSING_IDLE_REPORTING (1 << 6) +#define CLKDM_STANDBY_FORCE_WAKEUP BIT(7) #define CLKDM_CAN_HWSUP (CLKDM_CAN_ENABLE_AUTO | CLKDM_CAN_DISABLE_AUTO) #define CLKDM_CAN_SWSUP (CLKDM_CAN_FORCE_SLEEP | CLKDM_CAN_FORCE_WAKEUP) diff --git a/arch/arm/mach-omap2/clockdomains33xx_data.c b/arch/arm/mach-omap2/clockdomains33xx_data.c index 87f4e927eb183..c05a3c07d4486 100644 --- a/arch/arm/mach-omap2/clockdomains33xx_data.c +++ b/arch/arm/mach-omap2/clockdomains33xx_data.c @@ -19,7 +19,7 @@ static struct clockdomain l4ls_am33xx_clkdm = { .pwrdm = { .name = "per_pwrdm" }, .cm_inst = AM33XX_CM_PER_MOD, .clkdm_offs = AM33XX_CM_PER_L4LS_CLKSTCTRL_OFFSET, - .flags = CLKDM_CAN_SWSUP, + .flags = CLKDM_CAN_SWSUP | CLKDM_STANDBY_FORCE_WAKEUP, }; static struct clockdomain l3s_am33xx_clkdm = { diff --git a/arch/arm/mach-omap2/cm33xx.c b/arch/arm/mach-omap2/cm33xx.c index c824d4e3db632..aaee67d097915 100644 --- a/arch/arm/mach-omap2/cm33xx.c +++ b/arch/arm/mach-omap2/cm33xx.c @@ -20,6 +20,9 @@ #include "cm-regbits-34xx.h" #include "cm-regbits-33xx.h" #include "prm33xx.h" +#if IS_ENABLED(CONFIG_SUSPEND) +#include +#endif /* * CLKCTRL_IDLEST_*: possible values for the CM_*_CLKCTRL.IDLEST bitfield: @@ -328,8 +331,17 @@ static int am33xx_clkdm_clk_disable(struct clockdomain *clkdm) { bool hwsup = false; +#if IS_ENABLED(CONFIG_SUSPEND) + /* + * In case of standby, Don't put the l4ls clk domain to sleep. + * Since CM3 PM FW doesn't wake-up/enable the l4ls clk domain + * upon wake-up, CM3 PM FW fails to wake-up th MPU. + */ + if (pm_suspend_target_state == PM_SUSPEND_STANDBY && + (clkdm->flags & CLKDM_STANDBY_FORCE_WAKEUP)) + return 0; +#endif hwsup = am33xx_cm_is_clkdm_in_hwsup(clkdm->cm_inst, clkdm->clkdm_offs); - if (!hwsup && (clkdm->flags & CLKDM_CAN_FORCE_SLEEP)) am33xx_clkdm_sleep(clkdm); diff --git a/arch/arm/mach-omap2/pm33xx-core.c b/arch/arm/mach-omap2/pm33xx-core.c index c907478be196e..4abb86dc98fda 100644 --- a/arch/arm/mach-omap2/pm33xx-core.c +++ b/arch/arm/mach-omap2/pm33xx-core.c @@ -388,12 +388,15 @@ static int __init amx3_idle_init(struct device_node *cpu_node, int cpu) if (!state_node) break; - if (!of_device_is_available(state_node)) + if (!of_device_is_available(state_node)) { + of_node_put(state_node); continue; + } if (i == CPUIDLE_STATE_MAX) { pr_warn("%s: cpuidle states reached max possible\n", __func__); + of_node_put(state_node); break; } @@ -403,6 +406,7 @@ static int __init amx3_idle_init(struct device_node *cpu_node, int cpu) states[state_count].wfi_flags |= WFI_FLAG_WAKE_M3 | WFI_FLAG_FLUSH_CACHE; + of_node_put(state_node); state_count++; } diff --git a/arch/arm/mach-omap2/pmic-cpcap.c b/arch/arm/mach-omap2/pmic-cpcap.c index 668dc84fd31e0..527cf4b7e3787 100644 --- a/arch/arm/mach-omap2/pmic-cpcap.c +++ b/arch/arm/mach-omap2/pmic-cpcap.c @@ -264,7 +264,11 @@ int __init omap4_cpcap_init(void) static int __init cpcap_late_init(void) { - omap4_vc_set_pmic_signaling(PWRDM_POWER_RET); + if (!of_find_compatible_node(NULL, NULL, "motorola,cpcap")) + return 0; + + if (soc_is_omap443x() || soc_is_omap446x() || soc_is_omap447x()) + omap4_vc_set_pmic_signaling(PWRDM_POWER_RET); return 0; } diff --git a/arch/arm/mach-rockchip/platsmp.c b/arch/arm/mach-rockchip/platsmp.c index 36915a073c234..f432d22bfed84 100644 --- a/arch/arm/mach-rockchip/platsmp.c +++ b/arch/arm/mach-rockchip/platsmp.c @@ -279,11 +279,6 @@ static void __init rockchip_smp_prepare_cpus(unsigned int max_cpus) } if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9) { - if (rockchip_smp_prepare_sram(node)) { - of_node_put(node); - return; - } - /* enable the SCU power domain */ pmu_set_power_domain(PMU_PWRDN_SCU, true); @@ -316,11 +311,19 @@ static void __init rockchip_smp_prepare_cpus(unsigned int max_cpus) asm ("mrc p15, 1, %0, c9, c0, 2\n" : "=r" (l2ctlr)); ncores = ((l2ctlr >> 24) & 0x3) + 1; } - of_node_put(node); /* Make sure that all cores except the first are really off */ for (i = 1; i < ncores; i++) pmu_set_power_domain(0 + i, false); + + if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9) { + if (rockchip_smp_prepare_sram(node)) { + of_node_put(node); + return; + } + } + + of_node_put(node); } static void __init rk3036_smp_prepare_cpus(unsigned int max_cpus) diff --git a/arch/arm/mach-tegra/reset.c b/arch/arm/mach-tegra/reset.c index d5c805adf7a82..ea706fac63587 100644 --- a/arch/arm/mach-tegra/reset.c +++ b/arch/arm/mach-tegra/reset.c @@ -63,7 +63,7 @@ static void __init tegra_cpu_reset_handler_enable(void) BUG_ON(is_enabled); BUG_ON(tegra_cpu_reset_handler_size > TEGRA_IRAM_RESET_HANDLER_SIZE); - memcpy(iram_base, (void *)__tegra_cpu_reset_handler_start, + memcpy_toio(iram_base, (void *)__tegra_cpu_reset_handler_start, tegra_cpu_reset_handler_size); err = call_firmware_op(set_cpu_boot_addr, 0, reset_address); diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 1c5aeba9bc27f..7801c5bb775d9 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -515,7 +515,5 @@ void __init early_ioremap_init(void) bool arch_memremap_can_ram_remap(resource_size_t offset, size_t size, unsigned long flags) { - unsigned long pfn = PHYS_PFN(offset); - - return memblock_is_map_memory(pfn); + return memblock_is_map_memory(offset); } diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index d8a41303f9fd9..a92686eddfa53 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -308,9 +308,9 @@ config ARCH_MMAP_RND_BITS_MAX default 24 if ARM64_VA_BITS=39 default 27 if ARM64_VA_BITS=42 default 30 if ARM64_VA_BITS=47 - default 29 if ARM64_VA_BITS=48 && ARM64_64K_PAGES - default 31 if ARM64_VA_BITS=48 && ARM64_16K_PAGES - default 33 if ARM64_VA_BITS=48 + default 29 if (ARM64_VA_BITS=48 || ARM64_VA_BITS=52) && ARM64_64K_PAGES + default 31 if (ARM64_VA_BITS=48 || ARM64_VA_BITS=52) && ARM64_16K_PAGES + default 33 if (ARM64_VA_BITS=48 || ARM64_VA_BITS=52) default 14 if ARM64_64K_PAGES default 16 if ARM64_16K_PAGES default 18 @@ -1098,6 +1098,7 @@ config ARM64_ERRATUM_3194386 * ARM Neoverse-V1 erratum 3324341 * ARM Neoverse V2 erratum 3324336 * ARM Neoverse-V3 erratum 3312417 + * ARM Neoverse-V3AE erratum 3312417 On affected cores "MSR SSBS, #0" instructions may not affect subsequent speculative instructions, which may permit unexepected diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts b/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts index 381d58cea092d..c854c7e310519 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts @@ -151,28 +151,12 @@ vcc-pg-supply = <®_aldo1>; }; -&r_ir { - linux,rc-map-name = "rc-beelink-gs1"; - status = "okay"; -}; - -&r_pio { - /* - * FIXME: We can't add that supply for now since it would - * create a circular dependency between pinctrl, the regulator - * and the RSB Bus. - * - * vcc-pl-supply = <®_aldo1>; - */ - vcc-pm-supply = <®_aldo1>; -}; - -&r_rsb { +&r_i2c { status = "okay"; - axp805: pmic@745 { + axp805: pmic@36 { compatible = "x-powers,axp805", "x-powers,axp806"; - reg = <0x745>; + reg = <0x36>; interrupt-parent = <&r_intc>; interrupts = ; interrupt-controller; @@ -290,6 +274,22 @@ }; }; +&r_ir { + linux,rc-map-name = "rc-beelink-gs1"; + status = "okay"; +}; + +&r_pio { + /* + * PL0 and PL1 are used for PMIC I2C + * don't enable the pl-supply else + * it will fail at boot + * + * vcc-pl-supply = <®_aldo1>; + */ + vcc-pm-supply = <®_aldo1>; +}; + &spdif { pinctrl-names = "default"; pinctrl-0 = <&spdif_tx_pin>; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi-3.dts b/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi-3.dts index 6fc65e8db2206..8c476e089185b 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi-3.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi-3.dts @@ -175,16 +175,12 @@ vcc-pg-supply = <®_vcc_wifi_io>; }; -&r_ir { - status = "okay"; -}; - -&r_rsb { +&r_i2c { status = "okay"; - axp805: pmic@745 { + axp805: pmic@36 { compatible = "x-powers,axp805", "x-powers,axp806"; - reg = <0x745>; + reg = <0x36>; interrupt-parent = <&r_intc>; interrupts = ; interrupt-controller; @@ -295,6 +291,10 @@ }; }; +&r_ir { + status = "okay"; +}; + &rtc { clocks = <&ext_osc32k>; }; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi.dtsi index 92745128fcfeb..4ec4996592bef 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-orangepi.dtsi @@ -112,20 +112,12 @@ vcc-pg-supply = <®_aldo1>; }; -&r_ir { - status = "okay"; -}; - -&r_pio { - vcc-pm-supply = <®_bldo3>; -}; - -&r_rsb { +&r_i2c { status = "okay"; - axp805: pmic@745 { + axp805: pmic@36 { compatible = "x-powers,axp805", "x-powers,axp806"; - reg = <0x745>; + reg = <0x36>; interrupt-parent = <&r_intc>; interrupts = ; interrupt-controller; @@ -240,6 +232,14 @@ }; }; +&r_ir { + status = "okay"; +}; + +&r_pio { + vcc-pm-supply = <®_bldo3>; +}; + &rtc { clocks = <&ext_osc32k>; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index 152f95fd49a21..7089ccf3ce556 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -21,6 +21,14 @@ }; }; +/* + * Adjust pcie0's iommu-map to account for the disabled port01. + */ +&pcie0 { + iommu-map = <0x100 &pcie0_dart_0 1 1>, + <0x200 &pcie0_dart_2 1 1>; +}; + &bluetooth0 { brcm,board-type = "apple,santorini"; }; @@ -36,10 +44,10 @@ */ &port02 { - bus-range = <3 3>; + bus-range = <2 2>; status = "okay"; ethernet0: ethernet@0,0 { - reg = <0x30000 0x0 0x0 0x0 0x0>; + reg = <0x20000 0x0 0x0 0x0 0x0>; /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 00]; }; diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 5988a4eb6efaa..cb78ce7af0b38 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -71,7 +71,7 @@ */ &port00 { bus-range = <1 1>; - wifi0: network@0,0 { + wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; /* To be filled by the loader */ diff --git a/arch/arm64/boot/dts/freescale/imx8mm-beacon-kit.dts b/arch/arm64/boot/dts/freescale/imx8mm-beacon-kit.dts index 905c98cb080d2..a6e6860bf0184 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-beacon-kit.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-beacon-kit.dts @@ -124,6 +124,7 @@ assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>; assigned-clock-rates = <24576000>; #sound-dai-cells = <0>; + fsl,sai-mclk-direction-output; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi index f264102bdb274..f93c2b604c57f 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi @@ -231,6 +231,7 @@ rtc: rtc@51 { compatible = "nxp,pcf85263"; reg = <0x51>; + quartz-load-femtofarads = <12500>; }; }; @@ -283,6 +284,8 @@ pinctrl-0 = <&pinctrl_usdhc3>; pinctrl-1 = <&pinctrl_usdhc3_100mhz>; pinctrl-2 = <&pinctrl_usdhc3_200mhz>; + assigned-clocks = <&clk IMX8MM_CLK_USDHC3>; + assigned-clock-rates = <400000000>; bus-width = <8>; non-removable; status = "okay"; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi index 6c48fa4b0d0c4..218b548219078 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi @@ -148,6 +148,19 @@ startup-delay-us = <20000>; }; + reg_usdhc2_vqmmc: regulator-usdhc2-vqmmc { + compatible = "regulator-gpio"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usdhc2_vsel>; + gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + regulator-max-microvolt = <3300000>; + regulator-min-microvolt = <1800000>; + states = <1800000 0x1>, + <3300000 0x0>; + regulator-name = "PMIC_USDHC_VSELECT"; + vin-supply = <®_nvcc_sd>; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -266,7 +279,7 @@ "SODIMM_19", "", "", - "", + "PMIC_USDHC_VSELECT", "", "", "", @@ -457,6 +470,7 @@ }; reg_nvcc_sd: LDO5 { + regulator-always-on; regulator-max-microvolt = <3300000>; regulator-min-microvolt = <1800000>; regulator-name = "On-module +V3.3_1.8_SD (LDO5)"; @@ -787,6 +801,7 @@ pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_cd>; pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_cd_sleep>; vmmc-supply = <®_usdhc2_vmmc>; + vqmmc-supply = <®_usdhc2_vqmmc>; }; &wdog1 { @@ -1209,13 +1224,17 @@ ; /* SODIMM 76 */ }; + pinctrl_usdhc2_vsel: usdhc2vselgrp { + fsl,pins = + ; /* PMIC_USDHC_VSELECT */ + }; + /* * Note: Due to ERR050080 we use discrete external on-module resistors pulling-up to the * on-module +V3.3_1.8_SD (LDO5) rail and explicitly disable the internal pull-ups here. */ pinctrl_usdhc2: usdhc2grp { fsl,pins = - , , /* SODIMM 78 */ , /* SODIMM 74 */ , /* SODIMM 80 */ @@ -1226,7 +1245,6 @@ pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp { fsl,pins = - , , , , @@ -1237,7 +1255,6 @@ pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp { fsl,pins = - , , , , @@ -1249,7 +1266,6 @@ /* Avoid backfeeding with removed card power */ pinctrl_usdhc2_sleep: usdhc2slpgrp { fsl,pins = - , , , , diff --git a/arch/arm64/boot/dts/freescale/imx8mn-beacon-kit.dts b/arch/arm64/boot/dts/freescale/imx8mn-beacon-kit.dts index 35b8d2060cd99..dfa08be33a4f2 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-beacon-kit.dts +++ b/arch/arm64/boot/dts/freescale/imx8mn-beacon-kit.dts @@ -126,6 +126,7 @@ assigned-clock-parents = <&clk IMX8MN_AUDIO_PLL1_OUT>; assigned-clock-rates = <24576000>; #sound-dai-cells = <0>; + fsl,sai-mclk-direction-output; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi index 90073b16536f4..f5f87b3891234 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi @@ -240,6 +240,7 @@ rtc: rtc@51 { compatible = "nxp,pcf85263"; reg = <0x51>; + quartz-load-femtofarads = <12500>; }; }; @@ -294,6 +295,8 @@ pinctrl-0 = <&pinctrl_usdhc3>; pinctrl-1 = <&pinctrl_usdhc3_100mhz>; pinctrl-2 = <&pinctrl_usdhc3_200mhz>; + assigned-clocks = <&clk IMX8MN_CLK_USDHC3>; + assigned-clock-rates = <400000000>; bus-width = <8>; non-removable; status = "okay"; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi index e5da908047808..24380f8a00850 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi @@ -192,6 +192,7 @@ rtc: rtc@51 { compatible = "nxp,pcf85263"; reg = <0x51>; + quartz-load-femtofarads = <12500>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts index cd44bf83745ca..678ecc9f81dbb 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts @@ -442,6 +442,7 @@ pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>; cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; vmmc-supply = <®_usdhc2_vmmc>; + vqmmc-supply = <&ldo5>; bus-width = <4>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi index eae39c1cb9856..2e93d922c8611 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi @@ -571,6 +571,7 @@ pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>; cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; vmmc-supply = <®_usdhc2_vmmc>; + vqmmc-supply = <&ldo5>; bus-width = <4>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts index faa370a5885ff..34619f085623b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts @@ -185,7 +185,7 @@ #address-cells = <0x1>; #size-cells = <0x1>; reg = <0x0>; - spi-max-frequency = <36000000>; + spi-max-frequency = <25000000>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp.dtsi index 69b213ed7a594..7f7bd8477efde 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp.dtsi @@ -228,7 +228,7 @@ cpu-thermal { polling-delay-passive = <250>; polling-delay = <2000>; - thermal-sensors = <&tmu 0>; + thermal-sensors = <&tmu 1>; trips { cpu_alert0: trip0 { temperature = <85000>; @@ -258,7 +258,7 @@ soc-thermal { polling-delay-passive = <250>; polling-delay = <2000>; - thermal-sensors = <&tmu 1>; + thermal-sensors = <&tmu 0>; trips { soc_alert0: trip0 { temperature = <85000>; diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi index 3f79923376fb2..37244e8816d9e 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi @@ -26,6 +26,8 @@ leds { compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&spi_quad_pins>; led-power1 { label = "udpu:green:power"; @@ -82,8 +84,6 @@ &spi0 { status = "okay"; - pinctrl-names = "default"; - pinctrl-0 = <&spi_quad_pins>; flash@0 { compatible = "jedec,spi-nor"; @@ -108,6 +108,10 @@ }; }; +&spi_quad_pins { + function = "gpio"; +}; + &pinctrl_nb { i2c2_recovery_pins: i2c2-recovery-pins { groups = "i2c2"; diff --git a/arch/arm64/boot/dts/mediatek/mt6331.dtsi b/arch/arm64/boot/dts/mediatek/mt6331.dtsi index d89858c73ab1b..243afbffa21fd 100644 --- a/arch/arm64/boot/dts/mediatek/mt6331.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt6331.dtsi @@ -6,12 +6,12 @@ #include &pwrap { - pmic: mt6331 { + pmic: pmic { compatible = "mediatek,mt6331"; interrupt-controller; #interrupt-cells = <2>; - mt6331regulator: mt6331regulator { + mt6331regulator: regulators { compatible = "mediatek,mt6331-regulator"; mt6331_vdvfs11_reg: buck-vdvfs11 { @@ -258,7 +258,7 @@ }; mt6331_vdig18_reg: ldo-vdig18 { - regulator-name = "dvdd18_dig"; + regulator-name = "vdig18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-ramp-delay = <0>; @@ -266,11 +266,11 @@ }; }; - mt6331rtc: mt6331rtc { + mt6331rtc: rtc { compatible = "mediatek,mt6331-rtc"; }; - mt6331keys: mt6331keys { + mt6331keys: keys { compatible = "mediatek,mt6331-keys"; power { linux,keycodes = ; diff --git a/arch/arm64/boot/dts/mediatek/mt6357.dtsi b/arch/arm64/boot/dts/mediatek/mt6357.dtsi index 5fafa842d312f..dca4e5c3d8e21 100644 --- a/arch/arm64/boot/dts/mediatek/mt6357.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt6357.dtsi @@ -60,7 +60,6 @@ }; mt6357_vfe28_reg: ldo-vfe28 { - compatible = "regulator-fixed"; regulator-name = "vfe28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; @@ -75,7 +74,6 @@ }; mt6357_vrf18_reg: ldo-vrf18 { - compatible = "regulator-fixed"; regulator-name = "vrf18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; @@ -83,7 +81,6 @@ }; mt6357_vrf12_reg: ldo-vrf12 { - compatible = "regulator-fixed"; regulator-name = "vrf12"; regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; @@ -112,7 +109,6 @@ }; mt6357_vcn28_reg: ldo-vcn28 { - compatible = "regulator-fixed"; regulator-name = "vcn28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; @@ -120,7 +116,6 @@ }; mt6357_vcn18_reg: ldo-vcn18 { - compatible = "regulator-fixed"; regulator-name = "vcn18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; @@ -142,7 +137,6 @@ }; mt6357_vcamio_reg: ldo-vcamio18 { - compatible = "regulator-fixed"; regulator-name = "vcamio"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; @@ -175,7 +169,6 @@ }; mt6357_vaux18_reg: ldo-vaux18 { - compatible = "regulator-fixed"; regulator-name = "vaux18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; @@ -183,7 +176,6 @@ }; mt6357_vaud28_reg: ldo-vaud28 { - compatible = "regulator-fixed"; regulator-name = "vaud28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; @@ -191,7 +183,6 @@ }; mt6357_vio28_reg: ldo-vio28 { - compatible = "regulator-fixed"; regulator-name = "vio28"; regulator-min-microvolt = <2800000>; regulator-max-microvolt = <2800000>; @@ -199,7 +190,6 @@ }; mt6357_vio18_reg: ldo-vio18 { - compatible = "regulator-fixed"; regulator-name = "vio18"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; diff --git a/arch/arm64/boot/dts/mediatek/mt6359.dtsi b/arch/arm64/boot/dts/mediatek/mt6359.dtsi index 8e1b8c85c6ede..779d6dfb55c00 100644 --- a/arch/arm64/boot/dts/mediatek/mt6359.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt6359.dtsi @@ -18,6 +18,8 @@ }; regulators { + compatible = "mediatek,mt6359-regulator"; + mt6359_vs1_buck_reg: buck_vs1 { regulator-name = "vs1"; regulator-min-microvolt = <800000>; @@ -296,7 +298,7 @@ }; }; - mt6359rtc: mt6359rtc { + mt6359rtc: rtc { compatible = "mediatek,mt6358-rtc"; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt6795-sony-xperia-m5.dts b/arch/arm64/boot/dts/mediatek/mt6795-sony-xperia-m5.dts index b5746e6d0b15a..110fae4ec11c9 100644 --- a/arch/arm64/boot/dts/mediatek/mt6795-sony-xperia-m5.dts +++ b/arch/arm64/boot/dts/mediatek/mt6795-sony-xperia-m5.dts @@ -136,7 +136,7 @@ &mmc0 { /* eMMC controller */ - mediatek,latch-ck = <0x14>; /* hs400 */ + mediatek,latch-ck = <4>; /* hs400 */ mediatek,hs200-cmd-int-delay = <1>; mediatek,hs400-cmd-int-delay = <1>; mediatek,hs400-ds-dly3 = <0x1a>; diff --git a/arch/arm64/boot/dts/mediatek/mt8195.dtsi b/arch/arm64/boot/dts/mediatek/mt8195.dtsi index 7ba30209ba9a9..4b701afe995e2 100644 --- a/arch/arm64/boot/dts/mediatek/mt8195.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8195.dtsi @@ -617,22 +617,6 @@ #size-cells = <0>; #power-domain-cells = <1>; - power-domain@MT8195_POWER_DOMAIN_VDEC1 { - reg = ; - clocks = <&vdecsys CLK_VDEC_LARB1>; - clock-names = "vdec1-0"; - mediatek,infracfg = <&infracfg_ao>; - #power-domain-cells = <0>; - }; - - power-domain@MT8195_POWER_DOMAIN_VENC_CORE1 { - reg = ; - clocks = <&vencsys_core1 CLK_VENC_CORE1_LARB>; - clock-names = "venc1-larb"; - mediatek,infracfg = <&infracfg_ao>; - #power-domain-cells = <0>; - }; - power-domain@MT8195_POWER_DOMAIN_VDOSYS0 { reg = ; clocks = <&topckgen CLK_TOP_CFG_VDO0>, @@ -678,15 +662,25 @@ clocks = <&vdecsys_soc CLK_VDEC_SOC_LARB1>; clock-names = "vdec0-0"; mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; #power-domain-cells = <0>; - }; - power-domain@MT8195_POWER_DOMAIN_VDEC2 { - reg = ; - clocks = <&vdecsys_core1 CLK_VDEC_CORE1_LARB1>; - clock-names = "vdec2-0"; - mediatek,infracfg = <&infracfg_ao>; - #power-domain-cells = <0>; + power-domain@MT8195_POWER_DOMAIN_VDEC1 { + reg = ; + clocks = <&vdecsys CLK_VDEC_LARB1>; + clock-names = "vdec1-0"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_VDEC2 { + reg = ; + clocks = <&vdecsys_core1 CLK_VDEC_CORE1_LARB1>; + clock-names = "vdec2-0"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; }; power-domain@MT8195_POWER_DOMAIN_VENC { @@ -694,7 +688,17 @@ clocks = <&vencsys CLK_VENC_LARB>; clock-names = "venc0-larb"; mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; #power-domain-cells = <0>; + + power-domain@MT8195_POWER_DOMAIN_VENC_CORE1 { + reg = ; + clocks = <&vencsys_core1 CLK_VENC_CORE1_LARB>; + clock-names = "venc1-larb"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; }; power-domain@MT8195_POWER_DOMAIN_VDOSYS1 { @@ -1520,9 +1524,6 @@ power-domains = <&spm MT8195_POWER_DOMAIN_PCIE_MAC_P0>; - resets = <&infracfg_ao MT8195_INFRA_RST2_PCIE_P0_SWRST>; - reset-names = "mac"; - #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 7>; interrupt-map = <0 0 0 1 &pcie_intc0 0>, diff --git a/arch/arm64/boot/dts/mediatek/mt8516-pumpkin.dts b/arch/arm64/boot/dts/mediatek/mt8516-pumpkin.dts index cce642c538128..3d3db33a64dc6 100644 --- a/arch/arm64/boot/dts/mediatek/mt8516-pumpkin.dts +++ b/arch/arm64/boot/dts/mediatek/mt8516-pumpkin.dts @@ -11,7 +11,7 @@ / { model = "Pumpkin MT8516"; - compatible = "mediatek,mt8516"; + compatible = "mediatek,mt8516-pumpkin", "mediatek,mt8516"; memory@40000000 { device_type = "memory"; diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi index 2b3bb5d0af17b..f0b7949df92c0 100644 --- a/arch/arm64/boot/dts/nvidia/tegra186.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi @@ -621,9 +621,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA186_CLK_UARTB>; - clock-names = "serial"; resets = <&bpmp TEGRA186_RESET_UARTB>; - reset-names = "serial"; status = "disabled"; }; @@ -633,9 +631,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA186_CLK_UARTD>; - clock-names = "serial"; resets = <&bpmp TEGRA186_RESET_UARTD>; - reset-names = "serial"; status = "disabled"; }; @@ -645,9 +641,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA186_CLK_UARTE>; - clock-names = "serial"; resets = <&bpmp TEGRA186_RESET_UARTE>; - reset-names = "serial"; status = "disabled"; }; @@ -657,9 +651,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA186_CLK_UARTF>; - clock-names = "serial"; resets = <&bpmp TEGRA186_RESET_UARTF>; - reset-names = "serial"; status = "disabled"; }; @@ -1236,9 +1228,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA186_CLK_UARTC>; - clock-names = "serial"; resets = <&bpmp TEGRA186_RESET_UARTC>; - reset-names = "serial"; status = "disabled"; }; @@ -1248,9 +1238,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA186_CLK_UARTG>; - clock-names = "serial"; resets = <&bpmp TEGRA186_RESET_UARTG>; - reset-names = "serial"; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi index 33f92b77cd9d9..c369507747851 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi @@ -766,9 +766,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA194_CLK_UARTD>; - clock-names = "serial"; resets = <&bpmp TEGRA194_RESET_UARTD>; - reset-names = "serial"; status = "disabled"; }; @@ -778,9 +776,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA194_CLK_UARTE>; - clock-names = "serial"; resets = <&bpmp TEGRA194_RESET_UARTE>; - reset-names = "serial"; status = "disabled"; }; @@ -790,9 +786,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA194_CLK_UARTF>; - clock-names = "serial"; resets = <&bpmp TEGRA194_RESET_UARTF>; - reset-names = "serial"; status = "disabled"; }; @@ -817,9 +811,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA194_CLK_UARTH>; - clock-names = "serial"; resets = <&bpmp TEGRA194_RESET_UARTH>; - reset-names = "serial"; status = "disabled"; }; @@ -1616,9 +1608,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA194_CLK_UARTC>; - clock-names = "serial"; resets = <&bpmp TEGRA194_RESET_UARTC>; - reset-names = "serial"; status = "disabled"; }; @@ -1628,9 +1618,7 @@ reg-shift = <2>; interrupts = ; clocks = <&bpmp TEGRA194_CLK_UARTG>; - clock-names = "serial"; resets = <&bpmp TEGRA194_RESET_UARTG>; - reset-names = "serial"; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi index b4a1108c2dd74..0639f5ce1bd9e 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi @@ -1635,7 +1635,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-always-on; - gpio = <&exp1 14 GPIO_ACTIVE_HIGH>; + gpio = <&exp1 9 GPIO_ACTIVE_HIGH>; enable-active-high; vin-supply = <&vdd_1v8>; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts b/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts index bac611d735c58..2fa48972b2a91 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts +++ b/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts @@ -102,6 +102,16 @@ }; pcie@141a0000 { + reg = <0x00 0x141a0000 0x0 0x00020000 /* appl registers (128K) */ + 0x00 0x3a000000 0x0 0x00040000 /* configuration space (256K) */ + 0x00 0x3a040000 0x0 0x00040000 /* iATU_DMA reg space (256K) */ + 0x00 0x3a080000 0x0 0x00040000 /* DBI reg space (256K) */ + 0x2e 0x20000000 0x0 0x10000000>; /* ECAM (256MB) */ + + ranges = <0x81000000 0x00 0x3a100000 0x00 0x3a100000 0x0 0x00100000 /* downstream I/O (1MB) */ + 0x82000000 0x00 0x40000000 0x2e 0x30000000 0x0 0x08000000 /* non-prefetchable memory (128MB) */ + 0xc3000000 0x28 0x00000000 0x28 0x00000000 0x6 0x20000000>; /* prefetchable memory (25088MB) */ + status = "okay"; vddio-pex-ctl-supply = <&vdd_1v8_ls>; phys = <&p2u_nvhs_0>, <&p2u_nvhs_1>, <&p2u_nvhs_2>, diff --git a/arch/arm64/boot/dts/qcom/ipq9574.dtsi b/arch/arm64/boot/dts/qcom/ipq9574.dtsi index 8a72ad4afd032..82e4fd5eb388f 100644 --- a/arch/arm64/boot/dts/qcom/ipq9574.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq9574.dtsi @@ -231,6 +231,8 @@ interrupts = ; #dma-cells = <1>; qcom,ee = <1>; + qcom,num-ees = <4>; + num-channels = <16>; qcom,controlled-remotely; }; diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi index 6f5f96853ba1c..701e139a9cbd0 100644 --- a/arch/arm64/boot/dts/qcom/msm8916.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi @@ -1529,6 +1529,8 @@ interrupts = ; + resets = <&gcc GCC_MDSS_BCR>; + interrupt-controller; #interrupt-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/msm8939.dtsi b/arch/arm64/boot/dts/qcom/msm8939.dtsi index c844e01f9aa15..86432ed25b23a 100644 --- a/arch/arm64/boot/dts/qcom/msm8939.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8939.dtsi @@ -1210,6 +1210,8 @@ power-domains = <&gcc MDSS_GDSC>; + resets = <&gcc GCC_MDSS_BCR>; + #address-cells = <1>; #size-cells = <1>; #interrupt-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/msm8976.dtsi b/arch/arm64/boot/dts/qcom/msm8976.dtsi index 4c5be22b47fee..e01f6e8a17f2f 100644 --- a/arch/arm64/boot/dts/qcom/msm8976.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8976.dtsi @@ -862,6 +862,7 @@ clock-names = "bam_clk"; #dma-cells = <1>; qcom,ee = <0>; + qcom,controlled-remotely; }; blsp1_uart1: serial@78af000 { @@ -982,6 +983,7 @@ clock-names = "bam_clk"; #dma-cells = <1>; qcom,ee = <0>; + qcom,controlled-remotely; }; blsp2_uart2: serial@7af0000 { diff --git a/arch/arm64/boot/dts/qcom/qcm2290.dtsi b/arch/arm64/boot/dts/qcom/qcm2290.dtsi index 40a8506553ef5..4051a7660eb06 100644 --- a/arch/arm64/boot/dts/qcom/qcm2290.dtsi +++ b/arch/arm64/boot/dts/qcom/qcm2290.dtsi @@ -1107,6 +1107,7 @@ snps,has-lpm-erratum; snps,hird-threshold = /bits/ 8 <0x10>; snps,usb3_lpm_capable; + snps,parkmode-disable-ss-quirk; maximum-speed = "super-speed"; dr_mode = "otg"; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi index 7758136d71d64..9dc00f759f19b 100644 --- a/arch/arm64/boot/dts/qcom/sc7180.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi @@ -3435,18 +3435,18 @@ #interrupt-cells = <4>; }; - sram@146aa000 { + sram@14680000 { compatible = "qcom,sc7180-imem", "syscon", "simple-mfd"; - reg = <0 0x146aa000 0 0x2000>; + reg = <0 0x14680000 0 0x2e000>; #address-cells = <1>; #size-cells = <1>; - ranges = <0 0 0x146aa000 0x2000>; + ranges = <0 0 0x14680000 0x2e000>; - pil-reloc@94c { + pil-reloc@2a94c { compatible = "qcom,pil-reloc-info"; - reg = <0x94c 0xc8>; + reg = <0x2a94c 0xc8>; }; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts index 5c2894fcfa4a0..5498e84bfead0 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts @@ -985,9 +985,6 @@ "VA DMIC0", "MIC BIAS1", "VA DMIC1", "MIC BIAS1", "VA DMIC2", "MIC BIAS3", - "VA DMIC0", "VA MIC BIAS1", - "VA DMIC1", "VA MIC BIAS1", - "VA DMIC2", "VA MIC BIAS3", "TX SWR_ADC1", "ADC2_OUTPUT"; wcd-playback-dai-link { diff --git a/arch/arm64/boot/dts/qcom/sda660-inforce-ifc6560.dts b/arch/arm64/boot/dts/qcom/sda660-inforce-ifc6560.dts index 2ed39d402d3f6..d687cfadee6a1 100644 --- a/arch/arm64/boot/dts/qcom/sda660-inforce-ifc6560.dts +++ b/arch/arm64/boot/dts/qcom/sda660-inforce-ifc6560.dts @@ -155,6 +155,7 @@ * BAM DMA interconnects support is in place. */ /delete-property/ clocks; + /delete-property/ clock-names; }; &blsp1_uart2 { @@ -167,6 +168,7 @@ * BAM DMA interconnects support is in place. */ /delete-property/ clocks; + /delete-property/ clock-names; }; &blsp2_uart1 { diff --git a/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts b/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts index 3c47410ba94c0..8221da390e1aa 100644 --- a/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts +++ b/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts @@ -107,6 +107,7 @@ status = "okay"; vdd-supply = <&vreg_l1b_0p925>; + vdda-pll-supply = <&vreg_l10a_1p8>; vdda-phy-dpdm-supply = <&vreg_l7b_3p125>; }; @@ -404,6 +405,8 @@ &sdhc_2 { status = "okay"; + cd-gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>; + vmmc-supply = <&vreg_l5b_2p95>; vqmmc-supply = <&vreg_l2b_2p95>; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts index d37a433130b98..5948b401165ce 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts @@ -135,8 +135,6 @@ vdda_sp_sensor: vdda_ufs1_core: vdda_ufs2_core: - vdda_usb1_ss_core: - vdda_usb2_ss_core: vreg_l1a_0p875: ldo1 { regulator-min-microvolt = <880000>; regulator-max-microvolt = <880000>; @@ -157,6 +155,7 @@ regulator-initial-mode = ; }; + vdda_usb1_ss_core: vdd_wcss_cx: vdd_wcss_mx: vdda_wcss_pll: @@ -383,8 +382,8 @@ }; &sdhc_2 { - pinctrl-names = "default"; pinctrl-0 = <&sdc2_clk_state &sdc2_cmd_state &sdc2_data_state &sd_card_det_n_state>; + pinctrl-names = "default"; cd-gpios = <&tlmm 126 GPIO_ACTIVE_LOW>; vmmc-supply = <&vreg_l21a_2p95>; vqmmc-supply = <&vddpx_2>; @@ -418,16 +417,9 @@ status = "okay"; }; -&wifi { - vdd-0.8-cx-mx-supply = <&vreg_l5a_0p8>; - vdd-1.8-xo-supply = <&vreg_l7a_1p8>; - vdd-1.3-rfa-supply = <&vreg_l17a_1p3>; - vdd-3.3-ch0-supply = <&vreg_l25a_3p3>; - status = "okay"; -}; - &tlmm { - gpio-reserved-ranges = <0 4>, <27 4>, <81 4>, <85 4>; + gpio-reserved-ranges = <27 4>, /* SPI (eSE - embedded Secure Element) */ + <85 4>; /* SPI (fingerprint reader) */ sdc2_clk_state: sdc2-clk-state { pins = "sdc2_clk"; diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi index 4ea693a075856..fa91e2036dd56 100644 --- a/arch/arm64/boot/dts/qcom/sdm845.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi @@ -4989,18 +4989,18 @@ #interrupt-cells = <4>; }; - sram@146bf000 { + sram@14680000 { compatible = "qcom,sdm845-imem", "syscon", "simple-mfd"; - reg = <0 0x146bf000 0 0x1000>; + reg = <0 0x14680000 0 0x40000>; #address-cells = <1>; #size-cells = <1>; - ranges = <0 0 0x146bf000 0x1000>; + ranges = <0 0 0x14680000 0x40000>; - pil-reloc@94c { + pil-reloc@3f94c { compatible = "qcom,pil-reloc-info"; - reg = <0x94c 0xc8>; + reg = <0x3f94c 0xc8>; }; }; @@ -5208,11 +5208,11 @@ compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0"; qcom,controlled-remotely; reg = <0 0x17184000 0 0x2a000>; - num-channels = <31>; + num-channels = <23>; interrupts = ; #dma-cells = <1>; qcom,ee = <1>; - qcom,num-ees = <2>; + qcom,num-ees = <4>; iommus = <&apps_smmu 0x1806 0x0>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index 21bbffc4e5a28..c9a7d1b75c658 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -601,7 +601,7 @@ }; cpu7_opp9: opp-1747200000 { - opp-hz = /bits/ 64 <1708800000>; + opp-hz = /bits/ 64 <1747200000>; opp-peak-kBps = <5412000 42393600>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index 2a4d950ac02bf..215782b1970df 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -442,7 +442,7 @@ no-map; }; - pil_camera_mem: mmeory@85200000 { + pil_camera_mem: memory@85200000 { reg = <0x0 0x85200000 0x0 0x500000>; no-map; }; @@ -1754,11 +1754,11 @@ interrupts = ; #dma-cells = <1>; qcom,ee = <0>; + qcom,num-ees = <4>; + num-channels = <16>; qcom,controlled-remotely; iommus = <&apps_smmu 0x594 0x0011>, <&apps_smmu 0x596 0x0011>; - /* FIXME: Probing BAM DMA causes some abort and system hang */ - status = "fail"; }; crypto: crypto@1dfa000 { @@ -1770,8 +1770,6 @@ <&apps_smmu 0x596 0x0011>; interconnects = <&aggre2_noc MASTER_CRYPTO 0 &mc_virt SLAVE_EBI1 0>; interconnect-names = "memory"; - /* FIXME: dependency BAM DMA is disabled */ - status = "disabled"; }; ipa: ipa@1e40000 { diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 3b4d788230089..c1ed39cac8c5b 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -4233,6 +4233,8 @@ interrupts = ; #dma-cells = <1>; qcom,ee = <0>; + qcom,num-ees = <4>; + num-channels = <16>; qcom,controlled-remotely; iommus = <&apps_smmu 0x584 0x11>, <&apps_smmu 0x588 0x0>, diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi index bc9a1fca2db3a..2f0f1c2ab7391 100644 --- a/arch/arm64/boot/dts/qcom/sm8550.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi @@ -1064,6 +1064,20 @@ status = "disabled"; }; + uart14: serial@898000 { + compatible = "qcom,geni-uart"; + reg = <0 0x898000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart14_default>, <&qup_uart14_cts_rts>; + interrupts = ; + interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + i2c15: i2c@89c000 { compatible = "qcom,geni-i2c"; reg = <0 0x0089c000 0 0x4000>; @@ -1866,6 +1880,8 @@ interrupts = ; #dma-cells = <1>; qcom,ee = <0>; + qcom,num-ees = <4>; + num-channels = <20>; qcom,controlled-remotely; iommus = <&apps_smmu 0x480 0x0>, <&apps_smmu 0x481 0x0>; @@ -3638,6 +3654,22 @@ bias-disable; }; + qup_uart14_default: qup-uart14-default-state { + /* TX, RX */ + pins = "gpio78", "gpio79"; + function = "qup2_se6"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_uart14_cts_rts: qup-uart14-cts-rts-state { + /* CTS, RTS */ + pins = "gpio76", "gpio77"; + function = "qup2_se6"; + drive-strength = <2>; + bias-pull-down; + }; + sdc2_sleep: sdc2-sleep-state { clk-pins { pins = "sdc2_clk"; diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ard-audio-da7212.dtso b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ard-audio-da7212.dtso index e6f53377ecd90..99f62574bc3c2 100644 --- a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ard-audio-da7212.dtso +++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ard-audio-da7212.dtso @@ -108,7 +108,7 @@ }; tpu0_pins: tpu0 { - groups = "tpu_to0_a"; + groups = "tpu_to0_b"; function = "tpu"; }; }; diff --git a/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi index 83fce96a25752..0234dbe95413c 100644 --- a/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi +++ b/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi @@ -38,7 +38,10 @@ #if (SW_SCIF_CAN || SW_RSPI_CAN) &canfd { pinctrl-0 = <&can1_pins>; - /delete-node/ channel@0; + + channel0 { + status = "disabled"; + }; }; #else &canfd { diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi index 2963d634baba9..403a295bde6ab 100644 --- a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi @@ -344,6 +344,18 @@ <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; }; }; + + spi1 { + spi1_csn0_gpio_pin: spi1-csn0-gpio-pin { + rockchip,pins = + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up_4ma>; + }; + + spi1_csn1_gpio_pin: spi1-csn1-gpio-pin { + rockchip,pins = + <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up_4ma>; + }; + }; }; &saradc { @@ -355,6 +367,17 @@ vqmmc-supply = <&vccio_sd>; }; +&spi1 { + /* + * Hardware CS has a very slow rise time of about 6us, + * causing transmission errors. + * With cs-gpios we have a rise time of about 20ns. + */ + cs-gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_LOW>, <&gpio3 RK_PB2 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&spi1_clk &spi1_csn0_gpio_pin &spi1_csn1_gpio_pin &spi1_miso &spi1_mosi>; +}; + &tsadc { status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts index f5e124b235c83..fb3012a6c9fc3 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts @@ -967,6 +967,7 @@ reg = <0>; m25p,fast-read; spi-max-frequency = <10000000>; + vcc-supply = <&vcc_3v0>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts index 115c14c0a3c68..396a6636073b5 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts @@ -251,14 +251,6 @@ status = "okay"; }; -&usb_host0_ehci { - status = "okay"; -}; - -&usb_host0_ohci { - status = "okay"; -}; - &vopb { status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dtsi index 93189f8306400..c30354268c8f5 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dtsi @@ -486,9 +486,12 @@ &sdhci { bus-width = <8>; max-frequency = <200000000>; + mmc-hs200-1_8v; non-removable; pinctrl-names = "default"; - pinctrl-0 = <&emmc_bus8 &emmc_clk &emmc_cmd>; + pinctrl-0 = <&emmc_bus8 &emmc_clk &emmc_cmd &emmc_datastrobe>; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&vcc_1v8>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-odroid-m1.dts b/arch/arm64/boot/dts/rockchip/rk3568-odroid-m1.dts index 6a02db4f073f2..a5426b82552ed 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-odroid-m1.dts +++ b/arch/arm64/boot/dts/rockchip/rk3568-odroid-m1.dts @@ -482,6 +482,8 @@ }; &i2s1_8ch { + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx &i2s1m0_lrcktx &i2s1m0_sdi0 &i2s1m0_sdo0>; rockchip,trcm-sync-tx-only; status = "okay"; }; diff --git a/arch/arm64/boot/dts/st/stm32mp251.dtsi b/arch/arm64/boot/dts/st/stm32mp251.dtsi index 5268a43218415..ce5409acae1ce 100644 --- a/arch/arm64/boot/dts/st/stm32mp251.dtsi +++ b/arch/arm64/boot/dts/st/stm32mp251.dtsi @@ -73,14 +73,13 @@ }; intc: interrupt-controller@4ac00000 { - compatible = "arm,cortex-a7-gic"; + compatible = "arm,gic-400"; #interrupt-cells = <3>; - #address-cells = <1>; interrupt-controller; reg = <0x0 0x4ac10000 0x0 0x1000>, - <0x0 0x4ac20000 0x0 0x2000>, - <0x0 0x4ac40000 0x0 0x2000>, - <0x0 0x4ac60000 0x0 0x2000>; + <0x0 0x4ac20000 0x0 0x20000>, + <0x0 0x4ac40000 0x0 0x20000>, + <0x0 0x4ac60000 0x0 0x20000>; }; psci { diff --git a/arch/arm64/boot/dts/ti/k3-am62-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62-main.dtsi index f156167b4e8a7..3f74767d63ab7 100644 --- a/arch/arm64/boot/dts/ti/k3-am62-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62-main.dtsi @@ -531,7 +531,6 @@ clock-names = "clk_ahb", "clk_xin"; assigned-clocks = <&k3_clks 57 6>; assigned-clock-parents = <&k3_clks 57 8>; - mmc-ddr-1_8v; mmc-hs200-1_8v; ti,trm-icp = <0x2>; bus-width = <8>; diff --git a/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi b/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi index e931c966b7f22..e98d043e57460 100644 --- a/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi @@ -448,16 +448,16 @@ /* Verdin I2C_2_DSI */ pinctrl_i2c2: main-i2c2-default-pins { pinctrl-single,pins = < - AM62X_IOPAD(0x00b0, PIN_INPUT, 1) /* (K22) GPMC0_CSn2.I2C2_SCL */ /* SODIMM 55 */ - AM62X_IOPAD(0x00b4, PIN_INPUT, 1) /* (K24) GPMC0_CSn3.I2C2_SDA */ /* SODIMM 53 */ + AM62X_IOPAD(0x00b0, PIN_INPUT_PULLUP, 1) /* (K22) GPMC0_CSn2.I2C2_SCL */ /* SODIMM 55 */ + AM62X_IOPAD(0x00b4, PIN_INPUT_PULLUP, 1) /* (K24) GPMC0_CSn3.I2C2_SDA */ /* SODIMM 53 */ >; }; /* Verdin I2C_4_CSI */ pinctrl_i2c3: main-i2c3-default-pins { pinctrl-single,pins = < - AM62X_IOPAD(0x01d0, PIN_INPUT, 2) /* (A15) UART0_CTSn.I2C3_SCL */ /* SODIMM 95 */ - AM62X_IOPAD(0x01d4, PIN_INPUT, 2) /* (B15) UART0_RTSn.I2C3_SDA */ /* SODIMM 93 */ + AM62X_IOPAD(0x01d0, PIN_INPUT_PULLUP, 2) /* (A15) UART0_CTSn.I2C3_SCL */ /* SODIMM 95 */ + AM62X_IOPAD(0x01d4, PIN_INPUT_PULLUP, 2) /* (B15) UART0_RTSn.I2C3_SDA */ /* SODIMM 93 */ >; }; @@ -729,8 +729,8 @@ /* Verdin I2C_3_HDMI */ pinctrl_mcu_i2c0: mcu-i2c0-default-pins { pinctrl-single,pins = < - AM62X_MCU_IOPAD(0x0044, PIN_INPUT, 0) /* (A8) MCU_I2C0_SCL */ /* SODIMM 59 */ - AM62X_MCU_IOPAD(0x0048, PIN_INPUT, 0) /* (D10) MCU_I2C0_SDA */ /* SODIMM 57 */ + AM62X_MCU_IOPAD(0x0044, PIN_INPUT_PULLUP, 0) /* (A8) MCU_I2C0_SCL */ /* SODIMM 59 */ + AM62X_MCU_IOPAD(0x0048, PIN_INPUT_PULLUP, 0) /* (D10) MCU_I2C0_SDA */ /* SODIMM 57 */ >; }; diff --git a/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi index 1497f7c8adfaf..dfdc6e659cc46 100644 --- a/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi @@ -185,7 +185,7 @@ main_pmx0: pinctrl@f4000 { compatible = "pinctrl-single"; - reg = <0x00 0xf4000 0x00 0x2ac>; + reg = <0x00 0xf4000 0x00 0x25c>; #pinctrl-cells = <1>; pinctrl-single,register-width = <32>; pinctrl-single,function-mask = <0xffffffff>; diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts index 99f2878de4c67..12cbf253a8910 100644 --- a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts @@ -144,8 +144,8 @@ main_uart1_pins_default: main-uart1-default-pins { pinctrl-single,pins = < - AM62AX_IOPAD(0x01e8, PIN_INPUT, 1) /* (C17) I2C1_SCL.UART1_RXD */ - AM62AX_IOPAD(0x01ec, PIN_OUTPUT, 1) /* (E17) I2C1_SDA.UART1_TXD */ + AM62AX_IOPAD(0x01ac, PIN_INPUT, 2) /* (B21) MCASP0_AFSR.UART1_RXD */ + AM62AX_IOPAD(0x01b0, PIN_OUTPUT, 2) /* (A21) MCASP0_ACLKR.UART1_TXD */ AM62AX_IOPAD(0x0194, PIN_INPUT, 2) /* (C19) MCASP0_AXR3.UART1_CTSn */ AM62AX_IOPAD(0x0198, PIN_OUTPUT, 2) /* (B19) MCASP0_AXR2.UART1_RTSn */ >; diff --git a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi index 57befcce93b97..0523bd0da80c7 100644 --- a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi @@ -436,6 +436,8 @@ interrupts = ; mmc-ddr-1_8v; mmc-hs200-1_8v; + ti,clkbuf-sel = <0x7>; + ti,trm-icp = <0x8>; ti,otap-del-sel-legacy = <0x0>; ti,otap-del-sel-mmc-hs = <0x0>; ti,otap-del-sel-sd-hs = <0x0>; @@ -446,8 +448,9 @@ ti,otap-del-sel-ddr50 = <0x5>; ti,otap-del-sel-ddr52 = <0x5>; ti,otap-del-sel-hs200 = <0x5>; - ti,otap-del-sel-hs400 = <0x0>; - ti,trm-icp = <0x8>; + ti,itap-del-sel-legacy = <0xa>; + ti,itap-del-sel-mmc-hs = <0x1>; + ti,itap-del-sel-ddr52 = <0x0>; dma-coherent; }; @@ -458,18 +461,22 @@ clocks = <&k3_clks 48 0>, <&k3_clks 48 1>; clock-names = "clk_ahb", "clk_xin"; interrupts = ; + ti,clkbuf-sel = <0x7>; + ti,trm-icp = <0x8>; ti,otap-del-sel-legacy = <0x0>; ti,otap-del-sel-mmc-hs = <0x0>; ti,otap-del-sel-sd-hs = <0x0>; - ti,otap-del-sel-sdr12 = <0x0>; - ti,otap-del-sel-sdr25 = <0x0>; + ti,otap-del-sel-sdr12 = <0xf>; + ti,otap-del-sel-sdr25 = <0xf>; ti,otap-del-sel-sdr50 = <0x8>; ti,otap-del-sel-sdr104 = <0x7>; ti,otap-del-sel-ddr50 = <0x4>; ti,otap-del-sel-ddr52 = <0x4>; ti,otap-del-sel-hs200 = <0x7>; - ti,clkbuf-sel = <0x7>; - ti,trm-icp = <0x8>; + ti,itap-del-sel-legacy = <0xa>; + ti,itap-del-sel-sd-hs = <0x1>; + ti,itap-del-sel-sdr12 = <0xa>; + ti,itap-del-sel-sdr25 = <0x1>; dma-coherent; }; diff --git a/arch/arm64/boot/dts/ti/k3-am68-sk-base-board.dts b/arch/arm64/boot/dts/ti/k3-am68-sk-base-board.dts index 5df5946687b34..2e92f4174b3cf 100644 --- a/arch/arm64/boot/dts/ti/k3-am68-sk-base-board.dts +++ b/arch/arm64/boot/dts/ti/k3-am68-sk-base-board.dts @@ -43,6 +43,17 @@ regulator-boot-on; }; + vsys_5v0: regulator-vsys5v0 { + /* Output of LM61460 */ + compatible = "regulator-fixed"; + regulator-name = "vsys_5v0"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vusb_main>; + regulator-always-on; + regulator-boot-on; + }; + vsys_3v3: regulator-vsys3v3 { /* Output of LM5141 */ compatible = "regulator-fixed"; @@ -75,7 +86,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; regulator-boot-on; - vin-supply = <&vsys_3v3>; + vin-supply = <&vsys_5v0>; gpios = <&main_gpio0 49 GPIO_ACTIVE_HIGH>; states = <1800000 0x0>, <3300000 0x1>; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts index fe5207ac7d85d..90ae8e948671d 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts +++ b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts @@ -557,6 +557,7 @@ &ospi1 { pinctrl-names = "default"; pinctrl-0 = <&mcu_fss0_ospi1_pins_default>; + status = "okay"; flash@0 { compatible = "jedec,spi-nor"; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-sk.dts b/arch/arm64/boot/dts/ti/k3-j721e-sk.dts index ccacb65683b5b..952d2c72628ef 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-j721e-sk.dts @@ -183,6 +183,17 @@ regulator-boot-on; }; + vsys_5v0: fixedregulator-vsys5v0 { + /* Output of LM61460 */ + compatible = "regulator-fixed"; + regulator-name = "vsys_5v0"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vusb_main>; + regulator-always-on; + regulator-boot-on; + }; + vdd_mmc1: fixedregulator-sd { compatible = "regulator-fixed"; pinctrl-names = "default"; @@ -210,6 +221,56 @@ <3300000 0x1>; }; + vdd_sd_dv: gpio-regulator-TLV71033 { + compatible = "regulator-gpio"; + pinctrl-names = "default"; + pinctrl-0 = <&vdd_sd_dv_pins_default>; + regulator-name = "tlv71033"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vsys_5v0>; + gpios = <&main_gpio0 118 GPIO_ACTIVE_HIGH>; + states = <1800000 0x0>, + <3300000 0x1>; + }; + + transceiver1: can-phy1 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + pinctrl-names = "default"; + pinctrl-0 = <&mcu_mcan0_gpio_pins_default>; + standby-gpios = <&wkup_gpio0 3 GPIO_ACTIVE_HIGH>; + }; + + transceiver2: can-phy2 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan0_gpio_pins_default>; + standby-gpios = <&main_gpio0 65 GPIO_ACTIVE_HIGH>; + }; + + transceiver3: can-phy3 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan5_gpio_pins_default>; + standby-gpios = <&main_gpio0 66 GPIO_ACTIVE_HIGH>; + }; + + transceiver4: can-phy4 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan9_gpio_pins_default>; + standby-gpios = <&main_gpio0 67 GPIO_ACTIVE_HIGH>; + }; + dp_pwr_3v3: fixedregulator-dp-prw { compatible = "regulator-fixed"; regulator-name = "dp-pwr"; @@ -286,6 +347,15 @@ }; }; }; + + csi_mux: mux-controller { + compatible = "gpio-mux"; + #mux-state-cells = <1>; + mux-gpios = <&main_gpio0 88 GPIO_ACTIVE_HIGH>; + idle-state = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&main_csi_mux_sel_pins_default>; + }; }; &main_pmx0 { @@ -352,6 +422,51 @@ >; }; + main_csi_mux_sel_pins_default: main-csi-mux-sel-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x164, PIN_OUTPUT, 7) /* (V29) RGMII5_TD2 */ + >; + }; + + main_mcan0_pins_default: main-mcan0-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x208, PIN_INPUT, 0) /* (W5) MCAN0_RX */ + J721E_IOPAD(0x20c, PIN_OUTPUT, 0) /* (W6) MCAN0_TX */ + >; + }; + + main_mcan0_gpio_pins_default: main-mcan0-gpio-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x108, PIN_INPUT, 7) /* (AD27) PRG0_PRU1_GPO2.GPIO0_65 */ + >; + }; + + main_mcan5_pins_default: main-mcan5-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x050, PIN_INPUT, 6) /* (AE21) PRG1_PRU0_GPO18.MCAN5_RX */ + J721E_IOPAD(0x04c, PIN_OUTPUT, 6) /* (AJ21) PRG1_PRU0_GPO17.MCAN5_TX */ + >; + }; + + main_mcan5_gpio_pins_default: main-mcan5-gpio-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x10c, PIN_INPUT, 7) /* (AC25) PRG0_PRU1_GPO3.GPIO0_66 */ + >; + }; + + main_mcan9_pins_default: main-mcan9-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x0d0, PIN_INPUT, 6) /* (AC27) PRG0_PRU0_GPO8.MCAN9_RX */ + J721E_IOPAD(0x0cc, PIN_OUTPUT, 6) /* (AC28) PRG0_PRU0_GPO7.MCAN9_TX */ + >; + }; + + main_mcan9_gpio_pins_default: main-mcan9-gpio-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x110, PIN_INPUT, 7) /* (AD29) PRG0_PRU1_GPO4.GPIO0_67 */ + >; + }; + dp0_pins_default: dp0-default-pins { pinctrl-single,pins = < J721E_IOPAD(0x1c4, PIN_INPUT, 5) /* SPI0_CS1.DP0_HPD */ @@ -511,6 +626,12 @@ >; }; + vdd_sd_dv_pins_default: vdd-sd-dv-default-pins { + pinctrl-single,pins = < + J721E_IOPAD(0x1dc, PIN_OUTPUT, 7) /* (Y1) SPI1_CLK.GPIO0_118 */ + >; + }; + wkup_uart0_pins_default: wkup-uart0-default-pins { pinctrl-single,pins = < J721E_WKUP_IOPAD(0xa0, PIN_INPUT, 0) /* (J29) WKUP_UART0_RXD */ @@ -534,6 +655,19 @@ >; }; + mcu_mcan0_pins_default: mcu-mcan0-default-pins { + pinctrl-single,pins = < + J721E_WKUP_IOPAD(0x0ac, PIN_INPUT, 0) /* (C29) MCU_MCAN0_RX */ + J721E_WKUP_IOPAD(0x0a8, PIN_OUTPUT, 0) /* (D29) MCU_MCAN0_TX */ + >; + }; + + mcu_mcan0_gpio_pins_default: mcu-mcan0-gpio-default-pins { + pinctrl-single,pins = < + J721E_WKUP_IOPAD(0x0bc, PIN_INPUT, 7) /* (F27) WKUP_GPIO0_3 */ + >; + }; + /* Reset for M.2 M Key slot on PCIe1 */ mkey_reset_pins_default: mkey-reset-pns-default-pins { pinctrl-single,pins = < @@ -707,14 +841,14 @@ reg = <0x70>; /* CSI0 I2C */ - i2c@0 { + cam0_i2c: i2c@0 { #address-cells = <1>; #size-cells = <0>; reg = <0>; }; /* CSI1 I2C */ - i2c@1 { + cam1_i2c: i2c@1 { #address-cells = <1>; #size-cells = <0>; reg = <1>; @@ -942,6 +1076,34 @@ num-lanes = <2>; }; +&mcu_mcan0 { + pinctrl-names = "default"; + pinctrl-0 = <&mcu_mcan0_pins_default>; + phys = <&transceiver1>; + status = "okay"; +}; + +&main_mcan0 { + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan0_pins_default>; + phys = <&transceiver2>; + status = "okay"; +}; + +&main_mcan5 { + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan5_pins_default>; + phys = <&transceiver3>; + status = "okay"; +}; + +&main_mcan9 { + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan9_pins_default>; + phys = <&transceiver4>; + status = "okay"; +}; + &ufs_wrapper { status = "disabled"; }; diff --git a/arch/arm64/boot/dts/ti/k3-pinctrl.h b/arch/arm64/boot/dts/ti/k3-pinctrl.h index 2a4e0e084d695..8e0869d9b7a0e 100644 --- a/arch/arm64/boot/dts/ti/k3-pinctrl.h +++ b/arch/arm64/boot/dts/ti/k3-pinctrl.h @@ -8,11 +8,16 @@ #ifndef DTS_ARM64_TI_K3_PINCTRL_H #define DTS_ARM64_TI_K3_PINCTRL_H +#define ST_EN_SHIFT (14) #define PULLUDEN_SHIFT (16) #define PULLTYPESEL_SHIFT (17) #define RXACTIVE_SHIFT (18) #define DEBOUNCE_SHIFT (11) +/* Schmitt trigger configuration */ +#define ST_DISABLE (0 << ST_EN_SHIFT) +#define ST_ENABLE (1 << ST_EN_SHIFT) + #define PULL_DISABLE (1 << PULLUDEN_SHIFT) #define PULL_ENABLE (0 << PULLUDEN_SHIFT) @@ -26,9 +31,13 @@ #define PIN_OUTPUT (INPUT_DISABLE | PULL_DISABLE) #define PIN_OUTPUT_PULLUP (INPUT_DISABLE | PULL_UP) #define PIN_OUTPUT_PULLDOWN (INPUT_DISABLE | PULL_DOWN) -#define PIN_INPUT (INPUT_EN | PULL_DISABLE) -#define PIN_INPUT_PULLUP (INPUT_EN | PULL_UP) -#define PIN_INPUT_PULLDOWN (INPUT_EN | PULL_DOWN) +#define PIN_INPUT (INPUT_EN | ST_ENABLE | PULL_DISABLE) +#define PIN_INPUT_PULLUP (INPUT_EN | ST_ENABLE | PULL_UP) +#define PIN_INPUT_PULLDOWN (INPUT_EN | ST_ENABLE | PULL_DOWN) +/* Input configurations with Schmitt Trigger disabled */ +#define PIN_INPUT_NOST (INPUT_EN | PULL_DISABLE) +#define PIN_INPUT_PULLUP_NOST (INPUT_EN | PULL_UP) +#define PIN_INPUT_PULLDOWN_NOST (INPUT_EN | PULL_DOWN) #define PIN_DEBOUNCE_DISABLE (0 << DEBOUNCE_SHIFT) #define PIN_DEBOUNCE_CONF1 (1 << DEBOUNCE_SHIFT) diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi index ccaca29200bb9..995bd8ce9d43a 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi @@ -10,39 +10,44 @@ #include / { - pss_ref_clk: pss_ref_clk { + pss_ref_clk: pss-ref-clk { bootph-all; compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <33333333>; + clock-output-names = "pss_ref_clk"; }; - video_clk: video_clk { + video_clk: video-clk { bootph-all; compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <27000000>; + clock-output-names = "video_clk"; }; - pss_alt_ref_clk: pss_alt_ref_clk { + pss_alt_ref_clk: pss-alt-ref-clk { bootph-all; compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <0>; + clock-output-names = "pss_alt_ref_clk"; }; - gt_crx_ref_clk: gt_crx_ref_clk { + gt_crx_ref_clk: gt-crx-ref-clk { bootph-all; compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <108000000>; + clock-output-names = "gt_crx_ref_clk"; }; - aux_ref_clk: aux_ref_clk { + aux_ref_clk: aux-ref-clk { bootph-all; compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <27000000>; + clock-output-names = "aux_ref_clk"; }; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts index 50c384aa253e4..8c3f9735e563b 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts @@ -808,8 +808,8 @@ pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", "MIO60", "MIO61", "MIO62", "MIO63"; bias-disable; - drive-strength = <4>; - slew-rate = ; + drive-strength = <12>; + slew-rate = ; }; }; diff --git a/arch/arm64/configs/consolidate.fragment b/arch/arm64/configs/consolidate.fragment new file mode 100755 index 0000000000000..c4d613e77752d --- /dev/null +++ b/arch/arm64/configs/consolidate.fragment @@ -0,0 +1,84 @@ +# CONFIG_ATOMIC64_SELFTEST is not set +# CONFIG_BACKTRACE_SELF_TEST is not set +# CONFIG_BITFIELD_KUNIT is not set +# CONFIG_BITS_TEST is not set +CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y +CONFIG_CMDLINE="kasan.stacktrace=off stack_depot_disable=off page_owner=on no_hash_pointers panic_on_taint=0x20" +CONFIG_DEBUG_ATOMIC_SLEEP=y +CONFIG_DEBUG_IRQFLAGS=y +CONFIG_DEBUG_KMEMLEAK=y +# CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN is not set +CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF=y +CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE=16000 +# CONFIG_DEBUG_KOBJECT_RELEASE is not set +CONFIG_DEBUG_LOCKDEP=y +CONFIG_DEBUG_LOCK_ALLOC=y +CONFIG_DEBUG_MUTEXES=y +CONFIG_DEBUG_OBJECTS=y +CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT=1 +CONFIG_DEBUG_OBJECTS_FREE=y +# CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER is not set +# CONFIG_DEBUG_OBJECTS_RCU_HEAD is not set +# CONFIG_DEBUG_OBJECTS_SELFTEST is not set +CONFIG_DEBUG_OBJECTS_TIMERS=y +CONFIG_DEBUG_OBJECTS_WORK=y +CONFIG_DEBUG_PREEMPT=y +CONFIG_DEBUG_RT_MUTEXES=y +CONFIG_DEBUG_RWSEMS=y +CONFIG_DEBUG_SPINLOCK=y +CONFIG_DPM_WATCHDOG=y +CONFIG_DPM_WATCHDOG_TIMEOUT=120 +CONFIG_DYNAMIC_DEBUG=y +# CONFIG_FIND_BIT_BENCHMARK is not set +CONFIG_GKI_HACKS_FOR_DEBUG_CONFIG=y +CONFIG_I2C_CHARDEV=y +# CONFIG_INTERVAL_TREE_TEST is not set +# CONFIG_KPROBES_SANITY_TEST is not set +# CONFIG_LINEAR_RANGES_TEST is not set +# CONFIG_LIST_KUNIT_TEST is not set +# CONFIG_LKDTM is not set +CONFIG_LOCALVERSION="-debug" +CONFIG_LOCKDEP=y +CONFIG_LOCKUP_DETECTOR=y +CONFIG_LOCK_STAT=y +CONFIG_PAGE_POISONING=y +# CONFIG_PERCPU_TEST is not set +CONFIG_PID_IN_CONTEXTIDR=y +CONFIG_PM_DEBUG=y +CONFIG_PM_SLEEP_DEBUG=y +# CONFIG_PM_TEST_SUSPEND is not set +CONFIG_PREEMPTIRQ_TRACEPOINTS=y +# CONFIG_RBTREE_TEST is not set +# CONFIG_REED_SOLOMON_TEST is not set +CONFIG_RUNTIME_TESTING_MENU=y +CONFIG_SOFTLOCKUP_DETECTOR=y +CONFIG_SPI_SPIDEV=y +# CONFIG_SYSCTL_KUNIT_TEST is not set +CONFIG_TASKS_RUDE_RCU=y +# CONFIG_TEST_BITMAP is not set +# CONFIG_TEST_BITOPS is not set +# CONFIG_TEST_BLACKHOLE_DEV is not set +# CONFIG_TEST_BPF is not set +# CONFIG_TEST_FIRMWARE is not set +# CONFIG_TEST_FREE_PAGES is not set +# CONFIG_TEST_HEXDUMP is not set +# CONFIG_TEST_IDA is not set +# CONFIG_TEST_KMOD is not set +# CONFIG_TEST_KSTRTOX is not set +# CONFIG_TEST_LIST_SORT is not set +# CONFIG_TEST_LKM is not set +# CONFIG_TEST_MEMCAT_P is not set +# CONFIG_TEST_MEMINIT is not set +# CONFIG_TEST_MIN_HEAP is not set +# CONFIG_TEST_PRINTF is not set +# CONFIG_TEST_RHASHTABLE is not set +# CONFIG_TEST_SORT is not set +# CONFIG_TEST_STATIC_KEYS is not set +# CONFIG_TEST_STRING_HELPERS is not set +# CONFIG_TEST_SYSCTL is not set +# CONFIG_TEST_UDELAY is not set +# CONFIG_TEST_USER_COPY is not set +# CONFIG_TEST_UUID is not set +# CONFIG_TEST_VMALLOC is not set +# CONFIG_TEST_XARRAY is not set +CONFIG_TRACE_PREEMPT_TOGGLE=y diff --git a/arch/arm64/configs/crashdump_defconfig b/arch/arm64/configs/crashdump_defconfig index 75866b00b4037..0f925c42f3b6e 100644 --- a/arch/arm64/configs/crashdump_defconfig +++ b/arch/arm64/configs/crashdump_defconfig @@ -1,6 +1,5 @@ # CONFIG_WERROR is not set # CONFIG_LOCALVERSION_AUTO is not set -# CONFIG_SWAP is not set # CONFIG_CROSS_MEMORY_ATTACH is not set CONFIG_NO_HZ_IDLE=y CONFIG_PREEMPT=y @@ -18,15 +17,11 @@ CONFIG_BLK_DEV_INITRD=y # CONFIG_RD_LZ4 is not set # CONFIG_RD_ZSTD is not set CONFIG_CC_OPTIMIZE_FOR_SIZE=y -# CONFIG_COMPAT_BRK is not set -# CONFIG_SLAB_MERGE_DEFAULT is not set -# CONFIG_SLUB_CPU_PARTIAL is not set +CONFIG_CRASH_DUMP=y CONFIG_ARM64_VA_BITS_48=y CONFIG_NR_CPUS=2 -CONFIG_CRASH_DUMP=y # CONFIG_RODATA_FULL_DEFAULT_ENABLED is not set # CONFIG_ARM64_TAGGED_ADDR_ABI is not set -# CONFIG_ARM64_SVE is not set # CONFIG_EFI is not set # CONFIG_SUSPEND is not set CONFIG_JUMP_LABEL=y @@ -37,7 +32,10 @@ CONFIG_JUMP_LABEL=y # CONFIG_MQ_IOSCHED_KYBER is not set # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set # CONFIG_BINFMT_SCRIPT is not set -# CONFIG_SPECULATIVE_PAGE_FAULT is not set +# CONFIG_SWAP is not set +# CONFIG_SLAB_MERGE_DEFAULT is not set +# CONFIG_SLUB_CPU_PARTIAL is not set +# CONFIG_COMPAT_BRK is not set CONFIG_PCI=y CONFIG_PCI_HOST_GENERIC=y CONFIG_PCI_ENDPOINT=y @@ -77,5 +75,6 @@ CONFIG_TMPFS=y CONFIG_TMPFS_XATTR=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_XZ_DEC=y +CONFIG_DMA_RESTRICTED_POOL=y # CONFIG_SYMBOLIC_ERRNAME is not set # CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 60af93c04b45a..b65b815db956a 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -6,6 +6,8 @@ CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y CONFIG_PREEMPT=y +CONFIG_SLIM_SCHED=y +CONFIG_SCHED_CLASS_EXT=y CONFIG_IRQ_TIME_ACCOUNTING=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_BSD_PROCESS_ACCT_V3=y @@ -1412,6 +1414,9 @@ CONFIG_PHY_HISTB_COMBPHY=y CONFIG_PHY_HISI_INNO_USB2=y CONFIG_PHY_MVEBU_CP110_COMPHY=y CONFIG_PHY_MTK_TPHY=y +CONFIG_PHY_MTK_HDMI=m +CONFIG_PHY_MTK_MIPI_DSI=m +CONFIG_PHY_MTK_DP=m CONFIG_PHY_QCOM_EDP=m CONFIG_PHY_QCOM_EUSB2_REPEATER=m CONFIG_PHY_QCOM_PCIE2=m diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 0587a3186f57b..9d7fc2a7515e8 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -10,6 +10,8 @@ CONFIG_BPF_JIT_ALWAYS_ON=y # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set CONFIG_BPF_LSM=y CONFIG_PREEMPT=y +CONFIG_SLIM_SCHED=y +CONFIG_SCHED_CLASS_EXT=y CONFIG_IRQ_TIME_ACCOUNTING=y CONFIG_TASKSTATS=y CONFIG_TASK_DELAY_ACCT=y @@ -186,6 +188,7 @@ CONFIG_NF_CONNTRACK_PPTP=y CONFIG_NF_CONNTRACK_SANE=y CONFIG_NF_CONNTRACK_TFTP=y CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_XTABLES_COMPAT=y CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y CONFIG_NETFILTER_XT_TARGET_CONNMARK=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y @@ -328,6 +331,8 @@ CONFIG_ARM_SCPI_PROTOCOL=y # CONFIG_EFI_ARMSTUB_DTB_LOADER is not set CONFIG_GNSS=y CONFIG_ZRAM=m +CONFIG_ZRAM_WRITEBACK=y +CONFIG_ZRAM_MULTI_COMP=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -738,6 +743,7 @@ CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y CONFIG_CRYPTO_SHA2_ARM64_CE=y CONFIG_CRYPTO_SHA512_ARM64_CE=y CONFIG_CRYPTO_POLYVAL_ARM64_CE=y diff --git a/arch/arm64/configs/microdroid_defconfig b/arch/arm64/configs/microdroid_defconfig index f67d2f9d0aa03..a92ce39a5f977 100644 --- a/arch/arm64/configs/microdroid_defconfig +++ b/arch/arm64/configs/microdroid_defconfig @@ -8,6 +8,8 @@ CONFIG_RCU_EXPERT=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 +CONFIG_CGROUPS=y +CONFIG_MEMCG=y # CONFIG_RD_GZIP is not set # CONFIG_RD_BZIP2 is not set # CONFIG_RD_LZMA is not set @@ -16,6 +18,7 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_RD_ZSTD is not set CONFIG_BOOT_CONFIG=y CONFIG_EXPERT=y +# CONFIG_BASE_FULL is not set # CONFIG_IO_URING is not set CONFIG_PROFILING=y CONFIG_KEXEC_FILE=y @@ -23,18 +26,19 @@ CONFIG_SCHED_MC=y CONFIG_NR_CPUS=32 CONFIG_PARAVIRT_TIME_ACCOUNTING=y CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_BTI_KERNEL is not set CONFIG_RANDOMIZE_BASE=y # CONFIG_RANDOMIZE_MODULE_REGION_FULL is not set +CONFIG_UNWIND_PATCH_PAC_INTO_SCS=y CONFIG_CMDLINE="stack_depot_disable=on kasan.stacktrace=off cgroup_disable=pressure ioremap_guard panic=-1 bootconfig" CONFIG_CMDLINE_EXTEND=y # CONFIG_EFI is not set -CONFIG_PM_WAKELOCKS=y -CONFIG_PM_WAKELOCKS_LIMIT=0 -# CONFIG_PM_WAKELOCKS_GC is not set +# CONFIG_SUSPEND is not set CONFIG_CPU_FREQ=y CONFIG_ANDROID_V_CPUFREQ_VIRT=y CONFIG_VIRTUALIZATION=y CONFIG_JUMP_LABEL=y +# CONFIG_SECCOMP is not set CONFIG_SHADOW_CALL_STACK=y CONFIG_CFI_CLANG=y # CONFIG_BLOCK_LEGACY_AUTOLOAD is not set @@ -50,17 +54,28 @@ CONFIG_SLAB_FREELIST_HARDENED=y CONFIG_SHUFFLE_PAGE_ALLOCATOR=y # CONFIG_COMPAT_BRK is not set CONFIG_DEFAULT_MMAP_MIN_ADDR=32768 -CONFIG_TRANSPARENT_HUGEPAGE=y -CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y +# CONFIG_ZONE_DMA is not set +# CONFIG_ZONE_DMA32 is not set CONFIG_ANON_VMA_NAME=y CONFIG_USERFAULTFD=y CONFIG_LRU_GEN=y CONFIG_NET=y CONFIG_UNIX=y CONFIG_INET=y +CONFIG_INET_TABLE_PERTURB_ORDER=0 +# CONFIG_INET_DIAG is not set +CONFIG_TCP_CONG_ADVANCED=y +# CONFIG_TCP_CONG_BIC is not set +# CONFIG_TCP_CONG_CUBIC is not set +# CONFIG_TCP_CONG_WESTWOOD is not set +# CONFIG_TCP_CONG_HTCP is not set +# CONFIG_IPV6 is not set CONFIG_VSOCKETS=y CONFIG_VIRTIO_VSOCKETS=y +# CONFIG_RPS is not set +# CONFIG_BQL is not set # CONFIG_WIRELESS is not set +# CONFIG_ETHTOOL_NETLINK is not set CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCIEAER=y @@ -68,32 +83,33 @@ CONFIG_PCI_IOV=y # CONFIG_VGA_ARB is not set CONFIG_PCI_HOST_GENERIC=y CONFIG_PCI_ENDPOINT=y -CONFIG_FW_LOADER_USER_HELPER=y -# CONFIG_FW_CACHE is not set +# CONFIG_FW_LOADER is not set CONFIG_ARM_SCMI_PROTOCOL=y # CONFIG_ARM_SCMI_POWER_DOMAIN is not set CONFIG_ZRAM=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=8192 CONFIG_VIRTIO_BLK=y CONFIG_OPEN_DICE=y CONFIG_VCPU_STALL_DETECTOR=y CONFIG_MD=y +# CONFIG_MD_BITMAP_FILE is not set CONFIG_BLK_DEV_DM=y CONFIG_DM_CRYPT=y CONFIG_DM_UEVENT=y CONFIG_DM_VERITY=y CONFIG_DM_VERITY_FEC=y +# CONFIG_DM_USER is not set # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set # CONFIG_LEGACY_PTYS is not set CONFIG_SERIAL_8250=y # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +# CONFIG_SERIAL_8250_16550A_VARIANTS is not set CONFIG_SERIAL_8250_CONSOLE=y # CONFIG_SERIAL_8250_PCI is not set CONFIG_SERIAL_8250_RUNTIME_UARTS=0 +# CONFIG_SERIAL_8250_PERICOM is not set CONFIG_SERIAL_OF_PLATFORM=y CONFIG_NULL_TTY=y CONFIG_VIRTIO_CONSOLE=y @@ -104,19 +120,20 @@ CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_WATCHDOG=y CONFIG_WATCHDOG_CORE=y -# CONFIG_HID is not set +# CONFIG_HID_SUPPORT is not set # CONFIG_USB_SUPPORT is not set CONFIG_EDAC=y CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set CONFIG_RTC_DRV_PL030=y CONFIG_RTC_DRV_PL031=y -CONFIG_DMABUF_HEAPS=y -CONFIG_DMABUF_SYSFS_STATS=y CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_BALLOON=y -CONFIG_STAGING=y +# CONFIG_SURFACE_PLATFORMS is not set CONFIG_HWSPINLOCK=y +# CONFIG_IOMMU_SUPPORT is not set +# CONFIG_ANDROID_KABI_RESERVE is not set +# CONFIG_ANDROID_VENDOR_OEM_DATA is not set CONFIG_EXT4_FS=y # CONFIG_EXT4_USE_FOR_EXT2 is not set CONFIG_EXT4_FS_POSIX_ACL=y @@ -136,9 +153,10 @@ CONFIG_STATIC_USERMODEHELPER_PATH="" CONFIG_SECURITY_SELINUX=y CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y CONFIG_BUG_ON_DATA_CORRUPTION=y -CONFIG_CRYPTO_SHA1=y CONFIG_CRYPTO_HCTR2=y CONFIG_CRYPTO_LZO=y +CONFIG_CRYPTO_LZ4=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y CONFIG_CRYPTO_SHA2_ARM64_CE=y CONFIG_CRYPTO_POLYVAL_ARM64_CE=y CONFIG_CRYPTO_AES_ARM64_CE_BLK=y @@ -146,7 +164,6 @@ CONFIG_TRACE_MMIO_ACCESS=y CONFIG_DMA_RESTRICTED_POOL=y CONFIG_PRINTK_TIME=y CONFIG_PRINTK_CALLER=y -CONFIG_DYNAMIC_DEBUG_CORE=y CONFIG_DEBUG_INFO_DWARF5=y CONFIG_DEBUG_INFO_REDUCED=y CONFIG_HEADERS_INSTALL=y @@ -166,6 +183,7 @@ CONFIG_PANIC_TIMEOUT=-1 CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_WQ_WATCHDOG=y CONFIG_SCHEDSTATS=y +# CONFIG_UPROBE_EVENTS is not set CONFIG_HIST_TRIGGERS=y CONFIG_PID_IN_CONTEXTIDR=y # CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/arm64/gunyah/gunyah_hypercall.c b/arch/arm64/gunyah/gunyah_hypercall.c index 3c2672d683ae2..7cfdfdb5ab30a 100644 --- a/arch/arm64/gunyah/gunyah_hypercall.c +++ b/arch/arm64/gunyah/gunyah_hypercall.c @@ -8,6 +8,7 @@ #include #include +#define GUNYAH_VCPU_RUN_STATE_PSCI_SYSTEM_RESET 256 /* {c1d58fcd-a453-5fdb-9265-ce36673d5f14} */ static const uuid_t GUNYAH_UUID = UUID_INIT(0xc1d58fcd, 0xa453, 0x5fdb, 0x92, 0x65, 0xce, 0x36, 0x67, 0x3d, 0x5f, @@ -271,6 +272,14 @@ gunyah_hypercall_vcpu_run(u64 capid, unsigned long *resume_data, resp->state_data[2] = res.a4; } + /* + * PSCI_SYSTEM_RESET is also a state where VM is shutdown + * Translate it to GUNYAH_VCPU_STATE_SYSTEM_OFF as VMM will + * be able to take the action based on the exit_info. + */ + if (resp->sized_state == GUNYAH_VCPU_RUN_STATE_PSCI_SYSTEM_RESET) + resp->sized_state = GUNYAH_VCPU_STATE_SYSTEM_OFF; + return res.a0; } EXPORT_SYMBOL_GPL(gunyah_hypercall_vcpu_run); diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h index a407f9cd549ed..c07a58b96329d 100644 --- a/arch/arm64/include/asm/acpi.h +++ b/arch/arm64/include/asm/acpi.h @@ -150,7 +150,7 @@ acpi_set_mailbox_entry(int cpu, struct acpi_madt_generic_interrupt *processor) {} #endif -static inline const char *acpi_get_enable_method(int cpu) +static __always_inline const char *acpi_get_enable_method(int cpu) { if (acpi_psci_present()) return "psci"; diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index af3b93354c945..210b07bfe3afd 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -45,6 +45,11 @@ /* * Save/restore interrupts. */ + .macro save_and_disable_daif, flags + mrs \flags, daif + msr daifset, #0xf + .endm + .macro save_and_disable_irq, flags mrs \flags, daif msr daifset, #3 diff --git a/arch/arm64/include/asm/cfi.h b/arch/arm64/include/asm/cfi.h new file mode 100644 index 0000000000000..ab90f0351b7ae --- /dev/null +++ b/arch/arm64/include/asm/cfi.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ARM64_CFI_H +#define _ASM_ARM64_CFI_H + +#define __bpfcall + +#endif /* _ASM_ARM64_CFI_H */ diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 2a4e686e633c6..c279a0a9b3660 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -81,6 +81,7 @@ #define ARM_CPU_PART_CORTEX_A78AE 0xD42 #define ARM_CPU_PART_CORTEX_X1 0xD44 #define ARM_CPU_PART_CORTEX_A510 0xD46 +#define ARM_CPU_PART_CORTEX_X1C 0xD4C #define ARM_CPU_PART_CORTEX_A520 0xD80 #define ARM_CPU_PART_CORTEX_A710 0xD47 #define ARM_CPU_PART_CORTEX_A715 0xD4D @@ -92,6 +93,7 @@ #define ARM_CPU_PART_NEOVERSE_V2 0xD4F #define ARM_CPU_PART_CORTEX_A720 0xD81 #define ARM_CPU_PART_CORTEX_X4 0xD82 +#define ARM_CPU_PART_NEOVERSE_V3AE 0xD83 #define ARM_CPU_PART_NEOVERSE_V3 0xD84 #define ARM_CPU_PART_CORTEX_X925 0xD85 #define ARM_CPU_PART_CORTEX_A725 0xD87 @@ -131,6 +133,7 @@ #define FUJITSU_CPU_PART_A64FX 0x001 #define HISI_CPU_PART_TSV110 0xD01 +#define HISI_CPU_PART_HIP09 0xD02 #define APPLE_CPU_PART_M1_ICESTORM 0x022 #define APPLE_CPU_PART_M1_FIRESTORM 0x023 @@ -166,6 +169,7 @@ #define MIDR_CORTEX_A78AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78AE) #define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1) #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510) +#define MIDR_CORTEX_X1C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1C) #define MIDR_CORTEX_A520 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520) #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) #define MIDR_CORTEX_A715 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A715) @@ -177,6 +181,7 @@ #define MIDR_NEOVERSE_V2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V2) #define MIDR_CORTEX_A720 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720) #define MIDR_CORTEX_X4 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X4) +#define MIDR_NEOVERSE_V3AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V3AE) #define MIDR_NEOVERSE_V3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V3) #define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925) #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) @@ -206,6 +211,7 @@ #define MIDR_NVIDIA_CARMEL MIDR_CPU_MODEL(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_CARMEL) #define MIDR_FUJITSU_A64FX MIDR_CPU_MODEL(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) #define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) +#define MIDR_HISI_HIP09 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_HIP09) #define MIDR_APPLE_M1_ICESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) #define MIDR_APPLE_M1_FIRESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) #define MIDR_APPLE_M1_ICESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_PRO) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 5cdb1f77f9af9..bcd716ab9e7db 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -366,12 +366,14 @@ /* * ISS values for SME traps */ - -#define ESR_ELx_SME_ISS_SME_DISABLED 0 -#define ESR_ELx_SME_ISS_ILL 1 -#define ESR_ELx_SME_ISS_SM_DISABLED 2 -#define ESR_ELx_SME_ISS_ZA_DISABLED 3 -#define ESR_ELx_SME_ISS_ZT_DISABLED 4 +#define ESR_ELx_SME_ISS_SMTC_MASK GENMASK(2, 0) +#define ESR_ELx_SME_ISS_SMTC(esr) ((esr) & ESR_ELx_SME_ISS_SMTC_MASK) + +#define ESR_ELx_SME_ISS_SMTC_SME_DISABLED 0 +#define ESR_ELx_SME_ISS_SMTC_ILL 1 +#define ESR_ELx_SME_ISS_SMTC_SM_DISABLED 2 +#define ESR_ELx_SME_ISS_SMTC_ZA_DISABLED 3 +#define ESR_ELx_SME_ISS_SMTC_ZT_DISABLED 4 /* ISS field definitions for MOPS exceptions */ #define ESR_ELx_MOPS_ISS_MEM_INST (UL(1) << 24) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 7415c63b41874..c4840ea6d137f 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -6,6 +6,7 @@ #define __ASM_FP_H #include +#include #include #include #include @@ -69,6 +70,8 @@ struct cpu_fp_state { enum fp_type to_save; }; +DECLARE_PER_CPU(struct cpu_fp_state, fpsimd_last_state); + extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state); extern void fpsimd_flush_task_state(struct task_struct *target); diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 0ccf51afde31a..1d54ea97218ef 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -570,6 +570,11 @@ static __always_inline u32 aarch64_insn_gen_nop(void) return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP); } +static __always_inline bool aarch64_insn_is_nop(u32 insn) +{ + return insn == aarch64_insn_gen_nop(); +} + u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg, enum aarch64_insn_branch_type type); u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg, @@ -687,6 +692,7 @@ u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, } #endif u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type); +u32 aarch64_insn_gen_dsb(enum aarch64_insn_mb_type type); s32 aarch64_get_branch_offset(u32 insn); u32 aarch64_set_branch_offset(u32 insn, s32 offset); diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 7a05d61cd8efc..bf50d172a8a3c 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -98,7 +98,7 @@ HCR_BSU_IS | HCR_FB | HCR_TACR | \ HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3) -#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA) +#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK) #define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 9cc0033513a9c..ee4dcc9855b88 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -115,6 +115,9 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_host_iommu_iova_to_phys, __KVM_HOST_SMCCC_FUNC___pkvm_host_hvc_pd, __KVM_HOST_SMCCC_FUNC___pkvm_stage2_snapshot, + __KVM_HOST_SMCCC_FUNC___pkvm_host_iommu_iotlb_sync_map, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_pool_report_free_pages, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_pool_report_min_free_pages, /* * Start of the dynamically registered hypercalls. Start a bit diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 4a16808c3ba89..a2b556031587f 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -559,11 +559,16 @@ static inline unsigned long __hyp_pgtable_moveable_regs_pages(void) return res; } +extern u64 kvm_nvhe_sym(hyp_lm_size_mb); + static inline unsigned long hyp_s1_pgtable_pages(void) { unsigned long res; - res = __hyp_pgtable_moveable_regs_pages(); + if (!kvm_nvhe_sym(hyp_lm_size_mb)) + res = __hyp_pgtable_moveable_regs_pages(); + else + res = __hyp_pgtable_max_pages(kvm_nvhe_sym(hyp_lm_size_mb) * SZ_1M / PAGE_SIZE); /* Allow 1 GiB for private mappings */ res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); @@ -593,7 +598,7 @@ static inline unsigned long host_s2_pgtable_pages(void) * Maximum number of consitutents allowed in a descriptor. This number is * arbitrary, see comment below on SG_MAX_SEGMENTS in hyp_ffa_proxy_pages(). */ -#define KVM_FFA_MAX_NR_CONSTITUENTS 4096 +#define KVM_FFA_MAX_NR_CONSTITUENTS 12288 static inline unsigned long hyp_ffa_proxy_pages(void) { @@ -661,4 +666,9 @@ int pkvm_call_hyp_nvhe_ppage(struct kvm_pinned_page *ppage, int pkvm_guest_stage2_pa(pkvm_handle_t handle, u64 ipa, phys_addr_t *phys); +#ifdef CONFIG_DEBUG_FS +void kvm_hyp_s1_pool_debugfs(void); +#else +static inline void kvm_hyp_s1_pool_debugfs(void) { } +#endif #endif /* __ARM64_KVM_PKVM_H__ */ diff --git a/arch/arm64/include/asm/kvm_pkvm_module.h b/arch/arm64/include/asm/kvm_pkvm_module.h index 2be3f122212e0..77c4b7c717f1f 100644 --- a/arch/arm64/include/asm/kvm_pkvm_module.h +++ b/arch/arm64/include/asm/kvm_pkvm_module.h @@ -336,5 +336,16 @@ static inline int pkvm_register_el2_mod_call(dyn_hcall_t hfn, \ res.a1; \ }) + +#define pkvm_el2_mod_call_smccc(id, ...) \ + ({ \ + struct arm_smccc_res res; \ + \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_ID(id), \ + ##__VA_ARGS__, &res); \ + WARN_ON(res.a0 != SMCCC_RET_SUCCESS); \ + \ + res; \ + }) #endif #endif diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h old mode 100644 new mode 100755 index a038d31a99db6..a28ac5af65858 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -74,7 +74,7 @@ #define KASAN_SHADOW_END ((UL(1) << (64 - KASAN_SHADOW_SCALE_SHIFT)) \ + KASAN_SHADOW_OFFSET) #define PAGE_END (KASAN_SHADOW_END - (1UL << (vabits_actual - KASAN_SHADOW_SCALE_SHIFT))) -#define KASAN_THREAD_SHIFT 1 +#define KASAN_THREAD_SHIFT 2 #else #define KASAN_THREAD_SHIFT 0 #define PAGE_END (_PAGE_END(VA_BITS_MIN)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 0d1e7cd2159fc..3a9e6c551617f 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -183,7 +183,8 @@ static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot) static inline pte_t pte_mkwrite_novma(pte_t pte) { pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); - pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); + if (pte_sw_dirty(pte)) + pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte; } @@ -696,7 +697,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) #define pud_none(pud) (!pud_val(pud)) -#define pud_bad(pud) (!pud_table(pud)) +#define pud_bad(pud) ((pud_val(pud) & PUD_TYPE_MASK) != \ + PUD_TYPE_TABLE) #define pud_present(pud) pte_present(pud_pte(pud)) #define pud_leaf(pud) (pud_present(pud) && !pud_table(pud)) #define pud_valid(pud) pte_valid(pud_pte(pud)) diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h index 0c2b47673922e..32475d19c15f4 100644 --- a/arch/arm64/include/asm/spectre.h +++ b/arch/arm64/include/asm/spectre.h @@ -97,6 +97,9 @@ enum mitigation_state arm64_get_meltdown_state(void); enum mitigation_state arm64_get_spectre_bhb_state(void); bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); +extern bool __nospectre_bhb; +u8 get_spectre_bhb_loop_value(void); +bool is_spectre_bhb_fw_mitigated(void); void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr); diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index a87cc5a29d124..fad8e4cf1daa4 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -304,13 +304,14 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) } /* - * If mprotect/munmap/etc occurs during TLB batched flushing, we need to - * synchronise all the TLBI issued with a DSB to avoid the race mentioned in - * flush_tlb_batched_pending(). + * If mprotect/munmap/etc occurs during TLB batched flushing, we need to ensure + * all the previously issued TLBIs targeting mm have completed. But since we + * can be executing on a remote CPU, a DSB cannot guarantee this like it can + * for arch_tlbbatch_flush(). Our only option is to flush the entire mm. */ static inline void arch_flush_tlb_batched_pending(struct mm_struct *mm) { - dsb(ish); + flush_tlb_mm(mm); } /* diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 9fab663dd2de9..a323b109b9c44 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -23,6 +23,7 @@ void update_freq_counters_refs(void); #define arch_set_freq_scale topology_set_freq_scale #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant +#define arch_scale_freq_ref topology_get_freq_ref #ifdef CONFIG_ACPI_CPPC_LIB #define arch_init_invariance_cppc topology_init_cpu_capacity_cppc diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 5a5201a29b103..60b2f2c84842f 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -6,6 +6,7 @@ * 2001-2002 Keith Owens * Copyright (C) 2012 ARM Ltd. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 463b48d0f9250..23dcd0c4aad36 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -471,6 +471,7 @@ static const struct midr_range erratum_spec_ssbs_list[] = { MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), {} }; #endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 1a810dca482d4..68807c60f4f0d 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -364,6 +364,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { }; static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ECBHB_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HCX_SHIFT, 4, 0), @@ -2174,17 +2175,21 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused) #ifdef CONFIG_ARM64_MTE static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) { + static bool cleared_zero_page = false; + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); mte_cpu_setup(); /* * Clear the tags in the zero page. This needs to be done via the - * linear map which has the Tagged attribute. + * linear map which has the Tagged attribute. Since this page is + * always mapped as pte_special(), set_pte_at() will not attempt to + * clear the tags or set PG_mte_tagged. */ - if (try_page_mte_tagging(ZERO_PAGE(0))) { + if (!cleared_zero_page) { + cleared_zero_page = true; mte_clear_page_tags(lm_alias(empty_zero_page)); - set_page_mte_tagged(ZERO_PAGE(0)); } kasan_init_hw_tags_cpu(); @@ -2805,6 +2810,13 @@ static bool has_sve_feature(const struct arm64_cpu_capabilities *cap, int scope) } #endif +#ifdef CONFIG_ARM64_SME +static bool has_sme_feature(const struct arm64_cpu_capabilities *cap, int scope) +{ + return system_supports_sme() && has_user_cpuid_feature(cap, scope); +} +#endif + static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR0_EL1, AES, PMULL, CAP_HWCAP, KERNEL_HWCAP_PMULL), HWCAP_CAP(ID_AA64ISAR0_EL1, AES, AES, CAP_HWCAP, KERNEL_HWCAP_AES), @@ -2876,20 +2888,20 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR2_EL1, MOPS, IMP, CAP_HWCAP, KERNEL_HWCAP_MOPS), HWCAP_CAP(ID_AA64ISAR2_EL1, BC, IMP, CAP_HWCAP, KERNEL_HWCAP_HBC), #ifdef CONFIG_ARM64_SME - HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), - HWCAP_CAP(ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), - HWCAP_CAP(ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), - HWCAP_CAP(ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), - HWCAP_CAP(ID_AA64SMFR0_EL1, I16I64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64), - HWCAP_CAP(ID_AA64SMFR0_EL1, F64F64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F64F64), - HWCAP_CAP(ID_AA64SMFR0_EL1, I16I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I32), - HWCAP_CAP(ID_AA64SMFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16B16), - HWCAP_CAP(ID_AA64SMFR0_EL1, F16F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F16), - HWCAP_CAP(ID_AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32), - HWCAP_CAP(ID_AA64SMFR0_EL1, F16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F32), - HWCAP_CAP(ID_AA64SMFR0_EL1, B16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16F32), - HWCAP_CAP(ID_AA64SMFR0_EL1, BI32I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_BI32I32), - HWCAP_CAP(ID_AA64SMFR0_EL1, F32F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F32F32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I16I64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F64F64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F64F64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I16I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16B16), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F16), + HWCAP_CAP(ID_MATCH_ID(has_sme_feature, AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, B16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16F32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, BI32I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_BI32I32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F32F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F32F32), #endif /* CONFIG_ARM64_SME */ {}, }; diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index eb23a50c1bad2..88ea826e03c26 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -29,7 +29,7 @@ #include #include - +#include /* * Handle IRQ/context state management when entering from kernel mode. * Before this function is called it is not safe to call regular kernel code, @@ -132,11 +132,13 @@ static __always_inline void __exit_to_user_mode(void) static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs) { unsigned long flags; + int thread_lazy_resched_flag = 0; local_daif_mask(); flags = read_thread_flags(); - if (unlikely(flags & _TIF_WORK_MASK)) + trace_android_vh_restore_curr_resched(&flags, &thread_lazy_resched_flag); + if (unlikely(flags & _TIF_WORK_MASK) || thread_lazy_resched_flag) do_notify_resume(regs, flags); lockdep_sys_exit(); @@ -363,20 +365,16 @@ static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) * As per the ABI exit SME streaming mode and clear the SVE state not * shared with FPSIMD on syscall entry. */ -static inline void fp_user_discard(void) +static inline void fpsimd_syscall_enter(void) { - /* - * If SME is active then exit streaming mode. If ZA is active - * then flush the SVE registers but leave userspace access to - * both SVE and SME enabled, otherwise disable SME for the - * task and fall through to disabling SVE too. This means - * that after a syscall we never have any streaming mode - * register state to track, if this changes the KVM code will - * need updating. - */ + /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */ if (system_supports_sme()) sme_smstop_sm(); + /* + * The CPU is not in streaming mode. If non-streaming SVE is not + * supported, there is no SVE state that needs to be discarded. + */ if (!system_supports_sve()) return; @@ -386,6 +384,33 @@ static inline void fp_user_discard(void) sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; sve_flush_live(true, sve_vq_minus_one); } + + /* + * Any live non-FPSIMD SVE state has been zeroed. Allow + * fpsimd_save_user_state() to lazily discard SVE state until either + * the live state is unbound or fpsimd_syscall_exit() is called. + */ + __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD); +} + +static __always_inline void fpsimd_syscall_exit(void) +{ + if (!system_supports_sve()) + return; + + /* + * The current task's user FPSIMD/SVE/SME state is now bound to this + * CPU. The fpsimd_last_state.to_save value is either: + * + * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU + * since fpsimd_syscall_enter(). + * + * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at + * any point. + * + * Reset this to FP_STATE_CURRENT to stop lazy discarding. + */ + __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); } UNHANDLED(el1t, 64, sync) @@ -678,10 +703,11 @@ static void noinstr el0_svc(struct pt_regs *regs) { enter_from_user_mode(regs); cortex_a76_erratum_1463225_svc_handler(); - fp_user_discard(); + fpsimd_syscall_enter(); local_daif_restore(DAIF_PROCCTX); do_el0_svc(regs); exit_to_user_mode(regs); + fpsimd_syscall_exit(); } static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 7fcbee0f6c0e4..60a3cf3da7181 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -824,6 +824,7 @@ SYM_CODE_END(__bp_harden_el1_vectors) * */ SYM_FUNC_START(cpu_switch_to) + save_and_disable_daif x11 mov x10, #THREAD_CPU_CONTEXT add x8, x0, x10 mov x9, sp @@ -847,6 +848,7 @@ SYM_FUNC_START(cpu_switch_to) ptrauth_keys_install_kernel x1, x8, x9, x10 scs_save x0 scs_load_current + restore_irq x11 ret SYM_FUNC_END(cpu_switch_to) NOKPROBE(cpu_switch_to) @@ -873,6 +875,7 @@ NOKPROBE(ret_from_fork) * Calls func(regs) using this CPU's irq stack and shadow irq stack. */ SYM_FUNC_START(call_on_irq_stack) + save_and_disable_daif x9 #ifdef CONFIG_SHADOW_CALL_STACK get_current_task x16 scs_save x16 @@ -887,8 +890,10 @@ SYM_FUNC_START(call_on_irq_stack) /* Move to the new stack and call the function there */ add sp, x16, #IRQ_STACK_SIZE + restore_irq x9 blr x1 + save_and_disable_daif x9 /* * Restore the SP from the FP, and restore the FP and LR from the frame * record. @@ -896,6 +901,7 @@ SYM_FUNC_START(call_on_irq_stack) mov sp, x29 ldp x29, x30, [sp], #16 scs_load_current + restore_irq x9 ret SYM_FUNC_END(call_on_irq_stack) NOKPROBE(call_on_irq_stack) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index bd4f6c6ee0f31..83827384982e4 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -119,7 +119,7 @@ * whatever is in the FPSIMD registers is not saved to memory, but discarded. */ -static DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state); +DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state); __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = { #ifdef CONFIG_ARM64_SVE @@ -473,12 +473,15 @@ static void fpsimd_save(void) return; /* - * If a task is in a syscall the ABI allows us to only - * preserve the state shared with FPSIMD so don't bother - * saving the full SVE state in that case. + * Save SVE state if it is live. + * + * The syscall ABI discards live SVE state at syscall entry. When + * entering a syscall, fpsimd_syscall_enter() sets to_save to + * FP_STATE_FPSIMD to allow the SVE state to be lazily discarded until + * either new SVE state is loaded+bound or fpsimd_syscall_exit() is + * called prior to a return to userspace. */ - if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE) && - !in_syscall(current_pt_regs())) || + if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) || last->to_save == FP_STATE_SVE) { save_sve_regs = true; save_ffr = true; @@ -1514,7 +1517,7 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) * If this not a trap due to SME being disabled then something * is being used in the wrong mode, report as SIGILL. */ - if (ESR_ELx_ISS(esr) != ESR_ELx_SME_ISS_SME_DISABLED) { + if (ESR_ELx_SME_ISS_SMTC(esr) != ESR_ELx_SME_ISS_SMTC_SME_DISABLED) { force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); return; } @@ -1538,6 +1541,8 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) sme_set_vq(vq_minus_one); fpsimd_bind_task_to_cpu(); + } else { + fpsimd_flush_task_state(current); } put_cpu_fpsimd_context(); @@ -1803,7 +1808,7 @@ void fpsimd_update_current_state(struct user_fpsimd_state const *state) get_cpu_fpsimd_context(); current->thread.uw.fpsimd_state = *state; - if (test_thread_flag(TIF_SVE)) + if (current->thread.fp_type == FP_STATE_SVE) fpsimd_to_sve(current); task_fpsimd_load(); @@ -1868,13 +1873,17 @@ static void fpsimd_flush_cpu_state(void) */ void fpsimd_save_and_flush_cpu_state(void) { + unsigned long flags; + if (!system_supports_fpsimd()) return; WARN_ON(preemptible()); + local_irq_save(flags); __get_cpu_fpsimd_context(); fpsimd_save(); fpsimd_flush_cpu_state(); __put_cpu_fpsimd_context(); + local_irq_restore(flags); } #ifdef CONFIG_KERNEL_MODE_NEON diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 6517bf2644a08..efb0aa2fb2e48 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -575,7 +575,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) msr sctlr_el2, x0 isb 0: - mov_q x0, HCR_HOST_NVHE_FLAGS + mov_q x0, HCR_HOST_NVHE_FLAGS | HCR_ATA msr hcr_el2, x0 isb diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index cea96ee75d22d..678cd6a455ade 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -428,7 +428,8 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, put_page(page); break; } - WARN_ON_ONCE(!page_mte_tagged(page)); + + WARN_ON_ONCE(!page_mte_tagged(page) && !is_zero_page(page)); /* limit access to the end of the page */ offset = offset_in_page(addr); diff --git a/arch/arm64/kernel/probes/decode-insn.c b/arch/arm64/kernel/probes/decode-insn.c index 42b69936cee34..1ff5d721b4c15 100644 --- a/arch/arm64/kernel/probes/decode-insn.c +++ b/arch/arm64/kernel/probes/decode-insn.c @@ -78,6 +78,15 @@ static bool __kprobes aarch64_insn_is_steppable(u32 insn) enum probe_insn __kprobes arm_probe_decode_insn(probe_opcode_t insn, struct arch_probe_insn *api) { + /* + * While 'nop' instruction can execute in the out-of-line slot, + * simulating them in breakpoint handling offers better performance. + */ + if (aarch64_insn_is_nop(insn)) { + api->handler = simulate_nop; + return INSN_GOOD_NO_SLOT; + } + /* * Instructions reading or modifying the PC won't work from the XOL * slot. diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 70b91a8c6bb3f..c0942cce3b687 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -131,9 +131,15 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) void *alloc_insn_page(void) { - return __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, VMALLOC_END, + void *addr; + + addr = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, __builtin_return_address(0)); + if (!addr) + return NULL; + set_memory_rox((unsigned long)addr, 1); + return addr; } /* arm kprobe: install breakpoint in text */ diff --git a/arch/arm64/kernel/probes/simulate-insn.c b/arch/arm64/kernel/probes/simulate-insn.c index b65334ab79d2b..4c6d2d712fbd3 100644 --- a/arch/arm64/kernel/probes/simulate-insn.c +++ b/arch/arm64/kernel/probes/simulate-insn.c @@ -196,3 +196,9 @@ simulate_ldrsw_literal(u32 opcode, long addr, struct pt_regs *regs) instruction_pointer_set(regs, instruction_pointer(regs) + 4); } + +void __kprobes +simulate_nop(u32 opcode, long addr, struct pt_regs *regs) +{ + arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); +} diff --git a/arch/arm64/kernel/probes/simulate-insn.h b/arch/arm64/kernel/probes/simulate-insn.h index e065dc92218e8..efb2803ec943d 100644 --- a/arch/arm64/kernel/probes/simulate-insn.h +++ b/arch/arm64/kernel/probes/simulate-insn.h @@ -16,5 +16,6 @@ void simulate_cbz_cbnz(u32 opcode, long addr, struct pt_regs *regs); void simulate_tbz_tbnz(u32 opcode, long addr, struct pt_regs *regs); void simulate_ldr_literal(u32 opcode, long addr, struct pt_regs *regs); void simulate_ldrsw_literal(u32 opcode, long addr, struct pt_regs *regs); +void simulate_nop(u32 opcode, long addr, struct pt_regs *regs); #endif /* _ARM_KERNEL_KPROBES_SIMULATE_INSN_H */ diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 36a1f9a16a73c..2c81e0efaf378 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -876,11 +876,22 @@ static u8 spectre_bhb_loop_affected(void) { u8 k = 0; + static const struct midr_range spectre_bhb_k132_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + {}, + }; + static const struct midr_range spectre_bhb_k38_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A715), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720), + {}, + }; static const struct midr_range spectre_bhb_k32_list[] = { MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), @@ -889,9 +900,11 @@ static u8 spectre_bhb_loop_affected(void) }; static const struct midr_range spectre_bhb_k24_list[] = { MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_GOLD), + MIDR_ALL_VERSIONS(MIDR_HISI_HIP09), {}, }; static const struct midr_range spectre_bhb_k11_list[] = { @@ -904,7 +917,11 @@ static u8 spectre_bhb_loop_affected(void) {}, }; - if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list)) + if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k132_list)) + k = 132; + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k38_list)) + k = 38; + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list)) k = 32; else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list)) k = 24; @@ -983,6 +1000,11 @@ bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, return true; } +u8 get_spectre_bhb_loop_value(void) +{ + return max_bhb_k; +} + static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) { const char *v = arm64_get_bp_hardening_vector(slot); @@ -1000,7 +1022,7 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) isb(); } -static bool __read_mostly __nospectre_bhb; +bool __read_mostly __nospectre_bhb; static int __init parse_spectre_bhb_param(char *str) { __nospectre_bhb = true; @@ -1078,6 +1100,11 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) update_mitigation_state(&spectre_bhb_state, state); } +bool is_spectre_bhb_fw_mitigated(void) +{ + return test_bit(BHB_FW, &system_bhb_mitigations); +} + /* Patched to NOP when enabled */ void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, __le32 *origptr, diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 046c533dd9caa..eb0f53deaa7a3 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -141,7 +141,7 @@ unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) addr += n; if (regs_within_kernel_stack(regs, (unsigned long)addr)) - return *addr; + return READ_ONCE_NOCHECK(*addr); else return 0; } diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index e5ed5d748e2e2..ce51632146529 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -1283,11 +1283,15 @@ static void do_signal(struct pt_regs *regs) void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) { int thread_lazy_flag = 0; + int thread_lazy_resched_flag = 0; + trace_android_vh_restore_curr_resched(&thread_flags, &thread_lazy_resched_flag); do { trace_android_vh_read_lazy_flag(&thread_lazy_flag, &thread_flags); - if ((thread_flags & _TIF_NEED_RESCHED) || thread_lazy_flag) { + if ((thread_flags & _TIF_NEED_RESCHED) || thread_lazy_flag || + thread_lazy_resched_flag) { /* Unmask Debug and SError for the next task */ + thread_lazy_resched_flag = 0; local_daif_restore(DAIF_PROCCTX_NOIRQ); schedule(); @@ -1315,7 +1319,8 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) local_daif_mask(); thread_flags = read_thread_flags(); - } while (thread_flags & _TIF_WORK_MASK); + trace_android_vh_restore_curr_resched(&thread_flags, &thread_lazy_resched_flag); + } while (thread_flags & _TIF_WORK_MASK || thread_lazy_resched_flag); } unsigned long __ro_after_init signal_minsigstksz; diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 7c93307bd9609..4756d4e5a2c5c 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -84,7 +84,12 @@ int __init parse_acpi_topology(void) #undef pr_fmt #define pr_fmt(fmt) "AMU: " fmt -static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale); +/* + * Ensure that amu_scale_freq_tick() will return SCHED_CAPACITY_SCALE until + * the CPU capacity and its associated frequency have been correctly + * initialized. + */ +static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale) = 1UL << (2 * SCHED_CAPACITY_SHIFT); static DEFINE_PER_CPU(u64, arch_const_cycles_prev); static DEFINE_PER_CPU(u64, arch_core_cycles_prev); static cpumask_var_t amu_fie_cpus; @@ -114,14 +119,14 @@ static inline bool freq_counters_valid(int cpu) return true; } -static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate) +void freq_inv_set_max_ratio(int cpu, u64 max_rate) { - u64 ratio; + u64 ratio, ref_rate = arch_timer_get_rate(); if (unlikely(!max_rate || !ref_rate)) { - pr_debug("CPU%d: invalid maximum or reference frequency.\n", + WARN_ONCE(1, "CPU%d: invalid maximum or reference frequency.\n", cpu); - return -EINVAL; + return; } /* @@ -141,12 +146,10 @@ static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate) ratio = div64_u64(ratio, max_rate); if (!ratio) { WARN_ONCE(1, "Reference frequency too low.\n"); - return -EINVAL; + return; } - per_cpu(arch_max_freq_scale, cpu) = (unsigned long)ratio; - - return 0; + WRITE_ONCE(per_cpu(arch_max_freq_scale, cpu), (unsigned long)ratio); } static void amu_scale_freq_tick(void) @@ -202,10 +205,7 @@ static void amu_fie_setup(const struct cpumask *cpus) return; for_each_cpu(cpu, cpus) { - if (!freq_counters_valid(cpu) || - freq_inv_set_max_ratio(cpu, - cpufreq_get_hw_max_freq(cpu) * 1000ULL, - arch_timer_get_rate())) + if (!freq_counters_valid(cpu)) return; } diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index dd83f62fe494d..d01d469ac2cda 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -959,6 +959,7 @@ void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigne void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr) { + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); console_verbose(); pr_crit("SError Interrupt on CPU%d, code 0x%016lx -- %s\n", diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 79d9a0cbeb4ce..3c8e9d0d35742 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1879,6 +1879,10 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS; else params->hcr_el2 = HCR_HOST_NVHE_FLAGS; + + if (system_supports_mte()) + params->hcr_el2 |= HCR_ATA; + if (cpus_have_final_cap(ARM64_KVM_HVHE)) params->hcr_el2 |= HCR_E2H; params->vttbr = params->vtcr = 0; diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 92bf61f15c318..51d6e902ee48a 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -267,6 +267,9 @@ static inline void ___activate_traps(struct kvm_vcpu *vcpu) if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) hcr |= HCR_TVM; + if (!system_supports_mte()) + hcr &= ~(HCR_DCT | HCR_ATA); + write_sysreg(hcr, hcr_el2); if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) diff --git a/arch/arm64/kvm/hyp/include/nvhe/gfp.h b/arch/arm64/kvm/hyp/include/nvhe/gfp.h index cecb2125b32e8..527b88091f7d1 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/gfp.h +++ b/arch/arm64/kvm/hyp/include/nvhe/gfp.h @@ -16,6 +16,7 @@ struct hyp_pool { phys_addr_t range_start; phys_addr_t range_end; u64 free_pages; + u64 min_free_pages; u8 max_order; }; @@ -26,6 +27,7 @@ void hyp_get_page(struct hyp_pool *pool, void *addr); void hyp_put_page(struct hyp_pool *pool, void *addr); u64 hyp_pool_free_pages(struct hyp_pool *pool); +u64 hyp_pool_min_free_pages(struct hyp_pool *pool); /* Used pages cannot be freed */ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index 4afb0b83115e5..388ea6e0bd6ef 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -54,6 +54,8 @@ void kvm_iommu_iotlb_gather_add_page(struct kvm_hyp_iommu_domain *domain, void kvm_iommu_host_stage2_idmap(phys_addr_t start, phys_addr_t end, enum kvm_pgtable_prot prot); int kvm_iommu_snapshot_host_stage2(struct kvm_hyp_iommu_domain *domain); +int kvm_iommu_iotlb_sync_map(pkvm_handle_t domain_id, + unsigned long iova, size_t size); #define KVM_IOMMU_PADDR_CACHE_MAX ((size_t)511) /** @@ -112,6 +114,7 @@ static inline void kvm_iommu_unlock(struct kvm_hyp_iommu *iommu) * @map_pages: Map pages in a domain. * @unmap_pages: Unmap pages from a domain. * @iova_to_phys: get physical address from IOVA in a domain. + * @iotlb_sync_map: Sync mapping created using @map_pages to the hardware. */ struct kvm_iommu_ops { int (*init)(unsigned long arg); @@ -138,7 +141,8 @@ struct kvm_iommu_ops { struct iommu_iotlb_gather *gather, struct kvm_iommu_paddr_cache *cache); phys_addr_t (*iova_to_phys)(struct kvm_hyp_iommu_domain *domain, unsigned long iova); - ANDROID_KABI_RESERVE(1); + ANDROID_KABI_USE(1, int (*iotlb_sync_map)(struct kvm_hyp_iommu_domain *domain, + unsigned long iova, size_t size)); ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); ANDROID_KABI_RESERVE(4); diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 75f1be8a15de6..8b33b55c92276 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -36,6 +36,9 @@ void pkvm_remove_mappings(void *from, void *to); phys_addr_t __pkvm_private_range_pa(void *va); int __hyp_allocator_map(unsigned long start, phys_addr_t phys); +u64 hpool_get_free_pages(void); +u64 hpool_get_min_free_pages(void); + int __pkvm_map_module_page(u64 pfn, void *va, enum kvm_pgtable_prot prot, bool is_protected); void __pkvm_unmap_module_page(u64 pfn, void *va); void *__pkvm_alloc_module_va(u64 nr_pages); diff --git a/arch/arm64/kvm/hyp/nvhe/alloc.c b/arch/arm64/kvm/hyp/nvhe/alloc.c index dc1a7371b694a..90b644f890348 100644 --- a/arch/arm64/kvm/hyp/nvhe/alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/alloc.c @@ -556,7 +556,13 @@ void *hyp_alloc(size_t size) unsigned long chunk_addr; int missing_map, ret = 0; - size = ALIGN(size, MIN_ALLOC); + /* constrained by chunk_hdr *_size types */ + if (size > U32_MAX) { + ret = -E2BIG; + goto end_unlocked; + } + + size = ALIGN(size ?: MIN_ALLOC, MIN_ALLOC); hyp_spin_lock(&allocator->lock); @@ -591,9 +597,11 @@ void *hyp_alloc(size_t size) } WARN_ON(chunk_install(chunk, size, last_chunk, allocator)); + end: hyp_spin_unlock(&allocator->lock); +end_unlocked: *(this_cpu_ptr(&hyp_allocator_errno)) = ret; /* Enforce zeroing allocated memory */ diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 7363574689854..6f7791fb54ab0 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -437,7 +437,7 @@ static void __do_ffa_mem_xfer(const u64 func_id, struct ffa_mem_region_attributes *ep_mem_access; struct ffa_composite_mem_region *reg; struct ffa_mem_region *buf; - u32 offset, nr_ranges; + u32 offset, nr_ranges, checked_offset; int ret = 0; if (addr_mbz || npages_mbz || fraglen > len || @@ -474,11 +474,16 @@ static void __do_ffa_mem_xfer(const u64 func_id, goto out_unlock; } - if (fraglen < offset + sizeof(struct ffa_composite_mem_region)) { + if (check_add_overflow(offset, sizeof(struct ffa_composite_mem_region), &checked_offset)) { ret = FFA_RET_INVALID_PARAMETERS; goto out_unlock; } + if (fraglen < checked_offset) { + ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS); + goto out_unlock; + } + reg = (void *)buf + offset; nr_ranges = ((void *)buf + fraglen) - (void *)reg->constituents; if (nr_ranges % sizeof(reg->constituents[0])) { diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index a5ffc4cd3f70b..796dc55a03afd 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1469,6 +1469,8 @@ static void handle___pkvm_hyp_alloc_mgt_refill(struct kvm_cpu_context *host_ctxt }; cpu_reg(host_ctxt, 1) = hyp_alloc_mgt_refill(id, &mc); + cpu_reg(host_ctxt, 2) = mc.head; + cpu_reg(host_ctxt, 3) = mc.nr_pages; } static void handle___pkvm_hyp_alloc_mgt_reclaimable(struct kvm_cpu_context *host_ctxt) @@ -1574,6 +1576,17 @@ static void handle___pkvm_host_iommu_iova_to_phys(struct kvm_cpu_context *host_c hyp_reqs_smccc_encode(ret, host_ctxt, this_cpu_ptr(&host_hyp_reqs)); } +static void handle___pkvm_host_iommu_iotlb_sync_map(struct kvm_cpu_context *host_ctxt) +{ + unsigned long ret; + DECLARE_REG(pkvm_handle_t, domain, host_ctxt, 1); + DECLARE_REG(unsigned long, iova, host_ctxt, 2); + DECLARE_REG(size_t, size, host_ctxt, 3); + + ret = kvm_iommu_iotlb_sync_map(domain, iova, size); + hyp_reqs_smccc_encode(ret, host_ctxt, this_cpu_ptr(&host_hyp_reqs)); +} + static void handle___pkvm_iommu_init(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_iommu_ops *, ops, host_ctxt, 1); @@ -1605,6 +1618,16 @@ static void handle___pkvm_stage2_snapshot(struct kvm_cpu_context *host_ctxt) #endif } +static void handle___pkvm_hyp_pool_report_free_pages(struct kvm_cpu_context *host_ctxt) +{ + cpu_reg(host_ctxt, 1) = hpool_get_free_pages(); +} + +static void handle___pkvm_hyp_pool_report_min_free_pages(struct kvm_cpu_context *host_ctxt) +{ + cpu_reg(host_ctxt, 1) = hpool_get_min_free_pages(); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1671,6 +1694,9 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_host_iommu_iova_to_phys), HANDLE_FUNC(__pkvm_host_hvc_pd), HANDLE_FUNC(__pkvm_stage2_snapshot), + HANDLE_FUNC(__pkvm_host_iommu_iotlb_sync_map), + HANDLE_FUNC(__pkvm_hyp_pool_report_free_pages), + HANDLE_FUNC(__pkvm_hyp_pool_report_min_free_pages), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c index f06b84357eb4b..66fd9ed618b45 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c @@ -221,15 +221,17 @@ void __repudiate_host_page(void *addr, unsigned long order, int kvm_iommu_refill(struct kvm_hyp_memcache *host_mc) { + struct kvm_hyp_memcache tmp = *host_mc; + if (!kvm_iommu_ops) return -EINVAL; /* Paired with smp_wmb() in kvm_iommu_init() */ smp_rmb(); - while (host_mc->nr_pages) { - unsigned long order = FIELD_GET(~PAGE_MASK, host_mc->head); - phys_addr_t phys = host_mc->head & PAGE_MASK; + while (tmp.nr_pages) { + unsigned long order = FIELD_GET(~PAGE_MASK, tmp.head); + phys_addr_t phys = tmp.head & PAGE_MASK; struct hyp_pool *pool = &iommu_system_pool; u64 nr_pages; void *addr; @@ -238,16 +240,18 @@ int kvm_iommu_refill(struct kvm_hyp_memcache *host_mc) !IS_ALIGNED(phys, PAGE_SIZE << order)) return -EINVAL; - addr = admit_host_page(host_mc, order); + addr = admit_host_page(&tmp, order); if (!addr) return -EINVAL; + *host_mc = tmp; if (kvm_iommu_donate_from_cma(phys, order)) { hyp_spin_lock(&__block_pools_lock); pool = __get_empty_block_pool(phys); hyp_spin_unlock(&__block_pools_lock); if (!pool) { - __repudiate_host_page(addr, order, host_mc); + __repudiate_host_page(addr, order, &tmp); + *host_mc = tmp; return -EBUSY; } } else { @@ -825,3 +829,28 @@ int kvm_iommu_snapshot_host_stage2(struct kvm_hyp_iommu_domain *domain) return ret; } + +int kvm_iommu_iotlb_sync_map(pkvm_handle_t domain_id, + unsigned long iova, size_t size) +{ + struct kvm_hyp_iommu_domain *domain; + int ret; + + if (!kvm_iommu_ops || !kvm_iommu_ops->iotlb_sync_map) + return -ENODEV; + + if (!size || (iova + size < iova)) + return -EINVAL; + + if (domain_id == KVM_IOMMU_DOMAIN_IDMAP_ID) + return -EINVAL; + + domain = handle_to_domain(domain_id); + + if (!domain || domain_get(domain)) + return -EINVAL; + + ret = kvm_iommu_ops->iotlb_sync_map(domain, iova, size); + domain_put(domain); + return ret; +} diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index bc1f8cb3faf3d..bf97ded87f2d0 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -219,7 +219,7 @@ static void *guest_s2_zalloc_pages_exact(size_t size) { void *addr = hyp_alloc_pages(¤t_vm->pool, get_order(size)); - WARN_ON(size != (PAGE_SIZE << get_order(size))); + WARN_ON(!addr || size != (PAGE_SIZE << get_order(size))); hyp_split_page(hyp_virt_to_page(addr)); return addr; @@ -398,11 +398,6 @@ static int relinquish_walker(const struct kvm_pgtable_visit_ctx *ctx, phys = kvm_pte_to_phys(pte); phys += ctx->addr - addr; - if (state == PKVM_PAGE_OWNED) { - hyp_poison_page(phys, PAGE_SIZE); - psci_mem_protect_dec(1); - } - data->pa = phys; return 0; @@ -441,7 +436,16 @@ int __pkvm_guest_relinquish_to_host(struct pkvm_hyp_vcpu *vcpu, if (ret) goto end; + if (pkvm_hyp_vcpu_is_protected(vcpu)) { + hyp_poison_page(data.pa, PAGE_SIZE); + psci_mem_protect_dec(1); + } + WARN_ON(host_stage2_set_owner_locked(data.pa, PAGE_SIZE, PKVM_ID_HOST)); + + if (pkvm_ipa_range_has_pvmfw(vm, ipa, ipa + PAGE_SIZE)) + vm->kvm.arch.pkvm.pvmfw_load_addr = PVMFW_INVALID_LOAD_ADDR; + end: guest_unlock_component(vm); host_unlock_component(); @@ -491,17 +495,9 @@ int __pkvm_prot_finalize(void) int host_stage2_unmap_reg_locked(phys_addr_t start, u64 size) { - int ret; - hyp_assert_lock_held(&host_mmu.lock); - ret = kvm_pgtable_stage2_reclaim_leaves(&host_mmu.pgt, start, size); - if (ret) - return ret; - - kvm_iommu_host_stage2_idmap(start, start + size, 0); - - return 0; + return kvm_pgtable_stage2_reclaim_leaves(&host_mmu.pgt, start, size); } static int host_stage2_unmap_unmoveable_regs(void) @@ -1057,7 +1053,10 @@ static int ___host_check_page_state_range(u64 addr, u64 size, .desired = state, .get_page_state = host_get_mmio_page_state, }; - u64 end = addr + size; + u64 end; + + if (check_add_overflow(addr, size, &end)) + return -EINVAL; hyp_assert_lock_held(&host_mmu.lock); @@ -1087,7 +1086,10 @@ static int __host_check_page_state_range(u64 addr, u64 size, { struct memblock_region *reg; struct kvm_mem_range range; - u64 end = addr + size; + u64 end; + + if (check_add_overflow(addr, size, &end)) + return -EINVAL; /* Can't check the state of both MMIO and memory regions at once */ reg = find_mem_range(addr, &range); @@ -1100,6 +1102,14 @@ static int __host_check_page_state_range(u64 addr, u64 size, static int __host_set_page_state_range(u64 addr, u64 size, enum pkvm_page_state state) { + u64 end; + + if (check_add_overflow(addr, size, &end)) + return -EINVAL; + + if (!range_is_memory(addr, end)) + return -EPERM; + if (hyp_phys_to_page(addr)->host_state & PKVM_NOPAGE) { int ret = host_stage2_idmap_locked(addr, size, PKVM_HOST_MEM_PROT, true); @@ -1398,6 +1408,10 @@ static int __guest_check_page_state_range(struct pkvm_hyp_vm *vm, u64 addr, .desired = state, .get_page_state = guest_get_page_state, }; + u64 end; + + if (check_add_overflow(addr, size, &end)) + return -EINVAL; hyp_assert_lock_held(&vm->pgtable_lock); return check_page_state_range(&vm->pgt, addr, size, &d); @@ -1746,6 +1760,11 @@ static int __do_share(struct pkvm_mem_transition *tx, break; case PKVM_ID_HYP: ret = hyp_complete_share(checked_tx, tx->completer.prot); + if (ret == -ENOMEM) { + WARN_ON(tx->initiator.id != PKVM_ID_HOST); + WARN_ON(host_initiate_unshare(checked_tx)); + return ret; + } break; case PKVM_ID_FFA: /* @@ -1761,7 +1780,7 @@ static int __do_share(struct pkvm_mem_transition *tx, ret = -EINVAL; } - return ret; + return WARN_ON(ret); } /* @@ -1787,7 +1806,7 @@ static int do_share(struct pkvm_mem_transition *share, return ret; ret = __do_share(share, &checked_tx); - if (WARN_ON(ret)) + if (ret) return ret; *nr_shared = checked_tx.nr_pages; @@ -1970,6 +1989,16 @@ static int __do_donate(struct pkvm_mem_transition *tx) break; case PKVM_ID_HYP: ret = hyp_complete_donation(completer_addr, tx); + if (ret == -ENOMEM) { + struct pkvm_mem_transition abort = { + .nr_pages = tx->nr_pages, + .completer.id = PKVM_ID_HOST, + }; + + WARN_ON(tx->initiator.id != PKVM_ID_HOST); + WARN_ON(host_complete_donation(completer_addr, &abort)); + return -ENOMEM; + } break; case PKVM_ID_GUEST: ret = guest_complete_donation(completer_addr, tx); @@ -1978,7 +2007,7 @@ static int __do_donate(struct pkvm_mem_transition *tx) ret = -EINVAL; } - return ret; + return WARN_ON(ret); } /* @@ -1998,7 +2027,7 @@ static int do_donate(struct pkvm_mem_transition *donation) if (ret) return ret; - return WARN_ON(__do_donate(donation)); + return __do_donate(donation); } int __pkvm_host_share_hyp(u64 pfn) @@ -2600,10 +2629,14 @@ static int guest_get_valid_pte(struct pkvm_hyp_vm *vm, u64 pfn, u64 ipa, size_t size = PAGE_SIZE << order; u64 phys = hyp_pfn_to_phys(pfn); u32 level; + u64 end; if (order && size != PMD_SIZE) return -EINVAL; + if (check_add_overflow(phys, size, &end)) + return -EINVAL; + WARN_ON(kvm_pgtable_get_leaf(&vm->pgt, ipa, pte, &level)); if (kvm_granule_size(level) != size) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index e78f42a6e39ec..8f44bccfd0c19 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -113,7 +113,13 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, int __hyp_allocator_map(unsigned long va, phys_addr_t phys) { - return __pkvm_create_mappings(va, PAGE_SIZE, phys, PAGE_HYP); + int ret = __pkvm_create_mappings(va, PAGE_SIZE, phys, PAGE_HYP); + + /* Let's not confuse the hyp_alloc callers who will try to top-up pointlessly on -ENOMEM */ + if (ret == -ENOMEM) + ret = -EBUSY; + + return ret; } #ifdef CONFIG_NVHE_EL2_DEBUG @@ -541,6 +547,7 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) return ret; } +/* Note: The caller has to use a local copy of the arg */ void *admit_host_page(void *arg, unsigned long order) { phys_addr_t p; @@ -599,12 +606,19 @@ int refill_hyp_pool(struct hyp_pool *pool, struct kvm_hyp_memcache *host_mc) { unsigned long order; void *p; + struct kvm_hyp_memcache tmp = *host_mc; + u64 nr_pages; - while (host_mc->nr_pages) { - order = FIELD_GET(~PAGE_MASK, host_mc->head); - p = admit_host_page(host_mc, order); + while (tmp.nr_pages) { + order = FIELD_GET(~PAGE_MASK, tmp.head); + if (check_shl_overflow(1UL, order, &nr_pages)) + return -EINVAL; + + p = admit_host_page(&tmp, order); if (!p) return -EINVAL; + *host_mc = tmp; + hyp_virt_to_page(p)->order = order; hyp_set_page_refcounted(hyp_virt_to_page(p)); hyp_put_page(pool, p); diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index e962cfa01a09e..4c470e2eaf9dc 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -155,15 +155,22 @@ static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool, return p; } +static void update_free_pages(struct hyp_pool *pool, u64 free_pages) +{ + WRITE_ONCE(pool->free_pages, free_pages); + if (pool->min_free_pages > free_pages) + WRITE_ONCE(pool->min_free_pages, free_pages); +} + static void __hyp_put_page(struct hyp_pool *pool, struct hyp_page *p) { u64 free_pages; if (hyp_page_ref_dec_and_test(p)) { hyp_spin_lock(&pool->lock); + free_pages = pool->free_pages + (1ULL << p->order); __hyp_attach_page(pool, p); - free_pages = pool->free_pages + (1 << p->order); - WRITE_ONCE(pool->free_pages, free_pages); + update_free_pages(pool, free_pages); hyp_spin_unlock(&pool->lock); } } @@ -219,8 +226,8 @@ void *hyp_alloc_pages(struct hyp_pool *pool, u8 order) hyp_set_page_refcounted(p); - free_pages = pool->free_pages - (1 << p->order); - WRITE_ONCE(pool->free_pages, free_pages); + free_pages = pool->free_pages - (1ULL << p->order); + update_free_pages(pool, free_pages); hyp_spin_unlock(&pool->lock); return hyp_page_to_virt(p); @@ -238,6 +245,11 @@ u64 hyp_pool_free_pages(struct hyp_pool *pool) return READ_ONCE(pool->free_pages); } +u64 hyp_pool_min_free_pages(struct hyp_pool *pool) +{ + return READ_ONCE(pool->min_free_pages); +} + /* * empty_alloc alloc set true when pool has no pages initially, but we still want * to use it in the future, which means nr_pages only has to be valid to init @@ -274,6 +286,8 @@ static int __hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages for (i = reserved_pages; i < nr_pages; i++) __hyp_put_page(pool, &p[i]); + pool->min_free_pages = pool->free_pages; + return 0; } diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index feaad44fe2046..4a381fe21d5df 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -662,11 +662,18 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, { int ret = 0; u32 mp_state; + struct kvm_hyp_req *hyp_reqs; if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) return -EBUSY; - hyp_vcpu->vcpu.arch.hyp_reqs = kern_hyp_va(host_vcpu->arch.hyp_reqs); + hyp_reqs = READ_ONCE(host_vcpu->arch.hyp_reqs); + if (!PAGE_ALIGNED(hyp_reqs)) { + hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); + return -EINVAL; + } + + hyp_vcpu->vcpu.arch.hyp_reqs = kern_hyp_va(hyp_reqs); if (hyp_pin_shared_mem(hyp_vcpu->vcpu.arch.hyp_reqs, hyp_vcpu->vcpu.arch.hyp_reqs + 1)) { hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); @@ -829,6 +836,8 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long pgd_hva) ret = -EINVAL; pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.vtcr); + if (!IS_ALIGNED(pgd_hva, pgd_size)) + goto err_free_last_ran; pgd = map_donated_memory_noclear(pgd_hva, pgd_size); if (!pgd) goto err_free_last_ran; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 140c170b4f09b..4db2a5bf45b26 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -29,6 +29,8 @@ phys_addr_t pvmfw_size; #define hyp_percpu_size ((unsigned long)__per_cpu_end - \ (unsigned long)__per_cpu_start) +u64 hyp_lm_size_mb; + static void *vmemmap_base; static void *vm_table_base; static void *hyp_pgt_base; @@ -198,6 +200,16 @@ static void hpool_put_page(void *addr) hyp_put_page(&hpool, addr); } +u64 hpool_get_free_pages(void) +{ + return hyp_pool_free_pages(&hpool); +} + +u64 hpool_get_min_free_pages(void) +{ + return hyp_pool_min_free_pages(&hpool); +} + static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, enum kvm_pgtable_walk_flags visit) { diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 1bd41b08d9681..78aab26f80984 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -39,6 +39,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu) *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); + __vcpu_sys_reg(vcpu, VBAR_EL1) = read_sysreg_el1(SYS_VBAR); kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); @@ -465,6 +466,7 @@ static const struct sys_reg_desc pvm_sys_reg_descs[] = { HOST_HANDLED(SYS_CCSIDR_EL1), HOST_HANDLED(SYS_CLIDR_EL1), + RAZ_WI(SYS_AIDR_EL1), HOST_HANDLED(SYS_CSSELR_EL1), HOST_HANDLED(SYS_CTR_EL0), @@ -528,7 +530,7 @@ static const struct sys_reg_desc_reset pvm_sys_reg_reset_vals[] = { RESET_VAL(CPACR_EL1, 0), RESET_VAL(ZCR_EL1, 0), RESET_VAL(TCR_EL1, 0), - RESET_VAL(VBAR_EL1, 0), + RESET_VAL(VBAR_EL1, 0x1de7ec7edbadc000ULL), RESET_VAL(CONTEXTIDR_EL1, 0), RESET_FUNC(AMAIR_EL1, reset_amair_el1), RESET_VAL(CNTKCTL_EL1, 0), diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 140e9a17a5ce9..20460aa89f997 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -472,7 +472,7 @@ int __pkvm_load_tracing(unsigned long desc_hva, size_t desc_size) struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva); struct trace_page_desc *trace_pdesc = &desc->page_desc; struct rb_page_desc *pdesc; - int ret, cpu; + int ret, pdesc_cpu; if (!desc_size || !PAGE_ALIGNED(desc_hva) || !PAGE_ALIGNED(desc_size)) return -EINVAL; @@ -486,9 +486,9 @@ int __pkvm_load_tracing(unsigned long desc_hva, size_t desc_size) trace_clock_update(&desc->clock_data); - for_each_rb_page_desc(pdesc, cpu, trace_pdesc) { + for_each_rb_page_desc(pdesc, pdesc_cpu, trace_pdesc) { struct hyp_rb_per_cpu *cpu_buffer; - int cpu; + unsigned int cpu; ret = -EINVAL; if (!rb_cpu_fits_desc(pdesc, desc_hva + desc_size)) diff --git a/arch/arm64/kvm/hyp_events.c b/arch/arm64/kvm/hyp_events.c index 424cd5189355c..66ae6c6a469fe 100644 --- a/arch/arm64/kvm/hyp_events.c +++ b/arch/arm64/kvm/hyp_events.c @@ -250,7 +250,10 @@ bool hyp_trace_init_event_early(void) } static struct dentry *event_tracefs; -static unsigned int last_event_id; +// Event IDs should be positive integers, hence starting from 1 here. +// NOTE: this introduces ID clash between hypervisor events and kernel events. +// For now this doesn't seem to cause problems, but we should fix it... +static unsigned int last_event_id = 1; struct hyp_event_table { struct hyp_event *start; @@ -266,30 +269,36 @@ static struct hyp_event_mod_tables { struct hyp_event *hyp_trace_find_event(int id) { - struct hyp_event *event = __hyp_events_start + id; + struct hyp_event *event; + struct hyp_event_table *table; + int i, j; - if ((unsigned long)event >= (unsigned long)__hyp_events_end) { - struct hyp_event_table *table; - - event = NULL; - id -= nr_events(__hyp_events_start, __hyp_events_end); - - rcu_read_lock(); - table = rcu_dereference(mod_event_tables.tables); + for (event = __hyp_events_start; event < __hyp_events_end; event++) { + if (event->id == id) + return event; + if (event->id > id) + return NULL; + } - for (int i = 0; i < mod_event_tables.nr_tables; i++) { - if (table->nr_events <= id) { - id -= table->nr_events; - table++; - continue; + event = NULL; + rcu_read_lock(); + table = rcu_dereference(mod_event_tables.tables); + for (i = 0; i < mod_event_tables.nr_tables; i++, table++) { + event = table->start; + for (j = 0; j < table->nr_events; j++, event++) { + if (event->id == id) + goto end; + if (event->id > id) { + event = NULL; + goto end; } - - event = table->start + id; - break; } - rcu_read_unlock(); + event = NULL; } +end: + rcu_read_unlock(); + return event; } diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index b4a5d117568a6..4eb85aad055fa 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -861,7 +861,9 @@ int hyp_trace_init_tracefs(void) tracefs_create_file("trace_pipe", TRACEFS_MODE_READ, per_cpu_dir, (void *)cpu, &hyp_trace_pipe_fops); tracefs_create_file("trace_pipe_raw", TRACEFS_MODE_READ, per_cpu_dir, - (void *)cpu, &hyp_trace_pipe_fops); + (void *)cpu, &hyp_trace_raw_fops); + tracefs_create_file("trace", TRACEFS_MODE_WRITE, per_cpu_dir, + (void *)cpu, &hyp_trace_fops); } hyp_trace_init_event_tracefs(root); diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 4523cc6f27255..db08cd6589ecd 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -4,6 +4,7 @@ * Author: Quentin Perret */ +#include #include #include #include @@ -65,6 +66,7 @@ static void __init sort_memblock_regions(void) static int __init register_memblock_regions(void) { struct memblock_region *reg; + bool pvmfw_in_mem = false; for_each_mem_region(reg) { if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS) @@ -72,6 +74,27 @@ static int __init register_memblock_regions(void) hyp_memory[*hyp_memblock_nr_ptr] = *reg; (*hyp_memblock_nr_ptr)++; + + if (!*pvmfw_size || pvmfw_in_mem || + !memblock_addrs_overlap(reg->base, reg->size, *pvmfw_base, *pvmfw_size)) + continue; + /* If the pvmfw region overlaps a memblock, it must be a subset */ + if (*pvmfw_base < reg->base || + (*pvmfw_base + *pvmfw_size) > (reg->base + reg->size)) + return -EINVAL; + pvmfw_in_mem = true; + } + + if (*pvmfw_size && !pvmfw_in_mem) { + if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS) + return -ENOMEM; + + hyp_memory[*hyp_memblock_nr_ptr] = (struct memblock_region) { + .base = *pvmfw_base, + .size = *pvmfw_size, + .flags = MEMBLOCK_NOMAP, + }; + (*hyp_memblock_nr_ptr)++; } sort_memblock_regions(); @@ -152,6 +175,12 @@ static int __init register_moveable_regions(void) return 0; } +static int __init early_hyp_lm_size_mb_cfg(char *arg) +{ + return kstrtoull(arg, 10, &kvm_nvhe_sym(hyp_lm_size_mb)); +} +early_param("kvm-arm.hyp_lm_size_mb", early_hyp_lm_size_mb_cfg); + void __init kvm_hyp_reserve(void) { u64 hyp_mem_pages = 0; @@ -519,6 +548,7 @@ static int __init finalize_pkvm(void) kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size); kvm_ptdump_host_register(); + kvm_hyp_s1_pool_debugfs(); ret = pkvm_drop_host_privileges(); if (ret) { @@ -1022,8 +1052,13 @@ EXPORT_SYMBOL(__pkvm_register_el2_call); int __pkvm_topup_hyp_alloc_mgt_mc(unsigned long id, struct kvm_hyp_memcache *mc) { - return kvm_call_hyp_nvhe(__pkvm_hyp_alloc_mgt_refill, id, mc->head, - mc->nr_pages); + struct arm_smccc_res res; + + res = kvm_call_hyp_nvhe_smccc(__pkvm_hyp_alloc_mgt_refill, + id, mc->head, mc->nr_pages); + mc->head = res.a2; + mc->nr_pages = res.a3; + return res.a1; } EXPORT_SYMBOL(__pkvm_topup_hyp_alloc_mgt_mc); @@ -1040,8 +1075,11 @@ int __pkvm_topup_hyp_alloc_mgt_gfp(unsigned long id, unsigned long nr_pages, return ret; ret = __pkvm_topup_hyp_alloc_mgt_mc(id, &mc); - if (ret) + if (ret) { + kvm_err("Failed topup %ld pages = %ld, size = %ld err = %d, freeing %ld pages\n", + id, nr_pages, sz_alloc, ret, mc.nr_pages); free_hyp_memcache(&mc); + } return ret; } @@ -1086,3 +1124,47 @@ unsigned long __pkvm_reclaim_hyp_alloc_mgt(unsigned long nr_pages) return reclaimed; } + +#ifdef CONFIG_DEBUG_FS +static int pool_free_get(void *data, u64 *val) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_pool_report_free_pages), &res); + if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) + return -EINVAL; + + *val = res.a1 * PAGE_SIZE; + + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(pool_free_fops, pool_free_get, NULL, "%llu\n"); + +static int pool_min_free_get(void *data, u64 *val) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_pool_report_min_free_pages), &res); + if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) + return -EINVAL; + + *val = res.a1 * PAGE_SIZE; + + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(pool_min_free_fops, pool_min_free_get, NULL, "%llu\n"); + +void kvm_hyp_s1_pool_debugfs(void) +{ + static u64 pool_size; + + if (!is_protected_kvm_enabled()) + return; + + pool_size = hyp_s1_pgtable_pages() * PAGE_SIZE; + debugfs_create_u64("hyp_s1_pool_size", 0400, kvm_debugfs_dir, &pool_size); + debugfs_create_file("hyp_s1_pool_free", 0400, kvm_debugfs_dir, NULL, &pool_free_fops); + debugfs_create_file("hyp_s1_pool_min_free", 0400, kvm_debugfs_dir, NULL, + &pool_min_free_fops); +} +#endif /* CONFIG_DEBUG_FS */ diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index a635ab83fee35..7232b1e70a125 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -5,6 +5,7 @@ * * Copyright (C) 2014-2016 Zi Shen Lim */ +#include #include #include #include @@ -1471,43 +1472,41 @@ u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant, return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm); } -u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) +static u32 __get_barrier_crm_val(enum aarch64_insn_mb_type type) { - u32 opt; - u32 insn; - switch (type) { case AARCH64_INSN_MB_SY: - opt = 0xf; - break; + return 0xf; case AARCH64_INSN_MB_ST: - opt = 0xe; - break; + return 0xe; case AARCH64_INSN_MB_LD: - opt = 0xd; - break; + return 0xd; case AARCH64_INSN_MB_ISH: - opt = 0xb; - break; + return 0xb; case AARCH64_INSN_MB_ISHST: - opt = 0xa; - break; + return 0xa; case AARCH64_INSN_MB_ISHLD: - opt = 0x9; - break; + return 0x9; case AARCH64_INSN_MB_NSH: - opt = 0x7; - break; + return 0x7; case AARCH64_INSN_MB_NSHST: - opt = 0x6; - break; + return 0x6; case AARCH64_INSN_MB_NSHLD: - opt = 0x5; - break; + return 0x5; default: - pr_err("%s: unknown dmb type %d\n", __func__, type); + pr_err("%s: unknown barrier type %d\n", __func__, type); return AARCH64_BREAK_FAULT; } +} + +u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) +{ + u32 opt; + u32 insn; + + opt = __get_barrier_crm_val(type); + if (opt == AARCH64_BREAK_FAULT) + return AARCH64_BREAK_FAULT; insn = aarch64_insn_get_dmb_value(); insn &= ~GENMASK(11, 8); @@ -1515,3 +1514,18 @@ u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) return insn; } + +u32 aarch64_insn_gen_dsb(enum aarch64_insn_mb_type type) +{ + u32 opt, insn; + + opt = __get_barrier_crm_val(type); + if (opt == AARCH64_BREAK_FAULT) + return AARCH64_BREAK_FAULT; + + insn = aarch64_insn_get_dsb_base_value(); + insn &= ~GENMASK(11, 8); + insn |= (opt << 8); + + return insn; +} diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index fd71e882d6c87..502be5fbaea10 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -787,6 +787,7 @@ static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs) */ siaddr = untagged_addr(far); } + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); trace_android_rvh_do_sea(siaddr, esr, regs); arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 2e686a67c56de..1839847c5a85c 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1253,7 +1253,8 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr) next = addr; end = addr + PUD_SIZE; do { - pmd_free_pte_page(pmdp, next); + if (pmd_present(pmdp_get(pmdp))) + pmd_free_pte_page(pmdp, next); } while (pmdp++, next += PMD_SIZE, next != end); pud_clear(pudp); diff --git a/arch/arm64/mm/ptdump_debugfs.c b/arch/arm64/mm/ptdump_debugfs.c index 68bf1a125502d..1e308328c0796 100644 --- a/arch/arm64/mm/ptdump_debugfs.c +++ b/arch/arm64/mm/ptdump_debugfs.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 #include -#include #include #include @@ -9,9 +8,7 @@ static int ptdump_show(struct seq_file *m, void *v) { struct ptdump_info *info = m->private; - get_online_mems(); ptdump_walk(m, info); - put_online_mems(); return 0; } DEFINE_SHOW_ATTRIBUTE(ptdump); diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index e310ee022a959..479474fe6f47c 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -7,8 +7,10 @@ #define pr_fmt(fmt) "bpf_jit: " fmt +#include #include #include +#include #include #include #include @@ -17,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -99,6 +102,14 @@ static inline void emit(const u32 insn, struct jit_ctx *ctx) ctx->idx++; } +static inline void emit_u32_data(const u32 data, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) + ctx->image[ctx->idx] = data; + + ctx->idx++; +} + static inline void emit_a64_mov_i(const int is64, const int reg, const s32 val, struct jit_ctx *ctx) { @@ -159,6 +170,12 @@ static inline void emit_bti(u32 insn, struct jit_ctx *ctx) emit(insn, ctx); } +static inline void emit_kcfi(u32 hash, struct jit_ctx *ctx) +{ + if (IS_ENABLED(CONFIG_CFI_CLANG)) + emit_u32_data(hash, ctx); +} + /* * Kernel addresses in the vmalloc space use at most 48 bits, and the * remaining bits are guaranteed to be 0x1. So we can compose the address @@ -298,7 +315,6 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) const u8 fp = bpf2a64[BPF_REG_FP]; const u8 tcc = bpf2a64[TCALL_CNT]; const u8 fpb = bpf2a64[FP_BOTTOM]; - const int idx0 = ctx->idx; int cur_offset; /* @@ -324,6 +340,9 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) * */ + emit_kcfi(is_main_prog ? cfi_bpf_hash : cfi_bpf_subprog_hash, ctx); + const int idx0 = ctx->idx; + /* bpf function may be invoked by 3 instruction types: * 1. bl, attached via freplace to bpf prog via short jump * 2. br, attached via freplace to bpf prog via long jump @@ -655,7 +674,51 @@ static void build_plt(struct jit_ctx *ctx) plt->target = (u64)&dummy_tramp; } -static void build_epilogue(struct jit_ctx *ctx) +/* Clobbers BPF registers 1-4, aka x0-x3 */ +static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx) +{ + const u8 r1 = bpf2a64[BPF_REG_1]; /* aka x0 */ + u8 k = get_spectre_bhb_loop_value(); + + if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) || + cpu_mitigations_off() || __nospectre_bhb || + arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) + return; + + if (capable(CAP_SYS_ADMIN)) + return; + + if (supports_clearbhb(SCOPE_SYSTEM)) { + emit(aarch64_insn_gen_hint(AARCH64_INSN_HINT_CLEARBHB), ctx); + return; + } + + if (k) { + emit_a64_mov_i64(r1, k, ctx); + emit(A64_B(1), ctx); + emit(A64_SUBS_I(true, r1, r1, 1), ctx); + emit(A64_B_(A64_COND_NE, -2), ctx); + emit(aarch64_insn_gen_dsb(AARCH64_INSN_MB_ISH), ctx); + emit(aarch64_insn_get_isb_value(), ctx); + } + + if (is_spectre_bhb_fw_mitigated()) { + emit(A64_ORR_I(false, r1, AARCH64_INSN_REG_ZR, + ARM_SMCCC_ARCH_WORKAROUND_3), ctx); + switch (arm_smccc_1_1_get_conduit()) { + case SMCCC_CONDUIT_HVC: + emit(aarch64_insn_get_hvc_value(), ctx); + break; + case SMCCC_CONDUIT_SMC: + emit(aarch64_insn_get_smc_value(), ctx); + break; + default: + pr_err_once("Firmware mitigation enabled with unknown conduit\n"); + } + } +} + +static void build_epilogue(struct jit_ctx *ctx, bool was_classic) { const u8 r0 = bpf2a64[BPF_REG_0]; const u8 r6 = bpf2a64[BPF_REG_6]; @@ -677,10 +740,13 @@ static void build_epilogue(struct jit_ctx *ctx) emit(A64_POP(r8, r9, A64_SP), ctx); emit(A64_POP(r6, r7, A64_SP), ctx); + if (was_classic) + build_bhb_mitigation(ctx); + /* Restore FP/LR registers */ emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); - /* Set return value */ + /* Move the return value from bpf:r0 (aka x7) to x0 */ emit(A64_MOV(1, A64_R(0), r0), ctx); /* Authenticate lr */ @@ -1588,7 +1654,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) } ctx.epilogue_offset = ctx.idx; - build_epilogue(&ctx); + build_epilogue(&ctx, was_classic); build_plt(&ctx); extable_align = __alignof__(struct exception_table_entry); @@ -1624,7 +1690,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto out_off; } - build_epilogue(&ctx); + build_epilogue(&ctx, was_classic); build_plt(&ctx); /* 3. Extra pass to validate JITed code. */ @@ -1658,9 +1724,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) jit_data->image = image_ptr; jit_data->header = header; } - prog->bpf_func = (void *)ctx.image; + prog->bpf_func = (void *)ctx.image + cfi_get_offset(); prog->jited = 1; - prog->jited_len = prog_size; + prog->jited_len = prog_size - cfi_get_offset(); if (!prog->is_func || extra_pass) { int i; @@ -1916,6 +1982,12 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, /* return address locates above FP */ retaddr_off = stack_size + 8; + if (flags & BPF_TRAMP_F_INDIRECT) { + /* + * Indirect call for bpf_struct_ops + */ + emit_kcfi(cfi_get_func_hash(orig_call), ctx); + } /* bpf trampoline may be invoked by 3 instruction types: * 1. bl, attached to bpf prog or kernel function via short jump * 2. br, attached to bpf prog or kernel function via long jump @@ -1956,7 +2028,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, emit(A64_STR64I(A64_R(20), A64_SP, regs_off + 8), ctx); if (flags & BPF_TRAMP_F_CALL_ORIG) { - emit_addr_mov_i64(A64_R(0), (const u64)im, ctx); + /* for the first pass, assume the worst case */ + if (!ctx->image) + ctx->idx += 4; + else + emit_a64_mov_i64(A64_R(0), (const u64)im, ctx); emit_call((const u64)__bpf_tramp_enter, ctx); } @@ -2000,7 +2076,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (flags & BPF_TRAMP_F_CALL_ORIG) { im->ip_epilogue = ctx->image + ctx->idx; - emit_addr_mov_i64(A64_R(0), (const u64)im, ctx); + /* for the first pass, assume the worst case */ + if (!ctx->image) + ctx->idx += 4; + else + emit_a64_mov_i64(A64_R(0), (const u64)im, ctx); emit_call((const u64)__bpf_tramp_exit, ctx); } diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S index 9d01361696a14..ae551b8571374 100644 --- a/arch/arm64/xen/hypercall.S +++ b/arch/arm64/xen/hypercall.S @@ -83,7 +83,26 @@ HYPERCALL3(vcpu_op); HYPERCALL1(platform_op_raw); HYPERCALL2(multicall); HYPERCALL2(vm_assist); -HYPERCALL3(dm_op); + +SYM_FUNC_START(HYPERVISOR_dm_op) + mov x16, #__HYPERVISOR_dm_op; \ + /* + * dm_op hypercalls are issued by the userspace. The kernel needs to + * enable access to TTBR0_EL1 as the hypervisor would issue stage 1 + * translations to user memory via AT instructions. Since AT + * instructions are not affected by the PAN bit (ARMv8.1), we only + * need the explicit uaccess_enable/disable if the TTBR0 PAN emulation + * is enabled (it implies that hardware UAO and PAN disabled). + */ + uaccess_ttbr0_enable x6, x7, x8 + hvc XEN_IMM + + /* + * Disable userspace access from kernel once the hyp call completed. + */ + uaccess_ttbr0_disable x6, x7 + ret +SYM_FUNC_END(HYPERVISOR_dm_op); SYM_FUNC_START(privcmd_call) mov x16, x0 diff --git a/arch/csky/kernel/asm-offsets.c b/arch/csky/kernel/asm-offsets.c index d1e9035794733..5525c8e7e1d9e 100644 --- a/arch/csky/kernel/asm-offsets.c +++ b/arch/csky/kernel/asm-offsets.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. +#define COMPILE_OFFSETS #include #include diff --git a/arch/hexagon/kernel/asm-offsets.c b/arch/hexagon/kernel/asm-offsets.c index 03a7063f94561..50eea9fa6f137 100644 --- a/arch/hexagon/kernel/asm-offsets.c +++ b/arch/hexagon/kernel/asm-offsets.c @@ -8,6 +8,7 @@ * * Copyright (c) 2010-2012, The Linux Foundation. All rights reserved. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 25aa993abebce..9f56c15ed22d0 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -503,10 +503,14 @@ config ARCH_STRICT_ALIGN -mstrict-align build parameter to prevent unaligned accesses. CPUs with h/w unaligned access support: - Loongson-2K2000/2K3000/3A5000/3C5000/3D5000. + Loongson-2K2000/2K3000 and all of Loongson-3 series processors + based on LoongArch. CPUs without h/w unaligned access support: - Loongson-2K500/2K1000. + Loongson-2K0300/2K0500/2K1000. + + If you want to make sure whether to support unaligned memory access + on your hardware, please read the bit 20 (UAL) of CPUCFG1 register. This option is enabled by default to make the kernel be able to run on all LoongArch systems. But you can disable it manually if you want diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 81e8089c9c4f1..9c6aff9376ec0 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -43,7 +43,7 @@ endif ifdef CONFIG_64BIT ld-emul = $(64bit-emul) -cflags-y += -mabi=lp64s +cflags-y += -mabi=lp64s -mcmodel=normal endif cflags-y += -pipe -msoft-float diff --git a/arch/loongarch/include/asm/acenv.h b/arch/loongarch/include/asm/acenv.h index 52f298f7293ba..483c955f2ae50 100644 --- a/arch/loongarch/include/asm/acenv.h +++ b/arch/loongarch/include/asm/acenv.h @@ -10,9 +10,8 @@ #ifndef _ASM_LOONGARCH_ACENV_H #define _ASM_LOONGARCH_ACENV_H -/* - * This header is required by ACPI core, but we have nothing to fill in - * right now. Will be updated later when needed. - */ +#ifdef CONFIG_ARCH_STRICT_ALIGN +#define ACPI_MISALIGNMENT_NOT_SUPPORTED +#endif /* CONFIG_ARCH_STRICT_ALIGN */ #endif /* _ASM_LOONGARCH_ACENV_H */ diff --git a/arch/loongarch/include/asm/hw_breakpoint.h b/arch/loongarch/include/asm/hw_breakpoint.h index 13b2462f3d8c9..5faa97a87a9e2 100644 --- a/arch/loongarch/include/asm/hw_breakpoint.h +++ b/arch/loongarch/include/asm/hw_breakpoint.h @@ -134,13 +134,13 @@ static inline void hw_breakpoint_thread_switch(struct task_struct *next) /* Determine number of BRP registers available. */ static inline int get_num_brps(void) { - return csr_read64(LOONGARCH_CSR_FWPC) & CSR_FWPC_NUM; + return csr_read32(LOONGARCH_CSR_FWPC) & CSR_FWPC_NUM; } /* Determine number of WRP registers available. */ static inline int get_num_wrps(void) { - return csr_read64(LOONGARCH_CSR_MWPC) & CSR_MWPC_NUM; + return csr_read32(LOONGARCH_CSR_MWPC) & CSR_MWPC_NUM; } #endif /* __KERNEL__ */ diff --git a/arch/loongarch/include/asm/irqflags.h b/arch/loongarch/include/asm/irqflags.h index 319a8c616f1f5..003172b8406be 100644 --- a/arch/loongarch/include/asm/irqflags.h +++ b/arch/loongarch/include/asm/irqflags.h @@ -14,40 +14,48 @@ static inline void arch_local_irq_enable(void) { u32 flags = CSR_CRMD_IE; + register u32 mask asm("t0") = CSR_CRMD_IE; + __asm__ __volatile__( "csrxchg %[val], %[mask], %[reg]\n\t" : [val] "+r" (flags) - : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD) : "memory"); } static inline void arch_local_irq_disable(void) { u32 flags = 0; + register u32 mask asm("t0") = CSR_CRMD_IE; + __asm__ __volatile__( "csrxchg %[val], %[mask], %[reg]\n\t" : [val] "+r" (flags) - : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD) : "memory"); } static inline unsigned long arch_local_irq_save(void) { u32 flags = 0; + register u32 mask asm("t0") = CSR_CRMD_IE; + __asm__ __volatile__( "csrxchg %[val], %[mask], %[reg]\n\t" : [val] "+r" (flags) - : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD) : "memory"); return flags; } static inline void arch_local_irq_restore(unsigned long flags) { + register u32 mask asm("t0") = CSR_CRMD_IE; + __asm__ __volatile__( "csrxchg %[val], %[mask], %[reg]\n\t" : [val] "+r" (flags) - : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD) : "memory"); } diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 29d9b12298bc8..fc51c6df7e998 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -448,6 +448,9 @@ static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { + if (pte_val(pte) & _PAGE_DIRTY) + pte_val(pte) |= _PAGE_MODIFIED; + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | (pgprot_val(newprot) & ~_PAGE_CHG_MASK)); } @@ -570,9 +573,11 @@ static inline struct page *pmd_page(pmd_t pmd) static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { - pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) | - (pgprot_val(newprot) & ~_HPAGE_CHG_MASK); - return pmd; + if (pmd_val(pmd) & _PAGE_DIRTY) + pmd_val(pmd) |= _PAGE_MODIFIED; + + return __pmd((pmd_val(pmd) & _HPAGE_CHG_MASK) | + (pgprot_val(newprot) & ~_HPAGE_CHG_MASK)); } static inline pmd_t pmd_mkinvalid(pmd_t pmd) diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h index a5b63c84f8541..e5d21e836d993 100644 --- a/arch/loongarch/include/asm/ptrace.h +++ b/arch/loongarch/include/asm/ptrace.h @@ -55,7 +55,7 @@ static inline void instruction_pointer_set(struct pt_regs *regs, unsigned long v /* Query offset/name of register from its name/offset */ extern int regs_query_register_offset(const char *name); -#define MAX_REG_OFFSET (offsetof(struct pt_regs, __last)) +#define MAX_REG_OFFSET (offsetof(struct pt_regs, __last) - sizeof(unsigned long)) /** * regs_get_register() - get register value from its offset diff --git a/arch/loongarch/include/asm/uprobes.h b/arch/loongarch/include/asm/uprobes.h index c8f59983f702d..d01b6704e1d8e 100644 --- a/arch/loongarch/include/asm/uprobes.h +++ b/arch/loongarch/include/asm/uprobes.h @@ -15,7 +15,6 @@ typedef u32 uprobe_opcode_t; #define UPROBE_XOLBP_INSN larch_insn_gen_break(BRK_UPROBE_XOLBP) struct arch_uprobe { - unsigned long resume_era; u32 insn[2]; u32 ixol[2]; bool simulate; diff --git a/arch/loongarch/include/uapi/asm/ptrace.h b/arch/loongarch/include/uapi/asm/ptrace.h index aafb3cd9e943e..215e0f9e8aa32 100644 --- a/arch/loongarch/include/uapi/asm/ptrace.h +++ b/arch/loongarch/include/uapi/asm/ptrace.h @@ -10,10 +10,6 @@ #include -#ifndef __KERNEL__ -#include -#endif - /* * For PTRACE_{POKE,PEEK}USR. 0 - 31 are GPRs, * 32 is syscall's original ARG0, 33 is PC, 34 is BADVADDR. @@ -41,44 +37,44 @@ struct user_pt_regs { } __attribute__((aligned(8))); struct user_fp_state { - uint64_t fpr[32]; - uint64_t fcc; - uint32_t fcsr; + __u64 fpr[32]; + __u64 fcc; + __u32 fcsr; }; struct user_lsx_state { /* 32 registers, 128 bits width per register. */ - uint64_t vregs[32*2]; + __u64 vregs[32*2]; }; struct user_lasx_state { /* 32 registers, 256 bits width per register. */ - uint64_t vregs[32*4]; + __u64 vregs[32*4]; }; struct user_lbt_state { - uint64_t scr[4]; - uint32_t eflags; - uint32_t ftop; + __u64 scr[4]; + __u32 eflags; + __u32 ftop; }; struct user_watch_state { - uint64_t dbg_info; + __u64 dbg_info; struct { - uint64_t addr; - uint64_t mask; - uint32_t ctrl; - uint32_t pad; + __u64 addr; + __u64 mask; + __u32 ctrl; + __u32 pad; } dbg_regs[8]; }; struct user_watch_state_v2 { - uint64_t dbg_info; + __u64 dbg_info; struct { - uint64_t addr; - uint64_t mask; - uint32_t ctrl; - uint32_t pad; + __u64 addr; + __u64 mask; + __u32 ctrl; + __u32 pad; } dbg_regs[14]; }; diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 8da0726777edb..110afd3cc8f34 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -4,6 +4,8 @@ * * Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ +#define COMPILE_OFFSETS + #include #include #include diff --git a/arch/loongarch/kernel/env.c b/arch/loongarch/kernel/env.c index 6b3bfb0092e60..2178b35896af1 100644 --- a/arch/loongarch/kernel/env.c +++ b/arch/loongarch/kernel/env.c @@ -72,6 +72,8 @@ static int __init boardinfo_init(void) struct kobject *loongson_kobj; loongson_kobj = kobject_create_and_add("loongson", firmware_kobj); + if (!loongson_kobj) + return -ENOMEM; return sysfs_create_file(loongson_kobj, &boardinfo_attr.attr); } diff --git a/arch/loongarch/kernel/kfpu.c b/arch/loongarch/kernel/kfpu.c index ec5b28e570c96..4c476904227f9 100644 --- a/arch/loongarch/kernel/kfpu.c +++ b/arch/loongarch/kernel/kfpu.c @@ -18,11 +18,28 @@ static unsigned int euen_mask = CSR_EUEN_FPEN; static DEFINE_PER_CPU(bool, in_kernel_fpu); static DEFINE_PER_CPU(unsigned int, euen_current); +static inline void fpregs_lock(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_disable(); + else + local_bh_disable(); +} + +static inline void fpregs_unlock(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_enable(); + else + local_bh_enable(); +} + void kernel_fpu_begin(void) { unsigned int *euen_curr; - preempt_disable(); + if (!irqs_disabled()) + fpregs_lock(); WARN_ON(this_cpu_read(in_kernel_fpu)); @@ -73,7 +90,8 @@ void kernel_fpu_end(void) this_cpu_write(in_kernel_fpu, false); - preempt_enable(); + if (!irqs_disabled()) + fpregs_unlock(); } EXPORT_SYMBOL_GPL(kernel_fpu_end); diff --git a/arch/loongarch/kernel/module-sections.c b/arch/loongarch/kernel/module-sections.c index e2f30ff9afde8..a43ba7f9f9872 100644 --- a/arch/loongarch/kernel/module-sections.c +++ b/arch/loongarch/kernel/module-sections.c @@ -8,6 +8,7 @@ #include #include #include +#include Elf_Addr module_emit_got_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr val) { @@ -61,39 +62,38 @@ Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr v return (Elf_Addr)&plt[nr]; } -static int is_rela_equal(const Elf_Rela *x, const Elf_Rela *y) -{ - return x->r_info == y->r_info && x->r_addend == y->r_addend; -} +#define cmp_3way(a, b) ((a) < (b) ? -1 : (a) > (b)) -static bool duplicate_rela(const Elf_Rela *rela, int idx) +static int compare_rela(const void *x, const void *y) { - int i; + int ret; + const Elf_Rela *rela_x = x, *rela_y = y; - for (i = 0; i < idx; i++) { - if (is_rela_equal(&rela[i], &rela[idx])) - return true; - } + ret = cmp_3way(rela_x->r_info, rela_y->r_info); + if (ret == 0) + ret = cmp_3way(rela_x->r_addend, rela_y->r_addend); - return false; + return ret; } static void count_max_entries(Elf_Rela *relas, int num, unsigned int *plts, unsigned int *gots) { - unsigned int i, type; + unsigned int i; + + sort(relas, num, sizeof(Elf_Rela), compare_rela, NULL); for (i = 0; i < num; i++) { - type = ELF_R_TYPE(relas[i].r_info); - switch (type) { + if (i && !compare_rela(&relas[i-1], &relas[i])) + continue; + + switch (ELF_R_TYPE(relas[i].r_info)) { case R_LARCH_SOP_PUSH_PLT_PCREL: case R_LARCH_B26: - if (!duplicate_rela(relas, i)) - (*plts)++; + (*plts)++; break; case R_LARCH_GOT_PC_HI20: - if (!duplicate_rela(relas, i)) - (*gots)++; + (*gots)++; break; default: break; /* Do nothing. */ diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c index 0eddd4a66b874..1432bb214e3f4 100644 --- a/arch/loongarch/kernel/relocate.c +++ b/arch/loongarch/kernel/relocate.c @@ -124,6 +124,10 @@ static inline __init bool kaslr_disabled(void) if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) return true; + str = strstr(boot_command_line, "kexec_file"); + if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) + return true; + return false; } diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 655dc2b1616f2..31be0d00976e2 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -363,10 +363,9 @@ void __init platform_init(void) arch_reserve_vmcore(); arch_parse_crashkernel(); -#ifdef CONFIG_ACPI_TABLE_UPGRADE - acpi_table_upgrade(); -#endif #ifdef CONFIG_ACPI + acpi_table_upgrade(); + acpi_gbl_use_global_lock = false; acpi_gbl_use_default_register_widths = false; acpi_boot_table_init(); #endif diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c index 4a3686d133494..0e90cd2df0ea3 100644 --- a/arch/loongarch/kernel/signal.c +++ b/arch/loongarch/kernel/signal.c @@ -697,6 +697,11 @@ static int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, for (i = 1; i < 32; i++) err |= __put_user(regs->regs[i], &sc->sc_regs[i]); +#ifdef CONFIG_CPU_HAS_LBT + if (extctx->lbt.addr) + err |= protected_save_lbt_context(extctx); +#endif + if (extctx->lasx.addr) err |= protected_save_lasx_context(extctx); else if (extctx->lsx.addr) @@ -704,11 +709,6 @@ static int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, else if (extctx->fpu.addr) err |= protected_save_fpu_context(extctx); -#ifdef CONFIG_CPU_HAS_LBT - if (extctx->lbt.addr) - err |= protected_save_lbt_context(extctx); -#endif - /* Set the "end" magic */ info = (struct sctx_info *)extctx->end.addr; err |= __put_user(0, &info->magic); diff --git a/arch/loongarch/kernel/time.c b/arch/loongarch/kernel/time.c index e7015f7b70e37..a3732f754b5d8 100644 --- a/arch/loongarch/kernel/time.c +++ b/arch/loongarch/kernel/time.c @@ -110,7 +110,7 @@ static unsigned long __init get_loops_per_jiffy(void) return lpj; } -static long init_offset __nosavedata; +static long init_offset; void save_counter(void) { diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index 2b4b99b4e6c94..d7291b8ea65aa 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -1097,8 +1097,8 @@ static void configure_exception_vector(void) tlbrentry = (unsigned long)exception_handlers + 80*VECSIZE; csr_write64(eentry, LOONGARCH_CSR_EENTRY); - csr_write64(eentry, LOONGARCH_CSR_MERRENTRY); - csr_write64(tlbrentry, LOONGARCH_CSR_TLBRENTRY); + csr_write64(__pa(eentry), LOONGARCH_CSR_MERRENTRY); + csr_write64(__pa(tlbrentry), LOONGARCH_CSR_TLBRENTRY); } void per_cpu_trap_init(int cpu) diff --git a/arch/loongarch/kernel/uprobes.c b/arch/loongarch/kernel/uprobes.c index 87abc7137b738..6022eb0f71dbc 100644 --- a/arch/loongarch/kernel/uprobes.c +++ b/arch/loongarch/kernel/uprobes.c @@ -42,7 +42,6 @@ int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) utask->autask.saved_trap_nr = current->thread.trap_nr; current->thread.trap_nr = UPROBE_TRAP_NR; instruction_pointer_set(regs, utask->xol_vaddr); - user_enable_single_step(current); return 0; } @@ -53,13 +52,7 @@ int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR); current->thread.trap_nr = utask->autask.saved_trap_nr; - - if (auprobe->simulate) - instruction_pointer_set(regs, auprobe->resume_era); - else - instruction_pointer_set(regs, utask->vaddr + LOONGARCH_INSN_SIZE); - - user_disable_single_step(current); + instruction_pointer_set(regs, utask->vaddr + LOONGARCH_INSN_SIZE); return 0; } @@ -70,7 +63,6 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) current->thread.trap_nr = utask->autask.saved_trap_nr; instruction_pointer_set(regs, utask->vaddr); - user_disable_single_step(current); } bool arch_uprobe_xol_was_trapped(struct task_struct *t) @@ -90,7 +82,6 @@ bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) insn.word = auprobe->insn[0]; arch_simulate_insn(insn, regs); - auprobe->resume_era = regs->csr_era; return true; } diff --git a/arch/loongarch/mm/hugetlbpage.c b/arch/loongarch/mm/hugetlbpage.c index 41308429f4461..58277fd9fb2f0 100644 --- a/arch/loongarch/mm/hugetlbpage.c +++ b/arch/loongarch/mm/hugetlbpage.c @@ -47,7 +47,8 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, pmd = pmd_offset(pud, addr); } } - return pmd_none(pmdp_get(pmd)) ? NULL : (pte_t *) pmd; + + return (!pmd || pmd_none(pmdp_get(pmd))) ? NULL : (pte_t *) pmd; } int pmd_huge(pmd_t pmd) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index dcb1428b458c8..869003f1c7036 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -203,11 +203,9 @@ bool bpf_jit_supports_kfunc_call(void) return true; } -/* initialized on the first pass of build_body() */ -static int out_offset = -1; -static int emit_bpf_tail_call(struct jit_ctx *ctx) +static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) { - int off; + int off, tc_ninsn = 0; u8 tcc = tail_call_reg(ctx); u8 a1 = LOONGARCH_GPR_A1; u8 a2 = LOONGARCH_GPR_A2; @@ -217,7 +215,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) const int idx0 = ctx->idx; #define cur_offset (ctx->idx - idx0) -#define jmp_offset (out_offset - (cur_offset)) +#define jmp_offset (tc_ninsn - (cur_offset)) /* * a0: &ctx @@ -227,6 +225,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) * if (index >= array->map.max_entries) * goto out; */ + tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ @@ -258,15 +257,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) emit_insn(ctx, ldd, t3, t2, off); __build_epilogue(ctx, true); - /* out: */ - if (out_offset == -1) - out_offset = cur_offset; - if (cur_offset != out_offset) { - pr_err_once("tail_call out_offset = %d, expected %d!\n", - cur_offset, out_offset); - return -1; - } - return 0; toofar: @@ -853,7 +843,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext /* tail call */ case BPF_JMP | BPF_TAIL_CALL: mark_tail_call(ctx); - if (emit_bpf_tail_call(ctx) < 0) + if (emit_bpf_tail_call(ctx, i) < 0) return -EINVAL; break; @@ -1251,7 +1241,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (tmp_blinded) bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); - out_offset = -1; return prog; } diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c index 2726639150bc7..927dd31f82b93 100644 --- a/arch/loongarch/pci/pci.c +++ b/arch/loongarch/pci/pci.c @@ -51,11 +51,11 @@ static int __init pcibios_init(void) */ lsize = cpu_last_level_cache_line_size(); - BUG_ON(!lsize); + if (lsize) { + pci_dfl_cache_line_size = lsize >> 2; - pci_dfl_cache_line_size = lsize >> 2; - - pr_debug("PCI: pci_cache_line_size set to %d bytes\n", lsize); + pr_debug("PCI: pci_cache_line_size set to %d bytes\n", lsize); + } return 0; } diff --git a/arch/loongarch/power/hibernate.c b/arch/loongarch/power/hibernate.c index 1e0590542f987..e7b7346592cb2 100644 --- a/arch/loongarch/power/hibernate.c +++ b/arch/loongarch/power/hibernate.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -14,6 +15,7 @@ struct pt_regs saved_regs; void save_processor_state(void) { + save_counter(); saved_crmd = csr_read32(LOONGARCH_CSR_CRMD); saved_prmd = csr_read32(LOONGARCH_CSR_PRMD); saved_euen = csr_read32(LOONGARCH_CSR_EUEN); @@ -26,6 +28,7 @@ void save_processor_state(void) void restore_processor_state(void) { + sync_counter(); csr_write32(saved_crmd, LOONGARCH_CSR_CRMD); csr_write32(saved_prmd, LOONGARCH_CSR_PRMD); csr_write32(saved_euen, LOONGARCH_CSR_EUEN); diff --git a/arch/m68k/Kconfig.debug b/arch/m68k/Kconfig.debug index 30638a6e8edcb..d036f903864c2 100644 --- a/arch/m68k/Kconfig.debug +++ b/arch/m68k/Kconfig.debug @@ -10,7 +10,7 @@ config BOOTPARAM_STRING config EARLY_PRINTK bool "Early printk" - depends on !(SUN3 || M68000 || COLDFIRE) + depends on MMU_MOTOROLA help Write kernel log output directly to a serial port. Where implemented, output goes to the framebuffer as well. diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index e984af71df6be..d86aa744cb8fc 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -329,12 +329,12 @@ arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) #include #else -static inline int find_first_zero_bit(const unsigned long *vaddr, - unsigned size) +static inline unsigned long find_first_zero_bit(const unsigned long *vaddr, + unsigned long size) { const unsigned long *p = vaddr; - int res = 32; - unsigned int words; + unsigned long res = 32; + unsigned long words; unsigned long num; if (!size) @@ -355,8 +355,9 @@ static inline int find_first_zero_bit(const unsigned long *vaddr, } #define find_first_zero_bit find_first_zero_bit -static inline int find_next_zero_bit(const unsigned long *vaddr, int size, - int offset) +static inline unsigned long find_next_zero_bit(const unsigned long *vaddr, + unsigned long size, + unsigned long offset) { const unsigned long *p = vaddr + (offset >> 5); int bit = offset & 31UL, res; @@ -385,11 +386,12 @@ static inline int find_next_zero_bit(const unsigned long *vaddr, int size, } #define find_next_zero_bit find_next_zero_bit -static inline int find_first_bit(const unsigned long *vaddr, unsigned size) +static inline unsigned long find_first_bit(const unsigned long *vaddr, + unsigned long size) { const unsigned long *p = vaddr; - int res = 32; - unsigned int words; + unsigned long res = 32; + unsigned long words; unsigned long num; if (!size) @@ -410,8 +412,9 @@ static inline int find_first_bit(const unsigned long *vaddr, unsigned size) } #define find_first_bit find_first_bit -static inline int find_next_bit(const unsigned long *vaddr, int size, - int offset) +static inline unsigned long find_next_bit(const unsigned long *vaddr, + unsigned long size, + unsigned long offset) { const unsigned long *p = vaddr + (offset >> 5); int bit = offset & 31UL, res; diff --git a/arch/m68k/kernel/asm-offsets.c b/arch/m68k/kernel/asm-offsets.c index 906d732305374..67a1990f9d748 100644 --- a/arch/m68k/kernel/asm-offsets.c +++ b/arch/m68k/kernel/asm-offsets.c @@ -9,6 +9,7 @@ * #defines from the assembly-language output. */ +#define COMPILE_OFFSETS #define ASM_OFFSETS_C #include diff --git a/arch/m68k/kernel/early_printk.c b/arch/m68k/kernel/early_printk.c index f11ef9f1f56fc..521cbb8a150c9 100644 --- a/arch/m68k/kernel/early_printk.c +++ b/arch/m68k/kernel/early_printk.c @@ -16,25 +16,10 @@ #include "../mvme147/mvme147.h" #include "../mvme16x/mvme16x.h" -asmlinkage void __init debug_cons_nputs(const char *s, unsigned n); - -static void __ref debug_cons_write(struct console *c, - const char *s, unsigned n) -{ -#if !(defined(CONFIG_SUN3) || defined(CONFIG_M68000) || \ - defined(CONFIG_COLDFIRE)) - if (MACH_IS_MVME147) - mvme147_scc_write(c, s, n); - else if (MACH_IS_MVME16x) - mvme16x_cons_write(c, s, n); - else - debug_cons_nputs(s, n); -#endif -} +asmlinkage void __init debug_cons_nputs(struct console *c, const char *s, unsigned int n); static struct console early_console_instance = { .name = "debug", - .write = debug_cons_write, .flags = CON_PRINTBUFFER | CON_BOOT, .index = -1 }; @@ -44,6 +29,12 @@ static int __init setup_early_printk(char *buf) if (early_console || buf) return 0; + if (MACH_IS_MVME147) + early_console_instance.write = mvme147_scc_write; + else if (MACH_IS_MVME16x) + early_console_instance.write = mvme16x_cons_write; + else + early_console_instance.write = debug_cons_nputs; early_console = &early_console_instance; register_console(early_console); @@ -51,20 +42,15 @@ static int __init setup_early_printk(char *buf) } early_param("earlyprintk", setup_early_printk); -/* - * debug_cons_nputs() defined in arch/m68k/kernel/head.S cannot be called - * after init sections are discarded (for platforms that use it). - */ -#if !(defined(CONFIG_SUN3) || defined(CONFIG_M68000) || \ - defined(CONFIG_COLDFIRE)) - static int __init unregister_early_console(void) { - if (!early_console || MACH_IS_MVME16x) - return 0; + /* + * debug_cons_nputs() defined in arch/m68k/kernel/head.S cannot be + * called after init sections are discarded (for platforms that use it). + */ + if (early_console && early_console->write == debug_cons_nputs) + return unregister_console(early_console); - return unregister_console(early_console); + return 0; } late_initcall(unregister_early_console); - -#endif diff --git a/arch/m68k/kernel/head.S b/arch/m68k/kernel/head.S index 9e812d8606be8..10f6aa4d8f057 100644 --- a/arch/m68k/kernel/head.S +++ b/arch/m68k/kernel/head.S @@ -3267,8 +3267,8 @@ func_return putn * turns around and calls the internal routines. This routine * is used by the boot console. * - * The calling parameters are: - * void debug_cons_nputs(const char *str, unsigned length) + * The function signature is - + * void debug_cons_nputs(struct console *c, const char *s, unsigned int n) * * This routine does NOT understand variable arguments only * simple strings! @@ -3277,8 +3277,8 @@ ENTRY(debug_cons_nputs) moveml %d0/%d1/%a0,%sp@- movew %sr,%sp@- ori #0x0700,%sr - movel %sp@(18),%a0 /* fetch parameter */ - movel %sp@(22),%d1 /* fetch parameter */ + movel %sp@(22),%a0 /* char *s */ + movel %sp@(26),%d1 /* unsigned int n */ jra 2f 1: #ifdef CONSOLE_DEBUG @@ -3404,6 +3404,7 @@ L(console_clear_loop): movel %d4,%d1 /* screen height in pixels */ divul %a0@(FONT_DESC_HEIGHT),%d1 /* d1 = max num rows */ + subql #1,%d1 /* row range is 0 to num - 1 */ movel %d0,%a2@(Lconsole_struct_num_columns) movel %d1,%a2@(Lconsole_struct_num_rows) @@ -3550,15 +3551,14 @@ func_start console_putc,%a0/%a1/%d0-%d7 cmpib #10,%d7 jne L(console_not_lf) movel %a0@(Lconsole_struct_cur_row),%d0 - addil #1,%d0 - movel %d0,%a0@(Lconsole_struct_cur_row) movel %a0@(Lconsole_struct_num_rows),%d1 cmpl %d1,%d0 jcs 1f - subil #1,%d0 - movel %d0,%a0@(Lconsole_struct_cur_row) console_scroll + jra L(console_exit) 1: + addql #1,%d0 + movel %d0,%a0@(Lconsole_struct_cur_row) jra L(console_exit) L(console_not_lf): @@ -3585,12 +3585,6 @@ L(console_not_cr): */ L(console_not_home): movel %a0@(Lconsole_struct_cur_column),%d0 - addql #1,%a0@(Lconsole_struct_cur_column) - movel %a0@(Lconsole_struct_num_columns),%d1 - cmpl %d1,%d0 - jcs 1f - console_putc #'\n' /* recursion is OK! */ -1: movel %a0@(Lconsole_struct_cur_row),%d1 /* @@ -3637,6 +3631,23 @@ L(console_do_font_scanline): addq #1,%d1 dbra %d7,L(console_read_char_scanline) + /* + * Register usage in the code below: + * a0 = pointer to console globals + * d0 = cursor column + * d1 = cursor column limit + */ + + lea %pc@(L(console_globals)),%a0 + + movel %a0@(Lconsole_struct_cur_column),%d0 + addql #1,%d0 + movel %d0,%a0@(Lconsole_struct_cur_column) /* Update cursor pos */ + movel %a0@(Lconsole_struct_num_columns),%d1 + cmpl %d1,%d0 + jcs L(console_exit) + console_putc #'\n' /* Line wrap using tail recursion */ + L(console_exit): func_return console_putc diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index 382f656c29eae..9f5603e01a688 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -801,7 +801,7 @@ static void __init mac_identify(void) } macintosh_config = mac_data_table; - for (m = macintosh_config; m->ident != -1; m++) { + for (m = &mac_data_table[1]; m->ident != -1; m++) { if (m->ident == model) { macintosh_config = m; break; diff --git a/arch/microblaze/kernel/asm-offsets.c b/arch/microblaze/kernel/asm-offsets.c index 104c3ac5f30c8..b4b67d58e7f6a 100644 --- a/arch/microblaze/kernel/asm-offsets.c +++ b/arch/microblaze/kernel/asm-offsets.c @@ -7,6 +7,7 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/mips/boot/dts/lantiq/danube.dtsi b/arch/mips/boot/dts/lantiq/danube.dtsi index 7a7ba66aa5349..650400bd5725f 100644 --- a/arch/mips/boot/dts/lantiq/danube.dtsi +++ b/arch/mips/boot/dts/lantiq/danube.dtsi @@ -5,8 +5,12 @@ compatible = "lantiq,xway", "lantiq,danube"; cpus { + #address-cells = <1>; + #size-cells = <0>; + cpu@0 { compatible = "mips,mips24Kc"; + reg = <0>; }; }; @@ -100,6 +104,8 @@ 0x1000000 0 0x00000000 0xae00000 0 0x200000>; /* io space */ reg = <0x7000000 0x8000 /* config space */ 0xe105400 0x400>; /* pci bridge */ + + device_type = "pci"; }; }; }; diff --git a/arch/mips/boot/dts/lantiq/danube_easy50712.dts b/arch/mips/boot/dts/lantiq/danube_easy50712.dts index 1ce20b7d05cb8..c9f7886f57b8c 100644 --- a/arch/mips/boot/dts/lantiq/danube_easy50712.dts +++ b/arch/mips/boot/dts/lantiq/danube_easy50712.dts @@ -4,6 +4,8 @@ /include/ "danube.dtsi" / { + model = "Intel EASY50712"; + chosen { bootargs = "console=ttyLTQ0,115200 init=/etc/preinit"; }; @@ -82,16 +84,19 @@ }; }; - etop@e180000 { + ethernet@e180000 { compatible = "lantiq,etop-xway"; reg = <0xe180000 0x40000>; interrupt-parent = <&icu0>; interrupts = <73 78>; + interrupt-names = "tx", "rx"; phy-mode = "rmii"; mac-address = [ 00 11 22 33 44 55 ]; + lantiq,rx-burst-length = <4>; + lantiq,tx-burst-length = <4>; }; - stp0: stp@e100bb0 { + stp0: gpio@e100bb0 { #gpio-cells = <2>; compatible = "lantiq,gpio-stp-xway"; gpio-controller; diff --git a/arch/mips/boot/dts/loongson/loongson64c_4core_ls7a.dts b/arch/mips/boot/dts/loongson/loongson64c_4core_ls7a.dts index c7ea4f1c0bb21..6c277ab83d4b9 100644 --- a/arch/mips/boot/dts/loongson/loongson64c_4core_ls7a.dts +++ b/arch/mips/boot/dts/loongson/loongson64c_4core_ls7a.dts @@ -29,6 +29,7 @@ compatible = "loongson,pch-msi-1.0"; reg = <0 0x2ff00000 0 0x8>; interrupt-controller; + #interrupt-cells = <1>; msi-controller; loongson,msi-base-vec = <64>; loongson,msi-num-vecs = <64>; diff --git a/arch/mips/crypto/chacha-core.S b/arch/mips/crypto/chacha-core.S index 5755f69cfe007..706aeb850fb0d 100644 --- a/arch/mips/crypto/chacha-core.S +++ b/arch/mips/crypto/chacha-core.S @@ -55,17 +55,13 @@ #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define MSB 0 #define LSB 3 -#define ROTx rotl -#define ROTR(n) rotr n, 24 #define CPU_TO_LE32(n) \ - wsbh n; \ + wsbh n, n; \ rotr n, 16; #else #define MSB 3 #define LSB 0 -#define ROTx rotr #define CPU_TO_LE32(n) -#define ROTR(n) #endif #define FOR_EACH_WORD(x) \ @@ -192,10 +188,10 @@ CONCAT3(.Lchacha_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \ xor X(W), X(B); \ xor X(Y), X(C); \ xor X(Z), X(D); \ - rotl X(V), S; \ - rotl X(W), S; \ - rotl X(Y), S; \ - rotl X(Z), S; + rotr X(V), 32 - S; \ + rotr X(W), 32 - S; \ + rotr X(Y), 32 - S; \ + rotr X(Z), 32 - S; .text .set reorder @@ -372,21 +368,19 @@ chacha_crypt_arch: /* First byte */ lbu T1, 0(IN) addiu $at, BYTES, 1 - CPU_TO_LE32(SAVED_X) - ROTR(SAVED_X) xor T1, SAVED_X sb T1, 0(OUT) beqz $at, .Lchacha_mips_xor_done /* Second byte */ lbu T1, 1(IN) addiu $at, BYTES, 2 - ROTx SAVED_X, 8 + rotr SAVED_X, 8 xor T1, SAVED_X sb T1, 1(OUT) beqz $at, .Lchacha_mips_xor_done /* Third byte */ lbu T1, 2(IN) - ROTx SAVED_X, 8 + rotr SAVED_X, 8 xor T1, SAVED_X sb T1, 2(OUT) b .Lchacha_mips_xor_done diff --git a/arch/mips/include/asm/ftrace.h b/arch/mips/include/asm/ftrace.h index db497a8167da2..e3212f44446fa 100644 --- a/arch/mips/include/asm/ftrace.h +++ b/arch/mips/include/asm/ftrace.h @@ -87,4 +87,20 @@ struct dyn_arch_ftrace { #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* __ASSEMBLY__ */ #endif /* CONFIG_FUNCTION_TRACER */ + +#ifdef CONFIG_FTRACE_SYSCALLS +#ifndef __ASSEMBLY__ +/* + * Some syscall entry functions on mips start with "__sys_" (fork and clone, + * for instance). We should also match the sys_ variant with those. + */ +#define ARCH_HAS_SYSCALL_MATCH_SYM_NAME +static inline bool arch_syscall_match_sym_name(const char *sym, + const char *name) +{ + return !strcmp(sym, name) || + (!strncmp(sym, "__sys_", 6) && !strcmp(sym + 6, name + 4)); +} +#endif /* __ASSEMBLY__ */ +#endif /* CONFIG_FTRACE_SYSCALLS */ #endif /* _ASM_MIPS_FTRACE_H */ diff --git a/arch/mips/include/asm/ptrace.h b/arch/mips/include/asm/ptrace.h index 4a2b40ce39e09..841612913f0d1 100644 --- a/arch/mips/include/asm/ptrace.h +++ b/arch/mips/include/asm/ptrace.h @@ -65,7 +65,8 @@ static inline void instruction_pointer_set(struct pt_regs *regs, /* Query offset/name of register from its name/offset */ extern int regs_query_register_offset(const char *name); -#define MAX_REG_OFFSET (offsetof(struct pt_regs, __last)) +#define MAX_REG_OFFSET \ + (offsetof(struct pt_regs, __last) - sizeof(unsigned long)) /** * regs_get_register() - get register value from its offset diff --git a/arch/mips/include/asm/vpe.h b/arch/mips/include/asm/vpe.h index 61fd4d0aeda41..c0769dc4b8532 100644 --- a/arch/mips/include/asm/vpe.h +++ b/arch/mips/include/asm/vpe.h @@ -119,4 +119,12 @@ void cleanup_tc(struct tc *tc); int __init vpe_module_init(void); void __exit vpe_module_exit(void); + +#ifdef CONFIG_MIPS_VPE_LOADER_MT +void *vpe_alloc(void); +int vpe_start(void *vpe, unsigned long start); +int vpe_stop(void *vpe); +int vpe_free(void *vpe); +#endif /* CONFIG_MIPS_VPE_LOADER_MT */ + #endif /* _ASM_VPE_H */ diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c index cb1045ebab062..22c99a2cd5707 100644 --- a/arch/mips/kernel/asm-offsets.c +++ b/arch/mips/kernel/asm-offsets.c @@ -9,6 +9,8 @@ * Kevin Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com * Copyright (C) 2000 MIPS Technologies, Inc. */ +#define COMPILE_OFFSETS + #include #include #include diff --git a/arch/mips/kernel/pm-cps.c b/arch/mips/kernel/pm-cps.c index 9bf60d7d44d36..a7bcf2b814c86 100644 --- a/arch/mips/kernel/pm-cps.c +++ b/arch/mips/kernel/pm-cps.c @@ -56,10 +56,7 @@ static DEFINE_PER_CPU_ALIGNED(u32*, ready_count); /* Indicates online CPUs coupled with the current CPU */ static DEFINE_PER_CPU_ALIGNED(cpumask_t, online_coupled); -/* - * Used to synchronize entry to deep idle states. Actually per-core rather - * than per-CPU. - */ +/* Used to synchronize entry to deep idle states */ static DEFINE_PER_CPU_ALIGNED(atomic_t, pm_barrier); /* Saved CPU state across the CPS_PM_POWER_GATED state */ @@ -118,9 +115,10 @@ int cps_pm_enter_state(enum cps_pm_state state) cps_nc_entry_fn entry; struct core_boot_config *core_cfg; struct vpe_boot_config *vpe_cfg; + atomic_t *barrier; /* Check that there is an entry function for this state */ - entry = per_cpu(nc_asm_enter, core)[state]; + entry = per_cpu(nc_asm_enter, cpu)[state]; if (!entry) return -EINVAL; @@ -156,7 +154,7 @@ int cps_pm_enter_state(enum cps_pm_state state) smp_mb__after_atomic(); /* Create a non-coherent mapping of the core ready_count */ - core_ready_count = per_cpu(ready_count, core); + core_ready_count = per_cpu(ready_count, cpu); nc_addr = kmap_noncoherent(virt_to_page(core_ready_count), (unsigned long)core_ready_count); nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK); @@ -164,7 +162,8 @@ int cps_pm_enter_state(enum cps_pm_state state) /* Ensure ready_count is zero-initialised before the assembly runs */ WRITE_ONCE(*nc_core_ready_count, 0); - coupled_barrier(&per_cpu(pm_barrier, core), online); + barrier = &per_cpu(pm_barrier, cpumask_first(&cpu_sibling_map[cpu])); + coupled_barrier(barrier, online); /* Run the generated entry code */ left = entry(online, nc_core_ready_count); @@ -635,12 +634,14 @@ static void *cps_gen_entry_code(unsigned cpu, enum cps_pm_state state) static int cps_pm_online_cpu(unsigned int cpu) { - enum cps_pm_state state; - unsigned core = cpu_core(&cpu_data[cpu]); + unsigned int sibling, core; void *entry_fn, *core_rc; + enum cps_pm_state state; + + core = cpu_core(&cpu_data[cpu]); for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) { - if (per_cpu(nc_asm_enter, core)[state]) + if (per_cpu(nc_asm_enter, cpu)[state]) continue; if (!test_bit(state, state_support)) continue; @@ -652,16 +653,19 @@ static int cps_pm_online_cpu(unsigned int cpu) clear_bit(state, state_support); } - per_cpu(nc_asm_enter, core)[state] = entry_fn; + for_each_cpu(sibling, &cpu_sibling_map[cpu]) + per_cpu(nc_asm_enter, sibling)[state] = entry_fn; } - if (!per_cpu(ready_count, core)) { + if (!per_cpu(ready_count, cpu)) { core_rc = kmalloc(sizeof(u32), GFP_KERNEL); if (!core_rc) { pr_err("Failed allocate core %u ready_count\n", core); return -ENOMEM; } - per_cpu(ready_count, core) = core_rc; + + for_each_cpu(sibling, &cpu_sibling_map[cpu]) + per_cpu(ready_count, sibling) = core_rc; } return 0; diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index b630604c577f9..02aa6a04a21da 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -690,18 +690,20 @@ unsigned long mips_stack_top(void) } /* Space for the VDSO, data page & GIC user page */ - top -= PAGE_ALIGN(current->thread.abi->vdso->size); - top -= PAGE_SIZE; - top -= mips_gic_present() ? PAGE_SIZE : 0; + if (current->thread.abi) { + top -= PAGE_ALIGN(current->thread.abi->vdso->size); + top -= PAGE_SIZE; + top -= mips_gic_present() ? PAGE_SIZE : 0; + + /* Space to randomize the VDSO base */ + if (current->flags & PF_RANDOMIZE) + top -= VDSO_RANDOMIZE_SIZE; + } /* Space for cache colour alignment */ if (cpu_has_dc_aliases) top -= shm_align_mask + 1; - /* Space to randomize the VDSO base */ - if (current->flags & PF_RANDOMIZE) - top -= VDSO_RANDOMIZE_SIZE; - return top; } diff --git a/arch/mips/lantiq/falcon/sysctrl.c b/arch/mips/lantiq/falcon/sysctrl.c index 1187729d8cbb1..357543996ee66 100644 --- a/arch/mips/lantiq/falcon/sysctrl.c +++ b/arch/mips/lantiq/falcon/sysctrl.c @@ -214,19 +214,16 @@ void __init ltq_soc_init(void) of_node_put(np_syseth); of_node_put(np_sysgpe); - if ((request_mem_region(res_status.start, resource_size(&res_status), - res_status.name) < 0) || - (request_mem_region(res_ebu.start, resource_size(&res_ebu), - res_ebu.name) < 0) || - (request_mem_region(res_sys[0].start, - resource_size(&res_sys[0]), - res_sys[0].name) < 0) || - (request_mem_region(res_sys[1].start, - resource_size(&res_sys[1]), - res_sys[1].name) < 0) || - (request_mem_region(res_sys[2].start, - resource_size(&res_sys[2]), - res_sys[2].name) < 0)) + if ((!request_mem_region(res_status.start, resource_size(&res_status), + res_status.name)) || + (!request_mem_region(res_ebu.start, resource_size(&res_ebu), + res_ebu.name)) || + (!request_mem_region(res_sys[0].start, resource_size(&res_sys[0]), + res_sys[0].name)) || + (!request_mem_region(res_sys[1].start, resource_size(&res_sys[1]), + res_sys[1].name)) || + (!request_mem_region(res_sys[2].start, resource_size(&res_sys[2]), + res_sys[2].name))) pr_err("Failed to request core resources"); status_membase = ioremap(res_status.start, diff --git a/arch/mips/lantiq/xway/sysctrl.c b/arch/mips/lantiq/xway/sysctrl.c index 3ed0782252229..492f375bf1290 100644 --- a/arch/mips/lantiq/xway/sysctrl.c +++ b/arch/mips/lantiq/xway/sysctrl.c @@ -466,7 +466,7 @@ void __init ltq_soc_init(void) /* add our generic xway clocks */ clkdev_add_pmu("10000000.fpi", NULL, 0, 0, PMU_FPI); clkdev_add_pmu("1e100a00.gptu", NULL, 1, 0, PMU_GPT); - clkdev_add_pmu("1e100bb0.stp", NULL, 1, 0, PMU_STP); + clkdev_add_pmu("1e100bb0.gpio", NULL, 1, 0, PMU_STP); clkdev_add_pmu("1e100c00.serial", NULL, 0, 0, PMU_ASC1); clkdev_add_pmu("1e104100.dma", NULL, 1, 0, PMU_DMA); clkdev_add_pmu("1e100800.spi", NULL, 1, 0, PMU_SPI); @@ -478,7 +478,7 @@ void __init ltq_soc_init(void) ifccr = CGU_IFCCR_VR9; pcicr = CGU_PCICR_VR9; } else { - clkdev_add_pmu("1e180000.etop", NULL, 1, 0, PMU_PPE); + clkdev_add_pmu("1e180000.ethernet", NULL, 1, 0, PMU_PPE); } if (!of_machine_is_compatible("lantiq,ase")) @@ -512,9 +512,9 @@ void __init ltq_soc_init(void) CLOCK_133M, CLOCK_133M); clkdev_add_pmu("1e101000.usb", "otg", 1, 0, PMU_USB0); clkdev_add_pmu("1f203018.usb2-phy", "phy", 1, 0, PMU_USB0_P); - clkdev_add_pmu("1e180000.etop", "ppe", 1, 0, PMU_PPE); - clkdev_add_cgu("1e180000.etop", "ephycgu", CGU_EPHY); - clkdev_add_pmu("1e180000.etop", "ephy", 1, 0, PMU_EPHY); + clkdev_add_pmu("1e180000.ethernet", "ppe", 1, 0, PMU_PPE); + clkdev_add_cgu("1e180000.ethernet", "ephycgu", CGU_EPHY); + clkdev_add_pmu("1e180000.ethernet", "ephy", 1, 0, PMU_EPHY); clkdev_add_pmu("1e103000.sdio", NULL, 1, 0, PMU_ASE_SDIO); clkdev_add_pmu("1e116000.mei", "dfe", 1, 0, PMU_DFE); } else if (of_machine_is_compatible("lantiq,grx390")) { @@ -573,7 +573,7 @@ void __init ltq_soc_init(void) clkdev_add_pmu("1e101000.usb", "otg", 1, 0, PMU_USB0 | PMU_AHBM); clkdev_add_pmu("1f203034.usb2-phy", "phy", 1, 0, PMU_USB1_P); clkdev_add_pmu("1e106000.usb", "otg", 1, 0, PMU_USB1 | PMU_AHBM); - clkdev_add_pmu("1e180000.etop", "switch", 1, 0, PMU_SWITCH); + clkdev_add_pmu("1e180000.ethernet", "switch", 1, 0, PMU_SWITCH); clkdev_add_pmu("1e103000.sdio", NULL, 1, 0, PMU_SDIO); clkdev_add_pmu("1e103100.deu", NULL, 1, 0, PMU_DEU); clkdev_add_pmu("1e116000.mei", "dfe", 1, 0, PMU_DFE); diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 93c2d695588a1..ab8102f94e066 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -506,6 +506,60 @@ static int __init set_ntlb(char *str) __setup("ntlb=", set_ntlb); +/* Initialise all TLB entries with unique values */ +static void r4k_tlb_uniquify(void) +{ + int entry = num_wired_entries(); + + htw_stop(); + write_c0_entrylo0(0); + write_c0_entrylo1(0); + + while (entry < current_cpu_data.tlbsize) { + unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data); + unsigned long asid = 0; + int idx; + + /* Skip wired MMID to make ginvt_mmid work */ + if (cpu_has_mmid) + asid = MMID_KERNEL_WIRED + 1; + + /* Check for match before using UNIQUE_ENTRYHI */ + do { + if (cpu_has_mmid) { + write_c0_memorymapid(asid); + write_c0_entryhi(UNIQUE_ENTRYHI(entry)); + } else { + write_c0_entryhi(UNIQUE_ENTRYHI(entry) | asid); + } + mtc0_tlbw_hazard(); + tlb_probe(); + tlb_probe_hazard(); + idx = read_c0_index(); + /* No match or match is on current entry */ + if (idx < 0 || idx == entry) + break; + /* + * If we hit a match, we need to try again with + * a different ASID. + */ + asid++; + } while (asid < asid_mask); + + if (idx >= 0 && idx != entry) + panic("Unable to uniquify TLB entry %d", idx); + + write_c0_index(entry); + mtc0_tlbw_hazard(); + tlb_write_indexed(); + entry++; + } + + tlbw_use_hazard(); + htw_start(); + flush_micro_tlb(); +} + /* * Configure TLB (for init or after a CPU has been powered off). */ @@ -545,7 +599,7 @@ static void r4k_tlb_configure(void) temp_tlb_entry = current_cpu_data.tlbsize - 1; /* From this point on the ARC firmware is dead. */ - local_flush_tlb_all(); + r4k_tlb_uniquify(); /* Did I tell you that ARC SUCKS? */ } diff --git a/arch/mips/mti-malta/malta-init.c b/arch/mips/mti-malta/malta-init.c index 000d6d50520a8..82b0fd8576a24 100644 --- a/arch/mips/mti-malta/malta-init.c +++ b/arch/mips/mti-malta/malta-init.c @@ -241,16 +241,22 @@ void __init prom_init(void) #endif /* - * Setup the Malta max (2GB) memory for PCI DMA in host bridge - * in transparent addressing mode. + * Set up memory mapping in host bridge for PCI DMA masters, + * in transparent addressing mode. For EVA use the Malta + * maximum of 2 GiB memory in the alias space at 0x80000000 + * as per PHYS_OFFSET. Otherwise use 256 MiB of memory in + * the regular space, avoiding mapping the PCI MMIO window + * for DMA as it seems to confuse the system controller's + * logic, causing PCI MMIO to stop working. */ - mask = PHYS_OFFSET | PCI_BASE_ADDRESS_MEM_PREFETCH; - MSC_WRITE(MSC01_PCI_BAR0, mask); - MSC_WRITE(MSC01_PCI_HEAD4, mask); + mask = PHYS_OFFSET ? PHYS_OFFSET : 0xf0000000; + MSC_WRITE(MSC01_PCI_BAR0, + mask | PCI_BASE_ADDRESS_MEM_PREFETCH); + MSC_WRITE(MSC01_PCI_HEAD4, + PHYS_OFFSET | PCI_BASE_ADDRESS_MEM_PREFETCH); - mask &= MSC01_PCI_BAR0_SIZE_MSK; MSC_WRITE(MSC01_PCI_P2SCMSKL, mask); - MSC_WRITE(MSC01_PCI_P2SCMAPL, mask); + MSC_WRITE(MSC01_PCI_P2SCMAPL, PHYS_OFFSET); /* Don't handle target retries indefinitely. */ if ((data & MSC01_PCI_CFG_MAXRTRY_MSK) == diff --git a/arch/mips/mti-malta/malta-setup.c b/arch/mips/mti-malta/malta-setup.c index 21cb3ac1237b7..020c38e6c5de0 100644 --- a/arch/mips/mti-malta/malta-setup.c +++ b/arch/mips/mti-malta/malta-setup.c @@ -47,7 +47,7 @@ static struct resource standard_io_resources[] = { .name = "keyboard", .start = 0x60, .end = 0x6f, - .flags = IORESOURCE_IO | IORESOURCE_BUSY + .flags = IORESOURCE_IO }, { .name = "dma page reg", diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index eb56581f6d734..0a3249647401d 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -30,6 +30,7 @@ endif # offsets. cflags-vdso := $(ccflags-vdso) \ $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ + $(filter -std=%,$(KBUILD_CFLAGS)) \ -O3 -g -fPIC -fno-strict-aliasing -fno-common -fno-builtin -G 0 \ -mrelax-pic-calls $(call cc-option, -mexplicit-relocs) \ -fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \ diff --git a/arch/nios2/kernel/asm-offsets.c b/arch/nios2/kernel/asm-offsets.c index e3d9b7b6fb48a..88190b503ce5d 100644 --- a/arch/nios2/kernel/asm-offsets.c +++ b/arch/nios2/kernel/asm-offsets.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2011 Tobias Klauser */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/nios2/kernel/setup.c b/arch/nios2/kernel/setup.c index 8582ed9658447..5308c76122817 100644 --- a/arch/nios2/kernel/setup.c +++ b/arch/nios2/kernel/setup.c @@ -147,6 +147,20 @@ static void __init find_limits(unsigned long *min, unsigned long *max_low, *max_high = PFN_DOWN(memblock_end_of_DRAM()); } +static void __init adjust_lowmem_bounds(void) +{ + phys_addr_t block_start, block_end; + u64 i; + phys_addr_t memblock_limit = 0; + + for_each_mem_range(i, &block_start, &block_end) { + if (block_end > memblock_limit) + memblock_limit = block_end; + } + + memblock_set_current_limit(memblock_limit); +} + void __init setup_arch(char **cmdline_p) { console_verbose(); @@ -160,6 +174,7 @@ void __init setup_arch(char **cmdline_p) /* Keep a copy of command line */ *cmdline_p = boot_command_line; + adjust_lowmem_bounds(); find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); max_mapnr = max_low_pfn; diff --git a/arch/openrisc/kernel/asm-offsets.c b/arch/openrisc/kernel/asm-offsets.c index 710651d5aaae1..3cc826f2216b1 100644 --- a/arch/openrisc/kernel/asm-offsets.c +++ b/arch/openrisc/kernel/asm-offsets.c @@ -18,6 +18,7 @@ * compile this file to assembler, and then extract the * #defines from the assembly-language output. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile index 920db57b6b4cc..5f0a1f1b00a78 100644 --- a/arch/parisc/Makefile +++ b/arch/parisc/Makefile @@ -39,7 +39,9 @@ endif export LD_BFD -# Set default 32 bits cross compilers for vdso +# Set default 32 bits cross compilers for vdso. +# This means that for 64BIT, both the 64-bit tools and the 32-bit tools +# need to be in the path. CC_ARCHES_32 = hppa hppa2.0 hppa1.1 CC_SUFFIXES = linux linux-gnu unknown-linux-gnu suse-linux CROSS32_COMPILE := $(call cc-cross-prefix, \ @@ -139,7 +141,7 @@ palo lifimage: vmlinuz fi @if test ! -f "$(PALOCONF)"; then \ cp $(srctree)/arch/parisc/defpalo.conf $(objtree)/palo.conf; \ - echo 'A generic palo config file ($(objree)/palo.conf) has been created for you.'; \ + echo 'A generic palo config file ($(objtree)/palo.conf) has been created for you.'; \ echo 'You should check it and re-run "make palo".'; \ echo 'WARNING: the "lifimage" file is now placed in this directory by default!'; \ false; \ diff --git a/arch/parisc/boot/compressed/Makefile b/arch/parisc/boot/compressed/Makefile index a294a1b58ee7e..9fba12b70d1b3 100644 --- a/arch/parisc/boot/compressed/Makefile +++ b/arch/parisc/boot/compressed/Makefile @@ -22,6 +22,7 @@ KBUILD_CFLAGS += -fno-PIE -mno-space-regs -mdisable-fpregs -Os ifndef CONFIG_64BIT KBUILD_CFLAGS += -mfast-indirect-calls endif +KBUILD_CFLAGS += -std=gnu11 LDFLAGS_vmlinux := -X -e startup --as-needed -T $(obj)/vmlinux: $(obj)/vmlinux.lds $(addprefix $(obj)/, $(OBJECTS)) $(LIBGCC) FORCE diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index babf65751e818..3446a5e2520b2 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -276,7 +276,7 @@ extern unsigned long *empty_zero_page; #define pte_none(x) (pte_val(x) == 0) #define pte_present(x) (pte_val(x) & _PAGE_PRESENT) #define pte_user(x) (pte_val(x) & _PAGE_USER) -#define pte_clear(mm, addr, xp) set_pte(xp, __pte(0)) +#define pte_clear(mm, addr, xp) set_pte_at((mm), (addr), (xp), __pte(0)) #define pmd_flag(x) (pmd_val(x) & PxD_FLAG_MASK) #define pmd_address(x) ((unsigned long)(pmd_val(x) &~ PxD_FLAG_MASK) << PxD_VALUE_SHIFT) @@ -398,6 +398,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, } } #define set_ptes set_ptes +#define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1) /* Used for deferring calls to flush_dcache_page() */ @@ -462,7 +463,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned if (!pte_young(pte)) { return 0; } - set_pte(ptep, pte_mkold(pte)); + set_pte_at(vma->vm_mm, addr, ptep, pte_mkold(pte)); return 1; } @@ -472,7 +473,7 @@ pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *pt struct mm_struct; static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - set_pte(ptep, pte_wrprotect(*ptep)); + set_pte_at(mm, addr, ptep, pte_wrprotect(*ptep)); } #define pte_same(A,B) (pte_val(A) == pte_val(B)) diff --git a/arch/parisc/include/asm/special_insns.h b/arch/parisc/include/asm/special_insns.h index 51f40eaf77806..1013eeba31e5b 100644 --- a/arch/parisc/include/asm/special_insns.h +++ b/arch/parisc/include/asm/special_insns.h @@ -32,6 +32,34 @@ pa; \ }) +/** + * prober_user() - Probe user read access + * @sr: Space regster. + * @va: Virtual address. + * + * Return: Non-zero if address is accessible. + * + * Due to the way _PAGE_READ is handled in TLB entries, we need + * a special check to determine whether a user address is accessible. + * The ldb instruction does the initial access check. If it is + * successful, the probe instruction checks user access rights. + */ +#define prober_user(sr, va) ({ \ + unsigned long read_allowed; \ + __asm__ __volatile__( \ + "copy %%r0,%0\n" \ + "8:\tldb 0(%%sr%1,%2),%%r0\n" \ + "\tproberi (%%sr%1,%2),%3,%0\n" \ + "9:\n" \ + ASM_EXCEPTIONTABLE_ENTRY(8b, 9b, \ + "or %%r0,%%r0,%%r0") \ + : "=&r" (read_allowed) \ + : "i" (sr), "r" (va), "i" (PRIV_USER) \ + : "memory" \ + ); \ + read_allowed; \ +}) + #define CR_EIEM 15 /* External Interrupt Enable Mask */ #define CR_CR16 16 /* CR16 Interval Timer */ #define CR_EIRR 23 /* External Interrupt Request Register */ diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index 88d0ae5769dde..6c531d2c847eb 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -42,9 +42,24 @@ __gu_err; \ }) -#define __get_user(val, ptr) \ -({ \ - __get_user_internal(SR_USER, val, ptr); \ +#define __probe_user_internal(sr, error, ptr) \ +({ \ + __asm__("\tproberi (%%sr%1,%2),%3,%0\n" \ + "\tcmpiclr,= 1,%0,%0\n" \ + "\tldi %4,%0\n" \ + : "=r"(error) \ + : "i"(sr), "r"(ptr), "i"(PRIV_USER), \ + "i"(-EFAULT)); \ +}) + +#define __get_user(val, ptr) \ +({ \ + register long __gu_err; \ + \ + __gu_err = __get_user_internal(SR_USER, val, ptr); \ + if (likely(!__gu_err)) \ + __probe_user_internal(SR_USER, __gu_err, ptr); \ + __gu_err; \ }) #define __get_user_asm(sr, val, ldx, ptr) \ diff --git a/arch/parisc/include/uapi/asm/ioctls.h b/arch/parisc/include/uapi/asm/ioctls.h index 82d1148c6379a..74b4027a4e808 100644 --- a/arch/parisc/include/uapi/asm/ioctls.h +++ b/arch/parisc/include/uapi/asm/ioctls.h @@ -10,10 +10,10 @@ #define TCSETS _IOW('T', 17, struct termios) /* TCSETATTR */ #define TCSETSW _IOW('T', 18, struct termios) /* TCSETATTRD */ #define TCSETSF _IOW('T', 19, struct termios) /* TCSETATTRF */ -#define TCGETA _IOR('T', 1, struct termio) -#define TCSETA _IOW('T', 2, struct termio) -#define TCSETAW _IOW('T', 3, struct termio) -#define TCSETAF _IOW('T', 4, struct termio) +#define TCGETA 0x40125401 +#define TCSETA 0x80125402 +#define TCSETAW 0x80125403 +#define TCSETAF 0x80125404 #define TCSBRK _IO('T', 5) #define TCXONC _IO('T', 6) #define TCFLSH _IO('T', 7) diff --git a/arch/parisc/kernel/asm-offsets.c b/arch/parisc/kernel/asm-offsets.c index 757816a7bd4b2..9abfe65492c65 100644 --- a/arch/parisc/kernel/asm-offsets.c +++ b/arch/parisc/kernel/asm-offsets.c @@ -13,6 +13,7 @@ * Copyright (C) 2002 Randolph Chung * Copyright (C) 2003 James Bottomley */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index f7953b0391cf6..1898956a70f25 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -425,7 +425,7 @@ static inline pte_t *get_ptep(struct mm_struct *mm, unsigned long addr) return ptep; } -static inline bool pte_needs_flush(pte_t pte) +static inline bool pte_needs_cache_flush(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_NO_CACHE)) == (_PAGE_PRESENT | _PAGE_ACCESSED); @@ -630,7 +630,7 @@ static void flush_cache_page_if_present(struct vm_area_struct *vma, ptep = get_ptep(vma->vm_mm, vmaddr); if (ptep) { pte = ptep_get(ptep); - needs_flush = pte_needs_flush(pte); + needs_flush = pte_needs_cache_flush(pte); pte_unmap(ptep); } if (needs_flush) @@ -841,7 +841,7 @@ void flush_cache_vmap(unsigned long start, unsigned long end) } vm = find_vm_area((void *)start); - if (WARN_ON_ONCE(!vm)) { + if (!vm) { flush_cache_all(); return; } diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index ea57bcc21dc5f..f4bf61a34701e 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -499,6 +499,12 @@ * this happens is quite subtle, read below */ .macro make_insert_tlb spc,pte,prot,tmp space_to_prot \spc \prot /* create prot id from space */ + +#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT + /* need to drop DMB bit, as it's used as SPECIAL flag */ + depi 0,_PAGE_SPECIAL_BIT,1,\pte +#endif + /* The following is the real subtlety. This is depositing * T <-> _PAGE_REFTRAP * D <-> _PAGE_DIRTY @@ -511,17 +517,18 @@ * Finally, _PAGE_READ goes in the top bit of PL1 (so we * trigger an access rights trap in user space if the user * tries to read an unreadable page */ -#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT - /* need to drop DMB bit, as it's used as SPECIAL flag */ - depi 0,_PAGE_SPECIAL_BIT,1,\pte -#endif depd \pte,8,7,\prot /* PAGE_USER indicates the page can be read with user privileges, * so deposit X1|11 to PL1|PL2 (remember the upper bit of PL1 - * contains _PAGE_READ) */ + * contains _PAGE_READ). While the kernel can't directly write + * user pages which have _PAGE_WRITE zero, it can read pages + * which have _PAGE_READ zero (PL <= PL1). Thus, the kernel + * exception fault handler doesn't trigger when reading pages + * that aren't user read accessible */ extrd,u,*= \pte,_PAGE_USER_BIT+32,1,%r0 depdi 7,11,3,\prot + /* If we're a gateway page, drop PL2 back to zero for promotion * to kernel privilege (so we can execute the page as kernel). * Any privilege promotion page always denys read and write */ diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S index 0fa81bf1466b1..f58c4bccfbce0 100644 --- a/arch/parisc/kernel/syscall.S +++ b/arch/parisc/kernel/syscall.S @@ -613,6 +613,9 @@ lws_compare_and_swap32: lws_compare_and_swap: /* Trigger memory reference interruptions without writing to memory */ 1: ldw 0(%r26), %r28 + proberi (%r26), PRIV_USER, %r28 + comb,=,n %r28, %r0, lws_fault /* backwards, likely not taken */ + nop 2: stbys,e %r0, 0(%r26) /* Calculate 8-bit hash index from virtual address */ @@ -767,6 +770,9 @@ cas2_lock_start: copy %r26, %r28 depi_safe 0, 31, 2, %r28 10: ldw 0(%r28), %r1 + proberi (%r28), PRIV_USER, %r1 + comb,=,n %r1, %r0, lws_fault /* backwards, likely not taken */ + nop 11: stbys,e %r0, 0(%r28) /* Calculate 8-bit hash index from virtual address */ @@ -951,41 +957,47 @@ atomic_xchg_begin: /* 8-bit exchange */ 1: ldb 0(%r24), %r20 + proberi (%r24), PRIV_USER, %r20 + comb,=,n %r20, %r0, lws_fault /* backwards, likely not taken */ + nop copy %r23, %r20 depi_safe 0, 31, 2, %r20 b atomic_xchg_start 2: stbys,e %r0, 0(%r20) - nop - nop - nop /* 16-bit exchange */ 3: ldh 0(%r24), %r20 + proberi (%r24), PRIV_USER, %r20 + comb,=,n %r20, %r0, lws_fault /* backwards, likely not taken */ + nop copy %r23, %r20 depi_safe 0, 31, 2, %r20 b atomic_xchg_start 4: stbys,e %r0, 0(%r20) - nop - nop - nop /* 32-bit exchange */ 5: ldw 0(%r24), %r20 + proberi (%r24), PRIV_USER, %r20 + comb,=,n %r20, %r0, lws_fault /* backwards, likely not taken */ + nop b atomic_xchg_start 6: stbys,e %r0, 0(%r23) nop nop - nop - nop - nop /* 64-bit exchange */ #ifdef CONFIG_64BIT 7: ldd 0(%r24), %r20 + proberi (%r24), PRIV_USER, %r20 + comb,=,n %r20, %r0, lws_fault /* backwards, likely not taken */ + nop 8: stdby,e %r0, 0(%r23) #else 7: ldw 0(%r24), %r20 8: ldw 4(%r24), %r20 + proberi (%r24), PRIV_USER, %r20 + comb,=,n %r20, %r0, lws_fault /* backwards, likely not taken */ + nop copy %r23, %r20 depi_safe 0, 31, 2, %r20 9: stbys,e %r0, 0(%r20) diff --git a/arch/parisc/kernel/unaligned.c b/arch/parisc/kernel/unaligned.c index a8e75e5b884a7..e673c71e16d97 100644 --- a/arch/parisc/kernel/unaligned.c +++ b/arch/parisc/kernel/unaligned.c @@ -24,7 +24,7 @@ #define DPRINTF(fmt, args...) #endif -#define RFMT "%#08lx" +#define RFMT "0x%08lx" /* 1111 1100 0000 0000 0001 0011 1100 0000 */ #define OPCODE1(a,b,c) ((a)<<26|(b)<<12|(c)<<6) diff --git a/arch/parisc/lib/memcpy.c b/arch/parisc/lib/memcpy.c index 5fc0c852c84c8..03165c82dfdbd 100644 --- a/arch/parisc/lib/memcpy.c +++ b/arch/parisc/lib/memcpy.c @@ -12,6 +12,7 @@ #include #include #include +#include #define get_user_space() mfsp(SR_USER) #define get_kernel_space() SR_KERNEL @@ -32,9 +33,24 @@ EXPORT_SYMBOL(raw_copy_to_user); unsigned long raw_copy_from_user(void *dst, const void __user *src, unsigned long len) { + unsigned long start = (unsigned long) src; + unsigned long end = start + len; + unsigned long newlen = len; + mtsp(get_user_space(), SR_TEMP1); mtsp(get_kernel_space(), SR_TEMP2); - return pa_memcpy(dst, (void __force *)src, len); + + /* Check region is user accessible */ + while (start < end) { + if (!prober_user(SR_TEMP1, start)) { + newlen = (start - (unsigned long) src); + break; + } + start += PAGE_SIZE; + /* align to page boundry which may have different permission */ + start = PAGE_ALIGN_DOWN(start); + } + return len - newlen + pa_memcpy(dst, (void __force *)src, newlen); } EXPORT_SYMBOL(raw_copy_from_user); diff --git a/arch/parisc/math-emu/driver.c b/arch/parisc/math-emu/driver.c index 6ce427b58836c..ecd27b48d61f9 100644 --- a/arch/parisc/math-emu/driver.c +++ b/arch/parisc/math-emu/driver.c @@ -103,9 +103,19 @@ handle_fpe(struct pt_regs *regs) memcpy(regs->fr, frcopy, sizeof regs->fr); if (signalcode != 0) { - force_sig_fault(signalcode >> 24, signalcode & 0xffffff, - (void __user *) regs->iaoq[0]); - return -1; + int sig = signalcode >> 24; + + if (sig == SIGFPE) { + /* + * Clear floating point trap bit to avoid trapping + * again on the first floating-point instruction in + * the userspace signal handler. + */ + regs->fr[0] &= ~(1ULL << 38); + } + force_sig_fault(sig, signalcode & 0xffffff, + (void __user *) regs->iaoq[0]); + return -1; } return signalcode ? -1 : 0; diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index c39de84e98b05..f1785640b049b 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -363,6 +363,10 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, mmap_read_unlock(mm); bad_area_nosemaphore: + if (!user_mode(regs) && fixup_exception(regs)) { + return; + } + if (user_mode(regs)) { int signo, si_code; diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index 968aee2025b81..99c39c9b2a71f 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -72,6 +72,7 @@ BOOTCPPFLAGS := -nostdinc $(LINUXINCLUDE) BOOTCPPFLAGS += -isystem $(shell $(BOOTCC) -print-file-name=include) BOOTCFLAGS := $(BOOTTARGETFLAGS) \ + -std=gnu11 \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -O2 \ -msoft-float -mno-altivec -mno-vsx \ diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper index 352d7de24018f..ddb02cf0caaf5 100755 --- a/arch/powerpc/boot/wrapper +++ b/arch/powerpc/boot/wrapper @@ -234,10 +234,8 @@ fi # suppress some warnings in recent ld versions nowarn="-z noexecstack" -if ! ld_is_lld; then - if [ "$LD_VERSION" -ge "$(echo 2.39 | ld_version)" ]; then - nowarn="$nowarn --no-warn-rwx-segments" - fi +if "${CROSS}ld" -v --no-warn-rwx-segments >/dev/null 2>&1; then + nowarn="$nowarn --no-warn-rwx-segments" fi platformo=$object/"$platform".o diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index eaf3273372a98..80989e3f6780f 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -254,7 +254,6 @@ CONFIG_NET_SCH_DSMARK=m CONFIG_NET_SCH_NETEM=m CONFIG_NET_SCH_INGRESS=m CONFIG_NET_CLS_BASIC=m -CONFIG_NET_CLS_TCINDEX=m CONFIG_NET_CLS_ROUTE4=m CONFIG_NET_CLS_FW=m CONFIG_NET_CLS_U32=m diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index fccf742c55c2c..5c1619f251885 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -129,6 +129,7 @@ config CRYPTO_CHACHA20_P10 config CRYPTO_POLY1305_P10 tristate "Hash functions: Poly1305 (P10 or later)" depends on PPC64 && CPU_LITTLE_ENDIAN && VSX + depends on BROKEN # Needs to be fixed to work in softirq context select CRYPTO_HASH select CRYPTO_LIB_POLY1305_GENERIC help diff --git a/arch/powerpc/include/asm/floppy.h b/arch/powerpc/include/asm/floppy.h index f8ce178b43b78..34abf8bea2ccd 100644 --- a/arch/powerpc/include/asm/floppy.h +++ b/arch/powerpc/include/asm/floppy.h @@ -144,9 +144,12 @@ static int hard_dma_setup(char *addr, unsigned long size, int mode, int io) bus_addr = 0; } - if (!bus_addr) /* need to map it */ + if (!bus_addr) { /* need to map it */ bus_addr = dma_map_single(&isa_bridge_pcidev->dev, addr, size, dir); + if (dma_mapping_error(&isa_bridge_pcidev->dev, bus_addr)) + return -ENOMEM; + } /* remember this one as prev */ prev_addr = addr; diff --git a/arch/powerpc/include/asm/mmzone.h b/arch/powerpc/include/asm/mmzone.h index da827d2d08666..f2c4457c94c39 100644 --- a/arch/powerpc/include/asm/mmzone.h +++ b/arch/powerpc/include/asm/mmzone.h @@ -35,6 +35,7 @@ extern cpumask_var_t node_to_cpumask_map[]; #ifdef CONFIG_MEMORY_HOTPLUG extern unsigned long max_pfn; u64 memory_hotplug_max(void); +u64 hot_add_drconf_memory_max(void); #else #define memory_hotplug_max() memblock_end_of_DRAM() #endif diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index db2fe941e4c8b..bc726db5fdde7 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -20,18 +20,6 @@ struct mm_struct; #include #endif /* !CONFIG_PPC_BOOK3S */ -/* - * Protection used for kernel text. We want the debuggers to be able to - * set breakpoints anywhere, so don't write protect the kernel text - * on platforms where such control is possible. - */ -#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \ - defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE) -#define PAGE_KERNEL_TEXT PAGE_KERNEL_X -#else -#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX -#endif - /* Make modules code happy. We don't set RO yet */ #define PAGE_KERNEL_EXEC PAGE_KERNEL_X diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index e7792aa135105..fa79265328665 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -183,7 +183,7 @@ /* * Used to name C functions called from asm */ -#ifdef CONFIG_PPC_KERNEL_PCREL +#if defined(__powerpc64__) && defined(CONFIG_PPC_KERNEL_PCREL) #define CFUNC(name) name@notoc #else #define CFUNC(name) name diff --git a/arch/powerpc/include/uapi/asm/ioctls.h b/arch/powerpc/include/uapi/asm/ioctls.h index 2c145da3b774a..b5211e413829a 100644 --- a/arch/powerpc/include/uapi/asm/ioctls.h +++ b/arch/powerpc/include/uapi/asm/ioctls.h @@ -23,10 +23,10 @@ #define TCSETSW _IOW('t', 21, struct termios) #define TCSETSF _IOW('t', 22, struct termios) -#define TCGETA _IOR('t', 23, struct termio) -#define TCSETA _IOW('t', 24, struct termio) -#define TCSETAW _IOW('t', 25, struct termio) -#define TCSETAF _IOW('t', 28, struct termio) +#define TCGETA 0x40147417 /* _IOR('t', 23, struct termio) */ +#define TCSETA 0x80147418 /* _IOW('t', 24, struct termio) */ +#define TCSETAW 0x80147419 /* _IOW('t', 25, struct termio) */ +#define TCSETAF 0x8014741c /* _IOW('t', 28, struct termio) */ #define TCSBRK _IO('t', 29) #define TCXONC _IO('t', 30) diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 2919433be3557..131c859b24679 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -165,9 +165,7 @@ endif obj64-$(CONFIG_PPC_TRANSACTIONAL_MEM) += tm.o -ifneq ($(CONFIG_XMON)$(CONFIG_KEXEC_CORE)(CONFIG_PPC_BOOK3S),) obj-y += ppc_save_regs.o -endif obj-$(CONFIG_EPAPR_PARAVIRT) += epapr_paravirt.o epapr_hcalls.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvm_emul.o diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 2affd30468bc4..e2cee2f2ededd 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -8,6 +8,7 @@ * compile this file to assembler, and then extract the * #defines from the assembly-language output. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index ab316e155ea9f..82626363a3090 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1130,6 +1130,7 @@ int eeh_unfreeze_pe(struct eeh_pe *pe) return ret; } +EXPORT_SYMBOL_GPL(eeh_unfreeze_pe); static struct pci_device_id eeh_reset_ids[] = { @@ -1516,6 +1517,8 @@ int eeh_pe_configure(struct eeh_pe *pe) /* Invalid PE ? */ if (!pe) return -ENODEV; + else + ret = eeh_ops->configure_bridge(pe); return ret; } diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 438568a472d03..cc8bedf410ea7 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -257,13 +257,12 @@ static void eeh_pe_report_edev(struct eeh_dev *edev, eeh_report_fn fn, struct pci_driver *driver; enum pci_ers_result new_result; - pci_lock_rescan_remove(); pdev = edev->pdev; if (pdev) get_device(&pdev->dev); - pci_unlock_rescan_remove(); if (!pdev) { eeh_edev_info(edev, "no device"); + *result = PCI_ERS_RESULT_DISCONNECT; return; } device_lock(&pdev->dev); @@ -304,8 +303,9 @@ static void eeh_pe_report(const char *name, struct eeh_pe *root, struct eeh_dev *edev, *tmp; pr_info("EEH: Beginning: '%s'\n", name); - eeh_for_each_pe(root, pe) eeh_pe_for_each_dev(pe, edev, tmp) - eeh_pe_report_edev(edev, fn, result); + eeh_for_each_pe(root, pe) + eeh_pe_for_each_dev(pe, edev, tmp) + eeh_pe_report_edev(edev, fn, result); if (result) pr_info("EEH: Finished:'%s' with aggregate recovery state:'%s'\n", name, pci_ers_result_name(*result)); @@ -334,7 +334,7 @@ static enum pci_ers_result eeh_report_error(struct eeh_dev *edev, rc = driver->err_handler->error_detected(pdev, pci_channel_io_frozen); edev->in_error = true; - pci_uevent_ers(pdev, PCI_ERS_RESULT_NONE); + pci_uevent_ers(pdev, rc); return rc; } @@ -383,6 +383,8 @@ static void eeh_dev_restore_state(struct eeh_dev *edev, void *userdata) if (!edev) return; + pci_lock_rescan_remove(); + /* * The content in the config space isn't saved because * the blocked config space on some adapters. We have @@ -393,14 +395,19 @@ static void eeh_dev_restore_state(struct eeh_dev *edev, void *userdata) if (list_is_last(&edev->entry, &edev->pe->edevs)) eeh_pe_restore_bars(edev->pe); + pci_unlock_rescan_remove(); return; } pdev = eeh_dev_to_pci_dev(edev); - if (!pdev) + if (!pdev) { + pci_unlock_rescan_remove(); return; + } pci_restore_state(pdev); + + pci_unlock_rescan_remove(); } /** @@ -647,9 +654,7 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, if (any_passed || driver_eeh_aware || (pe->type & EEH_PE_VF)) { eeh_pe_dev_traverse(pe, eeh_rmv_device, rmv_data); } else { - pci_lock_rescan_remove(); pci_hp_remove_devices(bus); - pci_unlock_rescan_remove(); } /* @@ -665,8 +670,6 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, if (rc) return rc; - pci_lock_rescan_remove(); - /* Restore PE */ eeh_ops->configure_bridge(pe); eeh_pe_restore_bars(pe); @@ -674,7 +677,6 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, /* Clear frozen state */ rc = eeh_clear_pe_frozen_state(pe, false); if (rc) { - pci_unlock_rescan_remove(); return rc; } @@ -709,7 +711,6 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, pe->tstamp = tstamp; pe->freeze_count = cnt; - pci_unlock_rescan_remove(); return 0; } @@ -843,10 +844,13 @@ void eeh_handle_normal_event(struct eeh_pe *pe) {LIST_HEAD_INIT(rmv_data.removed_vf_list), 0}; int devices = 0; + pci_lock_rescan_remove(); + bus = eeh_pe_bus_get(pe); if (!bus) { pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n", __func__, pe->phb->global_number, pe->addr); + pci_unlock_rescan_remove(); return; } @@ -1085,10 +1089,15 @@ void eeh_handle_normal_event(struct eeh_pe *pe) eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - pci_lock_rescan_remove(); - pci_hp_remove_devices(bus); - pci_unlock_rescan_remove(); + bus = eeh_pe_bus_get(pe); + if (bus) + pci_hp_remove_devices(bus); + else + pr_err("%s: PCI bus for PHB#%x-PE#%x disappeared\n", + __func__, pe->phb->global_number, pe->addr); + /* The passed PE should no longer be used */ + pci_unlock_rescan_remove(); return; } @@ -1105,6 +1114,8 @@ void eeh_handle_normal_event(struct eeh_pe *pe) eeh_clear_slot_attention(edev->pdev); eeh_pe_state_clear(pe, EEH_PE_RECOVERING, true); + + pci_unlock_rescan_remove(); } /** @@ -1123,6 +1134,7 @@ void eeh_handle_special_event(void) unsigned long flags; int rc; + pci_lock_rescan_remove(); do { rc = eeh_ops->next_error(&pe); @@ -1162,10 +1174,12 @@ void eeh_handle_special_event(void) break; case EEH_NEXT_ERR_NONE: + pci_unlock_rescan_remove(); return; default: pr_warn("%s: Invalid value %d from next_error()\n", __func__, rc); + pci_unlock_rescan_remove(); return; } @@ -1177,7 +1191,9 @@ void eeh_handle_special_event(void) if (rc == EEH_NEXT_ERR_FROZEN_PE || rc == EEH_NEXT_ERR_FENCED_PHB) { eeh_pe_state_mark(pe, EEH_PE_RECOVERING); + pci_unlock_rescan_remove(); eeh_handle_normal_event(pe); + pci_lock_rescan_remove(); } else { eeh_for_each_pe(pe, tmp_pe) eeh_pe_for_each_dev(tmp_pe, edev, tmp_edev) @@ -1190,7 +1206,6 @@ void eeh_handle_special_event(void) eeh_report_failure, NULL); eeh_set_channel_state(pe, pci_channel_io_perm_failure); - pci_lock_rescan_remove(); list_for_each_entry(hose, &hose_list, list_node) { phb_pe = eeh_phb_pe_get(hose); if (!phb_pe || @@ -1209,7 +1224,6 @@ void eeh_handle_special_event(void) } pci_hp_remove_devices(bus); } - pci_unlock_rescan_remove(); } /* @@ -1219,4 +1233,6 @@ void eeh_handle_special_event(void) if (rc == EEH_NEXT_ERR_DEAD_IOC) break; } while (rc != EEH_NEXT_ERR_NONE); + + pci_unlock_rescan_remove(); } diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index 7d1b50599dd6c..08095aeba5c98 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -671,10 +671,12 @@ static void eeh_bridge_check_link(struct eeh_dev *edev) eeh_ops->write_config(edev, cap + PCI_EXP_LNKCTL, 2, val); /* Check link */ - if (!edev->pdev->link_active_reporting) { - eeh_edev_dbg(edev, "No link reporting capability\n"); - msleep(1000); - return; + if (edev->pdev) { + if (!edev->pdev->link_active_reporting) { + eeh_edev_dbg(edev, "No link reporting capability\n"); + msleep(1000); + return; + } } /* Wait the link is up until timeout (5s) */ diff --git a/arch/powerpc/kernel/kvm.c b/arch/powerpc/kernel/kvm.c index 5b3c093611baf..7209d00a9c257 100644 --- a/arch/powerpc/kernel/kvm.c +++ b/arch/powerpc/kernel/kvm.c @@ -632,19 +632,19 @@ static void __init kvm_check_ins(u32 *inst, u32 features) #endif } - switch (inst_no_rt & ~KVM_MASK_RB) { #ifdef CONFIG_PPC_BOOK3S_32 + switch (inst_no_rt & ~KVM_MASK_RB) { case KVM_INST_MTSRIN: if (features & KVM_MAGIC_FEAT_SR) { u32 inst_rb = _inst & KVM_MASK_RB; kvm_patch_ins_mtsrin(inst, inst_rt, inst_rb); } break; -#endif } +#endif - switch (_inst) { #ifdef CONFIG_BOOKE + switch (_inst) { case KVM_INST_WRTEEI_0: kvm_patch_ins_wrteei_0(inst); break; @@ -652,8 +652,8 @@ static void __init kvm_check_ins(u32 *inst, u32 features) case KVM_INST_WRTEEI_1: kvm_patch_ins_wrtee(inst, 0, 1); break; -#endif } +#endif } extern u32 kvm_template_start[]; diff --git a/arch/powerpc/kernel/pci-hotplug.c b/arch/powerpc/kernel/pci-hotplug.c index 0fe251c6ac2ce..ac70e85b0df85 100644 --- a/arch/powerpc/kernel/pci-hotplug.c +++ b/arch/powerpc/kernel/pci-hotplug.c @@ -111,6 +111,9 @@ void pci_hp_add_devices(struct pci_bus *bus) struct pci_controller *phb; struct device_node *dn = pci_bus_to_OF_node(bus); + if (!dn) + return; + phb = pci_bus_to_host(bus); mode = PCI_PROBE_NORMAL; diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index a6090896f7497..ac669e58e2023 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -2974,11 +2974,11 @@ static void __init fixup_device_tree_pmac(void) char type[8]; phandle node; - // Some pmacs are missing #size-cells on escc nodes + // Some pmacs are missing #size-cells on escc or i2s nodes for (node = 0; prom_next_node(&node); ) { type[0] = '\0'; prom_getprop(node, "device_type", type, sizeof(type)); - if (prom_strcmp(type, "escc")) + if (prom_strcmp(type, "escc") && prom_strcmp(type, "i2s")) continue; if (prom_getproplen(node, "#size-cells") != PROM_ERROR) diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index d5defff8472da..47a9533a4dc1d 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -50,7 +50,7 @@ ldflags-$(CONFIG_LD_ORPHAN_WARN) += -Wl,--orphan-handling=$(CONFIG_LD_ORPHAN_WAR ldflags-y += $(filter-out $(CC_AUTO_VAR_INIT_ZERO_ENABLER) $(CC_FLAGS_FTRACE) -Wa$(comma)%, $(KBUILD_CFLAGS)) CC32FLAGS := -m32 -CC32FLAGSREMOVE := -mcmodel=medium -mabi=elfv1 -mabi=elfv2 -mcall-aixdesc +CC32FLAGSREMOVE := -mcmodel=medium -mabi=elfv1 -mabi=elfv2 -mcall-aixdesc -mpcrel ifdef CONFIG_CC_IS_CLANG # This flag is supported by clang for 64-bit but not 32-bit so it will cause # an unused command line flag warning for this file. diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c index ef5c2d25ec397..61552bbb1ea8a 100644 --- a/arch/powerpc/kexec/crash.c +++ b/arch/powerpc/kexec/crash.c @@ -356,7 +356,10 @@ void default_machine_crash_shutdown(struct pt_regs *regs) if (TRAP(regs) == INTERRUPT_SYSTEM_RESET) is_via_system_reset = 1; - crash_smp_send_stop(); + if (IS_ENABLED(CONFIG_SMP)) + crash_smp_send_stop(); + else + crash_kexec_prepare(); crash_save_cpu(regs, crashing_cpu); diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index 850783cfa9c73..1b1848761a000 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -204,7 +204,7 @@ void mmu_mark_initmem_nx(void) for (i = 0; i < nb - 1 && base < top;) { size = bat_block_size(base, top); - setibat(i++, PAGE_OFFSET + base, base, size, PAGE_KERNEL_TEXT); + setibat(i++, PAGE_OFFSET + base, base, size, PAGE_KERNEL_X); base += size; } if (base < top) { @@ -215,7 +215,7 @@ void mmu_mark_initmem_nx(void) pr_warn("Some RW data is getting mapped X. " "Adjust CONFIG_DATA_SHIFT to avoid that.\n"); } - setibat(i++, PAGE_OFFSET + base, base, size, PAGE_KERNEL_TEXT); + setibat(i++, PAGE_OFFSET + base, base, size, PAGE_KERNEL_X); base += size; } for (; i < nb; i++) diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index c6a4ac766b2bf..aff3b37e32d64 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -912,7 +912,7 @@ int __meminit radix__vmemmap_create_mapping(unsigned long start, return 0; } - +#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { if (radix_enabled()) @@ -920,6 +920,7 @@ bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) return false; } +#endif int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node, unsigned long addr, unsigned long next) @@ -1056,6 +1057,19 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in pmd_t *pmd; pte_t *pte; + /* + * Make sure we align the start vmemmap addr so that we calculate + * the correct start_pfn in altmap boundary check to decided whether + * we should use altmap or RAM based backing memory allocation. Also + * the address need to be aligned for set_pte operation. + + * If the start addr is already PMD_SIZE aligned we will try to use + * a pmd mapping. We don't want to be too aggressive here beacause + * that will cause more allocations in RAM. So only if the namespace + * vmemmap start addr is PMD_SIZE aligned we will use PMD mapping. + */ + + start = ALIGN_DOWN(start, PAGE_SIZE); for (addr = start; addr < end; addr = next) { next = pmd_addr_end(addr, end); @@ -1081,8 +1095,8 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in * in altmap block allocation failures, in which case * we fallback to RAM for vmemmap allocation. */ - if (altmap && (!IS_ALIGNED(addr, PMD_SIZE) || - altmap_cross_boundary(altmap, addr, PMD_SIZE))) { + if (!IS_ALIGNED(addr, PMD_SIZE) || (altmap && + altmap_cross_boundary(altmap, addr, PMD_SIZE))) { /* * make sure we don't create altmap mappings * covering things outside the device. diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index f6c4ace3b2219..65a9df0b9e5a0 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -1342,7 +1342,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr) return nid; } -static u64 hot_add_drconf_memory_max(void) +u64 hot_add_drconf_memory_max(void) { struct device_node *memory = NULL; struct device_node *dn = NULL; diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index 5c02fd08d61ef..69fac96c2dcd1 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -109,7 +109,7 @@ static void __init __mapin_ram_chunk(unsigned long offset, unsigned long top) p = memstart_addr + s; for (; s < top; s += PAGE_SIZE) { ktext = core_kernel_text(v); - map_kernel_page(v, p, ktext ? PAGE_KERNEL_TEXT : PAGE_KERNEL); + map_kernel_page(v, p, ktext ? PAGE_KERNEL_X : PAGE_KERNEL); v += PAGE_SIZE; p += PAGE_SIZE; } diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 10b946e9c6e75..4bb84dc4393fc 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -2229,6 +2229,10 @@ static struct pmu power_pmu = { #define PERF_SAMPLE_ADDR_TYPE (PERF_SAMPLE_ADDR | \ PERF_SAMPLE_PHYS_ADDR | \ PERF_SAMPLE_DATA_PAGE_SIZE) + +#define SIER_TYPE_SHIFT 15 +#define SIER_TYPE_MASK (0x7ull << SIER_TYPE_SHIFT) + /* * A counter has overflowed; update its count and record * things if requested. Note that interrupts are hard-disabled @@ -2297,6 +2301,22 @@ static void record_and_restart(struct perf_event *event, unsigned long val, is_kernel_addr(mfspr(SPRN_SIAR))) record = 0; + /* + * SIER[46-48] presents instruction type of the sampled instruction. + * In ISA v3.0 and before values "0" and "7" are considered reserved. + * In ISA v3.1, value "7" has been used to indicate "larx/stcx". + * Drop the sample if "type" has reserved values for this field with a + * ISA version check. + */ + if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC && + ppmu->get_mem_data_src) { + val = (regs->dar & SIER_TYPE_MASK) >> SIER_TYPE_SHIFT; + if (val == 0 || (val == 7 && !cpu_has_feature(CPU_FTR_ARCH_31))) { + record = 0; + atomic64_inc(&event->lost_samples); + } + } + /* * Finally record data if requested. */ diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index 56301b2bc8ae8..031a2b63c171d 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -321,8 +321,10 @@ void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags, sier = mfspr(SPRN_SIER); val = (sier & ISA207_SIER_TYPE_MASK) >> ISA207_SIER_TYPE_SHIFT; - if (val != 1 && val != 2 && !(val == 7 && cpu_has_feature(CPU_FTR_ARCH_31))) + if (val != 1 && val != 2 && !(val == 7 && cpu_has_feature(CPU_FTR_ARCH_31))) { + dsrc->val = 0; return; + } idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT; sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT; diff --git a/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c b/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c index 4a25b6b486158..f1e353fc6594e 100644 --- a/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c +++ b/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c @@ -240,10 +240,8 @@ static int mpc512x_lpbfifo_kick(void) dma_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; /* Make DMA channel work with LPB FIFO data register */ - if (dma_dev->device_config(lpbfifo.chan, &dma_conf)) { - ret = -EINVAL; - goto err_dma_prep; - } + if (dma_dev->device_config(lpbfifo.chan, &dma_conf)) + return -EINVAL; sg_init_table(&sg, 1); diff --git a/arch/powerpc/platforms/book3s/vas-api.c b/arch/powerpc/platforms/book3s/vas-api.c index 0b6365d85d117..dc6f75d3ac6ef 100644 --- a/arch/powerpc/platforms/book3s/vas-api.c +++ b/arch/powerpc/platforms/book3s/vas-api.c @@ -521,6 +521,15 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) return -EINVAL; } + /* + * Map complete page to the paste address. So the user + * space should pass 0ULL to the offset parameter. + */ + if (vma->vm_pgoff) { + pr_debug("Page offset unsupported to map paste address\n"); + return -EINVAL; + } + /* Ensure instance has an open send window */ if (!txwin) { pr_err("No send window open?\n"); diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c index 877720c645151..35471b679638a 100644 --- a/arch/powerpc/platforms/powernv/memtrace.c +++ b/arch/powerpc/platforms/powernv/memtrace.c @@ -48,11 +48,15 @@ static ssize_t memtrace_read(struct file *filp, char __user *ubuf, static int memtrace_mmap(struct file *filp, struct vm_area_struct *vma) { struct memtrace_entry *ent = filp->private_data; + unsigned long ent_nrpages = ent->size >> PAGE_SHIFT; + unsigned long vma_nrpages = vma_pages(vma); - if (ent->size < vma->vm_end - vma->vm_start) + /* The requested page offset should be within object's page count */ + if (vma->vm_pgoff >= ent_nrpages) return -EINVAL; - if (vma->vm_pgoff << PAGE_SHIFT >= ent->size) + /* The requested mapping range should remain within the bounds */ + if (vma_nrpages > ent_nrpages - vma->vm_pgoff) return -EINVAL; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 28fac47700731..5337ad8a2e340 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1895,7 +1895,7 @@ static int pnv_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; out: - irq_domain_free_irqs_parent(domain, virq, i - 1); + irq_domain_free_irqs_parent(domain, virq, i); msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, nr_irqs); return ret; } diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index b1e6d275cda9e..bf02f94a973db 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -1183,17 +1183,13 @@ static LIST_HEAD(failed_ddw_pdn_list); static phys_addr_t ddw_memory_hotplug_max(void) { - resource_size_t max_addr = memory_hotplug_max(); - struct device_node *memory; + resource_size_t max_addr; - for_each_node_by_type(memory, "memory") { - struct resource res; - - if (of_address_to_resource(memory, 0, &res)) - continue; - - max_addr = max_t(resource_size_t, max_addr, res.end + 1); - } +#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG) + max_addr = hot_add_drconf_memory_max(); +#else + max_addr = memblock_end_of_DRAM(); +#endif return max_addr; } @@ -1471,7 +1467,7 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) window->direct = true; /* DDW maps the whole partition, so enable direct DMA mapping */ - ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT, + ret = walk_system_ram_range(0, ddw_memory_hotplug_max() >> PAGE_SHIFT, win64->value, tce_setrange_multi_pSeriesLP_walk); if (ret) { dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n", @@ -1658,11 +1654,17 @@ static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, struct memory_notify *arg = data; int ret = 0; + /* This notifier can get called when onlining persistent memory as well. + * TCEs are not pre-mapped for persistent memory. Persistent memory will + * always be above ddw_memory_hotplug_max() + */ + switch (action) { case MEM_GOING_ONLINE: spin_lock(&dma_win_list_lock); list_for_each_entry(window, &dma_win_list, list) { - if (window->direct) { + if (window->direct && (arg->start_pfn << PAGE_SHIFT) < + ddw_memory_hotplug_max()) { ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, arg->nr_pages, window->prop); } @@ -1674,7 +1676,8 @@ static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, case MEM_OFFLINE: spin_lock(&dma_win_list_lock); list_for_each_entry(window, &dma_win_list, list) { - if (window->direct) { + if (window->direct && (arg->start_pfn << PAGE_SHIFT) < + ddw_memory_hotplug_max()) { ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, arg->nr_pages, window->prop); } diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index 423ee1d5bd944..b53f817fc421f 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -519,7 +519,12 @@ static struct msi_domain_info pseries_msi_domain_info = { static void pseries_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) { - __pci_read_msi_msg(irq_data_get_msi_desc(data), msg); + struct pci_dev *dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(data)); + + if (dev->current_state == PCI_D0) + __pci_read_msi_msg(irq_data_get_msi_desc(data), msg); + else + get_cached_msi_msg(data->irq, msg); } static struct irq_chip pseries_msi_irq_chip = { @@ -582,7 +587,7 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq out: /* TODO: handle RTAS cleanup in ->msi_finish() ? */ - irq_domain_free_irqs_parent(domain, virq, i - 1); + irq_domain_free_irqs_parent(domain, virq, i); return ret; } diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index b5b84c6be01e1..da818b39a24cc 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -90,7 +90,7 @@ #endif .macro asm_per_cpu dst sym tmp - REG_L \tmp, TASK_TI_CPU_NUM(tp) + lw \tmp, TASK_TI_CPU_NUM(tp) slli \tmp, \tmp, PER_CPU_OFFSET_SHIFT la \dst, __per_cpu_offset add \dst, \dst, \tmp diff --git a/arch/riscv/include/asm/cfi.h b/arch/riscv/include/asm/cfi.h index 56bf9d69d5e38..4508aaa7a2fdb 100644 --- a/arch/riscv/include/asm/cfi.h +++ b/arch/riscv/include/asm/cfi.h @@ -7,11 +7,13 @@ * * Copyright (C) 2023 Google LLC */ +#include -#include +struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); +#define __bpfcall #else static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { diff --git a/arch/riscv/include/asm/compat.h b/arch/riscv/include/asm/compat.h index 2ac955b51148f..6b79287baecc0 100644 --- a/arch/riscv/include/asm/compat.h +++ b/arch/riscv/include/asm/compat.h @@ -9,7 +9,6 @@ */ #include #include -#include #include static inline int is_compat_task(void) diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 4d1f58848129e..dbb9d0d0f405e 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -26,12 +26,9 @@ * When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#ifdef CONFIG_64BIT #ifdef CONFIG_MMU +#ifdef CONFIG_64BIT #define PAGE_OFFSET kernel_map.page_offset -#else -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) -#endif /* * By default, CONFIG_PAGE_OFFSET value corresponds to SV57 address space so * define the PAGE_OFFSET value for SV48 and SV39. @@ -41,6 +38,9 @@ #else #define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) #endif /* CONFIG_64BIT */ +#else +#define PAGE_OFFSET ((unsigned long)phys_ram_base) +#endif /* CONFIG_MMU */ #ifndef __ASSEMBLY__ @@ -97,11 +97,7 @@ typedef struct page *pgtable_t; #define MIN_MEMBLOCK_ADDR 0 #endif -#ifdef CONFIG_MMU #define ARCH_PFN_OFFSET (PFN_DOWN((unsigned long)phys_ram_base)) -#else -#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) -#endif /* CONFIG_MMU */ struct kernel_mapping { unsigned long page_offset; diff --git a/arch/riscv/include/asm/patch.h b/arch/riscv/include/asm/patch.h index 9f5d6e14c4055..7228e266b9a1a 100644 --- a/arch/riscv/include/asm/patch.h +++ b/arch/riscv/include/asm/patch.h @@ -9,7 +9,7 @@ int patch_insn_write(void *addr, const void *insn, size_t len); int patch_text_nosync(void *addr, const void *insns, size_t len); int patch_text_set_nosync(void *addr, u8 c, size_t len); -int patch_text(void *addr, u32 *insns, int ninsns); +int patch_text(void *addr, u32 *insns, size_t len); extern int riscv_patch_in_stop_machine; diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 7915ba3c039d6..092fdb735f4f5 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -12,7 +12,7 @@ #include #ifndef CONFIG_MMU -#define KERNEL_LINK_ADDR PAGE_OFFSET +#define KERNEL_LINK_ADDR _AC(CONFIG_PAGE_OFFSET, UL) #define KERN_VIRT_SIZE (UL(-1)) #else @@ -620,6 +620,8 @@ static inline pgprot_t pgprot_writecombine(pgprot_t _prot) return __pgprot(prot); } +#define pgprot_dmacoherent pgprot_writecombine + /* * THP functions */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index e19f19cb763a8..f7d789b8e6a45 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -13,42 +13,19 @@ #include -#ifdef CONFIG_64BIT -#define DEFAULT_MAP_WINDOW (UL(1) << (MMAP_VA_BITS - 1)) -#define STACK_TOP_MAX TASK_SIZE - #define arch_get_mmap_end(addr, len, flags) \ ({ \ - unsigned long mmap_end; \ - typeof(addr) _addr = (addr); \ - if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \ - mmap_end = STACK_TOP_MAX; \ - else if ((_addr) >= VA_USER_SV57) \ - mmap_end = STACK_TOP_MAX; \ - else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \ - mmap_end = VA_USER_SV48; \ - else \ - mmap_end = VA_USER_SV39; \ - mmap_end; \ + STACK_TOP_MAX; \ }) #define arch_get_mmap_base(addr, base) \ ({ \ - unsigned long mmap_base; \ - typeof(addr) _addr = (addr); \ - typeof(base) _base = (base); \ - unsigned long rnd_gap = DEFAULT_MAP_WINDOW - (_base); \ - if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \ - mmap_base = (_base); \ - else if (((_addr) >= VA_USER_SV57) && (VA_BITS >= VA_BITS_SV57)) \ - mmap_base = VA_USER_SV57 - rnd_gap; \ - else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \ - mmap_base = VA_USER_SV48 - rnd_gap; \ - else \ - mmap_base = VA_USER_SV39 - rnd_gap; \ - mmap_base; \ + base; \ }) +#ifdef CONFIG_64BIT +#define DEFAULT_MAP_WINDOW (UL(1) << (MMAP_VA_BITS - 1)) +#define STACK_TOP_MAX TASK_SIZE #else #define DEFAULT_MAP_WINDOW TASK_SIZE #define STACK_TOP_MAX TASK_SIZE diff --git a/arch/riscv/include/asm/topology.h b/arch/riscv/include/asm/topology.h index e316ab3b77f34..61183688bdd54 100644 --- a/arch/riscv/include/asm/topology.h +++ b/arch/riscv/include/asm/topology.h @@ -9,6 +9,7 @@ #define arch_set_freq_scale topology_set_freq_scale #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant +#define arch_scale_freq_ref topology_get_freq_ref /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 6a992cba2f287..e4589457e6085 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -3,6 +3,7 @@ * Copyright (C) 2012 Regents of the University of California * Copyright (C) 2017 SiFive */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/riscv/kernel/cfi.c b/arch/riscv/kernel/cfi.c index 820158d7a2913..6ec9dbd7292ee 100644 --- a/arch/riscv/kernel/cfi.c +++ b/arch/riscv/kernel/cfi.c @@ -4,7 +4,7 @@ * * Copyright (C) 2023 Google LLC */ -#include +#include #include /* diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index 6b710ef9d9aef..2114903ee3ad4 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -61,6 +61,7 @@ void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) pr_notice("CPU%u: off\n", cpu); + clear_tasks_mm_cpumask(cpu); /* Verify from the firmware if the cpu is really stopped*/ if (cpu_ops[cpu]->cpu_is_stopped) ret = cpu_ops[cpu]->cpu_is_stopped(cpu); diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 88732abecd023..93e794d0e5231 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -61,10 +61,8 @@ int __init riscv_early_of_processor_hartid(struct device_node *node, unsigned lo return -ENODEV; } - if (!of_device_is_available(node)) { - pr_info("CPU with hartid=%lu is not available\n", *hart); + if (!of_device_is_available(node)) return -ENODEV; - } if (of_property_read_string(node, "riscv,isa-base", &isa)) goto old_interface; diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 19f97a6a2e598..572a042520a45 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -87,7 +87,6 @@ _save_context: call riscv_v_context_nesting_start #endif move a0, sp /* pt_regs */ - la ra, ret_from_exception /* * MSB of cause differentiates between @@ -96,7 +95,8 @@ _save_context: bge s4, zero, 1f /* Handle interrupts */ - tail do_irq + call do_irq + j ret_from_exception 1: /* Handle other exceptions */ slli t0, s4, RISCV_LGPTR @@ -104,11 +104,14 @@ _save_context: la t2, excp_vect_table_end add t0, t1, t0 /* Check if exception code lies within bounds */ - bgeu t0, t2, 1f - REG_L t0, 0(t0) - jr t0 -1: - tail do_trap_unknown + bgeu t0, t2, 3f + REG_L t1, 0(t0) +2: jalr t1 + j ret_from_exception +3: + + la t1, do_trap_unknown + j 2b SYM_CODE_END(handle_exception) ASM_NOKPROBE(handle_exception) @@ -180,6 +183,7 @@ SYM_CODE_START_NOALIGN(ret_from_exception) #else sret #endif +SYM_INNER_LABEL(ret_from_exception_end, SYM_L_GLOBAL) SYM_CODE_END(ret_from_exception) ASM_NOKPROBE(ret_from_exception) diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 78387d843aa56..aeda87240dbc1 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -19,7 +19,7 @@ struct patch_insn { void *addr; u32 *insns; - int ninsns; + size_t len; atomic_t cpu_count; }; @@ -234,14 +234,10 @@ NOKPROBE_SYMBOL(patch_text_nosync); static int patch_text_cb(void *data) { struct patch_insn *patch = data; - unsigned long len; - int i, ret = 0; + int ret = 0; if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { - for (i = 0; ret == 0 && i < patch->ninsns; i++) { - len = GET_INSN_LENGTH(patch->insns[i]); - ret = patch_insn_write(patch->addr + i * len, &patch->insns[i], len); - } + ret = patch_insn_write(patch->addr, patch->insns, patch->len); /* * Make sure the patching store is effective *before* we * increment the counter which releases all waiting CPUs @@ -262,13 +258,13 @@ static int patch_text_cb(void *data) } NOKPROBE_SYMBOL(patch_text_cb); -int patch_text(void *addr, u32 *insns, int ninsns) +int patch_text(void *addr, u32 *insns, size_t len) { int ret; struct patch_insn patch = { .addr = addr, .insns = insns, - .ninsns = ninsns, + .len = len, .cpu_count = ATOMIC_INIT(0), }; diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index 4fbc70e823f0f..8a6ea7d270188 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -23,13 +23,13 @@ post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *); static void __kprobes arch_prepare_ss_slot(struct kprobe *p) { + size_t len = GET_INSN_LENGTH(p->opcode); u32 insn = __BUG_INSN_32; - unsigned long offset = GET_INSN_LENGTH(p->opcode); - p->ainsn.api.restore = (unsigned long)p->addr + offset; + p->ainsn.api.restore = (unsigned long)p->addr + len; - patch_text_nosync(p->ainsn.api.insn, &p->opcode, 1); - patch_text_nosync((void *)p->ainsn.api.insn + offset, &insn, 1); + patch_text_nosync(p->ainsn.api.insn, &p->opcode, len); + patch_text_nosync((void *)p->ainsn.api.insn + len, &insn, GET_INSN_LENGTH(insn)); } static void __kprobes arch_prepare_simulate(struct kprobe *p) @@ -48,10 +48,15 @@ static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) post_kprobe_handler(p, kcb, regs); } -static bool __kprobes arch_check_kprobe(struct kprobe *p) +static bool __kprobes arch_check_kprobe(unsigned long addr) { - unsigned long tmp = (unsigned long)p->addr - p->offset; - unsigned long addr = (unsigned long)p->addr; + unsigned long tmp, offset; + + /* start iterating at the closest preceding symbol */ + if (!kallsyms_lookup_size_offset(addr, NULL, &offset)) + return false; + + tmp = addr - offset; while (tmp <= addr) { if (tmp == addr) @@ -70,7 +75,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) if ((unsigned long)insn & 0x1) return -EILSEQ; - if (!arch_check_kprobe(p)) + if (!arch_check_kprobe((unsigned long)p->addr)) return -EILSEQ; /* copy instruction */ @@ -116,16 +121,18 @@ void *alloc_insn_page(void) /* install breakpoint in text */ void __kprobes arch_arm_kprobe(struct kprobe *p) { - u32 insn = (p->opcode & __INSN_LENGTH_MASK) == __INSN_LENGTH_32 ? - __BUG_INSN_32 : __BUG_INSN_16; + size_t len = GET_INSN_LENGTH(p->opcode); + u32 insn = len == 4 ? __BUG_INSN_32 : __BUG_INSN_16; - patch_text(p->addr, &insn, 1); + patch_text(p->addr, &insn, len); } /* remove breakpoint from text */ void __kprobes arch_disarm_kprobe(struct kprobe *p) { - patch_text(p->addr, &p->opcode, 1); + size_t len = GET_INSN_LENGTH(p->opcode); + + patch_text(p->addr, &p->opcode, len); } void __kprobes arch_remove_kprobe(struct kprobe *p) diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index f598e0eb3b0a0..ee269b1c99a19 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -318,11 +318,14 @@ void __init setup_arch(char **cmdline_p) /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); + if (acpi_disabled) { #if IS_ENABLED(CONFIG_BUILTIN_DTB) - unflatten_and_copy_device_tree(); + unflatten_and_copy_device_tree(); #else - unflatten_device_tree(); + unflatten_device_tree(); #endif + } + misc_mem_init(); init_resources(); diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 10e311b2759d3..124ad8fe626fb 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -16,7 +16,24 @@ #ifdef CONFIG_FRAME_POINTER -extern asmlinkage void ret_from_exception(void); +/* + * This disables KASAN checking when reading a value from another task's stack, + * since the other task could be running on another CPU and could have poisoned + * the stack in the meantime. + */ +#define READ_ONCE_TASK_STACK(task, x) \ +({ \ + unsigned long val; \ + unsigned long addr = x; \ + if ((task) == current) \ + val = READ_ONCE(addr); \ + else \ + val = READ_ONCE_NOCHECK(addr); \ + val; \ +}) + +extern asmlinkage void handle_exception(void); +extern unsigned long ret_from_exception_end; static inline int fp_is_valid(unsigned long fp, unsigned long sp) { @@ -68,10 +85,12 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, fp = frame->ra; pc = regs->ra; } else { - fp = frame->fp; - pc = ftrace_graph_ret_addr(current, &graph_idx, frame->ra, + fp = READ_ONCE_TASK_STACK(task, frame->fp); + pc = READ_ONCE_TASK_STACK(task, frame->ra); + pc = ftrace_graph_ret_addr(current, &graph_idx, pc, &frame->ra); - if (pc == (unsigned long)ret_from_exception) { + if (pc >= (unsigned long)handle_exception && + pc < (unsigned long)&ret_from_exception_end) { if (unlikely(!__kernel_text_address(pc) || !fn(arg, pc))) break; diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index be43278109f4e..a71d33cd81d3d 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -103,9 +103,9 @@ void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu, struct kvm_vcpu *tmp; kvm_for_each_vcpu(i, tmp, vcpu->kvm) { - spin_lock(&vcpu->arch.mp_state_lock); + spin_lock(&tmp->arch.mp_state_lock); WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); - spin_unlock(&vcpu->arch.mp_state_lock); + spin_unlock(&tmp->arch.mp_state_lock); } kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP); diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 87ec68ed52d76..36f5386c60db7 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -103,7 +103,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_SENT); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: - if (cp->a2 == 0 && cp->a3 == 0) + if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL) kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask); else kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask, @@ -111,7 +111,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_SENT); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: - if (cp->a2 == 0 && cp->a3 == 0) + if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL) kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask, cp->a4); else @@ -127,9 +127,9 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: /* * Until nested virtualization is implemented, the - * SBI HFENCE calls should be treated as NOPs + * SBI HFENCE calls should return not supported + * hence fallthrough. */ - break; default: retdata->err_val = SBI_ERR_NOT_SUPPORTED; } diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index e9090b38f8117..52cc3d9380c08 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -22,7 +22,7 @@ #define pt_dump_seq_puts(m, fmt) \ ({ \ if (m) \ - seq_printf(m, fmt); \ + seq_puts(m, fmt); \ }) /* diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index a5ce1ab76ecee..cbaecbbe62e08 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -1089,7 +1089,7 @@ static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) #endif /* __riscv_xlen == 64 */ -void bpf_jit_build_prologue(struct rv_jit_context *ctx); +void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog); void bpf_jit_build_epilogue(struct rv_jit_context *ctx); int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 529a83b85c1c9..f5ba73bb153d7 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -1301,7 +1301,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return 0; } -void bpf_jit_build_prologue(struct rv_jit_context *ctx) +void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { const s8 *fp = bpf2rv32[BPF_REG_FP]; const s8 *r1 = bpf2rv32[BPF_REG_1]; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 26eeb39736319..e3de385a328c4 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -11,9 +11,11 @@ #include #include #include +#include #include "bpf_jit.h" #define RV_FENTRY_NINSNS 2 +#define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) #define RV_REG_TCC RV_REG_A6 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ @@ -480,6 +482,12 @@ static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); } +static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) +{ + if (IS_ENABLED(CONFIG_CFI_CLANG)) + emit(hash, ctx); +} + static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, struct rv_jit_context *ctx) { @@ -681,7 +689,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, if (ret) return ret; - if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4)) + if (memcmp(ip, old_insns, RV_FENTRY_NBYTES)) return -EFAULT; ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); @@ -690,8 +698,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, cpus_read_lock(); mutex_lock(&text_mutex); - if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4)) - ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS); + if (memcmp(ip, new_insns, RV_FENTRY_NBYTES)) + ret = patch_text(ip, new_insns, RV_FENTRY_NBYTES); mutex_unlock(&text_mutex); cpus_read_unlock(); @@ -854,10 +862,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size += 16; save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); - if (save_ret) { + if (save_ret) stack_size += 16; /* Save both A5 (BPF R0) and A0 */ - retval_off = stack_size; - } + retval_off = stack_size; stack_size += nregs * 8; args_off = stack_size; @@ -893,6 +900,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); } else { + /* emit kcfi hash */ + emit_kcfi(cfi_get_func_hash(func_addr), ctx); /* For the trampoline called directly, just handle * the frame of trampoline. */ @@ -1773,7 +1782,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return 0; } -void bpf_jit_build_prologue(struct rv_jit_context *ctx) +void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { int i, stack_adjust = 0, store_offset, bpf_stack_adjust; @@ -1802,6 +1811,9 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx) store_offset = stack_adjust - 8; + /* emit kcfi type preamble immediately before the first insn */ + emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 7b70ccb7fec34..6b3acac30c061 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "bpf_jit.h" /* Number of iterations to try until offsets converge. */ @@ -100,7 +101,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) pass++; ctx->ninsns = 0; - bpf_jit_build_prologue(ctx); + bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); ctx->prologue_len = ctx->ninsns; if (build_body(ctx, extra_pass, ctx->offset)) { @@ -160,7 +161,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx->ninsns = 0; ctx->nexentries = 0; - bpf_jit_build_prologue(ctx); + bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); if (build_body(ctx, extra_pass, NULL)) { prog = orig_prog; goto out_free_hdr; @@ -170,9 +171,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); - prog->bpf_func = (void *)ctx->ro_insns; + prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); prog->jited = 1; - prog->jited_len = prog_size; + prog->jited_len = prog_size - cfi_get_offset(); if (!prog->is_func || extra_pass) { if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index bd4782f23f66d..e99dae26500d2 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -128,7 +128,6 @@ config S390 select ARCH_WANT_DEFAULT_BPF_JIT select ARCH_WANT_IPC_PARSE_VERSION select ARCH_WANT_KERNEL_PMD_MKWRITE - select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 select DMA_OPS if PCI diff --git a/arch/s390/hypfs/hypfs_dbfs.c b/arch/s390/hypfs/hypfs_dbfs.c index 4024599eb448e..3612af9b4890b 100644 --- a/arch/s390/hypfs/hypfs_dbfs.c +++ b/arch/s390/hypfs/hypfs_dbfs.c @@ -6,6 +6,7 @@ * Author(s): Michael Holzheu */ +#include #include #include "hypfs.h" @@ -64,24 +65,28 @@ static long dbfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) long rc; mutex_lock(&df->lock); - if (df->unlocked_ioctl) - rc = df->unlocked_ioctl(file, cmd, arg); - else - rc = -ENOTTY; + rc = df->unlocked_ioctl(file, cmd, arg); mutex_unlock(&df->lock); return rc; } -static const struct file_operations dbfs_ops = { +static const struct file_operations dbfs_ops_ioctl = { .read = dbfs_read, .llseek = no_llseek, .unlocked_ioctl = dbfs_ioctl, }; +static const struct file_operations dbfs_ops = { + .read = dbfs_read, +}; + void hypfs_dbfs_create_file(struct hypfs_dbfs_file *df) { - df->dentry = debugfs_create_file(df->name, 0400, dbfs_dir, df, - &dbfs_ops); + const struct file_operations *fops = &dbfs_ops; + + if (df->unlocked_ioctl && !security_locked_down(LOCKDOWN_DEBUGFS)) + fops = &dbfs_ops_ioctl; + df->dentry = debugfs_create_file(df->name, 0400, dbfs_dir, df, fops); mutex_init(&df->lock); } diff --git a/arch/s390/hypfs/hypfs_diag_fs.c b/arch/s390/hypfs/hypfs_diag_fs.c index 00a6d370a2803..280266a74f378 100644 --- a/arch/s390/hypfs/hypfs_diag_fs.c +++ b/arch/s390/hypfs/hypfs_diag_fs.c @@ -208,6 +208,8 @@ static int hypfs_create_cpu_files(struct dentry *cpus_dir, void *cpu_info) snprintf(buffer, TMP_SIZE, "%d", cpu_info__cpu_addr(diag204_get_info_type(), cpu_info)); cpu_dir = hypfs_mkdir(cpus_dir, buffer); + if (IS_ERR(cpu_dir)) + return PTR_ERR(cpu_dir); rc = hypfs_create_u64(cpu_dir, "mgmtime", cpu_info__acc_time(diag204_get_info_type(), cpu_info) - cpu_info__lp_time(diag204_get_info_type(), cpu_info)); diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index b248694e00247..30e8e6baa5f85 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -138,7 +138,6 @@ struct zpci_dev { u8 has_resources : 1; u8 is_physfn : 1; u8 util_str_avail : 1; - u8 irqs_registered : 1; u8 reserved : 2; unsigned int devfn; /* DEVFN part of the RID*/ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 0250073f522fe..7f42a023d6bf6 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1065,17 +1065,15 @@ static inline pte_t pte_mkhuge(pte_t pte) #define IPTE_NODAT 0x400 #define IPTE_GUEST_ASCE 0x800 -static __always_inline void __ptep_rdp(unsigned long addr, pte_t *ptep, - unsigned long opt, unsigned long asce, - int local) +static __always_inline void __ptep_rdp(unsigned long addr, pte_t *ptep, int local) { unsigned long pto; pto = __pa(ptep) & ~(PTRS_PER_PTE * sizeof(pte_t) - 1); - asm volatile(".insn rrf,0xb98b0000,%[r1],%[r2],%[asce],%[m4]" + asm volatile(".insn rrf,0xb98b0000,%[r1],%[r2],%%r0,%[m4]" : "+m" (*ptep) - : [r1] "a" (pto), [r2] "a" ((addr & PAGE_MASK) | opt), - [asce] "a" (asce), [m4] "i" (local)); + : [r1] "a" (pto), [r2] "a" (addr & PAGE_MASK), + [m4] "i" (local)); } static __always_inline void __ptep_ipte(unsigned long address, pte_t *ptep, @@ -1259,7 +1257,7 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, * A local RDP can be used to do the flush. */ if (MACHINE_HAS_RDP && !(pte_val(*ptep) & _PAGE_PROTECT)) - __ptep_rdp(address, ptep, 0, 0, 1); + __ptep_rdp(address, ptep, 1); } #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index 4d646659a5f58..f7a44af12c2f4 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -192,13 +192,6 @@ static inline unsigned long get_tod_clock_fast(void) asm volatile("stckf %0" : "=Q" (clk) : : "cc"); return clk; } - -static inline cycles_t get_cycles(void) -{ - return (cycles_t) get_tod_clock() >> 2; -} -#define get_cycles get_cycles - int get_phys_clock(unsigned long *clock); void init_cpu_timer(void); @@ -226,6 +219,12 @@ static inline unsigned long get_tod_clock_monotonic(void) return tod; } +static inline cycles_t get_cycles(void) +{ + return (cycles_t)get_tod_clock_monotonic() >> 2; +} +#define get_cycles get_cycles + /** * tod_to_ns - convert a TOD format value to nanoseconds * @todval: to be converted TOD format value diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index fa5f6885c74aa..73a989dcfe208 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -4,6 +4,7 @@ * This code generates raw asm output which is post-processed to extract * and format the required data. */ +#define COMPILE_OFFSETS #define ASM_OFFSETS_C diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index ebad8c8b8c57d..0476ce7700dfa 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -639,7 +639,7 @@ SYM_CODE_START(stack_overflow) stmg %r0,%r7,__PT_R0(%r11) stmg %r8,%r9,__PT_PSW(%r11) mvc __PT_R8(64,%r11),0(%r14) - stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 + mvc __PT_ORIG_GPR2(8,%r11),__LC_PGM_LAST_BREAK xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs jg kernel_stack_overflow diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 65a66df5bb865..e590b4c096253 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -757,8 +757,6 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) break; case PERF_TYPE_HARDWARE: - if (is_sampling_event(event)) /* No sampling support */ - return -ENOENT; ev = attr->config; if (!attr->exclude_user && attr->exclude_kernel) { /* @@ -854,15 +852,15 @@ static int cpumf_pmu_event_type(struct perf_event *event) static int cpumf_pmu_event_init(struct perf_event *event) { unsigned int type = event->attr.type; - int err; + int err = -ENOENT; + if (is_sampling_event(event)) /* No sampling support */ + return err; if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_RAW) err = __hw_perf_event_init(event, type); else if (event->pmu->type == type) /* Registered as unknown PMU */ err = __hw_perf_event_init(event, cpumf_pmu_event_type(event)); - else - return -ENOENT; if (unlikely(err) && event->destroy) event->destroy(event); diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index d34d3548c046c..086d3e3ffdea5 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -579,7 +579,7 @@ static int stp_sync_clock(void *data) atomic_dec(&sync->cpus); /* Wait for in_sync to be set. */ while (READ_ONCE(sync->in_sync) == 0) - __udelay(1); + ; } if (sync->in_sync != 1) /* Didn't work. Clear per-cpu in sync bit again. */ diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 090dc38334336..969a5e1569192 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -490,7 +490,7 @@ enum prot_type { PROT_TYPE_DAT = 3, PROT_TYPE_IEP = 4, /* Dummy value for passing an initialized value when code != PGM_PROTECTION */ - PROT_NONE, + PROT_TYPE_DUMMY, }; static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, u8 ar, @@ -506,7 +506,7 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, switch (code) { case PGM_PROTECTION: switch (prot) { - case PROT_NONE: + case PROT_TYPE_DUMMY: /* We should never get here, acts like termination */ WARN_ON_ONCE(1); break; @@ -976,7 +976,7 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, gpa = kvm_s390_real_to_abs(vcpu, ga); if (kvm_is_error_gpa(vcpu->kvm, gpa)) { rc = PGM_ADDRESSING; - prot = PROT_NONE; + prot = PROT_TYPE_DUMMY; } } if (rc) @@ -1134,7 +1134,7 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, if (rc == PGM_PROTECTION) prot = PROT_TYPE_KEYC; else - prot = PROT_NONE; + prot = PROT_TYPE_DUMMY; rc = trans_exc_ending(vcpu, rc, ga, ar, mode, prot, terminate); } out_unlock: diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index b51666967aa1f..4721ada81a02d 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -249,11 +249,9 @@ static int ptdump_show(struct seq_file *m, void *v) .marker = address_markers, }; - get_online_mems(); mutex_lock(&cpa_mutex); ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); mutex_unlock(&cpa_mutex); - put_online_mems(); return 0; } DEFINE_SHOW_ATTRIBUTE(ptdump); diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 9355fbe5f51e9..2f534b26fda6c 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -456,11 +456,6 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) page = virt_to_page(pgtable); SetPageActive(page); page_table_free(mm, (unsigned long *)pgtable); - /* - * page_table_free() does not do the pgste gmap_unlink() which - * page_table_free_rcu() does: warn us if pgste ever reaches here. - */ - WARN_ON_ONCE(mm_has_pgste(mm)); } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 9ccb803eacd47..f7c599ce38f71 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -312,9 +312,9 @@ void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep, preempt_disable(); atomic_inc(&mm->context.flush_count); if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) - __ptep_rdp(addr, ptep, 0, 0, 1); + __ptep_rdp(addr, ptep, 1); else - __ptep_rdp(addr, ptep, 0, 0, 0); + __ptep_rdp(addr, ptep, 0); /* * PTE is not invalidated by RDP, only _PAGE_PROTECT is cleared. That * means it is still valid and active, and must not be changed according diff --git a/arch/s390/net/bpf_jit.h b/arch/s390/net/bpf_jit.h deleted file mode 100644 index 7822ea92e54af..0000000000000 --- a/arch/s390/net/bpf_jit.h +++ /dev/null @@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF Jit compiler defines - * - * Copyright IBM Corp. 2012,2015 - * - * Author(s): Martin Schwidefsky - * Michael Holzheu - */ - -#ifndef __ARCH_S390_NET_BPF_JIT_H -#define __ARCH_S390_NET_BPF_JIT_H - -#ifndef __ASSEMBLY__ - -#include -#include - -#endif /* __ASSEMBLY__ */ - -/* - * Stackframe layout (packed stack): - * - * ^ high - * +---------------+ | - * | old backchain | | - * +---------------+ | - * | r15 - r6 | | - * +---------------+ | - * | 4 byte align | | - * | tail_call_cnt | | - * BFP -> +===============+ | - * | | | - * | BPF stack | | - * | | | - * R15+160 -> +---------------+ | - * | new backchain | | - * R15+152 -> +---------------+ | - * | + 152 byte SA | | - * R15 -> +---------------+ + low - * - * We get 160 bytes stack space from calling function, but only use - * 12 * 8 byte for old backchain, r15..r6, and tail_call_cnt. - * - * The stack size used by the BPF program ("BPF stack" above) is passed - * via "aux->stack_depth". - */ -#define STK_SPACE_ADD (160) -#define STK_160_UNUSED (160 - 12 * 8) -#define STK_OFF (STK_SPACE_ADD - STK_160_UNUSED) - -#define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */ -#define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */ - -#endif /* __ARCH_S390_NET_BPF_JIT_H */ diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 62ee557d4b499..5a64d34a37482 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -31,11 +31,10 @@ #include #include #include -#include "bpf_jit.h" struct bpf_jit { u32 seen; /* Flags to remember seen eBPF instructions */ - u32 seen_reg[16]; /* Array to remember which registers are used */ + u16 seen_regs; /* Mask to remember which registers are used */ u32 *addrs; /* Array with relative instruction addresses */ u8 *prg_buf; /* Start of program */ int size; /* Size of program and literal pool */ @@ -53,6 +52,7 @@ struct bpf_jit { int excnt; /* Number of exception table entries */ int prologue_plt_ret; /* Return address for prologue hotpatch PLT */ int prologue_plt; /* Start of prologue hotpatch PLT */ + u32 frame_off; /* Offset of struct bpf_prog from %r15 */ }; #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */ @@ -118,8 +118,8 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) { u32 r1 = reg2hex[b1]; - if (r1 >= 6 && r1 <= 15 && !jit->seen_reg[r1]) - jit->seen_reg[r1] = 1; + if (r1 >= 6 && r1 <= 15) + jit->seen_regs |= (1 << r1); } #define REG_SET_SEEN(b1) \ @@ -127,8 +127,6 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) reg_set_seen(jit, b1); \ }) -#define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]] - /* * EMIT macros for code generation */ @@ -400,12 +398,26 @@ static void jit_fill_hole(void *area, unsigned int size) memset(area, 0, size); } +/* + * Caller-allocated part of the frame. + * Thanks to packed stack, its otherwise unused initial part can be used for + * the BPF stack and for the next frame. + */ +struct prog_frame { + u64 unused[8]; + /* BPF stack starts here and grows towards 0 */ + u32 tail_call_cnt; + u32 pad; + u64 r6[10]; /* r6 - r15 */ + u64 backchain; +} __packed; + /* * Save registers from "rs" (register start) to "re" (register end) on stack */ static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) { - u32 off = STK_OFF_R6 + (rs - 6) * 8; + u32 off = offsetof(struct prog_frame, r6) + (rs - 6) * 8; if (rs == re) /* stg %rs,off(%r15) */ @@ -418,12 +430,9 @@ static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) /* * Restore registers from "rs" (register start) to "re" (register end) on stack */ -static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth) +static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re) { - u32 off = STK_OFF_R6 + (rs - 6) * 8; - - if (jit->seen & SEEN_STACK) - off += STK_OFF + stack_depth; + u32 off = jit->frame_off + offsetof(struct prog_frame, r6) + (rs - 6) * 8; if (rs == re) /* lg %rs,off(%r15) */ @@ -436,12 +445,12 @@ static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth) /* * Return first seen register (from start) */ -static int get_start(struct bpf_jit *jit, int start) +static int get_start(u16 seen_regs, int start) { int i; for (i = start; i <= 15; i++) { - if (jit->seen_reg[i]) + if (seen_regs & (1 << i)) return i; } return 0; @@ -450,15 +459,15 @@ static int get_start(struct bpf_jit *jit, int start) /* * Return last seen register (from start) (gap >= 2) */ -static int get_end(struct bpf_jit *jit, int start) +static int get_end(u16 seen_regs, int start) { int i; for (i = start; i < 15; i++) { - if (!jit->seen_reg[i] && !jit->seen_reg[i + 1]) + if (!(seen_regs & (3 << i))) return i - 1; } - return jit->seen_reg[15] ? 15 : 14; + return (seen_regs & (1 << 15)) ? 15 : 14; } #define REGS_SAVE 1 @@ -467,8 +476,9 @@ static int get_end(struct bpf_jit *jit, int start) * Save and restore clobbered registers (6-15) on stack. * We save/restore registers in chunks with gap >= 2 registers. */ -static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth) +static void save_restore_regs(struct bpf_jit *jit, int op, u16 extra_regs) { + u16 seen_regs = jit->seen_regs | extra_regs; const int last = 15, save_restore_size = 6; int re = 6, rs; @@ -482,14 +492,14 @@ static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth) } do { - rs = get_start(jit, re); + rs = get_start(seen_regs, re); if (!rs) break; - re = get_end(jit, rs + 1); + re = get_end(seen_regs, rs + 1); if (op == REGS_SAVE) save_regs(jit, rs, re); else - restore_regs(jit, rs, re, stack_depth); + restore_regs(jit, rs, re); re++; } while (re <= last); } @@ -539,18 +549,27 @@ static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target) { memcpy(plt, &bpf_plt, sizeof(*plt)); plt->ret = ret; - plt->target = target; + /* + * (target == NULL) implies that the branch to this PLT entry was + * patched and became a no-op. However, some CPU could have jumped + * to this PLT entry before patching and may be still executing it. + * + * Since the intention in this case is to make the PLT entry a no-op, + * make the target point to the return label instead of NULL. + */ + plt->target = target ?: ret; } /* * Emit function prologue * * Save registers and create stack frame if necessary. - * See stack frame layout description in "bpf_jit.h"! + * Stack frame layout is described by struct prog_frame. */ -static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp, - u32 stack_depth) +static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) { + BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD); + /* No-op for hotpatching */ /* brcl 0,prologue_plt */ EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); @@ -558,8 +577,9 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp, if (fp->aux->func_idx == 0) { /* Initialize the tail call counter in the main program. */ - /* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */ - _EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT); + /* xc tail_call_cnt(4,%r15),tail_call_cnt(%r15) */ + _EMIT6(0xd703f000 | offsetof(struct prog_frame, tail_call_cnt), + 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); } else { /* * Skip the tail call counter initialization in subprograms. @@ -571,7 +591,7 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp, /* Tail calls have to skip above initialization */ jit->tail_call_start = jit->prg; /* Save registers */ - save_restore_regs(jit, REGS_SAVE, stack_depth); + save_restore_regs(jit, REGS_SAVE, 0); /* Setup literal pool */ if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) { if (!is_first_pass(jit) && @@ -587,17 +607,17 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp, } /* Setup stack and backchain */ if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) { - if (is_first_pass(jit) || (jit->seen & SEEN_FUNC)) - /* lgr %w1,%r15 (backchain) */ - EMIT4(0xb9040000, REG_W1, REG_15); - /* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */ - EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED); - /* aghi %r15,-STK_OFF */ - EMIT4_IMM(0xa70b0000, REG_15, -(STK_OFF + stack_depth)); - if (is_first_pass(jit) || (jit->seen & SEEN_FUNC)) - /* stg %w1,152(%r15) (backchain) */ - EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, - REG_15, 152); + /* lgr %w1,%r15 (backchain) */ + EMIT4(0xb9040000, REG_W1, REG_15); + /* la %bfp,unused_end(%r15) (BPF frame pointer) */ + EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, + offsetofend(struct prog_frame, unused)); + /* aghi %r15,-frame_off */ + EMIT4_IMM(0xa70b0000, REG_15, -jit->frame_off); + /* stg %w1,backchain(%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, + REG_15, + offsetof(struct prog_frame, backchain)); } } @@ -641,13 +661,13 @@ static void call_r1(struct bpf_jit *jit) /* * Function epilogue */ -static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) +static void bpf_jit_epilogue(struct bpf_jit *jit) { jit->exit_ip = jit->prg; /* Load exit code: lgr %r2,%b0 */ EMIT4(0xb9040000, REG_2, BPF_REG_0); /* Restore registers */ - save_restore_regs(jit, REGS_RESTORE, stack_depth); + save_restore_regs(jit, REGS_RESTORE, 0); if (nospec_uses_trampoline()) { jit->r14_thunk_ip = jit->prg; /* Generate __s390_indirect_jump_r14 thunk */ @@ -773,7 +793,7 @@ static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) * stack space for the large switch statement. */ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, - int i, bool extra_pass, u32 stack_depth) + int i, bool extra_pass) { struct bpf_insn *insn = &fp->insnsi[i]; u32 dst_reg = insn->dst_reg; @@ -1419,17 +1439,11 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, jit->seen |= SEEN_FUNC; /* * Copy the tail call counter to where the callee expects it. - * - * Note 1: The callee can increment the tail call counter, but - * we do not load it back, since the x86 JIT does not do this - * either. - * - * Note 2: We assume that the verifier does not let us call the - * main program, which clears the tail call counter on entry. */ - /* mvc STK_OFF_TCCNT(4,%r15),N(%r15) */ - _EMIT6(0xd203f000 | STK_OFF_TCCNT, - 0xf000 | (STK_OFF_TCCNT + STK_OFF + stack_depth)); + /* mvc tail_call_cnt(4,%r15),frame_off+tail_call_cnt(%r15) */ + _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), + 0xf000 | (jit->frame_off + + offsetof(struct prog_frame, tail_call_cnt))); /* Sign-extend the kfunc arguments. */ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { @@ -1451,6 +1465,22 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, call_r1(jit); /* lgr %b0,%r2: load return value into %b0 */ EMIT4(0xb9040000, BPF_REG_0, REG_2); + + /* + * Copy the potentially updated tail call counter back. + */ + + if (insn->src_reg == BPF_PSEUDO_CALL) + /* + * mvc frame_off+tail_call_cnt(%r15), + * tail_call_cnt(4,%r15) + */ + _EMIT6(0xd203f000 | (jit->frame_off + + offsetof(struct prog_frame, + tail_call_cnt)), + 0xf000 | offsetof(struct prog_frame, + tail_call_cnt)); + break; } case BPF_JMP | BPF_TAIL_CALL: { @@ -1480,10 +1510,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, * goto out; */ - if (jit->seen & SEEN_STACK) - off = STK_OFF_TCCNT + STK_OFF + stack_depth; - else - off = STK_OFF_TCCNT; + off = jit->frame_off + + offsetof(struct prog_frame, tail_call_cnt); /* lhi %w0,1 */ EMIT4_IMM(0xa7080000, REG_W0, 1); /* laal %w1,%w0,off(%r15) */ @@ -1513,7 +1541,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * Restore registers before calling function */ - save_restore_regs(jit, REGS_RESTORE, stack_depth); + save_restore_regs(jit, REGS_RESTORE, 0); /* * goto *(prog->bpf_func + tail_call_start); @@ -1816,7 +1844,7 @@ static int bpf_set_addr(struct bpf_jit *jit, int i) * Compile eBPF program into s390x code */ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, - bool extra_pass, u32 stack_depth) + bool extra_pass) { int i, insn_count, lit32_size, lit64_size; @@ -1824,19 +1852,25 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, jit->lit64 = jit->lit64_start; jit->prg = 0; jit->excnt = 0; + if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) + jit->frame_off = sizeof(struct prog_frame) - + offsetofend(struct prog_frame, unused) + + round_up(fp->aux->stack_depth, 8); + else + jit->frame_off = 0; - bpf_jit_prologue(jit, fp, stack_depth); + bpf_jit_prologue(jit, fp); if (bpf_set_addr(jit, 0) < 0) return -1; for (i = 0; i < fp->len; i += insn_count) { - insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth); + insn_count = bpf_jit_insn(jit, fp, i, extra_pass); if (insn_count < 0) return -1; /* Next instruction address */ if (bpf_set_addr(jit, i + insn_count) < 0) return -1; } - bpf_jit_epilogue(jit, stack_depth); + bpf_jit_epilogue(jit); lit32_size = jit->lit32 - jit->lit32_start; lit64_size = jit->lit64 - jit->lit64_start; @@ -1896,7 +1930,6 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, */ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) { - u32 stack_depth = round_up(fp->aux->stack_depth, 8); struct bpf_prog *tmp, *orig_fp = fp; struct bpf_binary_header *header; struct s390_jit_data *jit_data; @@ -1949,7 +1982,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) * - 3: Calculate program size and addrs array */ for (pass = 1; pass <= 3; pass++) { - if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) { + if (bpf_jit_prog(&jit, fp, extra_pass)) { fp = orig_fp; goto free_addrs; } @@ -1963,7 +1996,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) goto free_addrs; } skip_init_ctx: - if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) { + if (bpf_jit_prog(&jit, fp, extra_pass)) { bpf_jit_binary_free(header); fp = orig_fp; goto free_addrs; @@ -2279,9 +2312,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* stg %r1,backchain_off(%r15) */ EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15, tjit->backchain_off); - /* mvc tccnt_off(4,%r15),stack_size+STK_OFF_TCCNT(%r15) */ + /* mvc tccnt_off(4,%r15),stack_size+tail_call_cnt(%r15) */ _EMIT6(0xd203f000 | tjit->tccnt_off, - 0xf000 | (tjit->stack_size + STK_OFF_TCCNT)); + 0xf000 | (tjit->stack_size + + offsetof(struct prog_frame, tail_call_cnt))); /* stmg %r2,%rN,fwd_reg_args_off(%r15) */ if (nr_reg_args) EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2, @@ -2418,8 +2452,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, (nr_stack_args * sizeof(u64) - 1) << 16 | tjit->stack_args_off, 0xf000 | tjit->orig_stack_args_off); - /* mvc STK_OFF_TCCNT(4,%r15),tccnt_off(%r15) */ - _EMIT6(0xd203f000 | STK_OFF_TCCNT, 0xf000 | tjit->tccnt_off); + /* mvc tail_call_cnt(4,%r15),tccnt_off(%r15) */ + _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), + 0xf000 | tjit->tccnt_off); /* lgr %r1,%r8 */ EMIT4(0xb9040000, REG_1, REG_8); /* %r1() */ @@ -2427,6 +2462,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* stg %r2,retval_off(%r15) */ EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, tjit->retval_off); + /* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */ + _EMIT6(0xd203f000 | tjit->tccnt_off, + 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); im->ip_after_call = jit->prg_buf + jit->prg; @@ -2476,8 +2514,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, if (flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET)) EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15, tjit->retval_off); - /* mvc stack_size+STK_OFF_TCCNT(4,%r15),tccnt_off(%r15) */ - _EMIT6(0xd203f000 | (tjit->stack_size + STK_OFF_TCCNT), + /* mvc stack_size+tail_call_cnt(4,%r15),tccnt_off(%r15) */ + _EMIT6(0xd203f000 | (tjit->stack_size + + offsetof(struct prog_frame, tail_call_cnt)), 0xf000 | tjit->tccnt_off); /* aghi %r15,stack_size */ EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index b3961f1016ea0..bd4172db26345 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -83,6 +83,7 @@ static pci_ers_result_t zpci_event_notify_error_detected(struct pci_dev *pdev, pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT; ers_res = driver->err_handler->error_detected(pdev, pdev->error_state); + pci_uevent_ers(pdev, ers_res); if (ers_result_indicates_abort(ers_res)) pr_info("%s: Automatic recovery failed after initial reporting\n", pci_name(pdev)); else if (ers_res == PCI_ERS_RESULT_NEED_RESET) @@ -98,6 +99,10 @@ static pci_ers_result_t zpci_event_do_error_state_clear(struct pci_dev *pdev, struct zpci_dev *zdev = to_zpci(pdev); int rc; + /* The underlying device may have been disabled by the event */ + if (!zdev_enabled(zdev)) + return PCI_ERS_RESULT_NEED_RESET; + pr_info("%s: Unblocking device access for examination\n", pci_name(pdev)); rc = zpci_reset_load_store_blocked(zdev); if (rc) { @@ -169,7 +174,7 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) * is unbound or probed and that userspace can't access its * configuration space while we perform recovery. */ - pci_dev_lock(pdev); + device_lock(&pdev->dev); if (pdev->error_state == pci_channel_io_perm_failure) { ers_res = PCI_ERS_RESULT_DISCONNECT; goto out_unlock; @@ -208,6 +213,7 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) ers_res = zpci_event_do_reset(pdev, driver); if (ers_res != PCI_ERS_RESULT_RECOVERED) { + pci_uevent_ers(pdev, PCI_ERS_RESULT_DISCONNECT); pr_err("%s: Automatic recovery failed; operator intervention is required\n", pci_name(pdev)); goto out_unlock; @@ -216,8 +222,9 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) pr_info("%s: The device is ready to resume operations\n", pci_name(pdev)); if (driver->err_handler->resume) driver->err_handler->resume(pdev); + pci_uevent_ers(pdev, PCI_ERS_RESULT_RECOVERED); out_unlock: - pci_dev_unlock(pdev); + device_unlock(&pdev->dev); return ers_res; } diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index 84482a9213322..e73be96ce5fe6 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -107,9 +107,6 @@ static int zpci_set_irq(struct zpci_dev *zdev) else rc = zpci_set_airq(zdev); - if (!rc) - zdev->irqs_registered = 1; - return rc; } @@ -123,9 +120,6 @@ static int zpci_clear_irq(struct zpci_dev *zdev) else rc = zpci_clear_airq(zdev); - if (!rc) - zdev->irqs_registered = 0; - return rc; } @@ -427,8 +421,7 @@ bool arch_restore_msi_irqs(struct pci_dev *pdev) { struct zpci_dev *zdev = to_zpci(pdev); - if (!zdev->irqs_registered) - zpci_set_irq(zdev); + zpci_set_irq(zdev); return true; } diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index a90499c087f0c..2ee7b5a5016c7 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -223,7 +223,7 @@ static inline int __pcilg_mio_inuser( [ioaddr_len] "+&d" (ioaddr_len.pair), [cc] "+d" (cc), [val] "=d" (val), [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), - [shift] "+d" (shift) + [shift] "+a" (shift) :: "cc", "memory"); /* did we write everything to the user space buffer? */ diff --git a/arch/sh/Makefile b/arch/sh/Makefile index cab2f9c011a8d..7b420424b6d7c 100644 --- a/arch/sh/Makefile +++ b/arch/sh/Makefile @@ -103,16 +103,16 @@ UTS_MACHINE := sh LDFLAGS_vmlinux += -e _stext ifdef CONFIG_CPU_LITTLE_ENDIAN -ld-bfd := elf32-sh-linux -LDFLAGS_vmlinux += --defsym jiffies=jiffies_64 --oformat $(ld-bfd) +ld_bfd := elf32-sh-linux +LDFLAGS_vmlinux += --defsym jiffies=jiffies_64 --oformat $(ld_bfd) KBUILD_LDFLAGS += -EL else -ld-bfd := elf32-shbig-linux -LDFLAGS_vmlinux += --defsym jiffies=jiffies_64+4 --oformat $(ld-bfd) +ld_bfd := elf32-shbig-linux +LDFLAGS_vmlinux += --defsym jiffies=jiffies_64+4 --oformat $(ld_bfd) KBUILD_LDFLAGS += -EB endif -export ld-bfd +export ld_bfd # Mach groups machdir-$(CONFIG_SOLUTION_ENGINE) += mach-se diff --git a/arch/sh/boot/compressed/Makefile b/arch/sh/boot/compressed/Makefile index b5e29f99c02c8..5d5b2da2a3f1f 100644 --- a/arch/sh/boot/compressed/Makefile +++ b/arch/sh/boot/compressed/Makefile @@ -29,7 +29,7 @@ endif ccflags-remove-$(CONFIG_MCOUNT) += -pg -LDFLAGS_vmlinux := --oformat $(ld-bfd) -Ttext $(IMAGE_OFFSET) -e startup \ +LDFLAGS_vmlinux := --oformat $(ld_bfd) -Ttext $(IMAGE_OFFSET) -e startup \ -T $(obj)/../../kernel/vmlinux.lds KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING @@ -53,7 +53,7 @@ $(obj)/vmlinux.bin.lzo: $(obj)/vmlinux.bin FORCE OBJCOPYFLAGS += -R .empty_zero_page -LDFLAGS_piggy.o := -r --format binary --oformat $(ld-bfd) -T +LDFLAGS_piggy.o := -r --format binary --oformat $(ld_bfd) -T $(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/vmlinux.bin.$(suffix_y) FORCE $(call if_changed,ld) diff --git a/arch/sh/boot/romimage/Makefile b/arch/sh/boot/romimage/Makefile index c7c8be58400cd..17b03df0a8de4 100644 --- a/arch/sh/boot/romimage/Makefile +++ b/arch/sh/boot/romimage/Makefile @@ -13,7 +13,7 @@ mmcif-obj-$(CONFIG_CPU_SUBTYPE_SH7724) := $(obj)/mmcif-sh7724.o load-$(CONFIG_ROMIMAGE_MMCIF) := $(mmcif-load-y) obj-$(CONFIG_ROMIMAGE_MMCIF) := $(mmcif-obj-y) -LDFLAGS_vmlinux := --oformat $(ld-bfd) -Ttext $(load-y) -e romstart \ +LDFLAGS_vmlinux := --oformat $(ld_bfd) -Ttext $(load-y) -e romstart \ -T $(obj)/../../kernel/vmlinux.lds $(obj)/vmlinux: $(obj)/head.o $(obj-y) $(obj)/piggy.o FORCE @@ -24,7 +24,7 @@ OBJCOPYFLAGS += -j .empty_zero_page $(obj)/zeropage.bin: vmlinux FORCE $(call if_changed,objcopy) -LDFLAGS_piggy.o := -r --format binary --oformat $(ld-bfd) -T +LDFLAGS_piggy.o := -r --format binary --oformat $(ld_bfd) -T $(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/zeropage.bin arch/sh/boot/zImage FORCE $(call if_changed,ld) diff --git a/arch/sh/kernel/asm-offsets.c b/arch/sh/kernel/asm-offsets.c index a0322e8328456..429b6a7631468 100644 --- a/arch/sh/kernel/asm-offsets.c +++ b/arch/sh/kernel/asm-offsets.c @@ -8,6 +8,7 @@ * compile this file to assembler, and then extract the * #defines from the assembly-language output. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h index 8fb09eec8c3e7..694ed081cf8d9 100644 --- a/arch/sparc/include/asm/elf_64.h +++ b/arch/sparc/include/asm/elf_64.h @@ -58,6 +58,7 @@ #define R_SPARC_7 43 #define R_SPARC_5 44 #define R_SPARC_6 45 +#define R_SPARC_UA64 54 /* Bits present in AT_HWCAP, primarily for Sparc32. */ #define HWCAP_SPARC_FLUSH 0x00000001 diff --git a/arch/sparc/include/asm/io_64.h b/arch/sparc/include/asm/io_64.h index 9303270b22f3c..f9d370324729e 100644 --- a/arch/sparc/include/asm/io_64.h +++ b/arch/sparc/include/asm/io_64.h @@ -250,19 +250,19 @@ void insl(unsigned long, void *, unsigned long); #define insw insw #define insl insl -static inline void readsb(void __iomem *port, void *buf, unsigned long count) +static inline void readsb(const volatile void __iomem *port, void *buf, unsigned long count) { insb((unsigned long __force)port, buf, count); } #define readsb readsb -static inline void readsw(void __iomem *port, void *buf, unsigned long count) +static inline void readsw(const volatile void __iomem *port, void *buf, unsigned long count) { insw((unsigned long __force)port, buf, count); } #define readsw readsw -static inline void readsl(void __iomem *port, void *buf, unsigned long count) +static inline void readsl(const volatile void __iomem *port, void *buf, unsigned long count) { insl((unsigned long __force)port, buf, count); } diff --git a/arch/sparc/kernel/asm-offsets.c b/arch/sparc/kernel/asm-offsets.c index 5784f2df489a4..f1e27a7f800f4 100644 --- a/arch/sparc/kernel/asm-offsets.c +++ b/arch/sparc/kernel/asm-offsets.c @@ -10,6 +10,7 @@ * * On sparc, thread_info data is static and TI_XXX offsets are computed by hand. */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index 66c45a2764bc8..a7780907fc2f9 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -117,6 +117,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, break; #ifdef CONFIG_SPARC64 case R_SPARC_64: + case R_SPARC_UA64: location[0] = v >> 56; location[1] = v >> 48; location[2] = v >> 40; diff --git a/arch/sparc/kernel/of_device_32.c b/arch/sparc/kernel/of_device_32.c index 06012e68bdcae..284a4cafa4324 100644 --- a/arch/sparc/kernel/of_device_32.c +++ b/arch/sparc/kernel/of_device_32.c @@ -387,6 +387,7 @@ static struct platform_device * __init scan_one_device(struct device_node *dp, if (of_device_register(op)) { printk("%pOF: Could not register of device.\n", dp); + put_device(&op->dev); kfree(op); op = NULL; } diff --git a/arch/sparc/kernel/of_device_64.c b/arch/sparc/kernel/of_device_64.c index d3842821a5a05..d2cd4f42e0cf0 100644 --- a/arch/sparc/kernel/of_device_64.c +++ b/arch/sparc/kernel/of_device_64.c @@ -680,6 +680,7 @@ static struct platform_device * __init scan_one_device(struct device_node *dp, if (of_device_register(op)) { printk("%pOF: Could not register of device.\n", dp); + put_device(&op->dev); kfree(op); op = NULL; } diff --git a/arch/sparc/lib/M7memcpy.S b/arch/sparc/lib/M7memcpy.S index cbd42ea7c3f7c..99357bfa8e82a 100644 --- a/arch/sparc/lib/M7memcpy.S +++ b/arch/sparc/lib/M7memcpy.S @@ -696,16 +696,16 @@ FUNC_NAME: EX_LD_FP(LOAD(ldd, %o4+40, %f26), memcpy_retl_o2_plus_o5_plus_40) faligndata %f24, %f26, %f10 EX_ST_FP(STORE(std, %f6, %o0+24), memcpy_retl_o2_plus_o5_plus_40) - EX_LD_FP(LOAD(ldd, %o4+48, %f28), memcpy_retl_o2_plus_o5_plus_40) + EX_LD_FP(LOAD(ldd, %o4+48, %f28), memcpy_retl_o2_plus_o5_plus_32) faligndata %f26, %f28, %f12 - EX_ST_FP(STORE(std, %f8, %o0+32), memcpy_retl_o2_plus_o5_plus_40) + EX_ST_FP(STORE(std, %f8, %o0+32), memcpy_retl_o2_plus_o5_plus_32) add %o4, 64, %o4 - EX_LD_FP(LOAD(ldd, %o4-8, %f30), memcpy_retl_o2_plus_o5_plus_40) + EX_LD_FP(LOAD(ldd, %o4-8, %f30), memcpy_retl_o2_plus_o5_plus_24) faligndata %f28, %f30, %f14 - EX_ST_FP(STORE(std, %f10, %o0+40), memcpy_retl_o2_plus_o5_plus_40) - EX_ST_FP(STORE(std, %f12, %o0+48), memcpy_retl_o2_plus_o5_plus_40) + EX_ST_FP(STORE(std, %f10, %o0+40), memcpy_retl_o2_plus_o5_plus_24) + EX_ST_FP(STORE(std, %f12, %o0+48), memcpy_retl_o2_plus_o5_plus_16) add %o0, 64, %o0 - EX_ST_FP(STORE(std, %f14, %o0-8), memcpy_retl_o2_plus_o5_plus_40) + EX_ST_FP(STORE(std, %f14, %o0-8), memcpy_retl_o2_plus_o5_plus_8) fsrc2 %f30, %f14 bgu,pt %xcc, .Lunalign_sloop prefetch [%o4 + (8 * BLOCK_SIZE)], 20 @@ -728,7 +728,7 @@ FUNC_NAME: add %o4, 8, %o4 faligndata %f0, %f2, %f16 subcc %o5, 8, %o5 - EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5) + EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_8) fsrc2 %f2, %f0 bgu,pt %xcc, .Lunalign_by8 add %o0, 8, %o0 @@ -772,7 +772,7 @@ FUNC_NAME: subcc %o5, 0x20, %o5 EX_ST(STORE(stx, %o3, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32) EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24) - EX_ST(STORE(stx, %g7, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, %g7, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_16) EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8) bne,pt %xcc, 1b add %o0, 0x20, %o0 @@ -804,12 +804,12 @@ FUNC_NAME: brz,pt %o3, 2f sub %o2, %o3, %o2 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_o3) add %o1, 1, %o1 subcc %o3, 1, %o3 add %o0, 1, %o0 bne,pt %xcc, 1b - EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_o3_plus_1) 2: and %o1, 0x7, %o3 brz,pn %o3, .Lmedium_noprefetch_cp diff --git a/arch/sparc/lib/Memcpy_utils.S b/arch/sparc/lib/Memcpy_utils.S index 64fbac28b3db1..207343367bb2d 100644 --- a/arch/sparc/lib/Memcpy_utils.S +++ b/arch/sparc/lib/Memcpy_utils.S @@ -137,6 +137,15 @@ ENTRY(memcpy_retl_o2_plus_63_8) ba,pt %xcc, __restore_asi add %o2, 8, %o0 ENDPROC(memcpy_retl_o2_plus_63_8) +ENTRY(memcpy_retl_o2_plus_o3) + ba,pt %xcc, __restore_asi + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3) +ENTRY(memcpy_retl_o2_plus_o3_plus_1) + add %o3, 1, %o3 + ba,pt %xcc, __restore_asi + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_plus_1) ENTRY(memcpy_retl_o2_plus_o5) ba,pt %xcc, __restore_asi add %o2, %o5, %o0 diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S index 7ad58ebe0d009..df0ec1bd19489 100644 --- a/arch/sparc/lib/NG4memcpy.S +++ b/arch/sparc/lib/NG4memcpy.S @@ -281,7 +281,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ subcc %o5, 0x20, %o5 EX_ST(STORE(stx, %g1, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32) EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24) - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_16) EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8) bne,pt %icc, 1b add %o0, 0x20, %o0 diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S index ee51c12306894..bbd3ea0a64822 100644 --- a/arch/sparc/lib/NGmemcpy.S +++ b/arch/sparc/lib/NGmemcpy.S @@ -79,8 +79,8 @@ #ifndef EX_RETVAL #define EX_RETVAL(x) x __restore_asi: - ret wr %g0, ASI_AIUS, %asi + ret restore ENTRY(NG_ret_i2_plus_i4_plus_1) ba,pt %xcc, __restore_asi @@ -125,15 +125,16 @@ ENTRY(NG_ret_i2_plus_g1_minus_56) ba,pt %xcc, __restore_asi add %i2, %g1, %i0 ENDPROC(NG_ret_i2_plus_g1_minus_56) -ENTRY(NG_ret_i2_plus_i4) +ENTRY(NG_ret_i2_plus_i4_plus_16) + add %i4, 16, %i4 ba,pt %xcc, __restore_asi add %i2, %i4, %i0 -ENDPROC(NG_ret_i2_plus_i4) -ENTRY(NG_ret_i2_plus_i4_minus_8) - sub %i4, 8, %i4 +ENDPROC(NG_ret_i2_plus_i4_plus_16) +ENTRY(NG_ret_i2_plus_i4_plus_8) + add %i4, 8, %i4 ba,pt %xcc, __restore_asi add %i2, %i4, %i0 -ENDPROC(NG_ret_i2_plus_i4_minus_8) +ENDPROC(NG_ret_i2_plus_i4_plus_8) ENTRY(NG_ret_i2_plus_8) ba,pt %xcc, __restore_asi add %i2, 8, %i0 @@ -160,6 +161,12 @@ ENTRY(NG_ret_i2_and_7_plus_i4) ba,pt %xcc, __restore_asi add %i2, %i4, %i0 ENDPROC(NG_ret_i2_and_7_plus_i4) +ENTRY(NG_ret_i2_and_7_plus_i4_plus_8) + and %i2, 7, %i2 + add %i4, 8, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_and_7_plus_i4) #endif .align 64 @@ -405,13 +412,13 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ andn %i2, 0xf, %i4 and %i2, 0xf, %i2 1: subcc %i4, 0x10, %i4 - EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4) + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4_plus_16) add %i1, 0x08, %i1 - EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4) + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4_plus_16) sub %i1, 0x08, %i1 - EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4_plus_16) add %i1, 0x8, %i1 - EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8) + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_plus_8) bgu,pt %XCC, 1b add %i1, 0x8, %i1 73: andcc %i2, 0x8, %g0 @@ -468,7 +475,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ subcc %i4, 0x8, %i4 srlx %g3, %i3, %i5 or %i5, %g2, %i5 - EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4) + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S index 635398ec7540e..154fbd35400ca 100644 --- a/arch/sparc/lib/U1memcpy.S +++ b/arch/sparc/lib/U1memcpy.S @@ -164,17 +164,18 @@ ENTRY(U1_gs_40_fp) retl add %o0, %o2, %o0 ENDPROC(U1_gs_40_fp) -ENTRY(U1_g3_0_fp) - VISExitHalf - retl - add %g3, %o2, %o0 -ENDPROC(U1_g3_0_fp) ENTRY(U1_g3_8_fp) VISExitHalf add %g3, 8, %g3 retl add %g3, %o2, %o0 ENDPROC(U1_g3_8_fp) +ENTRY(U1_g3_16_fp) + VISExitHalf + add %g3, 16, %g3 + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_16_fp) ENTRY(U1_o2_0_fp) VISExitHalf retl @@ -547,18 +548,18 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 62: FINISH_VISCHUNK(o0, f44, f46) 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) -93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp) +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_8_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f0, %f2, %f8 - EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_16_fp) bl,pn %xcc, 95f add %o0, 8, %o0 - EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp) + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_8_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f2, %f0, %f8 - EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_16_fp) bge,pt %xcc, 93b add %o0, 8, %o0 diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S index 9248d59c734ce..bace3a18f836f 100644 --- a/arch/sparc/lib/U3memcpy.S +++ b/arch/sparc/lib/U3memcpy.S @@ -267,6 +267,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ faligndata %f10, %f12, %f26 EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2) + and %o2, 0x3f, %o2 subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE add %o1, 0x40, %o1 bgu,pt %XCC, 1f @@ -336,7 +337,6 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ * Also notice how this code is careful not to perform a * load past the end of the src buffer. */ - and %o2, 0x3f, %o2 andcc %o2, 0x38, %g2 be,pn %XCC, 2f subcc %g2, 0x8, %g2 diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 8dad5d0995dcc..5bf77c07f6b62 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -133,6 +133,26 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift) { + unsigned long hugepage_size = _PAGE_SZ4MB_4U; + + pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4U; + + switch (shift) { + case HPAGE_256MB_SHIFT: + hugepage_size = _PAGE_SZ256MB_4U; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_SHIFT: + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_64K_SHIFT: + hugepage_size = _PAGE_SZ64K_4U; + break; + default: + WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); + } + + pte_val(entry) = pte_val(entry) | hugepage_size; return entry; } diff --git a/arch/um/Makefile b/arch/um/Makefile index 34957dcb88b9c..744c5d0bdeb8f 100644 --- a/arch/um/Makefile +++ b/arch/um/Makefile @@ -151,5 +151,6 @@ MRPROPER_FILES += $(HOST_DIR)/include/generated archclean: @find . \( -name '*.bb' -o -name '*.bbg' -o -name '*.da' \ -o -name '*.gcov' \) -type f -print | xargs rm -f + $(Q)$(MAKE) -f $(srctree)/Makefile ARCH=$(HEADER_ARCH) clean export HEADER_ARCH SUBARCH USER_CFLAGS CFLAGS_NO_HARDENING DEV_NULL_PATH diff --git a/arch/um/drivers/mconsole_user.c b/arch/um/drivers/mconsole_user.c index e24298a734bef..a04cd13c6315a 100644 --- a/arch/um/drivers/mconsole_user.c +++ b/arch/um/drivers/mconsole_user.c @@ -71,7 +71,9 @@ static struct mconsole_command *mconsole_parse(struct mc_request *req) return NULL; } +#ifndef MIN #define MIN(a,b) ((a)<(b) ? (a):(b)) +#endif #define STRINGX(x) #x #define STRING(x) STRINGX(x) diff --git a/arch/um/drivers/rtc_user.c b/arch/um/drivers/rtc_user.c index 7c3cec4c68cff..006a5a164ea91 100644 --- a/arch/um/drivers/rtc_user.c +++ b/arch/um/drivers/rtc_user.c @@ -28,7 +28,7 @@ int uml_rtc_start(bool timetravel) int err; if (timetravel) { - int err = os_pipe(uml_rtc_irq_fds, 1, 1); + err = os_pipe(uml_rtc_irq_fds, 1, 1); if (err) goto fail; } else { diff --git a/arch/um/drivers/ssl.c b/arch/um/drivers/ssl.c index 277cea3d30eb5..8006a5bd578c2 100644 --- a/arch/um/drivers/ssl.c +++ b/arch/um/drivers/ssl.c @@ -199,4 +199,7 @@ static int ssl_non_raw_setup(char *str) return 1; } __setup("ssl-non-raw", ssl_non_raw_setup); -__channel_help(ssl_non_raw_setup, "set serial lines to non-raw mode"); +__uml_help(ssl_non_raw_setup, +"ssl-non-raw\n" +" Set serial lines to non-raw mode.\n\n" +); diff --git a/arch/um/drivers/ubd_user.c b/arch/um/drivers/ubd_user.c index a1afe414ce481..fb5b1e7c133d8 100644 --- a/arch/um/drivers/ubd_user.c +++ b/arch/um/drivers/ubd_user.c @@ -41,7 +41,7 @@ int start_io_thread(unsigned long sp, int *fd_out) *fd_out = fds[1]; err = os_set_fd_block(*fd_out, 0); - err = os_set_fd_block(kernel_fd, 0); + err |= os_set_fd_block(kernel_fd, 0); if (err) { printk("start_io_thread - failed to set nonblocking I/O.\n"); goto out_close; diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index 2baa8d4a33ed3..1a068859a4185 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -1600,35 +1600,19 @@ static void vector_eth_configure( device->dev = dev; - *vp = ((struct vector_private) - { - .list = LIST_HEAD_INIT(vp->list), - .dev = dev, - .unit = n, - .options = get_transport_options(def), - .rx_irq = 0, - .tx_irq = 0, - .parsed = def, - .max_packet = get_mtu(def) + ETH_HEADER_OTHER, - /* TODO - we need to calculate headroom so that ip header - * is 16 byte aligned all the time - */ - .headroom = get_headroom(def), - .form_header = NULL, - .verify_header = NULL, - .header_rxbuffer = NULL, - .header_txbuffer = NULL, - .header_size = 0, - .rx_header_size = 0, - .rexmit_scheduled = false, - .opened = false, - .transport_data = NULL, - .in_write_poll = false, - .coalesce = 2, - .req_size = get_req_size(def), - .in_error = false, - .bpf = NULL - }); + INIT_LIST_HEAD(&vp->list); + vp->dev = dev; + vp->unit = n; + vp->options = get_transport_options(def); + vp->parsed = def; + vp->max_packet = get_mtu(def) + ETH_HEADER_OTHER; + /* + * TODO - we need to calculate headroom so that ip header + * is 16 byte aligned all the time + */ + vp->headroom = get_headroom(def); + vp->coalesce = 2; + vp->req_size = get_req_size(def); dev->features = dev->hw_features = (NETIF_F_SG | NETIF_F_FRAGLIST); INIT_WORK(&vp->reset_tx, vector_reset_tx); diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index 8adca2000e519..d790acfc2c674 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -1229,10 +1229,12 @@ static int virtio_uml_probe(struct platform_device *pdev) device_set_wakeup_capable(&vu_dev->vdev.dev, true); rc = register_virtio_device(&vu_dev->vdev); - if (rc) + if (rc) { put_device(&vu_dev->vdev.dev); + return rc; + } vu_dev->registered = 1; - return rc; + return 0; error_init: os_close_file(vu_dev->sock); diff --git a/arch/um/include/asm/asm-prototypes.h b/arch/um/include/asm/asm-prototypes.h index 5898a26daa0dd..408b31d591279 100644 --- a/arch/um/include/asm/asm-prototypes.h +++ b/arch/um/include/asm/asm-prototypes.h @@ -1 +1,6 @@ #include +#include + +#ifdef CONFIG_UML_X86 +extern void cmpxchg8b_emu(void); +#endif diff --git a/arch/um/include/asm/thread_info.h b/arch/um/include/asm/thread_info.h index c7b4b49826a2a..40d823f36c094 100644 --- a/arch/um/include/asm/thread_info.h +++ b/arch/um/include/asm/thread_info.h @@ -68,7 +68,11 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) #define _TIF_MEMDIE (1 << TIF_MEMDIE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) #define _TIF_SECCOMP (1 << TIF_SECCOMP) #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) +#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL | \ + _TIF_NOTIFY_RESUME) + #endif diff --git a/arch/um/kernel/asm-offsets.c b/arch/um/kernel/asm-offsets.c index 1fb12235ab9c8..a69873aa697f4 100644 --- a/arch/um/kernel/asm-offsets.c +++ b/arch/um/kernel/asm-offsets.c @@ -1 +1,3 @@ +#define COMPILE_OFFSETS + #include diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index 38d5a71a579bc..f6c766b2bdf5e 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -68,6 +68,7 @@ void __init mem_init(void) map_memory(brk_end, __pa(brk_end), uml_reserved - brk_end, 1, 1, 0); memblock_free((void *)brk_end, uml_reserved - brk_end); uml_reserved = brk_end; + min_low_pfn = PFN_UP(__pa(uml_reserved)); /* this will put all low memory onto the freelists */ memblock_free_all(); diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index afe67d8161467..e7fbf610bda86 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -98,14 +98,18 @@ void *__switch_to(struct task_struct *from, struct task_struct *to) void interrupt_end(void) { struct pt_regs *regs = ¤t->thread.regs; - - if (need_resched()) - schedule(); - if (test_thread_flag(TIF_SIGPENDING) || - test_thread_flag(TIF_NOTIFY_SIGNAL)) - do_signal(regs); - if (test_thread_flag(TIF_NOTIFY_RESUME)) - resume_user_mode_work(regs); + unsigned long thread_flags; + + thread_flags = read_thread_flags(); + while (thread_flags & _TIF_WORK_MASK) { + if (thread_flags & _TIF_NEED_RESCHED) + schedule(); + if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + if (thread_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); + thread_flags = read_thread_flags(); + } } int get_current_pid(void) diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 6d8ae86ae978f..c16b80011adaa 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -17,6 +17,122 @@ #include #include +/* + * NOTE: UML does not have exception tables. As such, this is almost a copy + * of the code in mm/memory.c, only adjusting the logic to simply check whether + * we are coming from the kernel instead of doing an additional lookup in the + * exception table. + * We can do this simplification because we never get here if the exception was + * fixable. + */ +static inline bool get_mmap_lock_carefully(struct mm_struct *mm, bool is_user) +{ + if (likely(mmap_read_trylock(mm))) + return true; + + if (!is_user) + return false; + + return !mmap_read_lock_killable(mm); +} + +static inline bool mmap_upgrade_trylock(struct mm_struct *mm) +{ + /* + * We don't have this operation yet. + * + * It should be easy enough to do: it's basically a + * atomic_long_try_cmpxchg_acquire() + * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but + * it also needs the proper lockdep magic etc. + */ + return false; +} + +static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, bool is_user) +{ + mmap_read_unlock(mm); + if (!is_user) + return false; + + return !mmap_write_lock_killable(mm); +} + +/* + * Helper for page fault handling. + * + * This is kind of equivalend to "mmap_read_lock()" followed + * by "find_extend_vma()", except it's a lot more careful about + * the locking (and will drop the lock on failure). + * + * For example, if we have a kernel bug that causes a page + * fault, we don't want to just use mmap_read_lock() to get + * the mm lock, because that would deadlock if the bug were + * to happen while we're holding the mm lock for writing. + * + * So this checks the exception tables on kernel faults in + * order to only do this all for instructions that are actually + * expected to fault. + * + * We can also actually take the mm lock for writing if we + * need to extend the vma, which helps the VM layer a lot. + */ +static struct vm_area_struct * +um_lock_mm_and_find_vma(struct mm_struct *mm, + unsigned long addr, bool is_user) +{ + struct vm_area_struct *vma; + + if (!get_mmap_lock_carefully(mm, is_user)) + return NULL; + + vma = find_vma(mm, addr); + if (likely(vma && (vma->vm_start <= addr))) + return vma; + + /* + * Well, dang. We might still be successful, but only + * if we can extend a vma to do so. + */ + if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) { + mmap_read_unlock(mm); + return NULL; + } + + /* + * We can try to upgrade the mmap lock atomically, + * in which case we can continue to use the vma + * we already looked up. + * + * Otherwise we'll have to drop the mmap lock and + * re-take it, and also look up the vma again, + * re-checking it. + */ + if (!mmap_upgrade_trylock(mm)) { + if (!upgrade_mmap_lock_carefully(mm, is_user)) + return NULL; + + vma = find_vma(mm, addr); + if (!vma) + goto fail; + if (vma->vm_start <= addr) + goto success; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto fail; + } + + if (expand_stack_locked(vma, addr)) + goto fail; + +success: + mmap_write_downgrade(mm); + return vma; + +fail: + mmap_write_unlock(mm); + return NULL; +} + /* * Note this is constrained to return 0, -EFAULT, -EACCES, -ENOMEM by * segv(). @@ -43,21 +159,10 @@ int handle_page_fault(unsigned long address, unsigned long ip, if (is_user) flags |= FAULT_FLAG_USER; retry: - mmap_read_lock(mm); - vma = find_vma(mm, address); - if (!vma) - goto out; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto out; - if (is_user && !ARCH_IS_STACKGROW(address)) - goto out; - vma = expand_stack(mm, address); + vma = um_lock_mm_and_find_vma(mm, address, is_user); if (!vma) goto out_nosemaphore; -good_area: *code_out = SEGV_ACCERR; if (is_write) { if (!(vma->vm_flags & VM_WRITE)) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index d874ea22512b5..37e22efbd1e1e 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -128,7 +128,7 @@ config X86 select ARCH_WANTS_DYNAMIC_TASK_STRUCT select ARCH_WANTS_NO_INSTR select ARCH_WANT_GENERAL_HUGETLB - select ARCH_WANT_HUGE_PMD_SHARE + select ARCH_WANT_HUGE_PMD_SHARE if X86_64 select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if X86_64 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if X86_64 @@ -2610,6 +2610,35 @@ config MITIGATION_SPECTRE_BHI indirect branches. See +config MITIGATION_ITS + bool "Enable Indirect Target Selection mitigation" + depends on CPU_SUP_INTEL && X86_64 + depends on RETPOLINE && RETHUNK + default y + help + Enable Indirect Target Selection (ITS) mitigation. ITS is a bug in + BPU on some Intel CPUs that may allow Spectre V2 style attacks. If + disabled, mitigation cannot be enabled via cmdline. + See + +config MITIGATION_TSA + bool "Mitigate Transient Scheduler Attacks" + depends on CPU_SUP_AMD + default y + help + Enable mitigation for Transient Scheduler Attacks. TSA is a hardware + security vulnerability on AMD CPUs which can lead to forwarding of + invalid info to subsequent instructions and thus can affect their + timing and thereby cause a leakage. + +config MITIGATION_VMSCAPE + bool "Mitigate VMSCAPE" + depends on KVM + default y + help + Enable mitigation for VMSCAPE attacks. VMSCAPE is a hardware security + vulnerability on Intel and AMD CPUs that may allow a guest to do + Spectre v2 style attacks on userspace hypervisor. endif config ARCH_HAS_ADD_PAGES diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 81c4bb5d399bc..15f052d582415 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -43,7 +43,7 @@ endif # How to compile the 16-bit code. Note we always compile for -march=i386; # that way we can complain to the user if the CPU is insufficient. -REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \ +REALMODE_CFLAGS := -std=gnu11 -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \ -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \ -fno-strict-aliasing -fomit-frame-pointer -fno-pic \ -mno-mmx -mno-sse $(call cc-option,-fcf-protection=none) diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 5616c3b258060..92c9f8b79f0dc 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -165,6 +165,13 @@ static void __page_state_change(unsigned long paddr, enum psc_op op) */ if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1)) sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); + + /* + * If validating memory (making it private) and affected by the + * cache-coherency vulnerability, perform the cache eviction mitigation. + */ + if (op == SNP_PAGE_STATE_PRIVATE && !has_cpuflag(X86_FEATURE_COHERENCY_SFW_NO)) + sev_evict_cache((void *)paddr, 1); } void snp_set_page_private(unsigned long paddr) diff --git a/arch/x86/boot/cpuflags.c b/arch/x86/boot/cpuflags.c index d75237ba7ce94..5660d3229d29c 100644 --- a/arch/x86/boot/cpuflags.c +++ b/arch/x86/boot/cpuflags.c @@ -115,5 +115,18 @@ void get_cpuflags(void) cpuid(0x80000001, &ignored, &ignored, &cpu.flags[6], &cpu.flags[1]); } + + if (max_amd_level >= 0x8000001f) { + u32 ebx; + + /* + * The X86_FEATURE_COHERENCY_SFW_NO feature bit is in + * the virtualization flags entry (word 8) and set by + * scattered.c, so the bit needs to be explicitly set. + */ + cpuid(0x8000001f, &ignored, &ebx, &ignored, &ignored); + if (ebx & BIT(31)) + set_bit(X86_FEATURE_COHERENCY_SFW_NO, cpu.flags); + } } } diff --git a/arch/x86/boot/genimage.sh b/arch/x86/boot/genimage.sh index c9299aeb7333e..3882ead513f74 100644 --- a/arch/x86/boot/genimage.sh +++ b/arch/x86/boot/genimage.sh @@ -22,6 +22,7 @@ # This script requires: # bash # syslinux +# genisoimage # mtools (for fdimage* and hdimage) # edk2/OVMF (for hdimage) # @@ -251,7 +252,9 @@ geniso() { cp "$isolinux" "$ldlinux" "$tmp_dir" cp "$FBZIMAGE" "$tmp_dir"/linux echo default linux "$KCMDLINE" > "$tmp_dir"/isolinux.cfg - cp "${FDINITRDS[@]}" "$tmp_dir"/ + if [ ${#FDINITRDS[@]} -gt 0 ]; then + cp "${FDINITRDS[@]}" "$tmp_dir"/ + fi genisoimage -J -r -appid 'LINUX_BOOT' -input-charset=utf-8 \ -quiet -o "$FIMAGE" -b isolinux.bin \ -c boot.cat -no-emul-boot -boot-load-size 4 \ diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 48e5f8e372056..2f3f258e48243 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -177,6 +177,7 @@ CONFIG_NF_CONNTRACK_PPTP=y CONFIG_NF_CONNTRACK_SANE=y CONFIG_NF_CONNTRACK_TFTP=y CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_XTABLES_COMPAT=y CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y CONFIG_NETFILTER_XT_TARGET_CONNMARK=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y @@ -309,6 +310,8 @@ CONFIG_REGMAP_KUNIT=m CONFIG_GNSS=y CONFIG_OF=y CONFIG_ZRAM=m +CONFIG_ZRAM_WRITEBACK=y +CONFIG_ZRAM_MULTI_COMP=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 CONFIG_BLK_DEV_RAM=y @@ -673,6 +676,7 @@ CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y CONFIG_CRYPTO_AES_NI_INTEL=y CONFIG_CRYPTO_POLYVAL_CLMUL_NI=y +CONFIG_CRYPTO_SHA1_SSSE3=y CONFIG_CRYPTO_SHA256_SSSE3=y CONFIG_CRYPTO_SHA512_SSSE3=y CONFIG_CRC_CCITT=y diff --git a/arch/x86/configs/microdroid_defconfig b/arch/x86/configs/microdroid_defconfig index d58c3dc697d0e..73ffd94325be3 100644 --- a/arch/x86/configs/microdroid_defconfig +++ b/arch/x86/configs/microdroid_defconfig @@ -14,18 +14,13 @@ CONFIG_UCLAMP_TASK=y CONFIG_UCLAMP_BUCKETS_COUNT=20 CONFIG_CGROUPS=y CONFIG_MEMCG=y -CONFIG_BLK_CGROUP=y -CONFIG_CGROUP_SCHED=y -CONFIG_UCLAMP_TASK_GROUP=y -CONFIG_CGROUP_FREEZER=y -CONFIG_CPUSETS=y -CONFIG_CGROUP_CPUACCT=y # CONFIG_RD_BZIP2 is not set # CONFIG_RD_LZMA is not set # CONFIG_RD_XZ is not set # CONFIG_RD_LZO is not set CONFIG_BOOT_CONFIG=y CONFIG_EXPERT=y +# CONFIG_BASE_FULL is not set CONFIG_PROFILING=y CONFIG_KEXEC_FILE=y CONFIG_SMP=y @@ -38,16 +33,14 @@ CONFIG_NR_CPUS=32 CONFIG_EFI=y CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="stack_depot_disable=on cgroup_disable=pressure ioremap_guard panic=-1 bootconfig acpi=noirq" -CONFIG_PM_WAKELOCKS=y -CONFIG_PM_WAKELOCKS_LIMIT=0 -# CONFIG_PM_WAKELOCKS_GC is not set +# CONFIG_SUSPEND is not set CONFIG_CPU_FREQ_STAT=y CONFIG_CPU_FREQ_TIMES=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y CONFIG_JUMP_LABEL=y +# CONFIG_SECCOMP is not set # CONFIG_BLOCK_LEGACY_AUTOLOAD is not set -CONFIG_BLK_CGROUP_IOCOST=y CONFIG_PARTITION_ADVANCED=y # CONFIG_MSDOS_PARTITION is not set # CONFIG_MQ_IOSCHED_DEADLINE is not set @@ -60,8 +53,7 @@ CONFIG_SLAB_FREELIST_HARDENED=y CONFIG_SHUFFLE_PAGE_ALLOCATOR=y # CONFIG_COMPAT_BRK is not set CONFIG_DEFAULT_MMAP_MIN_ADDR=32768 -CONFIG_TRANSPARENT_HUGEPAGE=y -CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y +# CONFIG_ZONE_DMA is not set CONFIG_ANON_VMA_NAME=y CONFIG_USERFAULTFD=y CONFIG_LRU_GEN=y @@ -71,9 +63,20 @@ CONFIG_DAMON_RECLAIM=y CONFIG_NET=y CONFIG_UNIX=y CONFIG_INET=y +CONFIG_INET_TABLE_PERTURB_ORDER=0 +# CONFIG_INET_DIAG is not set +CONFIG_TCP_CONG_ADVANCED=y +# CONFIG_TCP_CONG_BIC is not set +# CONFIG_TCP_CONG_CUBIC is not set +# CONFIG_TCP_CONG_WESTWOOD is not set +# CONFIG_TCP_CONG_HTCP is not set +# CONFIG_IPV6 is not set CONFIG_VSOCKETS=y CONFIG_VIRTIO_VSOCKETS=y +# CONFIG_RPS is not set +# CONFIG_BQL is not set # CONFIG_WIRELESS is not set +# CONFIG_ETHTOOL_NETLINK is not set CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCIEAER=y @@ -81,29 +84,32 @@ CONFIG_PCI_MSI=y CONFIG_PCI_IOV=y # CONFIG_VGA_ARB is not set CONFIG_PCI_ENDPOINT=y -CONFIG_FW_LOADER_USER_HELPER=y -# CONFIG_FW_CACHE is not set +# CONFIG_FW_LOADER is not set CONFIG_OF=y CONFIG_ZRAM=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_LOOP_MIN_COUNT=16 -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=8192 CONFIG_VIRTIO_BLK=y CONFIG_MD=y +# CONFIG_MD_BITMAP_FILE is not set CONFIG_BLK_DEV_DM=y CONFIG_DM_CRYPT=y CONFIG_DM_UEVENT=y CONFIG_DM_VERITY=y CONFIG_DM_VERITY_FEC=y +# CONFIG_DM_USER is not set # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set # CONFIG_LEGACY_PTYS is not set CONFIG_SERIAL_8250=y # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +# CONFIG_SERIAL_8250_PNP is not set CONFIG_SERIAL_8250_CONSOLE=y # CONFIG_SERIAL_8250_PCI is not set CONFIG_SERIAL_8250_RUNTIME_UARTS=0 +# CONFIG_SERIAL_8250_LPSS is not set +# CONFIG_SERIAL_8250_MID is not set +# CONFIG_SERIAL_8250_PERICOM is not set CONFIG_SERIAL_OF_PLATFORM=y CONFIG_NULL_TTY=y CONFIG_VIRTIO_CONSOLE=y @@ -129,11 +135,12 @@ CONFIG_MFD_SYSCON=y # CONFIG_USB_SUPPORT is not set CONFIG_EDAC=y CONFIG_RTC_CLASS=y -CONFIG_DMABUF_HEAPS=y -CONFIG_DMABUF_SYSFS_STATS=y CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_BALLOON=y CONFIG_STAGING=y +# CONFIG_IOMMU_SUPPORT is not set +# CONFIG_ANDROID_KABI_RESERVE is not set +# CONFIG_ANDROID_VENDOR_OEM_DATA is not set CONFIG_LIBNVDIMM=y CONFIG_EXT4_FS=y # CONFIG_EXT4_USE_FOR_EXT2 is not set @@ -207,12 +214,13 @@ CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y CONFIG_BUG_ON_DATA_CORRUPTION=y CONFIG_CRYPTO_HCTR2=y CONFIG_CRYPTO_LZO=y +CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_AES_NI_INTEL=y CONFIG_CRYPTO_POLYVAL_CLMUL_NI=y +CONFIG_CRYPTO_SHA1_SSSE3=y CONFIG_CRYPTO_SHA256_SSSE3=y CONFIG_CRYPTO_SHA512_SSSE3=y CONFIG_PRINTK_TIME=y -CONFIG_DYNAMIC_DEBUG_CORE=y CONFIG_DEBUG_INFO_DWARF5=y CONFIG_DEBUG_INFO_REDUCED=y CONFIG_HEADERS_INSTALL=y @@ -233,5 +241,6 @@ CONFIG_PANIC_TIMEOUT=-1 CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_WQ_WATCHDOG=y CONFIG_SCHEDSTATS=y +# CONFIG_UPROBE_EVENTS is not set CONFIG_HIST_TRIGGERS=y CONFIG_UNWINDER_FRAME_POINTER=y diff --git a/arch/x86/entry/entry.S b/arch/x86/entry/entry.S index 78fd2442b49dc..4e7ecffee762a 100644 --- a/arch/x86/entry/entry.S +++ b/arch/x86/entry/entry.S @@ -31,20 +31,20 @@ EXPORT_SYMBOL_GPL(entry_ibpb); /* * Define the VERW operand that is disguised as entry code so that - * it can be referenced with KPTI enabled. This ensure VERW can be + * it can be referenced with KPTI enabled. This ensures VERW can be * used late in exit-to-user path after page tables are switched. */ .pushsection .entry.text, "ax" .align L1_CACHE_BYTES, 0xcc -SYM_CODE_START_NOALIGN(mds_verw_sel) +SYM_CODE_START_NOALIGN(x86_verw_sel) UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR .word __KERNEL_DS .align L1_CACHE_BYTES, 0xcc -SYM_CODE_END(mds_verw_sel); +SYM_CODE_END(x86_verw_sel); /* For KVM */ -EXPORT_SYMBOL_GPL(mds_verw_sel); +EXPORT_SYMBOL_GPL(x86_verw_sel); .popsection @@ -59,7 +59,7 @@ EXPORT_SYMBOL_GPL(mds_verw_sel); * entirely in the C code, and use an alias emitted by the linker script * instead. */ -#ifdef CONFIG_STACKPROTECTOR +#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP) EXPORT_SYMBOL(__ref_stack_chk_guard); #endif #endif diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 2192b6c33ea00..1f9e508ac075c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1569,7 +1569,9 @@ SYM_CODE_END(rewind_stack_and_make_dead) * ORC to unwind properly. * * The alignment is for performance and not for safety, and may be safely - * refactored in the future if needed. + * refactored in the future if needed. The .skips are for safety, to ensure + * that all RETs are in the second half of a cacheline to mitigate Indirect + * Target Selection, rather than taking the slowpath via its_return_thunk. */ SYM_FUNC_START(clear_bhb_loop) push %rbp @@ -1579,10 +1581,22 @@ SYM_FUNC_START(clear_bhb_loop) call 1f jmp 5f .align 64, 0xcc + /* + * Shift instructions so that the RET is in the upper half of the + * cacheline and don't take the slowpath to its_return_thunk. + */ + .skip 32 - (.Lret1 - 1f), 0xcc ANNOTATE_INTRA_FUNCTION_CALL 1: call 2f - RET +.Lret1: RET .align 64, 0xcc + /* + * As above shift instructions for RET at .Lret2 as well. + * + * This should be ideally be: .skip 32 - (.Lret2 - 2f), 0xcc + * but some Clang versions (e.g. 18) don't like this. + */ + .skip 32 - 18, 0xcc 2: movl $5, %eax 3: jmp 4f nop @@ -1590,7 +1604,7 @@ SYM_FUNC_START(clear_bhb_loop) jnz 3b sub $1, %ecx jnz 1b - RET +.Lret2: RET 5: lfence pop %rbp RET diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index 5949fc117d9ab..9f545b315f99b 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -125,7 +125,12 @@ bool emulate_vsyscall(unsigned long error_code, if ((error_code & (X86_PF_WRITE | X86_PF_USER)) != X86_PF_USER) return false; - if (!(error_code & X86_PF_INSTR)) { + /* + * Assume that faults at regs->ip are because of an + * instruction fetch. Return early and avoid + * emulation for faults during data accesses: + */ + if (address != regs->ip) { /* Failed vsyscall read */ if (vsyscall_mode == EMULATE) return false; @@ -137,13 +142,19 @@ bool emulate_vsyscall(unsigned long error_code, return false; } + /* + * X86_PF_INSTR is only set when NX is supported. When + * available, use it to double-check that the emulation code + * is only being used for instruction fetches: + */ + if (cpu_feature_enabled(X86_FEATURE_NX)) + WARN_ON_ONCE(!(error_code & X86_PF_INSTR)); + /* * No point in checking CS -- the only way to get here is a user mode * trap to a high address, which means that we're in 64-bit user code. */ - WARN_ON_ONCE(address != regs->ip); - if (vsyscall_mode == NONE) { warn_bad_vsyscall(KERN_INFO, regs, "vsyscall attempted with vsyscall=none"); diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index f483874fa20f1..fac3d97111b09 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -272,7 +272,7 @@ static int perf_ibs_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; struct perf_ibs *perf_ibs; - u64 max_cnt, config; + u64 config; int ret; perf_ibs = get_ibs_pmu(event->attr.type); @@ -306,10 +306,19 @@ static int perf_ibs_init(struct perf_event *event) if (!hwc->sample_period) hwc->sample_period = 0x10; } else { - max_cnt = config & perf_ibs->cnt_mask; + u64 period = 0; + + if (perf_ibs == &perf_ibs_op) { + period = (config & IBS_OP_MAX_CNT) << 4; + if (ibs_caps & IBS_CAPS_OPCNTEXT) + period |= config & IBS_OP_MAX_CNT_EXT_MASK; + } else { + period = (config & IBS_FETCH_MAX_CNT) << 4; + } + config &= ~perf_ibs->cnt_mask; - event->attr.sample_period = max_cnt << 4; - hwc->sample_period = event->attr.sample_period; + event->attr.sample_period = period; + hwc->sample_period = period; } if (!hwc->sample_period) @@ -1219,7 +1228,8 @@ static __init int perf_ibs_op_init(void) if (ibs_caps & IBS_CAPS_OPCNTEXT) { perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK; perf_ibs_op.config_mask |= IBS_OP_MAX_CNT_EXT_MASK; - perf_ibs_op.cnt_mask |= IBS_OP_MAX_CNT_EXT_MASK; + perf_ibs_op.cnt_mask |= (IBS_OP_MAX_CNT_EXT_MASK | + IBS_OP_CUR_CNT_EXT_MASK); } if (ibs_caps & IBS_CAPS_ZEN4) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 66d5782df18f8..87ea4339e03a2 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2734,7 +2734,7 @@ static void intel_pmu_read_event(struct perf_event *event) if (pmu_enabled) intel_pmu_disable_all(); - if (is_topdown_event(event)) + if (is_topdown_count(event)) static_call(intel_pmu_update_topdown_event)(event); else intel_pmu_drain_pebs_buffer(); @@ -4206,7 +4206,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) arr[pebs_enable] = (struct perf_guest_switch_msr){ .msr = MSR_IA32_PEBS_ENABLE, .host = cpuc->pebs_enabled & ~cpuc->intel_ctrl_guest_mask, - .guest = pebs_mask & ~cpuc->intel_ctrl_host_mask, + .guest = pebs_mask & ~cpuc->intel_ctrl_host_mask & kvm_pmu->pebs_enable, }; if (arr[pebs_enable].host) { diff --git a/arch/x86/hyperv/irqdomain.c b/arch/x86/hyperv/irqdomain.c index 42c70d28ef272..865ae4be233b3 100644 --- a/arch/x86/hyperv/irqdomain.c +++ b/arch/x86/hyperv/irqdomain.c @@ -192,7 +192,6 @@ static void hv_irq_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) struct pci_dev *dev; struct hv_interrupt_entry out_entry, *stored_entry; struct irq_cfg *cfg = irqd_cfg(data); - const cpumask_t *affinity; int cpu; u64 status; @@ -204,8 +203,7 @@ static void hv_irq_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) return; } - affinity = irq_data_get_effective_affinity_mask(data); - cpu = cpumask_first_and(affinity, cpu_online_mask); + cpu = cpumask_first(irq_data_get_effective_affinity_mask(data)); if (data->chip_data) { /* diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index cb9ce0f9e78e0..6740b839153a0 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -5,6 +5,7 @@ #include #include #include +#include #define ALT_FLAGS_SHIFT 16 @@ -130,6 +131,37 @@ static __always_inline int x86_call_depth_emit_accounting(u8 **pprog, } #endif +#ifdef CONFIG_MITIGATION_ITS +extern void its_init_mod(struct module *mod); +extern void its_fini_mod(struct module *mod); +extern void its_free_mod(struct module *mod); +extern u8 *its_static_thunk(int reg); +#else /* CONFIG_MITIGATION_ITS */ +static inline void its_init_mod(struct module *mod) { } +static inline void its_fini_mod(struct module *mod) { } +static inline void its_free_mod(struct module *mod) { } +static inline u8 *its_static_thunk(int reg) +{ + WARN_ONCE(1, "ITS not compiled in"); + + return NULL; +} +#endif + +#if defined(CONFIG_RETHUNK) && defined(CONFIG_OBJTOOL) +extern bool cpu_wants_rethunk(void); +extern bool cpu_wants_rethunk_at(void *addr); +#else +static __always_inline bool cpu_wants_rethunk(void) +{ + return false; +} +static __always_inline bool cpu_wants_rethunk_at(void *addr) +{ + return false; +} +#endif + #ifdef CONFIG_SMP extern void alternatives_smp_module_add(struct module *mod, char *name, void *locks, void *locks_end, diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index 806649c7f23dc..9a0f29be1a9ea 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -22,8 +22,9 @@ #define SECOND_BYTE_OPCODE_UD2 0x0b #define BUG_NONE 0xffff -#define BUG_UD1 0xfffe -#define BUG_UD2 0xfffd +#define BUG_UD2 0xfffe +#define BUG_UD1 0xfffd +#define BUG_UD1_UBSAN 0xfffc #ifdef CONFIG_GENERIC_BUG diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index 58dacd90daefc..894afeb158fc8 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -7,11 +7,125 @@ * * Copyright (C) 2022 Google LLC */ +#include +#include -#include +/* + * An overview of the various calling conventions... + * + * Traditional: + * + * foo: + * ... code here ... + * ret + * + * direct caller: + * call foo + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * call *%r11 + * + * + * IBT: + * + * foo: + * endbr64 + * ... code here ... + * ret + * + * direct caller: + * call foo / call foo+4 + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * call *%r11 + * + * + * kCFI: + * + * __cfi_foo: + * movl $0x12345678, %eax + * # 11 nops when CONFIG_CALL_PADDING + * foo: + * endbr64 # when IBT + * ... code here ... + * ret + * + * direct call: + * call foo # / call foo+4 when IBT + * + * indirect call: + * lea foo(%rip), %r11 + * ... + * movl $(-0x12345678), %r10d + * addl -4(%r11), %r10d # -15 when CONFIG_CALL_PADDING + * jz 1f + * ud2 + * 1:call *%r11 + * + * + * FineIBT (builds as kCFI + CALL_PADDING + IBT + RETPOLINE and runtime patches into): + * + * __cfi_foo: + * endbr64 + * subl 0x12345678, %r10d + * jz foo + * ud2 + * nop + * foo: + * osp nop3 # was endbr64 + * ... code here ... + * ret + * + * direct caller: + * call foo / call foo+4 + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * movl $0x12345678, %r10d + * subl $16, %r11 + * nop4 + * call *%r11 + * + */ +enum cfi_mode { + CFI_DEFAULT, /* FineIBT if hardware has IBT, otherwise kCFI */ + CFI_OFF, /* Taditional / IBT depending on .config */ + CFI_KCFI, /* Optionally CALL_PADDING, IBT, RETPOLINE */ + CFI_FINEIBT, /* see arch/x86/kernel/alternative.c */ +}; + +extern enum cfi_mode cfi_mode; + +struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); +#define __bpfcall + +static inline int cfi_get_offset(void) +{ + switch (cfi_mode) { + case CFI_FINEIBT: + return 16; + case CFI_KCFI: + if (IS_ENABLED(CONFIG_CALL_PADDING)) + return 16; + return 5; + default: + return 0; + } +} +#define cfi_get_offset cfi_get_offset + +extern u32 cfi_get_func_hash(void *func); +#define cfi_get_func_hash cfi_get_func_hash + + #else static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { @@ -19,4 +133,8 @@ static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) } #endif /* CONFIG_CFI_CLANG */ +#if HAS_KERNEL_IBT == 1 +#define CFI_NOSEAL(x) asm(IBT_NOSEAL(__stringify(x))) +#endif + #endif /* _ASM_X86_CFI_H */ diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h index fecc4fe1d68af..9c67f8b4c9197 100644 --- a/arch/x86/include/asm/cpu.h +++ b/arch/x86/include/asm/cpu.h @@ -81,4 +81,16 @@ int intel_microcode_sanity_check(void *mc, bool print_err, int hdr_type); extern struct cpumask cpus_stop_mask; +union zen_patch_rev { + struct { + __u32 rev : 8, + stepping : 4, + model : 4, + __reserved : 4, + ext_model : 4, + ext_fam : 8; + }; + __u32 ucode_rev; +}; + #endif /* _ASM_X86_CPU_H */ diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 55d18eef6775a..ae4ea1f9594f7 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -228,6 +228,7 @@ #define X86_FEATURE_FLEXPRIORITY ( 8*32+ 1) /* Intel FlexPriority */ #define X86_FEATURE_EPT ( 8*32+ 2) /* Intel Extended Page Table */ #define X86_FEATURE_VPID ( 8*32+ 3) /* Intel Virtual Processor ID */ +#define X86_FEATURE_COHERENCY_SFW_NO ( 8*32+ 4) /* "" SNP cache coherency software work around not needed */ #define X86_FEATURE_VMMCALL ( 8*32+15) /* Prefer VMMCALL to VMCALL */ #define X86_FEATURE_XENPV ( 8*32+16) /* "" Xen paravirtual guest */ @@ -449,6 +450,7 @@ /* AMD-defined Extended Feature 2 EAX, CPUID level 0x80000021 (EAX), word 20 */ #define X86_FEATURE_NO_NESTED_DATA_BP (20*32+ 0) /* "" No Nested Data Breakpoints */ #define X86_FEATURE_LFENCE_RDTSC (20*32+ 2) /* "" LFENCE always serializing / synchronizes RDTSC */ +#define X86_FEATURE_VERW_CLEAR (20*32+ 5) /* "" The memory form of VERW mitigates TSA */ #define X86_FEATURE_NULL_SEL_CLR_BASE (20*32+ 6) /* "" Null Selector Clears Base */ #define X86_FEATURE_AUTOIBRS (20*32+ 8) /* "" Automatic IBRS */ #define X86_FEATURE_NO_SMM_CTL_MSR (20*32+ 9) /* "" SMM_CTL MSR is not present */ @@ -468,6 +470,12 @@ #define X86_FEATURE_BHI_CTRL (21*32+ 2) /* "" BHI_DIS_S HW control available */ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* "" BHI_DIS_S HW control enabled */ #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */ +#define X86_FEATURE_INDIRECT_THUNK_ITS (21*32 + 5) /* "" Use thunk for indirect branches in lower half of cacheline */ + +#define X86_FEATURE_TSA_SQ_NO (21*32+11) /* "" AMD CPU not vulnerable to TSA-SQ */ +#define X86_FEATURE_TSA_L1_NO (21*32+12) /* "" AMD CPU not vulnerable to TSA-L1 */ +#define X86_FEATURE_CLEAR_CPU_BUF_VM (21*32+13) /* "" Clear CPU buffers using VERW before VMRUN */ +#define X86_FEATURE_IBPB_EXIT_TO_USER (21*32+14) /* Use IBPB on exit-to-userspace, see VMSCAPE bug */ /* * BUG word(s) @@ -518,4 +526,8 @@ #define X86_BUG_RFDS X86_BUG(1*32 + 2) /* CPU is vulnerable to Register File Data Sampling */ #define X86_BUG_BHI X86_BUG(1*32 + 3) /* CPU is affected by Branch History Injection */ #define X86_BUG_IBPB_NO_RET X86_BUG(1*32 + 4) /* "ibpb_no_ret" IBPB omits return target predictions */ +#define X86_BUG_ITS X86_BUG(1*32 + 5) /* CPU is affected by Indirect Target Selection */ +#define X86_BUG_ITS_NATIVE_ONLY X86_BUG(1*32 + 6) /* CPU is affected by ITS, VMX is not affected */ +#define X86_BUG_TSA X86_BUG(1*32+ 9) /* "tsa" CPU is affected by Transient Scheduler Attacks */ +#define X86_BUG_VMSCAPE X86_BUG( 1*32+10) /* "vmscape" CPU is affected by VMSCAPE attacks from guests */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h index fb2809b20b0ac..bb0a5ecc807fe 100644 --- a/arch/x86/include/asm/entry-common.h +++ b/arch/x86/include/asm/entry-common.h @@ -83,6 +83,13 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, * 8 (ia32) bits. */ choose_random_kstack_offset(rdtsc()); + + /* Avoid unnecessary reads of 'x86_ibpb_exit_to_user' */ + if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) && + this_cpu_read(x86_ibpb_exit_to_user)) { + indirect_branch_prediction_barrier(); + this_cpu_write(x86_ibpb_exit_to_user, false); + } } #define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare diff --git a/arch/x86/include/asm/ibt.h b/arch/x86/include/asm/ibt.h index 1e59581d500ca..b778ae6e67ee8 100644 --- a/arch/x86/include/asm/ibt.h +++ b/arch/x86/include/asm/ibt.h @@ -41,7 +41,7 @@ _ASM_PTR fname "\n\t" \ ".popsection\n\t" -static inline __attribute_const__ u32 gen_endbr(void) +static __always_inline __attribute_const__ u32 gen_endbr(void) { u32 endbr; @@ -56,7 +56,7 @@ static inline __attribute_const__ u32 gen_endbr(void) return endbr; } -static inline __attribute_const__ u32 gen_endbr_poison(void) +static __always_inline __attribute_const__ u32 gen_endbr_poison(void) { /* * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index 9acfe2bcf1fd5..9bfb7b90e2990 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -44,13 +44,13 @@ static __always_inline void native_irq_enable(void) static __always_inline void native_safe_halt(void) { - mds_idle_clear_cpu_buffers(); + x86_idle_clear_cpu_buffers(); asm volatile("sti; hlt": : :"memory"); } static __always_inline void native_halt(void) { - mds_idle_clear_cpu_buffers(); + x86_idle_clear_cpu_buffers(); asm volatile("hlt": : :"memory"); } diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 9b419f0de713c..a0a4fc684e63b 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -101,7 +101,6 @@ KVM_X86_OP(write_tsc_multiplier) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff) -KVM_X86_OP(request_immediate_exit) KVM_X86_OP(sched_in) KVM_X86_OP_OPTIONAL(update_cpu_dirty_logging) KVM_X86_OP_OPTIONAL(vcpu_blocking) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 39672561c6be8..813887324d52d 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -733,6 +733,7 @@ struct kvm_vcpu_arch { u32 pkru; u32 hflags; u64 efer; + u64 host_debugctl; u64 apic_base; struct kvm_lapic *apic; /* kernel irqchip context */ bool load_eoi_exitmap_pending; @@ -1549,6 +1550,12 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL; } +enum kvm_x86_run_flags { + KVM_RUN_FORCE_IMMEDIATE_EXIT = BIT(0), + KVM_RUN_LOAD_GUEST_DR6 = BIT(1), + KVM_RUN_LOAD_DEBUGCTL = BIT(2), +}; + struct kvm_x86_ops { const char *name; @@ -1574,6 +1581,12 @@ struct kvm_x86_ops { void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); void (*vcpu_put)(struct kvm_vcpu *vcpu); + /* + * Mask of DEBUGCTL bits that are owned by the host, i.e. that need to + * match the host's value even while the guest is active. + */ + const u64 HOST_OWNED_DEBUGCTL; + void (*update_exception_bitmap)(struct kvm_vcpu *vcpu); int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr); @@ -1622,7 +1635,8 @@ struct kvm_x86_ops { void (*flush_tlb_guest)(struct kvm_vcpu *vcpu); int (*vcpu_pre_run)(struct kvm_vcpu *vcpu); - enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu); + enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu, + u64 run_flags); int (*handle_exit)(struct kvm_vcpu *vcpu, enum exit_fastpath_completion exit_fastpath); int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); @@ -1656,7 +1670,7 @@ struct kvm_x86_ops { bool allow_apicv_in_x2apic_without_x2apic_virtualization; void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr); - void (*hwapic_isr_update)(int isr); + void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu); void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); @@ -1692,8 +1706,6 @@ struct kvm_x86_ops { struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); - void (*request_immediate_exit)(struct kvm_vcpu *vcpu); - void (*sched_in)(struct kvm_vcpu *kvm, int cpu); /* @@ -2179,7 +2191,6 @@ extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); -void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu); void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size); diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 0ee6ed0ff2bf2..79c35947e20cb 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -17,10 +17,12 @@ struct ucode_cpu_info { void load_ucode_bsp(void); void load_ucode_ap(void); void microcode_bsp_resume(void); +bool __init microcode_loader_disabled(void); #else static inline void load_ucode_bsp(void) { } static inline void load_ucode_ap(void) { } static inline void microcode_bsp_resume(void) { } +static inline bool __init microcode_loader_disabled(void) { return false; } #endif extern unsigned long initrd_start_early; diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 623bb48774d44..425980eacaa84 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -180,6 +180,14 @@ * VERW clears CPU Register * File. */ +#define ARCH_CAP_ITS_NO BIT_ULL(62) /* + * Not susceptible to + * Indirect Target Selection. + * This bit is not set by + * HW, but is synthesized by + * VMMs for guests to know + * their affected status. + */ #define ARCH_CAP_XAPIC_DISABLE BIT(21) /* * IA32_XAPIC_DISABLE_STATUS MSR @@ -372,6 +380,7 @@ #define DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI (1UL << 12) #define DEBUGCTLMSR_FREEZE_IN_SMM_BIT 14 #define DEBUGCTLMSR_FREEZE_IN_SMM (1UL << DEBUGCTLMSR_FREEZE_IN_SMM_BIT) +#define DEBUGCTLMSR_RTM_DEBUG BIT(15) #define MSR_PEBS_FRONTEND 0x000003f7 @@ -567,6 +576,7 @@ #define MSR_AMD64_OSVW_STATUS 0xc0010141 #define MSR_AMD_PPIN_CTL 0xc00102f0 #define MSR_AMD_PPIN 0xc00102f1 +#define MSR_AMD64_CPUID_FN_7 0xc0011002 #define MSR_AMD64_CPUID_FN_1 0xc0011004 #define MSR_AMD64_LS_CFG 0xc0011020 #define MSR_AMD64_DC_CFG 0xc0011022 @@ -651,6 +661,7 @@ #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 +#define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET 0xc0000303 /* AMD Last Branch Record MSRs */ #define MSR_AMD64_LBR_SELECT 0xc000010e diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index bae83810505bf..ae7a83e3f743e 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -44,8 +44,6 @@ static __always_inline void __monitorx(const void *eax, unsigned long ecx, static __always_inline void __mwait(unsigned long eax, unsigned long ecx) { - mds_idle_clear_cpu_buffers(); - /* "mwait %eax, %ecx;" */ asm volatile(".byte 0x0f, 0x01, 0xc9;" :: "a" (eax), "c" (ecx)); @@ -80,7 +78,7 @@ static __always_inline void __mwait(unsigned long eax, unsigned long ecx) static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx, unsigned long ecx) { - /* No MDS buffer clear as this is AMD/HYGON only */ + /* No need for TSA buffer clearing on AMD */ /* "mwaitx %eax, %ebx, %ecx;" */ asm volatile(".byte 0x0f, 0x01, 0xfb;" @@ -89,7 +87,7 @@ static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx, static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx) { - mds_idle_clear_cpu_buffers(); + /* "mwait %eax, %ecx;" */ asm volatile("sti; .byte 0x0f, 0x01, 0xc9;" :: "a" (eax), "c" (ecx)); @@ -107,24 +105,29 @@ static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx) */ static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) { + if (need_resched()) + return; + + x86_idle_clear_cpu_buffers(); + if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) { - if (static_cpu_has_bug(X86_BUG_CLFLUSH_MONITOR)) { - mb(); - clflush((void *)¤t_thread_info()->flags); - mb(); - } + const void *addr = ¤t_thread_info()->flags; + + alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr)); + __monitor(addr, 0, 0); - __monitor((void *)¤t_thread_info()->flags, 0, 0); + if (need_resched()) + goto out; - if (!need_resched()) { - if (ecx & 1) { - __mwait(eax, ecx); - } else { - __sti_mwait(eax, ecx); - raw_local_irq_disable(); - } + if (ecx & 1) { + __mwait(eax, ecx); + } else { + __sti_mwait(eax, ecx); + raw_local_irq_disable(); } } + +out: current_clr_polling(); } diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h index 5c5f1e56c4048..6f3d145670a95 100644 --- a/arch/x86/include/asm/nmi.h +++ b/arch/x86/include/asm/nmi.h @@ -59,6 +59,8 @@ int __register_nmi_handler(unsigned int, struct nmiaction *); void unregister_nmi_handler(unsigned int, const char *); +void set_emergency_nmi_handler(unsigned int type, nmi_handler_t handler); + void stop_nmi(void); void restart_nmi(void); void local_touch_nmi(void); diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index ee642d26e3045..fb469ace38393 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -219,9 +219,8 @@ .endm /* - * Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call - * to the retpoline thunk with a CS prefix when the register requires - * a RAX prefix byte to encode. Also see apply_retpolines(). + * Emits a conditional CS prefix that is compatible with + * -mindirect-branch-cs-prefix. */ .macro __CS_PREFIX reg:req .irp rs,r8,r9,r10,r11,r12,r13,r14,r15 @@ -325,25 +324,31 @@ .endm /* - * Macro to execute VERW instruction that mitigate transient data sampling - * attacks such as MDS. On affected systems a microcode update overloaded VERW - * instruction to also clear the CPU buffers. VERW clobbers CFLAGS.ZF. - * + * Macro to execute VERW insns that mitigate transient data sampling + * attacks such as MDS or TSA. On affected systems a microcode update + * overloaded VERW insns to also clear the CPU buffers. VERW clobbers + * CFLAGS.ZF. * Note: Only the memory operand variant of VERW clears the CPU buffers. */ -.macro CLEAR_CPU_BUFFERS +.macro __CLEAR_CPU_BUFFERS feature #ifdef CONFIG_X86_64 - ALTERNATIVE "", "verw mds_verw_sel(%rip)", X86_FEATURE_CLEAR_CPU_BUF + ALTERNATIVE "", "verw x86_verw_sel(%rip)", \feature #else /* * In 32bit mode, the memory operand must be a %cs reference. The data * segments may not be usable (vm86 mode), and the stack segment may not * be flat (ESPFIX32). */ - ALTERNATIVE "", "verw %cs:mds_verw_sel", X86_FEATURE_CLEAR_CPU_BUF + ALTERNATIVE "", "verw %cs:x86_verw_sel", \feature #endif .endm +#define CLEAR_CPU_BUFFERS \ + __CLEAR_CPU_BUFFERS X86_FEATURE_CLEAR_CPU_BUF + +#define VM_CLEAR_CPU_BUFFERS \ + __CLEAR_CPU_BUFFERS X86_FEATURE_CLEAR_CPU_BUF_VM + #ifdef CONFIG_X86_64 .macro CLEAR_BRANCH_HISTORY ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_LOOP @@ -365,10 +370,14 @@ ".long 999b\n\t" \ ".popsection\n\t" +#define ITS_THUNK_SIZE 64 + typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; +typedef u8 its_thunk_t[ITS_THUNK_SIZE]; extern retpoline_thunk_t __x86_indirect_thunk_array[]; extern retpoline_thunk_t __x86_indirect_call_thunk_array[]; extern retpoline_thunk_t __x86_indirect_jump_thunk_array[]; +extern its_thunk_t __x86_indirect_its_thunk_array[]; #ifdef CONFIG_RETHUNK extern void __x86_return_thunk(void); @@ -392,6 +401,12 @@ static inline void srso_return_thunk(void) {} static inline void srso_alias_return_thunk(void) {} #endif +#ifdef CONFIG_MITIGATION_ITS +extern void its_return_thunk(void); +#else +static inline void its_return_thunk(void) {} +#endif + extern void retbleed_return_thunk(void); extern void srso_return_thunk(void); extern void srso_alias_return_thunk(void); @@ -412,11 +427,6 @@ extern void (*x86_return_thunk)(void); #ifdef CONFIG_CALL_DEPTH_TRACKING extern void __x86_return_skl(void); -static inline void x86_set_skl_return_thunk(void) -{ - x86_return_thunk = &__x86_return_skl; -} - #define CALL_DEPTH_ACCOUNT \ ALTERNATIVE("", \ __stringify(INCREMENT_CALL_DEPTH), \ @@ -429,7 +439,6 @@ DECLARE_PER_CPU(u64, __x86_stuffs_count); DECLARE_PER_CPU(u64, __x86_ctxsw_count); #endif #else -static inline void x86_set_skl_return_thunk(void) {} #define CALL_DEPTH_ACCOUNT "" @@ -454,20 +463,23 @@ static inline void x86_set_skl_return_thunk(void) {} #ifdef CONFIG_X86_64 +/* + * Emits a conditional CS prefix that is compatible with + * -mindirect-branch-cs-prefix. + */ +#define __CS_PREFIX(reg) \ + ".irp rs,r8,r9,r10,r11,r12,r13,r14,r15\n" \ + ".ifc \\rs," reg "\n" \ + ".byte 0x2e\n" \ + ".endif\n" \ + ".endr\n" + /* * Inline asm uses the %V modifier which is only in newer GCC * which is ensured when CONFIG_RETPOLINE is defined. */ -# define CALL_NOSPEC \ - ALTERNATIVE_2( \ - ANNOTATE_RETPOLINE_SAFE \ - "call *%[thunk_target]\n", \ - "call __x86_indirect_thunk_%V[thunk_target]\n", \ - X86_FEATURE_RETPOLINE, \ - "lfence;\n" \ - ANNOTATE_RETPOLINE_SAFE \ - "call *%[thunk_target]\n", \ - X86_FEATURE_RETPOLINE_LFENCE) +#define CALL_NOSPEC __CS_PREFIX("%V[thunk_target]") \ + "call __x86_indirect_thunk_%V[thunk_target]\n" # define THUNK_TARGET(addr) [thunk_target] "r" (addr) @@ -547,6 +559,8 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) extern u64 x86_pred_cmd; +DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user); + static inline void indirect_branch_prediction_barrier(void) { alternative_msr_write(MSR_IA32_PRED_CMD, x86_pred_cmd, X86_FEATURE_USE_IBPB); @@ -586,24 +600,24 @@ DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); -DECLARE_STATIC_KEY_FALSE(mds_idle_clear); +DECLARE_STATIC_KEY_FALSE(cpu_buf_idle_clear); DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear); -extern u16 mds_verw_sel; +extern u16 x86_verw_sel; #include /** - * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability + * x86_clear_cpu_buffers - Buffer clearing support for different x86 CPU vulns * * This uses the otherwise unused and obsolete VERW instruction in * combination with microcode which triggers a CPU buffer flush when the * instruction is executed. */ -static __always_inline void mds_clear_cpu_buffers(void) +static __always_inline void x86_clear_cpu_buffers(void) { static const u16 ds = __KERNEL_DS; @@ -620,14 +634,15 @@ static __always_inline void mds_clear_cpu_buffers(void) } /** - * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability + * x86_idle_clear_cpu_buffers - Buffer clearing support in idle for the MDS + * and TSA vulnerabilities. * * Clear CPU buffers if the corresponding static key is enabled */ -static __always_inline void mds_idle_clear_cpu_buffers(void) +static __always_inline void x86_idle_clear_cpu_buffers(void) { - if (static_branch_likely(&mds_idle_clear)) - mds_clear_cpu_buffers(); + if (static_branch_likely(&cpu_buf_idle_clear)) + x86_clear_cpu_buffers(); } #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 384e8a7db4827..ba2a3935dc624 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -501,6 +501,7 @@ struct pebs_xmm { */ #define IBS_OP_CUR_CNT (0xFFF80ULL<<32) #define IBS_OP_CUR_CNT_RAND (0x0007FULL<<32) +#define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL<<52) #define IBS_OP_CNT_CTL (1ULL<<19) #define IBS_OP_VAL (1ULL<<18) #define IBS_OP_ENABLE (1ULL<<17) diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h index 35c416f061552..ea95303ab15b8 100644 --- a/arch/x86/include/asm/pgtable_64_types.h +++ b/arch/x86/include/asm/pgtable_64_types.h @@ -41,6 +41,9 @@ static inline bool pgtable_l5_enabled(void) #define pgtable_l5_enabled() 0 #endif /* CONFIG_X86_5LEVEL */ +#define ARCH_PAGE_TABLE_SYNC_MASK \ + (pgtable_l5_enabled() ? PGTBL_PGD_MODIFIED : PGTBL_P4D_MODIFIED) + extern unsigned int pgdir_shift; extern unsigned int ptrs_per_p4d; diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index 9d6411c659205..00cefbb59fa98 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/x86/include/asm/segment.h @@ -244,7 +244,7 @@ static inline unsigned long vdso_encode_cpunode(int cpu, unsigned long node) static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node) { - unsigned int p; + unsigned long p; /* * Load CPU and node number from the GDT. LSL is faster than RDTSCP @@ -254,10 +254,10 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node) * * If RDPID is available, use it. */ - alternative_io ("lsl %[seg],%[p]", - ".byte 0xf3,0x0f,0xc7,0xf8", /* RDPID %eax/rax */ + alternative_io ("lsl %[seg],%k[p]", + "rdpid %[p]", X86_FEATURE_RDPID, - [p] "=a" (p), [seg] "r" (__CPUNODE_SEG)); + [p] "=r" (p), [seg] "r" (__CPUNODE_SEG)); if (cpu) *cpu = (p & VDSO_CPUNODE_MASK); diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h index 97771b9d33af3..2759524b8ffc3 100644 --- a/arch/x86/include/asm/xen/hypercall.h +++ b/arch/x86/include/asm/xen/hypercall.h @@ -94,12 +94,13 @@ DECLARE_STATIC_CALL(xen_hypercall, xen_hypercall_func); #ifdef MODULE #define __ADDRESSABLE_xen_hypercall #else -#define __ADDRESSABLE_xen_hypercall __ADDRESSABLE_ASM_STR(__SCK__xen_hypercall) +#define __ADDRESSABLE_xen_hypercall \ + __stringify(.global STATIC_CALL_KEY(xen_hypercall);) #endif #define __HYPERCALL \ __ADDRESSABLE_xen_hypercall \ - "call __SCT__xen_hypercall" + __stringify(call STATIC_CALL_TRAMP(xen_hypercall)) #define __HYPERCALL_ENTRY(x) "a" (x) diff --git a/arch/x86/include/uapi/asm/debugreg.h b/arch/x86/include/uapi/asm/debugreg.h index 0007ba077c0c2..41da492dfb01f 100644 --- a/arch/x86/include/uapi/asm/debugreg.h +++ b/arch/x86/include/uapi/asm/debugreg.h @@ -15,7 +15,26 @@ which debugging register was responsible for the trap. The other bits are either reserved or not of interest to us. */ -/* Define reserved bits in DR6 which are always set to 1 */ +/* + * Define bits in DR6 which are set to 1 by default. + * + * This is also the DR6 architectural value following Power-up, Reset or INIT. + * + * Note, with the introduction of Bus Lock Detection (BLD) and Restricted + * Transactional Memory (RTM), the DR6 register has been modified: + * + * 1) BLD flag (bit 11) is no longer reserved to 1 if the CPU supports + * Bus Lock Detection. The assertion of a bus lock could clear it. + * + * 2) RTM flag (bit 16) is no longer reserved to 1 if the CPU supports + * restricted transactional memory. #DB occurred inside an RTM region + * could clear it. + * + * Apparently, DR6.BLD and DR6.RTM are active low bits. + * + * As a result, DR6_RESERVED is an incorrect name now, but it is kept for + * compatibility. + */ #define DR6_RESERVED (0xFFFF0FF0) #define DR_TRAP0 (0x1) /* db0 */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index aae7456ece070..b9dc620f731d3 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only #define pr_fmt(fmt) "SMP alternatives: " fmt +#include #include #include #include @@ -18,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -30,6 +33,8 @@ #include #include #include +#include +#include int __read_mostly alternatives_patched; @@ -123,6 +128,135 @@ const unsigned char * const x86_nops[ASM_NOP_MAX+1] = #endif }; +#ifdef CONFIG_MITIGATION_ITS + +#ifdef CONFIG_MODULES +static struct module *its_mod; +static void *its_page; +static unsigned int its_offset; + +/* Initialize a thunk with the "jmp *reg; int3" instructions. */ +static void *its_init_thunk(void *thunk, int reg) +{ + u8 *bytes = thunk; + int i = 0; + + if (reg >= 8) { + bytes[i++] = 0x41; /* REX.B prefix */ + reg -= 8; + } + bytes[i++] = 0xff; + bytes[i++] = 0xe0 + reg; /* jmp *reg */ + bytes[i++] = 0xcc; + + return thunk; +} + +void its_init_mod(struct module *mod) +{ + if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) + return; + + mutex_lock(&text_mutex); + its_mod = mod; + its_page = NULL; +} + +void its_fini_mod(struct module *mod) +{ + if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) + return; + + WARN_ON_ONCE(its_mod != mod); + + its_mod = NULL; + its_page = NULL; + mutex_unlock(&text_mutex); + + for (int i = 0; i < mod->its_num_pages; i++) { + void *page = mod->its_page_array[i]; + set_memory_rox((unsigned long)page, 1); + } +} + +void its_free_mod(struct module *mod) +{ + if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) + return; + + for (int i = 0; i < mod->its_num_pages; i++) { + void *page = mod->its_page_array[i]; + module_memfree(page); + } + kfree(mod->its_page_array); +} + +DEFINE_FREE(its_execmem, void *, if (_T) module_memfree(_T)); + +static void *its_alloc(void) +{ + void *page __free(its_execmem) = module_alloc(PAGE_SIZE); + + if (!page) + return NULL; + + if (its_mod) { + void *tmp = krealloc(its_mod->its_page_array, + (its_mod->its_num_pages+1) * sizeof(void *), + GFP_KERNEL); + if (!tmp) + return NULL; + + its_mod->its_page_array = tmp; + its_mod->its_page_array[its_mod->its_num_pages++] = page; + } + + return no_free_ptr(page); +} + +static void *its_allocate_thunk(int reg) +{ + int size = 3 + (reg / 8); + void *thunk; + + if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) { + its_page = its_alloc(); + if (!its_page) { + pr_err("ITS page allocation failed\n"); + return NULL; + } + memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE); + its_offset = 32; + } + + /* + * If the indirect branch instruction will be in the lower half + * of a cacheline, then update the offset to reach the upper half. + */ + if ((its_offset + size - 1) % 64 < 32) + its_offset = ((its_offset - 1) | 0x3F) + 33; + + thunk = its_page + its_offset; + its_offset += size; + + set_memory_rw((unsigned long)its_page, 1); + thunk = its_init_thunk(thunk, reg); + set_memory_rox((unsigned long)its_page, 1); + + return thunk; +} + +#else /* CONFIG_MODULES */ + +static void *its_allocate_thunk(int reg) +{ + return NULL; +} + +#endif /* CONFIG_MODULES */ + +#endif /* CONFIG_MITIGATION_ITS */ + /* * Fill the buffer with a single effective instruction of size @len. * @@ -521,7 +655,8 @@ static int emit_indirect(int op, int reg, u8 *bytes) return i; } -static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) +static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes, + void *call_dest, void *jmp_dest) { u8 op = insn->opcode.bytes[0]; int i = 0; @@ -542,7 +677,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 switch (op) { case CALL_INSN_OPCODE: __text_gen_insn(bytes+i, op, addr+i, - __x86_indirect_call_thunk_array[reg], + call_dest, CALL_INSN_SIZE); i += CALL_INSN_SIZE; break; @@ -550,7 +685,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 case JMP32_INSN_OPCODE: clang_jcc: __text_gen_insn(bytes+i, op, addr+i, - __x86_indirect_jump_thunk_array[reg], + jmp_dest, JMP32_INSN_SIZE); i += JMP32_INSN_SIZE; break; @@ -565,6 +700,47 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 return i; } +static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) +{ + return __emit_trampoline(addr, insn, bytes, + __x86_indirect_call_thunk_array[reg], + __x86_indirect_jump_thunk_array[reg]); +} + +#ifdef CONFIG_MITIGATION_ITS +static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes) +{ + u8 *thunk = __x86_indirect_its_thunk_array[reg]; + u8 *tmp = its_allocate_thunk(reg); + + if (tmp) + thunk = tmp; + + return __emit_trampoline(addr, insn, bytes, thunk, thunk); +} + +/* Check if an indirect branch is at ITS-unsafe address */ +static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg) +{ + if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) + return false; + + /* Indirect branch opcode is 2 or 3 bytes depending on reg */ + addr += 1 + reg / 8; + + /* Lower-half of the cacheline? */ + return !(addr & 0x20); +} + +u8 *its_static_thunk(int reg) +{ + u8 *thunk = __x86_indirect_its_thunk_array[reg]; + + return thunk; +} + +#endif /* CONFIG_MITIGATION_ITS */ + /* * Rewrite the compiler generated retpoline thunk calls. * @@ -639,6 +815,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) bytes[i++] = 0xe8; /* LFENCE */ } +#ifdef CONFIG_MITIGATION_ITS + /* + * Check if the address of last byte of emitted-indirect is in + * lower-half of the cacheline. Such branches need ITS mitigation. + */ + if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg)) + return emit_its_trampoline(addr, insn, reg, bytes); +#endif + ret = emit_indirect(op, reg, bytes + i); if (ret < 0) return ret; @@ -710,6 +895,21 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) #ifdef CONFIG_RETHUNK +bool cpu_wants_rethunk(void) +{ + return cpu_feature_enabled(X86_FEATURE_RETHUNK); +} + +bool cpu_wants_rethunk_at(void *addr) +{ + if (!cpu_feature_enabled(X86_FEATURE_RETHUNK)) + return false; + if (x86_return_thunk != its_return_thunk) + return true; + + return !((unsigned long)addr & 0x20); +} + /* * Rewrite the compiler generated return thunk tail-calls. * @@ -726,7 +926,7 @@ static int patch_return(void *addr, struct insn *insn, u8 *bytes) int i = 0; /* Patch the custom return thunks... */ - if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) { + if (cpu_wants_rethunk_at(addr)) { i = JMP32_INSN_SIZE; __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i); } else { @@ -743,7 +943,7 @@ void __init_or_module noinline apply_returns(s32 *start, s32 *end) { s32 *s; - if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) + if (cpu_wants_rethunk()) static_call_force_reinit(); for (s = start; s < end; s++) { @@ -842,15 +1042,41 @@ void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { } #endif /* CONFIG_X86_KERNEL_IBT */ #ifdef CONFIG_FINEIBT +#define __CFI_DEFAULT CFI_DEFAULT +#elif defined(CONFIG_CFI_CLANG) +#define __CFI_DEFAULT CFI_KCFI +#else +#define __CFI_DEFAULT CFI_OFF +#endif -enum cfi_mode { - CFI_DEFAULT, - CFI_OFF, - CFI_KCFI, - CFI_FINEIBT, -}; +enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT; + +#ifdef CONFIG_CFI_CLANG +u32 cfi_get_func_hash(void *func) +{ + u32 hash; + + func -= cfi_get_offset(); + switch (cfi_mode) { + case CFI_FINEIBT: + func += 7; + break; + case CFI_KCFI: + func += 1; + break; + default: + return 0; + } + + if (get_kernel_nofault(hash, func)) + return 0; + + return hash; +} +#endif + +#ifdef CONFIG_FINEIBT -static enum cfi_mode cfi_mode __ro_after_init = CFI_DEFAULT; static bool cfi_rand __ro_after_init = true; static u32 cfi_seed __ro_after_init; @@ -1159,8 +1385,11 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, goto err; if (cfi_rand) { - if (builtin) + if (builtin) { cfi_seed = get_random_u32(); + cfi_bpf_hash = cfi_rehash(cfi_bpf_hash); + cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash); + } ret = cfi_rand_preamble(start_cfi, end_cfi); if (ret) @@ -1575,6 +1804,8 @@ static noinline void __init alt_reloc_selftest(void) void __init alternative_instructions(void) { + u64 ibt; + int3_selftest(); /* @@ -1612,6 +1843,9 @@ void __init alternative_instructions(void) */ paravirt_set_cap(); + /* Keep CET-IBT disabled until caller/callee are patched */ + ibt = ibt_save(/*disable*/ true); + /* * First patch paravirt functions, such that we overwrite the indirect * call with the direct call. @@ -1645,6 +1879,8 @@ void __init alternative_instructions(void) */ apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); + ibt_restore(ibt); + #ifdef CONFIG_SMP /* Patch to UP if other cpus not imminent. */ if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { diff --git a/arch/x86/kernel/cfi.c b/arch/x86/kernel/cfi.c index 8674a5c0c031d..e6bf78fac1462 100644 --- a/arch/x86/kernel/cfi.c +++ b/arch/x86/kernel/cfi.c @@ -4,10 +4,10 @@ * * Copyright (C) 2022 Google LLC */ -#include +#include +#include #include #include -#include /* * Returns the target address and the expected type when regs->ip points diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 498f275377729..864d62e946141 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -539,6 +539,66 @@ static void early_init_amd_mc(struct cpuinfo_x86 *c) #endif } +static bool amd_check_tsa_microcode(void) +{ + struct cpuinfo_x86 *c = &boot_cpu_data; + union zen_patch_rev p; + u32 min_rev = 0; + + p.ext_fam = c->x86 - 0xf; + p.model = c->x86_model; + p.ext_model = c->x86_model >> 4; + p.stepping = c->x86_stepping; + /* reserved bits are expected to be 0 in test below */ + p.__reserved = 0; + + if (cpu_has(c, X86_FEATURE_ZEN3) || + cpu_has(c, X86_FEATURE_ZEN4)) { + switch (p.ucode_rev >> 8) { + case 0xa0011: min_rev = 0x0a0011d7; break; + case 0xa0012: min_rev = 0x0a00123b; break; + case 0xa0082: min_rev = 0x0a00820d; break; + case 0xa1011: min_rev = 0x0a10114c; break; + case 0xa1012: min_rev = 0x0a10124c; break; + case 0xa1081: min_rev = 0x0a108109; break; + case 0xa2010: min_rev = 0x0a20102e; break; + case 0xa2012: min_rev = 0x0a201211; break; + case 0xa4041: min_rev = 0x0a404108; break; + case 0xa5000: min_rev = 0x0a500012; break; + case 0xa6012: min_rev = 0x0a60120a; break; + case 0xa7041: min_rev = 0x0a704108; break; + case 0xa7052: min_rev = 0x0a705208; break; + case 0xa7080: min_rev = 0x0a708008; break; + case 0xa70c0: min_rev = 0x0a70c008; break; + case 0xaa002: min_rev = 0x0aa00216; break; + default: + pr_debug("%s: ucode_rev: 0x%x, current revision: 0x%x\n", + __func__, p.ucode_rev, c->microcode); + return false; + } + } + + if (!min_rev) + return false; + + return c->microcode >= min_rev; +} + +static void tsa_init(struct cpuinfo_x86 *c) +{ + if (cpu_has(c, X86_FEATURE_HYPERVISOR)) + return; + + if (cpu_has(c, X86_FEATURE_ZEN3) || + cpu_has(c, X86_FEATURE_ZEN4)) { + if (amd_check_tsa_microcode()) + setup_force_cpu_cap(X86_FEATURE_VERW_CLEAR); + } else { + setup_force_cpu_cap(X86_FEATURE_TSA_SQ_NO); + setup_force_cpu_cap(X86_FEATURE_TSA_L1_NO); + } +} + static void bsp_init_amd(struct cpuinfo_x86 *c) { if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { @@ -645,6 +705,9 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) break; } + + tsa_init(c); + return; warn: @@ -1093,6 +1156,13 @@ static void init_amd_zen2(struct cpuinfo_x86 *c) { fix_erratum_1386(c); zen2_zenbleed_check(c); + + /* Disable RDSEED on AMD Cyan Skillfish because of an error. */ + if (c->x86_model == 0x47 && c->x86_stepping == 0x0) { + clear_cpu_cap(c, X86_FEATURE_RDSEED); + msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18); + pr_emerg("RDSEED is not reliable on this platform; disabling.\n"); + } } static void init_amd_zen3(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 78545f7e9cc6c..ef1d3a5024ed4 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -49,6 +49,9 @@ static void __init srbds_select_mitigation(void); static void __init l1d_flush_select_mitigation(void); static void __init srso_select_mitigation(void); static void __init gds_select_mitigation(void); +static void __init its_select_mitigation(void); +static void __init tsa_select_mitigation(void); +static void __init vmscape_select_mitigation(void); /* The base value of the SPEC_CTRL MSR without task-specific bits set */ u64 x86_spec_ctrl_base; @@ -58,6 +61,14 @@ EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); DEFINE_PER_CPU(u64, x86_spec_ctrl_current); EXPORT_SYMBOL_GPL(x86_spec_ctrl_current); +/* + * Set when the CPU has run a potentially malicious guest. An IBPB will + * be needed to before running userspace. That IBPB will flush the branch + * predictor content. + */ +DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user); +EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user); + u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; EXPORT_SYMBOL_GPL(x86_pred_cmd); @@ -67,6 +78,13 @@ static DEFINE_MUTEX(spec_ctrl_mutex); void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; +static void __init set_return_thunk(void *thunk) +{ + x86_return_thunk = thunk; + + pr_info("active return thunk: %ps\n", thunk); +} + /* Update SPEC_CTRL MSR and its cached copy unconditionally */ static void update_spec_ctrl(u64 val) { @@ -113,9 +131,9 @@ DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); /* Control unconditional IBPB in switch_mm() */ DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); -/* Control MDS CPU buffer clear before idling (halt, mwait) */ -DEFINE_STATIC_KEY_FALSE(mds_idle_clear); -EXPORT_SYMBOL_GPL(mds_idle_clear); +/* Control CPU buffer clear before idling (halt, mwait) */ +DEFINE_STATIC_KEY_FALSE(cpu_buf_idle_clear); +EXPORT_SYMBOL_GPL(cpu_buf_idle_clear); /* * Controls whether l1d flush based mitigations are enabled, @@ -175,6 +193,9 @@ void __init cpu_select_mitigations(void) */ srso_select_mitigation(); gds_select_mitigation(); + its_select_mitigation(); + tsa_select_mitigation(); + vmscape_select_mitigation(); } /* @@ -435,7 +456,7 @@ static void __init mmio_select_mitigation(void) * is required irrespective of SMT state. */ if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) - static_branch_enable(&mds_idle_clear); + static_branch_enable(&cpu_buf_idle_clear); /* * Check if the system has the right microcode. @@ -1102,7 +1123,7 @@ static void __init retbleed_select_mitigation(void) setup_force_cpu_cap(X86_FEATURE_RETHUNK); setup_force_cpu_cap(X86_FEATURE_UNRET); - x86_return_thunk = retbleed_return_thunk; + set_return_thunk(retbleed_return_thunk); if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) @@ -1136,7 +1157,9 @@ static void __init retbleed_select_mitigation(void) case RETBLEED_MITIGATION_STUFF: setup_force_cpu_cap(X86_FEATURE_RETHUNK); setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); - x86_set_skl_return_thunk(); +#ifdef CONFIG_CALL_DEPTH_TRACKING + set_return_thunk(&__x86_return_skl); +#endif break; default: @@ -1162,14 +1185,156 @@ static void __init retbleed_select_mitigation(void) retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; break; default: - if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) + if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) { pr_err(RETBLEED_INTEL_MSG); + retbleed_mitigation = RETBLEED_MITIGATION_NONE; + } } } pr_info("%s\n", retbleed_strings[retbleed_mitigation]); } +#undef pr_fmt +#define pr_fmt(fmt) "ITS: " fmt + +enum its_mitigation_cmd { + ITS_CMD_OFF, + ITS_CMD_ON, + ITS_CMD_VMEXIT, + ITS_CMD_RSB_STUFF, +}; + +enum its_mitigation { + ITS_MITIGATION_OFF, + ITS_MITIGATION_VMEXIT_ONLY, + ITS_MITIGATION_ALIGNED_THUNKS, + ITS_MITIGATION_RETPOLINE_STUFF, +}; + +static const char * const its_strings[] = { + [ITS_MITIGATION_OFF] = "Vulnerable", + [ITS_MITIGATION_VMEXIT_ONLY] = "Mitigation: Vulnerable, KVM: Not affected", + [ITS_MITIGATION_ALIGNED_THUNKS] = "Mitigation: Aligned branch/return thunks", + [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", +}; + +static enum its_mitigation its_mitigation __ro_after_init = ITS_MITIGATION_ALIGNED_THUNKS; + +static enum its_mitigation_cmd its_cmd __ro_after_init = + IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_CMD_ON : ITS_CMD_OFF; + +static int __init its_parse_cmdline(char *str) +{ + if (!str) + return -EINVAL; + + if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) { + pr_err("Mitigation disabled at compile time, ignoring option (%s)", str); + return 0; + } + + if (!strcmp(str, "off")) { + its_cmd = ITS_CMD_OFF; + } else if (!strcmp(str, "on")) { + its_cmd = ITS_CMD_ON; + } else if (!strcmp(str, "force")) { + its_cmd = ITS_CMD_ON; + setup_force_cpu_bug(X86_BUG_ITS); + } else if (!strcmp(str, "vmexit")) { + its_cmd = ITS_CMD_VMEXIT; + } else if (!strcmp(str, "stuff")) { + its_cmd = ITS_CMD_RSB_STUFF; + } else { + pr_err("Ignoring unknown indirect_target_selection option (%s).", str); + } + + return 0; +} +early_param("indirect_target_selection", its_parse_cmdline); + +static void __init its_select_mitigation(void) +{ + enum its_mitigation_cmd cmd = its_cmd; + + if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) { + its_mitigation = ITS_MITIGATION_OFF; + return; + } + + /* Retpoline+CDT mitigates ITS, bail out */ + if (boot_cpu_has(X86_FEATURE_RETPOLINE) && + boot_cpu_has(X86_FEATURE_CALL_DEPTH)) { + its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; + goto out; + } + + /* Exit early to avoid irrelevant warnings */ + if (cmd == ITS_CMD_OFF) { + its_mitigation = ITS_MITIGATION_OFF; + goto out; + } + if (spectre_v2_enabled == SPECTRE_V2_NONE) { + pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); + its_mitigation = ITS_MITIGATION_OFF; + goto out; + } + if (!IS_ENABLED(CONFIG_RETPOLINE) || !IS_ENABLED(CONFIG_RETHUNK)) { + pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n"); + its_mitigation = ITS_MITIGATION_OFF; + goto out; + } + if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) { + pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n"); + its_mitigation = ITS_MITIGATION_OFF; + goto out; + } + if (boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { + pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n"); + its_mitigation = ITS_MITIGATION_OFF; + goto out; + } + + if (cmd == ITS_CMD_RSB_STUFF && + (!boot_cpu_has(X86_FEATURE_RETPOLINE) || !IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING))) { + pr_err("RSB stuff mitigation not supported, using default\n"); + cmd = ITS_CMD_ON; + } + + switch (cmd) { + case ITS_CMD_OFF: + its_mitigation = ITS_MITIGATION_OFF; + break; + case ITS_CMD_VMEXIT: + if (boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) { + its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; + goto out; + } + fallthrough; + case ITS_CMD_ON: + its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; + if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) + setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + set_return_thunk(its_return_thunk); + break; + case ITS_CMD_RSB_STUFF: + its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); +#ifdef CONFIG_CALL_DEPTH_TRACKING + set_return_thunk(&__x86_return_skl); +#endif + if (retbleed_mitigation == RETBLEED_MITIGATION_NONE) { + retbleed_mitigation = RETBLEED_MITIGATION_STUFF; + pr_info("Retbleed mitigation updated to stuffing\n"); + } + break; + } +out: + pr_info("%s\n", its_strings[its_mitigation]); +} + #undef pr_fmt #define pr_fmt(fmt) "Spectre V2 : " fmt @@ -1429,7 +1594,7 @@ spectre_v2_user_select_mitigation(void) static const char * const spectre_v2_strings[] = { [SPECTRE_V2_NONE] = "Vulnerable", [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", - [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", + [SPECTRE_V2_LFENCE] = "Vulnerable: LFENCE", [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", @@ -1677,10 +1842,11 @@ static void __init bhi_select_mitigation(void) return; } - if (spec_ctrl_bhi_dis()) + if (!IS_ENABLED(CONFIG_X86_64)) return; - if (!IS_ENABLED(CONFIG_X86_64)) + /* Mitigate in hardware if supported */ + if (spec_ctrl_bhi_dis()) return; /* Mitigate KVM by default */ @@ -1929,10 +2095,10 @@ static void update_mds_branch_idle(void) return; if (sched_smt_active()) { - static_branch_enable(&mds_idle_clear); + static_branch_enable(&cpu_buf_idle_clear); } else if (mmio_mitigation == MMIO_MITIGATION_OFF || (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) { - static_branch_disable(&mds_idle_clear); + static_branch_disable(&cpu_buf_idle_clear); } } @@ -1940,60 +2106,92 @@ static void update_mds_branch_idle(void) #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" -void cpu_bugs_smt_update(void) +#undef pr_fmt +#define pr_fmt(fmt) "Transient Scheduler Attacks: " fmt + +enum tsa_mitigations { + TSA_MITIGATION_NONE, + TSA_MITIGATION_UCODE_NEEDED, + TSA_MITIGATION_USER_KERNEL, + TSA_MITIGATION_VM, + TSA_MITIGATION_FULL, +}; + +static const char * const tsa_strings[] = { + [TSA_MITIGATION_NONE] = "Vulnerable", + [TSA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", + [TSA_MITIGATION_USER_KERNEL] = "Mitigation: Clear CPU buffers: user/kernel boundary", + [TSA_MITIGATION_VM] = "Mitigation: Clear CPU buffers: VM", + [TSA_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", +}; + +static enum tsa_mitigations tsa_mitigation __ro_after_init = + IS_ENABLED(CONFIG_MITIGATION_TSA) ? TSA_MITIGATION_FULL : TSA_MITIGATION_NONE; + +static int __init tsa_parse_cmdline(char *str) { - mutex_lock(&spec_ctrl_mutex); + if (!str) + return -EINVAL; - if (sched_smt_active() && unprivileged_ebpf_enabled() && - spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) - pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); + if (!strcmp(str, "off")) + tsa_mitigation = TSA_MITIGATION_NONE; + else if (!strcmp(str, "on")) + tsa_mitigation = TSA_MITIGATION_FULL; + else if (!strcmp(str, "user")) + tsa_mitigation = TSA_MITIGATION_USER_KERNEL; + else if (!strcmp(str, "vm")) + tsa_mitigation = TSA_MITIGATION_VM; + else + pr_err("Ignoring unknown tsa=%s option.\n", str); - switch (spectre_v2_user_stibp) { - case SPECTRE_V2_USER_NONE: - break; - case SPECTRE_V2_USER_STRICT: - case SPECTRE_V2_USER_STRICT_PREFERRED: - update_stibp_strict(); - break; - case SPECTRE_V2_USER_PRCTL: - case SPECTRE_V2_USER_SECCOMP: - update_indir_branch_cond(); - break; - } + return 0; +} +early_param("tsa", tsa_parse_cmdline); - switch (mds_mitigation) { - case MDS_MITIGATION_FULL: - case MDS_MITIGATION_VMWERV: - if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) - pr_warn_once(MDS_MSG_SMT); - update_mds_branch_idle(); - break; - case MDS_MITIGATION_OFF: - break; +static void __init tsa_select_mitigation(void) +{ + if (tsa_mitigation == TSA_MITIGATION_NONE) + return; + + if (cpu_mitigations_off() || !boot_cpu_has_bug(X86_BUG_TSA)) { + tsa_mitigation = TSA_MITIGATION_NONE; + return; } - switch (taa_mitigation) { - case TAA_MITIGATION_VERW: - case TAA_MITIGATION_UCODE_NEEDED: - if (sched_smt_active()) - pr_warn_once(TAA_MSG_SMT); + if (!boot_cpu_has(X86_FEATURE_VERW_CLEAR)) + tsa_mitigation = TSA_MITIGATION_UCODE_NEEDED; + + switch (tsa_mitigation) { + case TSA_MITIGATION_USER_KERNEL: + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); break; - case TAA_MITIGATION_TSX_DISABLED: - case TAA_MITIGATION_OFF: + + case TSA_MITIGATION_VM: + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); break; - } - switch (mmio_mitigation) { - case MMIO_MITIGATION_VERW: - case MMIO_MITIGATION_UCODE_NEEDED: - if (sched_smt_active()) - pr_warn_once(MMIO_MSG_SMT); + case TSA_MITIGATION_UCODE_NEEDED: + if (!boot_cpu_has(X86_FEATURE_HYPERVISOR)) + goto out; + + pr_notice("Forcing mitigation on in a VM\n"); + + /* + * On the off-chance that microcode has been updated + * on the host, enable the mitigation in the guest just + * in case. + */ + fallthrough; + case TSA_MITIGATION_FULL: + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); break; - case MMIO_MITIGATION_OFF: + default: break; } - mutex_unlock(&spec_ctrl_mutex); +out: + pr_info("%s\n", tsa_strings[tsa_mitigation]); } #undef pr_fmt @@ -2606,10 +2804,10 @@ static void __init srso_select_mitigation(void) if (boot_cpu_data.x86 == 0x19) { setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); - x86_return_thunk = srso_alias_return_thunk; + set_return_thunk(srso_alias_return_thunk); } else { setup_force_cpu_cap(X86_FEATURE_SRSO); - x86_return_thunk = srso_return_thunk; + set_return_thunk(srso_return_thunk); } if (has_microcode) srso_mitigation = SRSO_MITIGATION_SAFE_RET; @@ -2681,9 +2879,169 @@ static void __init srso_select_mitigation(void) x86_pred_cmd = PRED_CMD_SBPB; } +#undef pr_fmt +#define pr_fmt(fmt) "VMSCAPE: " fmt + +enum vmscape_mitigations { + VMSCAPE_MITIGATION_NONE, + VMSCAPE_MITIGATION_AUTO, + VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER, + VMSCAPE_MITIGATION_IBPB_ON_VMEXIT, +}; + +static const char * const vmscape_strings[] = { + [VMSCAPE_MITIGATION_NONE] = "Vulnerable", + /* [VMSCAPE_MITIGATION_AUTO] */ + [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER] = "Mitigation: IBPB before exit to userspace", + [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT", +}; + +static enum vmscape_mitigations vmscape_mitigation __ro_after_init = + IS_ENABLED(CONFIG_MITIGATION_VMSCAPE) ? VMSCAPE_MITIGATION_AUTO : VMSCAPE_MITIGATION_NONE; + +static int __init vmscape_parse_cmdline(char *str) +{ + if (!str) + return -EINVAL; + + if (!strcmp(str, "off")) { + vmscape_mitigation = VMSCAPE_MITIGATION_NONE; + } else if (!strcmp(str, "ibpb")) { + vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; + } else if (!strcmp(str, "force")) { + setup_force_cpu_bug(X86_BUG_VMSCAPE); + vmscape_mitigation = VMSCAPE_MITIGATION_AUTO; + } else { + pr_err("Ignoring unknown vmscape=%s option.\n", str); + } + + return 0; +} +early_param("vmscape", vmscape_parse_cmdline); + +static void __init vmscape_select_mitigation(void) +{ + if (cpu_mitigations_off() || + !boot_cpu_has_bug(X86_BUG_VMSCAPE) || + !boot_cpu_has(X86_FEATURE_IBPB)) { + vmscape_mitigation = VMSCAPE_MITIGATION_NONE; + return; + } + + if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO) + vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; + + if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB || + srso_mitigation == SRSO_MITIGATION_IBPB_ON_VMEXIT) + vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_ON_VMEXIT; + + if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER) + setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER); + + pr_info("%s\n", vmscape_strings[vmscape_mitigation]); +} + #undef pr_fmt #define pr_fmt(fmt) fmt +#define VMSCAPE_MSG_SMT "VMSCAPE: SMT on, STIBP is required for full protection. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/vmscape.html for more details.\n" + +void cpu_bugs_smt_update(void) +{ + mutex_lock(&spec_ctrl_mutex); + + if (sched_smt_active() && unprivileged_ebpf_enabled() && + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) + pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); + + switch (spectre_v2_user_stibp) { + case SPECTRE_V2_USER_NONE: + break; + case SPECTRE_V2_USER_STRICT: + case SPECTRE_V2_USER_STRICT_PREFERRED: + update_stibp_strict(); + break; + case SPECTRE_V2_USER_PRCTL: + case SPECTRE_V2_USER_SECCOMP: + update_indir_branch_cond(); + break; + } + + switch (mds_mitigation) { + case MDS_MITIGATION_FULL: + case MDS_MITIGATION_VMWERV: + if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) + pr_warn_once(MDS_MSG_SMT); + update_mds_branch_idle(); + break; + case MDS_MITIGATION_OFF: + break; + } + + switch (taa_mitigation) { + case TAA_MITIGATION_VERW: + case TAA_MITIGATION_UCODE_NEEDED: + if (sched_smt_active()) + pr_warn_once(TAA_MSG_SMT); + break; + case TAA_MITIGATION_TSX_DISABLED: + case TAA_MITIGATION_OFF: + break; + } + + switch (mmio_mitigation) { + case MMIO_MITIGATION_VERW: + case MMIO_MITIGATION_UCODE_NEEDED: + if (sched_smt_active()) + pr_warn_once(MMIO_MSG_SMT); + break; + case MMIO_MITIGATION_OFF: + break; + } + + switch (tsa_mitigation) { + case TSA_MITIGATION_USER_KERNEL: + case TSA_MITIGATION_VM: + case TSA_MITIGATION_FULL: + case TSA_MITIGATION_UCODE_NEEDED: + /* + * TSA-SQ can potentially lead to info leakage between + * SMT threads. + */ + if (sched_smt_active()) + static_branch_enable(&cpu_buf_idle_clear); + else + static_branch_disable(&cpu_buf_idle_clear); + break; + case TSA_MITIGATION_NONE: + break; + } + + switch (vmscape_mitigation) { + case VMSCAPE_MITIGATION_NONE: + case VMSCAPE_MITIGATION_AUTO: + break; + case VMSCAPE_MITIGATION_IBPB_ON_VMEXIT: + case VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER: + /* + * Hypervisors can be attacked across-threads, warn for SMT when + * STIBP is not already enabled system-wide. + * + * Intel eIBRS (!AUTOIBRS) implies STIBP on. + */ + if (!sched_smt_active() || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED || + (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && + !boot_cpu_has(X86_FEATURE_AUTOIBRS))) + break; + pr_warn_once(VMSCAPE_MSG_SMT); + break; + } + + mutex_unlock(&spec_ctrl_mutex); +} + #ifdef CONFIG_SYSFS #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" @@ -2793,6 +3151,11 @@ static ssize_t rfds_show_state(char *buf) return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); } +static ssize_t its_show_state(char *buf) +{ + return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]); +} + static char *stibp_state(void) { if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && @@ -2859,9 +3222,6 @@ static const char *spectre_bhi_state(void) static ssize_t spectre_v2_show_state(char *buf) { - if (spectre_v2_enabled == SPECTRE_V2_LFENCE) - return sysfs_emit(buf, "Vulnerable: LFENCE\n"); - if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); @@ -2917,6 +3277,16 @@ static ssize_t gds_show_state(char *buf) return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); } +static ssize_t tsa_show_state(char *buf) +{ + return sysfs_emit(buf, "%s\n", tsa_strings[tsa_mitigation]); +} + +static ssize_t vmscape_show_state(char *buf) +{ + return sysfs_emit(buf, "%s\n", vmscape_strings[vmscape_mitigation]); +} + static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, char *buf, unsigned int bug) { @@ -2975,6 +3345,15 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr case X86_BUG_RFDS: return rfds_show_state(buf); + case X86_BUG_ITS: + return its_show_state(buf); + + case X86_BUG_TSA: + return tsa_show_state(buf); + + case X86_BUG_VMSCAPE: + return vmscape_show_state(buf); + default: break; } @@ -3054,4 +3433,19 @@ ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attrib { return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); } + +ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf) +{ + return cpu_show_common(dev, attr, buf, X86_BUG_ITS); +} + +ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf) +{ + return cpu_show_common(dev, attr, buf, X86_BUG_TSA); +} + +ssize_t cpu_show_vmscape(struct device *dev, struct device_attribute *attr, char *buf) +{ + return cpu_show_common(dev, attr, buf, X86_BUG_VMSCAPE); +} #endif diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a844110691f97..cf455968f27b6 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1066,17 +1066,18 @@ void get_cpu_cap(struct cpuinfo_x86 *c) c->x86_capability[CPUID_D_1_EAX] = eax; } - /* AMD-defined flags: level 0x80000001 */ + /* + * Check if extended CPUID leaves are implemented: Max extended + * CPUID leaf must be in the 0x80000001-0x8000ffff range. + */ eax = cpuid_eax(0x80000000); - c->extended_cpuid_level = eax; + c->extended_cpuid_level = ((eax & 0xffff0000) == 0x80000000) ? eax : 0; - if ((eax & 0xffff0000) == 0x80000000) { - if (eax >= 0x80000001) { - cpuid(0x80000001, &eax, &ebx, &ecx, &edx); + if (c->extended_cpuid_level >= 0x80000001) { + cpuid(0x80000001, &eax, &ebx, &ecx, &edx); - c->x86_capability[CPUID_8000_0001_ECX] = ecx; - c->x86_capability[CPUID_8000_0001_EDX] = edx; - } + c->x86_capability[CPUID_8000_0001_ECX] = ecx; + c->x86_capability[CPUID_8000_0001_EDX] = edx; } if (c->extended_cpuid_level >= 0x80000007) { @@ -1272,51 +1273,74 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { #define GDS BIT(6) /* CPU is affected by Register File Data Sampling */ #define RFDS BIT(7) +/* CPU is affected by Indirect Target Selection */ +#define ITS BIT(8) +/* CPU is affected by Indirect Target Selection, but guest-host isolation is not affected */ +#define ITS_NATIVE_ONLY BIT(9) +/* CPU is affected by Transient Scheduler Attacks */ +#define TSA BIT(10) +/* CPU is affected by VMSCAPE */ +#define VMSCAPE BIT(11) static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { - VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(HASWELL, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(HASWELL_L, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(HASWELL_G, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(HASWELL_X, X86_STEPPING_ANY, MMIO), - VULNBL_INTEL_STEPPINGS(BROADWELL_D, X86_STEPPING_ANY, MMIO), - VULNBL_INTEL_STEPPINGS(BROADWELL_G, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(BROADWELL_X, X86_STEPPING_ANY, MMIO), - VULNBL_INTEL_STEPPINGS(BROADWELL, X86_STEPPING_ANY, SRBDS), - VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPING_ANY, MMIO | RETBLEED | GDS), - VULNBL_INTEL_STEPPINGS(SKYLAKE_L, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), - VULNBL_INTEL_STEPPINGS(SKYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), - VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), - VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS), - VULNBL_INTEL_STEPPINGS(CANNONLAKE_L, X86_STEPPING_ANY, RETBLEED), - VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS), - VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPING_ANY, MMIO | GDS), - VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO | GDS), - VULNBL_INTEL_STEPPINGS(COMETLAKE, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS), - VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x0, 0x0), MMIO | RETBLEED), - VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS), - VULNBL_INTEL_STEPPINGS(TIGERLAKE_L, X86_STEPPING_ANY, GDS), - VULNBL_INTEL_STEPPINGS(TIGERLAKE, X86_STEPPING_ANY, GDS), + VULNBL_INTEL_STEPPINGS(SANDYBRIDGE_X, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(SANDYBRIDGE, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(IVYBRIDGE_X, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(HASWELL, X86_STEPPING_ANY, SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(HASWELL_L, X86_STEPPING_ANY, SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(HASWELL_G, X86_STEPPING_ANY, SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(HASWELL_X, X86_STEPPING_ANY, MMIO | VMSCAPE), + VULNBL_INTEL_STEPPINGS(BROADWELL_D, X86_STEPPING_ANY, MMIO | VMSCAPE), + VULNBL_INTEL_STEPPINGS(BROADWELL_X, X86_STEPPING_ANY, MMIO | VMSCAPE), + VULNBL_INTEL_STEPPINGS(BROADWELL_G, X86_STEPPING_ANY, SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(BROADWELL, X86_STEPPING_ANY, SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x0, 0x5), MMIO | RETBLEED | GDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(SKYLAKE_X, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | ITS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(SKYLAKE_L, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(SKYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPINGS(0x0, 0xb), MMIO | RETBLEED | GDS | SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS | ITS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPINGS(0x0, 0xc), MMIO | RETBLEED | GDS | SRBDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | SRBDS | ITS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(CANNONLAKE_L, X86_STEPPING_ANY, RETBLEED | VMSCAPE), + VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPING_ANY, MMIO | GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPING_ANY, MMIO | GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(COMETLAKE, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x0, 0x0), MMIO | RETBLEED | ITS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(TIGERLAKE_L, X86_STEPPING_ANY, GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(TIGERLAKE, X86_STEPPING_ANY, GDS | ITS | ITS_NATIVE_ONLY), VULNBL_INTEL_STEPPINGS(LAKEFIELD, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED), - VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS), - VULNBL_INTEL_STEPPINGS(ALDERLAKE, X86_STEPPING_ANY, RFDS), - VULNBL_INTEL_STEPPINGS(ALDERLAKE_L, X86_STEPPING_ANY, RFDS), - VULNBL_INTEL_STEPPINGS(RAPTORLAKE, X86_STEPPING_ANY, RFDS), - VULNBL_INTEL_STEPPINGS(RAPTORLAKE_P, X86_STEPPING_ANY, RFDS), - VULNBL_INTEL_STEPPINGS(RAPTORLAKE_S, X86_STEPPING_ANY, RFDS), - VULNBL_INTEL_STEPPINGS(ATOM_GRACEMONT, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), + VULNBL_INTEL_STEPPINGS(ALDERLAKE, X86_STEPPING_ANY, RFDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(ALDERLAKE_L, X86_STEPPING_ANY, RFDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE, X86_STEPPING_ANY, RFDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE_P, X86_STEPPING_ANY, RFDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE_S, X86_STEPPING_ANY, RFDS | VMSCAPE), + VULNBL_INTEL_STEPPINGS(METEORLAKE_L, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(ARROWLAKE_H, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(ARROWLAKE, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(ARROWLAKE_U, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(LUNARLAKE_M, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(SAPPHIRERAPIDS_X,X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(GRANITERAPIDS_X, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(EMERALDRAPIDS_X, X86_STEPPING_ANY, VMSCAPE), + VULNBL_INTEL_STEPPINGS(ATOM_GRACEMONT, X86_STEPPING_ANY, RFDS | VMSCAPE), VULNBL_INTEL_STEPPINGS(ATOM_TREMONT, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RFDS), VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPING_ANY, MMIO | RFDS), VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RFDS), VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT, X86_STEPPING_ANY, RFDS), VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_D, X86_STEPPING_ANY, RFDS), VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_PLUS, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_CRESTMONT_X, X86_STEPPING_ANY, VMSCAPE), VULNBL_AMD(0x15, RETBLEED), VULNBL_AMD(0x16, RETBLEED), - VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO), - VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO), - VULNBL_AMD(0x19, SRSO), + VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO | VMSCAPE), + VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO | VMSCAPE), + VULNBL_AMD(0x19, SRSO | TSA | VMSCAPE), {} }; @@ -1362,6 +1386,32 @@ static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr) return cpu_matches(cpu_vuln_blacklist, RFDS); } +static bool __init vulnerable_to_its(u64 x86_arch_cap_msr) +{ + /* The "immunity" bit trumps everything else: */ + if (x86_arch_cap_msr & ARCH_CAP_ITS_NO) + return false; + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return false; + + /* None of the affected CPUs have BHI_CTRL */ + if (boot_cpu_has(X86_FEATURE_BHI_CTRL)) + return false; + + /* + * If a VMM did not expose ITS_NO, assume that a guest could + * be running on a vulnerable hardware or may migrate to such + * hardware. + */ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + return true; + + if (cpu_matches(cpu_vuln_blacklist, ITS)) + return true; + + return false; +} + static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) { u64 x86_arch_cap_msr = x86_read_arch_cap_msr(); @@ -1476,9 +1526,12 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) if (vulnerable_to_rfds(x86_arch_cap_msr)) setup_force_cpu_bug(X86_BUG_RFDS); - /* When virtualized, eIBRS could be hidden, assume vulnerable */ - if (!(x86_arch_cap_msr & ARCH_CAP_BHI_NO) && - !cpu_matches(cpu_vuln_whitelist, NO_BHI) && + /* + * Intel parts with eIBRS are vulnerable to BHI attacks. Parts with + * BHI_NO still need to use the BHI mitigation to prevent Intra-mode + * attacks. When virtualized, eIBRS could be hidden, assume vulnerable. + */ + if (!cpu_matches(cpu_vuln_whitelist, NO_BHI) && (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) || boot_cpu_has(X86_FEATURE_HYPERVISOR))) setup_force_cpu_bug(X86_BUG_BHI); @@ -1486,6 +1539,30 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) if (cpu_has(c, X86_FEATURE_AMD_IBPB) && !cpu_has(c, X86_FEATURE_AMD_IBPB_RET)) setup_force_cpu_bug(X86_BUG_IBPB_NO_RET); + if (vulnerable_to_its(x86_arch_cap_msr)) { + setup_force_cpu_bug(X86_BUG_ITS); + if (cpu_matches(cpu_vuln_blacklist, ITS_NATIVE_ONLY)) + setup_force_cpu_bug(X86_BUG_ITS_NATIVE_ONLY); + } + + if (c->x86_vendor == X86_VENDOR_AMD) { + if (!cpu_has(c, X86_FEATURE_TSA_SQ_NO) || + !cpu_has(c, X86_FEATURE_TSA_L1_NO)) { + if (cpu_matches(cpu_vuln_blacklist, TSA) || + /* Enable bug on Zen guests to allow for live migration. */ + (cpu_has(c, X86_FEATURE_HYPERVISOR) && cpu_has(c, X86_FEATURE_ZEN))) + setup_force_cpu_bug(X86_BUG_TSA); + } + } + + /* + * Set the bug only on bare-metal. A nested hypervisor should already be + * deploying IBPB to isolate itself from nested guests. + */ + if (cpu_matches(cpu_vuln_blacklist, VMSCAPE) && + !boot_cpu_has(X86_FEATURE_HYPERVISOR)) + setup_force_cpu_bug(X86_BUG_VMSCAPE); + if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN)) return; @@ -2172,20 +2249,16 @@ EXPORT_PER_CPU_SYMBOL(__stack_chk_guard); #endif /* CONFIG_X86_64 */ -/* - * Clear all 6 debug registers: - */ -static void clear_all_debug_regs(void) +static void initialize_debug_regs(void) { - int i; - - for (i = 0; i < 8; i++) { - /* Ignore db4, db5 */ - if ((i == 4) || (i == 5)) - continue; - - set_debugreg(0, i); - } + /* Control register first -- to make sure everything is disabled. */ + set_debugreg(0, 7); + set_debugreg(DR6_RESERVED, 6); + /* dr5 and dr4 don't exist */ + set_debugreg(0, 3); + set_debugreg(0, 2); + set_debugreg(0, 1); + set_debugreg(0, 0); } #ifdef CONFIG_KGDB @@ -2328,7 +2401,7 @@ void cpu_init(void) load_mm_ldt(&init_mm); - clear_all_debug_regs(); + initialize_debug_regs(); dbg_restore_debug_regs(); doublefault_init_cpu_tss(); diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 6e738759779e8..5c594781b463d 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "cpu.h" @@ -240,6 +241,8 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c) x86_amd_ls_cfg_ssbd_mask = 1ULL << 10; } } + + resctrl_cpu_detect(c); } static void early_init_hygon(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index c267f43de39ea..b89c5a3854034 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -335,7 +335,6 @@ static void smca_configure(unsigned int bank, unsigned int cpu) struct thresh_restart { struct threshold_block *b; - int reset; int set_lvt_off; int lvt_off; u16 old_limit; @@ -430,13 +429,13 @@ static void threshold_restart_bank(void *_tr) rdmsr(tr->b->address, lo, hi); - if (tr->b->threshold_limit < (hi & THRESHOLD_MAX)) - tr->reset = 1; /* limit cannot be lower than err count */ - - if (tr->reset) { /* reset err count and overflow bit */ - hi = - (hi & ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI)) | - (THRESHOLD_MAX - tr->b->threshold_limit); + /* + * Reset error count and overflow bit. + * This is done during init or after handling an interrupt. + */ + if (hi & MASK_OVERFLOW_HI || tr->set_lvt_off) { + hi &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI); + hi |= THRESHOLD_MAX - tr->b->threshold_limit; } else if (tr->old_limit) { /* change limit w/o reset */ int new_count = (hi & THRESHOLD_MAX) + (tr->old_limit - tr->b->threshold_limit); @@ -1049,13 +1048,20 @@ static const char *get_name(unsigned int cpu, unsigned int bank, struct threshol } bank_type = smca_get_bank_type(cpu, bank); - if (bank_type >= N_SMCA_BANK_TYPES) - return NULL; if (b && (bank_type == SMCA_UMC || bank_type == SMCA_UMC_V2)) { if (b->block < ARRAY_SIZE(smca_umc_block_names)) return smca_umc_block_names[b->block]; - return NULL; + } + + if (b && b->block) { + snprintf(buf_mcatype, MAX_MCATYPE_NAME_LEN, "th_block_%u", b->block); + return buf_mcatype; + } + + if (bank_type >= N_SMCA_BANK_TYPES) { + snprintf(buf_mcatype, MAX_MCATYPE_NAME_LEN, "th_bank_%u", bank); + return buf_mcatype; } if (per_cpu(smca_bank_counts, cpu)[bank_type] == 1) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index e103c227acd3a..106436ec3c005 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -2704,15 +2704,9 @@ static int mce_cpu_dead(unsigned int cpu) static int mce_cpu_online(unsigned int cpu) { struct timer_list *t = this_cpu_ptr(&mce_timer); - int ret; mce_device_create(cpu); - - ret = mce_threshold_create_device(cpu); - if (ret) { - mce_device_remove(cpu); - return ret; - } + mce_threshold_create_device(cpu); mce_reenable_cpu(); mce_start_timer(t); return 0; diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index f5323551c1a9a..0a9ce4f8a88cf 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -517,6 +517,7 @@ void mce_intel_feature_init(struct cpuinfo_x86 *c) void mce_intel_feature_clear(struct cpuinfo_x86 *c) { intel_clear_lmce(); + cmci_clear(); } bool intel_filter_mce(struct mce *m) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index ce78e39004e0e..9952c774eaa69 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -96,18 +96,6 @@ static struct equiv_cpu_table { struct equiv_cpu_entry *entry; } equiv_table; -union zen_patch_rev { - struct { - __u32 rev : 8, - stepping : 4, - model : 4, - __reserved : 4, - ext_model : 4, - ext_fam : 8; - }; - __u32 ucode_rev; -}; - union cpuid_1_eax { struct { __u32 stepping : 4, @@ -173,10 +161,30 @@ static int cmp_id(const void *key, const void *elem) return 1; } +static u32 cpuid_to_ucode_rev(unsigned int val) +{ + union zen_patch_rev p = {}; + union cpuid_1_eax c; + + c.full = val; + + p.stepping = c.stepping; + p.model = c.model; + p.ext_model = c.ext_model; + p.ext_fam = c.ext_fam; + + return p.ucode_rev; +} + static bool need_sha_check(u32 cur_rev) { + if (!cur_rev) { + cur_rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax); + pr_info_once("No current revision, generating the lowest one: 0x%x\n", cur_rev); + } + switch (cur_rev >> 8) { - case 0x80012: return cur_rev <= 0x800126f; break; + case 0x80012: return cur_rev <= 0x8001277; break; case 0x80082: return cur_rev <= 0x800820f; break; case 0x83010: return cur_rev <= 0x830107c; break; case 0x86001: return cur_rev <= 0x860010e; break; @@ -202,10 +210,13 @@ static bool need_sha_check(u32 cur_rev) case 0xaa001: return cur_rev <= 0xaa00116; break; case 0xaa002: return cur_rev <= 0xaa00218; break; case 0xb0021: return cur_rev <= 0xb002146; break; + case 0xb0081: return cur_rev <= 0xb008111; break; case 0xb1010: return cur_rev <= 0xb101046; break; case 0xb2040: return cur_rev <= 0xb204031; break; case 0xb4040: return cur_rev <= 0xb404031; break; + case 0xb4041: return cur_rev <= 0xb404101; break; case 0xb6000: return cur_rev <= 0xb600031; break; + case 0xb6080: return cur_rev <= 0xb608031; break; case 0xb7000: return cur_rev <= 0xb700031; break; default: break; } @@ -215,6 +226,24 @@ static bool need_sha_check(u32 cur_rev) return true; } +static bool cpu_has_entrysign(void) +{ + unsigned int fam = x86_family(bsp_cpuid_1_eax); + unsigned int model = x86_model(bsp_cpuid_1_eax); + + if (fam == 0x17 || fam == 0x19) + return true; + + if (fam == 0x1a) { + if (model <= 0x2f || + (0x40 <= model && model <= 0x4f) || + (0x60 <= model && model <= 0x6f)) + return true; + } + + return false; +} + static bool verify_sha256_digest(u32 patch_id, u32 cur_rev, const u8 *data, unsigned int len) { struct patch_digest *pd = NULL; @@ -222,7 +251,7 @@ static bool verify_sha256_digest(u32 patch_id, u32 cur_rev, const u8 *data, unsi struct sha256_state s; int i; - if (x86_family(bsp_cpuid_1_eax) < 0x17) + if (!cpu_has_entrysign()) return true; if (!need_sha_check(cur_rev)) @@ -756,8 +785,6 @@ static struct ucode_patch *cache_find_patch(struct ucode_cpu_info *uci, u16 equi n.equiv_cpu = equiv_cpu; n.patch_id = uci->cpu_sig.rev; - WARN_ON_ONCE(!n.patch_id); - list_for_each_entry(p, µcode_cache, plist) if (patch_cpus_equivalent(p, &n, false)) return p; @@ -1102,15 +1129,17 @@ static enum ucode_state load_microcode_amd(u8 family, const u8 *data, size_t siz static int __init save_microcode_in_initrd(void) { - unsigned int cpuid_1_eax = native_cpuid_eax(1); struct cpuinfo_x86 *c = &boot_cpu_data; struct cont_desc desc = { 0 }; + unsigned int cpuid_1_eax; enum ucode_state ret; struct cpio_data cp; - if (dis_ucode_ldr || c->x86_vendor != X86_VENDOR_AMD || c->x86 < 0x10) + if (microcode_loader_disabled() || c->x86_vendor != X86_VENDOR_AMD || c->x86 < 0x10) return 0; + cpuid_1_eax = native_cpuid_eax(1); + if (!find_blobs_in_containers(&cp)) return -EINVAL; diff --git a/arch/x86/kernel/cpu/microcode/amd_shas.c b/arch/x86/kernel/cpu/microcode/amd_shas.c index 2a1655b1fdd88..1fd349cfc8024 100644 --- a/arch/x86/kernel/cpu/microcode/amd_shas.c +++ b/arch/x86/kernel/cpu/microcode/amd_shas.c @@ -231,6 +231,13 @@ static const struct patch_digest phashes[] = { 0x0d,0x5b,0x65,0x34,0x69,0xb2,0x62,0x21, } }, + { 0xa0011d7, { + 0x35,0x07,0xcd,0x40,0x94,0xbc,0x81,0x6b, + 0xfc,0x61,0x56,0x1a,0xe2,0xdb,0x96,0x12, + 0x1c,0x1c,0x31,0xb1,0x02,0x6f,0xe5,0xd2, + 0xfe,0x1b,0x04,0x03,0x2c,0x8f,0x4c,0x36, + } + }, { 0xa001223, { 0xfb,0x32,0x5f,0xc6,0x83,0x4f,0x8c,0xb8, 0xa4,0x05,0xf9,0x71,0x53,0x01,0x16,0xc4, @@ -294,6 +301,13 @@ static const struct patch_digest phashes[] = { 0xc0,0xcd,0x33,0xf2,0x8d,0xf9,0xef,0x59, } }, + { 0xa00123b, { + 0xef,0xa1,0x1e,0x71,0xf1,0xc3,0x2c,0xe2, + 0xc3,0xef,0x69,0x41,0x7a,0x54,0xca,0xc3, + 0x8f,0x62,0x84,0xee,0xc2,0x39,0xd9,0x28, + 0x95,0xa7,0x12,0x49,0x1e,0x30,0x71,0x72, + } + }, { 0xa00820c, { 0xa8,0x0c,0x81,0xc0,0xa6,0x00,0xe7,0xf3, 0x5f,0x65,0xd3,0xb9,0x6f,0xea,0x93,0x63, @@ -301,6 +315,13 @@ static const struct patch_digest phashes[] = { 0xe1,0x3b,0x8d,0xb2,0xf8,0x22,0x03,0xe2, } }, + { 0xa00820d, { + 0xf9,0x2a,0xc0,0xf4,0x9e,0xa4,0x87,0xa4, + 0x7d,0x87,0x00,0xfd,0xab,0xda,0x19,0xca, + 0x26,0x51,0x32,0xc1,0x57,0x91,0xdf,0xc1, + 0x05,0xeb,0x01,0x7c,0x5a,0x95,0x21,0xb7, + } + }, { 0xa10113e, { 0x05,0x3c,0x66,0xd7,0xa9,0x5a,0x33,0x10, 0x1b,0xf8,0x9c,0x8f,0xed,0xfc,0xa7,0xa0, @@ -322,6 +343,13 @@ static const struct patch_digest phashes[] = { 0xf1,0x5e,0xb0,0xde,0xb4,0x98,0xae,0xc4, } }, + { 0xa10114c, { + 0x9e,0xb6,0xa2,0xd9,0x87,0x38,0xc5,0x64, + 0xd8,0x88,0xfa,0x78,0x98,0xf9,0x6f,0x74, + 0x39,0x90,0x1b,0xa5,0xcf,0x5e,0xb4,0x2a, + 0x02,0xff,0xd4,0x8c,0x71,0x8b,0xe2,0xc0, + } + }, { 0xa10123e, { 0x03,0xb9,0x2c,0x76,0x48,0x93,0xc9,0x18, 0xfb,0x56,0xfd,0xf7,0xe2,0x1d,0xca,0x4d, @@ -343,6 +371,13 @@ static const struct patch_digest phashes[] = { 0x1b,0x7d,0x64,0x9d,0x4b,0x53,0x13,0x75, } }, + { 0xa10124c, { + 0x29,0xea,0xf1,0x2c,0xb2,0xe4,0xef,0x90, + 0xa4,0xcd,0x1d,0x86,0x97,0x17,0x61,0x46, + 0xfc,0x22,0xcb,0x57,0x75,0x19,0xc8,0xcc, + 0x0c,0xf5,0xbc,0xac,0x81,0x9d,0x9a,0xd2, + } + }, { 0xa108108, { 0xed,0xc2,0xec,0xa1,0x15,0xc6,0x65,0xe9, 0xd0,0xef,0x39,0xaa,0x7f,0x55,0x06,0xc6, @@ -350,6 +385,13 @@ static const struct patch_digest phashes[] = { 0x28,0x1e,0x9c,0x59,0x69,0x99,0x4d,0x16, } }, + { 0xa108109, { + 0x85,0xb4,0xbd,0x7c,0x49,0xa7,0xbd,0xfa, + 0x49,0x36,0x80,0x81,0xc5,0xb7,0x39,0x1b, + 0x9a,0xaa,0x50,0xde,0x9b,0xe9,0x32,0x35, + 0x42,0x7e,0x51,0x4f,0x52,0x2c,0x28,0x59, + } + }, { 0xa20102d, { 0xf9,0x6e,0xf2,0x32,0xd3,0x0f,0x5f,0x11, 0x59,0xa1,0xfe,0xcc,0xcd,0x9b,0x42,0x89, @@ -357,6 +399,13 @@ static const struct patch_digest phashes[] = { 0x8c,0xe9,0x19,0x3e,0xcc,0x3f,0x7b,0xb4, } }, + { 0xa20102e, { + 0xbe,0x1f,0x32,0x04,0x0d,0x3c,0x9c,0xdd, + 0xe1,0xa4,0xbf,0x76,0x3a,0xec,0xc2,0xf6, + 0x11,0x00,0xa7,0xaf,0x0f,0xe5,0x02,0xc5, + 0x54,0x3a,0x1f,0x8c,0x16,0xb5,0xff,0xbe, + } + }, { 0xa201210, { 0xe8,0x6d,0x51,0x6a,0x8e,0x72,0xf3,0xfe, 0x6e,0x16,0xbc,0x62,0x59,0x40,0x17,0xe9, @@ -364,6 +413,13 @@ static const struct patch_digest phashes[] = { 0xf7,0x55,0xf0,0x13,0xbb,0x22,0xf6,0x41, } }, + { 0xa201211, { + 0x69,0xa1,0x17,0xec,0xd0,0xf6,0x6c,0x95, + 0xe2,0x1e,0xc5,0x59,0x1a,0x52,0x0a,0x27, + 0xc4,0xed,0xd5,0x59,0x1f,0xbf,0x00,0xff, + 0x08,0x88,0xb5,0xe1,0x12,0xb6,0xcc,0x27, + } + }, { 0xa404107, { 0xbb,0x04,0x4e,0x47,0xdd,0x5e,0x26,0x45, 0x1a,0xc9,0x56,0x24,0xa4,0x4c,0x82,0xb0, @@ -371,6 +427,13 @@ static const struct patch_digest phashes[] = { 0x13,0xbc,0xc5,0x25,0xe4,0xc5,0xc3,0x99, } }, + { 0xa404108, { + 0x69,0x67,0x43,0x06,0xf8,0x0c,0x62,0xdc, + 0xa4,0x21,0x30,0x4f,0x0f,0x21,0x2c,0xcb, + 0xcc,0x37,0xf1,0x1c,0xc3,0xf8,0x2f,0x19, + 0xdf,0x53,0x53,0x46,0xb1,0x15,0xea,0x00, + } + }, { 0xa500011, { 0x23,0x3d,0x70,0x7d,0x03,0xc3,0xc4,0xf4, 0x2b,0x82,0xc6,0x05,0xda,0x80,0x0a,0xf1, @@ -378,6 +441,13 @@ static const struct patch_digest phashes[] = { 0x11,0x5e,0x96,0x7e,0x71,0xe9,0xfc,0x74, } }, + { 0xa500012, { + 0xeb,0x74,0x0d,0x47,0xa1,0x8e,0x09,0xe4, + 0x93,0x4c,0xad,0x03,0x32,0x4c,0x38,0x16, + 0x10,0x39,0xdd,0x06,0xaa,0xce,0xd6,0x0f, + 0x62,0x83,0x9d,0x8e,0x64,0x55,0xbe,0x63, + } + }, { 0xa601209, { 0x66,0x48,0xd4,0x09,0x05,0xcb,0x29,0x32, 0x66,0xb7,0x9a,0x76,0xcd,0x11,0xf3,0x30, @@ -385,6 +455,13 @@ static const struct patch_digest phashes[] = { 0xe8,0x73,0xe2,0xd6,0xdb,0xd2,0x77,0x1d, } }, + { 0xa60120a, { + 0x0c,0x8b,0x3d,0xfd,0x52,0x52,0x85,0x7d, + 0x20,0x3a,0xe1,0x7e,0xa4,0x21,0x3b,0x7b, + 0x17,0x86,0xae,0xac,0x13,0xb8,0x63,0x9d, + 0x06,0x01,0xd0,0xa0,0x51,0x9a,0x91,0x2c, + } + }, { 0xa704107, { 0xf3,0xc6,0x58,0x26,0xee,0xac,0x3f,0xd6, 0xce,0xa1,0x72,0x47,0x3b,0xba,0x2b,0x93, @@ -392,6 +469,13 @@ static const struct patch_digest phashes[] = { 0x64,0x39,0x71,0x8c,0xce,0xe7,0x41,0x39, } }, + { 0xa704108, { + 0xd7,0x55,0x15,0x2b,0xfe,0xc4,0xbc,0x93, + 0xec,0x91,0xa0,0xae,0x45,0xb7,0xc3,0x98, + 0x4e,0xff,0x61,0x77,0x88,0xc2,0x70,0x49, + 0xe0,0x3a,0x1d,0x84,0x38,0x52,0xbf,0x5a, + } + }, { 0xa705206, { 0x8d,0xc0,0x76,0xbd,0x58,0x9f,0x8f,0xa4, 0x12,0x9d,0x21,0xfb,0x48,0x21,0xbc,0xe7, @@ -399,6 +483,13 @@ static const struct patch_digest phashes[] = { 0x03,0x35,0xe9,0xbe,0xfb,0x06,0xdf,0xfc, } }, + { 0xa705208, { + 0x30,0x1d,0x55,0x24,0xbc,0x6b,0x5a,0x19, + 0x0c,0x7d,0x1d,0x74,0xaa,0xd1,0xeb,0xd2, + 0x16,0x62,0xf7,0x5b,0xe1,0x1f,0x18,0x11, + 0x5c,0xf0,0x94,0x90,0x26,0xec,0x69,0xff, + } + }, { 0xa708007, { 0x6b,0x76,0xcc,0x78,0xc5,0x8a,0xa3,0xe3, 0x32,0x2d,0x79,0xe4,0xc3,0x80,0xdb,0xb2, @@ -406,6 +497,13 @@ static const struct patch_digest phashes[] = { 0xdf,0x92,0x73,0x84,0x87,0x3c,0x73,0x93, } }, + { 0xa708008, { + 0x08,0x6e,0xf0,0x22,0x4b,0x8e,0xc4,0x46, + 0x58,0x34,0xe6,0x47,0xa2,0x28,0xfd,0xab, + 0x22,0x3d,0xdd,0xd8,0x52,0x9e,0x1d,0x16, + 0xfa,0x01,0x68,0x14,0x79,0x3e,0xe8,0x6b, + } + }, { 0xa70c005, { 0x88,0x5d,0xfb,0x79,0x64,0xd8,0x46,0x3b, 0x4a,0x83,0x8e,0x77,0x7e,0xcf,0xb3,0x0f, @@ -413,6 +511,13 @@ static const struct patch_digest phashes[] = { 0xee,0x49,0xac,0xe1,0x8b,0x13,0xc5,0x13, } }, + { 0xa70c008, { + 0x0f,0xdb,0x37,0xa1,0x10,0xaf,0xd4,0x21, + 0x94,0x0d,0xa4,0xa2,0xe9,0x86,0x6c,0x0e, + 0x85,0x7c,0x36,0x30,0xa3,0x3a,0x78,0x66, + 0x18,0x10,0x60,0x0d,0x78,0x3d,0x44,0xd0, + } + }, { 0xaa00116, { 0xe8,0x4c,0x2c,0x88,0xa1,0xac,0x24,0x63, 0x65,0xe5,0xaa,0x2d,0x16,0xa9,0xc3,0xf5, @@ -441,4 +546,11 @@ static const struct patch_digest phashes[] = { 0x68,0x2f,0x46,0xee,0xfe,0xc6,0x6d,0xef, } }, + { 0xaa00216, { + 0x79,0xfb,0x5b,0x9f,0xb6,0xe6,0xa8,0xf5, + 0x4e,0x7c,0x4f,0x8e,0x1d,0xad,0xd0,0x08, + 0xc2,0x43,0x7c,0x8b,0xe6,0xdb,0xd0,0xd2, + 0xe8,0x39,0x26,0xc1,0xe5,0x5a,0x48,0xf1, + } + }, }; diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index c15c7b862bec1..fc539346599cb 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -43,8 +43,8 @@ #define DRIVER_VERSION "2.2" -static struct microcode_ops *microcode_ops; -bool dis_ucode_ldr = true; +static struct microcode_ops *microcode_ops; +static bool dis_ucode_ldr = false; bool force_minrev = IS_ENABLED(CONFIG_MICROCODE_LATE_FORCE_MINREV); module_param(force_minrev, bool, S_IRUSR | S_IWUSR); @@ -91,6 +91,9 @@ static bool amd_check_current_patch_level(void) u32 lvl, dummy, i; u32 *levels; + if (x86_cpuid_vendor() != X86_VENDOR_AMD) + return false; + native_rdmsr(MSR_AMD64_PATCH_LEVEL, lvl, dummy); levels = final_levels; @@ -102,27 +105,29 @@ static bool amd_check_current_patch_level(void) return false; } -static bool __init check_loader_disabled_bsp(void) +bool __init microcode_loader_disabled(void) { - static const char *__dis_opt_str = "dis_ucode_ldr"; - const char *cmdline = boot_command_line; - const char *option = __dis_opt_str; + if (dis_ucode_ldr) + return true; /* - * CPUID(1).ECX[31]: reserved for hypervisor use. This is still not - * completely accurate as xen pv guests don't see that CPUID bit set but - * that's good enough as they don't land on the BSP path anyway. + * Disable when: + * + * 1) The CPU does not support CPUID. + * + * 2) Bit 31 in CPUID[1]:ECX is clear + * The bit is reserved for hypervisor use. This is still not + * completely accurate as XEN PV guests don't see that CPUID bit + * set, but that's good enough as they don't land on the BSP + * path anyway. + * + * 3) Certain AMD patch levels are not allowed to be + * overwritten. */ - if (native_cpuid_ecx(1) & BIT(31)) - return true; - - if (x86_cpuid_vendor() == X86_VENDOR_AMD) { - if (amd_check_current_patch_level()) - return true; - } - - if (cmdline_find_option_bool(cmdline, option) <= 0) - dis_ucode_ldr = false; + if (!have_cpuid_p() || + native_cpuid_ecx(1) & BIT(31) || + amd_check_current_patch_level()) + dis_ucode_ldr = true; return dis_ucode_ldr; } @@ -132,7 +137,10 @@ void __init load_ucode_bsp(void) unsigned int cpuid_1_eax; bool intel = true; - if (!have_cpuid_p()) + if (cmdline_find_option_bool(boot_command_line, "dis_ucode_ldr") > 0) + dis_ucode_ldr = true; + + if (microcode_loader_disabled()) return; cpuid_1_eax = native_cpuid_eax(1); @@ -153,9 +161,6 @@ void __init load_ucode_bsp(void) return; } - if (check_loader_disabled_bsp()) - return; - if (intel) load_ucode_intel_bsp(&early_data); else @@ -166,6 +171,11 @@ void load_ucode_ap(void) { unsigned int cpuid_1_eax; + /* + * Can't use microcode_loader_disabled() here - .init section + * hell. It doesn't have to either - the BSP variant must've + * parsed cmdline already anyway. + */ if (dis_ucode_ldr) return; @@ -693,6 +703,8 @@ static int load_late_locked(void) return load_late_stop_cpus(true); case UCODE_NFOUND: return -ENOENT; + case UCODE_OK: + return 0; default: return -EBADFD; } @@ -817,7 +829,7 @@ static int __init microcode_init(void) struct cpuinfo_x86 *c = &boot_cpu_data; int error; - if (dis_ucode_ldr) + if (microcode_loader_disabled()) return -EINVAL; if (c->x86_vendor == X86_VENDOR_INTEL) diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index 9d7baf2573bcd..4e5a71796fbdb 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -389,7 +389,7 @@ static int __init save_builtin_microcode(void) if (xchg(&ucode_patch_va, NULL) != UCODE_BSP_LOADED) return 0; - if (dis_ucode_ldr || boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + if (microcode_loader_disabled() || boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return 0; uci.mc = get_microcode_blob(&uci, true); diff --git a/arch/x86/kernel/cpu/microcode/internal.h b/arch/x86/kernel/cpu/microcode/internal.h index 21776c529fa97..54f2b9582a7ce 100644 --- a/arch/x86/kernel/cpu/microcode/internal.h +++ b/arch/x86/kernel/cpu/microcode/internal.h @@ -94,7 +94,6 @@ static inline unsigned int x86_cpuid_family(void) return x86_family(eax); } -extern bool dis_ucode_ldr; extern bool force_minrev; #ifdef CONFIG_CPU_SUP_AMD diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c index 2d6aa5d2e3d77..6839440a4b31e 100644 --- a/arch/x86/kernel/cpu/mtrr/generic.c +++ b/arch/x86/kernel/cpu/mtrr/generic.c @@ -582,7 +582,7 @@ static void get_fixed_ranges(mtrr_type *frs) void mtrr_save_fixed_ranges(void *info) { - if (boot_cpu_has(X86_FEATURE_MTRR)) + if (mtrr_state.have_fixed) get_fixed_ranges(mtrr_state.fixed_ranges); } diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 3a6c069614eb8..976bdf15be22f 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -241,11 +241,15 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask)) return -EINVAL; + am = get_arch_mbm_state(hw_dom, rmid, eventid); + ret = __rmid_read(rmid, eventid, &msr_val); - if (ret) + if (ret) { + if (am && ret == -EINVAL) + am->prev_msr = 0; return ret; + } - am = get_arch_mbm_state(hw_dom, rmid, eventid); if (am) { am->chunks += mbm_overflow_count(am->prev_msr, msr_val, hw_res->mbm_width); diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index af5aa2c754c22..8fad19ec85551 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -46,8 +46,11 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 }, { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, + { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, { X86_FEATURE_BMEC, CPUID_EBX, 3, 0x80000020, 0 }, + { X86_FEATURE_TSA_SQ_NO, CPUID_ECX, 1, 0x80000021, 0 }, + { X86_FEATURE_TSA_L1_NO, CPUID_ECX, 2, 0x80000021, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_PMC_FREEZE, CPUID_EAX, 2, 0x80000022, 0 }, diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index c7f8c3200e8d7..0db6eeeeb6720 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -718,6 +718,8 @@ int arch_memory_failure(unsigned long pfn, int flags) goto out; } + sgx_unmark_page_reclaimable(page); + /* * TBD: Add additional plumbing to enable pre-emptive * action for asynchronous poison notification. Until diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index aaed20f46be4c..175c14567cf14 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -757,6 +757,9 @@ void fpu__clear_user_states(struct fpu *fpu) !fpregs_state_valid(fpu, smp_processor_id())) os_xrstor_supervisor(fpu->fpstate); + /* Ensure XFD state is in sync before reloading XSTATE */ + xfd_update_state(fpu->fpstate); + /* Reset user states in registers. */ restore_fpregs_from_init_fpstate(XFEATURE_MASK_USER_RESTORE); diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 12df54ff0e817..50f8c8a8483be 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -363,7 +363,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) goto fail; ip = trampoline + size; - if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) + if (cpu_wants_rethunk_at(ip)) __text_gen_insn(ip, JMP32_INSN_OPCODE, ip, x86_return_thunk, JMP32_INSN_SIZE); else memcpy(ip, retq, sizeof(retq)); diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c index de001b2146abf..375f2d7f1762d 100644 --- a/arch/x86/kernel/head32.c +++ b/arch/x86/kernel/head32.c @@ -145,10 +145,6 @@ void __init __no_stack_protector mk_early_pgtbl_32(void) *ptr = (unsigned long)ptep + PAGE_OFFSET; #ifdef CONFIG_MICROCODE_INITRD32 - /* Running on a hypervisor? */ - if (native_cpuid_ecx(1) & BIT(31)) - return; - params = (struct boot_params *)__pa_nodebug(&boot_params); if (!params->hdr.ramdisk_size || !params->hdr.ramdisk_image) return; diff --git a/arch/x86/kernel/ioport.c b/arch/x86/kernel/ioport.c index e2fab3ceb09fb..9a101150376db 100644 --- a/arch/x86/kernel/ioport.c +++ b/arch/x86/kernel/ioport.c @@ -33,8 +33,9 @@ void io_bitmap_share(struct task_struct *tsk) set_tsk_thread_flag(tsk, TIF_IO_BITMAP); } -static void task_update_io_bitmap(struct task_struct *tsk) +static void task_update_io_bitmap(void) { + struct task_struct *tsk = current; struct thread_struct *t = &tsk->thread; if (t->iopl_emul == 3 || t->io_bitmap) { @@ -54,7 +55,12 @@ void io_bitmap_exit(struct task_struct *tsk) struct io_bitmap *iobm = tsk->thread.io_bitmap; tsk->thread.io_bitmap = NULL; - task_update_io_bitmap(tsk); + /* + * Don't touch the TSS when invoked on a failed fork(). TSS + * reflects the state of @current and not the state of @tsk. + */ + if (tsk == current) + task_update_io_bitmap(); if (iobm && refcount_dec_and_test(&iobm->refcnt)) kfree(iobm); } @@ -192,8 +198,7 @@ SYSCALL_DEFINE1(iopl, unsigned int, level) } t->iopl_emul = level; - task_update_io_bitmap(current); - + task_update_io_bitmap(); return 0; } diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index b8ab9ee5896c1..b5ef2c2899f2a 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -1066,16 +1066,6 @@ static void kvm_wait(u8 *ptr, u8 val) */ void __init kvm_spinlock_init(void) { - /* - * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an - * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is - * preferred over native qspinlock when vCPU is preempted. - */ - if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) { - pr_info("PV spinlocks disabled, no host support\n"); - return; - } - /* * Disable PV spinlocks and use native qspinlock when dedicated pCPUs * are available. @@ -1095,6 +1085,16 @@ void __init kvm_spinlock_init(void) goto out; } + /* + * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an + * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is + * preferred over native qspinlock when vCPU is preempted. + */ + if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) { + pr_info("PV spinlocks disabled, no host support\n"); + return; + } + pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index 5f71a0cf4399a..cfb7163cf90e9 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -312,6 +312,9 @@ int module_finalize(const Elf_Ehdr *hdr, void *pseg = (void *)para->sh_addr; apply_paravirt(pseg, pseg + para->sh_size); } + + its_init_mod(me); + if (retpolines || cfi) { void *rseg = NULL, *cseg = NULL; unsigned int rsize = 0, csize = 0; @@ -332,6 +335,9 @@ int module_finalize(const Elf_Ehdr *hdr, void *rseg = (void *)retpolines->sh_addr; apply_retpolines(rseg, rseg + retpolines->sh_size); } + + its_fini_mod(me); + if (returns) { void *rseg = (void *)returns->sh_addr; apply_returns(rseg, rseg + returns->sh_size); @@ -379,4 +385,5 @@ int module_finalize(const Elf_Ehdr *hdr, void module_arch_cleanup(struct module *mod) { alternatives_smp_module_del(mod); + its_free_mod(mod); } diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 6da2cfa23c293..35fd5f1444fdb 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -39,8 +39,12 @@ #define CREATE_TRACE_POINTS #include +/* + * An emergency handler can be set in any context including NMI + */ struct nmi_desc { raw_spinlock_t lock; + nmi_handler_t emerg_handler; struct list_head head; }; @@ -131,9 +135,22 @@ static void nmi_check_duration(struct nmiaction *action, u64 duration) static int nmi_handle(unsigned int type, struct pt_regs *regs) { struct nmi_desc *desc = nmi_to_desc(type); + nmi_handler_t ehandler; struct nmiaction *a; int handled=0; + /* + * Call the emergency handler, if set + * + * In the case of crash_nmi_callback() emergency handler, it will + * return in the case of the crashing CPU to enable it to complete + * other necessary crashing actions ASAP. Other handlers in the + * linked list won't need to be run. + */ + ehandler = desc->emerg_handler; + if (ehandler) + return ehandler(type, regs); + rcu_read_lock(); /* @@ -223,6 +240,31 @@ void unregister_nmi_handler(unsigned int type, const char *name) } EXPORT_SYMBOL_GPL(unregister_nmi_handler); +/** + * set_emergency_nmi_handler - Set emergency handler + * @type: NMI type + * @handler: the emergency handler to be stored + * + * Set an emergency NMI handler which, if set, will preempt all the other + * handlers in the linked list. If a NULL handler is passed in, it will clear + * it. It is expected that concurrent calls to this function will not happen + * or the system is screwed beyond repair. + */ +void set_emergency_nmi_handler(unsigned int type, nmi_handler_t handler) +{ + struct nmi_desc *desc = nmi_to_desc(type); + + if (WARN_ON_ONCE(desc->emerg_handler == handler)) + return; + desc->emerg_handler = handler; + + /* + * Ensure the emergency handler is visible to other CPUs before + * function return + */ + smp_wmb(); +} + static void pci_serr_error(unsigned char reason, struct pt_regs *regs) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index c6af118ce9a08..84726a69497c7 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -181,6 +181,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) frame->ret_addr = (unsigned long) ret_from_fork_asm; p->thread.sp = (unsigned long) fork_frame; p->thread.io_bitmap = NULL; + clear_tsk_thread_flag(p, TIF_IO_BITMAP); p->thread.iopl_warn = 0; memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps)); @@ -469,6 +470,11 @@ void native_tss_update_io_bitmap(void) } else { struct io_bitmap *iobm = t->io_bitmap; + if (WARN_ON_ONCE(!iobm)) { + clear_thread_flag(TIF_IO_BITMAP); + native_tss_invalidate_io_bitmap(); + } + /* * Only copy bitmap data when the sequence number differs. The * update time is accounted to the incoming task. @@ -923,19 +929,24 @@ static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c) */ static __cpuidle void mwait_idle(void) { + if (need_resched()) + return; + + x86_idle_clear_cpu_buffers(); + if (!current_set_polling_and_test()) { - if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) { - mb(); /* quirk */ - clflush((void *)¤t_thread_info()->flags); - mb(); /* quirk */ - } + const void *addr = ¤t_thread_info()->flags; - __monitor((void *)¤t_thread_info()->flags, 0, 0); - if (!need_resched()) { - __sti_mwait(0, 0); - raw_local_irq_disable(); - } + alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr)); + __monitor(addr, 0, 0); + if (need_resched()) + goto out; + + __sti_mwait(0, 0); + raw_local_irq_disable(); } + +out: __current_clr_polling(); } diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 830425e6d38e2..456e61070a730 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -908,15 +908,11 @@ void nmi_shootdown_cpus(nmi_shootdown_cb callback) shootdown_callback = callback; atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1); - /* Would it be better to replace the trap vector here? */ - if (register_nmi_handler(NMI_LOCAL, crash_nmi_callback, - NMI_FLAG_FIRST, "crash")) - return; /* Return what? */ + /* - * Ensure the new callback function is set before sending - * out the NMI + * Set emergency handler to preempt other handlers. */ - wmb(); + set_emergency_nmi_handler(NMI_LOCAL, crash_nmi_callback); apic_send_IPI_allbutself(NMI_VECTOR); diff --git a/arch/x86/kernel/sev-shared.c b/arch/x86/kernel/sev-shared.c index acbec4de3ec31..b90dfa46ec5b5 100644 --- a/arch/x86/kernel/sev-shared.c +++ b/arch/x86/kernel/sev-shared.c @@ -1068,6 +1068,24 @@ static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info) } } +static inline void sev_evict_cache(void *va, int npages) +{ + volatile u8 val __always_unused; + u8 *bytes = va; + int page_idx; + + /* + * For SEV guests, a read from the first/last cache-lines of a 4K page + * using the guest key is sufficient to cause a flush of all cache-lines + * associated with that 4K page without incurring all the overhead of a + * full CLFLUSH sequence. + */ + for (page_idx = 0; page_idx < npages; page_idx++) { + val = bytes[page_idx * PAGE_SIZE]; + val = bytes[page_idx * PAGE_SIZE + PAGE_SIZE - 1]; + } +} + static void pvalidate_pages(struct snp_psc_desc *desc) { struct psc_entry *e; @@ -1100,6 +1118,24 @@ static void pvalidate_pages(struct snp_psc_desc *desc) sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); } } + + /* + * If not affected by the cache-coherency vulnerability there is no need + * to perform the cache eviction mitigation. + */ + if (cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO)) + return; + + for (i = 0; i <= desc->hdr.end_entry; i++) { + e = &desc->entries[i]; + + /* + * If validating memory (making it private) perform the cache + * eviction mitigation. + */ + if (e->operation == SNP_PAGE_STATE_PRIVATE) + sev_evict_cache(pfn_to_kaddr(e->gfn), e->pagesize ? 512 : 1); + } } static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc) diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c index 9905dc0e0b096..c6a9a9d3ff2f3 100644 --- a/arch/x86/kernel/sev.c +++ b/arch/x86/kernel/sev.c @@ -688,12 +688,14 @@ static void __head early_set_pages_state(unsigned long vaddr, unsigned long paddr, unsigned long npages, enum psc_op op) { - unsigned long paddr_end; + unsigned long vaddr_begin, paddr_end; u64 val; int ret; vaddr = vaddr & PAGE_MASK; + vaddr_begin = vaddr; + paddr = paddr & PAGE_MASK; paddr_end = paddr + (npages << PAGE_SHIFT); @@ -736,6 +738,13 @@ early_set_pages_state(unsigned long vaddr, unsigned long paddr, paddr += PAGE_SIZE; } + /* + * If validating memory (making it private) and affected by the + * cache-coherency vulnerability, perform the cache eviction mitigation. + */ + if (op == SNP_PAGE_STATE_PRIVATE && !cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO)) + sev_evict_cache((void *)vaddr_begin, npages); + return; e_term: diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index 07961b362e2a0..bcea882e022f5 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -81,7 +81,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, break; case RET: - if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) + if (cpu_wants_rethunk_at(insn)) code = text_gen_insn(JMP32_INSN_OPCODE, insn, x86_return_thunk); else code = &retinsn; @@ -90,7 +90,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, case JCC: if (!func) { func = __static_call_return; - if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) + if (cpu_wants_rethunk()) func = x86_return_thunk; } diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d8d9bc5a9b328..a52db362a65d1 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -92,10 +92,17 @@ __always_inline int is_valid_bugaddr(unsigned long addr) /* * Check for UD1 or UD2, accounting for Address Size Override Prefixes. - * If it's a UD1, get the ModRM byte to pass along to UBSan. + * If it's a UD1, further decode to determine its use: + * + * UBSan{0}: 67 0f b9 00 ud1 (%eax),%eax + * UBSan{10}: 67 0f b9 40 10 ud1 0x10(%eax),%eax + * static_call: 0f b9 cc ud1 %esp,%ecx + * + * Notably UBSAN uses EAX, static_call uses ECX. */ -__always_inline int decode_bug(unsigned long addr, u32 *imm) +__always_inline int decode_bug(unsigned long addr, s32 *imm, int *len) { + unsigned long start = addr; u8 v; if (addr < TASK_SIZE_MAX) @@ -108,24 +115,42 @@ __always_inline int decode_bug(unsigned long addr, u32 *imm) return BUG_NONE; v = *(u8 *)(addr++); - if (v == SECOND_BYTE_OPCODE_UD2) + if (v == SECOND_BYTE_OPCODE_UD2) { + *len = addr - start; return BUG_UD2; + } - if (!IS_ENABLED(CONFIG_UBSAN_TRAP) || v != SECOND_BYTE_OPCODE_UD1) + if (v != SECOND_BYTE_OPCODE_UD1) return BUG_NONE; - /* Retrieve the immediate (type value) for the UBSAN UD1 */ - v = *(u8 *)(addr++); - if (X86_MODRM_RM(v) == 4) - addr++; - *imm = 0; - if (X86_MODRM_MOD(v) == 1) - *imm = *(u8 *)addr; - else if (X86_MODRM_MOD(v) == 2) - *imm = *(u32 *)addr; - else - WARN_ONCE(1, "Unexpected MODRM_MOD: %u\n", X86_MODRM_MOD(v)); + v = *(u8 *)(addr++); /* ModRM */ + + if (X86_MODRM_MOD(v) != 3 && X86_MODRM_RM(v) == 4) + addr++; /* SIB */ + + /* Decode immediate, if present */ + switch (X86_MODRM_MOD(v)) { + case 0: if (X86_MODRM_RM(v) == 5) + addr += 4; /* RIP + disp32 */ + break; + + case 1: *imm = *(s8 *)addr; + addr += 1; + break; + + case 2: *imm = *(s32 *)addr; + addr += 4; + break; + + case 3: break; + } + + /* record instruction length */ + *len = addr - start; + + if (X86_MODRM_REG(v) == 0) /* EAX */ + return BUG_UD1_UBSAN; return BUG_UD1; } @@ -256,10 +281,10 @@ static inline void handle_invalid_op(struct pt_regs *regs) static noinstr bool handle_bug(struct pt_regs *regs) { bool handled = false; - int ud_type; - u32 imm; + int ud_type, ud_len; + s32 ud_imm; - ud_type = decode_bug(regs->ip, &imm); + ud_type = decode_bug(regs->ip, &ud_imm, &ud_len); if (ud_type == BUG_NONE) return handled; @@ -279,15 +304,28 @@ static noinstr bool handle_bug(struct pt_regs *regs) */ if (regs->flags & X86_EFLAGS_IF) raw_local_irq_enable(); - if (ud_type == BUG_UD2) { + + switch (ud_type) { + case BUG_UD2: if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN || handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) { - regs->ip += LEN_UD2; + regs->ip += ud_len; handled = true; } - } else if (IS_ENABLED(CONFIG_UBSAN_TRAP)) { - pr_crit("%s at %pS\n", report_ubsan_failure(regs, imm), (void *)regs->ip); + break; + + case BUG_UD1_UBSAN: + if (IS_ENABLED(CONFIG_UBSAN_TRAP)) { + pr_crit("%s at %pS\n", + report_ubsan_failure(regs, ud_imm), + (void *)regs->ip); + } + break; + + default: + break; } + if (regs->flags & X86_EFLAGS_IF) raw_local_irq_disable(); instrumentation_end(); @@ -937,24 +975,32 @@ static bool is_sysenter_singlestep(struct pt_regs *regs) #endif } -static __always_inline unsigned long debug_read_clear_dr6(void) +static __always_inline unsigned long debug_read_reset_dr6(void) { unsigned long dr6; + get_debugreg(dr6, 6); + dr6 ^= DR6_RESERVED; /* Flip to positive polarity */ + /* * The Intel SDM says: * - * Certain debug exceptions may clear bits 0-3. The remaining - * contents of the DR6 register are never cleared by the - * processor. To avoid confusion in identifying debug - * exceptions, debug handlers should clear the register before - * returning to the interrupted task. + * Certain debug exceptions may clear bits 0-3 of DR6. + * + * BLD induced #DB clears DR6.BLD and any other debug + * exception doesn't modify DR6.BLD. * - * Keep it simple: clear DR6 immediately. + * RTM induced #DB clears DR6.RTM and any other debug + * exception sets DR6.RTM. + * + * To avoid confusion in identifying debug exceptions, + * debug handlers should set DR6.BLD and DR6.RTM, and + * clear other DR6 bits before returning. + * + * Keep it simple: write DR6 with its architectural reset + * value 0xFFFF0FF0, defined as DR6_RESERVED, immediately. */ - get_debugreg(dr6, 6); set_debugreg(DR6_RESERVED, 6); - dr6 ^= DR6_RESERVED; /* Flip to positive polarity */ return dr6; } @@ -1150,19 +1196,19 @@ static __always_inline void exc_debug_user(struct pt_regs *regs, /* IST stack entry */ DEFINE_IDTENTRY_DEBUG(exc_debug) { - exc_debug_kernel(regs, debug_read_clear_dr6()); + exc_debug_kernel(regs, debug_read_reset_dr6()); } /* User entry, runs on regular task stack */ DEFINE_IDTENTRY_DEBUG_USER(exc_debug) { - exc_debug_user(regs, debug_read_clear_dr6()); + exc_debug_user(regs, debug_read_reset_dr6()); } #else /* 32 bit does not have separate entry points. */ DEFINE_IDTENTRY_RAW(exc_debug) { - unsigned long dr6 = debug_read_clear_dr6(); + unsigned long dr6 = debug_read_reset_dr6(); if (user_mode(regs)) exc_debug_user(regs, dr6); diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c index 5a4b21389b1d9..d432f3824f0c2 100644 --- a/arch/x86/kernel/umip.c +++ b/arch/x86/kernel/umip.c @@ -156,15 +156,26 @@ static int identify_insn(struct insn *insn) if (!insn->modrm.nbytes) return -EINVAL; - /* All the instructions of interest start with 0x0f. */ - if (insn->opcode.bytes[0] != 0xf) + /* The instructions of interest have 2-byte opcodes: 0F 00 or 0F 01. */ + if (insn->opcode.nbytes < 2 || insn->opcode.bytes[0] != 0xf) return -EINVAL; if (insn->opcode.bytes[1] == 0x1) { switch (X86_MODRM_REG(insn->modrm.value)) { case 0: + /* The reg form of 0F 01 /0 encodes VMX instructions. */ + if (X86_MODRM_MOD(insn->modrm.value) == 3) + return -EINVAL; + return UMIP_INST_SGDT; case 1: + /* + * The reg form of 0F 01 /1 encodes MONITOR/MWAIT, + * STAC/CLAC, and ENCLS. + */ + if (X86_MODRM_MOD(insn->modrm.value) == 3) + return -EINVAL; + return UMIP_INST_SIDT; case 4: return UMIP_INST_SMSW; diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 60eb8baa44d7b..0929d7fe7e274 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -500,10 +500,18 @@ SECTIONS PROVIDE(__ref_stack_chk_guard = __stack_chk_guard); /* - * The ASSERT() sink to . is intentional, for binutils 2.14 compatibility: + * COMPILE_TEST kernels can be large - CONFIG_KASAN, for example, can cause + * this. Let's assume that nobody will be running a COMPILE_TEST kernel and + * let's assert that fuller build coverage is more valuable than being able to + * run a COMPILE_TEST kernel. + */ +#ifndef CONFIG_COMPILE_TEST +/* + * The ASSERT() sync to . is intentional, for binutils 2.14 compatibility: */ . = ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE), "kernel image bigger than KERNEL_IMAGE_SIZE"); +#endif #ifdef CONFIG_X86_64 /* @@ -541,4 +549,14 @@ INIT_PER_CPU(irq_stack_backing_store); "SRSO function pair won't alias"); #endif +#if defined(CONFIG_MITIGATION_ITS) && !defined(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B) +. = ASSERT(__x86_indirect_its_thunk_rax & 0x20, "__x86_indirect_thunk_rax not in second half of cacheline"); +. = ASSERT(((__x86_indirect_its_thunk_rcx - __x86_indirect_its_thunk_rax) % 64) == 0, "Indirect thunks are not cacheline apart"); +. = ASSERT(__x86_indirect_its_thunk_array == __x86_indirect_its_thunk_rax, "Gap in ITS thunk array"); +#endif + +#if defined(CONFIG_MITIGATION_ITS) && !defined(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B) +. = ASSERT(its_return_thunk & 0x20, "its_return_thunk not in second half of cacheline"); +#endif + #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index a6cffeff75d40..2c0bc6a93ec3e 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -780,6 +780,7 @@ void kvm_set_cpu_caps(void) kvm_cpu_cap_mask(CPUID_8000_0021_EAX, F(NO_NESTED_DATA_BP) | F(LFENCE_RDTSC) | 0 /* SmmPgCfgLock */ | + F(VERW_CLEAR) | F(NULL_SEL_CLR_BASE) | F(AUTOIBRS) | 0 /* PrefetchCtlMsr */ ); @@ -790,6 +791,15 @@ void kvm_set_cpu_caps(void) F(PERFMON_V2) ); + kvm_cpu_cap_check_and_set(X86_FEATURE_VERW_CLEAR); + + kvm_cpu_cap_init_kvm_defined(CPUID_8000_0021_ECX, + F(TSA_SQ_NO) | F(TSA_L1_NO) + ); + + kvm_cpu_cap_check_and_set(X86_FEATURE_TSA_SQ_NO); + kvm_cpu_cap_check_and_set(X86_FEATURE_TSA_L1_NO); + /* * Synthesize "LFENCE is serializing" into the AMD-defined entry in * KVM's supported CPUID if the feature is reported as supported by the @@ -1296,8 +1306,9 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->eax = entry->ebx = entry->ecx = entry->edx = 0; break; case 0x80000021: - entry->ebx = entry->ecx = entry->edx = 0; + entry->ebx = entry->edx = 0; cpuid_entry_override(entry, CPUID_8000_0021_EAX); + cpuid_entry_override(entry, CPUID_8000_0021_ECX); break; /* AMD Extended Performance Monitoring and Debug */ case 0x80000022: { diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 2673cd5c46cb4..313cb2ffd6aae 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -5134,12 +5134,11 @@ void init_decode_cache(struct x86_emulate_ctxt *ctxt) ctxt->mem_read.end = 0; } -int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) +int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, bool check_intercepts) { const struct x86_emulate_ops *ops = ctxt->ops; int rc = X86EMUL_CONTINUE; int saved_dst_type = ctxt->dst.type; - bool is_guest_mode = ctxt->ops->is_guest_mode(ctxt); ctxt->mem_read.pos = 0; @@ -5187,7 +5186,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) fetch_possible_mmx_operand(&ctxt->dst); } - if (unlikely(is_guest_mode) && ctxt->intercept) { + if (unlikely(check_intercepts) && ctxt->intercept) { rc = emulator_check_intercept(ctxt, ctxt->intercept, X86_ICPT_PRE_EXCEPT); if (rc != X86EMUL_CONTINUE) @@ -5216,7 +5215,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) goto done; } - if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) { + if (unlikely(check_intercepts) && (ctxt->d & Intercept)) { rc = emulator_check_intercept(ctxt, ctxt->intercept, X86_ICPT_POST_EXCEPT); if (rc != X86EMUL_CONTINUE) @@ -5270,7 +5269,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) special_insn: - if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) { + if (unlikely(check_intercepts) && (ctxt->d & Intercept)) { rc = emulator_check_intercept(ctxt, ctxt->intercept, X86_ICPT_POST_MEMACCESS); if (rc != X86EMUL_CONTINUE) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index bd3fbd5be5da6..223f4fa6a849b 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1929,6 +1929,9 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) goto out_flush_all; + if (is_noncanonical_address(entries[i], vcpu)) + continue; + /* * Lower 12 bits of 'address' encode the number of additional * pages to flush. diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index be7aeb9b8ea3b..b940ee604bd84 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -220,7 +220,6 @@ struct x86_emulate_ops { void (*set_nmi_mask)(struct x86_emulate_ctxt *ctxt, bool masked); bool (*is_smm)(struct x86_emulate_ctxt *ctxt); - bool (*is_guest_mode)(struct x86_emulate_ctxt *ctxt); int (*leave_smm)(struct x86_emulate_ctxt *ctxt); void (*triple_fault)(struct x86_emulate_ctxt *ctxt); int (*set_xcr)(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr); @@ -498,7 +497,7 @@ bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt); #define EMULATION_RESTART 1 #define EMULATION_INTERCEPTED 2 void init_decode_cache(struct x86_emulate_ctxt *ctxt); -int x86_emulate_insn(struct x86_emulate_ctxt *ctxt); +int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, bool check_intercepts); int emulator_task_switch(struct x86_emulate_ctxt *ctxt, u16 tss_selector, int idt_index, int reason, bool has_error_code, u32 error_code); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 66c7f2367bb34..af4ae9216667e 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -750,7 +750,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic) * just set SVI. */ if (unlikely(apic->apicv_active)) - static_call_cond(kvm_x86_hwapic_isr_update)(vec); + static_call_cond(kvm_x86_hwapic_isr_update)(apic->vcpu, vec); else { ++apic->isr_count; BUG_ON(apic->isr_count > MAX_APIC_VECTOR); @@ -795,7 +795,7 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) * and must be left alone. */ if (unlikely(apic->apicv_active)) - static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); + static_call_cond(kvm_x86_hwapic_isr_update)(apic->vcpu, apic_find_highest_isr(apic)); else { --apic->isr_count; BUG_ON(apic->isr_count < 0); @@ -803,6 +803,17 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) } } +void kvm_apic_update_hwapic_isr(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (WARN_ON_ONCE(!lapic_in_kernel(vcpu)) || !apic->apicv_active) + return; + + static_call(kvm_x86_hwapic_isr_update)(vcpu, apic_find_highest_isr(apic)); +} +EXPORT_SYMBOL_GPL(kvm_apic_update_hwapic_isr); + int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) { /* This may race with setting of irr in __apic_accept_irq() and @@ -836,6 +847,8 @@ static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map, if (min > map->max_apic_id) return 0; + min = array_index_nospec(min, map->max_apic_id + 1); + for_each_set_bit(i, ipi_bitmap, min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { if (map->phys_map[min + i]) { @@ -2772,7 +2785,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) if (apic->apicv_active) { static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, -1); - static_call_cond(kvm_x86_hwapic_isr_update)(-1); + static_call_cond(kvm_x86_hwapic_isr_update)(vcpu, -1); } vcpu->arch.apic_arb_prio = 0; @@ -3072,7 +3085,7 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) if (apic->apicv_active) { static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, apic_find_highest_irr(apic)); - static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); + static_call_cond(kvm_x86_hwapic_isr_update)(vcpu, apic_find_highest_isr(apic)); } kvm_make_request(KVM_REQ_EVENT, vcpu); if (ioapic_in_kernel(vcpu->kvm)) diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 0a0ea4b5dd8ce..0dd069b8d6d11 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -124,6 +124,7 @@ int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info); int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu); +void kvm_apic_update_hwapic_isr(struct kvm_vcpu *vcpu); int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index da2d82e3a8735..f2cd8cfb0ef55 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -588,6 +588,7 @@ int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = pmu->global_ctrl; break; case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: case MSR_CORE_PERF_GLOBAL_OVF_CTRL: msr_info->data = 0; break; @@ -649,6 +650,10 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!msr_info->host_initiated) pmu->global_status &= ~data; break; + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: + if (!msr_info->host_initiated) + pmu->global_status |= data & ~pmu->global_status_mask; + break; default: kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info); diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 2f4e155080bad..be23712354bd8 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -17,6 +17,7 @@ enum kvm_only_cpuid_leafs { CPUID_8000_0007_EDX, CPUID_8000_0022_EAX, CPUID_7_2_EDX, + CPUID_8000_0021_ECX, NR_KVM_CPU_CAPS, NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS, @@ -61,6 +62,10 @@ enum kvm_only_cpuid_leafs { /* CPUID level 0x80000022 (EAX) */ #define KVM_X86_FEATURE_PERFMON_V2 KVM_X86_FEATURE(CPUID_8000_0022_EAX, 0) +/* CPUID level 0x80000021 (ECX) */ +#define KVM_X86_FEATURE_TSA_SQ_NO KVM_X86_FEATURE(CPUID_8000_0021_ECX, 1) +#define KVM_X86_FEATURE_TSA_L1_NO KVM_X86_FEATURE(CPUID_8000_0021_ECX, 2) + struct cpuid_reg { u32 function; u32 index; @@ -90,6 +95,7 @@ static const struct cpuid_reg reverse_cpuid[] = { [CPUID_8000_0021_EAX] = {0x80000021, 0, CPUID_EAX}, [CPUID_8000_0022_EAX] = {0x80000022, 0, CPUID_EAX}, [CPUID_7_2_EDX] = { 7, 2, CPUID_EDX}, + [CPUID_8000_0021_ECX] = {0x80000021, 0, CPUID_ECX}, }; /* @@ -129,6 +135,8 @@ static __always_inline u32 __feature_translate(int x86_feature) KVM_X86_TRANSLATE_FEATURE(PERFMON_V2); KVM_X86_TRANSLATE_FEATURE(RRSBA_CTRL); KVM_X86_TRANSLATE_FEATURE(BHI_CTRL); + KVM_X86_TRANSLATE_FEATURE(TSA_SQ_NO); + KVM_X86_TRANSLATE_FEATURE(TSA_L1_NO); default: return x86_feature; } diff --git a/arch/x86/kvm/smm.c b/arch/x86/kvm/smm.c index b42111a24cc28..8e38c51359d05 100644 --- a/arch/x86/kvm/smm.c +++ b/arch/x86/kvm/smm.c @@ -131,6 +131,7 @@ void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm) kvm_mmu_reset_context(vcpu); } +EXPORT_SYMBOL_GPL(kvm_smm_changed); void process_smi(struct kvm_vcpu *vcpu) { diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 3fd47de14b38a..0bad24f763d22 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -117,6 +117,7 @@ static bool amd_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: return pmu->version > 1; default: if (msr > MSR_F15H_PERF_CTR5 && diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 99e72b8a96ac0..67c01bd332f6f 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -1782,6 +1782,10 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) struct kvm_vcpu *src_vcpu; unsigned long i; + if (src->created_vcpus != atomic_read(&src->online_vcpus) || + dst->created_vcpus != atomic_read(&dst->online_vcpus)) + return -EBUSY; + if (!sev_es_guest(src)) return 0; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 1d06b8fc15a85..99180d0af3ea6 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1503,7 +1503,7 @@ static void svm_clear_current_vmcb(struct vmcb *vmcb) { int i; - for_each_online_cpu(i) + for_each_possible_cpu(i) cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL); } @@ -2014,11 +2014,11 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) svm->asid = sd->next_asid++; } -static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value) +static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value) { - struct vmcb *vmcb = svm->vmcb; + struct vmcb *vmcb = to_svm(vcpu)->vmcb; - if (svm->vcpu.arch.guest_state_protected) + if (vcpu->arch.guest_state_protected) return; if (unlikely(value != vmcb->save.dr6)) { @@ -2211,12 +2211,6 @@ static int shutdown_interception(struct kvm_vcpu *vcpu) struct kvm_run *kvm_run = vcpu->run; struct vcpu_svm *svm = to_svm(vcpu); - /* - * The VM save area has already been encrypted so it - * cannot be reinitialized - just terminate. - */ - if (sev_es_guest(vcpu->kvm)) - return -EINVAL; /* * VMCB is undefined after a SHUTDOWN intercept. INIT the vCPU to put @@ -2225,9 +2219,18 @@ static int shutdown_interception(struct kvm_vcpu *vcpu) * userspace. At a platform view, INIT is acceptable behavior as * there exist bare metal platforms that automatically INIT the CPU * in response to shutdown. + * + * The VM save area for SEV-ES guests has already been encrypted so it + * cannot be reinitialized, i.e. synthesizing INIT is futile. */ - clear_page(svm->vmcb); - kvm_vcpu_reset(vcpu, true); + if (!sev_es_guest(vcpu->kvm)) { + clear_page(svm->vmcb); +#ifdef CONFIG_KVM_SMM + if (is_smm(vcpu)) + kvm_smm_changed(vcpu, false); +#endif + kvm_vcpu_reset(vcpu, true); + } kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; return 0; @@ -3180,7 +3183,11 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) if (data & DEBUGCTL_RESERVED_BITS) return 1; + if (svm_get_lbr_vmcb(svm)->save.dbgctl == data) + break; + svm_get_lbr_vmcb(svm)->save.dbgctl = data; + vmcb_mark_dirty(svm->vmcb, VMCB_LBR); svm_update_lbrv(vcpu); break; case MSR_VM_HSAVE_PA: @@ -4026,8 +4033,7 @@ static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); u64 cr8; - if (nested_svm_virtualize_tpr(vcpu) || - kvm_vcpu_apicv_active(vcpu)) + if (nested_svm_virtualize_tpr(vcpu)) return; cr8 = kvm_get_cr8(vcpu); @@ -4154,8 +4160,20 @@ static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu) static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) { - if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && - to_svm(vcpu)->vmcb->control.exit_info_1) + struct vcpu_svm *svm = to_svm(vcpu); + + /* + * Next RIP must be provided as IRQs are disabled, and accessing guest + * memory to decode the instruction might fault, i.e. might sleep. + */ + if (!nrips || !svm->vmcb->control.next_rip) + return EXIT_FASTPATH_NONE; + + if (is_guest_mode(vcpu)) + return EXIT_FASTPATH_NONE; + + if (svm->vmcb->control.exit_code == SVM_EXIT_MSR && + svm->vmcb->control.exit_info_1) return handle_fastpath_set_msr_irqoff(vcpu); return EXIT_FASTPATH_NONE; @@ -4167,6 +4185,18 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in guest_state_enter_irqoff(); + /* + * Set RFLAGS.IF prior to VMRUN, as the host's RFLAGS.IF at the time of + * VMRUN controls whether or not physical IRQs are masked (KVM always + * runs with V_INTR_MASKING_MASK). Toggle RFLAGS.IF here to avoid the + * temptation to do STI+VMRUN+CLI, as AMD CPUs bleed the STI shadow + * into guest state if delivery of an event during VMRUN triggers a + * #VMEXIT, and the guest_state transitions already tell lockdep that + * IRQs are being enabled/disabled. Note! GIF=0 for the entirety of + * this path, so IRQs aren't actually unmasked while running host code. + */ + raw_local_irq_enable(); + amd_clear_divider(); if (sev_es_guest(vcpu->kvm)) @@ -4174,15 +4204,18 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in else __svm_vcpu_run(svm, spec_ctrl_intercepted); + raw_local_irq_disable(); + guest_state_exit_irqoff(); } -static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) +static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) { + bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT; struct vcpu_svm *svm = to_svm(vcpu); bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL); - trace_kvm_entry(vcpu); + trace_kvm_entry(vcpu, force_immediate_exit); svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; @@ -4201,9 +4234,12 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) * is enough to force an immediate vmexit. */ disable_nmi_singlestep(svm); - smp_send_reschedule(vcpu->cpu); + force_immediate_exit = true; } + if (force_immediate_exit) + smp_send_reschedule(vcpu->cpu); + pre_svm_run(vcpu); sync_lapic_to_cr8(vcpu); @@ -4217,13 +4253,14 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) svm_hv_update_vp_id(svm->vmcb, vcpu); /* - * Run with all-zero DR6 unless needed, so that we can get the exact cause - * of a #DB. + * Run with all-zero DR6 unless the guest can write DR6 freely, so that + * KVM can get the exact cause of a #DB. Note, loading guest DR6 from + * KVM's snapshot is only necessary when DR accesses won't exit. */ - if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) - svm_set_dr6(svm, vcpu->arch.dr6); - else - svm_set_dr6(svm, DR6_ACTIVE_LOW); + if (unlikely(run_flags & KVM_RUN_LOAD_GUEST_DR6)) + svm_set_dr6(vcpu, vcpu->arch.dr6); + else if (likely(!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))) + svm_set_dr6(vcpu, DR6_ACTIVE_LOW); clgi(); kvm_load_guest_xsave_state(vcpu); @@ -4299,9 +4336,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) svm_complete_interrupts(vcpu); - if (is_guest_mode(vcpu)) - return EXIT_FASTPATH_NONE; - return svm_exit_handlers_fastpath(vcpu); } @@ -5058,8 +5092,6 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, - .request_immediate_exit = __kvm_request_immediate_exit, - .sched_in = svm_sched_in, .nested_ops = &svm_nested_ops, diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index ef2ebabb059c8..81ecb9e1101d7 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -167,13 +167,12 @@ SYM_FUNC_START(__svm_vcpu_run) #endif mov VCPU_RDI(%_ASM_DI), %_ASM_DI - /* Enter guest mode */ - sti + /* Clobbers EFLAGS.ZF */ + VM_CLEAR_CPU_BUFFERS + /* Enter guest mode */ 3: vmrun %_ASM_AX 4: - cli - /* Pop @svm to RAX while it's the only available register. */ pop %_ASM_AX @@ -334,12 +333,12 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX - /* Enter guest mode */ - sti + /* Clobbers EFLAGS.ZF */ + VM_CLEAR_CPU_BUFFERS + /* Enter guest mode */ 1: vmrun %_ASM_AX - -2: cli +2: /* Pop @svm to RDI, guest registers have been saved already. */ pop %_ASM_DI diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index b82e6ed4f0241..c6b4b1728006d 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -15,20 +15,23 @@ * Tracepoint for guest mode entry. */ TRACE_EVENT(kvm_entry, - TP_PROTO(struct kvm_vcpu *vcpu), - TP_ARGS(vcpu), + TP_PROTO(struct kvm_vcpu *vcpu, bool force_immediate_exit), + TP_ARGS(vcpu, force_immediate_exit), TP_STRUCT__entry( __field( unsigned int, vcpu_id ) __field( unsigned long, rip ) + __field( bool, immediate_exit ) ), TP_fast_assign( __entry->vcpu_id = vcpu->vcpu_id; __entry->rip = kvm_rip_read(vcpu); + __entry->immediate_exit = force_immediate_exit; ), - TP_printk("vcpu %u, rip 0x%lx", __entry->vcpu_id, __entry->rip) + TP_printk("vcpu %u, rip 0x%lx%s", __entry->vcpu_id, __entry->rip, + __entry->immediate_exit ? "[immediate exit]" : "") ); /* diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index d3e346a574f11..d2fa192d7ce77 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2564,10 +2564,11 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, if (vmx->nested.nested_run_pending && (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); - vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); + vmx_guest_debugctl_write(vcpu, vmcs12->guest_ia32_debugctl & + vmx_get_supported_debugctl(vcpu, false)); } else { kvm_set_dr(vcpu, 7, vcpu->arch.dr7); - vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.pre_vmenter_debugctl); + vmx_guest_debugctl_write(vcpu, vmx->nested.pre_vmenter_debugctl); } if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) @@ -3045,7 +3046,8 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, return -EINVAL; if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && - CC(!kvm_dr7_valid(vmcs12->guest_dr7))) + (CC(!kvm_dr7_valid(vmcs12->guest_dr7)) || + CC(!vmx_is_valid_debugctl(vcpu, vmcs12->guest_ia32_debugctl, false)))) return -EINVAL; if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && @@ -3431,7 +3433,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, if (!vmx->nested.nested_run_pending || !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) - vmx->nested.pre_vmenter_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); + vmx->nested.pre_vmenter_debugctl = vmx_guest_debugctl_read(); if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) @@ -4435,6 +4437,12 @@ static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); + /* + * Note! Save DR7, but intentionally don't grab DEBUGCTL from vmcs02. + * Writes to DEBUGCTL that aren't intercepted by L1 are immediately + * propagated to vmcs12 (see vmx_set_msr()), as the value loaded into + * vmcs02 doesn't strictly track vmcs12. + */ if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); @@ -4625,7 +4633,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, __vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR); kvm_set_dr(vcpu, 7, 0x400); - vmcs_write64(GUEST_IA32_DEBUGCTL, 0); + vmx_guest_debugctl_write(vcpu, 0); if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, vmcs12->vm_exit_msr_load_count)) @@ -4680,6 +4688,9 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); } + /* Reload DEBUGCTL to ensure vmcs01 has a fresh FREEZE_IN_SMM value. */ + vmx_reload_guest_debugctl(vcpu); + /* * Note that calling vmx_set_{efer,cr0,cr4} is important as they * handle a variety of side effects to KVM's software model. @@ -4900,6 +4911,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); } + if (vmx->nested.update_vmcs01_hwapic_isr) { + vmx->nested.update_vmcs01_hwapic_isr = false; + kvm_apic_update_hwapic_isr(vcpu); + } + if ((vm_exit_reason != -1) && (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr))) vmx->nested.need_vmcs12_to_shadow_sync = true; diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 48a2f77f62ef3..50364e00e4e90 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -633,11 +633,11 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu) */ static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu) { - u64 data = vmcs_read64(GUEST_IA32_DEBUGCTL); + u64 data = vmx_guest_debugctl_read(); if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) { data &= ~DEBUGCTLMSR_LBR; - vmcs_write64(GUEST_IA32_DEBUGCTL, data); + vmx_guest_debugctl_write(vcpu, data); } } @@ -707,7 +707,7 @@ void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu) if (!lbr_desc->event) { vmx_disable_lbr_msrs_passthrough(vcpu); - if (vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR) + if (vmx_guest_debugctl_read() & DEBUGCTLMSR_LBR) goto warn; if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use)) goto warn; @@ -729,7 +729,7 @@ void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu) static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) { - if (!(vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR)) + if (!(vmx_guest_debugctl_read() & DEBUGCTLMSR_LBR)) intel_pmu_release_guest_lbr_event(vcpu); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 52098844290ad..9b1f22bcb7165 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -49,6 +49,8 @@ #include #include +#include + #include "capabilities.h" #include "cpuid.h" #include "hyperv.h" @@ -787,8 +789,11 @@ static void vmx_emergency_disable(void) return; list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), - loaded_vmcss_on_cpu_link) + loaded_vmcss_on_cpu_link) { vmcs_clear(v->vmcs); + if (v->shadow_vmcs) + vmcs_clear(v->shadow_vmcs); + } kvm_cpu_vmxoff(); } @@ -1301,8 +1306,6 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) u16 fs_sel, gs_sel; int i; - vmx->req_immediate_exit = false; - /* * Note that guest MSRs to be saved/restored can also be changed * when guest state is loaded. This happens when guest transitions @@ -1496,13 +1499,9 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, */ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); - vmx_vcpu_load_vmcs(vcpu, cpu, NULL); vmx_vcpu_pi_load(vcpu, cpu); - - vmx->host_debugctlmsr = get_debugctlmsr(); } static void vmx_vcpu_put(struct kvm_vcpu *vcpu) @@ -2125,7 +2124,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; break; case MSR_IA32_DEBUGCTLMSR: - msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL); + msr_info->data = vmx_guest_debugctl_read(); break; default: find_uret_msr: @@ -2150,7 +2149,7 @@ static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, return (unsigned long)data; } -static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) +u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) { u64 debugctl = 0; @@ -2162,9 +2161,25 @@ static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; + if (boot_cpu_has(X86_FEATURE_RTM) && + (host_initiated || guest_cpuid_has(vcpu, X86_FEATURE_RTM))) + debugctl |= DEBUGCTLMSR_RTM_DEBUG; + return debugctl; } +bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated) +{ + u64 invalid; + + invalid = data & ~vmx_get_supported_debugctl(vcpu, host_initiated); + if (invalid & (DEBUGCTLMSR_BTF | DEBUGCTLMSR_LBR)) { + kvm_pr_unimpl_wrmsr(vcpu, MSR_IA32_DEBUGCTLMSR, data); + invalid &= ~(DEBUGCTLMSR_BTF | DEBUGCTLMSR_LBR); + } + return !invalid; +} + /* * Writes msr value into the appropriate "register". * Returns 0 on success, non-0 otherwise. @@ -2233,29 +2248,22 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) } vmcs_writel(GUEST_SYSENTER_ESP, data); break; - case MSR_IA32_DEBUGCTLMSR: { - u64 invalid; - - invalid = data & ~vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); - if (invalid & (DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR)) { - kvm_pr_unimpl_wrmsr(vcpu, msr_index, data); - data &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); - invalid &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); - } - - if (invalid) + case MSR_IA32_DEBUGCTLMSR: + if (!vmx_is_valid_debugctl(vcpu, data, msr_info->host_initiated)) return 1; + data &= vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); + if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) get_vmcs12(vcpu)->guest_ia32_debugctl = data; - vmcs_write64(GUEST_IA32_DEBUGCTL, data); + vmx_guest_debugctl_write(vcpu, data); + if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && (data & DEBUGCTLMSR_LBR)) intel_pmu_create_guest_lbr_event(vcpu); return 0; - } case MSR_IA32_BNDCFGS: if (!kvm_mpx_supported() || (!msr_info->host_initiated && @@ -4819,7 +4827,8 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write32(GUEST_SYSENTER_CS, 0); vmcs_writel(GUEST_SYSENTER_ESP, 0); vmcs_writel(GUEST_SYSENTER_EIP, 0); - vmcs_write64(GUEST_IA32_DEBUGCTL, 0); + + vmx_guest_debugctl_write(&vmx->vcpu, 0); if (cpu_has_vmx_tpr_shadow()) { vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); @@ -6010,22 +6019,46 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) return 1; } -static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu) +static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu, + bool force_immediate_exit) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (!vmx->req_immediate_exit && - !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) { - kvm_lapic_expired_hv_timer(vcpu); + /* + * In the *extremely* unlikely scenario that this is a spurious VM-Exit + * due to the timer expiring while it was "soft" disabled, just eat the + * exit and re-enter the guest. + */ + if (unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) return EXIT_FASTPATH_REENTER_GUEST; - } - return EXIT_FASTPATH_NONE; + /* + * If the timer expired because KVM used it to force an immediate exit, + * then mission accomplished. + */ + if (force_immediate_exit) + return EXIT_FASTPATH_EXIT_HANDLED; + + /* + * If L2 is active, go down the slow path as emulating the guest timer + * expiration likely requires synthesizing a nested VM-Exit. + */ + if (is_guest_mode(vcpu)) + return EXIT_FASTPATH_NONE; + + kvm_lapic_expired_hv_timer(vcpu); + return EXIT_FASTPATH_REENTER_GUEST; } static int handle_preemption_timer(struct kvm_vcpu *vcpu) { - handle_fastpath_preemption_timer(vcpu); + /* + * This non-fastpath handler is reached if and only if the preemption + * timer was being used to emulate a guest timer while L2 is active. + * All other scenarios are supposed to be handled in the fastpath. + */ + WARN_ON_ONCE(!is_guest_mode(vcpu)); + kvm_lapic_expired_hv_timer(vcpu); return 1; } @@ -6825,11 +6858,27 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) kvm_release_pfn_clean(pfn); } -static void vmx_hwapic_isr_update(int max_isr) +static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) { u16 status; u8 old; + /* + * If L2 is active, defer the SVI update until vmcs01 is loaded, as SVI + * is only relevant for if and only if Virtual Interrupt Delivery is + * enabled in vmcs12, and if VID is enabled then L2 EOIs affect L2's + * vAPIC, not L1's vAPIC. KVM must update vmcs01 on the next nested + * VM-Exit, otherwise L1 with run with a stale SVI. + */ + if (is_guest_mode(vcpu)) { + /* + * KVM is supposed to forward intercepted L2 EOIs to L1 if VID + * is enabled in vmcs12; as above, the EOIs affect L2's vAPIC. + */ + to_vmx(vcpu)->nested.update_vmcs01_hwapic_isr = true; + return; + } + if (max_isr == -1) max_isr = 0; @@ -7166,13 +7215,13 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) msrs[i].host, false); } -static void vmx_update_hv_timer(struct kvm_vcpu *vcpu) +static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) { struct vcpu_vmx *vmx = to_vmx(vcpu); u64 tscl; u32 delta_tsc; - if (vmx->req_immediate_exit) { + if (force_immediate_exit) { vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); vmx->loaded_vmcs->hv_timer_soft_disabled = false; } else if (vmx->hv_deadline_tsc != -1) { @@ -7225,13 +7274,22 @@ void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, barrier_nospec(); } -static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) +static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, + bool force_immediate_exit) { + /* + * If L2 is active, some VMX preemption timer exits can be handled in + * the fastpath even, all other exits must use the slow path. + */ + if (is_guest_mode(vcpu) && + to_vmx(vcpu)->exit_reason.basic != EXIT_REASON_PREEMPTION_TIMER) + return EXIT_FASTPATH_NONE; + switch (to_vmx(vcpu)->exit_reason.basic) { case EXIT_REASON_MSR_WRITE: return handle_fastpath_set_msr_irqoff(vcpu); case EXIT_REASON_PREEMPTION_TIMER: - return handle_fastpath_preemption_timer(vcpu); + return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); default: return EXIT_FASTPATH_NONE; } @@ -7254,7 +7312,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, vmx_l1d_flush(vcpu); else if (static_branch_unlikely(&mmio_stale_data_clear) && kvm_arch_has_assigned_device(vcpu->kvm)) - mds_clear_cpu_buffers(); + x86_clear_cpu_buffers(); vmx_disable_fb_clear(vmx); @@ -7291,8 +7349,9 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, guest_state_exit_irqoff(); } -static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) +static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) { + bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT; struct vcpu_vmx *vmx = to_vmx(vcpu); unsigned long cr3, cr4; @@ -7318,7 +7377,7 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) return EXIT_FASTPATH_NONE; } - trace_kvm_entry(vcpu); + trace_kvm_entry(vcpu, force_immediate_exit); if (vmx->ple_window_dirty) { vmx->ple_window_dirty = false; @@ -7337,6 +7396,12 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); vcpu->arch.regs_dirty = 0; + if (run_flags & KVM_RUN_LOAD_GUEST_DR6) + set_debugreg(vcpu->arch.dr6, 6); + + if (run_flags & KVM_RUN_LOAD_DEBUGCTL) + vmx_reload_guest_debugctl(vcpu); + /* * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time @@ -7356,10 +7421,6 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) vmx->loaded_vmcs->host_state.cr4 = cr4; } - /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */ - if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) - set_debugreg(vcpu->arch.dr6, 6); - /* When single-stepping over STI and MOV SS, we must clear the * corresponding interruptibility bits in the guest state. Otherwise * vmentry fails as it then expects bit 14 (BS) in pending debug @@ -7377,7 +7438,9 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) vmx_passthrough_lbr_msrs(vcpu); if (enable_preemption_timer) - vmx_update_hv_timer(vcpu); + vmx_update_hv_timer(vcpu, force_immediate_exit); + else if (force_immediate_exit) + smp_send_reschedule(vcpu->cpu); kvm_wait_lapic_expire(vcpu); @@ -7393,8 +7456,8 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) } /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ - if (vmx->host_debugctlmsr) - update_debugctlmsr(vmx->host_debugctlmsr); + if (vcpu->arch.host_debugctl) + update_debugctlmsr(vcpu->arch.host_debugctl); #ifndef CONFIG_X86_64 /* @@ -7441,10 +7504,7 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) vmx_recover_nmi_blocking(vmx); vmx_complete_interrupts(vmx); - if (is_guest_mode(vcpu)) - return EXIT_FASTPATH_NONE; - - return vmx_exit_handlers_fastpath(vcpu); + return vmx_exit_handlers_fastpath(vcpu, force_immediate_exit); } static void vmx_vcpu_free(struct kvm_vcpu *vcpu) @@ -7943,11 +8003,6 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); } -static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu) -{ - to_vmx(vcpu)->req_immediate_exit = true; -} - static int vmx_check_intercept_io(struct kvm_vcpu *vcpu, struct x86_instruction_info *info) { @@ -8274,6 +8329,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .vcpu_load = vmx_vcpu_load, .vcpu_put = vmx_vcpu_put, + .HOST_OWNED_DEBUGCTL = DEBUGCTLMSR_FREEZE_IN_SMM, + .update_exception_bitmap = vmx_update_exception_bitmap, .get_msr_feature = vmx_get_msr_feature, .get_msr = vmx_get_msr, @@ -8358,8 +8415,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .check_intercept = vmx_check_intercept, .handle_exit_irqoff = vmx_handle_exit_irqoff, - .request_immediate_exit = vmx_request_immediate_exit, - .sched_in = vmx_sched_in, .cpu_dirty_log_size = PML_ENTITY_NUM, @@ -8617,7 +8672,6 @@ static __init int hardware_setup(void) if (!enable_preemption_timer) { vmx_x86_ops.set_hv_timer = NULL; vmx_x86_ops.cancel_hv_timer = NULL; - vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit; } kvm_caps.supported_mce_cap |= MCG_LMCE_P; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 6be1627d888e5..5d73d3e570d78 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -177,6 +177,7 @@ struct nested_vmx { bool reload_vmcs01_apic_access_page; bool update_vmcs01_cpu_dirty_logging; bool update_vmcs01_apicv_status; + bool update_vmcs01_hwapic_isr; /* * Enlightened VMCS has been enabled. It does not mean that L1 has to @@ -330,8 +331,6 @@ struct vcpu_vmx { unsigned int ple_window; bool ple_window_dirty; - bool req_immediate_exit; - /* Support for PML */ #define PML_ENTITY_NUM 512 struct page *pml_pg; @@ -339,8 +338,6 @@ struct vcpu_vmx { /* apic deadline value in host tsc */ u64 hv_deadline_tsc; - unsigned long host_debugctlmsr; - /* * Only bits masked by msr_ia32_feature_control_valid_bits can be set in * msr_ia32_feature_control. FEAT_CTL_LOCKED is always included @@ -432,6 +429,32 @@ static inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated); +bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated); + +static inline void vmx_guest_debugctl_write(struct kvm_vcpu *vcpu, u64 val) +{ + WARN_ON_ONCE(val & DEBUGCTLMSR_FREEZE_IN_SMM); + + val |= vcpu->arch.host_debugctl & DEBUGCTLMSR_FREEZE_IN_SMM; + vmcs_write64(GUEST_IA32_DEBUGCTL, val); +} + +static inline u64 vmx_guest_debugctl_read(void) +{ + return vmcs_read64(GUEST_IA32_DEBUGCTL) & ~DEBUGCTLMSR_FREEZE_IN_SMM; +} + +static inline void vmx_reload_guest_debugctl(struct kvm_vcpu *vcpu) +{ + u64 val = vmcs_read64(GUEST_IA32_DEBUGCTL); + + if (!((val ^ vcpu->arch.host_debugctl) & DEBUGCTLMSR_FREEZE_IN_SMM)) + return; + + vmx_guest_debugctl_write(vcpu, val & ~DEBUGCTLMSR_FREEZE_IN_SMM); +} + /* * Note, early Intel manuals have the write-low and read-high bitmap offsets * the wrong way round. The bitmaps control MSRs 0x00000000-0x00001fff and diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f67fe8a65820c..a589a5781e906 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1495,6 +1495,7 @@ static const u32 msrs_to_save_pmu[] = { MSR_AMD64_PERF_CNTR_GLOBAL_CTL, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, + MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, }; static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) + @@ -1621,7 +1622,7 @@ static bool kvm_is_immutable_feature_msr(u32 msr) ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \ ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \ ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \ - ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO) + ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO | ARCH_CAP_ITS_NO) static u64 kvm_get_arch_capabilities(void) { @@ -1655,6 +1656,8 @@ static u64 kvm_get_arch_capabilities(void) data |= ARCH_CAP_MDS_NO; if (!boot_cpu_has_bug(X86_BUG_RFDS)) data |= ARCH_CAP_RFDS_NO; + if (!boot_cpu_has_bug(X86_BUG_ITS)) + data |= ARCH_CAP_ITS_NO; if (!boot_cpu_has(X86_FEATURE_RTM)) { /* @@ -7192,6 +7195,7 @@ static void kvm_probe_msr_to_save(u32 msr_index) case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) return; break; @@ -8306,11 +8310,6 @@ static bool emulator_is_smm(struct x86_emulate_ctxt *ctxt) return is_smm(emul_to_vcpu(ctxt)); } -static bool emulator_is_guest_mode(struct x86_emulate_ctxt *ctxt) -{ - return is_guest_mode(emul_to_vcpu(ctxt)); -} - #ifndef CONFIG_KVM_SMM static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt) { @@ -8377,7 +8376,6 @@ static const struct x86_emulate_ops emulate_ops = { .guest_has_rdpid = emulator_guest_has_rdpid, .set_nmi_mask = emulator_set_nmi_mask, .is_smm = emulator_is_smm, - .is_guest_mode = emulator_is_guest_mode, .leave_smm = emulator_leave_smm, .triple_fault = emulator_triple_fault, .set_xcr = emulator_set_xcr, @@ -8997,7 +8995,14 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, ctxt->exception.address = 0; } - r = x86_emulate_insn(ctxt); + /* + * Check L1's instruction intercepts when emulating instructions for + * L2, unless KVM is re-emulating a previously decoded instruction, + * e.g. to complete userspace I/O, in which case KVM has already + * checked the intercepts. + */ + r = x86_emulate_insn(ctxt, is_guest_mode(vcpu) && + !(emulation_type & EMULTYPE_NO_DECODE)); if (r == EMULATION_INTERCEPTED) return 1; @@ -9800,8 +9805,11 @@ static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) rcu_read_lock(); map = rcu_dereference(vcpu->kvm->arch.apic_map); - if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id]) - target = map->phys_map[dest_id]->vcpu; + if (likely(map) && dest_id <= map->max_apic_id) { + dest_id = array_index_nospec(dest_id, map->max_apic_id + 1); + if (map->phys_map[dest_id]) + target = map->phys_map[dest_id]->vcpu; + } rcu_read_unlock(); @@ -10503,12 +10511,6 @@ static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu); } -void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu) -{ - smp_send_reschedule(vcpu->cpu); -} -EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit); - /* * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without @@ -10522,6 +10524,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) dm_request_for_irq_injection(vcpu) && kvm_cpu_accept_dm_intr(vcpu); fastpath_t exit_fastpath; + u64 run_flags, debug_ctl; bool req_immediate_exit = false; @@ -10754,9 +10757,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto cancel_injection; } + run_flags = 0; if (req_immediate_exit) { + run_flags |= KVM_RUN_FORCE_IMMEDIATE_EXIT; kvm_make_request(KVM_REQ_EVENT, vcpu); - static_call(kvm_x86_request_immediate_exit)(vcpu); } fpregs_assert_state_consistent(); @@ -10772,10 +10776,25 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) set_debugreg(vcpu->arch.eff_db[1], 1); set_debugreg(vcpu->arch.eff_db[2], 2); set_debugreg(vcpu->arch.eff_db[3], 3); + /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */ + if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) + run_flags |= KVM_RUN_LOAD_GUEST_DR6; } else if (unlikely(hw_breakpoint_active())) { set_debugreg(0, 7); } + /* + * Refresh the host DEBUGCTL snapshot after disabling IRQs, as DEBUGCTL + * can be modified in IRQ context, e.g. via SMP function calls. Inform + * vendor code if any host-owned bits were changed, e.g. so that the + * value loaded into hardware while running the guest can be updated. + */ + debug_ctl = get_debugctlmsr(); + if ((debug_ctl ^ vcpu->arch.host_debugctl) & kvm_x86_ops.HOST_OWNED_DEBUGCTL && + !vcpu->arch.guest_state_protected) + run_flags |= KVM_RUN_LOAD_DEBUGCTL; + vcpu->arch.host_debugctl = debug_ctl; + guest_timing_enter_irqoff(); for (;;) { @@ -10788,7 +10807,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) && (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED)); - exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu); + exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu, run_flags); if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST)) break; @@ -10800,6 +10819,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) break; } + run_flags = 0; + /* Note, VM-Exits that go down the "slow" path are accounted below. */ ++vcpu->stat.exits; } @@ -10846,6 +10867,15 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (vcpu->arch.guest_fpu.xfd_err) wrmsrl(MSR_IA32_XFD_ERR, 0); + /* + * Mark this CPU as needing a branch predictor flush before running + * userspace. Must be done before enabling preemption to ensure it gets + * set for the CPU that actually ran the guest, and not the CPU that it + * may migrate to. + */ + if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER)) + this_cpu_write(x86_ibpb_exit_to_user, true); + /* * Consume any pending interrupts, including the possible source of * VM-Exit on SVM and any ticks that occur between VM-Exit and now. @@ -13251,16 +13281,22 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, { struct kvm_kernel_irqfd *irqfd = container_of(cons, struct kvm_kernel_irqfd, consumer); + struct kvm *kvm = irqfd->kvm; int ret; - irqfd->producer = prod; kvm_arch_start_assignment(irqfd->kvm); + + spin_lock_irq(&kvm->irqfds.lock); + irqfd->producer = prod; + ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 1); - if (ret) kvm_arch_end_assignment(irqfd->kvm); + spin_unlock_irq(&kvm->irqfds.lock); + + return ret; } @@ -13270,9 +13306,9 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, int ret; struct kvm_kernel_irqfd *irqfd = container_of(cons, struct kvm_kernel_irqfd, consumer); + struct kvm *kvm = irqfd->kvm; WARN_ON(irqfd->producer != prod); - irqfd->producer = NULL; /* * When producer of consumer is unregistered, we change back to @@ -13280,11 +13316,17 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, * when the irq is masked/disabled or the consumer side (KVM * int this case doesn't want to receive the interrupts. */ + spin_lock_irq(&kvm->irqfds.lock); + irqfd->producer = NULL; + ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0); if (ret) printk(KERN_INFO "irq bypass consumer (token %p) unregistration" " fails: %d\n", irqfd->consumer.token, ret); + spin_unlock_irq(&kvm->irqfds.lock); + + kvm_arch_end_assignment(irqfd->kvm); } diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 0ea6016ad132a..94c7fee7a2259 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1260,7 +1260,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, if (kvm_read_guest_virt(vcpu, (gva_t)sched_poll.ports, ports, sched_poll.nr_ports * sizeof(*ports), &e)) { *r = -EFAULT; - return true; + goto out; } for (i = 0; i < sched_poll.nr_ports; i++) { @@ -1737,8 +1737,19 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, { struct kvm_vcpu *vcpu; - if (ue->u.xen_evtchn.port >= max_evtchn_port(kvm)) - return -EINVAL; + /* + * Don't check for the port being within range of max_evtchn_port(). + * Userspace can configure what ever targets it likes; events just won't + * be delivered if/while the target is invalid, just like userspace can + * configure MSIs which target non-existent APICs. + * + * This allow on Live Migration and Live Update, the IRQ routing table + * can be restored *independently* of other things like creating vCPUs, + * without imposing an ordering dependency on userspace. In this + * particular case, the problematic ordering would be with setting the + * Xen 'long mode' flag, which changes max_evtchn_port() to allow 4096 + * instead of 1024 event channels. + */ /* We only support 2 level event channels for now */ if (ue->u.xen_evtchn.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index ffa51f392e17a..91ce427d5af19 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -360,6 +360,45 @@ SYM_FUNC_END(__x86_return_skl) #endif /* CONFIG_CALL_DEPTH_TRACKING */ +#ifdef CONFIG_MITIGATION_ITS + +.macro ITS_THUNK reg + +SYM_INNER_LABEL(__x86_indirect_its_thunk_\reg, SYM_L_GLOBAL) + UNWIND_HINT_UNDEFINED + ANNOTATE_NOENDBR + ANNOTATE_RETPOLINE_SAFE + jmp *%\reg + int3 + .align 32, 0xcc /* fill to the end of the line */ + .skip 32, 0xcc /* skip to the next upper half */ +.endm + +/* ITS mitigation requires thunks be aligned to upper half of cacheline */ +.align 64, 0xcc +.skip 32, 0xcc +SYM_CODE_START(__x86_indirect_its_thunk_array) + +#define GEN(reg) ITS_THUNK reg +#include +#undef GEN + + .align 64, 0xcc +SYM_CODE_END(__x86_indirect_its_thunk_array) + +.align 64, 0xcc +.skip 32, 0xcc +SYM_CODE_START(its_return_thunk) + UNWIND_HINT_FUNC + ANNOTATE_NOENDBR + ANNOTATE_UNRET_SAFE + ret + int3 +SYM_CODE_END(its_return_thunk) +EXPORT_SYMBOL(its_return_thunk) + +#endif /* CONFIG_MITIGATION_ITS */ + /* * This function name is magical and is used by -mfunction-return=thunk-extern * for the compiler to generate JMPs to it. diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 2354c0156e51c..94195d750e65c 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -121,13 +121,12 @@ static bool ex_handler_sgx(const struct exception_table_entry *fixup, static bool ex_handler_fprestore(const struct exception_table_entry *fixup, struct pt_regs *regs) { - regs->ip = ex_fixup_addr(fixup); - WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.", (void *)instruction_pointer(regs)); fpu_reset_from_exception_fixup(); - return true; + + return ex_handler_default(fixup, regs); } /* diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 3a41d496e8723..0e4434373f6de 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -650,8 +650,13 @@ static void __init memory_map_top_down(unsigned long map_start, */ addr = memblock_phys_alloc_range(PMD_SIZE, PMD_SIZE, map_start, map_end); - memblock_phys_free(addr, PMD_SIZE); - real_end = addr + PMD_SIZE; + if (!addr) { + pr_warn("Failed to release memory for alloc_low_pages()"); + real_end = max(map_start, ALIGN_DOWN(map_end, PMD_SIZE)); + } else { + memblock_phys_free(addr, PMD_SIZE); + real_end = addr + PMD_SIZE; + } /* step_size need to be small so pgt_buf from BRK could cover it */ step_size = PMD_SIZE; diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index aa69353da49f2..cf080c96b7dd6 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -223,6 +223,24 @@ static void sync_global_pgds(unsigned long start, unsigned long end) sync_global_pgds_l4(start, end); } +/* + * Make kernel mappings visible in all page tables in the system. + * This is necessary except when the init task populates kernel mappings + * during the boot process. In that case, all processes originating from + * the init task copies the kernel mappings, so there is no issue. + * Otherwise, missing synchronization could lead to kernel crashes due + * to missing page table entries for certain kernel mappings. + * + * Synchronization is performed at the top level, which is the PGD in + * 5-level paging systems. But in 4-level paging systems, however, + * pgd_populate() is a no-op, so synchronization is done at the P4D level. + * sync_global_pgds() handles this difference between paging levels. + */ +void arch_sync_kernel_mappings(unsigned long start, unsigned long end) +{ + sync_global_pgds(start, end); +} + /* * NOTE: This function is marked __ref because it calls __init function * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0. @@ -959,9 +977,18 @@ int add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages, ret = __add_pages(nid, start_pfn, nr_pages, params); WARN_ON_ONCE(ret); - /* update max_pfn, max_low_pfn and high_memory */ - update_end_of_memory_vars(start_pfn << PAGE_SHIFT, - nr_pages << PAGE_SHIFT); + /* + * Special case: add_pages() is called by memremap_pages() for adding device + * private pages. Do not bump up max_pfn in the device private path, + * because max_pfn changes affect dma_addressing_limited(). + * + * dma_addressing_limited() returning true when max_pfn is the device's + * addressable memory can force device drivers to use bounce buffers + * and impact their performance negatively: + */ + if (!params->pgmap) + /* update max_pfn, max_low_pfn and high_memory */ + update_end_of_memory_vars(start_pfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT); return ret; } diff --git a/arch/x86/mm/kaslr.c b/arch/x86/mm/kaslr.c index 230f1dee4f095..e0b0ec0f82457 100644 --- a/arch/x86/mm/kaslr.c +++ b/arch/x86/mm/kaslr.c @@ -109,8 +109,14 @@ void __init kernel_randomize_memory(void) memory_tb = DIV_ROUND_UP(max_pfn << PAGE_SHIFT, 1UL << TB_SHIFT) + CONFIG_RANDOMIZE_MEMORY_PHYSICAL_PADDING; - /* Adapt physical memory region size based on available memory */ - if (memory_tb < kaslr_regions[0].size_tb) + /* + * Adapt physical memory region size based on available memory, + * except when CONFIG_PCI_P2PDMA is enabled. P2PDMA exposes the + * device BAR space assuming the direct map space is large enough + * for creating a ZONE_DEVICE mapping in the direct map corresponding + * to the physical BAR address. + */ + if (!IS_ENABLED(CONFIG_PCI_P2PDMA) && (memory_tb < kaslr_regions[0].size_tb)) kaslr_regions[0].size_tb = memory_tb; /* diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index b18f5a71e679e..98aa303ad0546 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -107,7 +107,7 @@ static inline void pgd_list_del(pgd_t *pgd) #define UNSHARED_PTRS_PER_PGD \ (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD) #define MAX_UNSHARED_PTRS_PER_PGD \ - max_t(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD) + MAX_T(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD) static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 4872bb082b193..20ce6a1bd6c57 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -630,7 +630,11 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); - /* Let nmi_uaccess_okay() know that we're changing CR3. */ + /* + * Indicate that CR3 is about to change. nmi_uaccess_okay() + * and others are sensitive to the window where mm_cpumask(), + * CR3 and cpu_tlbstate.loaded_mm are not all in sync. + */ this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING); barrier(); } @@ -900,8 +904,16 @@ static void flush_tlb_func(void *info) static bool should_flush_tlb(int cpu, void *data) { + struct mm_struct *loaded_mm = per_cpu(cpu_tlbstate.loaded_mm, cpu); struct flush_tlb_info *info = data; + /* + * Order the 'loaded_mm' and 'is_lazy' against their + * write ordering in switch_mm_irqs_off(). Ensure + * 'is_lazy' is at least as new as 'loaded_mm'. + */ + smp_rmb(); + /* Lazy TLB will get flushed at the next context switch. */ if (per_cpu(cpu_tlbstate_shared.is_lazy, cpu)) return false; @@ -910,8 +922,15 @@ static bool should_flush_tlb(int cpu, void *data) if (!info->mm) return true; + /* + * While switching, the remote CPU could have state from + * either the prev or next mm. Assume the worst and flush. + */ + if (loaded_mm == LOADED_MM_SWITCHING) + return true; + /* The target mm is loaded, and the CPU is not lazy. */ - if (per_cpu(cpu_tlbstate.loaded_mm, cpu) == info->mm) + if (loaded_mm == info->mm) return true; /* In cpumask, but not the loaded mm? Periodically remove by flushing. */ diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index a50c99e9b5c01..208a59d8fc4a1 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -16,6 +16,7 @@ #include #include #include +#include static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) { @@ -37,6 +38,8 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) +#define EMIT5(b1, b2, b3, b4, b5) \ + do { EMIT1(b1); EMIT4(b2, b3, b4, b5); } while (0) #define EMIT1_off32(b1, off) \ do { EMIT1(b1); EMIT(off, 4); } while (0) @@ -48,9 +51,11 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) #ifdef CONFIG_X86_KERNEL_IBT -#define EMIT_ENDBR() EMIT(gen_endbr(), 4) +#define EMIT_ENDBR() EMIT(gen_endbr(), 4) +#define EMIT_ENDBR_POISON() EMIT(gen_endbr_poison(), 4) #else #define EMIT_ENDBR() +#define EMIT_ENDBR_POISON() #endif static bool is_imm8(int value) @@ -335,6 +340,69 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) *pprog = prog; } +/* + * Emit the various CFI preambles, see asm/cfi.h and the comments about FineIBT + * in arch/x86/kernel/alternative.c + */ + +static void emit_fineibt(u8 **pprog, u32 hash) +{ + u8 *prog = *pprog; + + EMIT_ENDBR(); + EMIT3_off32(0x41, 0x81, 0xea, hash); /* subl $hash, %r10d */ + EMIT2(0x74, 0x07); /* jz.d8 +7 */ + EMIT2(0x0f, 0x0b); /* ud2 */ + EMIT1(0x90); /* nop */ + EMIT_ENDBR_POISON(); + + *pprog = prog; +} + +static void emit_kcfi(u8 **pprog, u32 hash) +{ + u8 *prog = *pprog; + + EMIT1_off32(0xb8, hash); /* movl $hash, %eax */ +#ifdef CONFIG_CALL_PADDING + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); +#endif + EMIT_ENDBR(); + + *pprog = prog; +} + +static void emit_cfi(u8 **pprog, u32 hash) +{ + u8 *prog = *pprog; + + switch (cfi_mode) { + case CFI_FINEIBT: + emit_fineibt(&prog, hash); + break; + + case CFI_KCFI: + emit_kcfi(&prog, hash); + break; + + default: + EMIT_ENDBR(); + break; + } + + *pprog = prog; +} + /* * Emit x86-64 prologue code for BPF program. * bpf_tail_call helper will skip the first X86_TAIL_CALL_OFFSET bytes @@ -345,10 +413,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, { u8 *prog = *pprog; + emit_cfi(&prog, is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash); /* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ - EMIT_ENDBR(); memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE; if (!ebpf_from_cbpf) { @@ -470,7 +538,11 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) { u8 *prog = *pprog; - if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + if (IS_ENABLED(CONFIG_MITIGATION_ITS) && + cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) { + OPTIMIZER_HIDE_VAR(reg); + emit_jump(&prog, its_static_thunk(reg), ip); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { EMIT_LFENCE(); EMIT2(0xFF, 0xE0 + reg); } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { @@ -492,7 +564,7 @@ static void emit_return(u8 **pprog, u8 *ip) { u8 *prog = *pprog; - if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) { + if (cpu_wants_rethunk()) { emit_jump(&prog, x86_return_thunk, ip); } else { EMIT1(0xC3); /* ret */ @@ -1072,6 +1144,48 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) #define RESTORE_TAIL_CALL_CNT(stack) \ EMIT3_off32(0x48, 0x8B, 0x85, -round_up(stack, 8) - 8) +static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, + struct bpf_prog *bpf_prog) +{ + u8 *prog = *pprog; + u8 *func; + + if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_LOOP)) { + /* The clearing sequence clobbers eax and ecx. */ + EMIT1(0x50); /* push rax */ + EMIT1(0x51); /* push rcx */ + ip += 2; + + func = (u8 *)clear_bhb_loop; + ip += x86_call_depth_emit_accounting(&prog, func); + + if (emit_call(&prog, func, ip)) + return -EINVAL; + EMIT1(0x59); /* pop rcx */ + EMIT1(0x58); /* pop rax */ + } + /* Insert IBHF instruction */ + if ((cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_LOOP) && + cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) || + cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_HW)) { + /* + * Add an Indirect Branch History Fence (IBHF). IBHF acts as a + * fence preventing branch history from before the fence from + * affecting indirect branches after the fence. This is + * specifically used in cBPF jitted code to prevent Intra-mode + * BHI attacks. The IBHF instruction is designed to be a NOP on + * hardware that doesn't need or support it. The REP and REX.W + * prefixes are required by the microcode, and they also ensure + * that the NOP is unlikely to be used in existing code. + * + * IBHF is not a valid instruction in 32-bit mode. + */ + EMIT5(0xF3, 0x48, 0x0F, 0x1E, 0xF8); /* ibhf */ + } + *pprog = prog; + return 0; +} + static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) { @@ -1945,6 +2059,15 @@ st: if (is_imm8(insn->off)) seen_exit = true; /* Update cleanup_addr */ ctx->cleanup_addr = proglen; + + if (bpf_prog_was_classic(bpf_prog) && + !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) { + u8 *ip = image + addrs[i - 1]; + + if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog)) + return -EINVAL; + } + pop_callee_regs(&prog, callee_regs_used); EMIT1(0xC9); /* leave */ emit_return(&prog, image + addrs[i - 1] + (prog - temp)); @@ -2410,10 +2533,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i u8 *prog; bool save_ret; + /* + * F_INDIRECT is only compatible with F_RET_FENTRY_RET, it is + * explicitly incompatible with F_CALL_ORIG | F_SKIP_FRAME | F_IP_ARG + * because @func_addr. + */ + WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && + (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + /* extra registers for struct arguments */ - for (i = 0; i < m->nr_args; i++) + for (i = 0; i < m->nr_args; i++) { if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) nr_regs += (m->arg_size[i] + 7) / 8 - 1; + } /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 * are passed through regs, the remains are through stack. @@ -2496,20 +2628,27 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i prog = image; - EMIT_ENDBR(); - /* - * This is the direct-call trampoline, as such it needs accounting - * for the __fentry__ call. - */ - x86_call_depth_emit_accounting(&prog, NULL); + if (flags & BPF_TRAMP_F_INDIRECT) { + /* + * Indirect call for bpf_struct_ops + */ + emit_cfi(&prog, cfi_get_func_hash(func_addr)); + } else { + /* + * Direct-call fentry stub, as such it needs accounting for the + * __fentry__ call. + */ + x86_call_depth_emit_accounting(&prog, NULL); + } EMIT1(0x55); /* push rbp */ EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ - if (!is_imm8(stack_size)) + if (!is_imm8(stack_size)) { /* sub rsp, stack_size */ EMIT3_off32(0x48, 0x81, 0xEC, stack_size); - else + } else { /* sub rsp, stack_size */ EMIT4(0x48, 0x83, 0xEC, stack_size); + } if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) EMIT1(0x50); /* push rax */ /* mov QWORD PTR [rbp - rbx_off], rbx */ @@ -2542,10 +2681,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } } - if (fentry->nr_links) + if (fentry->nr_links) { if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) return -EINVAL; + } if (fmod_ret->nr_links) { branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *), @@ -2564,11 +2704,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i restore_regs(m, &prog, regs_off); save_args(m, &prog, arg_stack_off, true); - if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, restore the * tail_call_cnt from stack to rax. */ RESTORE_TAIL_CALL_CNT(stack_size); + } if (flags & BPF_TRAMP_F_ORIG_STACK) { emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8); @@ -2597,16 +2738,18 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i /* Update the branches saved in invoke_bpf_mod_ret with the * aligned address of do_fexit. */ - for (i = 0; i < fmod_ret->nr_links; i++) + for (i = 0; i < fmod_ret->nr_links; i++) { emit_cond_near_jump(&branches[i], prog, branches[i], X86_JNE); + } } - if (fexit->nr_links) + if (fexit->nr_links) { if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, false)) { ret = -EINVAL; goto cleanup; } + } if (flags & BPF_TRAMP_F_RESTORE_REGS) restore_regs(m, &prog, regs_off); @@ -2623,11 +2766,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i ret = -EINVAL; goto cleanup; } - } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before running the original function, restore the * tail_call_cnt from stack to rax. */ RESTORE_TAIL_CALL_CNT(stack_size); + } /* restore return value of orig_call or fentry prog back into RAX */ if (save_ret) @@ -2635,9 +2779,10 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off); EMIT1(0xC9); /* leave */ - if (flags & BPF_TRAMP_F_SKIP_FRAME) + if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* skip our return address and return to parent */ EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ + } emit_return(&prog, prog); /* Make sure the trampoline generation logic doesn't overflow */ if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) { @@ -2908,9 +3053,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) jit_data->header = header; jit_data->rw_header = rw_header; } - prog->bpf_func = (void *)image; + /* + * ctx.prog_offset is used when CFI preambles put code *before* + * the function. See emit_cfi(). For FineIBT specifically this code + * can also be executed and bpf_prog_kallsyms_add() will + * generate an additional symbol to cover this, hence also + * decrement proglen. + */ + prog->bpf_func = (void *)image + cfi_get_offset(); prog->jited = 1; - prog->jited_len = proglen; + prog->jited_len = proglen - cfi_get_offset(); } else { prog = orig_prog; } @@ -2965,6 +3117,7 @@ void bpf_jit_free(struct bpf_prog *prog) kvfree(jit_data->addrs); kfree(jit_data); } + prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset(); hdr = bpf_jit_binary_pack_hdr(prog); bpf_jit_binary_pack_free(hdr, NULL); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); diff --git a/arch/x86/tools/insn_decoder_test.c b/arch/x86/tools/insn_decoder_test.c index 472540aeabc23..08cd913cbd4e9 100644 --- a/arch/x86/tools/insn_decoder_test.c +++ b/arch/x86/tools/insn_decoder_test.c @@ -10,8 +10,7 @@ #include #include #include - -#define unlikely(cond) (cond) +#include #include #include @@ -106,7 +105,7 @@ static void parse_args(int argc, char **argv) } } -#define BUFSIZE 256 +#define BUFSIZE (256 + KSYM_NAME_LEN) int main(int argc, char **argv) { diff --git a/arch/x86/um/asm/checksum.h b/arch/x86/um/asm/checksum.h index b07824500363f..ddc144657efad 100644 --- a/arch/x86/um/asm/checksum.h +++ b/arch/x86/um/asm/checksum.h @@ -20,6 +20,9 @@ */ extern __wsum csum_partial(const void *buff, int len, __wsum sum); +/* Do not call this directly. Declared for export type visibility. */ +extern __visible __wsum csum_partial_copy_generic(const void *src, void *dst, int len); + /** * csum_fold - Fold and invert a 32bit checksum. * sum: 32bit unfolded sum diff --git a/arch/x86/um/os-Linux/mcontext.c b/arch/x86/um/os-Linux/mcontext.c index 49c3744cac371..81b9d1f9f4e68 100644 --- a/arch/x86/um/os-Linux/mcontext.c +++ b/arch/x86/um/os-Linux/mcontext.c @@ -26,7 +26,6 @@ void get_regs_from_mc(struct uml_pt_regs *regs, mcontext_t *mc) COPY(RIP); COPY2(EFLAGS, EFL); COPY2(CS, CSGSFS); - regs->gp[CS / sizeof(unsigned long)] &= 0xffff; - regs->gp[CS / sizeof(unsigned long)] |= 3; + regs->gp[SS / sizeof(unsigned long)] = mc->gregs[REG_CSGSFS] >> 48; #endif } diff --git a/arch/xtensa/kernel/asm-offsets.c b/arch/xtensa/kernel/asm-offsets.c index da38de20ae598..cfbced95e944a 100644 --- a/arch/xtensa/kernel/asm-offsets.c +++ b/arch/xtensa/kernel/asm-offsets.c @@ -11,6 +11,7 @@ * * Chris Zankel */ +#define COMPILE_OFFSETS #include #include diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 178cf96ca10ac..f878bbb8b0ac2 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -230,10 +230,14 @@ static ssize_t proc_read_simdisk(struct file *file, char __user *buf, static ssize_t proc_write_simdisk(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char *tmp = memdup_user_nul(buf, count); + char *tmp; struct simdisk *dev = pde_data(file_inode(file)); int err; + if (count == 0 || count > PAGE_SIZE) + return -EINVAL; + + tmp = memdup_user_nul(buf, count); if (IS_ERR(tmp)) return PTR_ERR(tmp); diff --git a/block/bdev.c b/block/bdev.c index 3fbae577a7341..b79ab8be8345a 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -152,9 +152,26 @@ int set_blocksize(struct block_device *bdev, int size) /* Don't change the size if it is same as current */ if (bdev->bd_inode->i_blkbits != blksize_bits(size)) { + /* + * Flush and truncate the pagecache before we reconfigure the + * mapping geometry because folio sizes are variable now. If a + * reader has already allocated a folio whose size is smaller + * than the new min_order but invokes readahead after the new + * min_order becomes visible, readahead will think there are + * "zero" blocks per folio and crash. Take the inode and + * invalidation locks to avoid racing with + * read/write/fallocate. + */ + inode_lock(bdev->bd_inode); + filemap_invalidate_lock(bdev->bd_inode->i_mapping); + sync_blockdev(bdev); + kill_bdev(bdev); + bdev->bd_inode->i_blkbits = blksize_bits(size); kill_bdev(bdev); + filemap_invalidate_unlock(bdev->bd_inode->i_mapping); + inode_unlock(bdev->bd_inode); } return 0; } diff --git a/block/bio.c b/block/bio.c index e66ec7594f082..ab418bb7be2c1 100644 --- a/block/bio.c +++ b/block/bio.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "blk.h" @@ -604,7 +605,7 @@ struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask) { struct bio *bio; - if (nr_vecs > UIO_MAXIOV) + if (nr_vecs > BIO_MAX_INLINE_VECS) return NULL; return kmalloc(struct_size(bio, bi_inline_vecs, nr_vecs), gfp_mask); } @@ -1097,13 +1098,15 @@ int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { bool same_page = false; + bool skip_merge = false; if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; if (bio->bi_iter.bi_size > UINT_MAX - len) return 0; - if (bio->bi_vcnt > 0 && + trace_android_vh_bio_add_page_merge_bypass(bio, &skip_merge); + if (!skip_merge && bio->bi_vcnt > 0 && bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1], page, len, offset, &same_page)) { bio->bi_iter.bi_size += len; @@ -1196,11 +1199,13 @@ static int bio_iov_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { bool same_page = false; + bool skip_merge = false; if (WARN_ON_ONCE(bio->bi_iter.bi_size > UINT_MAX - len)) return -EIO; - if (bio->bi_vcnt > 0 && + trace_android_vh_bio_add_page_merge_bypass(bio, &skip_merge); + if (!skip_merge && bio->bi_vcnt > 0 && bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1], page, len, offset, &same_page)) { bio->bi_iter.bi_size += len; diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 64551b0aa51e6..916cf4f907b36 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -848,14 +848,8 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, disk = ctx->bdev->bd_disk; q = disk->queue; - /* - * blkcg_deactivate_policy() requires queue to be frozen, we can grab - * q_usage_counter to prevent concurrent with blkcg_deactivate_policy(). - */ - ret = blk_queue_enter(q, 0); - if (ret) - goto fail; - + /* Prevent concurrent with blkcg_deactivate_policy() */ + mutex_lock(&q->blkcg_mutex); spin_lock_irq(&q->queue_lock); if (!blkcg_policy_enabled(q, pol)) { @@ -885,16 +879,16 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, /* Drop locks to do new blkg allocation with GFP_KERNEL. */ spin_unlock_irq(&q->queue_lock); - new_blkg = blkg_alloc(pos, disk, GFP_KERNEL); + new_blkg = blkg_alloc(pos, disk, GFP_NOIO); if (unlikely(!new_blkg)) { ret = -ENOMEM; - goto fail_exit_queue; + goto fail_exit; } if (radix_tree_preload(GFP_KERNEL)) { blkg_free(new_blkg); ret = -ENOMEM; - goto fail_exit_queue; + goto fail_exit; } spin_lock_irq(&q->queue_lock); @@ -922,7 +916,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, goto success; } success: - blk_queue_exit(q); + mutex_unlock(&q->blkcg_mutex); ctx->blkg = blkg; return 0; @@ -930,9 +924,8 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, radix_tree_preload_end(); fail_unlock: spin_unlock_irq(&q->queue_lock); -fail_exit_queue: - blk_queue_exit(q); -fail: +fail_exit: + mutex_unlock(&q->blkcg_mutex); /* * If queue was bypassing, we should retry. Do so after a * short msleep(). It isn't strictly necessary but queue @@ -1566,6 +1559,14 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (blkcg_policy_enabled(q, pol)) return 0; + /* + * Policy is allowed to be registered without pd_alloc_fn/pd_free_fn, + * for example, ioprio. Such policy will work on blkcg level, not disk + * level, and don't need to be activated. + */ + if (WARN_ON_ONCE(!pol->pd_alloc_fn || !pol->pd_free_fn)) + return -EINVAL; + if (queue_is_mq(q)) blk_mq_freeze_queue(q); retry: @@ -1745,9 +1746,12 @@ int blkcg_policy_register(struct blkcg_policy *pol) goto err_unlock; } - /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */ + /* + * Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs, and policy + * without pd_alloc_fn/pd_free_fn can't be activated. + */ if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) || - (!pol->pd_alloc_fn ^ !pol->pd_free_fn)) + (!pol->pd_alloc_fn ^ !pol->pd_free_fn)) goto err_unlock; /* register @pol */ diff --git a/block/blk-core.c b/block/blk-core.c index 4ad61822e3afc..1c4a5e61d5ba7 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -887,6 +887,9 @@ static void bio_set_ioprio(struct bio *bio) void submit_bio(struct bio *bio) { if (bio_op(bio) == REQ_OP_READ) { + //#ifdef OPLUS_STORAGE_FS debug for bugid 7760993 + BUG_ON(!bio->bi_iter.bi_size); + //#endif task_io_account_read(bio->bi_iter.bi_size); count_vm_events(PGPGIN, bio_sectors(bio)); } else if (bio_op(bio) == REQ_OP_WRITE) { diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 638a747596b59..0b572fbce1f58 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "blk-cgroup.h" #include "blk-crypto-internal.h" @@ -232,7 +233,9 @@ static bool blk_crypto_fallback_split_bio_if_needed(struct bio **bio_ptr) bio->bi_status = BLK_STS_RESOURCE; return false; } + bio_chain(split_bio, bio); + trace_block_split(split_bio, bio->bi_iter.bi_sector); submit_bio_noacct(bio); *bio_ptr = split_bio; } diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c index 156e9bb07abf1..2fb234ab467b1 100644 --- a/block/blk-mq-sysfs.c +++ b/block/blk-mq-sysfs.c @@ -150,9 +150,11 @@ static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx) return; hctx_for_each_ctx(hctx, ctx, i) - kobject_del(&ctx->kobj); + if (ctx->kobj.state_in_sysfs) + kobject_del(&ctx->kobj); - kobject_del(&hctx->kobj); + if (hctx->kobj.state_in_sysfs) + kobject_del(&hctx->kobj); } static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) diff --git a/block/blk-mq.c b/block/blk-mq.c index 12eee9e1382ca..841a1bf040e61 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3545,11 +3545,13 @@ struct rq_iter_data { static bool blk_mq_has_request(struct request *rq, void *data) { struct rq_iter_data *iter_data = data; + bool ret = false; if (rq->mq_hctx != iter_data->hctx) return true; iter_data->has_rq = true; - return false; + trace_android_rvh_blk_mq_has_request(iter_data->hctx, rq, &ret); + return ret; } static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) diff --git a/block/blk-settings.c b/block/blk-settings.c index f2d1f0b22721d..1527f3f3ce072 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -624,7 +624,8 @@ static unsigned int blk_round_down_sectors(unsigned int sectors, unsigned int lb int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, sector_t start) { - unsigned int top, bottom, alignment, ret = 0; + unsigned int top, bottom, alignment; + int ret = 0; t->max_sectors = min_not_zero(t->max_sectors, b->max_sectors); t->max_hw_sectors = min_not_zero(t->max_hw_sectors, b->max_hw_sectors); @@ -705,7 +706,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, } /* chunk_sectors a multiple of the physical block size? */ - if ((t->chunk_sectors << 9) & (t->physical_block_size - 1)) { + if (t->chunk_sectors % (t->physical_block_size >> SECTOR_SHIFT)) { t->chunk_sectors = 0; t->misaligned = 1; ret = -1; diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 74839f6f2e0cb..8d15c73a520bd 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -909,4 +909,5 @@ void blk_unregister_queue(struct gendisk *disk) mutex_unlock(&q->sysfs_dir_lock); blk_debugfs_remove(disk); + kobject_put(&disk->queue_kobj); } diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 9a5ca71c5b131..37a9196b65154 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -406,6 +406,7 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, op = REQ_OP_ZONE_RESET; /* Invalidate the page cache, including dirty pages. */ + inode_lock(bdev->bd_inode); filemap_invalidate_lock(bdev->bd_inode->i_mapping); ret = blkdev_truncate_zone_range(bdev, mode, &zrange); if (ret) @@ -428,8 +429,10 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, GFP_KERNEL); fail: - if (cmd == BLKRESETZONE) + if (cmd == BLKRESETZONE) { filemap_invalidate_unlock(bdev->bd_inode->i_mapping); + inode_unlock(bdev->bd_inode); + } return ret; } diff --git a/block/fops.c b/block/fops.c index b91cb52991132..40e6a6ef5feb0 100644 --- a/block/fops.c +++ b/block/fops.c @@ -684,7 +684,14 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) ret = direct_write_fallback(iocb, from, ret, blkdev_buffered_write(iocb, from)); } else { + /* + * Take i_rwsem and invalidate_lock to avoid racing with + * set_blocksize changing i_blkbits/folio order and punching + * out the pagecache. + */ + inode_lock_shared(bd_inode); ret = blkdev_buffered_write(iocb, from); + inode_unlock_shared(bd_inode); } if (ret > 0) @@ -696,6 +703,7 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host); + struct inode *bd_inode = bdev->bd_inode; loff_t size = bdev_nr_bytes(bdev); loff_t pos = iocb->ki_pos; size_t shorted = 0; @@ -731,7 +739,13 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) goto reexpand; } + /* + * Take i_rwsem and invalidate_lock to avoid racing with set_blocksize + * changing i_blkbits/folio order and punching out the pagecache. + */ + inode_lock_shared(bd_inode); ret = filemap_read(iocb, to, ret); + inode_unlock_shared(bd_inode); reexpand: if (unlikely(shorted)) @@ -774,6 +788,7 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, if ((start | len) & (bdev_logical_block_size(bdev) - 1)) return -EINVAL; + inode_lock(inode); filemap_invalidate_lock(inode->i_mapping); /* @@ -814,6 +829,7 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, fail: filemap_invalidate_unlock(inode->i_mapping); + inode_unlock(inode); return error; } diff --git a/block/ioctl.c b/block/ioctl.c index 231537f79a8cb..024767fa1e52d 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -114,6 +114,7 @@ static int blk_ioctl_discard(struct block_device *bdev, blk_mode_t mode, end > bdev_nr_bytes(bdev)) return -EINVAL; + inode_lock(inode); filemap_invalidate_lock(inode->i_mapping); err = truncate_bdev_range(bdev, mode, start, end - 1); if (err) @@ -121,6 +122,7 @@ static int blk_ioctl_discard(struct block_device *bdev, blk_mode_t mode, err = blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_KERNEL); fail: filemap_invalidate_unlock(inode->i_mapping); + inode_unlock(inode); return err; } @@ -146,12 +148,14 @@ static int blk_ioctl_secure_erase(struct block_device *bdev, blk_mode_t mode, end > bdev_nr_bytes(bdev)) return -EINVAL; + inode_lock(bdev->bd_inode); filemap_invalidate_lock(bdev->bd_inode->i_mapping); err = truncate_bdev_range(bdev, mode, start, end - 1); if (!err) err = blkdev_issue_secure_erase(bdev, start >> 9, len >> 9, GFP_KERNEL); filemap_invalidate_unlock(bdev->bd_inode->i_mapping); + inode_unlock(bdev->bd_inode); return err; } @@ -184,6 +188,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, blk_mode_t mode, return -EINVAL; /* Invalidate the page cache, including dirty pages */ + inode_lock(inode); filemap_invalidate_lock(inode->i_mapping); err = truncate_bdev_range(bdev, mode, start, end); if (err) @@ -194,6 +199,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, blk_mode_t mode, fail: filemap_invalidate_unlock(inode->i_mapping); + inode_unlock(inode); return err; } diff --git a/build.config.gki_consolidate.aarch64 b/build.config.gki_consolidate.aarch64 new file mode 100644 index 0000000000000..7c835ad18cbdc --- /dev/null +++ b/build.config.gki_consolidate.aarch64 @@ -0,0 +1,5 @@ +KERNEL_DIR=common +DEFCONFIG=consolidate_defconfig +FRAGMENT_CONFIG=${KERNEL_DIR}/arch/arm64/configs/consolidate.fragment +PRE_DEFCONFIG_CMDS="KCONFIG_CONFIG=${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG} ${ROOT_DIR}/${KERNEL_DIR}/scripts/kconfig/merge_config.sh -m -r ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/gki_defconfig ${ROOT_DIR}/${FRAGMENT_CONFIG}" +POST_DEFCONFIG_CMDS="rm ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG}" diff --git a/consolidate.bzl b/consolidate.bzl new file mode 100644 index 0000000000000..367aac84ae681 --- /dev/null +++ b/consolidate.bzl @@ -0,0 +1,70 @@ +load( + "//build/kernel/kleaf:kernel.bzl", + "kernel_build", + "kernel_build_config", + "kernel_images", + "kernel_modules_install", +) +load("//build/kernel/kleaf:constants.bzl", "aarch64_outs") +load(":modules.bzl", "get_gki_modules_list", "get_kunit_modules_list") + +rule_base = "kernel_aarch64_consolidate" + +def _gen_config_without_source_lines(build_config, target): + rule_name = "{}.{}".format(target, build_config) + out_file_name = rule_name + ".generated" + native.genrule( + name = rule_name, + srcs = [build_config], + outs = [out_file_name], + cmd_bash = "sed -e '/^\\. /d' $(location {}) > $@".format(build_config), + ) + + return ":" + rule_name + +def define_consolidate(): + kernel_build_config( + name = rule_base + "_build_config", + srcs = [ + # do not sort + "build.config.constants", + _gen_config_without_source_lines("build.config.common", rule_base), + "build.config.aarch64", + "build.config.gki_consolidate.aarch64", + _gen_config_without_source_lines("build.config.gki.aarch64", rule_base), + ], + ) + + kernel_build( + name = rule_base, + outs = aarch64_outs, + implicit_outs = [ + "scripts/sign-file", + "certs/signing_key.pem", + "certs/signing_key.x509", + ], + make_goals = [ + "Image", + "modules", + "Image.lz4", + "Image.gz", + ], + page_size = "4k", + module_implicit_outs = get_gki_modules_list("arm64") + get_kunit_modules_list("arm64"), + build_config = rule_base + "_build_config", + trim_nonlisted_kmi = False, + ) + + kernel_modules_install( + name = "{}_modules_install".format(rule_base), + kernel_build = ":{}".format(rule_base), + ) + + kernel_images( + name = "{}_images".format(rule_base), + kernel_build = ":{}".format(rule_base), + build_system_dlkm = True, + build_system_dlkm_flatten = True, + system_dlkm_fs_types = ["ext4"], + kernel_modules_install = ":{}_modules_install".format(rule_base), + ) diff --git a/consolidate_16k.bzl b/consolidate_16k.bzl new file mode 100644 index 0000000000000..af6ad37b03e81 --- /dev/null +++ b/consolidate_16k.bzl @@ -0,0 +1,70 @@ +load( + "//build/kernel/kleaf:kernel.bzl", + "kernel_build", + "kernel_build_config", + "kernel_images", + "kernel_modules_install", +) +load("//build/kernel/kleaf:constants.bzl", "aarch64_outs") +load(":modules.bzl", "get_gki_modules_list", "get_kunit_modules_list") + +rule_base = "kernel_aarch64_consolidate_16k" + +def _gen_config_without_source_lines(build_config, target): + rule_name = "{}.{}".format(target, build_config) + out_file_name = rule_name + ".generated" + native.genrule( + name = rule_name, + srcs = [build_config], + outs = [out_file_name], + cmd_bash = "sed -e '/^\\. /d' $(location {}) > $@".format(build_config), + ) + + return ":" + rule_name + +def define_consolidate_16k(): + kernel_build_config( + name = rule_base + "_build_config", + srcs = [ + # do not sort + "build.config.constants", + _gen_config_without_source_lines("build.config.common", rule_base), + "build.config.aarch64", + "build.config.gki_consolidate.aarch64", + _gen_config_without_source_lines("build.config.gki.aarch64", rule_base), + ], + ) + + kernel_build( + name = rule_base, + outs = aarch64_outs, + implicit_outs = [ + "scripts/sign-file", + "certs/signing_key.pem", + "certs/signing_key.x509", + ], + make_goals = [ + "Image", + "modules", + "Image.lz4", + "Image.gz", + ], + page_size = "16k", + module_implicit_outs = get_gki_modules_list("arm64") + get_kunit_modules_list("arm64"), + build_config = rule_base + "_build_config", + trim_nonlisted_kmi = False, + ) + + kernel_modules_install( + name = "{}_modules_install".format(rule_base), + kernel_build = ":{}".format(rule_base), + ) + + kernel_images( + name = "{}_images".format(rule_base), + kernel_build = ":{}".format(rule_base), + build_system_dlkm = True, + build_system_dlkm_flatten = True, + system_dlkm_fs_types = ["ext4"], + kernel_modules_install = ":{}_modules_install".format(rule_base), + ) diff --git a/crypto/Kconfig b/crypto/Kconfig index 93c2f20d88954..b74df31f5355d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1360,9 +1360,69 @@ config CRYPTO_JITTERENTROPY See https://www.chronox.de/jent/ +if CRYPTO_JITTERENTROPY +if CRYPTO_FIPS && EXPERT + +choice + prompt "CPU Jitter RNG Memory Size" + default CRYPTO_JITTERENTROPY_MEMSIZE_2 + help + The Jitter RNG measures the execution time of memory accesses. + Multiple consecutive memory accesses are performed. If the memory + size fits into a cache (e.g. L1), only the memory access timing + to that cache is measured. The closer the cache is to the CPU + the less variations are measured and thus the less entropy is + obtained. Thus, if the memory size fits into the L1 cache, the + obtained entropy is less than if the memory size fits within + L1 + L2, which in turn is less if the memory fits into + L1 + L2 + L3. Thus, by selecting a different memory size, + the entropy rate produced by the Jitter RNG can be modified. + + config CRYPTO_JITTERENTROPY_MEMSIZE_2 + bool "2048 Bytes (default)" + + config CRYPTO_JITTERENTROPY_MEMSIZE_128 + bool "128 kBytes" + + config CRYPTO_JITTERENTROPY_MEMSIZE_1024 + bool "1024 kBytes" + + config CRYPTO_JITTERENTROPY_MEMSIZE_8192 + bool "8192 kBytes" +endchoice + +config CRYPTO_JITTERENTROPY_MEMORY_BLOCKS + int + default 64 if CRYPTO_JITTERENTROPY_MEMSIZE_2 + default 512 if CRYPTO_JITTERENTROPY_MEMSIZE_128 + default 1024 if CRYPTO_JITTERENTROPY_MEMSIZE_1024 + default 4096 if CRYPTO_JITTERENTROPY_MEMSIZE_8192 + +config CRYPTO_JITTERENTROPY_MEMORY_BLOCKSIZE + int + default 32 if CRYPTO_JITTERENTROPY_MEMSIZE_2 + default 256 if CRYPTO_JITTERENTROPY_MEMSIZE_128 + default 1024 if CRYPTO_JITTERENTROPY_MEMSIZE_1024 + default 2048 if CRYPTO_JITTERENTROPY_MEMSIZE_8192 + +config CRYPTO_JITTERENTROPY_OSR + int "CPU Jitter RNG Oversampling Rate" + range 1 15 + default 3 + help + The Jitter RNG allows the specification of an oversampling rate (OSR). + The Jitter RNG operation requires a fixed amount of timing + measurements to produce one output block of random numbers. The + OSR value is multiplied with the amount of timing measurements to + generate one output block. Thus, the timing measurement is oversampled + by the OSR factor. The oversampling allows the Jitter RNG to operate + on hardware whose timers deliver limited amount of entropy (e.g. + the timer is coarse) by setting the OSR to a higher value. The + trade-off, however, is that the Jitter RNG now requires more time + to generate random numbers. + config CRYPTO_JITTERENTROPY_TESTINTERFACE bool "CPU Jitter RNG Test Interface" - depends on CRYPTO_JITTERENTROPY help The test interface allows a privileged process to capture the raw unconditioned high resolution time stamp noise that @@ -1380,6 +1440,28 @@ config CRYPTO_JITTERENTROPY_TESTINTERFACE If unsure, select N. +endif # if CRYPTO_FIPS && EXPERT + +if !(CRYPTO_FIPS && EXPERT) + +config CRYPTO_JITTERENTROPY_MEMORY_BLOCKS + int + default 64 + +config CRYPTO_JITTERENTROPY_MEMORY_BLOCKSIZE + int + default 32 + +config CRYPTO_JITTERENTROPY_OSR + int + default 1 + +config CRYPTO_JITTERENTROPY_TESTINTERFACE + bool + +endif # if !(CRYPTO_FIPS && EXPERT) +endif # if CRYPTO_JITTERENTROPY + config CRYPTO_KDF800108_CTR tristate select CRYPTO_HMAC diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 68cc9290cabe9..886eccb97b041 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -969,6 +969,12 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, } lock_sock(sk); + if (ctx->write) { + release_sock(sk); + return -EBUSY; + } + ctx->write = true; + if (ctx->init && !ctx->more) { if (ctx->used) { err = -EINVAL; @@ -1018,6 +1024,8 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, continue; } + ctx->merge = 0; + if (!af_alg_writable(sk)) { err = af_alg_wait_for_wmem(sk, msg->msg_flags); if (err) @@ -1057,7 +1065,6 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, ctx->used += plen; copied += plen; size -= plen; - ctx->merge = 0; } else { do { struct page *pg; @@ -1103,6 +1110,7 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, unlock: af_alg_data_wakeup(sk); + ctx->write = false; release_sock(sk); return copied ?: err; diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c index e24c829d7a015..5ab7441734b8e 100644 --- a/crypto/algif_hash.c +++ b/crypto/algif_hash.c @@ -265,10 +265,6 @@ static int hash_accept(struct socket *sock, struct socket *newsock, int flags, goto out_free_state; err = crypto_ahash_import(&ctx2->req, state); - if (err) { - sock_orphan(sk2); - sock_put(sk2); - } out_free_state: kfree_sensitive(state); diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index 0a7049b470c18..5201f47e31d8b 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -618,11 +618,14 @@ int x509_process_extension(void *context, size_t hdrlen, /* * Get hold of the basicConstraints * v[1] is the encoding size - * (Expect 0x2 or greater, making it 1 or more bytes) + * (Expect 0x00 for empty SEQUENCE with CA:FALSE, or + * 0x03 or greater for non-empty SEQUENCE) * v[2] is the encoding type * (Expect an ASN1_BOOL for the CA) - * v[3] is the contents of the ASN1_BOOL - * (Expect 1 if the CA is TRUE) + * v[3] is the length of the ASN1_BOOL + * (Expect 1 for a single byte boolean) + * v[4] is the contents of the ASN1_BOOL + * (Expect 0xFF if the CA is TRUE) * vlen should match the entire extension size */ if (v[0] != (ASN1_CONS_BIT | ASN1_SEQ)) @@ -631,8 +634,13 @@ int x509_process_extension(void *context, size_t hdrlen, return -EBADMSG; if (v[1] != vlen - 2) return -EBADMSG; - if (vlen >= 4 && v[1] != 0 && v[2] == ASN1_BOOL && v[3] == 1) + /* Empty SEQUENCE means CA:FALSE (default value omitted per DER) */ + if (v[1] == 0) + return 0; + if (vlen >= 5 && v[2] == ASN1_BOOL && v[3] == 1 && v[4] == 0xFF) ctx->cert->pub->key_eflags |= 1 << KEY_EFLAG_CA; + else + return -EBADMSG; return 0; } diff --git a/crypto/ecc.c b/crypto/ecc.c index 21504280aca2e..27fcecf3a61fb 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -69,7 +69,7 @@ EXPORT_SYMBOL(ecc_get_curve); void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes, u64 *out, unsigned int ndigits) { - int diff = ndigits - DIV_ROUND_UP(nbytes, sizeof(u64)); + int diff = ndigits - DIV_ROUND_UP_POW2(nbytes, sizeof(u64)); unsigned int o = nbytes & 7; __be64 msd = 0; diff --git a/crypto/essiv.c b/crypto/essiv.c index f7d4ef4837e54..4dbec116ddc3e 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -186,9 +186,14 @@ static int essiv_aead_crypt(struct aead_request *req, bool enc) const struct essiv_tfm_ctx *tctx = crypto_aead_ctx(tfm); struct essiv_aead_request_ctx *rctx = aead_request_ctx(req); struct aead_request *subreq = &rctx->aead_req; + int ivsize = crypto_aead_ivsize(tfm); + int ssize = req->assoclen - ivsize; struct scatterlist *src = req->src; int err; + if (ssize < 0) + return -EINVAL; + crypto_cipher_encrypt_one(tctx->essiv_cipher, req->iv, req->iv); /* @@ -198,19 +203,12 @@ static int essiv_aead_crypt(struct aead_request *req, bool enc) */ rctx->assoc = NULL; if (req->src == req->dst || !enc) { - scatterwalk_map_and_copy(req->iv, req->dst, - req->assoclen - crypto_aead_ivsize(tfm), - crypto_aead_ivsize(tfm), 1); + scatterwalk_map_and_copy(req->iv, req->dst, ssize, ivsize, 1); } else { u8 *iv = (u8 *)aead_request_ctx(req) + tctx->ivoffset; - int ivsize = crypto_aead_ivsize(tfm); - int ssize = req->assoclen - ivsize; struct scatterlist *sg; int nents; - if (ssize < 0) - return -EINVAL; - nents = sg_nents_for_len(req->src, ssize); if (nents < 0) return -EINVAL; diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 7d1463a1562ac..e3164f0625d7e 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -54,6 +54,17 @@ * Helper function ***************************************************************************/ +void *jent_kvzalloc(unsigned int len) +{ + return kvzalloc(len, GFP_KERNEL); +} + +void jent_kvzfree(void *ptr, unsigned int len) +{ + memzero_explicit(ptr, len); + kvfree(ptr); +} + void *jent_zalloc(unsigned int len) { return kzalloc(len, GFP_KERNEL); @@ -107,6 +118,7 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, pr_warn_ratelimited("Unexpected digest size\n"); return -EINVAL; } + kmsan_unpoison_memory(intermediary, sizeof(intermediary)); /* * This loop fills a buffer which is injected into the entropy pool. @@ -134,7 +146,7 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, * Inject the data from the previous loop into the pool. This data is * not considered to contain any entropy, but it stirs the pool a bit. */ - ret = crypto_shash_update(desc, intermediary, sizeof(intermediary)); + ret = crypto_shash_update(hash_state_desc, intermediary, sizeof(intermediary)); if (ret) goto err; @@ -147,11 +159,12 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, * conditioning operation to have an identical amount of input data * according to section 3.1.5. */ - if (!stuck) { - ret = crypto_shash_update(hash_state_desc, (u8 *)&time, - sizeof(__u64)); + if (stuck) { + time = 0; } + ret = crypto_shash_update(hash_state_desc, (u8 *)&time, sizeof(__u64)); + err: shash_desc_zero(desc); memzero_explicit(intermediary, sizeof(intermediary)); @@ -245,7 +258,9 @@ static int jent_kcapi_init(struct crypto_tfm *tfm) crypto_shash_init(sdesc); rng->sdesc = sdesc; - rng->entropy_collector = jent_entropy_collector_alloc(1, 0, sdesc); + rng->entropy_collector = + jent_entropy_collector_alloc(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, + sdesc); if (!rng->entropy_collector) { ret = -ENOMEM; goto err; @@ -334,7 +349,7 @@ static int __init jent_mod_init(void) desc->tfm = tfm; crypto_shash_init(desc); - ret = jent_entropy_init(desc); + ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, desc, NULL); shash_desc_zero(desc); crypto_free_shash(tfm); if (ret) { diff --git a/crypto/jitterentropy-testing.c b/crypto/jitterentropy-testing.c index 5cb6a77b8e3b2..21c9d7c3269a2 100644 --- a/crypto/jitterentropy-testing.c +++ b/crypto/jitterentropy-testing.c @@ -15,7 +15,7 @@ #define JENT_TEST_RINGBUFFER_MASK (JENT_TEST_RINGBUFFER_SIZE - 1) struct jent_testing { - u32 jent_testing_rb[JENT_TEST_RINGBUFFER_SIZE]; + u64 jent_testing_rb[JENT_TEST_RINGBUFFER_SIZE]; u32 rb_reader; atomic_t rb_writer; atomic_t jent_testing_enabled; @@ -72,7 +72,7 @@ static void jent_testing_fini(struct jent_testing *data, u32 boot) pr_warn("Disabling data collection\n"); } -static bool jent_testing_store(struct jent_testing *data, u32 value, +static bool jent_testing_store(struct jent_testing *data, u64 value, u32 *boot) { unsigned long flags; @@ -156,20 +156,20 @@ static int jent_testing_reader(struct jent_testing *data, u32 *boot, } /* We copy out word-wise */ - if (outbuflen < sizeof(u32)) { + if (outbuflen < sizeof(u64)) { spin_unlock_irqrestore(&data->lock, flags); goto out; } memcpy(outbuf, &data->jent_testing_rb[data->rb_reader], - sizeof(u32)); + sizeof(u64)); data->rb_reader++; spin_unlock_irqrestore(&data->lock, flags); - outbuf += sizeof(u32); - outbuflen -= sizeof(u32); - collected_data += sizeof(u32); + outbuf += sizeof(u64); + outbuflen -= sizeof(u64); + collected_data += sizeof(u64); } out: @@ -189,16 +189,17 @@ static int jent_testing_extract_user(struct file *file, char __user *buf, /* * The intention of this interface is for collecting at least - * 1000 samples due to the SP800-90B requirements. So, we make no - * effort in avoiding allocating more memory that actually needed - * by the user. Hence, we allocate sufficient memory to always hold - * that amount of data. + * 1000 samples due to the SP800-90B requirements. However, due to + * memory and performance constraints, it is not desirable to allocate + * 8000 bytes of memory. Instead, we allocate space for only 125 + * samples, which will allow the user to collect all 1000 samples using + * 8 calls to this interface. */ - tmp = kmalloc(JENT_TEST_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); + tmp = kmalloc(125 * sizeof(u64) + sizeof(u64), GFP_KERNEL); if (!tmp) return -ENOMEM; - tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); + tmp_aligned = PTR_ALIGN(tmp, sizeof(u64)); while (nbytes) { int i; @@ -212,7 +213,7 @@ static int jent_testing_extract_user(struct file *file, char __user *buf, schedule(); } - i = min_t(int, nbytes, JENT_TEST_RINGBUFFER_SIZE); + i = min_t(int, nbytes, 125 * sizeof(u64)); i = reader(tmp_aligned, i); if (i <= 0) { if (i < 0) @@ -251,7 +252,7 @@ static struct jent_testing jent_raw_hires = { .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(jent_raw_hires.read_wait) }; -int jent_raw_hires_entropy_store(__u32 value) +int jent_raw_hires_entropy_store(__u64 value) { return jent_testing_store(&jent_raw_hires, value, &boot_raw_hires_test); } diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index fe9c233ec7693..26a9048bc893d 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -72,11 +72,13 @@ struct rand_data { __u64 prev_time; /* SENSITIVE Previous time stamp */ __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ + + unsigned int flags; /* Flags used to initialize */ unsigned int osr; /* Oversample rate */ -#define JENT_MEMORY_BLOCKS 64 -#define JENT_MEMORY_BLOCKSIZE 32 #define JENT_MEMORY_ACCESSLOOPS 128 -#define JENT_MEMORY_SIZE (JENT_MEMORY_BLOCKS*JENT_MEMORY_BLOCKSIZE) +#define JENT_MEMORY_SIZE \ + (CONFIG_CRYPTO_JITTERENTROPY_MEMORY_BLOCKS * \ + CONFIG_CRYPTO_JITTERENTROPY_MEMORY_BLOCKSIZE) unsigned char *mem; /* Memory access location with size of * memblocks * memblocksize */ unsigned int memlocation; /* Pointer to byte in *mem */ @@ -88,16 +90,9 @@ struct rand_data { /* Repetition Count Test */ unsigned int rct_count; /* Number of stuck values */ - /* Intermittent health test failure threshold of 2^-30 */ - /* From an SP800-90B perspective, this RCT cutoff value is equal to 31. */ - /* However, our RCT implementation starts at 1, so we subtract 1 here. */ -#define JENT_RCT_CUTOFF (31 - 1) /* Taken from SP800-90B sec 4.4.1 */ -#define JENT_APT_CUTOFF 325 /* Taken from SP800-90B sec 4.4.2 */ - /* Permanent health test failure threshold of 2^-60 */ - /* From an SP800-90B perspective, this RCT cutoff value is equal to 61. */ - /* However, our RCT implementation starts at 1, so we subtract 1 here. */ -#define JENT_RCT_CUTOFF_PERMANENT (61 - 1) -#define JENT_APT_CUTOFF_PERMANENT 355 + /* Adaptive Proportion Test cutoff values */ + unsigned int apt_cutoff; /* Intermittent health test failure */ + unsigned int apt_cutoff_permanent; /* Permanent health test failure */ #define JENT_APT_WINDOW_SIZE 512 /* Data window size */ /* LSB of time stamp to process */ #define JENT_APT_LSB 16 @@ -105,6 +100,8 @@ struct rand_data { unsigned int apt_observations; /* Number of collected observations */ unsigned int apt_count; /* APT counter */ unsigned int apt_base; /* APT base reference */ + unsigned int health_failure; /* Record health failure */ + unsigned int apt_base_set:1; /* APT base reference set? */ }; @@ -122,6 +119,16 @@ struct rand_data { * zero). */ #define JENT_ESTUCK 8 /* Too many stuck results during init. */ #define JENT_EHEALTH 9 /* Health test failed during initialization */ +#define JENT_ERCT 10 /* RCT failed during initialization */ +#define JENT_EHASH 11 /* Hash self test failed */ +#define JENT_EMEM 12 /* Can't allocate memory for initialization */ + +#define JENT_RCT_FAILURE 1 /* Failure in RCT health test. */ +#define JENT_APT_FAILURE 2 /* Failure in APT health test. */ +#define JENT_PERMANENT_FAILURE_SHIFT 16 +#define JENT_PERMANENT_FAILURE(x) (x << JENT_PERMANENT_FAILURE_SHIFT) +#define JENT_RCT_FAILURE_PERMANENT JENT_PERMANENT_FAILURE(JENT_RCT_FAILURE) +#define JENT_APT_FAILURE_PERMANENT JENT_PERMANENT_FAILURE(JENT_APT_FAILURE) /* * The output n bits can receive more than n bits of min entropy, of course, @@ -147,6 +154,48 @@ struct rand_data { * This test complies with SP800-90B section 4.4.2. ***************************************************************************/ +/* + * See the SP 800-90B comment #10b for the corrected cutoff for the SP 800-90B + * APT. + * http://www.untruth.org/~josh/sp80090b/UL%20SP800-90B-final%20comments%20v1.9%2020191212.pdf + * In in the syntax of R, this is C = 2 + qbinom(1 − 2^(−30), 511, 2^(-1/osr)). + * (The original formula wasn't correct because the first symbol must + * necessarily have been observed, so there is no chance of observing 0 of these + * symbols.) + * + * For the alpha < 2^-53, R cannot be used as it uses a float data type without + * arbitrary precision. A SageMath script is used to calculate those cutoff + * values. + * + * For any value above 14, this yields the maximal allowable value of 512 + * (by FIPS 140-2 IG 7.19 Resolution # 16, we cannot choose a cutoff value that + * renders the test unable to fail). + */ +static const unsigned int jent_apt_cutoff_lookup[15] = { + 325, 422, 459, 477, 488, 494, 499, 502, + 505, 507, 508, 509, 510, 511, 512 }; +static const unsigned int jent_apt_cutoff_permanent_lookup[15] = { + 355, 447, 479, 494, 502, 507, 510, 512, + 512, 512, 512, 512, 512, 512, 512 }; +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +static void jent_apt_init(struct rand_data *ec, unsigned int osr) +{ + /* + * Establish the apt_cutoff based on the presumed entropy rate of + * 1/osr. + */ + if (osr >= ARRAY_SIZE(jent_apt_cutoff_lookup)) { + ec->apt_cutoff = jent_apt_cutoff_lookup[ + ARRAY_SIZE(jent_apt_cutoff_lookup) - 1]; + ec->apt_cutoff_permanent = jent_apt_cutoff_permanent_lookup[ + ARRAY_SIZE(jent_apt_cutoff_permanent_lookup) - 1]; + } else { + ec->apt_cutoff = jent_apt_cutoff_lookup[osr - 1]; + ec->apt_cutoff_permanent = + jent_apt_cutoff_permanent_lookup[osr - 1]; + } +} /* * Reset the APT counter * @@ -175,26 +224,22 @@ static void jent_apt_insert(struct rand_data *ec, unsigned int delta_masked) return; } - if (delta_masked == ec->apt_base) + if (delta_masked == ec->apt_base) { ec->apt_count++; + /* Note, ec->apt_count starts with one. */ + if (ec->apt_count >= ec->apt_cutoff_permanent) + ec->health_failure |= JENT_APT_FAILURE_PERMANENT; + else if (ec->apt_count >= ec->apt_cutoff) + ec->health_failure |= JENT_APT_FAILURE; + } + ec->apt_observations++; if (ec->apt_observations >= JENT_APT_WINDOW_SIZE) jent_apt_reset(ec, delta_masked); } -/* APT health test failure detection */ -static int jent_apt_permanent_failure(struct rand_data *ec) -{ - return (ec->apt_count >= JENT_APT_CUTOFF_PERMANENT) ? 1 : 0; -} - -static int jent_apt_failure(struct rand_data *ec) -{ - return (ec->apt_count >= JENT_APT_CUTOFF) ? 1 : 0; -} - /*************************************************************************** * Stuck Test and its use as Repetition Count Test * @@ -221,6 +266,30 @@ static void jent_rct_insert(struct rand_data *ec, int stuck) { if (stuck) { ec->rct_count++; + + /* + * The cutoff value is based on the following consideration: + * alpha = 2^-30 or 2^-60 as recommended in SP800-90B. + * In addition, we require an entropy value H of 1/osr as this + * is the minimum entropy required to provide full entropy. + * Note, we collect (DATA_SIZE_BITS + ENTROPY_SAFETY_FACTOR)*osr + * deltas for inserting them into the entropy pool which should + * then have (close to) DATA_SIZE_BITS bits of entropy in the + * conditioned output. + * + * Note, ec->rct_count (which equals to value B in the pseudo + * code of SP800-90B section 4.4.1) starts with zero. Hence + * we need to subtract one from the cutoff value as calculated + * following SP800-90B. Thus C = ceil(-log_2(alpha)/H) = 30*osr + * or 60*osr. + */ + if ((unsigned int)ec->rct_count >= (60 * ec->osr)) { + ec->rct_count = -1; + ec->health_failure |= JENT_RCT_FAILURE_PERMANENT; + } else if ((unsigned int)ec->rct_count >= (30 * ec->osr)) { + ec->rct_count = -1; + ec->health_failure |= JENT_RCT_FAILURE; + } } else { /* Reset RCT */ ec->rct_count = 0; @@ -275,26 +344,25 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta) return 0; } -/* RCT health test failure detection */ -static int jent_rct_permanent_failure(struct rand_data *ec) -{ - return (ec->rct_count >= JENT_RCT_CUTOFF_PERMANENT) ? 1 : 0; -} - -static int jent_rct_failure(struct rand_data *ec) -{ - return (ec->rct_count >= JENT_RCT_CUTOFF) ? 1 : 0; -} - -/* Report of health test failures */ -static int jent_health_failure(struct rand_data *ec) +/* + * Report any health test failures + * + * @ec [in] Reference to entropy collector + * + * @return a bitmask indicating which tests failed + * 0 No health test failure + * 1 RCT failure + * 2 APT failure + * 1<health_failure; } /*************************************************************************** @@ -448,7 +516,7 @@ static void jent_memaccess(struct rand_data *ec, __u64 loop_cnt) * * @return result of stuck test */ -static int jent_measure_jitter(struct rand_data *ec) +static int jent_measure_jitter(struct rand_data *ec, __u64 *ret_current_delta) { __u64 time = 0; __u64 current_delta = 0; @@ -472,6 +540,10 @@ static int jent_measure_jitter(struct rand_data *ec) if (jent_condition_data(ec, current_delta, stuck)) stuck = 1; + /* return the raw entropy value */ + if (ret_current_delta) + *ret_current_delta = current_delta; + return stuck; } @@ -489,11 +561,11 @@ static void jent_gen_entropy(struct rand_data *ec) safety_factor = JENT_ENTROPY_SAFETY_FACTOR; /* priming of the ->prev_time value */ - jent_measure_jitter(ec); + jent_measure_jitter(ec, NULL); while (!jent_health_failure(ec)) { /* If a stuck measurement is received, repeat measurement */ - if (jent_measure_jitter(ec)) + if (jent_measure_jitter(ec, NULL)) continue; /* @@ -537,11 +609,12 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, return -1; while (len > 0) { - unsigned int tocopy; + unsigned int tocopy, health_test_result; jent_gen_entropy(ec); - if (jent_permanent_health_failure(ec)) { + health_test_result = jent_health_failure(ec); + if (health_test_result > JENT_PERMANENT_FAILURE_SHIFT) { /* * At this point, the Jitter RNG instance is considered * as a failed instance. There is no rerun of the @@ -549,13 +622,18 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, * is assumed to not further use this instance. */ return -3; - } else if (jent_health_failure(ec)) { + } else if (health_test_result) { /* * Perform startup health tests and return permanent * error if it fails. */ - if (jent_entropy_init(ec->hash_state)) + if (jent_entropy_init(0, 0, NULL, ec)) { + /* Mark the permanent error */ + ec->health_failure &= + JENT_RCT_FAILURE_PERMANENT | + JENT_APT_FAILURE_PERMANENT; return -3; + } return -2; } @@ -592,23 +670,29 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr, /* Allocate memory for adding variations based on memory * access */ - entropy_collector->mem = jent_zalloc(JENT_MEMORY_SIZE); + entropy_collector->mem = jent_kvzalloc(JENT_MEMORY_SIZE); if (!entropy_collector->mem) { jent_zfree(entropy_collector); return NULL; } - entropy_collector->memblocksize = JENT_MEMORY_BLOCKSIZE; - entropy_collector->memblocks = JENT_MEMORY_BLOCKS; + entropy_collector->memblocksize = + CONFIG_CRYPTO_JITTERENTROPY_MEMORY_BLOCKSIZE; + entropy_collector->memblocks = + CONFIG_CRYPTO_JITTERENTROPY_MEMORY_BLOCKS; entropy_collector->memaccessloops = JENT_MEMORY_ACCESSLOOPS; } /* verify and set the oversampling rate */ if (osr == 0) - osr = 1; /* minimum sampling rate is 1 */ + osr = 1; /* H_submitter = 1 / osr */ entropy_collector->osr = osr; + entropy_collector->flags = flags; entropy_collector->hash_state = hash_state; + /* Initialize the APT */ + jent_apt_init(entropy_collector, osr); + /* fill the data pad with non-zero values */ jent_gen_entropy(entropy_collector); @@ -617,25 +701,39 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr, void jent_entropy_collector_free(struct rand_data *entropy_collector) { - jent_zfree(entropy_collector->mem); + jent_kvzfree(entropy_collector->mem, JENT_MEMORY_SIZE); entropy_collector->mem = NULL; jent_zfree(entropy_collector); } -int jent_entropy_init(void *hash_state) +int jent_entropy_init(unsigned int osr, unsigned int flags, void *hash_state, + struct rand_data *p_ec) { - int i; - __u64 delta_sum = 0; - __u64 old_delta = 0; - unsigned int nonstuck = 0; - int time_backwards = 0; - int count_mod = 0; - int count_stuck = 0; - struct rand_data ec = { 0 }; - - /* Required for RCT */ - ec.osr = 1; - ec.hash_state = hash_state; + /* + * If caller provides an allocated ec, reuse it which implies that the + * health test entropy data is used to further still the available + * entropy pool. + */ + struct rand_data *ec = p_ec; + int i, time_backwards = 0, ret = 0, ec_free = 0; + unsigned int health_test_result; + + if (!ec) { + ec = jent_entropy_collector_alloc(osr, flags, hash_state); + if (!ec) + return JENT_EMEM; + ec_free = 1; + } else { + /* Reset the APT */ + jent_apt_reset(ec, 0); + /* Ensure that a new APT base is obtained */ + ec->apt_base_set = 0; + /* Reset the RCT */ + ec->rct_count = 0; + /* Reset intermittent, leave permanent health test result */ + ec->health_failure &= (~JENT_RCT_FAILURE); + ec->health_failure &= (~JENT_APT_FAILURE); + } /* We could perform statistical tests here, but the problem is * that we only have a few loop counts to do testing. These @@ -664,31 +762,28 @@ int jent_entropy_init(void *hash_state) #define TESTLOOPCOUNT 1024 #define CLEARCACHE 100 for (i = 0; (TESTLOOPCOUNT + CLEARCACHE) > i; i++) { - __u64 time = 0; - __u64 time2 = 0; - __u64 delta = 0; - unsigned int lowdelta = 0; - int stuck; + __u64 start_time = 0, end_time = 0, delta = 0; /* Invoke core entropy collection logic */ - jent_get_nstime(&time); - ec.prev_time = time; - jent_condition_data(&ec, time, 0); - jent_get_nstime(&time2); + jent_measure_jitter(ec, &delta); + end_time = ec->prev_time; + start_time = ec->prev_time - delta; /* test whether timer works */ - if (!time || !time2) - return JENT_ENOTIME; - delta = jent_delta(time, time2); + if (!start_time || !end_time) { + ret = JENT_ENOTIME; + goto out; + } + /* * test whether timer is fine grained enough to provide * delta even when called shortly after each other -- this * implies that we also have a high resolution timer */ - if (!delta) - return JENT_ECOARSETIME; - - stuck = jent_stuck(&ec, delta); + if (!delta || (end_time == start_time)) { + ret = JENT_ECOARSETIME; + goto out; + } /* * up to here we did not modify any variable that will be @@ -700,49 +795,9 @@ int jent_entropy_init(void *hash_state) if (i < CLEARCACHE) continue; - if (stuck) - count_stuck++; - else { - nonstuck++; - - /* - * Ensure that the APT succeeded. - * - * With the check below that count_stuck must be less - * than 10% of the overall generated raw entropy values - * it is guaranteed that the APT is invoked at - * floor((TESTLOOPCOUNT * 0.9) / 64) == 14 times. - */ - if ((nonstuck % JENT_APT_WINDOW_SIZE) == 0) { - jent_apt_reset(&ec, - delta & JENT_APT_WORD_MASK); - } - } - - /* Validate health test result */ - if (jent_health_failure(&ec)) - return JENT_EHEALTH; - /* test whether we have an increasing timer */ - if (!(time2 > time)) + if (!(end_time > start_time)) time_backwards++; - - /* use 32 bit value to ensure compilation on 32 bit arches */ - lowdelta = time2 - time; - if (!(lowdelta % 100)) - count_mod++; - - /* - * ensure that we have a varying delta timer which is necessary - * for the calculation of entropy -- perform this check - * only after the first loop is executed as we need to prime - * the old_data value - */ - if (delta > old_delta) - delta_sum += (delta - old_delta); - else - delta_sum += (old_delta - delta); - old_delta = delta; } /* @@ -752,31 +807,22 @@ int jent_entropy_init(void *hash_state) * should not fail. The value of 3 should cover the NTP case being * performed during our test run. */ - if (time_backwards > 3) - return JENT_ENOMONOTONIC; - - /* - * Variations of deltas of time must on average be larger - * than 1 to ensure the entropy estimation - * implied with 1 is preserved - */ - if ((delta_sum) <= 1) - return JENT_EVARVAR; + if (time_backwards > 3) { + ret = JENT_ENOMONOTONIC; + goto out; + } - /* - * Ensure that we have variations in the time stamp below 10 for at - * least 10% of all checks -- on some platforms, the counter increments - * in multiples of 100, but not always - */ - if ((TESTLOOPCOUNT/10 * 9) < count_mod) - return JENT_ECOARSETIME; + /* Did we encounter a health test failure? */ + health_test_result = jent_health_failure(ec); + if (health_test_result) { + ret = (health_test_result & JENT_RCT_FAILURE) ? JENT_ERCT : + JENT_EHEALTH; + goto out; + } - /* - * If we have more than 90% stuck results, then this Jitter RNG is - * likely to not work well. - */ - if ((TESTLOOPCOUNT/10 * 9) < count_stuck) - return JENT_ESTUCK; +out: + if (ec_free) + jent_entropy_collector_free(ec); - return 0; + return ret; } diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h index 4c92176ea2b1d..4c5dbf2a8d8f9 100644 --- a/crypto/jitterentropy.h +++ b/crypto/jitterentropy.h @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +extern void *jent_kvzalloc(unsigned int len); +extern void jent_kvzfree(void *ptr, unsigned int len); extern void *jent_zalloc(unsigned int len); extern void jent_zfree(void *ptr); extern void jent_get_nstime(__u64 *out); @@ -9,7 +11,8 @@ extern int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len); struct rand_data; -extern int jent_entropy_init(void *hash_state); +extern int jent_entropy_init(unsigned int osr, unsigned int flags, + void *hash_state, struct rand_data *p_ec); extern int jent_read_entropy(struct rand_data *ec, unsigned char *data, unsigned int len); @@ -19,11 +22,11 @@ extern struct rand_data *jent_entropy_collector_alloc(unsigned int osr, extern void jent_entropy_collector_free(struct rand_data *entropy_collector); #ifdef CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE -int jent_raw_hires_entropy_store(__u32 value); +int jent_raw_hires_entropy_store(__u64 value); void jent_testing_init(void); void jent_testing_exit(void); #else /* CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE */ -static inline int jent_raw_hires_entropy_store(__u32 value) { return 0; } +static inline int jent_raw_hires_entropy_store(__u64 value) { return 0; } static inline void jent_testing_init(void) { } static inline void jent_testing_exit(void) { } #endif /* CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE */ diff --git a/crypto/lrw.c b/crypto/lrw.c index 59260aefed280..5536ec7bf18f1 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -322,7 +322,7 @@ static int lrw_create(struct crypto_template *tmpl, struct rtattr **tb) err = crypto_grab_skcipher(spawn, skcipher_crypto_instance(inst), cipher_name, 0, mask); - if (err == -ENOENT) { + if (err == -ENOENT && memcmp(cipher_name, "ecb(", 4)) { err = -ENAMETOOLONG; if (snprintf(ecb_name, CRYPTO_MAX_ALG_NAME, "ecb(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) @@ -356,7 +356,7 @@ static int lrw_create(struct crypto_template *tmpl, struct rtattr **tb) /* Alas we screwed up the naming so we have to mangle the * cipher name. */ - if (!strncmp(cipher_name, "ecb(", 4)) { + if (!memcmp(cipher_name, "ecb(", 4)) { int len; len = strscpy(ecb_name, cipher_name + 4, sizeof(ecb_name)); diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index 0631d975bfac1..0abc2d87f0420 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -55,7 +55,7 @@ static int __lzorle_compress(const u8 *src, unsigned int slen, size_t tmp_len = *dlen; /* size_t(ulong) <-> uint on 64 bit */ int err; - err = lzorle1x_1_compress(src, slen, dst, &tmp_len, ctx); + err = lzorle1x_1_compress_safe(src, slen, dst, &tmp_len, ctx); if (err != LZO_E_OK) return -EINVAL; diff --git a/crypto/lzo.c b/crypto/lzo.c index ebda132dd22bf..8338851c7406a 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -55,7 +55,7 @@ static int __lzo_compress(const u8 *src, unsigned int slen, size_t tmp_len = *dlen; /* size_t(ulong) <-> uint on 64 bit */ int err; - err = lzo1x_1_compress(src, slen, dst, &tmp_len, ctx); + err = lzo1x_1_compress_safe(src, slen, dst, &tmp_len, ctx); if (err != LZO_E_OK) return -EINVAL; diff --git a/crypto/rng.c b/crypto/rng.c index 279dffdebf598..e462031fef68c 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -197,6 +197,11 @@ int crypto_del_default_rng(void) EXPORT_SYMBOL_GPL(crypto_del_default_rng); #endif +static void rng_default_set_ent(struct crypto_rng *tfm, const u8 *data, + unsigned int len) +{ +} + int crypto_register_rng(struct rng_alg *alg) { struct crypto_istat_rng *istat = rng_get_stat(alg); @@ -212,6 +217,9 @@ int crypto_register_rng(struct rng_alg *alg) if (IS_ENABLED(CONFIG_CRYPTO_STATS)) memset(istat, 0, sizeof(*istat)); + if (!alg->set_ent) + alg->set_ent = rng_default_set_ent; + return crypto_register_alg(base); } EXPORT_SYMBOL_GPL(crypto_register_rng); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 7b275716cf4e3..acc879ed6031a 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -811,6 +811,7 @@ struct crypto_sync_skcipher *crypto_alloc_sync_skcipher( /* Only sync algorithms allowed. */ mask |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE; + type &= ~(CRYPTO_ALG_ASYNC | CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE); tfm = crypto_alloc_tfm(alg_name, &crypto_skcipher_type, type, mask); diff --git a/crypto/xts.c b/crypto/xts.c index 038f60dd512d9..97fd0fb8757c2 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -363,7 +363,7 @@ static int xts_create(struct crypto_template *tmpl, struct rtattr **tb) err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), cipher_name, 0, mask); - if (err == -ENOENT) { + if (err == -ENOENT && memcmp(cipher_name, "ecb(", 4)) { err = -ENAMETOOLONG; if (snprintf(name, CRYPTO_MAX_ALG_NAME, "ecb(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) @@ -397,7 +397,7 @@ static int xts_create(struct crypto_template *tmpl, struct rtattr **tb) /* Alas we screwed up the naming so we have to mangle the * cipher name. */ - if (!strncmp(cipher_name, "ecb(", 4)) { + if (!memcmp(cipher_name, "ecb(", 4)) { int len; len = strscpy(name, cipher_name + 4, sizeof(name)); diff --git a/drivers/accel/habanalabs/common/memory.c b/drivers/accel/habanalabs/common/memory.c index 5b7d9a351133f..33a4246dd7358 100644 --- a/drivers/accel/habanalabs/common/memory.c +++ b/drivers/accel/habanalabs/common/memory.c @@ -2323,7 +2323,7 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size, if (rc < 0) goto destroy_pages; npages = rc; - rc = -EFAULT; + rc = -ENOMEM; goto put_pages; } userptr->npages = npages; diff --git a/drivers/accel/habanalabs/gaudi/gaudi.c b/drivers/accel/habanalabs/gaudi/gaudi.c index 056e2ef44afb5..ceed6cfe2f919 100644 --- a/drivers/accel/habanalabs/gaudi/gaudi.c +++ b/drivers/accel/habanalabs/gaudi/gaudi.c @@ -4173,10 +4173,29 @@ static int gaudi_mmap(struct hl_device *hdev, struct vm_area_struct *vma, vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_NORESERVE); +#ifdef _HAS_DMA_MMAP_COHERENT + /* + * If dma_alloc_coherent() returns a vmalloc address, set VM_MIXEDMAP + * so vm_insert_page() can handle it safely. Without this, the kernel + * may BUG_ON due to VM_PFNMAP. + */ + if (is_vmalloc_addr(cpu_addr)) + vm_flags_set(vma, VM_MIXEDMAP); + rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, (dma_addr - HOST_PHYS_BASE), size); if (rc) dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); +#else + + rc = remap_pfn_range(vma, vma->vm_start, + virt_to_phys(cpu_addr) >> PAGE_SHIFT, + size, vma->vm_page_prot); + if (rc) + dev_err(hdev->dev, "remap_pfn_range error %d", rc); + + #endif + return rc; } diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2.c b/drivers/accel/habanalabs/gaudi2/gaudi2.c index 31c74ca70a2e5..44b5678ea615c 100644 --- a/drivers/accel/habanalabs/gaudi2/gaudi2.c +++ b/drivers/accel/habanalabs/gaudi2/gaudi2.c @@ -2985,7 +2985,6 @@ static int gaudi2_early_init(struct hl_device *hdev) rc = hl_fw_read_preboot_status(hdev); if (rc) { if (hdev->reset_on_preboot_fail) - /* we are already on failure flow, so don't check if hw_fini fails. */ hdev->asic_funcs->hw_fini(hdev, true, false); goto pci_fini; } @@ -2997,6 +2996,13 @@ static int gaudi2_early_init(struct hl_device *hdev) dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); goto pci_fini; } + + rc = hl_fw_read_preboot_status(hdev); + if (rc) { + if (hdev->reset_on_preboot_fail) + hdev->asic_funcs->hw_fini(hdev, true, false); + goto pci_fini; + } } return 0; @@ -6339,6 +6345,13 @@ static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, VM_DONTCOPY | VM_NORESERVE); #ifdef _HAS_DMA_MMAP_COHERENT + /* + * If dma_alloc_coherent() returns a vmalloc address, set VM_MIXEDMAP + * so vm_insert_page() can handle it safely. Without this, the kernel + * may BUG_ON due to VM_PFNMAP. + */ + if (is_vmalloc_addr(cpu_addr)) + vm_flags_set(vma, VM_MIXEDMAP); rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); if (rc) diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c b/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c index 25b5368f37dde..9ff00b65ae34b 100644 --- a/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c +++ b/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c @@ -2409,7 +2409,7 @@ static int gaudi2_config_bmon(struct hl_device *hdev, struct hl_debug_params *pa WREG32(base_reg + mmBMON_ADDRH_E3_OFFSET, 0); WREG32(base_reg + mmBMON_REDUCTION_OFFSET, 0); WREG32(base_reg + mmBMON_STM_TRC_OFFSET, 0x7 | (0xA << 8)); - WREG32(base_reg + mmBMON_CR_OFFSET, 0x77 | 0xf << 24); + WREG32(base_reg + mmBMON_CR_OFFSET, 0x41); } return 0; diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c index f3db3fa91dd52..08b78f5678532 100644 --- a/drivers/accel/qaic/qaic_control.c +++ b/drivers/accel/qaic/qaic_control.c @@ -407,7 +407,7 @@ static int find_and_map_user_pages(struct qaic_device *qdev, return -EINVAL; remaining = in_trans->size - resources->xferred_dma_size; if (remaining == 0) - return 0; + return -EINVAL; if (check_add_overflow(xfer_start_addr, remaining, &end)) return -EINVAL; diff --git a/drivers/accel/qaic/qaic_drv.c b/drivers/accel/qaic/qaic_drv.c index b5de82e6eb4d5..e69bfb30b44e0 100644 --- a/drivers/accel/qaic/qaic_drv.c +++ b/drivers/accel/qaic/qaic_drv.c @@ -400,7 +400,7 @@ static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev) int bars; int ret; - bars = pci_select_bars(pdev, IORESOURCE_MEM); + bars = pci_select_bars(pdev, IORESOURCE_MEM) & 0x3f; /* make sure the device has the expected BARs */ if (bars != (BIT(0) | BIT(2) | BIT(4))) { diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index cee82b473dc50..648228831f5e8 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -438,7 +438,7 @@ config ACPI_SBS the modules will be called sbs and sbshc. config ACPI_HED - tristate "Hardware Error Device" + bool "Hardware Error Device" help This driver supports the Hardware Error Device (PNP0C33), which is used to report some hardware errors notified via diff --git a/drivers/acpi/acpi_dbg.c b/drivers/acpi/acpi_dbg.c index d50261d05f3a1..515b20d0b698a 100644 --- a/drivers/acpi/acpi_dbg.c +++ b/drivers/acpi/acpi_dbg.c @@ -569,11 +569,11 @@ static int acpi_aml_release(struct inode *inode, struct file *file) return 0; } -static int acpi_aml_read_user(char __user *buf, int len) +static ssize_t acpi_aml_read_user(char __user *buf, size_t len) { - int ret; struct circ_buf *crc = &acpi_aml_io.out_crc; - int n; + ssize_t ret; + size_t n; char *p; ret = acpi_aml_lock_read(crc, ACPI_AML_OUT_USER); @@ -582,7 +582,7 @@ static int acpi_aml_read_user(char __user *buf, int len) /* sync head before removing logs */ smp_rmb(); p = &crc->buf[crc->tail]; - n = min(len, circ_count_to_end(crc)); + n = min_t(size_t, len, circ_count_to_end(crc)); if (copy_to_user(buf, p, n)) { ret = -EFAULT; goto out; @@ -599,8 +599,8 @@ static int acpi_aml_read_user(char __user *buf, int len) static ssize_t acpi_aml_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - int ret = 0; - int size = 0; + ssize_t ret = 0; + ssize_t size = 0; if (!count) return 0; @@ -639,11 +639,11 @@ static ssize_t acpi_aml_read(struct file *file, char __user *buf, return size > 0 ? size : ret; } -static int acpi_aml_write_user(const char __user *buf, int len) +static ssize_t acpi_aml_write_user(const char __user *buf, size_t len) { - int ret; struct circ_buf *crc = &acpi_aml_io.in_crc; - int n; + ssize_t ret; + size_t n; char *p; ret = acpi_aml_lock_write(crc, ACPI_AML_IN_USER); @@ -652,7 +652,7 @@ static int acpi_aml_write_user(const char __user *buf, int len) /* sync tail before inserting cmds */ smp_mb(); p = &crc->buf[crc->head]; - n = min(len, circ_space_to_end(crc)); + n = min_t(size_t, len, circ_space_to_end(crc)); if (copy_from_user(p, buf, n)) { ret = -EFAULT; goto out; @@ -663,14 +663,14 @@ static int acpi_aml_write_user(const char __user *buf, int len) ret = n; out: acpi_aml_unlock_fifo(ACPI_AML_IN_USER, ret >= 0); - return n; + return ret; } static ssize_t acpi_aml_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - int ret = 0; - int size = 0; + ssize_t ret = 0; + ssize_t size = 0; if (!count) return 0; diff --git a/drivers/acpi/acpi_pnp.c b/drivers/acpi/acpi_pnp.c index 01abf26764b00..3f5a1840f5733 100644 --- a/drivers/acpi/acpi_pnp.c +++ b/drivers/acpi/acpi_pnp.c @@ -355,8 +355,10 @@ static bool acpi_pnp_match(const char *idstr, const struct acpi_device_id **matc * device represented by it. */ static const struct acpi_device_id acpi_nonpnp_device_ids[] = { + {"INT3F0D"}, {"INTC1080"}, {"INTC1081"}, + {"INTC1099"}, {""}, }; diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 7053f1b9fc1dd..c0f9cf9768ea9 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -250,7 +250,7 @@ static inline int acpi_processor_hotadd_init(struct acpi_processor *pr) static int acpi_processor_get_info(struct acpi_device *device) { - union acpi_object object = { 0 }; + union acpi_object object = { .processor = { 0 } }; struct acpi_buffer buffer = { sizeof(union acpi_object), &object }; struct acpi_processor *pr = acpi_driver_data(device); int device_declaration = 0; diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c index 33c3b16af556b..ecba82ac7cd5f 100644 --- a/drivers/acpi/acpi_tad.c +++ b/drivers/acpi/acpi_tad.c @@ -564,6 +564,9 @@ static int acpi_tad_remove(struct platform_device *pdev) pm_runtime_get_sync(dev); + if (dd->capabilities & ACPI_TAD_RT) + sysfs_remove_group(&dev->kobj, &acpi_tad_time_attr_group); + if (dd->capabilities & ACPI_TAD_DC_WAKE) sysfs_remove_group(&dev->kobj, &acpi_tad_dc_attr_group); diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index a971770e24ff9..841db33e94756 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -1952,8 +1952,10 @@ static void acpi_video_bus_remove_notify_handler(struct acpi_video_bus *video) struct acpi_video_device *dev; mutex_lock(&video->device_list_lock); - list_for_each_entry(dev, &video->video_device_list, entry) + list_for_each_entry(dev, &video->video_device_list, entry) { acpi_video_dev_remove_notify_handler(dev); + cancel_delayed_work_sync(&dev->switch_brightness_work); + } mutex_unlock(&video->device_list_lock); acpi_video_bus_stop_devices(video); diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h index 82563b44af351..261dc8f87556d 100644 --- a/drivers/acpi/acpica/aclocal.h +++ b/drivers/acpi/acpica/aclocal.h @@ -1139,7 +1139,7 @@ struct acpi_port_info { #define ACPI_RESOURCE_NAME_PIN_GROUP_FUNCTION 0x91 #define ACPI_RESOURCE_NAME_PIN_GROUP_CONFIG 0x92 #define ACPI_RESOURCE_NAME_CLOCK_INPUT 0x93 -#define ACPI_RESOURCE_NAME_LARGE_MAX 0x94 +#define ACPI_RESOURCE_NAME_LARGE_MAX 0x93 /***************************************************************************** * diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c index e809c2aed78ae..6168597a96e6f 100644 --- a/drivers/acpi/acpica/dsmethod.c +++ b/drivers/acpi/acpica/dsmethod.c @@ -462,7 +462,6 @@ acpi_ds_call_control_method(struct acpi_thread_state *thread, struct acpi_walk_state *next_walk_state = NULL; union acpi_operand_object *obj_desc; struct acpi_evaluate_info *info; - u32 i; ACPI_FUNCTION_TRACE_PTR(ds_call_control_method, this_walk_state); @@ -483,6 +482,13 @@ acpi_ds_call_control_method(struct acpi_thread_state *thread, return_ACPI_STATUS(AE_NULL_OBJECT); } + if (this_walk_state->num_operands < obj_desc->method.param_count) { + ACPI_ERROR((AE_INFO, "Missing argument for method [%4.4s]", + acpi_ut_get_node_name(method_node))); + + return_ACPI_STATUS(AE_AML_UNINITIALIZED_ARG); + } + /* Init for new method, possibly wait on method mutex */ status = @@ -539,14 +545,7 @@ acpi_ds_call_control_method(struct acpi_thread_state *thread, * Delete the operands on the previous walkstate operand stack * (they were copied to new objects) */ - for (i = 0; i < obj_desc->method.param_count; i++) { - acpi_ut_remove_reference(this_walk_state->operands[i]); - this_walk_state->operands[i] = NULL; - } - - /* Clear the operand stack */ - - this_walk_state->num_operands = 0; + acpi_ds_clear_operands(this_walk_state); ACPI_DEBUG_PRINT((ACPI_DB_DISPATCH, "**** Begin nested execution of [%4.4s] **** WalkState=%p\n", diff --git a/drivers/acpi/acpica/dsutils.c b/drivers/acpi/acpica/dsutils.c index fb9ed5e1da89d..2bdae8a25e084 100644 --- a/drivers/acpi/acpica/dsutils.c +++ b/drivers/acpi/acpica/dsutils.c @@ -668,6 +668,8 @@ acpi_ds_create_operands(struct acpi_walk_state *walk_state, union acpi_parse_object *arguments[ACPI_OBJ_NUM_OPERANDS]; u32 arg_count = 0; u32 index = walk_state->num_operands; + u32 prev_num_operands = walk_state->num_operands; + u32 new_num_operands; u32 i; ACPI_FUNCTION_TRACE_PTR(ds_create_operands, first_arg); @@ -696,6 +698,7 @@ acpi_ds_create_operands(struct acpi_walk_state *walk_state, /* Create the interpreter arguments, in reverse order */ + new_num_operands = index; index--; for (i = 0; i < arg_count; i++) { arg = arguments[index]; @@ -720,7 +723,11 @@ acpi_ds_create_operands(struct acpi_walk_state *walk_state, * pop everything off of the operand stack and delete those * objects */ - acpi_ds_obj_stack_pop_and_delete(arg_count, walk_state); + walk_state->num_operands = i; + acpi_ds_obj_stack_pop_and_delete(new_num_operands, walk_state); + + /* Restore operand count */ + walk_state->num_operands = prev_num_operands; ACPI_EXCEPTION((AE_INFO, status, "While creating Arg %u", index)); return_ACPI_STATUS(status); diff --git a/drivers/acpi/acpica/evglock.c b/drivers/acpi/acpica/evglock.c index 989dc01af03fb..bc205b3309043 100644 --- a/drivers/acpi/acpica/evglock.c +++ b/drivers/acpi/acpica/evglock.c @@ -42,6 +42,10 @@ acpi_status acpi_ev_init_global_lock_handler(void) return_ACPI_STATUS(AE_OK); } + if (!acpi_gbl_use_global_lock) { + return_ACPI_STATUS(AE_OK); + } + /* Attempt installation of the global lock handler */ status = acpi_install_fixed_event_handler(ACPI_EVENT_GLOBAL, diff --git a/drivers/acpi/acpica/exserial.c b/drivers/acpi/acpica/exserial.c index 5241f4c01c765..89a4ac447a2be 100644 --- a/drivers/acpi/acpica/exserial.c +++ b/drivers/acpi/acpica/exserial.c @@ -201,6 +201,12 @@ acpi_ex_read_serial_bus(union acpi_operand_object *obj_desc, function = ACPI_READ; break; + case ACPI_ADR_SPACE_FIXED_HARDWARE: + + buffer_length = ACPI_FFH_INPUT_BUFFER_SIZE; + function = ACPI_READ; + break; + default: return_ACPI_STATUS(AE_AML_INVALID_SPACE_ID); } diff --git a/drivers/acpi/acpica/psobject.c b/drivers/acpi/acpica/psobject.c index 54471083ba545..0bce1baaa62b3 100644 --- a/drivers/acpi/acpica/psobject.c +++ b/drivers/acpi/acpica/psobject.c @@ -636,7 +636,8 @@ acpi_status acpi_ps_complete_final_op(struct acpi_walk_state *walk_state, union acpi_parse_object *op, acpi_status status) { - acpi_status status2; + acpi_status return_status = status; + u8 ascending = TRUE; ACPI_FUNCTION_TRACE_PTR(ps_complete_final_op, walk_state); @@ -650,7 +651,7 @@ acpi_ps_complete_final_op(struct acpi_walk_state *walk_state, op)); do { if (op) { - if (walk_state->ascending_callback != NULL) { + if (ascending && walk_state->ascending_callback != NULL) { walk_state->op = op; walk_state->op_info = acpi_ps_get_opcode_info(op->common. @@ -672,49 +673,26 @@ acpi_ps_complete_final_op(struct acpi_walk_state *walk_state, } if (status == AE_CTRL_TERMINATE) { - status = AE_OK; - - /* Clean up */ - do { - if (op) { - status2 = - acpi_ps_complete_this_op - (walk_state, op); - if (ACPI_FAILURE - (status2)) { - return_ACPI_STATUS - (status2); - } - } - - acpi_ps_pop_scope(& - (walk_state-> - parser_state), - &op, - &walk_state-> - arg_types, - &walk_state-> - arg_count); - - } while (op); - - return_ACPI_STATUS(status); + ascending = FALSE; + return_status = AE_CTRL_TERMINATE; } else if (ACPI_FAILURE(status)) { /* First error is most important */ - (void) - acpi_ps_complete_this_op(walk_state, - op); - return_ACPI_STATUS(status); + ascending = FALSE; + return_status = status; } } - status2 = acpi_ps_complete_this_op(walk_state, op); - if (ACPI_FAILURE(status2)) { - return_ACPI_STATUS(status2); + status = acpi_ps_complete_this_op(walk_state, op); + if (ACPI_FAILURE(status)) { + ascending = FALSE; + if (ACPI_SUCCESS(return_status) || + return_status == AE_CTRL_TERMINATE) { + return_status = status; + } } } @@ -724,5 +702,5 @@ acpi_ps_complete_final_op(struct acpi_walk_state *walk_state, } while (op); - return_ACPI_STATUS(status); + return_ACPI_STATUS(return_status); } diff --git a/drivers/acpi/acpica/tbprint.c b/drivers/acpi/acpica/tbprint.c index 58b02e4b254b8..544964c9c530a 100644 --- a/drivers/acpi/acpica/tbprint.c +++ b/drivers/acpi/acpica/tbprint.c @@ -95,6 +95,11 @@ acpi_tb_print_table_header(acpi_physical_address address, { struct acpi_table_header local_header; +#pragma GCC diagnostic push +#if defined(__GNUC__) && __GNUC__ >= 11 +#pragma GCC diagnostic ignored "-Wstringop-overread" +#endif + if (ACPI_COMPARE_NAMESEG(header->signature, ACPI_SIG_FACS)) { /* FACS only has signature and length fields */ @@ -135,4 +140,5 @@ acpi_tb_print_table_header(acpi_physical_address address, local_header.asl_compiler_id, local_header.asl_compiler_revision)); } +#pragma GCC diagnostic pop } diff --git a/drivers/acpi/acpica/utprint.c b/drivers/acpi/acpica/utprint.c index 42b30b9f93128..7fad03c5252c3 100644 --- a/drivers/acpi/acpica/utprint.c +++ b/drivers/acpi/acpica/utprint.c @@ -333,11 +333,8 @@ int vsnprintf(char *string, acpi_size size, const char *format, va_list args) pos = string; - if (size != ACPI_UINT32_MAX) { - end = string + size; - } else { - end = ACPI_CAST_PTR(char, ACPI_UINT32_MAX); - } + size = ACPI_MIN(size, ACPI_PTR_DIFF(ACPI_MAX_PTR, string)); + end = string + size; for (; *format; ++format) { if (*format != '%') { diff --git a/drivers/acpi/apei/Kconfig b/drivers/acpi/apei/Kconfig index 6b18f8bc7be35..71e0d64a7792e 100644 --- a/drivers/acpi/apei/Kconfig +++ b/drivers/acpi/apei/Kconfig @@ -23,6 +23,7 @@ config ACPI_APEI_GHES select ACPI_HED select IRQ_WORK select GENERIC_ALLOCATOR + select ARM_SDE_INTERFACE if ARM64 help Generic Hardware Error Source provides a way to report platform hardware errors (such as that from chipset). It diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 3aadc632d7dd5..ec364c2541124 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -715,6 +715,17 @@ static bool ghes_do_proc(struct ghes *ghes, } } + /* + * If no memory failure work is queued for abnormal synchronous + * errors, do a force kill. + */ + if (sync && !queued) { + dev_err(ghes->dev, + HW_ERR GHES_PFX "%s:%d: synchronous unrecoverable error (SIGBUS)\n", + current->comm, task_pid_nr(current)); + force_sig(SIGBUS); + } + return queued; } @@ -901,6 +912,8 @@ static void __ghes_panic(struct ghes *ghes, __ghes_print_estatus(KERN_EMERG, ghes->generic, estatus); + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); + ghes_clear_estatus(ghes, estatus, buf_paddr, fixmap_idx); if (!panic_timeout) @@ -1523,7 +1536,7 @@ void __init acpi_ghes_init(void) { int rc; - sdei_init(); + acpi_sdei_init(); if (acpi_disabled) return; diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 1a31106a14e44..6ee8382a32302 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -937,8 +937,10 @@ static u32 *iort_rmr_alloc_sids(u32 *sids, u32 count, u32 id_start, new_sids = krealloc_array(sids, count + new_count, sizeof(*new_sids), GFP_KERNEL); - if (!new_sids) + if (!new_sids) { + kfree(sids); return NULL; + } for (i = count; i < total_count; i++) new_sids[i] = id_start++; diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index e3cbaf3c3bbc1..a70b6db3bf059 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -94,7 +94,7 @@ enum { struct acpi_battery { struct mutex lock; - struct mutex sysfs_lock; + struct mutex update_lock; struct power_supply *bat; struct power_supply_desc bat_desc; struct acpi_device *device; @@ -888,15 +888,12 @@ static int sysfs_add_battery(struct acpi_battery *battery) static void sysfs_remove_battery(struct acpi_battery *battery) { - mutex_lock(&battery->sysfs_lock); - if (!battery->bat) { - mutex_unlock(&battery->sysfs_lock); + if (!battery->bat) return; - } + battery_hook_remove_battery(battery); power_supply_unregister(battery->bat); battery->bat = NULL; - mutex_unlock(&battery->sysfs_lock); } static void find_battery(const struct dmi_header *dm, void *private) @@ -1056,6 +1053,9 @@ static void acpi_battery_notify(acpi_handle handle, u32 event, void *data) if (!battery) return; + + guard(mutex)(&battery->update_lock); + old = battery->bat; /* * On Acer Aspire V5-573G notifications are sometimes triggered too @@ -1078,21 +1078,22 @@ static void acpi_battery_notify(acpi_handle handle, u32 event, void *data) } static int battery_notify(struct notifier_block *nb, - unsigned long mode, void *_unused) + unsigned long mode, void *_unused) { struct acpi_battery *battery = container_of(nb, struct acpi_battery, pm_nb); - int result; - switch (mode) { - case PM_POST_HIBERNATION: - case PM_POST_SUSPEND: + if (mode == PM_POST_SUSPEND || mode == PM_POST_HIBERNATION) { + guard(mutex)(&battery->update_lock); + if (!acpi_battery_present(battery)) return 0; if (battery->bat) { acpi_battery_refresh(battery); } else { + int result; + result = acpi_battery_get_info(battery); if (result) return result; @@ -1104,7 +1105,6 @@ static int battery_notify(struct notifier_block *nb, acpi_battery_init_alarm(battery); acpi_battery_get_state(battery); - break; } return 0; @@ -1182,6 +1182,8 @@ static int acpi_battery_update_retry(struct acpi_battery *battery) { int retry, ret; + guard(mutex)(&battery->update_lock); + for (retry = 5; retry; retry--) { ret = acpi_battery_update(battery, false); if (!ret) @@ -1192,6 +1194,13 @@ static int acpi_battery_update_retry(struct acpi_battery *battery) return ret; } +static void sysfs_battery_cleanup(struct acpi_battery *battery) +{ + guard(mutex)(&battery->update_lock); + + sysfs_remove_battery(battery); +} + static int acpi_battery_add(struct acpi_device *device) { int result = 0; @@ -1203,15 +1212,21 @@ static int acpi_battery_add(struct acpi_device *device) if (device->dep_unmet) return -EPROBE_DEFER; - battery = kzalloc(sizeof(struct acpi_battery), GFP_KERNEL); + battery = devm_kzalloc(&device->dev, sizeof(*battery), GFP_KERNEL); if (!battery) return -ENOMEM; battery->device = device; strcpy(acpi_device_name(device), ACPI_BATTERY_DEVICE_NAME); strcpy(acpi_device_class(device), ACPI_BATTERY_CLASS); device->driver_data = battery; - mutex_init(&battery->lock); - mutex_init(&battery->sysfs_lock); + result = devm_mutex_init(&device->dev, &battery->lock); + if (result) + return result; + + result = devm_mutex_init(&device->dev, &battery->update_lock); + if (result) + return result; + if (acpi_has_method(battery->device->handle, "_BIX")) set_bit(ACPI_BATTERY_XINFO_PRESENT, &battery->flags); @@ -1238,10 +1253,7 @@ static int acpi_battery_add(struct acpi_device *device) device_init_wakeup(&device->dev, 0); unregister_pm_notifier(&battery->pm_nb); fail: - sysfs_remove_battery(battery); - mutex_destroy(&battery->lock); - mutex_destroy(&battery->sysfs_lock); - kfree(battery); + sysfs_battery_cleanup(battery); return result; } @@ -1260,11 +1272,10 @@ static void acpi_battery_remove(struct acpi_device *device) device_init_wakeup(&device->dev, 0); unregister_pm_notifier(&battery->pm_nb); - sysfs_remove_battery(battery); - mutex_destroy(&battery->lock); - mutex_destroy(&battery->sysfs_lock); - kfree(battery); + guard(mutex)(&battery->update_lock); + + sysfs_remove_battery(battery); } #ifdef CONFIG_PM_SLEEP @@ -1281,6 +1292,9 @@ static int acpi_battery_resume(struct device *dev) return -EINVAL; battery->update_time = 0; + + guard(mutex)(&battery->update_lock); + acpi_battery_update(battery, true); return 0; } diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index a4aa53b7e2bb3..645464f02363f 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -1396,8 +1396,10 @@ static int __init acpi_init(void) } acpi_kobj = kobject_create_and_add("acpi", firmware_kobj); - if (!acpi_kobj) - pr_debug("%s: kset create error\n", __func__); + if (!acpi_kobj) { + pr_err("Failed to register kobject\n"); + return -ENOMEM; + } init_prmt(); acpi_init_pcc(); diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c index 906a7bfa448b3..c760e38df981f 100644 --- a/drivers/acpi/button.c +++ b/drivers/acpi/button.c @@ -603,8 +603,10 @@ static int acpi_button_add(struct acpi_device *device) input_set_drvdata(input, device); error = input_register_device(input); - if (error) + if (error) { + input_free_device(input); goto err_remove_fs; + } switch (device->device_type) { case ACPI_BUS_TYPE_POWER_BUTTON: diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index ed02a2a9970aa..888c7838579a8 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -445,7 +445,7 @@ bool acpi_cpc_valid(void) if (acpi_disabled) return false; - for_each_present_cpu(cpu) { + for_each_online_cpu(cpu) { cpc_ptr = per_cpu(cpc_desc_ptr, cpu); if (!cpc_ptr) return false; @@ -461,7 +461,7 @@ bool cppc_allow_fast_switch(void) struct cpc_desc *cpc_ptr; int cpu; - for_each_possible_cpu(cpu) { + for_each_online_cpu(cpu) { cpc_ptr = per_cpu(cpc_desc_ptr, cpu); desired_reg = &cpc_ptr->cpc_regs[DESIRED_PERF]; if (!CPC_IN_SYSTEM_MEMORY(desired_reg) && @@ -1368,7 +1368,7 @@ bool cppc_perf_ctrs_in_pcc(void) { int cpu; - for_each_present_cpu(cpu) { + for_each_online_cpu(cpu) { struct cpc_register_resource *ref_perf_reg; struct cpc_desc *cpc_desc; diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 77d6af6115893..8e304efde3429 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -2329,6 +2329,12 @@ static const struct dmi_system_id acpi_ec_no_wakeup[] = { DMI_MATCH(DMI_PRODUCT_NAME, "83Q3"), } }, + { + // TUXEDO InfinityBook Pro AMD Gen9 + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GXxHRXx"), + }, + }, { }, }; diff --git a/drivers/acpi/hed.c b/drivers/acpi/hed.c index 46c6f8c35b436..2e01eaa8d8cd5 100644 --- a/drivers/acpi/hed.c +++ b/drivers/acpi/hed.c @@ -80,7 +80,12 @@ static struct acpi_driver acpi_hed_driver = { .remove = acpi_hed_remove, }, }; -module_acpi_driver(acpi_hed_driver); + +static int __init acpi_hed_driver_init(void) +{ + return acpi_bus_register_driver(&acpi_hed_driver); +} +subsys_initcall(acpi_hed_driver_init); MODULE_AUTHOR("Huang Ying"); MODULE_DESCRIPTION("ACPI Hardware Error Device Driver"); diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index a466ad6e5d93a..5a1ced5bf7f6f 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -2643,7 +2643,7 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, if (ndr_desc->target_node == NUMA_NO_NODE) { ndr_desc->target_node = phys_to_target_node(spa->address); dev_info(acpi_desc->dev, "changing target node from %d to %d for nfit region [%pa-%pa]", - NUMA_NO_NODE, ndr_desc->numa_node, &res.start, &res.end); + NUMA_NO_NODE, ndr_desc->target_node, &res.start, &res.end); } /* diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index bba268ecd802f..1d6450c139225 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -24,6 +24,7 @@ #include #include #include +#include static u8 hmat_revision; static int hmat_disable __initdata; @@ -57,14 +58,20 @@ struct target_cache { struct node_cache_attrs cache_attrs; }; +enum { + NODE_ACCESS_CLASS_GENPORT_SINK = ACCESS_COORDINATE_MAX, + NODE_ACCESS_CLASS_MAX, +}; + struct memory_target { struct list_head node; unsigned int memory_pxm; unsigned int processor_pxm; struct resource memregions; - struct node_hmem_attrs hmem_attrs[2]; + struct access_coordinate coord[NODE_ACCESS_CLASS_MAX]; struct list_head caches; struct node_cache_attrs cache_attrs; + u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; bool registered; }; @@ -119,8 +126,7 @@ static __init void alloc_memory_initiator(unsigned int cpu_pxm) list_add_tail(&initiator->node, &initiators); } -static __init void alloc_memory_target(unsigned int mem_pxm, - resource_size_t start, resource_size_t len) +static __init struct memory_target *alloc_target(unsigned int mem_pxm) { struct memory_target *target; @@ -128,7 +134,7 @@ static __init void alloc_memory_target(unsigned int mem_pxm, if (!target) { target = kzalloc(sizeof(*target), GFP_KERNEL); if (!target) - return; + return NULL; target->memory_pxm = mem_pxm; target->processor_pxm = PXM_INVAL; target->memregions = (struct resource) { @@ -141,6 +147,19 @@ static __init void alloc_memory_target(unsigned int mem_pxm, INIT_LIST_HEAD(&target->caches); } + return target; +} + +static __init void alloc_memory_target(unsigned int mem_pxm, + resource_size_t start, + resource_size_t len) +{ + struct memory_target *target; + + target = alloc_target(mem_pxm); + if (!target) + return; + /* * There are potentially multiple ranges per PXM, so record each * in the per-target memregions resource tree. @@ -151,6 +170,18 @@ static __init void alloc_memory_target(unsigned int mem_pxm, start, start + len, mem_pxm); } +static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle) +{ + struct memory_target *target; + + target = alloc_target(mem_pxm); + if (!target) + return; + + memcpy(target->gen_port_device_handle, handle, + ACPI_SRAT_DEVICE_HANDLE_SIZE); +} + static __init const char *hmat_data_type(u8 type) { switch (type) { @@ -227,24 +258,24 @@ static void hmat_update_target_access(struct memory_target *target, { switch (type) { case ACPI_HMAT_ACCESS_LATENCY: - target->hmem_attrs[access].read_latency = value; - target->hmem_attrs[access].write_latency = value; + target->coord[access].read_latency = value; + target->coord[access].write_latency = value; break; case ACPI_HMAT_READ_LATENCY: - target->hmem_attrs[access].read_latency = value; + target->coord[access].read_latency = value; break; case ACPI_HMAT_WRITE_LATENCY: - target->hmem_attrs[access].write_latency = value; + target->coord[access].write_latency = value; break; case ACPI_HMAT_ACCESS_BANDWIDTH: - target->hmem_attrs[access].read_bandwidth = value; - target->hmem_attrs[access].write_bandwidth = value; + target->coord[access].read_bandwidth = value; + target->coord[access].write_bandwidth = value; break; case ACPI_HMAT_READ_BANDWIDTH: - target->hmem_attrs[access].read_bandwidth = value; + target->coord[access].read_bandwidth = value; break; case ACPI_HMAT_WRITE_BANDWIDTH: - target->hmem_attrs[access].write_bandwidth = value; + target->coord[access].write_bandwidth = value; break; default: break; @@ -290,11 +321,28 @@ static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) } } +static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm, + u8 mem_hier, u8 type, u32 value) +{ + struct memory_target *target = find_mem_target(tgt_pxm); + + if (mem_hier != ACPI_HMAT_MEMORY) + return; + + if (target && target->processor_pxm == init_pxm) { + hmat_update_target_access(target, type, value, + ACCESS_COORDINATE_LOCAL); + /* If the node has a CPU, update access 1 */ + if (node_state(pxm_to_node(init_pxm), N_CPU)) + hmat_update_target_access(target, type, value, + ACCESS_COORDINATE_CPU); + } +} + static __init int hmat_parse_locality(union acpi_subtable_headers *header, const unsigned long end) { struct acpi_hmat_locality *hmat_loc = (void *)header; - struct memory_target *target; unsigned int init, targ, total_size, ipds, tpds; u32 *inits, *targs, value; u16 *entries; @@ -335,15 +383,8 @@ static __init int hmat_parse_locality(union acpi_subtable_headers *header, inits[init], targs[targ], value, hmat_data_type_suffix(type)); - if (mem_hier == ACPI_HMAT_MEMORY) { - target = find_mem_target(targs[targ]); - if (target && target->processor_pxm == inits[init]) { - hmat_update_target_access(target, type, value, 0); - /* If the node has a CPU, update access 1 */ - if (node_state(pxm_to_node(inits[init]), N_CPU)) - hmat_update_target_access(target, type, value, 1); - } - } + hmat_update_target(targs[targ], inits[init], + mem_hier, type, value); } } @@ -490,6 +531,27 @@ static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header, return 0; } +static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_srat_generic_affinity *ga = (void *)header; + + if (!ga) + return -EINVAL; + + if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED)) + return 0; + + /* Skip PCI device_handle for now */ + if (ga->device_handle_type != 0) + return 0; + + alloc_genport_target(ga->proximity_domain, + (u8 *)ga->device_handle); + + return 0; +} + static u32 hmat_initiator_perf(struct memory_target *target, struct memory_initiator *initiator, struct acpi_hmat_locality *hmat_loc) @@ -582,28 +644,31 @@ static int initiators_to_nodemask(unsigned long *p_nodes) return 0; } -static void hmat_register_target_initiators(struct memory_target *target) +static void hmat_update_target_attrs(struct memory_target *target, + unsigned long *p_nodes, int access) { - static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); struct memory_initiator *initiator; - unsigned int mem_nid, cpu_nid; + unsigned int cpu_nid; struct memory_locality *loc = NULL; u32 best = 0; - bool access0done = false; int i; - mem_nid = pxm_to_node(target->memory_pxm); + /* Don't update for generic port if there's no device handle */ + if (access == NODE_ACCESS_CLASS_GENPORT_SINK && + !(*(u16 *)target->gen_port_device_handle)) + return; + + bitmap_zero(p_nodes, MAX_NUMNODES); /* - * If the Address Range Structure provides a local processor pxm, link + * If the Address Range Structure provides a local processor pxm, set * only that one. Otherwise, find the best performance attributes and - * register all initiators that match. + * collect all initiators that match. */ if (target->processor_pxm != PXM_INVAL) { cpu_nid = pxm_to_node(target->processor_pxm); - register_memory_node_under_compute_node(mem_nid, cpu_nid, 0); - access0done = true; - if (node_state(cpu_nid, N_CPU)) { - register_memory_node_under_compute_node(mem_nid, cpu_nid, 1); + if (access == ACCESS_COORDINATE_LOCAL || + node_state(cpu_nid, N_CPU)) { + set_bit(target->processor_pxm, p_nodes); return; } } @@ -617,47 +682,10 @@ static void hmat_register_target_initiators(struct memory_target *target) * We'll also use the sorting to prime the candidate nodes with known * initiators. */ - bitmap_zero(p_nodes, MAX_NUMNODES); list_sort(NULL, &initiators, initiator_cmp); if (initiators_to_nodemask(p_nodes) < 0) return; - if (!access0done) { - for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { - loc = localities_types[i]; - if (!loc) - continue; - - best = 0; - list_for_each_entry(initiator, &initiators, node) { - u32 value; - - if (!test_bit(initiator->processor_pxm, p_nodes)) - continue; - - value = hmat_initiator_perf(target, initiator, - loc->hmat_loc); - if (hmat_update_best(loc->hmat_loc->data_type, value, &best)) - bitmap_clear(p_nodes, 0, initiator->processor_pxm); - if (value != best) - clear_bit(initiator->processor_pxm, p_nodes); - } - if (best) - hmat_update_target_access(target, loc->hmat_loc->data_type, - best, 0); - } - - for_each_set_bit(i, p_nodes, MAX_NUMNODES) { - cpu_nid = pxm_to_node(i); - register_memory_node_under_compute_node(mem_nid, cpu_nid, 0); - } - } - - /* Access 1 ignores Generic Initiators */ - bitmap_zero(p_nodes, MAX_NUMNODES); - if (initiators_to_nodemask(p_nodes) < 0) - return; - for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { loc = localities_types[i]; if (!loc) @@ -667,7 +695,8 @@ static void hmat_register_target_initiators(struct memory_target *target) list_for_each_entry(initiator, &initiators, node) { u32 value; - if (!initiator->has_cpu) { + if (access == ACCESS_COORDINATE_CPU && + !initiator->has_cpu) { clear_bit(initiator->processor_pxm, p_nodes); continue; } @@ -681,14 +710,43 @@ static void hmat_register_target_initiators(struct memory_target *target) clear_bit(initiator->processor_pxm, p_nodes); } if (best) - hmat_update_target_access(target, loc->hmat_loc->data_type, best, 1); + hmat_update_target_access(target, loc->hmat_loc->data_type, best, access); } +} + +static void __hmat_register_target_initiators(struct memory_target *target, + unsigned long *p_nodes, + int access) +{ + unsigned int mem_nid, cpu_nid; + int i; + + mem_nid = pxm_to_node(target->memory_pxm); + hmat_update_target_attrs(target, p_nodes, access); for_each_set_bit(i, p_nodes, MAX_NUMNODES) { cpu_nid = pxm_to_node(i); - register_memory_node_under_compute_node(mem_nid, cpu_nid, 1); + register_memory_node_under_compute_node(mem_nid, cpu_nid, access); } } +static void hmat_update_generic_target(struct memory_target *target) +{ + static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); + + hmat_update_target_attrs(target, p_nodes, + NODE_ACCESS_CLASS_GENPORT_SINK); +} + +static void hmat_register_target_initiators(struct memory_target *target) +{ + static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); + + __hmat_register_target_initiators(target, p_nodes, + ACCESS_COORDINATE_LOCAL); + __hmat_register_target_initiators(target, p_nodes, + ACCESS_COORDINATE_CPU); +} + static void hmat_register_target_cache(struct memory_target *target) { unsigned mem_nid = pxm_to_node(target->memory_pxm); @@ -701,7 +759,7 @@ static void hmat_register_target_cache(struct memory_target *target) static void hmat_register_target_perf(struct memory_target *target, int access) { unsigned mem_nid = pxm_to_node(target->memory_pxm); - node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access); + node_set_perf_attrs(mem_nid, &target->coord[access], access); } static void hmat_register_target_devices(struct memory_target *target) @@ -722,10 +780,32 @@ static void hmat_register_target_devices(struct memory_target *target) } } -static void hmat_register_target(struct memory_target *target) +static void hmat_hotplug_target(struct memory_target *target) { int nid = pxm_to_node(target->memory_pxm); + /* + * Skip offline nodes. This can happen when memory marked EFI_MEMORY_SP, + * "specific purpose", is applied to all the memory in a proximity + * domain leading to * the node being marked offline / unplugged, or if + * memory-only "hotplug" node is offline. + */ + if (nid == NUMA_NO_NODE || !node_online(nid)) + return; + + guard(mutex)(&target_lock); + if (target->registered) + return; + + hmat_register_target_initiators(target); + hmat_register_target_cache(target); + hmat_register_target_perf(target, ACCESS_COORDINATE_LOCAL); + hmat_register_target_perf(target, ACCESS_COORDINATE_CPU); + target->registered = true; +} + +static void hmat_register_target(struct memory_target *target) +{ /* * Devices may belong to either an offline or online * node, so unconditionally add them. @@ -733,24 +813,17 @@ static void hmat_register_target(struct memory_target *target) hmat_register_target_devices(target); /* - * Skip offline nodes. This can happen when memory - * marked EFI_MEMORY_SP, "specific purpose", is applied - * to all the memory in a proximity domain leading to - * the node being marked offline / unplugged, or if - * memory-only "hotplug" node is offline. + * Register generic port perf numbers. The nid may not be + * initialized and is still NUMA_NO_NODE. */ - if (nid == NUMA_NO_NODE || !node_online(nid)) - return; - mutex_lock(&target_lock); - if (!target->registered) { - hmat_register_target_initiators(target); - hmat_register_target_cache(target); - hmat_register_target_perf(target, 0); - hmat_register_target_perf(target, 1); + if (*(u16 *)target->gen_port_device_handle) { + hmat_update_generic_target(target); target->registered = true; } mutex_unlock(&target_lock); + + hmat_hotplug_target(target); } static void hmat_register_targets(void) @@ -776,10 +849,65 @@ static int hmat_callback(struct notifier_block *self, if (!target) return NOTIFY_OK; - hmat_register_target(target); + hmat_hotplug_target(target); return NOTIFY_OK; } +static int hmat_set_default_dram_perf(void) +{ + int rc; + int nid, pxm; + struct memory_target *target; + struct access_coordinate *attrs; + + if (!default_dram_type) + return -EIO; + + for_each_node_mask(nid, default_dram_type->nodes) { + pxm = node_to_pxm(nid); + target = find_mem_target(pxm); + if (!target) + continue; + attrs = &target->coord[1]; + rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT"); + if (rc) + return rc; + } + + return 0; +} + +static int hmat_calculate_adistance(struct notifier_block *self, + unsigned long nid, void *data) +{ + static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); + struct memory_target *target; + struct access_coordinate *perf; + int *adist = data; + int pxm; + + pxm = node_to_pxm(nid); + target = find_mem_target(pxm); + if (!target) + return NOTIFY_OK; + + mutex_lock(&target_lock); + hmat_update_target_attrs(target, p_nodes, ACCESS_COORDINATE_CPU); + mutex_unlock(&target_lock); + + perf = &target->coord[1]; + + if (mt_perf_to_adistance(perf, adist)) + return NOTIFY_OK; + + return NOTIFY_STOP; +} + +static struct notifier_block hmat_adist_nb __meminitdata = { + .notifier_call = hmat_calculate_adistance, + .priority = 100, +}; + static __init void hmat_free_structures(void) { struct memory_target *target, *tnext; @@ -835,6 +963,13 @@ static __init int hmat_init(void) ACPI_SRAT_TYPE_MEMORY_AFFINITY, srat_parse_mem_affinity, 0) < 0) goto out_put; + + if (acpi_table_parse_entries(ACPI_SIG_SRAT, + sizeof(struct acpi_table_srat), + ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY, + srat_parse_genport_affinity, 0) < 0) + goto out_put; + acpi_put_table(tbl); status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl); @@ -862,8 +997,13 @@ static __init int hmat_init(void) hmat_register_targets(); /* Keep the table and structures if the notifier may use them */ - if (!hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) - return 0; + if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) + goto out_put; + + if (!hmat_set_default_dram_perf()) + register_mt_adistance_algorithm(&hmat_adist_nb); + + return 0; out_put: hmat_free_structures(); acpi_put_table(tbl); diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index a44c0761fd1c0..848942bf883cb 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -140,7 +140,7 @@ acpi_table_print_srat_entry(struct acpi_subtable_header *header) struct acpi_srat_generic_affinity *p = (struct acpi_srat_generic_affinity *)header; - if (p->device_handle_type == 0) { + if (p->device_handle_type == 1) { /* * For pci devices this may be the only place they * are assigned a proximity domain diff --git a/drivers/acpi/osi.c b/drivers/acpi/osi.c index d4405e1ca9b97..ae9620757865b 100644 --- a/drivers/acpi/osi.c +++ b/drivers/acpi/osi.c @@ -42,7 +42,6 @@ static struct acpi_osi_entry osi_setup_entries[OSI_STRING_ENTRIES_MAX] __initdata = { {"Module Device", true}, {"Processor Device", true}, - {"3.0 _SCP Extensions", true}, {"Processor Aggregator Device", true}, }; diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 84030804a7633..ddc2b3e891116 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -1008,7 +1008,6 @@ struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root, int node = acpi_get_node(device->handle); struct pci_bus *bus; struct pci_host_bridge *host_bridge; - union acpi_object *obj; info->root = root; info->bridge = device; @@ -1050,17 +1049,6 @@ struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root, if (!(root->osc_ext_control_set & OSC_CXL_ERROR_REPORTING_CONTROL)) host_bridge->native_cxl_error = 0; - /* - * Evaluate the "PCI Boot Configuration" _DSM Function. If it - * exists and returns 0, we must preserve any PCI resource - * assignments made by firmware for this host bridge. - */ - obj = acpi_evaluate_dsm(ACPI_HANDLE(bus->bridge), &pci_acpi_dsm_guid, 1, - DSM_PCI_PRESERVE_BOOT_CONFIG, NULL); - if (obj && obj->type == ACPI_TYPE_INTEGER && obj->integer.value == 0) - host_bridge->preserve_config = 1; - ACPI_FREE(obj); - acpi_dev_power_up_children_with_adr(device); pci_scan_child_bus(bus); diff --git a/drivers/acpi/pfr_update.c b/drivers/acpi/pfr_update.c index 98267f163e2bd..aedf7e40145e0 100644 --- a/drivers/acpi/pfr_update.c +++ b/drivers/acpi/pfr_update.c @@ -310,7 +310,7 @@ static bool applicable_image(const void *data, struct pfru_update_cap_info *cap, if (type == PFRU_CODE_INJECT_TYPE) return payload_hdr->rt_ver >= cap->code_rt_version; - return payload_hdr->rt_ver >= cap->drv_rt_version; + return payload_hdr->svn_ver >= cap->drv_svn; } static void print_update_debug_info(struct pfru_updated_result *result, diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c index f73ce6e13065d..54676e3d82dd5 100644 --- a/drivers/acpi/pptt.c +++ b/drivers/acpi/pptt.c @@ -231,16 +231,18 @@ static int acpi_pptt_leaf_node(struct acpi_table_header *table_hdr, sizeof(struct acpi_table_pptt)); proc_sz = sizeof(struct acpi_pptt_processor); - while ((unsigned long)entry + proc_sz < table_end) { + /* ignore subtable types that are smaller than a processor node */ + while ((unsigned long)entry + proc_sz <= table_end) { cpu_node = (struct acpi_pptt_processor *)entry; + if (entry->type == ACPI_PPTT_TYPE_PROCESSOR && cpu_node->parent == node_entry) return 0; if (entry->length == 0) return 0; + entry = ACPI_ADD_PTR(struct acpi_subtable_header, entry, entry->length); - } return 1; } @@ -273,15 +275,18 @@ static struct acpi_pptt_processor *acpi_find_processor_node(struct acpi_table_he proc_sz = sizeof(struct acpi_pptt_processor); /* find the processor structure associated with this cpuid */ - while ((unsigned long)entry + proc_sz < table_end) { + while ((unsigned long)entry + proc_sz <= table_end) { cpu_node = (struct acpi_pptt_processor *)entry; if (entry->length == 0) { pr_warn("Invalid zero length subtable\n"); break; } + /* entry->length may not equal proc_sz, revalidate the processor structure length */ if (entry->type == ACPI_PPTT_TYPE_PROCESSOR && acpi_cpu_id == cpu_node->acpi_processor_id && + (unsigned long)entry + entry->length <= table_end && + entry->length == proc_sz + cpu_node->number_of_priv_resources * sizeof(u32) && acpi_pptt_leaf_node(table_hdr, cpu_node)) { return (struct acpi_pptt_processor *)entry; } diff --git a/drivers/acpi/prmt.c b/drivers/acpi/prmt.c index a34f7d37877c9..6efaeda01d166 100644 --- a/drivers/acpi/prmt.c +++ b/drivers/acpi/prmt.c @@ -85,8 +85,6 @@ static u64 efi_pa_va_lookup(efi_guid_t *guid, u64 pa) } } - pr_warn("Failed to find VA for GUID: %pUL, PA: 0x%llx", guid, pa); - return 0; } @@ -152,15 +150,52 @@ acpi_parse_prmt(union acpi_subtable_headers *header, const unsigned long end) th = &tm->handlers[cur_handler]; guid_copy(&th->guid, (guid_t *)handler_info->handler_guid); + + /* + * Print an error message if handler_address is NULL, the parse of VA also + * can be skipped. + */ + if (unlikely(!handler_info->handler_address)) { + pr_info("Skipping handler with NULL address for GUID: %pUL", + (guid_t *)handler_info->handler_guid); + continue; + } + th->handler_addr = (void *)efi_pa_va_lookup(&th->guid, handler_info->handler_address); + /* + * Print a warning message and skip the parse of VA if handler_addr is zero + * which is not expected to ever happen. + */ + if (unlikely(!th->handler_addr)) { + pr_warn("Failed to find VA of handler for GUID: %pUL, PA: 0x%llx", + &th->guid, handler_info->handler_address); + continue; + } th->static_data_buffer_addr = efi_pa_va_lookup(&th->guid, handler_info->static_data_buffer_address); + /* + * According to the PRM specification, static_data_buffer_address can be zero, + * so avoid printing a warning message in that case. Otherwise, if the + * return value of efi_pa_va_lookup() is zero, print the message. + */ + if (unlikely(!th->static_data_buffer_addr && handler_info->static_data_buffer_address)) + pr_warn("Failed to find VA of static data buffer for GUID: %pUL, PA: 0x%llx", + &th->guid, handler_info->static_data_buffer_address); th->acpi_param_buffer_addr = efi_pa_va_lookup(&th->guid, handler_info->acpi_param_buffer_address); + /* + * According to the PRM specification, acpi_param_buffer_address can be zero, + * so avoid printing a warning message in that case. Otherwise, if the + * return value of efi_pa_va_lookup() is zero, print the message. + */ + if (unlikely(!th->acpi_param_buffer_addr && handler_info->acpi_param_buffer_address)) + pr_warn("Failed to find VA of acpi param buffer for GUID: %pUL, PA: 0x%llx", + &th->guid, handler_info->acpi_param_buffer_address); + } while (++cur_handler < tm->handler_count && (handler_info = get_next_handler(handler_info))); return 0; diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 0888e4d618d53..b524cf27213d4 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -1410,6 +1410,9 @@ int acpi_processor_power_init(struct acpi_processor *pr) if (retval) { if (acpi_processor_registered == 0) cpuidle_unregister_driver(&acpi_idle_driver); + + per_cpu(acpi_cpuidle_device, pr->id) = NULL; + kfree(dev); return retval; } acpi_processor_registered++; diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c index 4265814c74f81..d81f30ce2341a 100644 --- a/drivers/acpi/processor_perflib.c +++ b/drivers/acpi/processor_perflib.c @@ -174,6 +174,9 @@ void acpi_processor_ppc_init(struct cpufreq_policy *policy) { unsigned int cpu; + if (ignore_ppc == 1) + return; + for_each_cpu(cpu, policy->related_cpus) { struct acpi_processor *pr = per_cpu(processors, cpu); int ret; @@ -194,6 +197,14 @@ void acpi_processor_ppc_init(struct cpufreq_policy *policy) if (ret < 0) pr_err("Failed to add freq constraint for CPU%d (%d)\n", cpu, ret); + + if (!pr->performance) + continue; + + ret = acpi_processor_get_platform_limit(pr); + if (ret) + pr_err("Failed to update freq constraint for CPU%d (%d)\n", + cpu, ret); } } diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index dca5682308cb3..d02b332744e64 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -74,6 +74,7 @@ static bool acpi_nondev_subnode_extract(union acpi_object *desc, struct fwnode_handle *parent) { struct acpi_data_node *dn; + acpi_handle scope = NULL; bool result; dn = kzalloc(sizeof(*dn), GFP_KERNEL); @@ -86,29 +87,35 @@ static bool acpi_nondev_subnode_extract(union acpi_object *desc, INIT_LIST_HEAD(&dn->data.properties); INIT_LIST_HEAD(&dn->data.subnodes); - result = acpi_extract_properties(handle, desc, &dn->data); - - if (handle) { - acpi_handle scope; - acpi_status status; + /* + * The scope for the completion of relative pathname segments and + * subnode object lookup is the one of the namespace node (device) + * containing the object that has returned the package. That is, it's + * the scope of that object's parent device. + */ + if (handle) + acpi_get_parent(handle, &scope); - /* - * The scope for the subnode object lookup is the one of the - * namespace node (device) containing the object that has - * returned the package. That is, it's the scope of that - * object's parent. - */ - status = acpi_get_parent(handle, &scope); - if (ACPI_SUCCESS(status) - && acpi_enumerate_nondev_subnodes(scope, desc, &dn->data, - &dn->fwnode)) - result = true; - } else if (acpi_enumerate_nondev_subnodes(NULL, desc, &dn->data, - &dn->fwnode)) { + /* + * Extract properties from the _DSD-equivalent package pointed to by + * desc and use scope (if not NULL) for the completion of relative + * pathname segments. + * + * The extracted properties will be held in the new data node dn. + */ + result = acpi_extract_properties(scope, desc, &dn->data); + /* + * Look for subnodes in the _DSD-equivalent package pointed to by desc + * and create child nodes of dn if there are any. + */ + if (acpi_enumerate_nondev_subnodes(scope, desc, &dn->data, &dn->fwnode)) result = true; - } if (result) { + /* + * This will be NULL if the desc package is embedded in an outer + * _DSD-equivalent package and its scope cannot be determined. + */ dn->handle = handle; dn->data.pointer = desc; list_add_tail(&dn->sibling, list); @@ -120,35 +127,21 @@ static bool acpi_nondev_subnode_extract(union acpi_object *desc, return false; } -static bool acpi_nondev_subnode_data_ok(acpi_handle handle, - const union acpi_object *link, - struct list_head *list, - struct fwnode_handle *parent) -{ - struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; - acpi_status status; - - status = acpi_evaluate_object_typed(handle, NULL, NULL, &buf, - ACPI_TYPE_PACKAGE); - if (ACPI_FAILURE(status)) - return false; - - if (acpi_nondev_subnode_extract(buf.pointer, handle, link, list, - parent)) - return true; - - ACPI_FREE(buf.pointer); - return false; -} - static bool acpi_nondev_subnode_ok(acpi_handle scope, const union acpi_object *link, struct list_head *list, struct fwnode_handle *parent) { + struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; acpi_handle handle; acpi_status status; + /* + * If the scope is unknown, the _DSD-equivalent package being parsed + * was embedded in an outer _DSD-equivalent package as a result of + * direct evaluation of an object pointed to by a reference. In that + * case, using a pathname as the target object pointer is invalid. + */ if (!scope) return false; @@ -157,7 +150,17 @@ static bool acpi_nondev_subnode_ok(acpi_handle scope, if (ACPI_FAILURE(status)) return false; - return acpi_nondev_subnode_data_ok(handle, link, list, parent); + status = acpi_evaluate_object_typed(handle, NULL, NULL, &buf, + ACPI_TYPE_PACKAGE); + if (ACPI_FAILURE(status)) + return false; + + if (acpi_nondev_subnode_extract(buf.pointer, handle, link, list, + parent)) + return true; + + ACPI_FREE(buf.pointer); + return false; } static bool acpi_add_nondev_subnodes(acpi_handle scope, @@ -168,9 +171,12 @@ static bool acpi_add_nondev_subnodes(acpi_handle scope, bool ret = false; int i; + /* + * Every element in the links package is expected to represent a link + * to a non-device node in a tree containing device-specific data. + */ for (i = 0; i < links->package.count; i++) { union acpi_object *link, *desc; - acpi_handle handle; bool result; link = &links->package.elements[i]; @@ -178,26 +184,53 @@ static bool acpi_add_nondev_subnodes(acpi_handle scope, if (link->package.count != 2) continue; - /* The first one must be a string. */ + /* The first one (the key) must be a string. */ if (link->package.elements[0].type != ACPI_TYPE_STRING) continue; - /* The second one may be a string, a reference or a package. */ + /* The second one (the target) may be a string or a package. */ switch (link->package.elements[1].type) { case ACPI_TYPE_STRING: + /* + * The string is expected to be a full pathname or a + * pathname segment relative to the given scope. That + * pathname is expected to point to an object returning + * a package that contains _DSD-equivalent information. + */ result = acpi_nondev_subnode_ok(scope, link, list, parent); break; - case ACPI_TYPE_LOCAL_REFERENCE: - handle = link->package.elements[1].reference.handle; - result = acpi_nondev_subnode_data_ok(handle, link, list, - parent); - break; case ACPI_TYPE_PACKAGE: + /* + * This happens when a reference is used in AML to + * point to the target. Since the target is expected + * to be a named object, a reference to it will cause it + * to be avaluated in place and its return package will + * be embedded in the links package at the location of + * the reference. + * + * The target package is expected to contain _DSD- + * equivalent information, but the scope in which it + * is located in the original AML is unknown. Thus + * it cannot contain pathname segments represented as + * strings because there is no way to build full + * pathnames out of them. + */ + acpi_handle_debug(scope, "subnode %s: Unknown scope\n", + link->package.elements[0].string.pointer); desc = &link->package.elements[1]; result = acpi_nondev_subnode_extract(desc, NULL, link, list, parent); break; + case ACPI_TYPE_LOCAL_REFERENCE: + /* + * It is not expected to see any local references in + * the links package because referencing a named object + * should cause it to be evaluated in place. + */ + acpi_handle_info(scope, "subnode %s: Unexpected reference\n", + link->package.elements[0].string.pointer); + fallthrough; default: result = false; break; @@ -357,6 +390,9 @@ static void acpi_untie_nondev_subnodes(struct acpi_device_data *data) struct acpi_data_node *dn; list_for_each_entry(dn, &data->subnodes, sibling) { + if (!dn->handle) + continue; + acpi_detach_data(dn->handle, acpi_nondev_subnode_tag); acpi_untie_nondev_subnodes(&dn->data); @@ -371,6 +407,9 @@ static bool acpi_tie_nondev_subnodes(struct acpi_device_data *data) acpi_status status; bool ret; + if (!dn->handle) + continue; + status = acpi_attach_data(dn->handle, acpi_nondev_subnode_tag, dn); if (ACPI_FAILURE(status) && status != AE_ALREADY_EXISTS) { acpi_handle_err(dn->handle, "Can't tag data node\n"); @@ -1247,6 +1286,28 @@ struct fwnode_handle *acpi_get_next_subnode(const struct fwnode_handle *fwnode, return NULL; } +/* + * acpi_get_next_present_subnode - Return the next present child node handle + * @fwnode: Firmware node to find the next child node for. + * @child: Handle to one of the device's child nodes or a null handle. + * + * Like acpi_get_next_subnode(), but the device nodes returned by + * acpi_get_next_present_subnode() are guaranteed to be present. + * + * Returns: The fwnode handle of the next present sub-node. + */ +static struct fwnode_handle * +acpi_get_next_present_subnode(const struct fwnode_handle *fwnode, + struct fwnode_handle *child) +{ + do { + child = acpi_get_next_subnode(fwnode, child); + } while (is_acpi_device_node(child) && + !acpi_device_is_present(to_acpi_device_node(child))); + + return child; +} + /** * acpi_node_get_parent - Return parent fwnode of this fwnode * @fwnode: Firmware node whose parent to get @@ -1590,7 +1651,7 @@ static int acpi_fwnode_irq_get(const struct fwnode_handle *fwnode, .property_read_string_array = \ acpi_fwnode_property_read_string_array, \ .get_parent = acpi_node_get_parent, \ - .get_next_child_node = acpi_get_next_subnode, \ + .get_next_child_node = acpi_get_next_present_subnode, \ .get_named_child_node = acpi_fwnode_get_named_child_node, \ .get_name = acpi_fwnode_get_name, \ .get_name_prefix = acpi_fwnode_get_name_prefix, \ diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index c0c5c5c58ae1e..5b5986e10c2d7 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -784,6 +784,8 @@ static bool acpi_info_matches_ids(struct acpi_device_info *info, static const char * const acpi_ignore_dep_ids[] = { "PNP0D80", /* Windows-compatible System Power Management Controller */ "INT33BD", /* Intel Baytrail Mailbox Device */ + "INTC10DE", /* Intel CVS LNL */ + "INTC10E0", /* Intel CVS ARL */ "LATT2021", /* Lattice FW Update Client Driver */ NULL }; diff --git a/drivers/android/binder.c b/drivers/android/binder.c old mode 100644 new mode 100755 index 929ae45e0745d..35643b0881021 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -717,7 +717,7 @@ static void binder_do_set_priority(struct binder_thread *thread, bool has_cap_nice; unsigned int policy = desired->sched_policy; - if (task->policy == policy && task->normal_prio == desired->prio) { + if (task->policy == policy && task->prio == desired->prio) { spin_lock(&thread->prio_lock); if (thread->prio_state == BINDER_PRIO_PENDING) thread->prio_state = BINDER_PRIO_SET; @@ -759,7 +759,7 @@ static void binder_do_set_priority(struct binder_thread *thread, task->pid, desired->prio, to_kernel_prio(policy, priority)); - trace_binder_set_priority(task->tgid, task->pid, task->normal_prio, + trace_binder_set_priority(task->tgid, task->pid, task->prio, to_kernel_prio(policy, priority), desired->prio); @@ -1024,17 +1024,8 @@ static int binder_inc_node_nilocked(struct binder_node *node, int strong, } else { if (!internal) node->local_weak_refs++; - if (!node->has_weak_ref && list_empty(&node->work.entry)) { - if (target_list == NULL) { - pr_err("invalid inc weak node for %d\n", - node->debug_id); - return -EINVAL; - } - /* - * See comment above - */ + if (!node->has_weak_ref && target_list && list_empty(&node->work.entry)) binder_enqueue_work_ilocked(&node->work, target_list); - } } return 0; } @@ -3495,7 +3486,7 @@ static void binder_transaction(struct binder_proc *proc, binder_supported_policy(current->policy)) { /* Inherit supported policies for synchronous transactions */ t->priority.sched_policy = current->policy; - t->priority.prio = current->normal_prio; + t->priority.prio = current->prio; } else { /* Otherwise, fall back to the default priority */ t->priority = target_proc->default_priority; @@ -6642,10 +6633,10 @@ static void print_binder_transaction_ilocked(struct seq_file *m, } static void print_binder_work_ilocked(struct seq_file *m, - struct binder_proc *proc, - const char *prefix, - const char *transaction_prefix, - struct binder_work *w) + struct binder_proc *proc, + const char *prefix, + const char *transaction_prefix, + struct binder_work *w, bool hash_ptrs) { struct binder_node *node; struct binder_transaction *t; @@ -6668,9 +6659,15 @@ static void print_binder_work_ilocked(struct seq_file *m, break; case BINDER_WORK_NODE: node = container_of(w, struct binder_node, work); - seq_printf(m, "%snode work %d: u%016llx c%016llx\n", - prefix, node->debug_id, - (u64)node->ptr, (u64)node->cookie); + if (hash_ptrs) + seq_printf(m, "%snode work %d: u%p c%p\n", + prefix, node->debug_id, + (void *)(long)node->ptr, + (void *)(long)node->cookie); + else + seq_printf(m, "%snode work %d: u%016llx c%016llx\n", + prefix, node->debug_id, + (u64)node->ptr, (u64)node->cookie); break; case BINDER_WORK_DEAD_BINDER: seq_printf(m, "%shas dead binder\n", prefix); @@ -6695,7 +6692,7 @@ static void print_binder_work_ilocked(struct seq_file *m, static void print_binder_thread_ilocked(struct seq_file *m, struct binder_thread *thread, - int print_always) + bool print_always, bool hash_ptrs) { struct binder_transaction *t; struct binder_work *w; @@ -6725,14 +6722,16 @@ static void print_binder_thread_ilocked(struct seq_file *m, } list_for_each_entry(w, &thread->todo, entry) { print_binder_work_ilocked(m, thread->proc, " ", - " pending transaction", w); + " pending transaction", + w, hash_ptrs); } if (!print_always && m->count == header_pos) m->count = start_pos; } static void print_binder_node_nilocked(struct seq_file *m, - struct binder_node *node) + struct binder_node *node, + bool hash_ptrs) { struct binder_ref *ref; struct binder_work *w; @@ -6742,8 +6741,13 @@ static void print_binder_node_nilocked(struct seq_file *m, hlist_for_each_entry(ref, &node->refs, node_entry) count++; - seq_printf(m, " node %d: u%016llx c%016llx pri %d:%d hs %d hw %d ls %d lw %d is %d iw %d tr %d", - node->debug_id, (u64)node->ptr, (u64)node->cookie, + if (hash_ptrs) + seq_printf(m, " node %d: u%p c%p", node->debug_id, + (void *)(long)node->ptr, (void *)(long)node->cookie); + else + seq_printf(m, " node %d: u%016llx c%016llx", node->debug_id, + (u64)node->ptr, (u64)node->cookie); + seq_printf(m, " pri %d:%d hs %d hw %d ls %d lw %d is %d iw %d tr %d", node->sched_policy, node->min_priority, node->has_strong_ref, node->has_weak_ref, node->local_strong_refs, node->local_weak_refs, @@ -6757,7 +6761,8 @@ static void print_binder_node_nilocked(struct seq_file *m, if (node->proc) { list_for_each_entry(w, &node->async_todo, entry) print_binder_work_ilocked(m, node->proc, " ", - " pending async transaction", w); + " pending async transaction", + w, hash_ptrs); } } @@ -6773,8 +6778,54 @@ static void print_binder_ref_olocked(struct seq_file *m, binder_node_unlock(ref->node); } -static void print_binder_proc(struct seq_file *m, - struct binder_proc *proc, int print_all) +/** + * print_next_binder_node_ilocked() - Print binder_node from a locked list + * @m: struct seq_file for output via seq_printf() + * @proc: struct binder_proc we hold the inner_proc_lock to (if any) + * @node: struct binder_node to print fields of + * @prev_node: struct binder_node we hold a temporary reference to (if any) + * @hash_ptrs: whether to hash @node's binder_uintptr_t fields + * + * Helper function to handle synchronization around printing a struct + * binder_node while iterating through @proc->nodes or the dead nodes list. + * Caller must hold either @proc->inner_lock (for live nodes) or + * binder_dead_nodes_lock. This lock will be released during the body of this + * function, but it will be reacquired before returning to the caller. + * + * Return: pointer to the struct binder_node we hold a tmpref on + */ +static struct binder_node * +print_next_binder_node_ilocked(struct seq_file *m, struct binder_proc *proc, + struct binder_node *node, + struct binder_node *prev_node, bool hash_ptrs) +{ + /* + * Take a temporary reference on the node so that isn't freed while + * we print it. + */ + binder_inc_node_tmpref_ilocked(node); + /* + * Live nodes need to drop the inner proc lock and dead nodes need to + * drop the binder_dead_nodes_lock before trying to take the node lock. + */ + if (proc) + binder_inner_proc_unlock(proc); + else + spin_unlock(&binder_dead_nodes_lock); + if (prev_node) + binder_put_node(prev_node); + binder_node_inner_lock(node); + print_binder_node_nilocked(m, node, hash_ptrs); + binder_node_inner_unlock(node); + if (proc) + binder_inner_proc_lock(proc); + else + spin_lock(&binder_dead_nodes_lock); + return node; +} + +static void print_binder_proc(struct seq_file *m, struct binder_proc *proc, + bool print_all, bool hash_ptrs) { struct binder_work *w; struct rb_node *n; @@ -6787,31 +6838,19 @@ static void print_binder_proc(struct seq_file *m, header_pos = m->count; binder_inner_proc_lock(proc); - for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) + for (n = rb_first(&proc->threads); n; n = rb_next(n)) print_binder_thread_ilocked(m, rb_entry(n, struct binder_thread, - rb_node), print_all); + rb_node), print_all, hash_ptrs); - for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) { + for (n = rb_first(&proc->nodes); n; n = rb_next(n)) { struct binder_node *node = rb_entry(n, struct binder_node, rb_node); if (!print_all && !node->has_async_transaction) continue; - /* - * take a temporary reference on the node so it - * survives and isn't removed from the tree - * while we print it. - */ - binder_inc_node_tmpref_ilocked(node); - /* Need to drop inner lock to take node lock */ - binder_inner_proc_unlock(proc); - if (last_node) - binder_put_node(last_node); - binder_node_inner_lock(node); - print_binder_node_nilocked(m, node); - binder_node_inner_unlock(node); - last_node = node; - binder_inner_proc_lock(proc); + last_node = print_next_binder_node_ilocked(m, proc, node, + last_node, + hash_ptrs); } binder_inner_proc_unlock(proc); if (last_node) @@ -6819,19 +6858,18 @@ static void print_binder_proc(struct seq_file *m, if (print_all) { binder_proc_lock(proc); - for (n = rb_first(&proc->refs_by_desc); - n != NULL; - n = rb_next(n)) + for (n = rb_first(&proc->refs_by_desc); n; n = rb_next(n)) print_binder_ref_olocked(m, rb_entry(n, - struct binder_ref, - rb_node_desc)); + struct binder_ref, + rb_node_desc)); binder_proc_unlock(proc); } binder_alloc_print_allocated(m, &proc->alloc); binder_inner_proc_lock(proc); list_for_each_entry(w, &proc->todo, entry) print_binder_work_ilocked(m, proc, " ", - " pending transaction", w); + " pending transaction", w, + hash_ptrs); list_for_each_entry(w, &proc->delivered_death, entry) { seq_puts(m, " has delivered dead binder\n"); break; @@ -6958,7 +6996,7 @@ static void print_binder_proc_stats(struct seq_file *m, count = 0; ready_threads = 0; binder_inner_proc_lock(proc); - for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) + for (n = rb_first(&proc->threads); n; n = rb_next(n)) count++; list_for_each_entry(thread, &proc->waiting_threads, waiting_thread_node) @@ -6972,7 +7010,7 @@ static void print_binder_proc_stats(struct seq_file *m, ready_threads, free_async_space); count = 0; - for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) + for (n = rb_first(&proc->nodes); n; n = rb_next(n)) count++; binder_inner_proc_unlock(proc); seq_printf(m, " nodes: %d\n", count); @@ -6980,7 +7018,7 @@ static void print_binder_proc_stats(struct seq_file *m, strong = 0; weak = 0; binder_proc_lock(proc); - for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) { + for (n = rb_first(&proc->refs_by_desc); n; n = rb_next(n)) { struct binder_ref *ref = rb_entry(n, struct binder_ref, rb_node_desc); count++; @@ -7007,7 +7045,7 @@ static void print_binder_proc_stats(struct seq_file *m, print_binder_stats(m, " ", &proc->stats); } -static int state_show(struct seq_file *m, void *unused) +static void print_binder_state(struct seq_file *m, bool hash_ptrs) { struct binder_proc *proc; struct binder_node *node; @@ -7018,31 +7056,40 @@ static int state_show(struct seq_file *m, void *unused) spin_lock(&binder_dead_nodes_lock); if (!hlist_empty(&binder_dead_nodes)) seq_puts(m, "dead nodes:\n"); - hlist_for_each_entry(node, &binder_dead_nodes, dead_node) { - /* - * take a temporary reference on the node so it - * survives and isn't removed from the list - * while we print it. - */ - node->tmp_refs++; - spin_unlock(&binder_dead_nodes_lock); - if (last_node) - binder_put_node(last_node); - binder_node_lock(node); - print_binder_node_nilocked(m, node); - binder_node_unlock(node); - last_node = node; - spin_lock(&binder_dead_nodes_lock); - } + hlist_for_each_entry(node, &binder_dead_nodes, dead_node) + last_node = print_next_binder_node_ilocked(m, NULL, node, + last_node, + hash_ptrs); spin_unlock(&binder_dead_nodes_lock); if (last_node) binder_put_node(last_node); mutex_lock(&binder_procs_lock); hlist_for_each_entry(proc, &binder_procs, proc_node) - print_binder_proc(m, proc, 1); + print_binder_proc(m, proc, true, hash_ptrs); mutex_unlock(&binder_procs_lock); +} +static void print_binder_transactions(struct seq_file *m, bool hash_ptrs) +{ + struct binder_proc *proc; + + seq_puts(m, "binder transactions:\n"); + mutex_lock(&binder_procs_lock); + hlist_for_each_entry(proc, &binder_procs, proc_node) + print_binder_proc(m, proc, false, hash_ptrs); + mutex_unlock(&binder_procs_lock); +} + +static int state_show(struct seq_file *m, void *unused) +{ + print_binder_state(m, false); + return 0; +} + +static int state_hashed_show(struct seq_file *m, void *unused) +{ + print_binder_state(m, true); return 0; } @@ -7064,14 +7111,13 @@ static int stats_show(struct seq_file *m, void *unused) static int transactions_show(struct seq_file *m, void *unused) { - struct binder_proc *proc; - - seq_puts(m, "binder transactions:\n"); - mutex_lock(&binder_procs_lock); - hlist_for_each_entry(proc, &binder_procs, proc_node) - print_binder_proc(m, proc, 0); - mutex_unlock(&binder_procs_lock); + print_binder_transactions(m, false); + return 0; +} +static int transactions_hashed_show(struct seq_file *m, void *unused) +{ + print_binder_transactions(m, true); return 0; } @@ -7084,7 +7130,7 @@ static int proc_show(struct seq_file *m, void *unused) hlist_for_each_entry(itr, &binder_procs, proc_node) { if (itr->pid == pid) { seq_puts(m, "binder proc state:\n"); - print_binder_proc(m, itr, 1); + print_binder_proc(m, itr, true, false); } } mutex_unlock(&binder_procs_lock); @@ -7151,8 +7197,10 @@ const struct file_operations binder_fops = { }; DEFINE_SHOW_ATTRIBUTE(state); +DEFINE_SHOW_ATTRIBUTE(state_hashed); DEFINE_SHOW_ATTRIBUTE(stats); DEFINE_SHOW_ATTRIBUTE(transactions); +DEFINE_SHOW_ATTRIBUTE(transactions_hashed); DEFINE_SHOW_ATTRIBUTE(transaction_log); const struct binder_debugfs_entry binder_debugfs_entries[] = { @@ -7162,6 +7210,12 @@ const struct binder_debugfs_entry binder_debugfs_entries[] = { .fops = &state_fops, .data = NULL, }, + { + .name = "state_hashed", + .mode = 0444, + .fops = &state_hashed_fops, + .data = NULL, + }, { .name = "stats", .mode = 0444, @@ -7174,6 +7228,12 @@ const struct binder_debugfs_entry binder_debugfs_entries[] = { .fops = &transactions_fops, .data = NULL, }, + { + .name = "transactions_hashed", + .mode = 0444, + .fops = &transactions_hashed_fops, + .data = NULL, + }, { .name = "transaction_log", .mode = 0444, diff --git a/drivers/android/binder_internal.h b/drivers/android/binder_internal.h old mode 100644 new mode 100755 diff --git a/drivers/android/dbitmap.h b/drivers/android/dbitmap.h index 956f1bd087d1c..c7299ce8b3741 100644 --- a/drivers/android/dbitmap.h +++ b/drivers/android/dbitmap.h @@ -37,6 +37,7 @@ static inline void dbitmap_free(struct dbitmap *dmap) { dmap->nbits = 0; kfree(dmap->map); + dmap->map = NULL; } /* Returns the nbits that a dbitmap can shrink to, 0 if not possible. */ diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c old mode 100644 new mode 100755 index 6753c73f7ae1e..2d64b03a1bec1 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -89,6 +89,7 @@ * Export tracepoints that act as a bare tracehook (ie: have no trace event * associated with them) to allow external modules to probe them. */ +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_put_super); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tcp_sendmsg); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tcp_recvmsg); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_udp_sendmsg); @@ -106,8 +107,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_build_skb_around); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_refrigerator); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_send_sig_info); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_killed_process); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_oom_reaper_delay_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rmqueue_smallest_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_free_one_page_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_thaw_killed_process); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_wait_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_wait_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_init); @@ -151,6 +154,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpufreq_resolve_freq); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpufreq_fast_switch); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpufreq_target); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpufreq_online); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpufreq_create_policy); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_cgroup_attach); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_cgroup_online); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpuinfo_c_show); @@ -243,8 +247,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_mmap_file); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_file_open); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_disable_thermal_cooling_stats); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_gic_v3_set_affinity); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_gic_set_affinity); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_gic_v3_affinity_init); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dc_send_copy); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dc_receive); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_bpf_syscall); @@ -306,6 +308,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cache_show); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_report_bug); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_watchdog_timer_softlockup); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_try_to_freeze_todo); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_try_to_freeze_todo_logging); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_try_to_freeze_todo_unfrozen); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ep_create_wakeup_source); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_timerfd_create); @@ -452,6 +455,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sd_setup_unmap_multi_segment); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_scan_type); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_queue_request_and_unlock); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_fuse_request_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpuset_attach_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpuset_css_online); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpuset_fork); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_pageout_swap_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_swapin_walk_pmd_entry); @@ -492,6 +497,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lruvec_add_folio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lruvec_del_folio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_add_lazyfree_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_async_mmap_readahead); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_free_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_cache_ra_unbounded); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_force_page_cache_ra); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_fault_folio_locked); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_read_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_update_page_mapcount); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_add_page_to_lrulist); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_del_page_from_lrulist); @@ -503,7 +513,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_event); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_group); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_update_triggers); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_cold_or_pageout_abort); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_free_page); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tcp_sock_error); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tcp_fastsyn); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tcp_state_change); @@ -527,6 +536,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_fill_rwbs); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_io_statistics); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_delay_run_hw_queue); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_kick_requeue_list); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_mq_has_request); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_f2fs_ra_op_flags); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_use_amu_fie); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_resume_end); @@ -574,12 +584,15 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_map_pages); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_alloc_pages_gfp); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_cache_readahead_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_cache_readahead_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_cache_ra_order_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_fault_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_fault_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dma_heap_buffer_alloc_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dma_heap_buffer_alloc_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zs_shrinker_adjust); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zs_shrinker_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_remove_mapping); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_remove_mapping_failed); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ogki_get_log_usertype); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ogki_hievent_to_jank); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_ogki_hievent_create); @@ -650,7 +663,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_compaction_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_compaction_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_read_fault); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_private_mod); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cma_alloc_fail); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_store_partner_src_caps); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpm_modify_src_caps); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpm_log); @@ -667,6 +679,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_reuse_whole_anon_folio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_swap_slot_cache); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_calculate_totalreserve_pages); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_set_ioprio); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_oom_evaluate_task_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_pages); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lru_gen_add_folio_skip); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lru_gen_del_folio_skip); @@ -675,9 +688,34 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_kswapd_shrink_node); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_keep_reclaimed_folio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_reclaimed_folio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_evict_folios_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_folio_add_lru); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_folio_add_lru_folio_activate); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_fault_pre_folio_locked); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_folio_mapped); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_folio_remove_rmap_ptes); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_get_folio_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_pageset_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_xhci_full_reset_on_remove); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mempool_alloc_skip_wait); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_ac); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_rmqueue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_suitable_zone); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_max_order); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_pageset); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_lru_add_dst); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_isolate_priv_lru); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_pgdat_balanced); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_file_is_tiny); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_nr_pcp_alloc); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_pcp_alloc_factor_adjust); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_fas_gpu_qos_update_tracer); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_loop_skip_queue_work); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_gup_longterm_locked); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_bio_add_page_merge_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_swap_bio_charge); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_resched_curr_lazy); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_restore_curr_resched); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_curr_lazy); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lock_delay_schedule); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lock_task_fork); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lock_task_exit); diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index 42b51c9812a0e..188707d2970ef 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -117,7 +117,7 @@ config SATA_AHCI config SATA_MOBILE_LPM_POLICY int "Default SATA Link Power Management policy for low power chipsets" - range 0 4 + range 0 5 default 0 depends on SATA_AHCI help @@ -126,15 +126,32 @@ config SATA_MOBILE_LPM_POLICY chipsets are typically found on most laptops but desktops and servers now also widely use chipsets supporting low power modes. - The value set has the following meanings: + Each policy combines power saving states and features: + - Partial: The Phy logic is powered but is in a reduced power + state. The exit latency from this state is no longer than + 10us). + - Slumber: The Phy logic is powered but is in an even lower power + state. The exit latency from this state is potentially + longer, but no longer than 10ms. + - DevSleep: The Phy logic may be powered down. The exit latency from + this state is no longer than 20 ms, unless otherwise + specified by DETO in the device Identify Device Data log. + - HIPM: Host Initiated Power Management (host automatically + transitions to partial and slumber). + - DIPM: Device Initiated Power Management (device automatically + transitions to partial and slumber). + + The possible values for the default SATA link power management + policies are: 0 => Keep firmware settings - 1 => Maximum performance - 2 => Medium power - 3 => Medium power with Device Initiated PM enabled - 4 => Minimum power - - Note "Minimum power" is known to cause issues, including disk - corruption, with some disks and should not be used. + 1 => No power savings (maximum performance) + 2 => HIPM (Partial) + 3 => HIPM (Partial) and DIPM (Partial and Slumber) + 4 => HIPM (Partial and DevSleep) and DIPM (Partial and Slumber) + 5 => HIPM (Slumber and DevSleep) and DIPM (Partial and Slumber) + + Excluding the value 0, higher values represent policies with higher + power savings. config SATA_AHCI_PLATFORM tristate "Platform AHCI SATA support" diff --git a/drivers/ata/libata-acpi.c b/drivers/ata/libata-acpi.c index d36e71f475abd..39a350755a1ba 100644 --- a/drivers/ata/libata-acpi.c +++ b/drivers/ata/libata-acpi.c @@ -514,15 +514,19 @@ unsigned int ata_acpi_gtm_xfermask(struct ata_device *dev, EXPORT_SYMBOL_GPL(ata_acpi_gtm_xfermask); /** - * ata_acpi_cbl_80wire - Check for 80 wire cable + * ata_acpi_cbl_pata_type - Return PATA cable type * @ap: Port to check - * @gtm: GTM data to use * - * Return 1 if the @gtm indicates the BIOS selected an 80wire mode. + * Return ATA_CBL_PATA* according to the transfer mode selected by BIOS */ -int ata_acpi_cbl_80wire(struct ata_port *ap, const struct ata_acpi_gtm *gtm) +int ata_acpi_cbl_pata_type(struct ata_port *ap) { struct ata_device *dev; + int ret = ATA_CBL_PATA_UNK; + const struct ata_acpi_gtm *gtm = ata_acpi_init_gtm(ap); + + if (!gtm) + return ATA_CBL_PATA40; ata_for_each_dev(dev, &ap->link, ENABLED) { unsigned int xfer_mask, udma_mask; @@ -530,13 +534,17 @@ int ata_acpi_cbl_80wire(struct ata_port *ap, const struct ata_acpi_gtm *gtm) xfer_mask = ata_acpi_gtm_xfermask(dev, gtm); ata_unpack_xfermask(xfer_mask, NULL, NULL, &udma_mask); - if (udma_mask & ~ATA_UDMA_MASK_40C) - return 1; + ret = ATA_CBL_PATA40; + + if (udma_mask & ~ATA_UDMA_MASK_40C) { + ret = ATA_CBL_PATA80; + break; + } } - return 0; + return ret; } -EXPORT_SYMBOL_GPL(ata_acpi_cbl_80wire); +EXPORT_SYMBOL_GPL(ata_acpi_cbl_pata_type); static void ata_acpi_gtf_to_tf(struct ata_device *dev, const struct ata_acpi_gtf *gtf, diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c index be72030a500d4..9e0a820d6961d 100644 --- a/drivers/ata/libata-sata.c +++ b/drivers/ata/libata-sata.c @@ -817,6 +817,11 @@ static ssize_t ata_scsi_lpm_store(struct device *device, spin_lock_irqsave(ap->lock, flags); + if (ap->flags & ATA_FLAG_NO_LPM) { + count = -EOPNOTSUPP; + goto out_unlock; + } + ata_for_each_link(link, ap, EDGE) { ata_for_each_dev(dev, &ap->link, ENABLED) { if (dev->horkage & ATA_HORKAGE_NOLPM) { diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 6a1460d35447c..8f6a7acf770e4 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -856,18 +856,14 @@ static void ata_to_sense_error(unsigned id, u8 drv_stat, u8 drv_err, u8 *sk, {0xFF, 0xFF, 0xFF, 0xFF}, // END mark }; static const unsigned char stat_table[][4] = { - /* Must be first because BUSY means no other bits valid */ - {0x80, ABORTED_COMMAND, 0x47, 0x00}, - // Busy, fake parity for now - {0x40, ILLEGAL_REQUEST, 0x21, 0x04}, - // Device ready, unaligned write command - {0x20, HARDWARE_ERROR, 0x44, 0x00}, - // Device fault, internal target failure - {0x08, ABORTED_COMMAND, 0x47, 0x00}, - // Timed out in xfer, fake parity for now - {0x04, RECOVERED_ERROR, 0x11, 0x00}, - // Recovered ECC error Medium error, recovered - {0xFF, 0xFF, 0xFF, 0xFF}, // END mark + /* Busy: must be first because BUSY means no other bits valid */ + { ATA_BUSY, ABORTED_COMMAND, 0x00, 0x00 }, + /* Device fault: INTERNAL TARGET FAILURE */ + { ATA_DF, HARDWARE_ERROR, 0x44, 0x00 }, + /* Corrected data error */ + { ATA_CORR, RECOVERED_ERROR, 0x00, 0x00 }, + + { 0xFF, 0xFF, 0xFF, 0xFF }, /* END mark */ }; /* @@ -939,6 +935,8 @@ static void ata_gen_passthru_sense(struct ata_queued_cmd *qc) if (!(qc->flags & ATA_QCFLAG_RTF_FILLED)) { ata_dev_dbg(dev, "missing result TF: can't generate ATA PT sense data\n"); + if (qc->err_mask) + ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND, 0, 0); return; } @@ -994,10 +992,17 @@ static void ata_gen_ata_sense(struct ata_queued_cmd *qc) return; } + if (ata_id_is_locked(dev->id)) { + /* Security locked */ + /* LOGICAL UNIT ACCESS NOT AUTHORIZED */ + ata_scsi_set_sense(dev, cmd, DATA_PROTECT, 0x74, 0x71); + return; + } + if (!(qc->flags & ATA_QCFLAG_RTF_FILLED)) { ata_dev_dbg(dev, - "missing result TF: can't generate sense data\n"); - return; + "Missing result TF: reporting aborted command\n"); + goto aborted; } /* Use ata_to_sense_error() to map status register bits @@ -1008,19 +1013,20 @@ static void ata_gen_ata_sense(struct ata_queued_cmd *qc) ata_to_sense_error(qc->ap->print_id, tf->status, tf->error, &sense_key, &asc, &ascq); ata_scsi_set_sense(dev, cmd, sense_key, asc, ascq); - } else { - /* Could not decode error */ - ata_dev_warn(dev, "could not decode error status 0x%x err_mask 0x%x\n", - tf->status, qc->err_mask); - ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND, 0, 0); - return; - } - block = ata_tf_read_block(&qc->result_tf, dev); - if (block == U64_MAX) + block = ata_tf_read_block(&qc->result_tf, dev); + if (block != U64_MAX) + scsi_set_sense_information(sb, SCSI_SENSE_BUFFERSIZE, + block); return; + } - scsi_set_sense_information(sb, SCSI_SENSE_BUFFERSIZE, block); + /* Could not decode error */ + ata_dev_warn(dev, + "Could not decode error 0x%x, status 0x%x (err_mask=0x%x)\n", + tf->error, tf->status, qc->err_mask); +aborted: + ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND, 0, 0); } void ata_scsi_sdev_config(struct scsi_device *sdev) @@ -3786,21 +3792,16 @@ static int ata_mselect_control_ata_feature(struct ata_queued_cmd *qc, /* Check cdl_ctrl */ switch (buf[0] & 0x03) { case 0: - /* Disable CDL if it is enabled */ - if (!(dev->flags & ATA_DFLAG_CDL_ENABLED)) - return 0; + /* Disable CDL */ ata_dev_dbg(dev, "Disabling CDL\n"); cdl_action = 0; dev->flags &= ~ATA_DFLAG_CDL_ENABLED; break; case 0x02: /* - * Enable CDL if not already enabled. Since this is mutually - * exclusive with NCQ priority, allow this only if NCQ priority - * is disabled. + * Enable CDL. Since CDL is mutually exclusive with NCQ + * priority, allow this only if NCQ priority is disabled. */ - if (dev->flags & ATA_DFLAG_CDL_ENABLED) - return 0; if (dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLED) { ata_dev_err(dev, "NCQ priority must be disabled to enable CDL\n"); @@ -4837,8 +4838,10 @@ void ata_scsi_dev_rescan(struct work_struct *work) spin_unlock_irqrestore(ap->lock, flags); if (do_resume) { ret = scsi_resume_device(sdev); - if (ret == -EWOULDBLOCK) + if (ret == -EWOULDBLOCK) { + scsi_device_put(sdev); goto unlock_scan; + } dev->flags &= ~ATA_DFLAG_RESUMING; } ret = scsi_rescan_device(sdev); diff --git a/drivers/ata/pata_cs5536.c b/drivers/ata/pata_cs5536.c index b811efd2cc346..73e81e160c91f 100644 --- a/drivers/ata/pata_cs5536.c +++ b/drivers/ata/pata_cs5536.c @@ -27,7 +27,7 @@ #include #include -#ifdef CONFIG_X86_32 +#if defined(CONFIG_X86) && defined(CONFIG_X86_32) #include static int use_msr; module_param_named(msr, use_msr, int, 0644); diff --git a/drivers/ata/pata_via.c b/drivers/ata/pata_via.c index 696b99720dcbd..bb80e7800dcbe 100644 --- a/drivers/ata/pata_via.c +++ b/drivers/ata/pata_via.c @@ -201,11 +201,9 @@ static int via_cable_detect(struct ata_port *ap) { two drives */ if (ata66 & (0x10100000 >> (16 * ap->port_no))) return ATA_CBL_PATA80; + /* Check with ACPI so we can spot BIOS reported SATA bridges */ - if (ata_acpi_init_gtm(ap) && - ata_acpi_cbl_80wire(ap, ata_acpi_init_gtm(ap))) - return ATA_CBL_PATA80; - return ATA_CBL_PATA40; + return ata_acpi_cbl_pata_type(ap); } static int via_pre_reset(struct ata_link *link, unsigned long deadline) @@ -368,7 +366,8 @@ static unsigned int via_mode_filter(struct ata_device *dev, unsigned int mask) } if (dev->class == ATA_DEV_ATAPI && - dmi_check_system(no_atapi_dma_dmi_table)) { + (dmi_check_system(no_atapi_dma_dmi_table) || + config->id == PCI_DEVICE_ID_VIA_6415)) { ata_dev_warn(dev, "controller locks up on ATAPI DMA, forcing PIO\n"); mask &= ATA_MASK_PIO; } diff --git a/drivers/atm/atmtcp.c b/drivers/atm/atmtcp.c index 96bea1ab1eccf..9c83fb29b2f1b 100644 --- a/drivers/atm/atmtcp.c +++ b/drivers/atm/atmtcp.c @@ -279,6 +279,19 @@ static struct atm_vcc *find_vcc(struct atm_dev *dev, short vpi, int vci) return NULL; } +static int atmtcp_c_pre_send(struct atm_vcc *vcc, struct sk_buff *skb) +{ + struct atmtcp_hdr *hdr; + + if (skb->len < sizeof(struct atmtcp_hdr)) + return -EINVAL; + + hdr = (struct atmtcp_hdr *)skb->data; + if (hdr->length == ATMTCP_HDR_MAGIC) + return -EINVAL; + + return 0; +} static int atmtcp_c_send(struct atm_vcc *vcc,struct sk_buff *skb) { @@ -288,7 +301,6 @@ static int atmtcp_c_send(struct atm_vcc *vcc,struct sk_buff *skb) struct sk_buff *new_skb; int result = 0; - if (!skb->len) return 0; dev = vcc->dev_data; hdr = (struct atmtcp_hdr *) skb->data; if (hdr->length == ATMTCP_HDR_MAGIC) { @@ -345,6 +357,7 @@ static const struct atmdev_ops atmtcp_v_dev_ops = { static const struct atmdev_ops atmtcp_c_dev_ops = { .close = atmtcp_c_close, + .pre_send = atmtcp_c_pre_send, .send = atmtcp_c_send }; diff --git a/drivers/atm/idt77252.c b/drivers/atm/idt77252.c index a876024d8a05f..63d41320cd5cf 100644 --- a/drivers/atm/idt77252.c +++ b/drivers/atm/idt77252.c @@ -852,6 +852,8 @@ queue_skb(struct idt77252_dev *card, struct vc_map *vc, IDT77252_PRV_PADDR(skb) = dma_map_single(&card->pcidev->dev, skb->data, skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(&card->pcidev->dev, IDT77252_PRV_PADDR(skb))) + return -ENOMEM; error = -EINVAL; @@ -1857,6 +1859,8 @@ add_rx_skb(struct idt77252_dev *card, int queue, paddr = dma_map_single(&card->pcidev->dev, skb->data, skb_end_pointer(skb) - skb->data, DMA_FROM_DEVICE); + if (dma_mapping_error(&card->pcidev->dev, paddr)) + goto outpoolrm; IDT77252_PRV_PADDR(skb) = paddr; if (push_rx_skb(card, skb, queue)) { @@ -1871,6 +1875,7 @@ add_rx_skb(struct idt77252_dev *card, int queue, dma_unmap_single(&card->pcidev->dev, IDT77252_PRV_PADDR(skb), skb_end_pointer(skb) - skb->data, DMA_FROM_DEVICE); +outpoolrm: handle = IDT77252_PRV_POOL(skb); card->sbpool[POOL_QUEUE(handle)].skb[POOL_INDEX(handle)] = NULL; diff --git a/drivers/auxdisplay/charlcd.c b/drivers/auxdisplay/charlcd.c index 6d309e4971b61..e243291a7e77c 100644 --- a/drivers/auxdisplay/charlcd.c +++ b/drivers/auxdisplay/charlcd.c @@ -594,18 +594,19 @@ static int charlcd_init(struct charlcd *lcd) return 0; } -struct charlcd *charlcd_alloc(void) +struct charlcd *charlcd_alloc(unsigned int drvdata_size) { struct charlcd_priv *priv; struct charlcd *lcd; - priv = kzalloc(sizeof(*priv), GFP_KERNEL); + priv = kzalloc(sizeof(*priv) + drvdata_size, GFP_KERNEL); if (!priv) return NULL; priv->esc_seq.len = -1; lcd = &priv->lcd; + lcd->drvdata = priv->drvdata; return lcd; } diff --git a/drivers/auxdisplay/charlcd.h b/drivers/auxdisplay/charlcd.h index eed80063a6d20..4bbf106b2dd8a 100644 --- a/drivers/auxdisplay/charlcd.h +++ b/drivers/auxdisplay/charlcd.h @@ -49,7 +49,7 @@ struct charlcd { unsigned long y; } addr; - void *drvdata; + void *drvdata; /* Set by charlcd_alloc() */ }; /** @@ -93,7 +93,8 @@ struct charlcd_ops { }; void charlcd_backlight(struct charlcd *lcd, enum charlcd_onoff on); -struct charlcd *charlcd_alloc(void); + +struct charlcd *charlcd_alloc(unsigned int drvdata_size); void charlcd_free(struct charlcd *lcd); int charlcd_register(struct charlcd *lcd); diff --git a/drivers/auxdisplay/hd44780.c b/drivers/auxdisplay/hd44780.c index 8b690f59df27d..ebaf0ff518f4c 100644 --- a/drivers/auxdisplay/hd44780.c +++ b/drivers/auxdisplay/hd44780.c @@ -226,7 +226,7 @@ static int hd44780_probe(struct platform_device *pdev) if (!hdc) return -ENOMEM; - lcd = charlcd_alloc(); + lcd = charlcd_alloc(0); if (!lcd) goto fail1; diff --git a/drivers/auxdisplay/lcd2s.c b/drivers/auxdisplay/lcd2s.c index 6422be0dfe20e..0ecf6a9469f24 100644 --- a/drivers/auxdisplay/lcd2s.c +++ b/drivers/auxdisplay/lcd2s.c @@ -307,7 +307,7 @@ static int lcd2s_i2c_probe(struct i2c_client *i2c) if (err < 0) return err; - lcd = charlcd_alloc(); + lcd = charlcd_alloc(0); if (!lcd) return -ENOMEM; diff --git a/drivers/auxdisplay/panel.c b/drivers/auxdisplay/panel.c index eba04c0de7eb3..0f3999b665e70 100644 --- a/drivers/auxdisplay/panel.c +++ b/drivers/auxdisplay/panel.c @@ -835,7 +835,7 @@ static void lcd_init(void) if (!hdc) return; - charlcd = charlcd_alloc(); + charlcd = charlcd_alloc(0); if (!charlcd) { kfree(hdc); return; diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 06163f87718ac..379c51576c320 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -19,6 +19,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -30,7 +31,8 @@ static DEFINE_PER_CPU(struct scale_freq_data __rcu *, sft_data); static struct cpumask scale_freq_counters_mask; static bool scale_freq_invariant; -static DEFINE_PER_CPU(u32, freq_factor) = 1; +DEFINE_PER_CPU(unsigned long, capacity_freq_ref) = 1; +EXPORT_PER_CPU_SYMBOL_GPL(capacity_freq_ref); static bool supports_scale_freq_counters(const struct cpumask *cpus) { @@ -183,9 +185,9 @@ EXPORT_PER_CPU_SYMBOL_GPL(thermal_pressure); * operating on stale data when hot-plug is used for some CPUs. The * @capped_freq reflects the currently allowed max CPUs frequency due to * thermal capping. It might be also a boost frequency value, which is bigger - * than the internal 'freq_factor' max frequency. In such case the pressure - * value should simply be removed, since this is an indication that there is - * no thermal throttling. The @capped_freq must be provided in kHz. + * than the internal 'capacity_freq_ref' max frequency. In such case the + * pressure value should simply be removed, since this is an indication that + * there is no thermal throttling. The @capped_freq must be provided in kHz. */ void topology_update_thermal_pressure(const struct cpumask *cpus, unsigned long capped_freq) @@ -196,10 +198,7 @@ void topology_update_thermal_pressure(const struct cpumask *cpus, cpu = cpumask_first(cpus); max_capacity = arch_scale_cpu_capacity(cpu); - max_freq = per_cpu(freq_factor, cpu); - - /* Convert to MHz scale which is used in 'freq_factor' */ - capped_freq /= 1000; + max_freq = arch_scale_freq_ref(cpu); /* * Handle properly the boost frequencies, which should simply clean @@ -301,13 +300,13 @@ void topology_normalize_cpu_scale(void) capacity_scale = 1; for_each_possible_cpu(cpu) { - capacity = raw_capacity[cpu] * per_cpu(freq_factor, cpu); + capacity = raw_capacity[cpu] * per_cpu(capacity_freq_ref, cpu); capacity_scale = max(capacity, capacity_scale); } pr_debug("cpu_capacity: capacity_scale=%llu\n", capacity_scale); for_each_possible_cpu(cpu) { - capacity = raw_capacity[cpu] * per_cpu(freq_factor, cpu); + capacity = raw_capacity[cpu] * per_cpu(capacity_freq_ref, cpu); capacity = div64_u64(capacity << SCHED_CAPACITY_SHIFT, capacity_scale); topology_set_cpu_scale(cpu, capacity); @@ -343,15 +342,15 @@ bool __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu) cpu_node, raw_capacity[cpu]); /* - * Update freq_factor for calculating early boot cpu capacities. + * Update capacity_freq_ref for calculating early boot CPU capacities. * For non-clk CPU DVFS mechanism, there's no way to get the * frequency value now, assuming they are running at the same - * frequency (by keeping the initial freq_factor value). + * frequency (by keeping the initial capacity_freq_ref value). */ cpu_clk = of_clk_get(cpu_node, 0); - if (!PTR_ERR_OR_ZERO(cpu_clk)) { - per_cpu(freq_factor, cpu) = - clk_get_rate(cpu_clk) / 1000; + if (!IS_ERR_OR_NULL(cpu_clk)) { + per_cpu(capacity_freq_ref, cpu) = + clk_get_rate(cpu_clk) / HZ_PER_KHZ; clk_put(cpu_clk); } } else { @@ -367,11 +366,16 @@ bool __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu) return !ret; } +void __weak freq_inv_set_max_ratio(int cpu, u64 max_rate) +{ +} + #ifdef CONFIG_ACPI_CPPC_LIB #include void topology_init_cpu_capacity_cppc(void) { + u64 capacity, capacity_scale = 0; struct cppc_perf_caps perf_caps; int cpu; @@ -388,6 +392,10 @@ void topology_init_cpu_capacity_cppc(void) (perf_caps.highest_perf >= perf_caps.nominal_perf) && (perf_caps.highest_perf >= perf_caps.lowest_perf)) { raw_capacity[cpu] = perf_caps.highest_perf; + capacity_scale = max_t(u64, capacity_scale, raw_capacity[cpu]); + + per_cpu(capacity_freq_ref, cpu) = cppc_perf_to_khz(&perf_caps, raw_capacity[cpu]); + pr_debug("cpu_capacity: CPU%d cpu_capacity=%u (raw).\n", cpu, raw_capacity[cpu]); continue; @@ -398,7 +406,18 @@ void topology_init_cpu_capacity_cppc(void) goto exit; } - topology_normalize_cpu_scale(); + for_each_possible_cpu(cpu) { + freq_inv_set_max_ratio(cpu, + per_cpu(capacity_freq_ref, cpu) * HZ_PER_KHZ); + + capacity = raw_capacity[cpu]; + capacity = div64_u64(capacity << SCHED_CAPACITY_SHIFT, + capacity_scale); + topology_set_cpu_scale(cpu, capacity); + pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n", + cpu, topology_get_cpu_scale(cpu)); + } + schedule_work(&update_topology_flags_work); pr_debug("cpu_capacity: cpu_capacity initialization done\n"); @@ -420,9 +439,6 @@ init_cpu_capacity_callback(struct notifier_block *nb, struct cpufreq_policy *policy = data; int cpu; - if (!raw_capacity) - return 0; - if (val != CPUFREQ_CREATE_POLICY) return 0; @@ -432,13 +448,18 @@ init_cpu_capacity_callback(struct notifier_block *nb, cpumask_andnot(cpus_to_visit, cpus_to_visit, policy->related_cpus); - for_each_cpu(cpu, policy->related_cpus) - per_cpu(freq_factor, cpu) = policy->cpuinfo.max_freq / 1000; + for_each_cpu(cpu, policy->related_cpus) { + per_cpu(capacity_freq_ref, cpu) = policy->cpuinfo.max_freq; + freq_inv_set_max_ratio(cpu, + per_cpu(capacity_freq_ref, cpu) * HZ_PER_KHZ); + } if (cpumask_empty(cpus_to_visit)) { - topology_normalize_cpu_scale(); - schedule_work(&update_topology_flags_work); - free_raw_capacity(); + if (raw_capacity) { + topology_normalize_cpu_scale(); + schedule_work(&update_topology_flags_work); + free_raw_capacity(); + } pr_debug("cpu_capacity: parsing done\n"); schedule_work(&parsing_done_work); } @@ -458,7 +479,7 @@ static int __init register_cpufreq_notifier(void) * On ACPI-based systems skip registering cpufreq notifier as cpufreq * information is not needed for cpu capacity initialization. */ - if (!acpi_disabled || !raw_capacity) + if (!acpi_disabled) return -EINVAL; if (!alloc_cpumask_var(&cpus_to_visit, GFP_KERNEL)) diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index ef427ee787a99..801a4b4f90260 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -566,6 +566,9 @@ CPU_SHOW_VULN_FALLBACK(retbleed); CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow); CPU_SHOW_VULN_FALLBACK(gds); CPU_SHOW_VULN_FALLBACK(reg_file_data_sampling); +CPU_SHOW_VULN_FALLBACK(indirect_target_selection); +CPU_SHOW_VULN_FALLBACK(tsa); +CPU_SHOW_VULN_FALLBACK(vmscape); static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL); static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL); @@ -581,6 +584,9 @@ static DEVICE_ATTR(retbleed, 0444, cpu_show_retbleed, NULL); static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NULL); static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL); static DEVICE_ATTR(reg_file_data_sampling, 0444, cpu_show_reg_file_data_sampling, NULL); +static DEVICE_ATTR(indirect_target_selection, 0444, cpu_show_indirect_target_selection, NULL); +static DEVICE_ATTR(tsa, 0444, cpu_show_tsa, NULL); +static DEVICE_ATTR(vmscape, 0444, cpu_show_vmscape, NULL); static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_meltdown.attr, @@ -597,6 +603,9 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_spec_rstack_overflow.attr, &dev_attr_gather_data_sampling.attr, &dev_attr_reg_file_data_sampling.attr, + &dev_attr_indirect_target_selection.attr, + &dev_attr_tsa.attr, + &dev_attr_vmscape.attr, NULL }; diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c index 7e2d1f0d903a6..6e6bf8be00664 100644 --- a/drivers/base/devcoredump.c +++ b/drivers/base/devcoredump.c @@ -26,50 +26,46 @@ struct devcd_entry { void *data; size_t datalen; /* - * Here, mutex is required to serialize the calls to del_wk work between - * user/kernel space which happens when devcd is added with device_add() - * and that sends uevent to user space. User space reads the uevents, - * and calls to devcd_data_write() which try to modify the work which is - * not even initialized/queued from devcoredump. + * There are 2 races for which mutex is required. * + * The first race is between device creation and userspace writing to + * schedule immediately destruction. * + * This race is handled by arming the timer before device creation, but + * when device creation fails the timer still exists. * - * cpu0(X) cpu1(Y) + * To solve this, hold the mutex during device_add(), and set + * init_completed on success before releasing the mutex. * - * dev_coredump() uevent sent to user space - * device_add() ======================> user space process Y reads the - * uevents writes to devcd fd - * which results into writes to + * That way the timer will never fire until device_add() is called, + * it will do nothing if init_completed is not set. The timer is also + * cancelled in that case. * - * devcd_data_write() - * mod_delayed_work() - * try_to_grab_pending() - * del_timer() - * debug_assert_init() - * INIT_DELAYED_WORK() - * schedule_delayed_work() - * - * - * Also, mutex alone would not be enough to avoid scheduling of - * del_wk work after it get flush from a call to devcd_free() - * mentioned as below. - * - * disabled_store() - * devcd_free() - * mutex_lock() devcd_data_write() - * flush_delayed_work() - * mutex_unlock() - * mutex_lock() - * mod_delayed_work() - * mutex_unlock() - * So, delete_work flag is required. + * The second race involves multiple parallel invocations of devcd_free(), + * add a deleted flag so only 1 can call the destructor. */ struct mutex mutex; - bool delete_work; + bool init_completed, deleted; struct module *owner; ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen); void (*free)(void *data); + /* + * If nothing interferes and device_add() was returns success, + * del_wk will destroy the device after the timer fires. + * + * Multiple userspace processes can interfere in the working of the timer: + * - Writing to the coredump will reschedule the timer to run immediately, + * if still armed. + * + * This is handled by using "if (cancel_delayed_work()) { + * schedule_delayed_work() }", to prevent re-arming after having + * been previously fired. + * - Writing to /sys/class/devcoredump/disabled will destroy the + * coredump synchronously. + * This is handled by using disable_delayed_work_sync(), and then + * checking if deleted flag is set with &devcd->mutex held. + */ struct delayed_work del_wk; struct device *failing_dev; }; @@ -98,14 +94,27 @@ static void devcd_dev_release(struct device *dev) kfree(devcd); } +static void __devcd_del(struct devcd_entry *devcd) +{ + devcd->deleted = true; + device_del(&devcd->devcd_dev); + put_device(&devcd->devcd_dev); +} + static void devcd_del(struct work_struct *wk) { struct devcd_entry *devcd; + bool init_completed; devcd = container_of(wk, struct devcd_entry, del_wk.work); - device_del(&devcd->devcd_dev); - put_device(&devcd->devcd_dev); + /* devcd->mutex serializes against dev_coredumpm_timeout */ + mutex_lock(&devcd->mutex); + init_completed = devcd->init_completed; + mutex_unlock(&devcd->mutex); + + if (init_completed) + __devcd_del(devcd); } static ssize_t devcd_data_read(struct file *filp, struct kobject *kobj, @@ -125,12 +134,12 @@ static ssize_t devcd_data_write(struct file *filp, struct kobject *kobj, struct device *dev = kobj_to_dev(kobj); struct devcd_entry *devcd = dev_to_devcd(dev); - mutex_lock(&devcd->mutex); - if (!devcd->delete_work) { - devcd->delete_work = true; - mod_delayed_work(system_wq, &devcd->del_wk, 0); - } - mutex_unlock(&devcd->mutex); + /* + * Although it's tempting to use mod_delayed work here, + * that will cause a reschedule if the timer already fired. + */ + if (cancel_delayed_work(&devcd->del_wk)) + schedule_delayed_work(&devcd->del_wk, 0); return count; } @@ -158,11 +167,21 @@ static int devcd_free(struct device *dev, void *data) { struct devcd_entry *devcd = dev_to_devcd(dev); + /* + * To prevent a race with devcd_data_write(), cancel work and + * complete manually instead. + * + * We cannot rely on the return value of + * cancel_delayed_work_sync() here, because it might be in the + * middle of a cancel_delayed_work + schedule_delayed_work pair. + * + * devcd->mutex here guards against multiple parallel invocations + * of devcd_free(). + */ + cancel_delayed_work_sync(&devcd->del_wk); mutex_lock(&devcd->mutex); - if (!devcd->delete_work) - devcd->delete_work = true; - - flush_delayed_work(&devcd->del_wk); + if (!devcd->deleted) + __devcd_del(devcd); mutex_unlock(&devcd->mutex); return 0; } @@ -186,12 +205,10 @@ static ssize_t disabled_show(const struct class *class, const struct class_attri * put_device() <- last reference * error = fn(dev, data) devcd_dev_release() * devcd_free(dev, data) kfree(devcd) - * mutex_lock(&devcd->mutex); * * - * In the above diagram, It looks like disabled_store() would be racing with parallely - * running devcd_del() and result in memory abort while acquiring devcd->mutex which - * is called after kfree of devcd memory after dropping its last reference with + * In the above diagram, it looks like disabled_store() would be racing with parallelly + * running devcd_del() and result in memory abort after dropping its last reference with * put_device(). However, this will not happens as fn(dev, data) runs * with its own reference to device via klist_node so it is not its last reference. * so, above situation would not occur. @@ -352,7 +369,7 @@ void dev_coredumpm(struct device *dev, struct module *owner, devcd->read = read; devcd->free = free; devcd->failing_dev = get_device(dev); - devcd->delete_work = false; + devcd->deleted = false; mutex_init(&devcd->mutex); device_initialize(&devcd->devcd_dev); @@ -361,8 +378,14 @@ void dev_coredumpm(struct device *dev, struct module *owner, atomic_inc_return(&devcd_count)); devcd->devcd_dev.class = &devcd_class; - mutex_lock(&devcd->mutex); dev_set_uevent_suppress(&devcd->devcd_dev, true); + + /* devcd->mutex prevents devcd_del() completing until init finishes */ + mutex_lock(&devcd->mutex); + devcd->init_completed = false; + INIT_DELAYED_WORK(&devcd->del_wk, devcd_del); + schedule_delayed_work(&devcd->del_wk, DEVCD_TIMEOUT); + if (device_add(&devcd->devcd_dev)) goto put_device; @@ -379,13 +402,20 @@ void dev_coredumpm(struct device *dev, struct module *owner, dev_set_uevent_suppress(&devcd->devcd_dev, false); kobject_uevent(&devcd->devcd_dev.kobj, KOBJ_ADD); - INIT_DELAYED_WORK(&devcd->del_wk, devcd_del); - schedule_delayed_work(&devcd->del_wk, DEVCD_TIMEOUT); + + /* + * Safe to run devcd_del() now that we are done with devcd_dev. + * Alternatively we could have taken a ref on devcd_dev before + * dropping the lock. + */ + devcd->init_completed = true; mutex_unlock(&devcd->mutex); return; put_device: - put_device(&devcd->devcd_dev); mutex_unlock(&devcd->mutex); + cancel_delayed_work_sync(&devcd->del_wk); + put_device(&devcd->devcd_dev); + put_module: module_put(owner); free: diff --git a/drivers/base/module.c b/drivers/base/module.c index a33663d92256d..955582b34e54a 100644 --- a/drivers/base/module.c +++ b/drivers/base/module.c @@ -42,16 +42,13 @@ int module_add_driver(struct module *mod, struct device_driver *drv) if (mod) mk = &mod->mkobj; else if (drv->mod_name) { - struct kobject *mkobj; - - /* Lookup built-in module entry in /sys/modules */ - mkobj = kset_find_obj(module_kset, drv->mod_name); - if (mkobj) { - mk = container_of(mkobj, struct module_kobject, kobj); + /* Lookup or create built-in module entry in /sys/modules */ + mk = lookup_or_create_module_kobject(drv->mod_name); + if (mk) { /* remember our module structure */ drv->p->mkobj = mk; - /* kset_find_obj took a reference */ - kobject_put(mkobj); + /* lookup_or_create_module_kobject took a reference */ + kobject_put(&mk->kobj); } } diff --git a/drivers/base/node.c b/drivers/base/node.c index 4d588f4658c85..2b398c8a0f06c 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -74,14 +74,14 @@ static BIN_ATTR_RO(cpulist, CPULIST_FILE_MAX_BYTES); * @dev: Device for this memory access class * @list_node: List element in the node's access list * @access: The access class rank - * @hmem_attrs: Heterogeneous memory performance attributes + * @coord: Heterogeneous memory performance coordinates */ struct node_access_nodes { struct device dev; struct list_head list_node; unsigned int access; #ifdef CONFIG_HMEM_REPORTING - struct node_hmem_attrs hmem_attrs; + struct access_coordinate coord; #endif }; #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev) @@ -126,7 +126,7 @@ static void node_access_release(struct device *dev) } static struct node_access_nodes *node_init_node_access(struct node *node, - unsigned int access) + enum access_coordinate_class access) { struct node_access_nodes *access_node; struct device *dev; @@ -167,7 +167,7 @@ static ssize_t property##_show(struct device *dev, \ char *buf) \ { \ return sysfs_emit(buf, "%u\n", \ - to_access_nodes(dev)->hmem_attrs.property); \ + to_access_nodes(dev)->coord.property); \ } \ static DEVICE_ATTR_RO(property) @@ -187,11 +187,11 @@ static struct attribute *access_attrs[] = { /** * node_set_perf_attrs - Set the performance values for given access class * @nid: Node identifier to be set - * @hmem_attrs: Heterogeneous memory performance attributes + * @coord: Heterogeneous memory performance coordinates * @access: The access class the for the given attributes */ -void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs, - unsigned int access) +void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access) { struct node_access_nodes *c; struct node *node; @@ -205,7 +205,7 @@ void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs, if (!c) return; - c->hmem_attrs = *hmem_attrs; + c->coord = *coord; for (i = 0; access_attrs[i] != NULL; i++) { if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i], "initiators")) { @@ -689,7 +689,7 @@ int register_cpu_under_node(unsigned int cpu, unsigned int nid) */ int register_memory_node_under_compute_node(unsigned int mem_nid, unsigned int cpu_nid, - unsigned int access) + enum access_coordinate_class access) { struct node *init_node, *targ_node; struct node_access_nodes *initiator, *target; @@ -878,6 +878,10 @@ int __register_one_node(int nid) node_devices[nid] = node; error = register_node(node_devices[nid], nid); + if (error) { + node_devices[nid] = NULL; + return error; + } /* link cpu under this node */ for_each_present_cpu(cpu) { diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index daaeea4d401e6..043f62f790a46 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2866,7 +2866,7 @@ struct device *genpd_dev_pm_attach_by_id(struct device *dev, /* Verify that the index is within a valid range. */ num_domains = of_count_phandle_with_args(dev->of_node, "power-domains", "#power-domain-cells"); - if (index >= num_domains) + if (num_domains < 0 || index >= num_domains) return NULL; /* Allocate and register device on the genpd bus. */ diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c index cc2c3a5a6d35a..4f09f9a5a0431 100644 --- a/drivers/base/power/domain_governor.c +++ b/drivers/base/power/domain_governor.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -345,6 +346,8 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) struct cpuidle_device *dev; ktime_t domain_wakeup, next_hrtimer; ktime_t now = ktime_get(); + struct device *cpu_dev; + s64 cpu_constraint, global_constraint; s64 idle_duration_ns; int cpu, i; @@ -355,6 +358,7 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) return true; + global_constraint = cpu_latency_qos_limit(); /* * Find the next wakeup for any of the online CPUs within the PM domain * and its subdomains. Note, we only need the genpd->cpus, as it already @@ -368,8 +372,16 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) if (ktime_before(next_hrtimer, domain_wakeup)) domain_wakeup = next_hrtimer; } + + cpu_dev = get_cpu_device(cpu); + if (cpu_dev) { + cpu_constraint = dev_pm_qos_raw_resume_latency(cpu_dev); + if (cpu_constraint < global_constraint) + global_constraint = cpu_constraint; + } } + global_constraint *= NSEC_PER_USEC; /* The minimum idle duration is from now - until the next wakeup. */ idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); if (idle_duration_ns <= 0) @@ -385,8 +397,10 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) */ i = genpd->state_idx; do { - if (idle_duration_ns >= (genpd->states[i].residency_ns + - genpd->states[i].power_off_latency_ns)) { + if ((idle_duration_ns >= (genpd->states[i].residency_ns + + genpd->states[i].power_off_latency_ns)) && + (global_constraint >= (genpd->states[i].power_on_latency_ns + + genpd->states[i].power_off_latency_ns))) { genpd->state_idx = i; return true; } diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 16b48dec992ab..3873e332b8abd 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -606,8 +606,20 @@ static void __device_resume_noirq(struct device *dev, pm_message_t state, bool a if (dev->power.syscore || dev->power.direct_complete) goto Out; - if (!dev->power.is_noirq_suspended) + if (!dev->power.is_noirq_suspended) { + /* + * This means that system suspend has been aborted in the noirq + * phase before invoking the noirq suspend callback for the + * device, so if device_suspend_late() has left it in suspend, + * device_resume_early() should leave it in suspend either in + * case the early resume of it depends on the noirq resume that + * has not run. + */ + if (dev_pm_skip_suspend(dev)) + dev->power.must_resume = false; + goto Out; + } if (!dpm_wait_for_superior(dev, async)) goto Out; @@ -903,6 +915,8 @@ static void __device_resume(struct device *dev, pm_message_t state, bool async) if (!dev->power.is_suspended) goto Complete; + dev->power.is_suspended = false; + if (dev->power.direct_complete) { /* Match the pm_runtime_disable() in __device_suspend(). */ pm_runtime_enable(dev); @@ -958,7 +972,6 @@ static void __device_resume(struct device *dev, pm_message_t state, bool async) End: error = dpm_run_callback(callback, dev, state, info); - dev->power.is_suspended = false; device_unlock(dev); dpm_watchdog_clear(&wd); diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index 0489db3a1f606..b7fbbd88b4a32 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "power.h" @@ -437,8 +438,11 @@ static int __dev_pm_qos_update_request(struct dev_pm_qos_request *req, trace_dev_pm_qos_update_request(dev_name(req->dev), req->type, new_value); - if (curr_value != new_value) + if (curr_value != new_value) { + if (req->type == DEV_PM_QOS_MAX_FREQUENCY) + trace_android_vh_fas_gpu_qos_update_tracer(&req->data.freq, &new_value); ret = apply_constraint(req, PM_QOS_UPDATE_REQ, new_value); + } return ret; } diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c old mode 100644 new mode 100755 index 20975a271b130..0ec0295274d1b --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1513,6 +1513,32 @@ void pm_runtime_enable(struct device *dev) } EXPORT_SYMBOL_GPL(pm_runtime_enable); +static void pm_runtime_set_suspended_action(void *data) +{ + pm_runtime_set_suspended(data); +} + +/** + * devm_pm_runtime_set_active_enabled - set_active version of devm_pm_runtime_enable. + * + * @dev: Device to handle. + */ +int devm_pm_runtime_set_active_enabled(struct device *dev) +{ + int err; + + err = pm_runtime_set_active(dev); + if (err) + return err; + + err = devm_add_action_or_reset(dev, pm_runtime_set_suspended_action, dev); + if (err) + return err; + + return devm_pm_runtime_enable(dev); +} +EXPORT_SYMBOL_GPL(devm_pm_runtime_set_active_enabled); + static void pm_runtime_disable_action(void *data) { pm_runtime_dont_use_autosuspend(data); @@ -1535,6 +1561,24 @@ int devm_pm_runtime_enable(struct device *dev) } EXPORT_SYMBOL_GPL(devm_pm_runtime_enable); +static void pm_runtime_put_noidle_action(void *data) +{ + pm_runtime_put_noidle(data); +} + +/** + * devm_pm_runtime_get_noresume - devres-enabled version of pm_runtime_get_noresume. + * + * @dev: Device to handle. + */ +int devm_pm_runtime_get_noresume(struct device *dev) +{ + pm_runtime_get_noresume(dev); + + return devm_add_action_or_reset(dev, pm_runtime_put_noidle_action, dev); +} +EXPORT_SYMBOL_GPL(devm_pm_runtime_get_noresume); + /** * pm_runtime_forbid - Block runtime PM of a device. * @dev: Device to handle. @@ -1744,17 +1788,24 @@ void pm_runtime_init(struct device *dev) */ void pm_runtime_reinit(struct device *dev) { - if (!pm_runtime_enabled(dev)) { - if (dev->power.runtime_status == RPM_ACTIVE) - pm_runtime_set_suspended(dev); - if (dev->power.irq_safe) { - spin_lock_irq(&dev->power.lock); - dev->power.irq_safe = 0; - spin_unlock_irq(&dev->power.lock); - if (dev->parent) - pm_runtime_put(dev->parent); - } + if (pm_runtime_enabled(dev)) + return; + + if (dev->power.runtime_status == RPM_ACTIVE) + pm_runtime_set_suspended(dev); + + if (dev->power.irq_safe) { + spin_lock_irq(&dev->power.lock); + dev->power.irq_safe = 0; + spin_unlock_irq(&dev->power.lock); + if (dev->parent) + pm_runtime_put(dev->parent); } + /* + * Clear power.needs_force_resume in case it has been set by + * pm_runtime_force_suspend() invoked from a driver remove callback. + */ + dev->power.needs_force_resume = false; } /** diff --git a/drivers/base/regmap/regmap-slimbus.c b/drivers/base/regmap/regmap-slimbus.c index 8075db788b39a..6a864433cdd9b 100644 --- a/drivers/base/regmap/regmap-slimbus.c +++ b/drivers/base/regmap/regmap-slimbus.c @@ -48,8 +48,7 @@ struct regmap *__regmap_init_slimbus(struct slim_device *slimbus, if (IS_ERR(bus)) return ERR_CAST(bus); - return __regmap_init(&slimbus->dev, bus, &slimbus->dev, config, - lock_key, lock_name); + return __regmap_init(&slimbus->dev, bus, slimbus, config, lock_key, lock_name); } EXPORT_SYMBOL_GPL(__regmap_init_slimbus); @@ -63,8 +62,7 @@ struct regmap *__devm_regmap_init_slimbus(struct slim_device *slimbus, if (IS_ERR(bus)) return ERR_CAST(bus); - return __devm_regmap_init(&slimbus->dev, bus, &slimbus, config, - lock_key, lock_name); + return __devm_regmap_init(&slimbus->dev, bus, slimbus, config, lock_key, lock_name); } EXPORT_SYMBOL_GPL(__devm_regmap_init_slimbus); diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 07dc53a9f2a58..51f812d2b6d99 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -826,7 +826,7 @@ struct regmap *__regmap_init(struct device *dev, map->read_flag_mask = bus->read_flag_mask; } - if (config && config->read && config->write) { + if (config->read && config->write) { map->reg_read = _regmap_bus_read; if (config->reg_update_bits) map->reg_update_bits = config->reg_update_bits; @@ -1172,6 +1172,8 @@ struct regmap *__regmap_init(struct device *dev, err_map: kfree(map); err: + if (bus && bus->free_on_exit) + kfree(bus); return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(__regmap_init); diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index 079bd14bdedc7..a7a3e3b66bb5e 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -518,7 +518,7 @@ software_node_get_reference_args(const struct fwnode_handle *fwnode, if (prop->is_inline) return -EINVAL; - if (index * sizeof(*ref) >= prop->length) + if ((index + 1) * sizeof(*ref) > prop->length) return -ENOENT; ref_array = prop->pointer; diff --git a/drivers/bcma/main.c b/drivers/bcma/main.c index 7061d3ee836a1..c69c05256b59f 100644 --- a/drivers/bcma/main.c +++ b/drivers/bcma/main.c @@ -294,6 +294,8 @@ static int bcma_register_devices(struct bcma_bus *bus) int err; list_for_each_entry(core, &bus->cores, list) { + struct device_node *np; + /* We support that core ourselves */ switch (core->id.id) { case BCMA_CORE_4706_CHIPCOMMON: @@ -311,6 +313,10 @@ static int bcma_register_devices(struct bcma_bus *bus) if (bcma_is_core_needed_early(core->id.id)) continue; + np = core->dev.of_node; + if (np && !of_device_is_available(np)) + continue; + /* Only first GMAC core on BCM4706 is connected and working */ if (core->id.id == BCMA_CORE_4706_MAC_GBIT && core->core_unit > 0) diff --git a/drivers/block/aoe/aoe.h b/drivers/block/aoe/aoe.h index 749ae1246f4cf..d35caa3c69e15 100644 --- a/drivers/block/aoe/aoe.h +++ b/drivers/block/aoe/aoe.h @@ -80,6 +80,7 @@ enum { DEVFL_NEWSIZE = (1<<6), /* need to update dev size in block layer */ DEVFL_FREEING = (1<<7), /* set when device is being cleaned up */ DEVFL_FREED = (1<<8), /* device has been cleaned up */ + DEVFL_DEAD = (1<<9), /* device has timed out of aoe_deadsecs */ }; enum { diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c index d1f4ddc576451..c4c5cf1ec71ba 100644 --- a/drivers/block/aoe/aoecmd.c +++ b/drivers/block/aoe/aoecmd.c @@ -754,7 +754,7 @@ rexmit_timer(struct timer_list *timer) utgts = count_targets(d, NULL); - if (d->flags & DEVFL_TKILL) { + if (d->flags & (DEVFL_TKILL | DEVFL_DEAD)) { spin_unlock_irqrestore(&d->lock, flags); return; } @@ -786,7 +786,8 @@ rexmit_timer(struct timer_list *timer) * to clean up. */ list_splice(&flist, &d->factive[0]); - aoedev_downdev(d); + d->flags |= DEVFL_DEAD; + queue_work(aoe_wq, &d->work); goto out; } @@ -898,6 +899,9 @@ aoecmd_sleepwork(struct work_struct *work) { struct aoedev *d = container_of(work, struct aoedev, work); + if (d->flags & DEVFL_DEAD) + aoedev_downdev(d); + if (d->flags & DEVFL_GDALLOC) aoeblk_gdalloc(d); diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c index 3523dd82d7a00..4240e11adfb76 100644 --- a/drivers/block/aoe/aoedev.c +++ b/drivers/block/aoe/aoedev.c @@ -198,9 +198,13 @@ aoedev_downdev(struct aoedev *d) { struct aoetgt *t, **tt, **te; struct list_head *head, *pos, *nx; + struct request *rq, *rqnext; int i; + unsigned long flags; - d->flags &= ~DEVFL_UP; + spin_lock_irqsave(&d->lock, flags); + d->flags &= ~(DEVFL_UP | DEVFL_DEAD); + spin_unlock_irqrestore(&d->lock, flags); /* clean out active and to-be-retransmitted buffers */ for (i = 0; i < NFACTIVE; i++) { @@ -223,6 +227,13 @@ aoedev_downdev(struct aoedev *d) /* clean out the in-process request (if any) */ aoe_failip(d); + /* clean out any queued block requests */ + list_for_each_entry_safe(rq, rqnext, &d->rq_list, queuelist) { + list_del_init(&rq->queuelist); + blk_mq_start_request(rq); + blk_mq_end_request(rq, BLK_STS_IOERR); + } + /* fast fail all pending I/O */ if (d->blkq) { /* UP is cleared, freeze+quiesce to insure all are errored */ diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 0c9f54197768d..ac18d36b0ea84 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -2500,7 +2500,11 @@ static int handle_write_conflicts(struct drbd_device *device, peer_req->w.cb = superseded ? e_send_superseded : e_send_retry_write; list_add_tail(&peer_req->w.list, &device->done_ee); - queue_work(connection->ack_sender, &peer_req->peer_device->send_acks_work); + /* put is in drbd_send_acks_wf() */ + kref_get(&device->kref); + if (!queue_work(connection->ack_sender, + &peer_req->peer_device->send_acks_work)) + kref_put(&device->kref, drbd_destroy_device); err = -ENOENT; goto out; diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 37c67bf30bee6..dfc7d3af4ebe5 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -36,6 +36,7 @@ #include #include #include +#include /* Possible states of device */ enum { @@ -830,8 +831,12 @@ static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd) struct loop_worker *cur_worker, *worker = NULL; struct work_struct *work; struct list_head *cmd_list; + bool skip = false; spin_lock_irq(&lo->lo_work_lock); + trace_android_vh_loop_skip_queue_work(blk_mq_rq_from_pdu(cmd), &skip); + if (skip) + goto skip_queue_work; if (queue_on_root_worker(cmd->blkcg_css)) goto queue_work; @@ -891,6 +896,7 @@ static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd) } list_add_tail(&cmd->list_entry, cmd_list); queue_work(lo->workqueue, work); +skip_queue_work: spin_unlock_irq(&lo->lo_work_lock); } @@ -1472,19 +1478,36 @@ static int loop_set_dio(struct loop_device *lo, unsigned long arg) return error; } -static int loop_set_block_size(struct loop_device *lo, unsigned long arg) +static int loop_set_block_size(struct loop_device *lo, blk_mode_t mode, + struct block_device *bdev, unsigned long arg) { int err = 0; - if (lo->lo_state != Lo_bound) - return -ENXIO; + /* + * If we don't hold exclusive handle for the device, upgrade to it + * here to avoid changing device under exclusive owner. + */ + if (!(mode & BLK_OPEN_EXCL)) { + err = bd_prepare_to_claim(bdev, loop_set_block_size, NULL); + if (err) + return err; + } + + err = mutex_lock_killable(&lo->lo_mutex); + if (err) + goto abort_claim; + + if (lo->lo_state != Lo_bound) { + err = -ENXIO; + goto unlock; + } err = blk_validate_block_size(arg); if (err) - return err; + goto unlock; if (lo->lo_queue->limits.logical_block_size == arg) - return 0; + goto unlock; sync_blockdev(lo->lo_device); invalidate_bdev(lo->lo_device); @@ -1496,6 +1519,11 @@ static int loop_set_block_size(struct loop_device *lo, unsigned long arg) loop_update_dio(lo); blk_mq_unfreeze_queue(lo->lo_queue); +unlock: + mutex_unlock(&lo->lo_mutex); +abort_claim: + if (!(mode & BLK_OPEN_EXCL)) + bd_abort_claiming(bdev, loop_set_block_size); return err; } @@ -1514,9 +1542,6 @@ static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd, case LOOP_SET_DIRECT_IO: err = loop_set_dio(lo, arg); break; - case LOOP_SET_BLOCK_SIZE: - err = loop_set_block_size(lo, arg); - break; default: err = -EINVAL; } @@ -1571,9 +1596,12 @@ static int lo_ioctl(struct block_device *bdev, blk_mode_t mode, break; case LOOP_GET_STATUS64: return loop_get_status64(lo, argp); + case LOOP_SET_BLOCK_SIZE: + if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + return loop_set_block_size(lo, mode, bdev, arg); case LOOP_SET_CAPACITY: case LOOP_SET_DIRECT_IO: - case LOOP_SET_BLOCK_SIZE: if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN)) return -EPERM; fallthrough; @@ -1964,6 +1992,13 @@ static void loop_process_work(struct loop_worker *worker, current->flags = orig_flags; } +void loop_process_cmd_list(struct list_head *cmd_list, struct gendisk *disk) +{ + struct loop_device *lo = disk->private_data; + loop_process_work(NULL, cmd_list, lo); +} +EXPORT_SYMBOL_GPL(loop_process_cmd_list); + static void loop_workfn(struct work_struct *work) { struct loop_worker *worker = diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 2203686156bfe..27a05b1521f69 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1128,6 +1128,14 @@ static struct socket *nbd_get_socket(struct nbd_device *nbd, unsigned long fd, if (!sock) return NULL; + if (!sk_is_tcp(sock->sk) && + !sk_is_stream_unix(sock->sk)) { + dev_err(disk_to_dev(nbd->disk), "Unsupported socket: should be TCP or UNIX.\n"); + *err = -EINVAL; + sockfd_put(sock); + return NULL; + } + if (sock->ops->shutdown == sock_no_shutdown) { dev_err(disk_to_dev(nbd->disk), "Unsupported socket: shutdown callout must be supported.\n"); *err = -EINVAL; @@ -2120,9 +2128,7 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) goto out; } } - ret = nbd_start_device(nbd); - if (ret) - goto out; + if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) { nbd->backend = nla_strdup(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER], GFP_KERNEL); @@ -2138,6 +2144,8 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) goto out; } set_bit(NBD_RT_HAS_BACKEND_FILE, &config->runtime_flags); + + ret = nbd_start_device(nbd); out: mutex_unlock(&nbd->config_lock); if (!ret) { diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index f771d1d02b55f..fc72dc60cc4e9 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -215,7 +215,7 @@ MODULE_PARM_DESC(discard, "Support discard operations (requires memory-backed nu static unsigned long g_cache_size; module_param_named(cache_size, g_cache_size, ulong, 0444); -MODULE_PARM_DESC(mbps, "Cache size in MiB for memory-backed device. Default: 0 (none)"); +MODULE_PARM_DESC(cache_size, "Cache size in MiB for memory-backed device. Default: 0 (none)"); static unsigned int g_mbps; module_param_named(mbps, g_mbps, uint, 0444); diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 7bf4b48e2282e..a379a37c94498 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -956,8 +956,10 @@ static bool vdc_port_mpgroup_check(struct vio_dev *vdev) dev = device_find_child(vdev->dev.parent, &port_data, vdc_device_probed); - if (dev) + if (dev) { + put_device(dev); return true; + } return false; } diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 95095500f93af..ec6d7a08104d9 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2030,7 +2030,7 @@ static void ublk_deinit_queues(struct ublk_device *ub) for (i = 0; i < nr_queues; i++) ublk_deinit_queue(ub, i); - kfree(ub->__queues); + kvfree(ub->__queues); } static int ublk_init_queues(struct ublk_device *ub) @@ -2041,7 +2041,7 @@ static int ublk_init_queues(struct ublk_device *ub) int i, ret = -ENOMEM; ub->queue_size = ubq_size; - ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL); + ub->__queues = kvcalloc(nr_queues, ubq_size, GFP_KERNEL); if (!ub->__queues) return ret; @@ -2323,6 +2323,10 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd) if (copy_from_user(&info, argp, sizeof(info))) return -EFAULT; + if (info.queue_depth > UBLK_MAX_QUEUE_DEPTH || !info.queue_depth || + info.nr_hw_queues > UBLK_MAX_NR_QUEUES || !info.nr_hw_queues) + return -EINVAL; + if (capable(CAP_SYS_ADMIN)) info.flags &= ~UBLK_F_UNPRIVILEGED_DEV; else if (!(info.flags & UBLK_F_UNPRIVILEGED_DEV)) diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index f9a3444753c2b..97659b4792e69 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -1257,6 +1257,12 @@ static void btmtksdio_cmd_timeout(struct hci_dev *hdev) sdio_claim_host(bdev->func); + /* set drv_pmctrl if BT is closed before doing reset */ + if (!test_bit(BTMTKSDIO_FUNC_ENABLED, &bdev->tx_state)) { + sdio_enable_func(bdev->func); + btmtksdio_drv_pmctrl(bdev); + } + sdio_writel(bdev->func, C_INT_EN_CLR, MTK_REG_CHLPCR, NULL); skb_queue_purge(&bdev->txq); cancel_work_sync(&bdev->txrx_work); @@ -1272,6 +1278,12 @@ static void btmtksdio_cmd_timeout(struct hci_dev *hdev) goto err; } + /* set fw_pmctrl back if BT is closed after doing reset */ + if (!test_bit(BTMTKSDIO_FUNC_ENABLED, &bdev->tx_state)) { + btmtksdio_fw_pmctrl(bdev); + sdio_disable_func(bdev->func); + } + clear_bit(BTMTKSDIO_PATCH_ENABLED, &bdev->tx_state); err: sdio_release_host(bdev->func); diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c index a4274d8c7faaf..f0d0c5a6d5127 100644 --- a/drivers/bluetooth/btnxpuart.c +++ b/drivers/bluetooth/btnxpuart.c @@ -601,8 +601,10 @@ static int nxp_download_firmware(struct hci_dev *hdev) &nxpdev->tx_state), msecs_to_jiffies(60000)); - release_firmware(nxpdev->fw); - memset(nxpdev->fw_name, 0, sizeof(nxpdev->fw_name)); + if (nxpdev->fw && strlen(nxpdev->fw_name)) { + release_firmware(nxpdev->fw); + memset(nxpdev->fw_name, 0, sizeof(nxpdev->fw_name)); + } if (err == 0) { bt_dev_err(hdev, "FW Download Timeout. offset: %d", diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c index 7f67e460f7f49..24dae5440c036 100644 --- a/drivers/bluetooth/btrtl.c +++ b/drivers/bluetooth/btrtl.c @@ -604,8 +604,10 @@ static int rtlbt_parse_firmware_v2(struct hci_dev *hdev, len += entry->len; } - if (!len) + if (!len) { + kvfree(ptr); return -EPERM; + } *_buf = ptr; return len; diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index bc3f63f1ccd86..74d264b64b534 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -65,6 +65,7 @@ static struct usb_driver btusb_driver; #define BTUSB_INTEL_BROKEN_INITIAL_NCMD BIT(25) #define BTUSB_INTEL_NO_WBS_SUPPORT BIT(26) #define BTUSB_ACTIONS_SEMI BIT(27) +#define BTUSB_BARROT BIT(28) static const struct usb_device_id btusb_table[] = { /* Generic Bluetooth USB device */ @@ -508,6 +509,12 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x13d3, 0x3549), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + /* Realtek 8851BU Bluetooth devices */ + { USB_DEVICE(0x3625, 0x010b), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x2001, 0x332a), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + /* Realtek 8852AE Bluetooth devices */ { USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, @@ -764,6 +771,10 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x0cb5, 0xc547), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + /* Barrot Technology Bluetooth devices */ + { USB_DEVICE(0x33fa, 0x0010), .driver_info = BTUSB_BARROT }, + { USB_DEVICE(0x33fa, 0x0012), .driver_info = BTUSB_BARROT }, + /* Actions Semiconductor ATS2851 based devices */ { USB_DEVICE(0x10d7, 0xb012), .driver_info = BTUSB_ACTIONS_SEMI }, @@ -1161,6 +1172,18 @@ static int btusb_recv_intr(struct btusb_data *data, void *buffer, int count) } if (!hci_skb_expect(skb)) { + /* Each chunk should correspond to at least 1 or more + * events so if there are still bytes left that doesn't + * constitute a new event this is likely a bug in the + * controller. + */ + if (count && count < HCI_EVENT_HDR_SIZE) { + bt_dev_warn(data->hdev, + "Unexpected continuation: %d bytes", + count); + count = 0; + } + /* Complete frame */ btusb_recv_event(data, skb); skb = NULL; @@ -3521,55 +3544,27 @@ static void btusb_coredump_qca(struct hci_dev *hdev) bt_dev_err(hdev, "%s: triggle crash failed (%d)", __func__, err); } -/* - * ==0: not a dump pkt. - * < 0: fails to handle a dump pkt - * > 0: otherwise. - */ +/* Return: 0 on success, negative errno on failure. */ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb) { - int ret = 1; + int ret = 0; + unsigned int skip = 0; u8 pkt_type; - u8 *sk_ptr; - unsigned int sk_len; u16 seqno; u32 dump_size; - struct hci_event_hdr *event_hdr; - struct hci_acl_hdr *acl_hdr; struct qca_dump_hdr *dump_hdr; struct btusb_data *btdata = hci_get_drvdata(hdev); struct usb_device *udev = btdata->udev; pkt_type = hci_skb_pkt_type(skb); - sk_ptr = skb->data; - sk_len = skb->len; + skip = sizeof(struct hci_event_hdr); + if (pkt_type == HCI_ACLDATA_PKT) + skip += sizeof(struct hci_acl_hdr); - if (pkt_type == HCI_ACLDATA_PKT) { - acl_hdr = hci_acl_hdr(skb); - if (le16_to_cpu(acl_hdr->handle) != QCA_MEMDUMP_ACL_HANDLE) - return 0; - sk_ptr += HCI_ACL_HDR_SIZE; - sk_len -= HCI_ACL_HDR_SIZE; - event_hdr = (struct hci_event_hdr *)sk_ptr; - } else { - event_hdr = hci_event_hdr(skb); - } + skb_pull(skb, skip); + dump_hdr = (struct qca_dump_hdr *)skb->data; - if ((event_hdr->evt != HCI_VENDOR_PKT) - || (event_hdr->plen != (sk_len - HCI_EVENT_HDR_SIZE))) - return 0; - - sk_ptr += HCI_EVENT_HDR_SIZE; - sk_len -= HCI_EVENT_HDR_SIZE; - - dump_hdr = (struct qca_dump_hdr *)sk_ptr; - if ((sk_len < offsetof(struct qca_dump_hdr, data)) - || (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS) - || (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE)) - return 0; - - /*it is dump pkt now*/ seqno = le16_to_cpu(dump_hdr->seqno); if (seqno == 0) { set_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags); @@ -3589,16 +3584,15 @@ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb) btdata->qca_dump.ram_dump_size = dump_size; btdata->qca_dump.ram_dump_seqno = 0; - sk_ptr += offsetof(struct qca_dump_hdr, data0); - sk_len -= offsetof(struct qca_dump_hdr, data0); + + skb_pull(skb, offsetof(struct qca_dump_hdr, data0)); usb_disable_autosuspend(udev); bt_dev_info(hdev, "%s memdump size(%u)\n", (pkt_type == HCI_ACLDATA_PKT) ? "ACL" : "event", dump_size); } else { - sk_ptr += offsetof(struct qca_dump_hdr, data); - sk_len -= offsetof(struct qca_dump_hdr, data); + skb_pull(skb, offsetof(struct qca_dump_hdr, data)); } if (!btdata->qca_dump.ram_dump_size) { @@ -3618,7 +3612,6 @@ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb) return ret; } - skb_pull(skb, skb->len - sk_len); hci_devcd_append(hdev, skb); btdata->qca_dump.ram_dump_seqno++; if (seqno == QCA_LAST_SEQUENCE_NUM) { @@ -3643,17 +3636,74 @@ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb) return ret; } +/* Return: true if the ACL packet is a dump packet, false otherwise. */ +static bool acl_pkt_is_dump_qca(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct hci_event_hdr *event_hdr; + struct hci_acl_hdr *acl_hdr; + struct qca_dump_hdr *dump_hdr; + struct sk_buff *clone = skb_clone(skb, GFP_ATOMIC); + bool is_dump = false; + + if (!clone) + return false; + + acl_hdr = skb_pull_data(clone, sizeof(*acl_hdr)); + if (!acl_hdr || (le16_to_cpu(acl_hdr->handle) != QCA_MEMDUMP_ACL_HANDLE)) + goto out; + + event_hdr = skb_pull_data(clone, sizeof(*event_hdr)); + if (!event_hdr || (event_hdr->evt != HCI_VENDOR_PKT)) + goto out; + + dump_hdr = skb_pull_data(clone, sizeof(*dump_hdr)); + if (!dump_hdr || (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS) || + (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE)) + goto out; + + is_dump = true; +out: + consume_skb(clone); + return is_dump; +} + +/* Return: true if the event packet is a dump packet, false otherwise. */ +static bool evt_pkt_is_dump_qca(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct hci_event_hdr *event_hdr; + struct qca_dump_hdr *dump_hdr; + struct sk_buff *clone = skb_clone(skb, GFP_ATOMIC); + bool is_dump = false; + + if (!clone) + return false; + + event_hdr = skb_pull_data(clone, sizeof(*event_hdr)); + if (!event_hdr || (event_hdr->evt != HCI_VENDOR_PKT)) + goto out; + + dump_hdr = skb_pull_data(clone, sizeof(*dump_hdr)); + if (!dump_hdr || (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS) || + (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE)) + goto out; + + is_dump = true; +out: + consume_skb(clone); + return is_dump; +} + static int btusb_recv_acl_qca(struct hci_dev *hdev, struct sk_buff *skb) { - if (handle_dump_pkt_qca(hdev, skb)) - return 0; + if (acl_pkt_is_dump_qca(hdev, skb)) + return handle_dump_pkt_qca(hdev, skb); return hci_recv_frame(hdev, skb); } static int btusb_recv_evt_qca(struct hci_dev *hdev, struct sk_buff *skb) { - if (handle_dump_pkt_qca(hdev, skb)) - return 0; + if (evt_pkt_is_dump_qca(hdev, skb)) + return handle_dump_pkt_qca(hdev, skb); return hci_recv_frame(hdev, skb); } @@ -3709,6 +3759,32 @@ static const struct qca_device_info qca_devices_table[] = { { 0x00190200, 40, 4, 16 }, /* WCN785x 2.0 */ }; +static u16 qca_extract_board_id(const struct qca_version *ver) +{ + u16 flag = le16_to_cpu(ver->flag); + u16 board_id = 0; + + if (((flag >> 8) & 0xff) == QCA_FLAG_MULTI_NVM) { + /* The board_id should be split into two bytes + * The 1st byte is chip ID, and the 2nd byte is platform ID + * For example, board ID 0x010A, 0x01 is platform ID. 0x0A is chip ID + * we have several platforms, and platform IDs are continuously added + * Platform ID: + * 0x00 is for Mobile + * 0x01 is for X86 + * 0x02 is for Automotive + * 0x03 is for Consumer electronic + */ + board_id = (ver->chip_id << 8) + ver->platform_id; + } + + /* Take 0xffff as invalid board ID */ + if (board_id == 0xffff) + board_id = 0; + + return board_id; +} + static int btusb_qca_send_vendor_req(struct usb_device *udev, u8 request, void *data, u16 size) { @@ -3865,44 +3941,28 @@ static void btusb_generate_qca_nvm_name(char *fwname, size_t max_size, const struct qca_version *ver) { u32 rom_version = le32_to_cpu(ver->rom_version); - u16 flag = le16_to_cpu(ver->flag); - - if (((flag >> 8) & 0xff) == QCA_FLAG_MULTI_NVM) { - /* The board_id should be split into two bytes - * The 1st byte is chip ID, and the 2nd byte is platform ID - * For example, board ID 0x010A, 0x01 is platform ID. 0x0A is chip ID - * we have several platforms, and platform IDs are continuously added - * Platform ID: - * 0x00 is for Mobile - * 0x01 is for X86 - * 0x02 is for Automotive - * 0x03 is for Consumer electronic - */ - u16 board_id = (ver->chip_id << 8) + ver->platform_id; - const char *variant; + const char *variant; + int len; + u16 board_id; - switch (le32_to_cpu(ver->ram_version)) { - case WCN6855_2_0_RAM_VERSION_GF: - case WCN6855_2_1_RAM_VERSION_GF: - variant = "_gf"; - break; - default: - variant = ""; - break; - } + board_id = qca_extract_board_id(ver); - if (board_id == 0) { - snprintf(fwname, max_size, "qca/nvm_usb_%08x%s.bin", - rom_version, variant); - } else { - snprintf(fwname, max_size, "qca/nvm_usb_%08x%s_%04x.bin", - rom_version, variant, board_id); - } - } else { - snprintf(fwname, max_size, "qca/nvm_usb_%08x.bin", - rom_version); + switch (le32_to_cpu(ver->ram_version)) { + case WCN6855_2_0_RAM_VERSION_GF: + case WCN6855_2_1_RAM_VERSION_GF: + variant = "_gf"; + break; + default: + variant = NULL; + break; } + len = snprintf(fwname, max_size, "qca/nvm_usb_%08x", rom_version); + if (variant) + len += snprintf(fwname + len, max_size - len, "%s", variant); + if (board_id) + len += snprintf(fwname + len, max_size - len, "_%04x", board_id); + len += snprintf(fwname + len, max_size - len, ".bin"); } static int btusb_setup_qca_load_nvm(struct hci_dev *hdev, @@ -4653,6 +4713,11 @@ static void btusb_disconnect(struct usb_interface *intf) hci_unregister_dev(hdev); + if (data->oob_wake_irq) + device_init_wakeup(&data->udev->dev, false); + if (data->reset_gpio) + gpiod_put(data->reset_gpio); + if (intf == data->intf) { if (data->isoc) usb_driver_release_interface(&btusb_driver, data->isoc); @@ -4663,17 +4728,11 @@ static void btusb_disconnect(struct usb_interface *intf) usb_driver_release_interface(&btusb_driver, data->diag); usb_driver_release_interface(&btusb_driver, data->intf); } else if (intf == data->diag) { - usb_driver_release_interface(&btusb_driver, data->intf); if (data->isoc) usb_driver_release_interface(&btusb_driver, data->isoc); + usb_driver_release_interface(&btusb_driver, data->intf); } - if (data->oob_wake_irq) - device_init_wakeup(&data->udev->dev, false); - - if (data->reset_gpio) - gpiod_put(data->reset_gpio); - hci_free_dev(hdev); } diff --git a/drivers/bluetooth/hci_bcsp.c b/drivers/bluetooth/hci_bcsp.c index 2a5a27d713f8a..e991d9e624869 100644 --- a/drivers/bluetooth/hci_bcsp.c +++ b/drivers/bluetooth/hci_bcsp.c @@ -582,6 +582,9 @@ static int bcsp_recv(struct hci_uart *hu, const void *data, int count) struct bcsp_struct *bcsp = hu->priv; const unsigned char *ptr; + if (!test_bit(HCI_UART_REGISTERED, &hu->flags)) + return -EUNATCH; + BT_DBG("hu %p count %d rx_state %d rx_count %ld", hu, count, bcsp->rx_state, bcsp->rx_count); diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 7a552387129ef..9c6f79d860532 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -2307,14 +2307,14 @@ static int qca_serdev_probe(struct serdev_device *serdev) qcadev->bt_en = devm_gpiod_get_optional(&serdev->dev, "enable", GPIOD_OUT_LOW); - if (IS_ERR(qcadev->bt_en) && - (data->soc_type == QCA_WCN6750 || - data->soc_type == QCA_WCN6855)) { - dev_err(&serdev->dev, "failed to acquire BT_EN gpio\n"); - return PTR_ERR(qcadev->bt_en); - } + if (IS_ERR(qcadev->bt_en)) + return dev_err_probe(&serdev->dev, + PTR_ERR(qcadev->bt_en), + "failed to acquire BT_EN gpio\n"); - if (!qcadev->bt_en) + if (!qcadev->bt_en && + (data->soc_type == QCA_WCN6750 || + data->soc_type == QCA_WCN6855)) power_ctrl_enabled = false; qcadev->sw_ctrl = devm_gpiod_get_optional(&serdev->dev, "swctrl", diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c index 4bfc78f9781ed..0935045051699 100644 --- a/drivers/bluetooth/hci_vhci.c +++ b/drivers/bluetooth/hci_vhci.c @@ -380,6 +380,28 @@ static const struct file_operations force_devcoredump_fops = { .write = force_devcd_write, }; +static void vhci_debugfs_init(struct vhci_data *data) +{ + struct hci_dev *hdev = data->hdev; + + debugfs_create_file("force_suspend", 0644, hdev->debugfs, data, + &force_suspend_fops); + + debugfs_create_file("force_wakeup", 0644, hdev->debugfs, data, + &force_wakeup_fops); + + if (IS_ENABLED(CONFIG_BT_MSFTEXT)) + debugfs_create_file("msft_opcode", 0644, hdev->debugfs, data, + &msft_opcode_fops); + + if (IS_ENABLED(CONFIG_BT_AOSPEXT)) + debugfs_create_file("aosp_capable", 0644, hdev->debugfs, data, + &aosp_capable_fops); + + debugfs_create_file("force_devcoredump", 0644, hdev->debugfs, data, + &force_devcoredump_fops); +} + static int __vhci_create_device(struct vhci_data *data, __u8 opcode) { struct hci_dev *hdev; @@ -435,22 +457,8 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode) return -EBUSY; } - debugfs_create_file("force_suspend", 0644, hdev->debugfs, data, - &force_suspend_fops); - - debugfs_create_file("force_wakeup", 0644, hdev->debugfs, data, - &force_wakeup_fops); - - if (IS_ENABLED(CONFIG_BT_MSFTEXT)) - debugfs_create_file("msft_opcode", 0644, hdev->debugfs, data, - &msft_opcode_fops); - - if (IS_ENABLED(CONFIG_BT_AOSPEXT)) - debugfs_create_file("aosp_capable", 0644, hdev->debugfs, data, - &aosp_capable_fops); - - debugfs_create_file("force_devcoredump", 0644, hdev->debugfs, data, - &force_devcoredump_fops); + if (!IS_ERR_OR_NULL(hdev->debugfs)) + vhci_debugfs_init(data); hci_skb_pkt_type(skb) = HCI_VENDOR_PKT; @@ -652,6 +660,21 @@ static int vhci_open(struct inode *inode, struct file *file) return 0; } +static void vhci_debugfs_remove(struct hci_dev *hdev) +{ + debugfs_lookup_and_remove("force_suspend", hdev->debugfs); + + debugfs_lookup_and_remove("force_wakeup", hdev->debugfs); + + if (IS_ENABLED(CONFIG_BT_MSFTEXT)) + debugfs_lookup_and_remove("msft_opcode", hdev->debugfs); + + if (IS_ENABLED(CONFIG_BT_AOSPEXT)) + debugfs_lookup_and_remove("aosp_capable", hdev->debugfs); + + debugfs_lookup_and_remove("force_devcoredump", hdev->debugfs); +} + static int vhci_release(struct inode *inode, struct file *file) { struct vhci_data *data = file->private_data; @@ -663,6 +686,8 @@ static int vhci_release(struct inode *inode, struct file *file) hdev = data->hdev; if (hdev) { + if (!IS_ERR_OR_NULL(hdev->debugfs)) + vhci_debugfs_remove(hdev); hci_unregister_dev(hdev); hci_free_dev(hdev); } diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 2f6d5002e43d5..03642df2df0bb 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -905,8 +905,10 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc, error_cleanup_dev: kfree(mc_dev->regions); - kfree(mc_bus); - kfree(mc_dev); + if (mc_bus) + kfree(mc_bus); + else + kfree(mc_dev); return error; } @@ -940,6 +942,7 @@ struct fsl_mc_device *fsl_mc_get_endpoint(struct fsl_mc_device *mc_dev, struct fsl_mc_obj_desc endpoint_desc = {{ 0 }}; struct dprc_endpoint endpoint1 = {{ 0 }}; struct dprc_endpoint endpoint2 = {{ 0 }}; + struct fsl_mc_bus *mc_bus; int state, err; mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent); @@ -963,6 +966,8 @@ struct fsl_mc_device *fsl_mc_get_endpoint(struct fsl_mc_device *mc_dev, strcpy(endpoint_desc.type, endpoint2.type); endpoint_desc.id = endpoint2.id; endpoint = fsl_mc_device_lookup(&endpoint_desc, mc_bus_dev); + if (endpoint) + return endpoint; /* * We know that the device has an endpoint because we verified by @@ -970,17 +975,13 @@ struct fsl_mc_device *fsl_mc_get_endpoint(struct fsl_mc_device *mc_dev, * yet discovered by the fsl-mc bus, thus the lookup returned NULL. * Force a rescan of the devices in this container and retry the lookup. */ - if (!endpoint) { - struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev); - - if (mutex_trylock(&mc_bus->scan_mutex)) { - err = dprc_scan_objects(mc_bus_dev, true); - mutex_unlock(&mc_bus->scan_mutex); - } - - if (err < 0) - return ERR_PTR(err); + mc_bus = to_fsl_mc_bus(mc_bus_dev); + if (mutex_trylock(&mc_bus->scan_mutex)) { + err = dprc_scan_objects(mc_bus_dev, true); + mutex_unlock(&mc_bus->scan_mutex); } + if (err < 0) + return ERR_PTR(err); endpoint = fsl_mc_device_lookup(&endpoint_desc, mc_bus_dev); /* @@ -1102,6 +1103,9 @@ static int fsl_mc_bus_probe(struct platform_device *pdev) * Get physical address of MC portal for the root DPRC: */ plat_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!plat_res) + return -EINVAL; + mc_portal_phys_addr = plat_res->start; mc_portal_size = resource_size(plat_res); mc_portal_base_phys_addr = mc_portal_phys_addr & ~0x3ffffff; diff --git a/drivers/bus/fsl-mc/fsl-mc-uapi.c b/drivers/bus/fsl-mc/fsl-mc-uapi.c index 9c4c1395fcdbf..a376ec6616534 100644 --- a/drivers/bus/fsl-mc/fsl-mc-uapi.c +++ b/drivers/bus/fsl-mc/fsl-mc-uapi.c @@ -275,13 +275,13 @@ static struct fsl_mc_cmd_desc fsl_mc_accepted_cmds[] = { .size = 8, }, [DPSW_GET_TAILDROP] = { - .cmdid_value = 0x0A80, + .cmdid_value = 0x0A90, .cmdid_mask = 0xFFF0, .token = true, .size = 14, }, [DPSW_SET_TAILDROP] = { - .cmdid_value = 0x0A90, + .cmdid_value = 0x0A80, .cmdid_mask = 0xFFF0, .token = true, .size = 24, diff --git a/drivers/bus/fsl-mc/mc-io.c b/drivers/bus/fsl-mc/mc-io.c index 95b10a6cf3073..8b7a34f4db94b 100644 --- a/drivers/bus/fsl-mc/mc-io.c +++ b/drivers/bus/fsl-mc/mc-io.c @@ -214,12 +214,19 @@ int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev, if (error < 0) goto error_cleanup_resource; - dpmcp_dev->consumer_link = device_link_add(&mc_dev->dev, - &dpmcp_dev->dev, - DL_FLAG_AUTOREMOVE_CONSUMER); - if (!dpmcp_dev->consumer_link) { - error = -EINVAL; - goto error_cleanup_mc_io; + /* If the DPRC device itself tries to allocate a portal (usually for + * UAPI interaction), don't add a device link between them since the + * DPMCP device is an actual child device of the DPRC and a reverse + * dependency is not allowed. + */ + if (mc_dev != mc_bus_dev) { + dpmcp_dev->consumer_link = device_link_add(&mc_dev->dev, + &dpmcp_dev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!dpmcp_dev->consumer_link) { + error = -EINVAL; + goto error_cleanup_mc_io; + } } *new_mc_io = mc_io; diff --git a/drivers/bus/fsl-mc/mc-sys.c b/drivers/bus/fsl-mc/mc-sys.c index f2052cd0a0517..b22c59d57c8f0 100644 --- a/drivers/bus/fsl-mc/mc-sys.c +++ b/drivers/bus/fsl-mc/mc-sys.c @@ -19,7 +19,7 @@ /* * Timeout in milliseconds to wait for the completion of an MC command */ -#define MC_CMD_COMPLETION_TIMEOUT_MS 500 +#define MC_CMD_COMPLETION_TIMEOUT_MS 15000 /* * usleep_range() min and max values used to throttle down polling diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c index c48f4d9f2c690..558cfb3a06c5e 100644 --- a/drivers/bus/mhi/ep/main.c +++ b/drivers/bus/mhi/ep/main.c @@ -387,17 +387,13 @@ static int mhi_ep_read_channel(struct mhi_ep_cntrl *mhi_cntrl, { struct mhi_ep_chan *mhi_chan = &mhi_cntrl->mhi_chan[ring->ch_id]; struct device *dev = &mhi_cntrl->mhi_dev->dev; - size_t tr_len, read_offset, write_offset; + size_t tr_len, read_offset; struct mhi_ep_buf_info buf_info = {}; u32 len = MHI_EP_DEFAULT_MTU; struct mhi_ring_element *el; - bool tr_done = false; void *buf_addr; - u32 buf_left; int ret; - buf_left = len; - do { /* Don't process the transfer ring if the channel is not in RUNNING state */ if (mhi_chan->state != MHI_CH_STATE_RUNNING) { @@ -410,24 +406,23 @@ static int mhi_ep_read_channel(struct mhi_ep_cntrl *mhi_cntrl, /* Check if there is data pending to be read from previous read operation */ if (mhi_chan->tre_bytes_left) { dev_dbg(dev, "TRE bytes remaining: %u\n", mhi_chan->tre_bytes_left); - tr_len = min(buf_left, mhi_chan->tre_bytes_left); + tr_len = min(len, mhi_chan->tre_bytes_left); } else { mhi_chan->tre_loc = MHI_TRE_DATA_GET_PTR(el); mhi_chan->tre_size = MHI_TRE_DATA_GET_LEN(el); mhi_chan->tre_bytes_left = mhi_chan->tre_size; - tr_len = min(buf_left, mhi_chan->tre_size); + tr_len = min(len, mhi_chan->tre_size); } read_offset = mhi_chan->tre_size - mhi_chan->tre_bytes_left; - write_offset = len - buf_left; buf_addr = kmem_cache_zalloc(mhi_cntrl->tre_buf_cache, GFP_KERNEL); if (!buf_addr) return -ENOMEM; buf_info.host_addr = mhi_chan->tre_loc + read_offset; - buf_info.dev_addr = buf_addr + write_offset; + buf_info.dev_addr = buf_addr; buf_info.size = tr_len; buf_info.cb = mhi_ep_read_completion; buf_info.cb_buf = buf_addr; @@ -443,16 +438,12 @@ static int mhi_ep_read_channel(struct mhi_ep_cntrl *mhi_cntrl, goto err_free_buf_addr; } - buf_left -= tr_len; mhi_chan->tre_bytes_left -= tr_len; - if (!mhi_chan->tre_bytes_left) { - if (MHI_TRE_DATA_GET_IEOT(el)) - tr_done = true; - + if (!mhi_chan->tre_bytes_left) mhi_chan->rd_offset = (mhi_chan->rd_offset + 1) % ring->ring_size; - } - } while (buf_left && !tr_done); + /* Read until the some buffer is left or the ring becomes not empty */ + } while (!mhi_ep_queue_is_empty(mhi_chan->mhi_dev, DMA_TO_DEVICE)); return 0; @@ -486,15 +477,11 @@ static int mhi_ep_process_ch_ring(struct mhi_ep_ring *ring) mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); } else { /* UL channel */ - do { - ret = mhi_ep_read_channel(mhi_cntrl, ring); - if (ret < 0) { - dev_err(&mhi_chan->mhi_dev->dev, "Failed to read channel\n"); - return ret; - } - - /* Read until the ring becomes empty */ - } while (!mhi_ep_queue_is_empty(mhi_chan->mhi_dev, DMA_TO_DEVICE)); + ret = mhi_ep_read_channel(mhi_cntrl, ring); + if (ret < 0) { + dev_err(&mhi_chan->mhi_dev->dev, "Failed to read channel\n"); + return ret; + } } return 0; diff --git a/drivers/bus/mhi/ep/ring.c b/drivers/bus/mhi/ep/ring.c index ba9f696d1aa80..fedf26392e8d2 100644 --- a/drivers/bus/mhi/ep/ring.c +++ b/drivers/bus/mhi/ep/ring.c @@ -131,19 +131,23 @@ int mhi_ep_ring_add_element(struct mhi_ep_ring *ring, struct mhi_ring_element *e } old_offset = ring->rd_offset; - mhi_ep_ring_inc_index(ring); dev_dbg(dev, "Adding an element to ring at offset (%zu)\n", ring->rd_offset); + buf_info.host_addr = ring->rbase + (old_offset * sizeof(*el)); + buf_info.dev_addr = el; + buf_info.size = sizeof(*el); + + ret = mhi_cntrl->write_sync(mhi_cntrl, &buf_info); + if (ret) + return ret; + + mhi_ep_ring_inc_index(ring); /* Update rp in ring context */ rp = cpu_to_le64(ring->rd_offset * sizeof(*el) + ring->rbase); memcpy_toio((void __iomem *) &ring->ring_ctx->generic.rp, &rp, sizeof(u64)); - buf_info.host_addr = ring->rbase + (old_offset * sizeof(*el)); - buf_info.dev_addr = el; - buf_info.size = sizeof(*el); - - return mhi_cntrl->write_sync(mhi_cntrl, &buf_info); + return ret; } void mhi_ep_ring_init(struct mhi_ep_ring *ring, enum mhi_ep_ring_type type, u32 id) diff --git a/drivers/bus/mhi/host/boot.c b/drivers/bus/mhi/host/boot.c index edc0ec5a09339..022571cfec5cb 100644 --- a/drivers/bus/mhi/host/boot.c +++ b/drivers/bus/mhi/host/boot.c @@ -31,8 +31,8 @@ int mhi_rddm_prepare(struct mhi_controller *mhi_cntrl, int ret; for (i = 0; i < img_info->entries - 1; i++, mhi_buf++, bhi_vec++) { - bhi_vec->dma_addr = mhi_buf->dma_addr; - bhi_vec->size = mhi_buf->len; + bhi_vec->dma_addr = cpu_to_le64(mhi_buf->dma_addr); + bhi_vec->size = cpu_to_le64(mhi_buf->len); } dev_dbg(dev, "BHIe programming for RDDM\n"); @@ -375,8 +375,8 @@ static void mhi_firmware_copy(struct mhi_controller *mhi_cntrl, while (remainder) { to_cpy = min(remainder, mhi_buf->len); memcpy(mhi_buf->buf, buf, to_cpy); - bhi_vec->dma_addr = mhi_buf->dma_addr; - bhi_vec->size = to_cpy; + bhi_vec->dma_addr = cpu_to_le64(mhi_buf->dma_addr); + bhi_vec->size = cpu_to_le64(to_cpy); buf += to_cpy; remainder -= to_cpy; diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c index cfd17c02fe20e..4af748ff50985 100644 --- a/drivers/bus/mhi/host/init.c +++ b/drivers/bus/mhi/host/init.c @@ -164,7 +164,6 @@ void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl) int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) { struct mhi_event *mhi_event = mhi_cntrl->mhi_event; - struct device *dev = &mhi_cntrl->mhi_dev->dev; unsigned long irq_flags = IRQF_SHARED | IRQF_NO_SUSPEND; int i, ret; @@ -191,7 +190,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) continue; if (mhi_event->irq >= mhi_cntrl->nr_irqs) { - dev_err(dev, "irq %d not available for event ring\n", + dev_err(mhi_cntrl->cntrl_dev, "irq %d not available for event ring\n", mhi_event->irq); ret = -EINVAL; goto error_request; @@ -202,7 +201,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) irq_flags, "mhi", mhi_event); if (ret) { - dev_err(dev, "Error requesting irq:%d for ev:%d\n", + dev_err(mhi_cntrl->cntrl_dev, "Error requesting irq:%d for ev:%d\n", mhi_cntrl->irq[mhi_event->irq], i); goto error_request; } diff --git a/drivers/bus/mhi/host/internal.h b/drivers/bus/mhi/host/internal.h index d2858236af52b..88c9bc11f1717 100644 --- a/drivers/bus/mhi/host/internal.h +++ b/drivers/bus/mhi/host/internal.h @@ -31,8 +31,8 @@ struct mhi_ctxt { }; struct bhi_vec_entry { - u64 dma_addr; - u64 size; + __le64 dma_addr; + __le64 size; }; enum mhi_ch_state_type { diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c index ad1e97222a0f4..196929fff243e 100644 --- a/drivers/bus/mhi/host/main.c +++ b/drivers/bus/mhi/host/main.c @@ -603,7 +603,7 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl, { dma_addr_t ptr = MHI_TRE_GET_EV_PTR(event); struct mhi_ring_element *local_rp, *ev_tre; - void *dev_rp; + void *dev_rp, *next_rp; struct mhi_buf_info *buf_info; u16 xfer_len; @@ -622,6 +622,16 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl, result.dir = mhi_chan->dir; local_rp = tre_ring->rp; + + next_rp = local_rp + 1; + if (next_rp >= tre_ring->base + tre_ring->len) + next_rp = tre_ring->base; + if (dev_rp != next_rp && !MHI_TRE_DATA_GET_CHAIN(local_rp)) { + dev_err(&mhi_cntrl->mhi_dev->dev, + "Event element points to an unexpected TRE\n"); + break; + } + while (local_rp != dev_rp) { buf_info = buf_ring->rp; /* If it's the last TRE, get length from the event */ diff --git a/drivers/bus/mhi/host/pm.c b/drivers/bus/mhi/host/pm.c index 27f8a40f288cf..e48bf44c785c9 100644 --- a/drivers/bus/mhi/host/pm.c +++ b/drivers/bus/mhi/host/pm.c @@ -586,6 +586,7 @@ static void mhi_pm_sys_error_transition(struct mhi_controller *mhi_cntrl) struct mhi_cmd *mhi_cmd; struct mhi_event_ctxt *er_ctxt; struct device *dev = &mhi_cntrl->mhi_dev->dev; + bool reset_device = false; int ret, i; dev_dbg(dev, "Transitioning from PM state: %s to: %s\n", @@ -614,8 +615,23 @@ static void mhi_pm_sys_error_transition(struct mhi_controller *mhi_cntrl) /* Wake up threads waiting for state transition */ wake_up_all(&mhi_cntrl->state_event); - /* Trigger MHI RESET so that the device will not access host memory */ if (MHI_REG_ACCESS_VALID(prev_state)) { + /* + * If the device is in PBL or SBL, it will only respond to + * RESET if the device is in SYSERR state. SYSERR might + * already be cleared at this point. + */ + enum mhi_state cur_state = mhi_get_mhi_state(mhi_cntrl); + enum mhi_ee_type cur_ee = mhi_get_exec_env(mhi_cntrl); + + if (cur_state == MHI_STATE_SYS_ERR) + reset_device = true; + else if (cur_ee != MHI_EE_PBL && cur_ee != MHI_EE_SBL) + reset_device = true; + } + + /* Trigger MHI RESET so that the device will not access host memory */ + if (reset_device) { u32 in_reset = -1; unsigned long timeout = msecs_to_jiffies(mhi_cntrl->timeout_ms); diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index 9ed9239b1228f..5716b381d7071 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -667,51 +667,6 @@ static int sysc_parse_and_check_child_range(struct sysc *ddata) return 0; } -/* Interconnect instances to probe before l4_per instances */ -static struct resource early_bus_ranges[] = { - /* am3/4 l4_wkup */ - { .start = 0x44c00000, .end = 0x44c00000 + 0x300000, }, - /* omap4/5 and dra7 l4_cfg */ - { .start = 0x4a000000, .end = 0x4a000000 + 0x300000, }, - /* omap4 l4_wkup */ - { .start = 0x4a300000, .end = 0x4a300000 + 0x30000, }, - /* omap5 and dra7 l4_wkup without dra7 dcan segment */ - { .start = 0x4ae00000, .end = 0x4ae00000 + 0x30000, }, -}; - -static atomic_t sysc_defer = ATOMIC_INIT(10); - -/** - * sysc_defer_non_critical - defer non_critical interconnect probing - * @ddata: device driver data - * - * We want to probe l4_cfg and l4_wkup interconnect instances before any - * l4_per instances as l4_per instances depend on resources on l4_cfg and - * l4_wkup interconnects. - */ -static int sysc_defer_non_critical(struct sysc *ddata) -{ - struct resource *res; - int i; - - if (!atomic_read(&sysc_defer)) - return 0; - - for (i = 0; i < ARRAY_SIZE(early_bus_ranges); i++) { - res = &early_bus_ranges[i]; - if (ddata->module_pa >= res->start && - ddata->module_pa <= res->end) { - atomic_set(&sysc_defer, 0); - - return 0; - } - } - - atomic_dec_if_positive(&sysc_defer); - - return -EPROBE_DEFER; -} - static struct device_node *stdout_path; static void sysc_init_stdout_path(struct sysc *ddata) @@ -937,10 +892,6 @@ static int sysc_map_and_check_registers(struct sysc *ddata) if (error) return error; - error = sysc_defer_non_critical(ddata); - if (error) - return error; - sysc_check_children(ddata); if (!of_property_present(np, "reg")) diff --git a/drivers/cdx/controller/cdx_rpmsg.c b/drivers/cdx/controller/cdx_rpmsg.c index 04b578a0be17c..61f1a290ff089 100644 --- a/drivers/cdx/controller/cdx_rpmsg.c +++ b/drivers/cdx/controller/cdx_rpmsg.c @@ -129,8 +129,7 @@ static int cdx_rpmsg_probe(struct rpmsg_device *rpdev) chinfo.src = RPMSG_ADDR_ANY; chinfo.dst = rpdev->dst; - strscpy(chinfo.name, cdx_rpmsg_id_table[0].name, - strlen(cdx_rpmsg_id_table[0].name)); + strscpy(chinfo.name, cdx_rpmsg_id_table[0].name, sizeof(chinfo.name)); cdx_mcdi->ept = rpmsg_create_ept(rpdev, cdx_rpmsg_cb, NULL, chinfo); if (!cdx_mcdi->ept) { diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 8de74dcfa18cf..ece2d794174d4 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -286,6 +286,7 @@ config HW_RANDOM_INGENIC_TRNG config HW_RANDOM_NOMADIK tristate "ST-Ericsson Nomadik Random Number Generator support" depends on ARCH_NOMADIK || COMPILE_TEST + depends on ARM_AMBA default HW_RANDOM help This driver provides kernel-side support for the Random Number diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c index 2f2f21f1b659e..d7b42888f25c2 100644 --- a/drivers/char/hw_random/ks-sa-rng.c +++ b/drivers/char/hw_random/ks-sa-rng.c @@ -240,6 +240,10 @@ static int ks_sa_rng_probe(struct platform_device *pdev) return -EINVAL; } + ks_sa_rng->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(ks_sa_rng->clk)) + return dev_err_probe(dev, PTR_ERR(ks_sa_rng->clk), "Failed to get clock\n"); + pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) { diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c index 1e3048f2bb38f..6c4e40d0365f0 100644 --- a/drivers/char/hw_random/mtk-rng.c +++ b/drivers/char/hw_random/mtk-rng.c @@ -142,7 +142,9 @@ static int mtk_rng_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, priv); pm_runtime_set_autosuspend_delay(&pdev->dev, RNG_AUTOSUSPEND_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); - devm_pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; dev_info(&pdev->dev, "registered RNG driver\n"); diff --git a/drivers/char/ipmi/ipmi_kcs_sm.c b/drivers/char/ipmi/ipmi_kcs_sm.c index ecfcb50302f6c..efda90dcf5b3d 100644 --- a/drivers/char/ipmi/ipmi_kcs_sm.c +++ b/drivers/char/ipmi/ipmi_kcs_sm.c @@ -122,10 +122,10 @@ struct si_sm_data { unsigned long error0_timeout; }; -static unsigned int init_kcs_data_with_state(struct si_sm_data *kcs, - struct si_sm_io *io, enum kcs_states state) +static unsigned int init_kcs_data(struct si_sm_data *kcs, + struct si_sm_io *io) { - kcs->state = state; + kcs->state = KCS_IDLE; kcs->io = io; kcs->write_pos = 0; kcs->write_count = 0; @@ -140,12 +140,6 @@ static unsigned int init_kcs_data_with_state(struct si_sm_data *kcs, return 2; } -static unsigned int init_kcs_data(struct si_sm_data *kcs, - struct si_sm_io *io) -{ - return init_kcs_data_with_state(kcs, io, KCS_IDLE); -} - static inline unsigned char read_status(struct si_sm_data *kcs) { return kcs->io->inputb(kcs->io, 1); @@ -276,7 +270,7 @@ static int start_kcs_transaction(struct si_sm_data *kcs, unsigned char *data, if (size > MAX_KCS_WRITE_SIZE) return IPMI_REQ_LEN_EXCEEDED_ERR; - if (kcs->state != KCS_IDLE) { + if ((kcs->state != KCS_IDLE) && (kcs->state != KCS_HOSED)) { dev_warn(kcs->io->dev, "KCS in invalid state %d\n", kcs->state); return IPMI_NOT_IN_MY_STATE_ERR; } @@ -501,7 +495,7 @@ static enum si_sm_result kcs_event(struct si_sm_data *kcs, long time) } if (kcs->state == KCS_HOSED) { - init_kcs_data_with_state(kcs, kcs->io, KCS_ERROR0); + init_kcs_data(kcs, kcs->io); return SI_SM_HOSED; } diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 186f1fee75340..b7d8bf202ed2d 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -39,7 +39,9 @@ #define IPMI_DRIVER_VERSION "39.2" -static struct ipmi_recv_msg *ipmi_alloc_recv_msg(void); +static struct ipmi_recv_msg *ipmi_alloc_recv_msg(struct ipmi_user *user); +static void ipmi_set_recv_msg_user(struct ipmi_recv_msg *msg, + struct ipmi_user *user); static int ipmi_init_msghandler(void); static void smi_recv_tasklet(struct tasklet_struct *t); static void handle_new_recv_msgs(struct ipmi_smi *intf); @@ -939,13 +941,11 @@ static int deliver_response(struct ipmi_smi *intf, struct ipmi_recv_msg *msg) * risk. At this moment, simply skip it in that case. */ ipmi_free_recv_msg(msg); - atomic_dec(&msg->user->nr_msgs); } else { int index; struct ipmi_user *user = acquire_ipmi_user(msg->user, &index); if (user) { - atomic_dec(&user->nr_msgs); user->handler->ipmi_recv_hndl(msg, user->handler_data); release_ipmi_user(user, index); } else { @@ -1241,7 +1241,7 @@ int ipmi_create_user(unsigned int if_num, } /* Not found, return an error */ rv = -EINVAL; - goto out_kfree; + goto out_unlock; found: if (atomic_add_return(1, &intf->nr_users) > max_users) { @@ -1283,6 +1283,7 @@ int ipmi_create_user(unsigned int if_num, out_kfree: atomic_dec(&intf->nr_users); +out_unlock: srcu_read_unlock(&ipmi_interfaces_srcu, index); vfree(new_user); return rv; @@ -1633,8 +1634,7 @@ int ipmi_set_gets_events(struct ipmi_user *user, bool val) spin_unlock_irqrestore(&intf->events_lock, flags); list_for_each_entry_safe(msg, msg2, &msgs, link) { - msg->user = user; - kref_get(&user->refcount); + ipmi_set_recv_msg_user(msg, user); deliver_local_response(intf, msg); } @@ -2308,22 +2308,18 @@ static int i_ipmi_request(struct ipmi_user *user, struct ipmi_recv_msg *recv_msg; int rv = 0; - if (user) { - if (atomic_add_return(1, &user->nr_msgs) > max_msgs_per_user) { - /* Decrement will happen at the end of the routine. */ - rv = -EBUSY; - goto out; - } - } - - if (supplied_recv) + if (supplied_recv) { recv_msg = supplied_recv; - else { - recv_msg = ipmi_alloc_recv_msg(); - if (recv_msg == NULL) { - rv = -ENOMEM; - goto out; + recv_msg->user = user; + if (user) { + atomic_inc(&user->nr_msgs); + /* The put happens when the message is freed. */ + kref_get(&user->refcount); } + } else { + recv_msg = ipmi_alloc_recv_msg(user); + if (IS_ERR(recv_msg)) + return PTR_ERR(recv_msg); } recv_msg->user_msg_data = user_msg_data; @@ -2334,8 +2330,7 @@ static int i_ipmi_request(struct ipmi_user *user, if (smi_msg == NULL) { if (!supplied_recv) ipmi_free_recv_msg(recv_msg); - rv = -ENOMEM; - goto out; + return -ENOMEM; } } @@ -2345,10 +2340,6 @@ static int i_ipmi_request(struct ipmi_user *user, goto out_err; } - recv_msg->user = user; - if (user) - /* The put happens when the message is freed. */ - kref_get(&user->refcount); recv_msg->msgid = msgid; /* * Store the message to send in the receive message so timeout @@ -2377,8 +2368,10 @@ static int i_ipmi_request(struct ipmi_user *user, if (rv) { out_err: - ipmi_free_smi_msg(smi_msg); - ipmi_free_recv_msg(recv_msg); + if (!supplied_smi) + ipmi_free_smi_msg(smi_msg); + if (!supplied_recv) + ipmi_free_recv_msg(recv_msg); } else { dev_dbg(intf->si_dev, "Send: %*ph\n", smi_msg->data_size, smi_msg->data); @@ -2387,9 +2380,6 @@ static int i_ipmi_request(struct ipmi_user *user, } rcu_read_unlock(); -out: - if (rv && user) - atomic_dec(&user->nr_msgs); return rv; } @@ -3882,7 +3872,7 @@ static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf, unsigned char chan; struct ipmi_user *user = NULL; struct ipmi_ipmb_addr *ipmb_addr; - struct ipmi_recv_msg *recv_msg; + struct ipmi_recv_msg *recv_msg = NULL; if (msg->rsp_size < 10) { /* Message not big enough, just ignore it. */ @@ -3903,9 +3893,8 @@ static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf, rcvr = find_cmd_rcvr(intf, netfn, cmd, chan); if (rcvr) { user = rcvr->user; - kref_get(&user->refcount); - } else - user = NULL; + recv_msg = ipmi_alloc_recv_msg(user); + } rcu_read_unlock(); if (user == NULL) { @@ -3940,47 +3929,41 @@ static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf, rv = -1; } rcu_read_unlock(); - } else { - recv_msg = ipmi_alloc_recv_msg(); - if (!recv_msg) { - /* - * We couldn't allocate memory for the - * message, so requeue it for handling - * later. - */ - rv = 1; - kref_put(&user->refcount, free_user); - } else { - /* Extract the source address from the data. */ - ipmb_addr = (struct ipmi_ipmb_addr *) &recv_msg->addr; - ipmb_addr->addr_type = IPMI_IPMB_ADDR_TYPE; - ipmb_addr->slave_addr = msg->rsp[6]; - ipmb_addr->lun = msg->rsp[7] & 3; - ipmb_addr->channel = msg->rsp[3] & 0xf; + } else if (!IS_ERR(recv_msg)) { + /* Extract the source address from the data. */ + ipmb_addr = (struct ipmi_ipmb_addr *) &recv_msg->addr; + ipmb_addr->addr_type = IPMI_IPMB_ADDR_TYPE; + ipmb_addr->slave_addr = msg->rsp[6]; + ipmb_addr->lun = msg->rsp[7] & 3; + ipmb_addr->channel = msg->rsp[3] & 0xf; - /* - * Extract the rest of the message information - * from the IPMB header. - */ - recv_msg->user = user; - recv_msg->recv_type = IPMI_CMD_RECV_TYPE; - recv_msg->msgid = msg->rsp[7] >> 2; - recv_msg->msg.netfn = msg->rsp[4] >> 2; - recv_msg->msg.cmd = msg->rsp[8]; - recv_msg->msg.data = recv_msg->msg_data; + /* + * Extract the rest of the message information + * from the IPMB header. + */ + recv_msg->recv_type = IPMI_CMD_RECV_TYPE; + recv_msg->msgid = msg->rsp[7] >> 2; + recv_msg->msg.netfn = msg->rsp[4] >> 2; + recv_msg->msg.cmd = msg->rsp[8]; + recv_msg->msg.data = recv_msg->msg_data; - /* - * We chop off 10, not 9 bytes because the checksum - * at the end also needs to be removed. - */ - recv_msg->msg.data_len = msg->rsp_size - 10; - memcpy(recv_msg->msg_data, &msg->rsp[9], - msg->rsp_size - 10); - if (deliver_response(intf, recv_msg)) - ipmi_inc_stat(intf, unhandled_commands); - else - ipmi_inc_stat(intf, handled_commands); - } + /* + * We chop off 10, not 9 bytes because the checksum + * at the end also needs to be removed. + */ + recv_msg->msg.data_len = msg->rsp_size - 10; + memcpy(recv_msg->msg_data, &msg->rsp[9], + msg->rsp_size - 10); + if (deliver_response(intf, recv_msg)) + ipmi_inc_stat(intf, unhandled_commands); + else + ipmi_inc_stat(intf, handled_commands); + } else { + /* + * We couldn't allocate memory for the message, so + * requeue it for handling later. + */ + rv = 1; } return rv; @@ -3993,7 +3976,7 @@ static int handle_ipmb_direct_rcv_cmd(struct ipmi_smi *intf, int rv = 0; struct ipmi_user *user = NULL; struct ipmi_ipmb_direct_addr *daddr; - struct ipmi_recv_msg *recv_msg; + struct ipmi_recv_msg *recv_msg = NULL; unsigned char netfn = msg->rsp[0] >> 2; unsigned char cmd = msg->rsp[3]; @@ -4002,9 +3985,8 @@ static int handle_ipmb_direct_rcv_cmd(struct ipmi_smi *intf, rcvr = find_cmd_rcvr(intf, netfn, cmd, 0); if (rcvr) { user = rcvr->user; - kref_get(&user->refcount); - } else - user = NULL; + recv_msg = ipmi_alloc_recv_msg(user); + } rcu_read_unlock(); if (user == NULL) { @@ -4031,44 +4013,38 @@ static int handle_ipmb_direct_rcv_cmd(struct ipmi_smi *intf, rv = -1; } rcu_read_unlock(); - } else { - recv_msg = ipmi_alloc_recv_msg(); - if (!recv_msg) { - /* - * We couldn't allocate memory for the - * message, so requeue it for handling - * later. - */ - rv = 1; - kref_put(&user->refcount, free_user); - } else { - /* Extract the source address from the data. */ - daddr = (struct ipmi_ipmb_direct_addr *)&recv_msg->addr; - daddr->addr_type = IPMI_IPMB_DIRECT_ADDR_TYPE; - daddr->channel = 0; - daddr->slave_addr = msg->rsp[1]; - daddr->rs_lun = msg->rsp[0] & 3; - daddr->rq_lun = msg->rsp[2] & 3; + } else if (!IS_ERR(recv_msg)) { + /* Extract the source address from the data. */ + daddr = (struct ipmi_ipmb_direct_addr *)&recv_msg->addr; + daddr->addr_type = IPMI_IPMB_DIRECT_ADDR_TYPE; + daddr->channel = 0; + daddr->slave_addr = msg->rsp[1]; + daddr->rs_lun = msg->rsp[0] & 3; + daddr->rq_lun = msg->rsp[2] & 3; - /* - * Extract the rest of the message information - * from the IPMB header. - */ - recv_msg->user = user; - recv_msg->recv_type = IPMI_CMD_RECV_TYPE; - recv_msg->msgid = (msg->rsp[2] >> 2); - recv_msg->msg.netfn = msg->rsp[0] >> 2; - recv_msg->msg.cmd = msg->rsp[3]; - recv_msg->msg.data = recv_msg->msg_data; - - recv_msg->msg.data_len = msg->rsp_size - 4; - memcpy(recv_msg->msg_data, msg->rsp + 4, - msg->rsp_size - 4); - if (deliver_response(intf, recv_msg)) - ipmi_inc_stat(intf, unhandled_commands); - else - ipmi_inc_stat(intf, handled_commands); - } + /* + * Extract the rest of the message information + * from the IPMB header. + */ + recv_msg->recv_type = IPMI_CMD_RECV_TYPE; + recv_msg->msgid = (msg->rsp[2] >> 2); + recv_msg->msg.netfn = msg->rsp[0] >> 2; + recv_msg->msg.cmd = msg->rsp[3]; + recv_msg->msg.data = recv_msg->msg_data; + + recv_msg->msg.data_len = msg->rsp_size - 4; + memcpy(recv_msg->msg_data, msg->rsp + 4, + msg->rsp_size - 4); + if (deliver_response(intf, recv_msg)) + ipmi_inc_stat(intf, unhandled_commands); + else + ipmi_inc_stat(intf, handled_commands); + } else { + /* + * We couldn't allocate memory for the message, so + * requeue it for handling later. + */ + rv = 1; } return rv; @@ -4182,7 +4158,7 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf, unsigned char chan; struct ipmi_user *user = NULL; struct ipmi_lan_addr *lan_addr; - struct ipmi_recv_msg *recv_msg; + struct ipmi_recv_msg *recv_msg = NULL; if (msg->rsp_size < 12) { /* Message not big enough, just ignore it. */ @@ -4203,9 +4179,8 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf, rcvr = find_cmd_rcvr(intf, netfn, cmd, chan); if (rcvr) { user = rcvr->user; - kref_get(&user->refcount); - } else - user = NULL; + recv_msg = ipmi_alloc_recv_msg(user); + } rcu_read_unlock(); if (user == NULL) { @@ -4217,49 +4192,44 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf, * them to be freed. */ rv = 0; - } else { - recv_msg = ipmi_alloc_recv_msg(); - if (!recv_msg) { - /* - * We couldn't allocate memory for the - * message, so requeue it for handling later. - */ - rv = 1; - kref_put(&user->refcount, free_user); - } else { - /* Extract the source address from the data. */ - lan_addr = (struct ipmi_lan_addr *) &recv_msg->addr; - lan_addr->addr_type = IPMI_LAN_ADDR_TYPE; - lan_addr->session_handle = msg->rsp[4]; - lan_addr->remote_SWID = msg->rsp[8]; - lan_addr->local_SWID = msg->rsp[5]; - lan_addr->lun = msg->rsp[9] & 3; - lan_addr->channel = msg->rsp[3] & 0xf; - lan_addr->privilege = msg->rsp[3] >> 4; + } else if (!IS_ERR(recv_msg)) { + /* Extract the source address from the data. */ + lan_addr = (struct ipmi_lan_addr *) &recv_msg->addr; + lan_addr->addr_type = IPMI_LAN_ADDR_TYPE; + lan_addr->session_handle = msg->rsp[4]; + lan_addr->remote_SWID = msg->rsp[8]; + lan_addr->local_SWID = msg->rsp[5]; + lan_addr->lun = msg->rsp[9] & 3; + lan_addr->channel = msg->rsp[3] & 0xf; + lan_addr->privilege = msg->rsp[3] >> 4; - /* - * Extract the rest of the message information - * from the IPMB header. - */ - recv_msg->user = user; - recv_msg->recv_type = IPMI_CMD_RECV_TYPE; - recv_msg->msgid = msg->rsp[9] >> 2; - recv_msg->msg.netfn = msg->rsp[6] >> 2; - recv_msg->msg.cmd = msg->rsp[10]; - recv_msg->msg.data = recv_msg->msg_data; + /* + * Extract the rest of the message information + * from the IPMB header. + */ + recv_msg->recv_type = IPMI_CMD_RECV_TYPE; + recv_msg->msgid = msg->rsp[9] >> 2; + recv_msg->msg.netfn = msg->rsp[6] >> 2; + recv_msg->msg.cmd = msg->rsp[10]; + recv_msg->msg.data = recv_msg->msg_data; - /* - * We chop off 12, not 11 bytes because the checksum - * at the end also needs to be removed. - */ - recv_msg->msg.data_len = msg->rsp_size - 12; - memcpy(recv_msg->msg_data, &msg->rsp[11], - msg->rsp_size - 12); - if (deliver_response(intf, recv_msg)) - ipmi_inc_stat(intf, unhandled_commands); - else - ipmi_inc_stat(intf, handled_commands); - } + /* + * We chop off 12, not 11 bytes because the checksum + * at the end also needs to be removed. + */ + recv_msg->msg.data_len = msg->rsp_size - 12; + memcpy(recv_msg->msg_data, &msg->rsp[11], + msg->rsp_size - 12); + if (deliver_response(intf, recv_msg)) + ipmi_inc_stat(intf, unhandled_commands); + else + ipmi_inc_stat(intf, handled_commands); + } else { + /* + * We couldn't allocate memory for the message, so + * requeue it for handling later. + */ + rv = 1; } return rv; @@ -4281,7 +4251,7 @@ static int handle_oem_get_msg_cmd(struct ipmi_smi *intf, unsigned char chan; struct ipmi_user *user = NULL; struct ipmi_system_interface_addr *smi_addr; - struct ipmi_recv_msg *recv_msg; + struct ipmi_recv_msg *recv_msg = NULL; /* * We expect the OEM SW to perform error checking @@ -4310,9 +4280,8 @@ static int handle_oem_get_msg_cmd(struct ipmi_smi *intf, rcvr = find_cmd_rcvr(intf, netfn, cmd, chan); if (rcvr) { user = rcvr->user; - kref_get(&user->refcount); - } else - user = NULL; + recv_msg = ipmi_alloc_recv_msg(user); + } rcu_read_unlock(); if (user == NULL) { @@ -4325,48 +4294,42 @@ static int handle_oem_get_msg_cmd(struct ipmi_smi *intf, */ rv = 0; - } else { - recv_msg = ipmi_alloc_recv_msg(); - if (!recv_msg) { - /* - * We couldn't allocate memory for the - * message, so requeue it for handling - * later. - */ - rv = 1; - kref_put(&user->refcount, free_user); - } else { - /* - * OEM Messages are expected to be delivered via - * the system interface to SMS software. We might - * need to visit this again depending on OEM - * requirements - */ - smi_addr = ((struct ipmi_system_interface_addr *) - &recv_msg->addr); - smi_addr->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE; - smi_addr->channel = IPMI_BMC_CHANNEL; - smi_addr->lun = msg->rsp[0] & 3; - - recv_msg->user = user; - recv_msg->user_msg_data = NULL; - recv_msg->recv_type = IPMI_OEM_RECV_TYPE; - recv_msg->msg.netfn = msg->rsp[0] >> 2; - recv_msg->msg.cmd = msg->rsp[1]; - recv_msg->msg.data = recv_msg->msg_data; + } else if (!IS_ERR(recv_msg)) { + /* + * OEM Messages are expected to be delivered via + * the system interface to SMS software. We might + * need to visit this again depending on OEM + * requirements + */ + smi_addr = ((struct ipmi_system_interface_addr *) + &recv_msg->addr); + smi_addr->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE; + smi_addr->channel = IPMI_BMC_CHANNEL; + smi_addr->lun = msg->rsp[0] & 3; + + recv_msg->user_msg_data = NULL; + recv_msg->recv_type = IPMI_OEM_RECV_TYPE; + recv_msg->msg.netfn = msg->rsp[0] >> 2; + recv_msg->msg.cmd = msg->rsp[1]; + recv_msg->msg.data = recv_msg->msg_data; - /* - * The message starts at byte 4 which follows the - * Channel Byte in the "GET MESSAGE" command - */ - recv_msg->msg.data_len = msg->rsp_size - 4; - memcpy(recv_msg->msg_data, &msg->rsp[4], - msg->rsp_size - 4); - if (deliver_response(intf, recv_msg)) - ipmi_inc_stat(intf, unhandled_commands); - else - ipmi_inc_stat(intf, handled_commands); - } + /* + * The message starts at byte 4 which follows the + * Channel Byte in the "GET MESSAGE" command + */ + recv_msg->msg.data_len = msg->rsp_size - 4; + memcpy(recv_msg->msg_data, &msg->rsp[4], + msg->rsp_size - 4); + if (deliver_response(intf, recv_msg)) + ipmi_inc_stat(intf, unhandled_commands); + else + ipmi_inc_stat(intf, handled_commands); + } else { + /* + * We couldn't allocate memory for the message, so + * requeue it for handling later. + */ + rv = 1; } return rv; @@ -4425,8 +4388,8 @@ static int handle_read_event_rsp(struct ipmi_smi *intf, if (!user->gets_events) continue; - recv_msg = ipmi_alloc_recv_msg(); - if (!recv_msg) { + recv_msg = ipmi_alloc_recv_msg(user); + if (IS_ERR(recv_msg)) { rcu_read_unlock(); list_for_each_entry_safe(recv_msg, recv_msg2, &msgs, link) { @@ -4445,8 +4408,6 @@ static int handle_read_event_rsp(struct ipmi_smi *intf, deliver_count++; copy_event_into_recv_msg(recv_msg, msg); - recv_msg->user = user; - kref_get(&user->refcount); list_add_tail(&recv_msg->link, &msgs); } srcu_read_unlock(&intf->users_srcu, index); @@ -4462,8 +4423,8 @@ static int handle_read_event_rsp(struct ipmi_smi *intf, * No one to receive the message, put it in queue if there's * not already too many things in the queue. */ - recv_msg = ipmi_alloc_recv_msg(); - if (!recv_msg) { + recv_msg = ipmi_alloc_recv_msg(NULL); + if (IS_ERR(recv_msg)) { /* * We couldn't allocate memory for the * message, so requeue it for handling @@ -4617,10 +4578,10 @@ static int handle_one_recv_msg(struct ipmi_smi *intf, * The NetFN and Command in the response is not even * marginally correct. */ - dev_warn(intf->si_dev, - "BMC returned incorrect response, expected netfn %x cmd %x, got netfn %x cmd %x\n", - (msg->data[0] >> 2) | 1, msg->data[1], - msg->rsp[0] >> 2, msg->rsp[1]); + dev_warn_ratelimited(intf->si_dev, + "BMC returned incorrect response, expected netfn %x cmd %x, got netfn %x cmd %x\n", + (msg->data[0] >> 2) | 1, msg->data[1], + msg->rsp[0] >> 2, msg->rsp[1]); goto return_unspecified; } @@ -5155,27 +5116,51 @@ static void free_recv_msg(struct ipmi_recv_msg *msg) kfree(msg); } -static struct ipmi_recv_msg *ipmi_alloc_recv_msg(void) +static struct ipmi_recv_msg *ipmi_alloc_recv_msg(struct ipmi_user *user) { struct ipmi_recv_msg *rv; + if (user) { + if (atomic_add_return(1, &user->nr_msgs) > max_msgs_per_user) { + atomic_dec(&user->nr_msgs); + return ERR_PTR(-EBUSY); + } + } + rv = kmalloc(sizeof(struct ipmi_recv_msg), GFP_ATOMIC); - if (rv) { - rv->user = NULL; - rv->done = free_recv_msg; - atomic_inc(&recv_msg_inuse_count); + if (!rv) { + if (user) + atomic_dec(&user->nr_msgs); + return ERR_PTR(-ENOMEM); } + + rv->user = user; + rv->done = free_recv_msg; + if (user) + kref_get(&user->refcount); + atomic_inc(&recv_msg_inuse_count); return rv; } void ipmi_free_recv_msg(struct ipmi_recv_msg *msg) { - if (msg->user && !oops_in_progress) + if (msg->user && !oops_in_progress) { + atomic_dec(&msg->user->nr_msgs); kref_put(&msg->user->refcount, free_user); + } msg->done(msg); } EXPORT_SYMBOL(ipmi_free_recv_msg); +static void ipmi_set_recv_msg_user(struct ipmi_recv_msg *msg, + struct ipmi_user *user) +{ + WARN_ON_ONCE(msg->user); /* User should not be set. */ + msg->user = user; + atomic_inc(&user->nr_msgs); + kref_get(&user->refcount); +} + static atomic_t panic_done_count = ATOMIC_INIT(0); static void dummy_smi_done_handler(struct ipmi_smi_msg *msg) diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c index 9a459257489f0..ca149ca8ccd66 100644 --- a/drivers/char/ipmi/ipmi_watchdog.c +++ b/drivers/char/ipmi/ipmi_watchdog.c @@ -1190,14 +1190,8 @@ static struct ipmi_smi_watcher smi_watcher = { .smi_gone = ipmi_smi_gone }; -static int action_op(const char *inval, char *outval) +static int action_op_set_val(const char *inval) { - if (outval) - strcpy(outval, action); - - if (!inval) - return 0; - if (strcmp(inval, "reset") == 0) action_val = WDOG_TIMEOUT_RESET; else if (strcmp(inval, "none") == 0) @@ -1208,18 +1202,26 @@ static int action_op(const char *inval, char *outval) action_val = WDOG_TIMEOUT_POWER_DOWN; else return -EINVAL; - strcpy(action, inval); return 0; } -static int preaction_op(const char *inval, char *outval) +static int action_op(const char *inval, char *outval) { + int rv; + if (outval) - strcpy(outval, preaction); + strcpy(outval, action); if (!inval) return 0; + rv = action_op_set_val(inval); + if (!rv) + strcpy(action, inval); + return rv; +} +static int preaction_op_set_val(const char *inval) +{ if (strcmp(inval, "pre_none") == 0) preaction_val = WDOG_PRETIMEOUT_NONE; else if (strcmp(inval, "pre_smi") == 0) @@ -1232,18 +1234,26 @@ static int preaction_op(const char *inval, char *outval) preaction_val = WDOG_PRETIMEOUT_MSG_INT; else return -EINVAL; - strcpy(preaction, inval); return 0; } -static int preop_op(const char *inval, char *outval) +static int preaction_op(const char *inval, char *outval) { + int rv; + if (outval) - strcpy(outval, preop); + strcpy(outval, preaction); if (!inval) return 0; + rv = preaction_op_set_val(inval); + if (!rv) + strcpy(preaction, inval); + return 0; +} +static int preop_op_set_val(const char *inval) +{ if (strcmp(inval, "preop_none") == 0) preop_val = WDOG_PREOP_NONE; else if (strcmp(inval, "preop_panic") == 0) @@ -1252,7 +1262,22 @@ static int preop_op(const char *inval, char *outval) preop_val = WDOG_PREOP_GIVE_DATA; else return -EINVAL; - strcpy(preop, inval); + return 0; +} + +static int preop_op(const char *inval, char *outval) +{ + int rv; + + if (outval) + strcpy(outval, preop); + + if (!inval) + return 0; + + rv = preop_op_set_val(inval); + if (!rv) + strcpy(preop, inval); return 0; } @@ -1289,18 +1314,18 @@ static int __init ipmi_wdog_init(void) { int rv; - if (action_op(action, NULL)) { + if (action_op_set_val(action)) { action_op("reset", NULL); pr_info("Unknown action '%s', defaulting to reset\n", action); } - if (preaction_op(preaction, NULL)) { + if (preaction_op_set_val(preaction)) { preaction_op("pre_none", NULL); pr_info("Unknown preaction '%s', defaulting to none\n", preaction); } - if (preop_op(preop, NULL)) { + if (preop_op_set_val(preop)) { preop_op("preop_none", NULL); pr_info("Unknown preop '%s', defaulting to none\n", preop); } diff --git a/drivers/char/misc.c b/drivers/char/misc.c index dda466f9181ac..8d8c4bcf07e1c 100644 --- a/drivers/char/misc.c +++ b/drivers/char/misc.c @@ -58,9 +58,8 @@ static LIST_HEAD(misc_list); static DEFINE_MUTEX(misc_mtx); /* - * Assigned numbers, used for dynamic minors + * Assigned numbers. */ -#define DYNAMIC_MINORS 128 /* like dynamic majors */ static DEFINE_IDA(misc_minors_ida); static int misc_minor_alloc(int minor) @@ -69,34 +68,17 @@ static int misc_minor_alloc(int minor) if (minor == MISC_DYNAMIC_MINOR) { /* allocate free id */ - ret = ida_alloc_max(&misc_minors_ida, DYNAMIC_MINORS - 1, GFP_KERNEL); - if (ret >= 0) { - ret = DYNAMIC_MINORS - ret - 1; - } else { - ret = ida_alloc_range(&misc_minors_ida, MISC_DYNAMIC_MINOR + 1, - MINORMASK, GFP_KERNEL); - } + ret = ida_alloc_range(&misc_minors_ida, MISC_DYNAMIC_MINOR + 1, + MINORMASK, GFP_KERNEL); } else { - /* specific minor, check if it is in dynamic or misc dynamic range */ - if (minor < DYNAMIC_MINORS) { - minor = DYNAMIC_MINORS - minor - 1; - ret = ida_alloc_range(&misc_minors_ida, minor, minor, GFP_KERNEL); - } else if (minor > MISC_DYNAMIC_MINOR) { - ret = ida_alloc_range(&misc_minors_ida, minor, minor, GFP_KERNEL); - } else { - /* case of non-dynamic minors, no need to allocate id */ - ret = 0; - } + ret = ida_alloc_range(&misc_minors_ida, minor, minor, GFP_KERNEL); } return ret; } static void misc_minor_free(int minor) { - if (minor < DYNAMIC_MINORS) - ida_free(&misc_minors_ida, DYNAMIC_MINORS - minor - 1); - else if (minor > MISC_DYNAMIC_MINOR) - ida_free(&misc_minors_ida, minor); + ida_free(&misc_minors_ida, minor); } #ifdef CONFIG_PROC_FS @@ -150,7 +132,8 @@ static int misc_open(struct inode *inode, struct file *file) break; } - if (!new_fops) { + /* Only request module for fixed minor code */ + if (!new_fops && minor < MISC_DYNAMIC_MINOR) { mutex_unlock(&misc_mtx); request_module("char-major-%d-%d", MISC_MAJOR, minor); mutex_lock(&misc_mtx); @@ -162,10 +145,11 @@ static int misc_open(struct inode *inode, struct file *file) new_fops = fops_get(iter->fops); break; } - if (!new_fops) - goto fail; } + if (!new_fops) + goto fail; + /* * Place the miscdevice in the file's * private_data so it can be used by the @@ -297,9 +281,11 @@ void misc_deregister(struct miscdevice *misc) return; mutex_lock(&misc_mtx); - list_del(&misc->list); + list_del_init(&misc->list); device_destroy(&misc_class, MKDEV(MISC_MAJOR, misc->minor)); misc_minor_free(misc->minor); + if (misc->minor > MISC_DYNAMIC_MINOR) + misc->minor = MISC_DYNAMIC_MINOR; mutex_unlock(&misc_mtx); } EXPORT_SYMBOL(misc_deregister); @@ -314,8 +300,8 @@ static int __init misc_init(void) if (err) goto fail_remove; - err = -EIO; - if (__register_chrdev(MISC_MAJOR, 0, MINORMASK + 1, "misc", &misc_fops)) + err = __register_chrdev(MISC_MAJOR, 0, MINORMASK + 1, "misc", &misc_fops); + if (err < 0) goto fail_printk; return 0; diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index c71e61ccb95a2..5e6ee5b82b8ff 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -977,8 +977,8 @@ static int tpm_tis_probe_irq_single(struct tpm_chip *chip, u32 intmask, * will call disable_irq which undoes all of the above. */ if (!(chip->flags & TPM_CHIP_FLAG_IRQ)) { - tpm_tis_write8(priv, original_int_vec, - TPM_INT_VECTOR(priv->locality)); + tpm_tis_write8(priv, TPM_INT_VECTOR(priv->locality), + original_int_vec); rc = -1; } diff --git a/drivers/char/tpm/tpm_tis_core.h b/drivers/char/tpm/tpm_tis_core.h index 369496a6aebf1..27e61ddfb6229 100644 --- a/drivers/char/tpm/tpm_tis_core.h +++ b/drivers/char/tpm/tpm_tis_core.h @@ -54,7 +54,7 @@ enum tis_int_flags { enum tis_defaults { TIS_MEM_LEN = 0x5000, TIS_SHORT_TIMEOUT = 750, /* ms */ - TIS_LONG_TIMEOUT = 2000, /* 2 sec */ + TIS_LONG_TIMEOUT = 4000, /* 4 secs */ TIS_TIMEOUT_MIN_ATML = 14700, /* usecs */ TIS_TIMEOUT_MAX_ATML = 15000, /* usecs */ }; diff --git a/drivers/clk/at91/clk-master.c b/drivers/clk/at91/clk-master.c index 15c46489ba850..4c87a0f789de1 100644 --- a/drivers/clk/at91/clk-master.c +++ b/drivers/clk/at91/clk-master.c @@ -580,6 +580,9 @@ clk_sama7g5_master_recalc_rate(struct clk_hw *hw, { struct clk_master *master = to_clk_master(hw); + if (master->div == MASTER_PRES_MAX) + return DIV_ROUND_CLOSEST_ULL(parent_rate, 3); + return DIV_ROUND_CLOSEST_ULL(parent_rate, (1 << master->div)); } diff --git a/drivers/clk/at91/clk-peripheral.c b/drivers/clk/at91/clk-peripheral.c index c173a44c800aa..629f050a855aa 100644 --- a/drivers/clk/at91/clk-peripheral.c +++ b/drivers/clk/at91/clk-peripheral.c @@ -279,8 +279,11 @@ static int clk_sam9x5_peripheral_determine_rate(struct clk_hw *hw, long best_diff = LONG_MIN; u32 shift; - if (periph->id < PERIPHERAL_ID_MIN || !periph->range.max) - return parent_rate; + if (periph->id < PERIPHERAL_ID_MIN || !periph->range.max) { + req->rate = parent_rate; + + return 0; + } /* Fist step: check the available dividers. */ for (shift = 0; shift <= PERIPHERAL_MAX_SHIFT; shift++) { diff --git a/drivers/clk/at91/clk-sam9x60-pll.c b/drivers/clk/at91/clk-sam9x60-pll.c index ff65f7b916f07..57d7aef1ea865 100644 --- a/drivers/clk/at91/clk-sam9x60-pll.c +++ b/drivers/clk/at91/clk-sam9x60-pll.c @@ -90,8 +90,8 @@ static int sam9x60_frac_pll_set(struct sam9x60_pll_core *core) spin_lock_irqsave(core->lock, flags); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_ID_MSK, core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_ID_MSK, core->id); regmap_read(regmap, AT91_PMC_PLL_CTRL1, &val); cmul = (val & core->layout->mul_mask) >> core->layout->mul_shift; cfrac = (val & core->layout->frac_mask) >> core->layout->frac_shift; @@ -125,17 +125,17 @@ static int sam9x60_frac_pll_set(struct sam9x60_pll_core *core) udelay(10); } - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, - AT91_PMC_PLL_UPDT_UPDATE | core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, + AT91_PMC_PLL_UPDT_UPDATE | core->id); regmap_update_bits(regmap, AT91_PMC_PLL_CTRL0, AT91_PMC_PLL_CTRL0_ENLOCK | AT91_PMC_PLL_CTRL0_ENPLL, AT91_PMC_PLL_CTRL0_ENLOCK | AT91_PMC_PLL_CTRL0_ENPLL); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, - AT91_PMC_PLL_UPDT_UPDATE | core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, + AT91_PMC_PLL_UPDT_UPDATE | core->id); while (!sam9x60_pll_ready(regmap, core->id)) cpu_relax(); @@ -161,8 +161,8 @@ static void sam9x60_frac_pll_unprepare(struct clk_hw *hw) spin_lock_irqsave(core->lock, flags); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_ID_MSK, core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_ID_MSK, core->id); regmap_update_bits(regmap, AT91_PMC_PLL_CTRL0, AT91_PMC_PLL_CTRL0_ENPLL, 0); @@ -170,9 +170,9 @@ static void sam9x60_frac_pll_unprepare(struct clk_hw *hw) regmap_update_bits(regmap, AT91_PMC_PLL_ACR, AT91_PMC_PLL_ACR_UTMIBG | AT91_PMC_PLL_ACR_UTMIVR, 0); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, - AT91_PMC_PLL_UPDT_UPDATE | core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, + AT91_PMC_PLL_UPDT_UPDATE | core->id); spin_unlock_irqrestore(core->lock, flags); } @@ -257,8 +257,8 @@ static int sam9x60_frac_pll_set_rate_chg(struct clk_hw *hw, unsigned long rate, spin_lock_irqsave(core->lock, irqflags); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, AT91_PMC_PLL_UPDT_ID_MSK, - core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, AT91_PMC_PLL_UPDT_ID_MSK, + core->id); regmap_read(regmap, AT91_PMC_PLL_CTRL1, &val); cmul = (val & core->layout->mul_mask) >> core->layout->mul_shift; cfrac = (val & core->layout->frac_mask) >> core->layout->frac_shift; @@ -270,18 +270,18 @@ static int sam9x60_frac_pll_set_rate_chg(struct clk_hw *hw, unsigned long rate, (frac->mul << core->layout->mul_shift) | (frac->frac << core->layout->frac_shift)); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, - AT91_PMC_PLL_UPDT_UPDATE | core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, + AT91_PMC_PLL_UPDT_UPDATE | core->id); regmap_update_bits(regmap, AT91_PMC_PLL_CTRL0, AT91_PMC_PLL_CTRL0_ENLOCK | AT91_PMC_PLL_CTRL0_ENPLL, AT91_PMC_PLL_CTRL0_ENLOCK | AT91_PMC_PLL_CTRL0_ENPLL); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, - AT91_PMC_PLL_UPDT_UPDATE | core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, + AT91_PMC_PLL_UPDT_UPDATE | core->id); while (!sam9x60_pll_ready(regmap, core->id)) cpu_relax(); @@ -333,7 +333,10 @@ static const struct clk_ops sam9x60_frac_pll_ops_chg = { .restore_context = sam9x60_frac_pll_restore_context, }; -/* This function should be called with spinlock acquired. */ +/* This function should be called with spinlock acquired. + * Warning: this function must be called only if the same PLL ID was set in + * PLL_UPDT register previously. + */ static void sam9x60_div_pll_set_div(struct sam9x60_pll_core *core, u32 div, bool enable) { @@ -345,9 +348,9 @@ static void sam9x60_div_pll_set_div(struct sam9x60_pll_core *core, u32 div, core->layout->div_mask | ena_msk, (div << core->layout->div_shift) | ena_val); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, - AT91_PMC_PLL_UPDT_UPDATE | core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, + AT91_PMC_PLL_UPDT_UPDATE | core->id); while (!sam9x60_pll_ready(regmap, core->id)) cpu_relax(); @@ -361,8 +364,8 @@ static int sam9x60_div_pll_set(struct sam9x60_pll_core *core) unsigned int val, cdiv; spin_lock_irqsave(core->lock, flags); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_ID_MSK, core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_ID_MSK, core->id); regmap_read(regmap, AT91_PMC_PLL_CTRL0, &val); cdiv = (val & core->layout->div_mask) >> core->layout->div_shift; @@ -393,15 +396,15 @@ static void sam9x60_div_pll_unprepare(struct clk_hw *hw) spin_lock_irqsave(core->lock, flags); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_ID_MSK, core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_ID_MSK, core->id); regmap_update_bits(regmap, AT91_PMC_PLL_CTRL0, core->layout->endiv_mask, 0); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, - AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, - AT91_PMC_PLL_UPDT_UPDATE | core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, + AT91_PMC_PLL_UPDT_UPDATE | AT91_PMC_PLL_UPDT_ID_MSK, + AT91_PMC_PLL_UPDT_UPDATE | core->id); spin_unlock_irqrestore(core->lock, flags); } @@ -507,8 +510,8 @@ static int sam9x60_div_pll_set_rate_chg(struct clk_hw *hw, unsigned long rate, div->div = DIV_ROUND_CLOSEST(parent_rate, rate) - 1; spin_lock_irqsave(core->lock, irqflags); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, AT91_PMC_PLL_UPDT_ID_MSK, - core->id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, AT91_PMC_PLL_UPDT_ID_MSK, + core->id); regmap_read(regmap, AT91_PMC_PLL_CTRL0, &val); cdiv = (val & core->layout->div_mask) >> core->layout->div_shift; @@ -563,8 +566,8 @@ static int sam9x60_div_pll_notifier_fn(struct notifier_block *notifier, div->div = div->safe_div; spin_lock_irqsave(core.lock, irqflags); - regmap_update_bits(regmap, AT91_PMC_PLL_UPDT, AT91_PMC_PLL_UPDT_ID_MSK, - core.id); + regmap_write_bits(regmap, AT91_PMC_PLL_UPDT, AT91_PMC_PLL_UPDT_ID_MSK, + core.id); regmap_read(regmap, AT91_PMC_PLL_CTRL0, &val); cdiv = (val & core.layout->div_mask) >> core.layout->div_shift; diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c index 4d411408e4afe..cc4ca336ac13a 100644 --- a/drivers/clk/bcm/clk-raspberrypi.c +++ b/drivers/clk/bcm/clk-raspberrypi.c @@ -271,6 +271,8 @@ static struct clk_hw *raspberrypi_clk_register(struct raspberrypi_clk *rpi, init.name = devm_kasprintf(rpi->dev, GFP_KERNEL, "fw-clk-%s", rpi_firmware_clk_names[id]); + if (!init.name) + return ERR_PTR(-ENOMEM); init.ops = &raspberrypi_firmware_clk_ops; init.flags = CLK_GET_RATE_NOCACHE; diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c index 934e53a96ddda..00bf799964c61 100644 --- a/drivers/clk/clk-axi-clkgen.c +++ b/drivers/clk/clk-axi-clkgen.c @@ -118,7 +118,7 @@ static const struct axi_clkgen_limits axi_clkgen_zynqmp_default_limits = { static const struct axi_clkgen_limits axi_clkgen_zynq_default_limits = { .fpfd_min = 10000, - .fpfd_max = 300000, + .fpfd_max = 450000, .fvco_min = 600000, .fvco_max = 1200000, }; diff --git a/drivers/clk/clk-gate.c b/drivers/clk/clk-gate.c index 64283807600b6..c9a5bc0fec5d3 100644 --- a/drivers/clk/clk-gate.c +++ b/drivers/clk/clk-gate.c @@ -26,6 +26,9 @@ static inline u32 clk_gate_readl(struct clk_gate *gate) { + if (!gate || !gate->reg) + return 0; + if (gate->flags & CLK_GATE_BIG_ENDIAN) return ioread32be(gate->reg); diff --git a/drivers/clk/clk-s2mps11.c b/drivers/clk/clk-s2mps11.c index 38c456540d1b9..337144570fafa 100644 --- a/drivers/clk/clk-s2mps11.c +++ b/drivers/clk/clk-s2mps11.c @@ -137,6 +137,8 @@ static int s2mps11_clk_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; + clk_data->num = S2MPS11_CLKS_NUM; + switch (hwid) { case S2MPS11X: s2mps11_reg = S2MPS11_REG_RTC_CTRL; @@ -186,7 +188,6 @@ static int s2mps11_clk_probe(struct platform_device *pdev) clk_data->hws[i] = &s2mps11_clks[i].hw; } - clk_data->num = S2MPS11_CLKS_NUM; of_clk_add_hw_provider(s2mps11_clks->clk_np, of_clk_hw_onecell_get, clk_data); diff --git a/drivers/clk/davinci/psc.c b/drivers/clk/davinci/psc.c index cd85d9f158b0c..6f7b8f082ad30 100644 --- a/drivers/clk/davinci/psc.c +++ b/drivers/clk/davinci/psc.c @@ -278,6 +278,11 @@ davinci_lpsc_clk_register(struct device *dev, const char *name, lpsc->pm_domain.name = devm_kasprintf(dev, GFP_KERNEL, "%s: %s", best_dev_name(dev), name); + if (!lpsc->pm_domain.name) { + clk_hw_unregister(&lpsc->hw); + kfree(lpsc); + return ERR_PTR(-ENOMEM); + } lpsc->pm_domain.attach_dev = davinci_psc_genpd_attach_dev; lpsc->pm_domain.detach_dev = davinci_psc_genpd_detach_dev; lpsc->pm_domain.flags = GENPD_FLAG_PM_CLK; diff --git a/drivers/clk/imx/clk-imx8mp.c b/drivers/clk/imx/clk-imx8mp.c index 747f5397692e5..2a0804dd4b846 100644 --- a/drivers/clk/imx/clk-imx8mp.c +++ b/drivers/clk/imx/clk-imx8mp.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -406,11 +407,151 @@ static const char * const imx8mp_clkout_sels[] = {"audio_pll1_out", "audio_pll2_ static struct clk_hw **hws; static struct clk_hw_onecell_data *clk_hw_data; +struct imx8mp_clock_constraints { + unsigned int clkid; + u32 maxrate; +}; + +/* + * Below tables are taken from IMX8MPCEC Rev. 2.1, 07/2023 + * Table 13. Maximum frequency of modules. + * Probable typos fixed are marked with a comment. + */ +static const struct imx8mp_clock_constraints imx8mp_clock_common_constraints[] = { + { IMX8MP_CLK_A53_DIV, 1000 * HZ_PER_MHZ }, + { IMX8MP_CLK_ENET_AXI, 266666667 }, /* Datasheet claims 266MHz */ + { IMX8MP_CLK_NAND_USDHC_BUS, 266666667 }, /* Datasheet claims 266MHz */ + { IMX8MP_CLK_MEDIA_APB, 200 * HZ_PER_MHZ }, + { IMX8MP_CLK_HDMI_APB, 133333333 }, /* Datasheet claims 133MHz */ + { IMX8MP_CLK_ML_AXI, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_AHB, 133333333 }, + { IMX8MP_CLK_IPG_ROOT, 66666667 }, + { IMX8MP_CLK_AUDIO_AHB, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_DISP2_PIX, 170 * HZ_PER_MHZ }, + { IMX8MP_CLK_DRAM_ALT, 666666667 }, + { IMX8MP_CLK_DRAM_APB, 200 * HZ_PER_MHZ }, + { IMX8MP_CLK_CAN1, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_CAN2, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_PCIE_AUX, 10 * HZ_PER_MHZ }, + { IMX8MP_CLK_I2C5, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_I2C6, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_SAI1, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_SAI2, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_SAI3, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_SAI5, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_SAI6, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_ENET_QOS, 125 * HZ_PER_MHZ }, + { IMX8MP_CLK_ENET_QOS_TIMER, 200 * HZ_PER_MHZ }, + { IMX8MP_CLK_ENET_REF, 125 * HZ_PER_MHZ }, + { IMX8MP_CLK_ENET_TIMER, 125 * HZ_PER_MHZ }, + { IMX8MP_CLK_ENET_PHY_REF, 125 * HZ_PER_MHZ }, + { IMX8MP_CLK_NAND, 500 * HZ_PER_MHZ }, + { IMX8MP_CLK_QSPI, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_USDHC1, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_USDHC2, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_I2C1, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_I2C2, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_I2C3, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_I2C4, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_UART1, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_UART2, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_UART3, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_UART4, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_ECSPI1, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_ECSPI2, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_PWM1, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_PWM2, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_PWM3, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_PWM4, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_GPT1, 100 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPT2, 100 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPT3, 100 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPT4, 100 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPT5, 100 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPT6, 100 * HZ_PER_MHZ }, + { IMX8MP_CLK_WDOG, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_IPP_DO_CLKO1, 200 * HZ_PER_MHZ }, + { IMX8MP_CLK_IPP_DO_CLKO2, 200 * HZ_PER_MHZ }, + { IMX8MP_CLK_HDMI_REF_266M, 266 * HZ_PER_MHZ }, + { IMX8MP_CLK_USDHC3, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_MIPI_PHY1_REF, 300 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_DISP1_PIX, 250 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_CAM2_PIX, 277 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_LDB, 595 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_MIPI_TEST_BYTE, 200 * HZ_PER_MHZ }, + { IMX8MP_CLK_ECSPI3, 80 * HZ_PER_MHZ }, + { IMX8MP_CLK_PDM, 200 * HZ_PER_MHZ }, + { IMX8MP_CLK_SAI7, 66666667 }, /* Datasheet claims 66MHz */ + { IMX8MP_CLK_MAIN_AXI, 400 * HZ_PER_MHZ }, + { /* Sentinel */ } +}; + +static const struct imx8mp_clock_constraints imx8mp_clock_nominal_constraints[] = { + { IMX8MP_CLK_M7_CORE, 600 * HZ_PER_MHZ }, + { IMX8MP_CLK_ML_CORE, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU3D_CORE, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU3D_SHADER_CORE, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU2D_CORE, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_AUDIO_AXI_SRC, 600 * HZ_PER_MHZ }, + { IMX8MP_CLK_HSIO_AXI, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_ISP, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_BUS, 600 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_AXI, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_HDMI_AXI, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU_AXI, 600 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU_AHB, 300 * HZ_PER_MHZ }, + { IMX8MP_CLK_NOC, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_NOC_IO, 600 * HZ_PER_MHZ }, + { IMX8MP_CLK_ML_AHB, 300 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_G1, 600 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_G2, 500 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_CAM1_PIX, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_VC8000E, 400 * HZ_PER_MHZ }, /* Datasheet claims 500MHz */ + { IMX8MP_CLK_DRAM_CORE, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_GIC, 400 * HZ_PER_MHZ }, + { /* Sentinel */ } +}; + +static const struct imx8mp_clock_constraints imx8mp_clock_overdrive_constraints[] = { + { IMX8MP_CLK_M7_CORE, 800 * HZ_PER_MHZ}, + { IMX8MP_CLK_ML_CORE, 1000 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU3D_CORE, 1000 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU3D_SHADER_CORE, 1000 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU2D_CORE, 1000 * HZ_PER_MHZ }, + { IMX8MP_CLK_AUDIO_AXI_SRC, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_HSIO_AXI, 500 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_ISP, 500 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_BUS, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_AXI, 500 * HZ_PER_MHZ }, + { IMX8MP_CLK_HDMI_AXI, 500 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU_AXI, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_GPU_AHB, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_NOC, 1000 * HZ_PER_MHZ }, + { IMX8MP_CLK_NOC_IO, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_ML_AHB, 400 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_G1, 800 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_G2, 700 * HZ_PER_MHZ }, + { IMX8MP_CLK_MEDIA_CAM1_PIX, 500 * HZ_PER_MHZ }, + { IMX8MP_CLK_VPU_VC8000E, 500 * HZ_PER_MHZ }, /* Datasheet claims 400MHz */ + { IMX8MP_CLK_DRAM_CORE, 1000 * HZ_PER_MHZ }, + { IMX8MP_CLK_GIC, 500 * HZ_PER_MHZ }, + { /* Sentinel */ } +}; + +static void imx8mp_clocks_apply_constraints(const struct imx8mp_clock_constraints constraints[]) +{ + const struct imx8mp_clock_constraints *constr; + + for (constr = constraints; constr->clkid; constr++) + clk_hw_set_rate_range(hws[constr->clkid], 0, constr->maxrate); +} + static int imx8mp_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np; void __iomem *anatop_base, *ccm_base; + const char *opmode; int err; np = of_find_compatible_node(NULL, NULL, "fsl,imx8mp-anatop"); @@ -715,6 +856,16 @@ static int imx8mp_clocks_probe(struct platform_device *pdev) imx_check_clk_hws(hws, IMX8MP_CLK_END); + imx8mp_clocks_apply_constraints(imx8mp_clock_common_constraints); + + err = of_property_read_string(np, "fsl,operating-mode", &opmode); + if (!err) { + if (!strcmp(opmode, "nominal")) + imx8mp_clocks_apply_constraints(imx8mp_clock_nominal_constraints); + else if (!strcmp(opmode, "overdrive")) + imx8mp_clocks_apply_constraints(imx8mp_clock_overdrive_constraints); + } + err = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); if (err < 0) { dev_err(dev, "failed to register hws for i.MX8MP\n"); diff --git a/drivers/clk/mediatek/clk-mt8195-infra_ao.c b/drivers/clk/mediatek/clk-mt8195-infra_ao.c index dfba6eb61ccfe..4ecdf9ae02443 100644 --- a/drivers/clk/mediatek/clk-mt8195-infra_ao.c +++ b/drivers/clk/mediatek/clk-mt8195-infra_ao.c @@ -103,7 +103,7 @@ static const struct mtk_gate infra_ao_clks[] = { GATE_INFRA_AO0(CLK_INFRA_AO_CQ_DMA_FPC, "infra_ao_cq_dma_fpc", "fpc", 28), GATE_INFRA_AO0(CLK_INFRA_AO_UART5, "infra_ao_uart5", "top_uart", 29), /* INFRA_AO1 */ - GATE_INFRA_AO1(CLK_INFRA_AO_HDMI_26M, "infra_ao_hdmi_26m", "clk26m", 0), + GATE_INFRA_AO1(CLK_INFRA_AO_HDMI_26M, "infra_ao_hdmi_26m", "top_hdmi_xtal", 0), GATE_INFRA_AO1(CLK_INFRA_AO_SPI0, "infra_ao_spi0", "top_spi", 1), GATE_INFRA_AO1(CLK_INFRA_AO_MSDC0, "infra_ao_msdc0", "top_msdc50_0_hclk", 2), GATE_INFRA_AO1(CLK_INFRA_AO_MSDC1, "infra_ao_msdc1", "top_axi", 4), diff --git a/drivers/clk/mediatek/clk-mux.c b/drivers/clk/mediatek/clk-mux.c index c93bc7f926e5d..359f92df826b5 100644 --- a/drivers/clk/mediatek/clk-mux.c +++ b/drivers/clk/mediatek/clk-mux.c @@ -132,9 +132,7 @@ static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) static int mtk_clk_mux_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { - struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); - - return clk_mux_determine_rate_flags(hw, req, mux->data->flags); + return clk_mux_determine_rate_flags(hw, req, 0); } const struct clk_ops mtk_mux_clr_set_upd_ops = { diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 233ce4a4c1c2f..69018efbf7f47 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -3971,6 +3971,7 @@ static const struct clk_parent_data spicc_sclk_parent_data[] = { { .hw = &g12a_clk81.hw }, { .hw = &g12a_fclk_div4.hw }, { .hw = &g12a_fclk_div3.hw }, + { .hw = &g12a_fclk_div2.hw }, { .hw = &g12a_fclk_div5.hw }, { .hw = &g12a_fclk_div7.hw }, }; diff --git a/drivers/clk/nxp/clk-lpc18xx-cgu.c b/drivers/clk/nxp/clk-lpc18xx-cgu.c index 69ebf65081b81..bbd7d64038fab 100644 --- a/drivers/clk/nxp/clk-lpc18xx-cgu.c +++ b/drivers/clk/nxp/clk-lpc18xx-cgu.c @@ -371,23 +371,25 @@ static unsigned long lpc18xx_pll0_recalc_rate(struct clk_hw *hw, return 0; } -static long lpc18xx_pll0_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) +static int lpc18xx_pll0_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) { unsigned long m; - if (*prate < rate) { + if (req->best_parent_rate < req->rate) { pr_warn("%s: pll dividers not supported\n", __func__); return -EINVAL; } - m = DIV_ROUND_UP_ULL(*prate, rate * 2); - if (m <= 0 && m > LPC18XX_PLL0_MSEL_MAX) { - pr_warn("%s: unable to support rate %lu\n", __func__, rate); + m = DIV_ROUND_UP_ULL(req->best_parent_rate, req->rate * 2); + if (m == 0 || m > LPC18XX_PLL0_MSEL_MAX) { + pr_warn("%s: unable to support rate %lu\n", __func__, req->rate); return -EINVAL; } - return 2 * *prate * m; + req->rate = 2 * req->best_parent_rate * m; + + return 0; } static int lpc18xx_pll0_set_rate(struct clk_hw *hw, unsigned long rate, @@ -403,7 +405,7 @@ static int lpc18xx_pll0_set_rate(struct clk_hw *hw, unsigned long rate, } m = DIV_ROUND_UP_ULL(parent_rate, rate * 2); - if (m <= 0 && m > LPC18XX_PLL0_MSEL_MAX) { + if (m == 0 || m > LPC18XX_PLL0_MSEL_MAX) { pr_warn("%s: unable to support rate %lu\n", __func__, rate); return -EINVAL; } @@ -444,7 +446,7 @@ static int lpc18xx_pll0_set_rate(struct clk_hw *hw, unsigned long rate, static const struct clk_ops lpc18xx_pll0_ops = { .recalc_rate = lpc18xx_pll0_recalc_rate, - .round_rate = lpc18xx_pll0_round_rate, + .determine_rate = lpc18xx_pll0_determine_rate, .set_rate = lpc18xx_pll0_set_rate, }; diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 1de1661037b1b..95cbea8d380c3 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -148,7 +148,7 @@ config IPQ_GCC_4019 config IPQ_GCC_5018 tristate "IPQ5018 Global Clock Controller" - depends on ARM64 || COMPILE_TEST + depends on ARM || ARM64 || COMPILE_TEST help Support for global clock controller on ipq5018 devices. Say Y if you want to use peripheral devices such as UART, SPI, diff --git a/drivers/clk/qcom/camcc-sm6350.c b/drivers/clk/qcom/camcc-sm6350.c index acba9f99d960c..eca36bd3ba5c9 100644 --- a/drivers/clk/qcom/camcc-sm6350.c +++ b/drivers/clk/qcom/camcc-sm6350.c @@ -1694,6 +1694,9 @@ static struct clk_branch camcc_sys_tmr_clk = { static struct gdsc bps_gdsc = { .gdscr = 0x6004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "bps_gdsc", }, @@ -1703,6 +1706,9 @@ static struct gdsc bps_gdsc = { static struct gdsc ipe_0_gdsc = { .gdscr = 0x7004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "ipe_0_gdsc", }, @@ -1712,6 +1718,9 @@ static struct gdsc ipe_0_gdsc = { static struct gdsc ife_0_gdsc = { .gdscr = 0x9004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "ife_0_gdsc", }, @@ -1720,6 +1729,9 @@ static struct gdsc ife_0_gdsc = { static struct gdsc ife_1_gdsc = { .gdscr = 0xa004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "ife_1_gdsc", }, @@ -1728,6 +1740,9 @@ static struct gdsc ife_1_gdsc = { static struct gdsc ife_2_gdsc = { .gdscr = 0xb004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "ife_2_gdsc", }, @@ -1736,6 +1751,9 @@ static struct gdsc ife_2_gdsc = { static struct gdsc titan_top_gdsc = { .gdscr = 0x14004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "titan_top_gdsc", }, diff --git a/drivers/clk/qcom/camcc-sm8250.c b/drivers/clk/qcom/camcc-sm8250.c index 9b32c56a5bc5a..e29706d782870 100644 --- a/drivers/clk/qcom/camcc-sm8250.c +++ b/drivers/clk/qcom/camcc-sm8250.c @@ -411,7 +411,7 @@ static struct clk_rcg2 cam_cc_bps_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -433,7 +433,7 @@ static struct clk_rcg2 cam_cc_camnoc_axi_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -454,7 +454,7 @@ static struct clk_rcg2 cam_cc_cci_0_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -469,7 +469,7 @@ static struct clk_rcg2 cam_cc_cci_1_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -490,7 +490,7 @@ static struct clk_rcg2 cam_cc_cphy_rx_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -511,7 +511,7 @@ static struct clk_rcg2 cam_cc_csi0phytimer_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -526,7 +526,7 @@ static struct clk_rcg2 cam_cc_csi1phytimer_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -556,7 +556,7 @@ static struct clk_rcg2 cam_cc_csi3phytimer_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -571,7 +571,7 @@ static struct clk_rcg2 cam_cc_csi4phytimer_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -586,7 +586,7 @@ static struct clk_rcg2 cam_cc_csi5phytimer_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -611,7 +611,7 @@ static struct clk_rcg2 cam_cc_fast_ahb_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -634,7 +634,7 @@ static struct clk_rcg2 cam_cc_fd_core_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -649,7 +649,7 @@ static struct clk_rcg2 cam_cc_icp_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -673,7 +673,7 @@ static struct clk_rcg2 cam_cc_ife_0_clk_src = { .parent_data = cam_cc_parent_data_2, .num_parents = ARRAY_SIZE(cam_cc_parent_data_2), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -710,7 +710,7 @@ static struct clk_rcg2 cam_cc_ife_0_csid_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -734,7 +734,7 @@ static struct clk_rcg2 cam_cc_ife_1_clk_src = { .parent_data = cam_cc_parent_data_3, .num_parents = ARRAY_SIZE(cam_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -749,7 +749,7 @@ static struct clk_rcg2 cam_cc_ife_1_csid_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -771,7 +771,7 @@ static struct clk_rcg2 cam_cc_ife_lite_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -786,7 +786,7 @@ static struct clk_rcg2 cam_cc_ife_lite_csid_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -810,7 +810,7 @@ static struct clk_rcg2 cam_cc_ipe_0_clk_src = { .parent_data = cam_cc_parent_data_4, .num_parents = ARRAY_SIZE(cam_cc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -825,7 +825,7 @@ static struct clk_rcg2 cam_cc_jpeg_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -847,7 +847,7 @@ static struct clk_rcg2 cam_cc_mclk0_clk_src = { .parent_data = cam_cc_parent_data_1, .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -862,7 +862,7 @@ static struct clk_rcg2 cam_cc_mclk1_clk_src = { .parent_data = cam_cc_parent_data_1, .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -877,7 +877,7 @@ static struct clk_rcg2 cam_cc_mclk2_clk_src = { .parent_data = cam_cc_parent_data_1, .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -892,7 +892,7 @@ static struct clk_rcg2 cam_cc_mclk3_clk_src = { .parent_data = cam_cc_parent_data_1, .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -907,7 +907,7 @@ static struct clk_rcg2 cam_cc_mclk4_clk_src = { .parent_data = cam_cc_parent_data_1, .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -922,7 +922,7 @@ static struct clk_rcg2 cam_cc_mclk5_clk_src = { .parent_data = cam_cc_parent_data_1, .num_parents = ARRAY_SIZE(cam_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -993,7 +993,7 @@ static struct clk_rcg2 cam_cc_slow_ahb_clk_src = { .parent_data = cam_cc_parent_data_0, .num_parents = ARRAY_SIZE(cam_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 80aadafffacdb..732ca46703ba3 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -645,14 +645,19 @@ clk_alpha_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); u32 alpha_width = pll_alpha_width(pll); - regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l); + if (regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l)) + return 0; + + if (regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl)) + return 0; - regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl); if (ctl & PLL_ALPHA_EN) { - regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &low); + if (regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &low)) + return 0; if (alpha_width > 32) { - regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL_U(pll), - &high); + if (regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL_U(pll), + &high)) + return 0; a = (u64)high << 32 | low; } else { a = low & GENMASK(alpha_width - 1, 0); @@ -844,8 +849,11 @@ alpha_pll_huayra_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); u32 l, alpha = 0, ctl, alpha_m, alpha_n; - regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l); - regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl); + if (regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l)) + return 0; + + if (regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl)) + return 0; if (ctl & PLL_ALPHA_EN) { regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &alpha); @@ -1039,8 +1047,11 @@ clk_trion_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); u32 l, frac, alpha_width = pll_alpha_width(pll); - regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l); - regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &frac); + if (regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l)) + return 0; + + if (regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &frac)) + return 0; return alpha_pll_calc_rate(parent_rate, l, frac, alpha_width); } @@ -1098,7 +1109,8 @@ clk_alpha_pll_postdiv_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); u32 ctl; - regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl); + if (regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl)) + return 0; ctl >>= PLL_POST_DIV_SHIFT; ctl &= PLL_POST_DIV_MASK(pll); @@ -1314,8 +1326,11 @@ static unsigned long alpha_pll_fabia_recalc_rate(struct clk_hw *hw, struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); u32 l, frac, alpha_width = pll_alpha_width(pll); - regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l); - regmap_read(pll->clkr.regmap, PLL_FRAC(pll), &frac); + if (regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l)) + return 0; + + if (regmap_read(pll->clkr.regmap, PLL_FRAC(pll), &frac)) + return 0; return alpha_pll_calc_rate(parent_rate, l, frac, alpha_width); } @@ -1465,7 +1480,8 @@ clk_trion_pll_postdiv_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) struct regmap *regmap = pll->clkr.regmap; u32 i, div = 1, val; - regmap_read(regmap, PLL_USER_CTL(pll), &val); + if (regmap_read(regmap, PLL_USER_CTL(pll), &val)) + return 0; val >>= pll->post_div_shift; val &= PLL_POST_DIV_MASK(pll); @@ -2339,9 +2355,12 @@ static unsigned long alpha_pll_lucid_evo_recalc_rate(struct clk_hw *hw, struct regmap *regmap = pll->clkr.regmap; u32 l, frac; - regmap_read(regmap, PLL_L_VAL(pll), &l); + if (regmap_read(regmap, PLL_L_VAL(pll), &l)) + return 0; l &= LUCID_EVO_PLL_L_VAL_MASK; - regmap_read(regmap, PLL_ALPHA_VAL(pll), &frac); + + if (regmap_read(regmap, PLL_ALPHA_VAL(pll), &frac)) + return 0; return alpha_pll_calc_rate(parent_rate, l, frac, pll_alpha_width(pll)); } @@ -2416,7 +2435,8 @@ static unsigned long clk_rivian_evo_pll_recalc_rate(struct clk_hw *hw, struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); u32 l; - regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l); + if (regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l)) + return 0; return parent_rate * l; } diff --git a/drivers/clk/qcom/dispcc-sm6350.c b/drivers/clk/qcom/dispcc-sm6350.c index ddacb4f76eca5..ea98a63746f0f 100644 --- a/drivers/clk/qcom/dispcc-sm6350.c +++ b/drivers/clk/qcom/dispcc-sm6350.c @@ -680,6 +680,9 @@ static struct clk_branch disp_cc_xo_clk = { static struct gdsc mdss_gdsc = { .gdscr = 0x1004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "mdss_gdsc", }, diff --git a/drivers/clk/qcom/gcc-ipq5018.c b/drivers/clk/qcom/gcc-ipq5018.c index 3136ba1c2a59c..915e84db3c97d 100644 --- a/drivers/clk/qcom/gcc-ipq5018.c +++ b/drivers/clk/qcom/gcc-ipq5018.c @@ -1370,7 +1370,7 @@ static struct clk_branch gcc_xo_clk = { &gcc_xo_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/gcc-msm8939.c b/drivers/clk/qcom/gcc-msm8939.c index b45f97c07eeb6..e4a44377b75f7 100644 --- a/drivers/clk/qcom/gcc-msm8939.c +++ b/drivers/clk/qcom/gcc-msm8939.c @@ -432,7 +432,7 @@ static const struct parent_map gcc_xo_gpll0_gpll1a_gpll6_sleep_map[] = { { P_XO, 0 }, { P_GPLL0, 1 }, { P_GPLL1_AUX, 2 }, - { P_GPLL6, 2 }, + { P_GPLL6, 3 }, { P_SLEEP_CLK, 6 }, }; @@ -1100,7 +1100,7 @@ static struct clk_rcg2 jpeg0_clk_src = { }; static const struct freq_tbl ftbl_gcc_camss_mclk0_1_clk[] = { - F(24000000, P_GPLL0, 1, 1, 45), + F(24000000, P_GPLL6, 1, 1, 45), F(66670000, P_GPLL0, 12, 0, 0), { } }; diff --git a/drivers/clk/qcom/gcc-sm6350.c b/drivers/clk/qcom/gcc-sm6350.c index 428cd99dcdcbe..4031613c6236f 100644 --- a/drivers/clk/qcom/gcc-sm6350.c +++ b/drivers/clk/qcom/gcc-sm6350.c @@ -2320,6 +2320,9 @@ static struct clk_branch gcc_video_xo_clk = { static struct gdsc usb30_prim_gdsc = { .gdscr = 0x1a004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "usb30_prim_gdsc", }, @@ -2328,6 +2331,9 @@ static struct gdsc usb30_prim_gdsc = { static struct gdsc ufs_phy_gdsc = { .gdscr = 0x3a004, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "ufs_phy_gdsc", }, diff --git a/drivers/clk/qcom/gpucc-sm6350.c b/drivers/clk/qcom/gpucc-sm6350.c index 0bcbba2a29436..86c8ad5b55bac 100644 --- a/drivers/clk/qcom/gpucc-sm6350.c +++ b/drivers/clk/qcom/gpucc-sm6350.c @@ -412,6 +412,9 @@ static struct clk_branch gpu_cc_gx_vsense_clk = { static struct gdsc gpu_cx_gdsc = { .gdscr = 0x106c, .gds_hw_ctrl = 0x1540, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x8, .pd = { .name = "gpu_cx_gdsc", }, @@ -422,6 +425,9 @@ static struct gdsc gpu_cx_gdsc = { static struct gdsc gpu_gx_gdsc = { .gdscr = 0x100c, .clamp_io_ctrl = 0x1508, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, .pd = { .name = "gpu_gx_gdsc", .power_on = gdsc_gx_do_nothing_enable, diff --git a/drivers/clk/rockchip/clk-rk3036.c b/drivers/clk/rockchip/clk-rk3036.c index d644bc155ec6e..f5f27535087a3 100644 --- a/drivers/clk/rockchip/clk-rk3036.c +++ b/drivers/clk/rockchip/clk-rk3036.c @@ -431,6 +431,7 @@ static const char *const rk3036_critical_clocks[] __initconst = { "hclk_peri", "pclk_peri", "pclk_ddrupctl", + "ddrphy", }; static void __init rk3036_clk_init(struct device_node *np) diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c index f95c3615ca772..98f107e96317e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c @@ -412,19 +412,23 @@ static const struct clk_parent_data mmc0_mmc1_parents[] = { { .hw = &pll_periph0_2x_clk.common.hw }, { .hw = &pll_audio1_div2_clk.common.hw }, }; -static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(mmc0_clk, "mmc0", mmc0_mmc1_parents, 0x830, - 0, 4, /* M */ - 8, 2, /* P */ - 24, 3, /* mux */ - BIT(31), /* gate */ - 0); - -static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(mmc1_clk, "mmc1", mmc0_mmc1_parents, 0x834, - 0, 4, /* M */ - 8, 2, /* P */ - 24, 3, /* mux */ - BIT(31), /* gate */ - 0); +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_POSTDIV(mmc0_clk, "mmc0", + mmc0_mmc1_parents, 0x830, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_POSTDIV(mmc1_clk, "mmc1", + mmc0_mmc1_parents, 0x834, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); static const struct clk_parent_data mmc2_parents[] = { { .fw_name = "hosc" }, @@ -433,12 +437,14 @@ static const struct clk_parent_data mmc2_parents[] = { { .hw = &pll_periph0_800M_clk.common.hw }, { .hw = &pll_audio1_div2_clk.common.hw }, }; -static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(mmc2_clk, "mmc2", mmc2_parents, 0x838, - 0, 4, /* M */ - 8, 2, /* P */ - 24, 3, /* mux */ - BIT(31), /* gate */ - 0); +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_POSTDIV(mmc2_clk, "mmc2", mmc2_parents, + 0x838, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); static SUNXI_CCU_GATE_HWS(bus_mmc0_clk, "bus-mmc0", psi_ahb_hws, 0x84c, BIT(0), 0); diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c index fdc8ccc586c99..ec1717ddaf275 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c +++ b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c @@ -325,6 +325,13 @@ static const struct sun6i_rtc_match_data sun50i_r329_rtc_ccu_data = { .osc32k_fanout_nparents = ARRAY_SIZE(sun50i_r329_osc32k_fanout_parents), }; +static const struct sun6i_rtc_match_data sun55i_a523_rtc_ccu_data = { + .have_ext_osc32k = true, + .have_iosc_calibration = true, + .osc32k_fanout_parents = sun50i_r329_osc32k_fanout_parents, + .osc32k_fanout_nparents = ARRAY_SIZE(sun50i_r329_osc32k_fanout_parents), +}; + static const struct of_device_id sun6i_rtc_ccu_match[] = { { .compatible = "allwinner,sun50i-h616-rtc", @@ -334,6 +341,10 @@ static const struct of_device_id sun6i_rtc_ccu_match[] = { .compatible = "allwinner,sun50i-r329-rtc", .data = &sun50i_r329_rtc_ccu_data, }, + { + .compatible = "allwinner,sun55i-a523-rtc", + .data = &sun55i_a523_rtc_ccu_data, + }, {}, }; diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c index f3ce8664b2883..b05553faed6d1 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c @@ -347,8 +347,7 @@ static SUNXI_CCU_GATE(dram_ohci_clk, "dram-ohci", "dram", static const char * const de_parents[] = { "pll-video", "pll-periph0" }; static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents, - 0x104, 0, 4, 24, 2, BIT(31), - CLK_SET_RATE_PARENT); + 0x104, 0, 4, 24, 3, BIT(31), 0); static const char * const tcon_parents[] = { "pll-video" }; static SUNXI_CCU_M_WITH_MUX_GATE(tcon_clk, "tcon", tcon_parents, diff --git a/drivers/clk/sunxi-ng/ccu_mp.h b/drivers/clk/sunxi-ng/ccu_mp.h index 6e50f3728fb5f..7d836a9fb3db3 100644 --- a/drivers/clk/sunxi-ng/ccu_mp.h +++ b/drivers/clk/sunxi-ng/ccu_mp.h @@ -52,6 +52,28 @@ struct ccu_mp { } \ } +#define SUNXI_CCU_MP_DATA_WITH_MUX_GATE_POSTDIV(_struct, _name, _parents, \ + _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + _gate, _postdiv, _flags)\ + struct ccu_mp _struct = { \ + .enable = _gate, \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .p = _SUNXI_CCU_DIV(_pshift, _pwidth), \ + .mux = _SUNXI_CCU_MUX(_muxshift, _muxwidth), \ + .fixed_post_div = _postdiv, \ + .common = { \ + .reg = _reg, \ + .features = CCU_FEATURE_FIXED_POSTDIV, \ + .hw.init = CLK_HW_INIT_PARENTS_DATA(_name, \ + _parents, \ + &ccu_mp_ops, \ + _flags), \ + } \ + } + #define SUNXI_CCU_MP_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ _mshift, _mwidth, \ _pshift, _pwidth, \ diff --git a/drivers/clk/tegra/clk-bpmp.c b/drivers/clk/tegra/clk-bpmp.c index 7bfba0afd7783..4ec408c3a26aa 100644 --- a/drivers/clk/tegra/clk-bpmp.c +++ b/drivers/clk/tegra/clk-bpmp.c @@ -635,7 +635,7 @@ static int tegra_bpmp_register_clocks(struct tegra_bpmp *bpmp, bpmp->num_clocks = count; - bpmp->clocks = devm_kcalloc(bpmp->dev, count, sizeof(struct tegra_bpmp_clk), GFP_KERNEL); + bpmp->clocks = devm_kcalloc(bpmp->dev, count, sizeof(*bpmp->clocks), GFP_KERNEL); if (!bpmp->clocks) return -ENOMEM; diff --git a/drivers/clk/tegra/clk-periph.c b/drivers/clk/tegra/clk-periph.c index 0626650a7011c..c9fc52a36fce9 100644 --- a/drivers/clk/tegra/clk-periph.c +++ b/drivers/clk/tegra/clk-periph.c @@ -51,7 +51,7 @@ static int clk_periph_determine_rate(struct clk_hw *hw, struct tegra_clk_periph *periph = to_clk_periph(hw); const struct clk_ops *div_ops = periph->div_ops; struct clk_hw *div_hw = &periph->divider.hw; - unsigned long rate; + long rate; __clk_hw_set_clk(div_hw, hw); @@ -59,7 +59,7 @@ static int clk_periph_determine_rate(struct clk_hw *hw, if (rate < 0) return rate; - req->rate = rate; + req->rate = (unsigned long)rate; return 0; } diff --git a/drivers/clk/ti/clk-33xx.c b/drivers/clk/ti/clk-33xx.c index 85c50ea39e6da..9269e6a0db6a4 100644 --- a/drivers/clk/ti/clk-33xx.c +++ b/drivers/clk/ti/clk-33xx.c @@ -258,6 +258,8 @@ static const char *enable_init_clks[] = { "dpll_ddr_m2_ck", "dpll_mpu_m2_ck", "l3_gclk", + /* WKUP_DEBUGSS_CLKCTRL - disable fails, AM335x Errata Advisory 1.0.42 */ + "l3-aon-clkctrl:0000:0", /* AM3_L3_L3_MAIN_CLKCTRL, needed during suspend */ "l3-clkctrl:00bc:0", "l4hs_gclk", diff --git a/drivers/clk/xilinx/xlnx_vcu.c b/drivers/clk/xilinx/xlnx_vcu.c index 60a3ed7c7263b..299332818ba3e 100644 --- a/drivers/clk/xilinx/xlnx_vcu.c +++ b/drivers/clk/xilinx/xlnx_vcu.c @@ -587,8 +587,8 @@ static void xvcu_unregister_clock_provider(struct xvcu_device *xvcu) xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_ENC_MCU]); if (!IS_ERR_OR_NULL(hws[CLK_XVCU_ENC_CORE])) xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_ENC_CORE]); - - clk_hw_unregister_fixed_factor(xvcu->pll_post); + if (!IS_ERR_OR_NULL(xvcu->pll_post)) + clk_hw_unregister_fixed_factor(xvcu->pll_post); } /** diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c index e95fdc49c2269..bbceb0289d457 100644 --- a/drivers/clocksource/clps711x-timer.c +++ b/drivers/clocksource/clps711x-timer.c @@ -78,24 +78,33 @@ static int __init clps711x_timer_init(struct device_node *np) unsigned int irq = irq_of_parse_and_map(np, 0); struct clk *clock = of_clk_get(np, 0); void __iomem *base = of_iomap(np, 0); + int ret = 0; if (!base) return -ENOMEM; - if (!irq) - return -EINVAL; - if (IS_ERR(clock)) - return PTR_ERR(clock); + if (!irq) { + ret = -EINVAL; + goto unmap_io; + } + if (IS_ERR(clock)) { + ret = PTR_ERR(clock); + goto unmap_io; + } switch (of_alias_get_id(np, "timer")) { case CLPS711X_CLKSRC_CLOCKSOURCE: clps711x_clksrc_init(clock, base); break; case CLPS711X_CLKSRC_CLOCKEVENT: - return _clps711x_clkevt_init(clock, base, irq); + ret = _clps711x_clkevt_init(clock, base, irq); + break; default: - return -EINVAL; + ret = -EINVAL; + break; } - return 0; +unmap_io: + iounmap(base); + return ret; } TIMER_OF_DECLARE(clps711x, "cirrus,ep7209-timer", clps711x_timer_init); diff --git a/drivers/clocksource/i8253.c b/drivers/clocksource/i8253.c index 39f7c2d736d16..b603c25f3dfaa 100644 --- a/drivers/clocksource/i8253.c +++ b/drivers/clocksource/i8253.c @@ -103,7 +103,7 @@ int __init clocksource_i8253_init(void) #ifdef CONFIG_CLKEVT_I8253 void clockevent_i8253_disable(void) { - raw_spin_lock(&i8253_lock); + guard(raw_spinlock_irqsave)(&i8253_lock); /* * Writing the MODE register should stop the counter, according to @@ -132,8 +132,6 @@ void clockevent_i8253_disable(void) outb_p(0, PIT_CH0); outb_p(0x30, PIT_MODE); - - raw_spin_unlock(&i8253_lock); } static int pit_shutdown(struct clock_event_device *evt) diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index b3ae38f367205..39c70b5ac44c9 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -114,6 +114,9 @@ static void gic_update_frequency(void *data) static int gic_starting_cpu(unsigned int cpu) { + /* Ensure the GIC counter is running */ + clear_gic_config(GIC_CONFIG_COUNTSTOP); + gic_clockevent_cpu_init(cpu, this_cpu_ptr(&gic_clockevent_device)); return 0; } @@ -248,9 +251,6 @@ static int __init gic_clocksource_of_init(struct device_node *node) pr_warn("Unable to register clock notifier\n"); } - /* And finally start the counter */ - clear_gic_config(GIC_CONFIG_COUNTSTOP); - /* * It's safe to use the MIPS GIC timer as a sched clock source only if * its ticks are stable, which is true on either the platforms with diff --git a/drivers/clocksource/timer-vf-pit.c b/drivers/clocksource/timer-vf-pit.c index 911c92146eca6..8041a8f62d1fa 100644 --- a/drivers/clocksource/timer-vf-pit.c +++ b/drivers/clocksource/timer-vf-pit.c @@ -35,30 +35,30 @@ static unsigned long cycle_per_jiffy; static inline void pit_timer_enable(void) { - __raw_writel(PITTCTRL_TEN | PITTCTRL_TIE, clkevt_base + PITTCTRL); + writel(PITTCTRL_TEN | PITTCTRL_TIE, clkevt_base + PITTCTRL); } static inline void pit_timer_disable(void) { - __raw_writel(0, clkevt_base + PITTCTRL); + writel(0, clkevt_base + PITTCTRL); } static inline void pit_irq_acknowledge(void) { - __raw_writel(PITTFLG_TIF, clkevt_base + PITTFLG); + writel(PITTFLG_TIF, clkevt_base + PITTFLG); } static u64 notrace pit_read_sched_clock(void) { - return ~__raw_readl(clksrc_base + PITCVAL); + return ~readl(clksrc_base + PITCVAL); } static int __init pit_clocksource_init(unsigned long rate) { /* set the max load value and start the clock source counter */ - __raw_writel(0, clksrc_base + PITTCTRL); - __raw_writel(~0UL, clksrc_base + PITLDVAL); - __raw_writel(PITTCTRL_TEN, clksrc_base + PITTCTRL); + writel(0, clksrc_base + PITTCTRL); + writel(~0UL, clksrc_base + PITLDVAL); + writel(PITTCTRL_TEN, clksrc_base + PITTCTRL); sched_clock_register(pit_read_sched_clock, 32, rate); return clocksource_mmio_init(clksrc_base + PITCVAL, "vf-pit", rate, @@ -76,7 +76,7 @@ static int pit_set_next_event(unsigned long delta, * hardware requirement. */ pit_timer_disable(); - __raw_writel(delta - 1, clkevt_base + PITLDVAL); + writel(delta - 1, clkevt_base + PITLDVAL); pit_timer_enable(); return 0; @@ -125,8 +125,8 @@ static struct clock_event_device clockevent_pit = { static int __init pit_clockevent_init(unsigned long rate, int irq) { - __raw_writel(0, clkevt_base + PITTCTRL); - __raw_writel(PITTFLG_TIF, clkevt_base + PITTFLG); + writel(0, clkevt_base + PITTCTRL); + writel(PITTFLG_TIF, clkevt_base + PITTFLG); BUG_ON(request_irq(irq, pit_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL, "VF pit timer", &clockevent_pit)); @@ -183,7 +183,7 @@ static int __init pit_timer_init(struct device_node *np) cycle_per_jiffy = clk_rate / (HZ); /* enable the pit module */ - __raw_writel(~PITMCR_MDIS, timer_base + PITMCR); + writel(~PITMCR_MDIS, timer_base + PITMCR); ret = pit_clocksource_init(clk_rate); if (ret) diff --git a/drivers/comedi/comedi_buf.c b/drivers/comedi/comedi_buf.c index 393966c097405..ef38e9ad98f62 100644 --- a/drivers/comedi/comedi_buf.c +++ b/drivers/comedi/comedi_buf.c @@ -368,7 +368,7 @@ static unsigned int comedi_buf_munge(struct comedi_subdevice *s, unsigned int count = 0; const unsigned int num_sample_bytes = comedi_bytes_per_sample(s); - if (!s->munge || (async->cmd.flags & CMDF_RAWDATA)) { + if (!s->munge || (async->cmd.flags & CMDF_RAWDATA) || async->cmd.chanlist_len == 0) { async->munge_count += num_bytes; return num_bytes; } diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c index 81763e3f94846..1f12f7ea3dc49 100644 --- a/drivers/comedi/comedi_fops.c +++ b/drivers/comedi/comedi_fops.c @@ -787,6 +787,7 @@ static int is_device_busy(struct comedi_device *dev) struct comedi_subdevice *s; int i; + lockdep_assert_held_write(&dev->attach_lock); lockdep_assert_held(&dev->mutex); if (!dev->attached) return 0; @@ -795,7 +796,16 @@ static int is_device_busy(struct comedi_device *dev) s = &dev->subdevices[i]; if (s->busy) return 1; - if (s->async && comedi_buf_is_mmapped(s)) + if (!s->async) + continue; + if (comedi_buf_is_mmapped(s)) + return 1; + /* + * There may be tasks still waiting on the subdevice's wait + * queue, although they should already be about to be removed + * from it since the subdevice has no active async command. + */ + if (wq_has_sleeper(&s->async->wait_head)) return 1; } @@ -825,15 +835,22 @@ static int do_devconfig_ioctl(struct comedi_device *dev, return -EPERM; if (!arg) { - if (is_device_busy(dev)) - return -EBUSY; + int rc = 0; + if (dev->attached) { - struct module *driver_module = dev->driver->module; + down_write(&dev->attach_lock); + if (is_device_busy(dev)) { + rc = -EBUSY; + } else { + struct module *driver_module = + dev->driver->module; - comedi_device_detach(dev); - module_put(driver_module); + comedi_device_detach_locked(dev); + module_put(driver_module); + } + up_write(&dev->attach_lock); } - return 0; + return rc; } if (copy_from_user(&it, arg, sizeof(it))) @@ -1556,21 +1573,30 @@ static int do_insnlist_ioctl(struct comedi_device *dev, } for (i = 0; i < n_insns; ++i) { + unsigned int n = insns[i].n; + if (insns[i].insn & INSN_MASK_WRITE) { if (copy_from_user(data, insns[i].data, - insns[i].n * sizeof(unsigned int))) { + n * sizeof(unsigned int))) { dev_dbg(dev->class_dev, "copy_from_user failed\n"); ret = -EFAULT; goto error; } + if (n < MIN_SAMPLES) { + memset(&data[n], 0, (MIN_SAMPLES - n) * + sizeof(unsigned int)); + } + } else { + memset(data, 0, max_t(unsigned int, n, MIN_SAMPLES) * + sizeof(unsigned int)); } ret = parse_insn(dev, insns + i, data, file); if (ret < 0) goto error; if (insns[i].insn & INSN_MASK_READ) { if (copy_to_user(insns[i].data, data, - insns[i].n * sizeof(unsigned int))) { + n * sizeof(unsigned int))) { dev_dbg(dev->class_dev, "copy_to_user failed\n"); ret = -EFAULT; @@ -1589,6 +1615,16 @@ static int do_insnlist_ioctl(struct comedi_device *dev, return i; } +#define MAX_INSNS MAX_SAMPLES +static int check_insnlist_len(struct comedi_device *dev, unsigned int n_insns) +{ + if (n_insns > MAX_INSNS) { + dev_dbg(dev->class_dev, "insnlist length too large\n"); + return -EINVAL; + } + return 0; +} + /* * COMEDI_INSN ioctl * synchronous instruction @@ -1633,6 +1669,12 @@ static int do_insn_ioctl(struct comedi_device *dev, ret = -EFAULT; goto error; } + if (insn->n < MIN_SAMPLES) { + memset(&data[insn->n], 0, + (MIN_SAMPLES - insn->n) * sizeof(unsigned int)); + } + } else { + memset(data, 0, n_data * sizeof(unsigned int)); } ret = parse_insn(dev, insn, data, file); if (ret < 0) @@ -2239,6 +2281,9 @@ static long comedi_unlocked_ioctl(struct file *file, unsigned int cmd, rc = -EFAULT; break; } + rc = check_insnlist_len(dev, insnlist.n_insns); + if (rc) + break; insns = kcalloc(insnlist.n_insns, sizeof(*insns), GFP_KERNEL); if (!insns) { rc = -ENOMEM; @@ -3090,6 +3135,9 @@ static int compat_insnlist(struct file *file, unsigned long arg) if (copy_from_user(&insnlist32, compat_ptr(arg), sizeof(insnlist32))) return -EFAULT; + rc = check_insnlist_len(dev, insnlist32.n_insns); + if (rc) + return rc; insns = kcalloc(insnlist32.n_insns, sizeof(*insns), GFP_KERNEL); if (!insns) return -ENOMEM; diff --git a/drivers/comedi/comedi_internal.h b/drivers/comedi/comedi_internal.h index 9b3631a654c89..cf10ba016ebc8 100644 --- a/drivers/comedi/comedi_internal.h +++ b/drivers/comedi/comedi_internal.h @@ -50,6 +50,7 @@ extern struct mutex comedi_drivers_list_lock; int insn_inval(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data); +void comedi_device_detach_locked(struct comedi_device *dev); void comedi_device_detach(struct comedi_device *dev); int comedi_device_attach(struct comedi_device *dev, struct comedi_devconfig *it); diff --git a/drivers/comedi/drivers.c b/drivers/comedi/drivers.c index d4e2ed709bfc7..ce4cde140518b 100644 --- a/drivers/comedi/drivers.c +++ b/drivers/comedi/drivers.c @@ -158,7 +158,7 @@ static void comedi_device_detach_cleanup(struct comedi_device *dev) int i; struct comedi_subdevice *s; - lockdep_assert_held(&dev->attach_lock); + lockdep_assert_held_write(&dev->attach_lock); lockdep_assert_held(&dev->mutex); if (dev->subdevices) { for (i = 0; i < dev->n_subdevices; i++) { @@ -195,16 +195,23 @@ static void comedi_device_detach_cleanup(struct comedi_device *dev) comedi_clear_hw_dev(dev); } -void comedi_device_detach(struct comedi_device *dev) +void comedi_device_detach_locked(struct comedi_device *dev) { + lockdep_assert_held_write(&dev->attach_lock); lockdep_assert_held(&dev->mutex); comedi_device_cancel_all(dev); - down_write(&dev->attach_lock); dev->attached = false; dev->detach_count++; if (dev->driver) dev->driver->detach(dev); comedi_device_detach_cleanup(dev); +} + +void comedi_device_detach(struct comedi_device *dev) +{ + lockdep_assert_held(&dev->mutex); + down_write(&dev->attach_lock); + comedi_device_detach_locked(dev); up_write(&dev->attach_lock); } @@ -338,10 +345,10 @@ int comedi_dio_insn_config(struct comedi_device *dev, unsigned int *data, unsigned int mask) { - unsigned int chan_mask = 1 << CR_CHAN(insn->chanspec); + unsigned int chan = CR_CHAN(insn->chanspec); - if (!mask) - mask = chan_mask; + if (!mask && chan < 32) + mask = 1U << chan; switch (data[0]) { case INSN_CONFIG_DIO_INPUT: @@ -381,7 +388,7 @@ EXPORT_SYMBOL_GPL(comedi_dio_insn_config); unsigned int comedi_dio_update_state(struct comedi_subdevice *s, unsigned int *data) { - unsigned int chanmask = (s->n_chan < 32) ? ((1 << s->n_chan) - 1) + unsigned int chanmask = (s->n_chan < 32) ? ((1U << s->n_chan) - 1) : 0xffffffff; unsigned int mask = data[0] & chanmask; unsigned int bits = data[1]; @@ -612,6 +619,7 @@ static int insn_rw_emulate_bits(struct comedi_device *dev, unsigned int chan = CR_CHAN(insn->chanspec); unsigned int base_chan = (chan < 32) ? 0 : chan; unsigned int _data[2]; + unsigned int i; int ret; memset(_data, 0, sizeof(_data)); @@ -624,18 +632,21 @@ static int insn_rw_emulate_bits(struct comedi_device *dev, if (insn->insn == INSN_WRITE) { if (!(s->subdev_flags & SDF_WRITABLE)) return -EINVAL; - _data[0] = 1 << (chan - base_chan); /* mask */ - _data[1] = data[0] ? (1 << (chan - base_chan)) : 0; /* bits */ + _data[0] = 1U << (chan - base_chan); /* mask */ } + for (i = 0; i < insn->n; i++) { + if (insn->insn == INSN_WRITE) + _data[1] = data[i] ? _data[0] : 0; /* bits */ - ret = s->insn_bits(dev, s, &_insn, _data); - if (ret < 0) - return ret; + ret = s->insn_bits(dev, s, &_insn, _data); + if (ret < 0) + return ret; - if (insn->insn == INSN_READ) - data[0] = (_data[1] >> (chan - base_chan)) & 1; + if (insn->insn == INSN_READ) + data[i] = (_data[1] >> (chan - base_chan)) & 1; + } - return 1; + return insn->n; } static int __comedi_device_postconfig_async(struct comedi_device *dev, @@ -708,7 +719,7 @@ static int __comedi_device_postconfig(struct comedi_device *dev) if (s->type == COMEDI_SUBD_DO) { if (s->n_chan < 32) - s->io_bits = (1 << s->n_chan) - 1; + s->io_bits = (1U << s->n_chan) - 1; else s->io_bits = 0xffffffff; } diff --git a/drivers/comedi/drivers/aio_iiro_16.c b/drivers/comedi/drivers/aio_iiro_16.c index b00fab0b89d4c..739cc4db52ac7 100644 --- a/drivers/comedi/drivers/aio_iiro_16.c +++ b/drivers/comedi/drivers/aio_iiro_16.c @@ -177,7 +177,8 @@ static int aio_iiro_16_attach(struct comedi_device *dev, * Digital input change of state interrupts are optionally supported * using IRQ 2-7, 10-12, 14, or 15. */ - if ((1 << it->options[1]) & 0xdcfc) { + if (it->options[1] > 0 && it->options[1] < 16 && + (1 << it->options[1]) & 0xdcfc) { ret = request_irq(it->options[1], aio_iiro_16_cos, 0, dev->board_name, dev); if (ret == 0) diff --git a/drivers/comedi/drivers/comedi_test.c b/drivers/comedi/drivers/comedi_test.c index 05ae9122823f8..e713ef611434d 100644 --- a/drivers/comedi/drivers/comedi_test.c +++ b/drivers/comedi/drivers/comedi_test.c @@ -790,7 +790,7 @@ static void waveform_detach(struct comedi_device *dev) { struct waveform_private *devpriv = dev->private; - if (devpriv) { + if (devpriv && dev->n_subdevices) { del_timer_sync(&devpriv->ai_timer); del_timer_sync(&devpriv->ao_timer); } diff --git a/drivers/comedi/drivers/das16m1.c b/drivers/comedi/drivers/das16m1.c index 275effb77746b..99e63d51c506c 100644 --- a/drivers/comedi/drivers/das16m1.c +++ b/drivers/comedi/drivers/das16m1.c @@ -522,7 +522,8 @@ static int das16m1_attach(struct comedi_device *dev, devpriv->extra_iobase = dev->iobase + DAS16M1_8255_IOBASE; /* only irqs 2, 3, 4, 5, 6, 7, 10, 11, 12, 14, and 15 are valid */ - if ((1 << it->options[1]) & 0xdcfc) { + if (it->options[1] >= 2 && it->options[1] <= 15 && + (1 << it->options[1]) & 0xdcfc) { ret = request_irq(it->options[1], das16m1_interrupt, 0, dev->board_name, dev); if (ret == 0) diff --git a/drivers/comedi/drivers/das6402.c b/drivers/comedi/drivers/das6402.c index 1af394591e748..e730668236c26 100644 --- a/drivers/comedi/drivers/das6402.c +++ b/drivers/comedi/drivers/das6402.c @@ -567,7 +567,8 @@ static int das6402_attach(struct comedi_device *dev, das6402_reset(dev); /* IRQs 2,3,5,6,7, 10,11,15 are valid for "enhanced" mode */ - if ((1 << it->options[1]) & 0x8cec) { + if (it->options[1] > 0 && it->options[1] < 16 && + (1 << it->options[1]) & 0x8cec) { ret = request_irq(it->options[1], das6402_interrupt, 0, dev->board_name, dev); if (ret == 0) { diff --git a/drivers/comedi/drivers/pcl726.c b/drivers/comedi/drivers/pcl726.c index 0430630e6ebb9..b542896fa0e42 100644 --- a/drivers/comedi/drivers/pcl726.c +++ b/drivers/comedi/drivers/pcl726.c @@ -328,7 +328,8 @@ static int pcl726_attach(struct comedi_device *dev, * Hook up the external trigger source interrupt only if the * user config option is valid and the board supports interrupts. */ - if (it->options[1] && (board->irq_mask & (1 << it->options[1]))) { + if (it->options[1] > 0 && it->options[1] < 16 && + (board->irq_mask & (1U << it->options[1]))) { ret = request_irq(it->options[1], pcl726_interrupt, 0, dev->board_name, dev); if (ret == 0) { diff --git a/drivers/comedi/drivers/pcl812.c b/drivers/comedi/drivers/pcl812.c index 70dbc129fcf55..e0459f286fa61 100644 --- a/drivers/comedi/drivers/pcl812.c +++ b/drivers/comedi/drivers/pcl812.c @@ -1149,7 +1149,8 @@ static int pcl812_attach(struct comedi_device *dev, struct comedi_devconfig *it) if (!dev->pacer) return -ENOMEM; - if ((1 << it->options[1]) & board->irq_bits) { + if (it->options[1] > 0 && it->options[1] < 16 && + (1 << it->options[1]) & board->irq_bits) { ret = request_irq(it->options[1], pcl812_interrupt, 0, dev->board_name, dev); if (ret == 0) diff --git a/drivers/counter/interrupt-cnt.c b/drivers/counter/interrupt-cnt.c index 229473855c5b3..bc762ba87a19b 100644 --- a/drivers/counter/interrupt-cnt.c +++ b/drivers/counter/interrupt-cnt.c @@ -3,12 +3,14 @@ * Copyright (c) 2021 Pengutronix, Oleksij Rempel */ +#include #include #include #include #include #include #include +#include #include #include @@ -19,6 +21,7 @@ struct interrupt_cnt_priv { struct gpio_desc *gpio; int irq; bool enabled; + struct mutex lock; struct counter_signal signals; struct counter_synapse synapses; struct counter_count cnts; @@ -41,6 +44,8 @@ static int interrupt_cnt_enable_read(struct counter_device *counter, { struct interrupt_cnt_priv *priv = counter_priv(counter); + guard(mutex)(&priv->lock); + *enable = priv->enabled; return 0; @@ -51,6 +56,8 @@ static int interrupt_cnt_enable_write(struct counter_device *counter, { struct interrupt_cnt_priv *priv = counter_priv(counter); + guard(mutex)(&priv->lock); + if (priv->enabled == enable) return 0; @@ -227,6 +234,8 @@ static int interrupt_cnt_probe(struct platform_device *pdev) if (ret) return ret; + mutex_init(&priv->lock); + ret = devm_counter_add(dev, counter); if (ret < 0) return dev_err_probe(dev, ret, "Failed to add counter\n"); diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 0615e7fa20ad7..0bfced81cc17e 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -659,7 +659,7 @@ static u64 get_max_boost_ratio(unsigned int cpu, u64 *nominal_freq) nominal_perf = perf_caps.nominal_perf; if (nominal_freq) - *nominal_freq = perf_caps.nominal_freq; + *nominal_freq = perf_caps.nominal_freq * 1000; if (!highest_perf || !nominal_perf) { pr_debug("CPU%d: highest or nominal performance missing\n", cpu); diff --git a/drivers/cpufreq/armada-8k-cpufreq.c b/drivers/cpufreq/armada-8k-cpufreq.c index 8afefdea4d80c..8a01032e57fa7 100644 --- a/drivers/cpufreq/armada-8k-cpufreq.c +++ b/drivers/cpufreq/armada-8k-cpufreq.c @@ -103,7 +103,7 @@ static void armada_8k_cpufreq_free_table(struct freq_table *freq_tables) { int opps_index, nb_cpus = num_possible_cpus(); - for (opps_index = 0 ; opps_index <= nb_cpus; opps_index++) { + for (opps_index = 0 ; opps_index < nb_cpus; opps_index++) { int i; /* If cpu_dev is NULL then we reached the end of the array */ diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index aa34af940cb53..4d96eed64fe0c 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -344,6 +344,16 @@ static int cppc_verify_policy(struct cpufreq_policy_data *policy) return 0; } +static unsigned int __cppc_cpufreq_get_transition_delay_us(unsigned int cpu) +{ + unsigned int transition_latency_ns = cppc_get_transition_latency(cpu); + + if (transition_latency_ns == CPUFREQ_ETERNAL) + return CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS / NSEC_PER_USEC; + + return transition_latency_ns / NSEC_PER_USEC; +} + /* * The PCC subspace describes the rate at which platform can accept commands * on the shared PCC channel (including READs which do not count towards freq @@ -366,12 +376,12 @@ static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) return 10000; } } - return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; + return __cppc_cpufreq_get_transition_delay_us(cpu); } #else static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) { - return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; + return __cppc_cpufreq_get_transition_delay_us(cpu); } #endif @@ -847,7 +857,7 @@ static struct freq_attr *cppc_cpufreq_attr[] = { }; static struct cpufreq_driver cppc_cpufreq_driver = { - .flags = CPUFREQ_CONST_LOOPS, + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, .verify = cppc_verify_policy, .target = cppc_cpufreq_set_target, .get = cppc_cpufreq_get_rate, diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 99c31837084c0..5a2feebc97b28 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -165,6 +165,7 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,sm8350", }, { .compatible = "qcom,sm8450", }, { .compatible = "qcom,sm8550", }, + { .compatible = "qcom,sm8650", }, { .compatible = "st,stih407", }, { .compatible = "st,stih410", }, diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c old mode 100644 new mode 100755 index 3a7bd62ef6b7b..ac670943574c0 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -458,7 +458,7 @@ void cpufreq_freq_transition_end(struct cpufreq_policy *policy, arch_set_freq_scale(policy->related_cpus, policy->cur, - policy->cpuinfo.max_freq); + arch_scale_freq_ref(policy->cpu)); spin_lock(&policy->transition_lock); policy->transition_ongoing = false; @@ -810,7 +810,7 @@ static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, /* * show_scaling_governor - show the current policy for the specified CPU */ -static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) +ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) { if (policy->policy == CPUFREQ_POLICY_POWERSAVE) return sprintf(buf, "powersave\n"); @@ -825,7 +825,7 @@ static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) /* * store_scaling_governor - store policy for the specified CPU */ -static ssize_t store_scaling_governor(struct cpufreq_policy *policy, +ssize_t store_scaling_governor(struct cpufreq_policy *policy, const char *buf, size_t count) { char str_governor[16]; @@ -1284,6 +1284,8 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) goto err_free_real_cpus; } + init_rwsem(&policy->rwsem); + freq_constraints_init(&policy->constraints); policy->nb_min.notifier_call = cpufreq_notifier_min; @@ -1306,7 +1308,6 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) } INIT_LIST_HEAD(&policy->policy_list); - init_rwsem(&policy->rwsem); spin_lock_init(&policy->transition_lock); init_waitqueue_head(&policy->transition_wait); INIT_WORK(&policy->update, handle_update); @@ -1499,6 +1500,7 @@ static int cpufreq_online(unsigned int cpu) } trace_android_vh_cpufreq_online(policy); + trace_android_rvh_cpufreq_create_policy(policy); blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); @@ -2206,7 +2208,7 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, policy->cur = freq; arch_set_freq_scale(policy->related_cpus, freq, - policy->cpuinfo.max_freq); + arch_scale_freq_ref(policy->cpu)); cpufreq_stats_record_transition(policy, freq); cpufreq_times_record_transition(policy, freq); trace_android_rvh_cpufreq_transition(policy); @@ -2697,10 +2699,12 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, pr_debug("starting governor %s failed\n", policy->governor->name); if (old_gov) { policy->governor = old_gov; - if (cpufreq_init_governor(policy)) + if (cpufreq_init_governor(policy)) { policy->governor = NULL; - else - cpufreq_start_governor(policy); + } else if (cpufreq_start_governor(policy)) { + cpufreq_exit_governor(policy); + policy->governor = NULL; + } } return ret; @@ -2936,15 +2940,6 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) cpufreq_driver = driver_data; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - /* - * Mark support for the scheduler's frequency invariance engine for - * drivers that implement target(), target_index() or fast_switch(). - */ - if (!cpufreq_driver->setpolicy) { - static_branch_enable_cpuslocked(&cpufreq_freq_invariance); - pr_debug("supports frequency invariance"); - } - if (driver_data->setpolicy) driver_data->flags |= CPUFREQ_CONST_LOOPS; @@ -2954,6 +2949,15 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) goto err_null_driver; } + /* + * Mark support for the scheduler's frequency invariance engine for + * drivers that implement target(), target_index() or fast_switch(). + */ + if (!cpufreq_driver->setpolicy) { + static_branch_enable_cpuslocked(&cpufreq_freq_invariance); + pr_debug("cpufreq: supports frequency invariance\n"); + } + ret = subsys_interface_register(&cpufreq_interface); if (ret) goto err_boost_unreg; @@ -2981,6 +2985,8 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) err_if_unreg: subsys_interface_unregister(&cpufreq_interface); err_boost_unreg: + if (!cpufreq_driver->setpolicy) + static_branch_disable_cpuslocked(&cpufreq_freq_invariance); remove_boost_sysfs_file(); err_null_driver: write_lock_irqsave(&cpufreq_driver_lock, flags); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 8a4fdf212ce0d..60b4e3b608c00 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -172,7 +172,6 @@ struct vid_data { * based on the MSR_IA32_MISC_ENABLE value and whether or * not the maximum reported turbo P-state is different from * the maximum reported non-turbo one. - * @turbo_disabled_mf: The @turbo_disabled value reflected by cpuinfo.max_freq. * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo * P-state capacity. * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo @@ -181,7 +180,6 @@ struct vid_data { struct global_params { bool no_turbo; bool turbo_disabled; - bool turbo_disabled_mf; int max_perf_pct; int min_perf_pct; }; @@ -592,16 +590,16 @@ static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq); } -static inline void update_turbo_state(void) +static bool turbo_is_disabled(void) { u64 misc_en; - struct cpudata *cpu; - cpu = all_cpu_data[0]; + if (!cpu_feature_enabled(X86_FEATURE_IDA)) + return true; + rdmsrl(MSR_IA32_MISC_ENABLE, misc_en); - global.turbo_disabled = - (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE || - cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); + + return !!(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); } static int min_perf_pct_min(void) @@ -1156,40 +1154,16 @@ static void intel_pstate_update_policies(void) static void __intel_pstate_update_max_freq(struct cpudata *cpudata, struct cpufreq_policy *policy) { - policy->cpuinfo.max_freq = global.turbo_disabled_mf ? + policy->cpuinfo.max_freq = global.turbo_disabled ? cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; refresh_frequency_limits(policy); } -static void intel_pstate_update_max_freq(unsigned int cpu) -{ - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); - - if (!policy) - return; - - __intel_pstate_update_max_freq(all_cpu_data[cpu], policy); - - cpufreq_cpu_release(policy); -} - static void intel_pstate_update_limits(unsigned int cpu) { mutex_lock(&intel_pstate_driver_lock); - update_turbo_state(); - /* - * If turbo has been turned on or off globally, policy limits for - * all CPUs need to be updated to reflect that. - */ - if (global.turbo_disabled_mf != global.turbo_disabled) { - global.turbo_disabled_mf = global.turbo_disabled; - arch_set_max_freq_ratio(global.turbo_disabled); - for_each_possible_cpu(cpu) - intel_pstate_update_max_freq(cpu); - } else { - cpufreq_update_policy(cpu); - } + cpufreq_update_policy(cpu); mutex_unlock(&intel_pstate_driver_lock); } @@ -1289,11 +1263,7 @@ static ssize_t show_no_turbo(struct kobject *kobj, return -EAGAIN; } - update_turbo_state(); - if (global.turbo_disabled) - ret = sprintf(buf, "%u\n", global.turbo_disabled); - else - ret = sprintf(buf, "%u\n", global.no_turbo); + ret = sprintf(buf, "%u\n", global.no_turbo); mutex_unlock(&intel_pstate_driver_lock); @@ -1304,32 +1274,39 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, const char *buf, size_t count) { unsigned int input; - int ret; + bool no_turbo; - ret = sscanf(buf, "%u", &input); - if (ret != 1) + if (sscanf(buf, "%u", &input) != 1) return -EINVAL; mutex_lock(&intel_pstate_driver_lock); if (!intel_pstate_driver) { - mutex_unlock(&intel_pstate_driver_lock); - return -EAGAIN; + count = -EAGAIN; + goto unlock_driver; } - mutex_lock(&intel_pstate_limits_lock); + no_turbo = !!clamp_t(int, input, 0, 1); - update_turbo_state(); - if (global.turbo_disabled) { - pr_notice_once("Turbo disabled by BIOS or unavailable on processor\n"); - mutex_unlock(&intel_pstate_limits_lock); - mutex_unlock(&intel_pstate_driver_lock); - return -EPERM; + WRITE_ONCE(global.turbo_disabled, turbo_is_disabled()); + if (global.turbo_disabled && !no_turbo) { + pr_notice("Turbo disabled by BIOS or unavailable on processor\n"); + count = -EPERM; + if (global.no_turbo) + goto unlock_driver; + else + no_turbo = 1; + } + + if (no_turbo == global.no_turbo) { + goto unlock_driver; } - global.no_turbo = clamp_t(int, input, 0, 1); + WRITE_ONCE(global.no_turbo, no_turbo); + + mutex_lock(&intel_pstate_limits_lock); - if (global.no_turbo) { + if (no_turbo) { struct cpudata *cpu = all_cpu_data[0]; int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate; @@ -1341,8 +1318,9 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, mutex_unlock(&intel_pstate_limits_lock); intel_pstate_update_policies(); - arch_set_max_freq_ratio(global.no_turbo); + arch_set_max_freq_ratio(no_turbo); +unlock_driver: mutex_unlock(&intel_pstate_driver_lock); return count; @@ -1363,10 +1341,10 @@ static void update_qos_request(enum freq_qos_req_type type) continue; req = policy->driver_data; - cpufreq_cpu_put(policy); - - if (!req) + if (!req) { + cpufreq_cpu_put(policy); continue; + } if (hwp_active) intel_pstate_get_hwp_cap(cpu); @@ -1382,6 +1360,8 @@ static void update_qos_request(enum freq_qos_req_type type) if (freq_qos_update_request(req, freq) < 0) pr_warn("Failed to update freq constraint: CPU%d\n", i); + + cpufreq_cpu_put(policy); } } @@ -1793,7 +1773,7 @@ static u64 atom_get_val(struct cpudata *cpudata, int pstate) u32 vid; val = (u64)pstate << 8; - if (global.no_turbo && !global.turbo_disabled) + if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled)) val |= (u64)1 << 32; vid_fp = cpudata->vid.min + mul_fp( @@ -1958,7 +1938,7 @@ static u64 core_get_val(struct cpudata *cpudata, int pstate) u64 val; val = (u64)pstate << 8; - if (global.no_turbo && !global.turbo_disabled) + if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled)) val |= (u64)1 << 32; return val; @@ -2031,14 +2011,6 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu) intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate); } -static void intel_pstate_max_within_limits(struct cpudata *cpu) -{ - int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio); - - update_turbo_state(); - intel_pstate_set_pstate(cpu, pstate); -} - static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) { int perf_ctl_max_phys = pstate_funcs.get_max_physical(cpu->cpu); @@ -2264,7 +2236,7 @@ static inline int32_t get_target_pstate(struct cpudata *cpu) sample->busy_scaled = busy_frac * 100; - target = global.no_turbo || global.turbo_disabled ? + target = READ_ONCE(global.no_turbo) ? cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; target += target >> 2; target = mul_fp(target, busy_frac); @@ -2308,8 +2280,6 @@ static void intel_pstate_adjust_pstate(struct cpudata *cpu) struct sample *sample; int target_pstate; - update_turbo_state(); - target_pstate = get_target_pstate(cpu); target_pstate = intel_pstate_prepare_request(cpu, target_pstate); trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu); @@ -2527,7 +2497,7 @@ static void intel_pstate_clear_update_util_hook(unsigned int cpu) static int intel_pstate_get_max_freq(struct cpudata *cpu) { - return global.turbo_disabled || global.no_turbo ? + return READ_ONCE(global.no_turbo) ? cpu->pstate.max_freq : cpu->pstate.turbo_freq; } @@ -2612,12 +2582,14 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) intel_pstate_update_perf_limits(cpu, policy->min, policy->max); if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { + int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio); + /* * NOHZ_FULL CPUs need this as the governor callback may not * be invoked on them. */ intel_pstate_clear_update_util_hook(policy->cpu); - intel_pstate_max_within_limits(cpu); + intel_pstate_set_pstate(cpu, pstate); } else { intel_pstate_set_update_util_hook(policy->cpu); } @@ -2660,10 +2632,9 @@ static void intel_pstate_verify_cpu_policy(struct cpudata *cpu, { int max_freq; - update_turbo_state(); if (hwp_active) { intel_pstate_get_hwp_cap(cpu); - max_freq = global.no_turbo || global.turbo_disabled ? + max_freq = READ_ONCE(global.no_turbo) ? cpu->pstate.max_freq : cpu->pstate.turbo_freq; } else { max_freq = intel_pstate_get_max_freq(cpu); @@ -2757,8 +2728,6 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) /* cpuinfo and default policy values */ policy->cpuinfo.min_freq = cpu->pstate.min_freq; - update_turbo_state(); - global.turbo_disabled_mf = global.turbo_disabled; policy->cpuinfo.max_freq = global.turbo_disabled ? cpu->pstate.max_freq : cpu->pstate.turbo_freq; @@ -2902,8 +2871,8 @@ static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy, int max_pstate = policy->strict_target ? target_pstate : cpu->max_perf_ratio; - intel_cpufreq_hwp_update(cpu, target_pstate, max_pstate, 0, - fast_switch); + intel_cpufreq_hwp_update(cpu, target_pstate, max_pstate, + target_pstate, fast_switch); } else if (target_pstate != old_pstate) { intel_cpufreq_perf_ctl_update(cpu, target_pstate, fast_switch); } @@ -2924,8 +2893,6 @@ static int intel_cpufreq_target(struct cpufreq_policy *policy, struct cpufreq_freqs freqs; int target_pstate; - update_turbo_state(); - freqs.old = policy->cur; freqs.new = target_freq; @@ -2947,8 +2914,6 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, struct cpudata *cpu = all_cpu_data[policy->cpu]; int target_pstate; - update_turbo_state(); - target_pstate = intel_pstate_freq_to_hwp(cpu, target_freq); target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true); @@ -2966,7 +2931,6 @@ static void intel_cpufreq_adjust_perf(unsigned int cpunum, int old_pstate = cpu->pstate.current_pstate; int cap_pstate, min_pstate, max_pstate, target_pstate; - update_turbo_state(); cap_pstate = global.turbo_disabled ? HWP_GUARANTEED_PERF(hwp_cap) : HWP_HIGHEST_PERF(hwp_cap); @@ -3156,6 +3120,10 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) memset(&global, 0, sizeof(global)); global.max_perf_pct = 100; + global.turbo_disabled = turbo_is_disabled(); + global.no_turbo = global.turbo_disabled; + + arch_set_max_freq_ratio(global.turbo_disabled); intel_pstate_driver = driver; ret = cpufreq_register_driver(intel_pstate_driver); diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c index 4c57c6725c134..1412d4617a04a 100644 --- a/drivers/cpufreq/longhaul.c +++ b/drivers/cpufreq/longhaul.c @@ -953,6 +953,9 @@ static void __exit longhaul_exit(void) struct cpufreq_policy *policy = cpufreq_cpu_get(0); int i; + if (unlikely(!policy)) + return; + for (i = 0; i < numscales; i++) { if (mults[i] == maxmult) { struct cpufreq_freqs freqs; diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index c4aa94559b94f..db2cd8f025a5d 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -323,6 +324,49 @@ static struct cpufreq_driver scmi_cpufreq_driver = { .register_em = scmi_cpufreq_register_em, }; +static bool scmi_dev_used_by_cpus(struct device *scmi_dev) +{ + struct device_node *scmi_np = dev_of_node(scmi_dev); + struct device_node *cpu_np, *np; + struct device *cpu_dev; + int cpu, idx; + + if (!scmi_np) + return false; + + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + continue; + + cpu_np = dev_of_node(cpu_dev); + + np = of_parse_phandle(cpu_np, "clocks", 0); + of_node_put(np); + + if (np == scmi_np) + return true; + + idx = of_property_match_string(cpu_np, "power-domain-names", "perf"); + np = of_parse_phandle(cpu_np, "power-domains", idx); + of_node_put(np); + + if (np == scmi_np) + return true; + } + + /* + * Older Broadcom STB chips had a "clocks" property for CPU node(s) + * that did not match the SCMI performance protocol node, if we got + * there, it means we had such an older Device Tree, therefore return + * true to preserve backwards compatibility. + */ + if (of_machine_is_compatible("brcm,brcmstb")) + return true; + + return false; +} + static int scmi_cpufreq_probe(struct scmi_device *sdev) { int ret; @@ -331,7 +375,7 @@ static int scmi_cpufreq_probe(struct scmi_device *sdev) handle = sdev->handle; - if (!handle) + if (!handle || !scmi_dev_used_by_cpus(dev)) return -ENODEV; perf_ops = handle->devm_protocol_get(sdev, SCMI_PROTOCOL_PERF, &ph); diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c index 7b8fcfa55038b..233c82a834086 100644 --- a/drivers/cpufreq/tegra186-cpufreq.c +++ b/drivers/cpufreq/tegra186-cpufreq.c @@ -86,10 +86,14 @@ static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy, { struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); struct cpufreq_frequency_table *tbl = policy->freq_table + index; - unsigned int edvd_offset = data->cpus[policy->cpu].edvd_offset; + unsigned int edvd_offset; u32 edvd_val = tbl->driver_data; + u32 cpu; - writel(edvd_val, data->regs + edvd_offset); + for_each_cpu(cpu, policy->cpus) { + edvd_offset = data->cpus[cpu].edvd_offset; + writel(edvd_val, data->regs + edvd_offset); + } return 0; } @@ -128,13 +132,14 @@ static struct cpufreq_driver tegra186_cpufreq_driver = { static struct cpufreq_frequency_table *init_vhint_table( struct platform_device *pdev, struct tegra_bpmp *bpmp, - struct tegra186_cpufreq_cluster *cluster, unsigned int cluster_id) + struct tegra186_cpufreq_cluster *cluster, unsigned int cluster_id, + int *num_rates) { struct cpufreq_frequency_table *table; struct mrq_cpu_vhint_request req; struct tegra_bpmp_message msg; struct cpu_vhint_data *data; - int err, i, j, num_rates = 0; + int err, i, j; dma_addr_t phys; void *virt; @@ -164,6 +169,7 @@ static struct cpufreq_frequency_table *init_vhint_table( goto free; } + *num_rates = 0; for (i = data->vfloor; i <= data->vceil; i++) { u16 ndiv = data->ndiv[i]; @@ -174,10 +180,10 @@ static struct cpufreq_frequency_table *init_vhint_table( if (i > 0 && ndiv == data->ndiv[i - 1]) continue; - num_rates++; + (*num_rates)++; } - table = devm_kcalloc(&pdev->dev, num_rates + 1, sizeof(*table), + table = devm_kcalloc(&pdev->dev, *num_rates + 1, sizeof(*table), GFP_KERNEL); if (!table) { table = ERR_PTR(-ENOMEM); @@ -219,7 +225,9 @@ static int tegra186_cpufreq_probe(struct platform_device *pdev) { struct tegra186_cpufreq_data *data; struct tegra_bpmp *bpmp; - unsigned int i = 0, err; + unsigned int i = 0, err, edvd_offset; + int num_rates = 0; + u32 edvd_val, cpu; data = devm_kzalloc(&pdev->dev, struct_size(data, clusters, TEGRA186_NUM_CLUSTERS), @@ -242,10 +250,21 @@ static int tegra186_cpufreq_probe(struct platform_device *pdev) for (i = 0; i < TEGRA186_NUM_CLUSTERS; i++) { struct tegra186_cpufreq_cluster *cluster = &data->clusters[i]; - cluster->table = init_vhint_table(pdev, bpmp, cluster, i); + cluster->table = init_vhint_table(pdev, bpmp, cluster, i, &num_rates); if (IS_ERR(cluster->table)) { err = PTR_ERR(cluster->table); goto put_bpmp; + } else if (!num_rates) { + err = -EINVAL; + goto put_bpmp; + } + + for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) { + if (data->cpus[cpu].bpmp_cluster_id == i) { + edvd_val = cluster->table[num_rates - 1].driver_data; + edvd_offset = data->cpus[cpu].edvd_offset; + writel(edvd_val, data->regs + edvd_offset); + } } } diff --git a/drivers/cpuidle/cpuidle-qcom-spm.c b/drivers/cpuidle/cpuidle-qcom-spm.c index 1fc9968eae199..b6b06a510fd86 100644 --- a/drivers/cpuidle/cpuidle-qcom-spm.c +++ b/drivers/cpuidle/cpuidle-qcom-spm.c @@ -96,20 +96,23 @@ static int spm_cpuidle_register(struct device *cpuidle_dev, int cpu) return -ENODEV; saw_node = of_parse_phandle(cpu_node, "qcom,saw", 0); + of_node_put(cpu_node); if (!saw_node) return -ENODEV; pdev = of_find_device_by_node(saw_node); of_node_put(saw_node); - of_node_put(cpu_node); if (!pdev) return -ENODEV; data = devm_kzalloc(cpuidle_dev, sizeof(*data), GFP_KERNEL); - if (!data) + if (!data) { + put_device(&pdev->dev); return -ENOMEM; + } data->spm = dev_get_drvdata(&pdev->dev); + put_device(&pdev->dev); if (!data->spm) return -EINVAL; diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 875f2a6b30b70..ca4ac5ff39854 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -647,8 +647,14 @@ static void __cpuidle_device_init(struct cpuidle_device *dev) static int __cpuidle_register_device(struct cpuidle_device *dev) { struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); + unsigned int cpu = dev->cpu; int i, ret; + if (per_cpu(cpuidle_devices, cpu)) { + pr_info("CPU%d: cpuidle device already registered\n", cpu); + return -EEXIST; + } + if (!try_module_get(drv->owner)) return -EINVAL; @@ -660,7 +666,7 @@ static int __cpuidle_register_device(struct cpuidle_device *dev) dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER; } - per_cpu(cpuidle_devices, dev->cpu) = dev; + per_cpu(cpuidle_devices, cpu) = dev; list_add(&dev->device_list, &cpuidle_detected_devices); ret = cpuidle_coupled_register_device(dev); diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index b96e3da0fedd0..8c591dde61023 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -21,7 +21,7 @@ #include "gov.h" -#define BUCKETS 12 +#define BUCKETS 6 #define INTERVAL_SHIFT 3 #define INTERVALS (1UL << INTERVAL_SHIFT) #define RESOLUTION 1024 @@ -31,12 +31,11 @@ /* * Concepts and ideas behind the menu governor * - * For the menu governor, there are 3 decision factors for picking a C + * For the menu governor, there are 2 decision factors for picking a C * state: * 1) Energy break even point - * 2) Performance impact - * 3) Latency tolerance (from pmqos infrastructure) - * These three factors are treated independently. + * 2) Latency tolerance (from pmqos infrastructure) + * These two factors are treated independently. * * Energy break even point * ----------------------- @@ -119,19 +118,10 @@ struct menu_device { int interval_ptr; }; -static inline int which_bucket(u64 duration_ns, unsigned int nr_iowaiters) +static inline int which_bucket(u64 duration_ns) { int bucket = 0; - /* - * We keep two groups of stats; one with no - * IO pending, one without. - * This allows us to calculate - * E(duration)|iowait - */ - if (nr_iowaiters) - bucket = BUCKETS/2; - if (duration_ns < 10ULL * NSEC_PER_USEC) return bucket; if (duration_ns < 100ULL * NSEC_PER_USEC) @@ -145,21 +135,16 @@ static inline int which_bucket(u64 duration_ns, unsigned int nr_iowaiters) return bucket + 5; } -/* - * Return a multiplier for the exit latency that is intended - * to take performance requirements into account. - * The more performance critical we estimate the system - * to be, the higher this multiplier, and thus the higher - * the barrier to go to an expensive C state. - */ -static inline int performance_multiplier(unsigned int nr_iowaiters) +static DEFINE_PER_CPU(struct menu_device, menu_devices); + +static void menu_update_intervals(struct menu_device *data, unsigned int interval_us) { - /* for IO wait tasks (per cpu!) we add 10x each */ - return 1 + 10 * nr_iowaiters; + /* Update the repeating-pattern data. */ + data->intervals[data->interval_ptr++] = interval_us; + if (data->interval_ptr >= INTERVALS) + data->interval_ptr = 0; } -static DEFINE_PER_CPU(struct menu_device, menu_devices); - static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev); /* @@ -245,8 +230,16 @@ static unsigned int get_typical_interval(struct menu_device *data) * * This can deal with workloads that have long pauses interspersed * with sporadic activity with a bunch of short pauses. + * + * However, if the number of remaining samples is too small to exclude + * any more outliers, allow the deepest available idle state to be + * selected because there are systems where the time spent by CPUs in + * deep idle states is correlated to the maximum frequency the CPUs + * can get to. On those systems, shallow idle states should be avoided + * unless there is a clear indication that the given CPU is most likley + * going to be woken up shortly. */ - if ((divisor * 4) <= INTERVALS * 3) + if (divisor * 4 <= INTERVALS * 3) return UINT_MAX; thresh = max - 1; @@ -265,18 +258,22 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, struct menu_device *data = this_cpu_ptr(&menu_devices); s64 latency_req = cpuidle_governor_latency_req(dev->cpu); u64 predicted_ns; - u64 interactivity_req; - unsigned int nr_iowaiters; ktime_t delta, delta_tick; int i, idx; if (data->needs_update) { menu_update(drv, dev); data->needs_update = 0; + } else if (!dev->last_residency_ns) { + /* + * This happens when the driver rejects the previously selected + * idle state and returns an error, so update the recent + * intervals table to prevent invalid information from being + * used going forward. + */ + menu_update_intervals(data, UINT_MAX); } - nr_iowaiters = nr_iowait_cpu(dev->cpu); - /* Find the shortest expected idle interval. */ predicted_ns = get_typical_interval(data) * NSEC_PER_USEC; if (predicted_ns > RESIDENCY_THRESHOLD_NS) { @@ -290,7 +287,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, } data->next_timer_ns = delta; - data->bucket = which_bucket(data->next_timer_ns, nr_iowaiters); + data->bucket = which_bucket(data->next_timer_ns); /* Round up the result for half microseconds. */ timer_us = div_u64((RESOLUTION * DECAY * NSEC_PER_USEC) / 2 + @@ -308,7 +305,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, */ data->next_timer_ns = KTIME_MAX; delta_tick = TICK_NSEC / 2; - data->bucket = which_bucket(KTIME_MAX, nr_iowaiters); + data->bucket = which_bucket(KTIME_MAX); } if (unlikely(drv->state_count <= 1 || latency_req == 0) || @@ -324,27 +321,15 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, return 0; } - if (tick_nohz_tick_stopped()) { - /* - * If the tick is already stopped, the cost of possible short - * idle duration misprediction is much higher, because the CPU - * may be stuck in a shallow idle state for a long time as a - * result of it. In that case say we might mispredict and use - * the known time till the closest timer event for the idle - * state selection. - */ - if (predicted_ns < TICK_NSEC) - predicted_ns = data->next_timer_ns; - } else { - /* - * Use the performance multiplier and the user-configurable - * latency_req to determine the maximum exit latency. - */ - interactivity_req = div64_u64(predicted_ns, - performance_multiplier(nr_iowaiters)); - if (latency_req > interactivity_req) - latency_req = interactivity_req; - } + /* + * If the tick is already stopped, the cost of possible short idle + * duration misprediction is much higher, because the CPU may be stuck + * in a shallow idle state for a long time as a result of it. In that + * case, say we might mispredict and use the known time till the closest + * timer event for the idle state selection. + */ + if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) + predicted_ns = data->next_timer_ns; /* * Find the idle state with the lowest power while satisfying @@ -360,48 +345,53 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, if (idx == -1) idx = i; /* first enabled state */ - if (s->target_residency_ns > predicted_ns) { - /* - * Use a physical idle state, not busy polling, unless - * a timer is going to trigger soon enough. - */ - if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && - s->exit_latency_ns <= latency_req && - s->target_residency_ns <= data->next_timer_ns) { - predicted_ns = s->target_residency_ns; - idx = i; - break; - } - if (predicted_ns < TICK_NSEC) - break; - - if (!tick_nohz_tick_stopped()) { - /* - * If the state selected so far is shallow, - * waking up early won't hurt, so retain the - * tick in that case and let the governor run - * again in the next iteration of the loop. - */ - predicted_ns = drv->states[idx].target_residency_ns; - break; - } + if (s->exit_latency_ns > latency_req) + break; - /* - * If the state selected so far is shallow and this - * state's target residency matches the time till the - * closest timer event, select this one to avoid getting - * stuck in the shallow one for too long. - */ - if (drv->states[idx].target_residency_ns < TICK_NSEC && - s->target_residency_ns <= delta_tick) - idx = i; + if (s->target_residency_ns <= predicted_ns) { + idx = i; + continue; + } - return idx; + /* + * Use a physical idle state, not busy polling, unless a timer + * is going to trigger soon enough or the exit latency of the + * idle state in question is greater than the predicted idle + * duration. + */ + if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && + s->target_residency_ns <= data->next_timer_ns && + s->exit_latency_ns <= predicted_ns) { + predicted_ns = s->target_residency_ns; + idx = i; + break; } - if (s->exit_latency_ns > latency_req) + + if (predicted_ns < TICK_NSEC) break; - idx = i; + if (!tick_nohz_tick_stopped()) { + /* + * If the state selected so far is shallow, waking up + * early won't hurt, so retain the tick in that case and + * let the governor run again in the next iteration of + * the idle loop. + */ + predicted_ns = drv->states[idx].target_residency_ns; + break; + } + + /* + * If the state selected so far is shallow and this state's + * target residency matches the time till the closest timer + * event, select this one to avoid getting stuck in the shallow + * one for too long. + */ + if (drv->states[idx].target_residency_ns < TICK_NSEC && + s->target_residency_ns <= delta_tick) + idx = i; + + return idx; } if (idx == -1) @@ -542,10 +532,7 @@ static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) data->correction_factor[data->bucket] = new_factor; - /* update the repeating-pattern data */ - data->intervals[data->interval_ptr++] = ktime_to_us(measured_ns); - if (data->interval_ptr >= INTERVALS) - data->interval_ptr = 0; + menu_update_intervals(data, ktime_to_us(measured_ns)); } /** diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index d2cf9619018b1..31f2b4adf3d9a 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -264,9 +264,8 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req goto theend_sgs; } - chan->timeout = areq->cryptlen; - rctx->nr_sgs = nr_sgs; - rctx->nr_sgd = nr_sgd; + rctx->nr_sgs = ns; + rctx->nr_sgd = nd; return 0; theend_sgs: @@ -275,13 +274,16 @@ static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req } else { if (nr_sgs > 0) dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); + + if (nr_sgd > 0) + dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); } theend_iv: if (areq->iv && ivsize > 0) { - if (rctx->addr_iv) + if (!dma_mapping_error(ce->dev, rctx->addr_iv)) dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); + offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { memcpy(areq->iv, chan->backup_iv, ivsize); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index d4ccd5254280b..eba186c5c6251 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -186,11 +186,10 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) mutex_unlock(&ce->mlock); wait_for_completion_interruptible_timeout(&ce->chanlist[flow].complete, - msecs_to_jiffies(ce->chanlist[flow].timeout)); + msecs_to_jiffies(CE_DMA_TIMEOUT_MS)); if (ce->chanlist[flow].status == 0) { - dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name, - ce->chanlist[flow].timeout, flow); + dev_err(ce->dev, "DMA timeout for %s on flow %d\n", name, flow); err = -EFAULT; } /* No need to lock for this read, the channel is locked so diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index d358334e59811..8ee1e94335c37 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -343,9 +343,8 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) u32 common; u64 byte_count; __le32 *bf; - void *buf = NULL; + void *buf, *result; int j, i, todo; - void *result = NULL; u64 bs; int digestsize; dma_addr_t addr_res, addr_pad; @@ -365,14 +364,14 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) buf = kzalloc(bs * 2, GFP_KERNEL | GFP_DMA); if (!buf) { err = -ENOMEM; - goto theend; + goto err_out; } bf = (__le32 *)buf; result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); if (!result) { err = -ENOMEM; - goto theend; + goto err_free_buf; } flow = rctx->flow; @@ -398,7 +397,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (nr_sgs <= 0 || nr_sgs > MAX_SG) { dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); err = -EINVAL; - goto theend; + goto err_free_result; } len = areq->nbytes; @@ -411,7 +410,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (len > 0) { dev_err(ce->dev, "remaining len %d\n", len); err = -EINVAL; - goto theend; + goto err_unmap_src; } addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE); cet->t_dst[0].addr = cpu_to_le32(addr_res); @@ -419,7 +418,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (dma_mapping_error(ce->dev, addr_res)) { dev_err(ce->dev, "DMA map dest\n"); err = -EINVAL; - goto theend; + goto err_unmap_src; } byte_count = areq->nbytes; @@ -441,7 +440,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) } if (!j) { err = -EINVAL; - goto theend; + goto err_unmap_result; } addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE); @@ -450,7 +449,7 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) if (dma_mapping_error(ce->dev, addr_pad)) { dev_err(ce->dev, "DMA error on padding SG\n"); err = -EINVAL; - goto theend; + goto err_unmap_result; } if (ce->variant->hash_t_dlen_in_bits) @@ -458,21 +457,28 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) else cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j); - chan->timeout = areq->nbytes; - err = sun8i_ce_run_task(ce, flow, crypto_ahash_alg_name(tfm)); dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE); - dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + +err_unmap_result: dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); + if (!err) + memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize); +err_unmap_src: + dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); - memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize); -theend: - kfree(buf); +err_free_result: kfree(result); + +err_free_buf: + kfree(buf); + +err_out: local_bh_disable(); crypto_finalize_hash_request(engine, breq, err); local_bh_enable(); + return 0; } diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c index 80815379f6fc5..b571d1d0c4c4c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c @@ -137,7 +137,6 @@ int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, cet->t_dst[0].addr = cpu_to_le32(dma_dst); cet->t_dst[0].len = cpu_to_le32(todo / 4); - ce->chanlist[flow].timeout = 2000; err = sun8i_ce_run_task(ce, 3, "PRNG"); mutex_unlock(&ce->rnglock); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c index 9c35f2a83eda8..630a0b84b494d 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -79,7 +79,6 @@ static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wa cet->t_dst[0].addr = cpu_to_le32(dma_dst); cet->t_dst[0].len = cpu_to_le32(todo / 4); - ce->chanlist[flow].timeout = todo; err = sun8i_ce_run_task(ce, 3, "TRNG"); mutex_unlock(&ce->rnglock); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 93d4985def87a..d817ce445f4a3 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -106,6 +106,7 @@ #define MAX_SG 8 #define CE_MAX_CLOCKS 4 +#define CE_DMA_TIMEOUT_MS 3000 #define MAXFLOW 4 @@ -195,7 +196,6 @@ struct sun8i_ce_flow { struct completion complete; int status; dma_addr_t t_phy; - int timeout; struct ce_task *tl; void *backup_iv; void *bounce_iv; @@ -293,8 +293,8 @@ struct sun8i_ce_hash_tfm_ctx { * @flow: the flow to use for this request */ struct sun8i_ce_hash_reqctx { - struct ahash_request fallback_req; int flow; + struct ahash_request fallback_req; // keep at the end }; /* diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 7fa359725ec75..9b18fb46a2c89 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -141,7 +141,7 @@ static int sun8i_ss_setup_ivs(struct skcipher_request *areq) /* we need to copy all IVs from source in case DMA is bi-directionnal */ while (sg && len) { - if (sg_dma_len(sg) == 0) { + if (sg->length == 0) { sg = sg_next(sg); continue; } diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c index 247c568aa8dfe..8ca0913d94abf 100644 --- a/drivers/crypto/aspeed/aspeed-acry.c +++ b/drivers/crypto/aspeed/aspeed-acry.c @@ -789,28 +789,24 @@ static int aspeed_acry_probe(struct platform_device *pdev) err_engine_rsa_start: crypto_engine_exit(acry_dev->crypt_engine_rsa); clk_exit: - clk_disable_unprepare(acry_dev->clk); return rc; } -static int aspeed_acry_remove(struct platform_device *pdev) +static void aspeed_acry_remove(struct platform_device *pdev) { struct aspeed_acry_dev *acry_dev = platform_get_drvdata(pdev); aspeed_acry_unregister(acry_dev); crypto_engine_exit(acry_dev->crypt_engine_rsa); tasklet_kill(&acry_dev->done_task); - clk_disable_unprepare(acry_dev->clk); - - return 0; } MODULE_DEVICE_TABLE(of, aspeed_acry_of_matches); static struct platform_driver aspeed_acry_driver = { .probe = aspeed_acry_probe, - .remove = aspeed_acry_remove, + .remove_new = aspeed_acry_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = aspeed_acry_of_matches, diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c index f0eddb7854e5d..15e843b4854d0 100644 --- a/drivers/crypto/aspeed/aspeed-hace-crypto.c +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -346,7 +346,7 @@ static int aspeed_sk_start_sg(struct aspeed_hace_dev *hace_dev) } else { dma_unmap_sg(hace_dev->dev, req->dst, rctx->dst_nents, - DMA_TO_DEVICE); + DMA_FROM_DEVICE); dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); } diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 099b32a10dd75..d234495f1115b 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -548,7 +548,7 @@ static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) if (err && (dd->flags & TDES_FLAGS_FAST)) { dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); } return err; diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index bdf367f3f6798..eb880f0435ee9 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -692,12 +692,12 @@ static int caam_ctrl_rng_init(struct device *dev) */ if (needs_entropy_delay_adjustment()) ent_delay = 12000; - if (!(ctrlpriv->rng4_sh_init || inst_handles)) { + if (!inst_handles) { dev_info(dev, "Entropy delay = %u\n", ent_delay); kick_trng(dev, ent_delay); - ent_delay += 400; + ent_delay = ent_delay * 2; } /* * if instantiate_rng(...) fails, the loop will rerun diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c index a1055554b47a2..dc26bc22c91d1 100644 --- a/drivers/crypto/ccp/ccp-debugfs.c +++ b/drivers/crypto/ccp/ccp-debugfs.c @@ -319,5 +319,8 @@ void ccp5_debugfs_setup(struct ccp_device *ccp) void ccp5_debugfs_destroy(void) { + mutex_lock(&ccp_debugfs_lock); debugfs_remove_recursive(ccp_debugfs_dir); + ccp_debugfs_dir = NULL; + mutex_unlock(&ccp_debugfs_lock); } diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c index bd205f1f2279e..573c0033a5afe 100644 --- a/drivers/crypto/hisilicon/debugfs.c +++ b/drivers/crypto/hisilicon/debugfs.c @@ -865,6 +865,7 @@ static int qm_diff_regs_init(struct hisi_qm *qm, dfx_regs_uninit(qm, qm->debug.qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); ret = PTR_ERR(qm->debug.acc_diff_regs); qm->debug.acc_diff_regs = NULL; + qm->debug.qm_diff_regs = NULL; return ret; } diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 9a1c61be32ccd..059319f7a7160 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -1482,11 +1482,13 @@ static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + /* Do unmap before data processing */ + hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); + p = sg_virt(areq->dst); memmove(p, p + ctx->key_sz - curve_sz, curve_sz); memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); - hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); kpp_request_complete(areq, ret); atomic64_inc(&dfx[HPRE_RECV_CNT].value); @@ -1796,9 +1798,11 @@ static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp) if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + /* Do unmap before data processing */ + hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); + hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE); - hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); kpp_request_complete(areq, ret); atomic64_inc(&dfx[HPRE_RECV_CNT].value); diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 762a2a54ca821..e3d113334d4e4 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -689,6 +689,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) /* Config data buffer pasid needed by Kunpeng 920 */ hpre_config_pasid(qm); + hpre_open_sva_prefetch(qm); hpre_enable_clock_gate(qm); @@ -1366,8 +1367,6 @@ static int hpre_pf_probe_init(struct hpre *hpre) if (ret) return ret; - hpre_open_sva_prefetch(qm); - hisi_qm_dev_err_init(qm); ret = hpre_show_last_regs_init(qm); if (ret) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 7921409791fb0..a2c6b28f4b84b 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3748,8 +3748,14 @@ static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, } pdev = container_of(dev, struct pci_dev, dev); + if (pci_physfn(pdev) != qm->pdev) { + pci_err(qm->pdev, "the pdev input does not match the pf!\n"); + put_device(dev); + return -EINVAL; + } *fun_index = pdev->devfn; + put_device(dev); return 0; } @@ -4363,9 +4369,6 @@ static void qm_restart_prepare(struct hisi_qm *qm) { u32 value; - if (qm->err_ini->open_sva_prefetch) - qm->err_ini->open_sva_prefetch(qm); - if (qm->ver >= QM_HW_V3) return; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 6aaaaf784ddc0..064fbc8db16fc 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -436,6 +436,45 @@ static void sec_set_endian(struct hisi_qm *qm) writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); } +static void sec_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val |= SEC_PREFETCH_DISABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, + val, !(val & SEC_SVA_DISABLE_READY), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); +} + +static void sec_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val &= SEC_PREFETCH_ENABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, + val, !(val & SEC_PREFETCH_DISABLE), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to open sva prefetch\n"); +} + static void sec_engine_sva_config(struct hisi_qm *qm) { u32 reg; @@ -469,45 +508,7 @@ static void sec_engine_sva_config(struct hisi_qm *qm) writel_relaxed(reg, qm->io_base + SEC_INTERFACE_USER_CTRL1_REG); } -} - -static void sec_open_sva_prefetch(struct hisi_qm *qm) -{ - u32 val; - int ret; - - if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) - return; - - /* Enable prefetch */ - val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); - val &= SEC_PREFETCH_ENABLE; - writel(val, qm->io_base + SEC_PREFETCH_CFG); - - ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, - val, !(val & SEC_PREFETCH_DISABLE), - SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to open sva prefetch\n"); -} - -static void sec_close_sva_prefetch(struct hisi_qm *qm) -{ - u32 val; - int ret; - - if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) - return; - - val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); - val |= SEC_PREFETCH_DISABLE; - writel(val, qm->io_base + SEC_PREFETCH_CFG); - - ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, - val, !(val & SEC_SVA_DISABLE_READY), - SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to close sva prefetch\n"); + sec_open_sva_prefetch(qm); } static void sec_enable_clock_gate(struct hisi_qm *qm) @@ -1090,7 +1091,6 @@ static int sec_pf_probe_init(struct sec_dev *sec) if (ret) return ret; - sec_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); sec_debug_regs_clear(qm); ret = sec_show_last_regs_init(qm); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 66e553115adfd..b70aa6032874e 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -469,10 +469,9 @@ bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) return false; } -static int hisi_zip_set_high_perf(struct hisi_qm *qm) +static void hisi_zip_set_high_perf(struct hisi_qm *qm) { u32 val; - int ret; val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET); if (perf_mode == HZIP_HIGH_COMP_PERF) @@ -482,13 +481,6 @@ static int hisi_zip_set_high_perf(struct hisi_qm *qm) /* Set perf mode */ writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET); - ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_HIGH_PERF_OFFSET, - val, val == perf_mode, HZIP_DELAY_1_US, - HZIP_POLL_TIMEOUT_US); - if (ret) - pci_err(qm->pdev, "failed to set perf mode\n"); - - return ret; } static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) @@ -585,6 +577,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); } + hisi_zip_open_sva_prefetch(qm); /* let's open all compression/decompression cores */ dcomp_bm = qm->cap_tables.dev_cap_table[ZIP_DECOMP_ENABLE_BITMAP_IDX].cap_val; @@ -596,6 +589,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); + hisi_zip_set_high_perf(qm); hisi_zip_enable_clock_gate(qm); return 0; @@ -1180,11 +1174,6 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) if (ret) return ret; - ret = hisi_zip_set_high_perf(qm); - if (ret) - return ret; - - hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 45063693859c0..de80c95309e62 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -436,7 +436,7 @@ static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev) struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); if (ctx->flags & DRIVER_FLAGS_SG) - dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE); + dma_unmap_sg(hdev->dev, ctx->sg, 1, DMA_TO_DEVICE); return 0; } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index f44c08f5f5ec4..af4b978189e51 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -249,7 +249,9 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, safexcel_complete(priv, ring); if (sreq->nents) { - dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); sreq->nents = 0; } @@ -497,7 +499,9 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, DMA_FROM_DEVICE); unmap_sg: if (req->nents) { - dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); req->nents = 0; } cdesc_rollback: diff --git a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c index daba8ca05dbe4..3b5d5b64e1414 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c @@ -68,6 +68,7 @@ struct ocs_hcu_ctx { * @sg_data_total: Total data in the SG list at any time. * @sg_data_offset: Offset into the data of the current individual SG node. * @sg_dma_nents: Number of sg entries mapped in dma_list. + * @nents: Number of entries in the scatterlist. */ struct ocs_hcu_rctx { struct ocs_hcu_dev *hcu_dev; @@ -91,6 +92,7 @@ struct ocs_hcu_rctx { unsigned int sg_data_total; unsigned int sg_data_offset; unsigned int sg_dma_nents; + unsigned int nents; }; /** @@ -199,7 +201,7 @@ static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, /* Unmap req->src (if mapped). */ if (rctx->sg_dma_nents) { - dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, rctx->nents, DMA_TO_DEVICE); rctx->sg_dma_nents = 0; } @@ -230,7 +232,7 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) struct device *dev = rctx->hcu_dev->dev; unsigned int remainder = 0; unsigned int total; - size_t nents; + int nents; size_t count; int rc; int i; @@ -251,6 +253,9 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) /* Determine the number of scatter gather list entries to process. */ nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); + if (nents < 0) + return nents; + /* If there are entries to process, map them. */ if (nents) { rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, @@ -260,6 +265,10 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req) rc = -ENOMEM; goto cleanup; } + + /* Save the value of nents to pass to dma_unmap_sg. */ + rctx->nents = nents; + /* * The value returned by dma_map_sg() can be < nents; so update * nents accordingly. diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index 79ff7982378d9..05d1402001a1e 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -193,6 +193,7 @@ void adf_exit_misc_wq(void); bool adf_misc_wq_queue_work(struct work_struct *work); bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, unsigned long delay); +void adf_misc_wq_flush(void); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c index 0f9e2d59ce385..9c1b587a87e5b 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_init.c +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -381,6 +381,7 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) hw_data->exit_admin_comms(accel_dev); adf_cleanup_etr_data(accel_dev); + adf_misc_wq_flush(); adf_dev_restore(accel_dev); } diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index 2aba194a7c292..ce7c9ef6346b3 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -386,3 +386,8 @@ bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, { return queue_delayed_work(adf_misc_wq, work, delay); } + +void adf_misc_wq_flush(void) +{ + flush_workqueue(adf_misc_wq); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c index e2dd568b87b51..621b5d3dfcef9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c @@ -31,8 +31,10 @@ static void *adf_ring_next(struct seq_file *sfile, void *v, loff_t *pos) struct adf_etr_ring_data *ring = sfile->private; if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / - ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) + ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) { + (*pos)++; return NULL; + } return ring->base_addr + (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++); diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c index 3c4bba4a87795..d69cc1e5e0239 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c @@ -1277,7 +1277,7 @@ static struct aead_alg qat_aeads[] = { { .base = { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha1", - .cra_priority = 4001, + .cra_priority = 100, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), @@ -1294,7 +1294,7 @@ static struct aead_alg qat_aeads[] = { { .base = { .cra_name = "authenc(hmac(sha256),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha256", - .cra_priority = 4001, + .cra_priority = 100, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), @@ -1311,7 +1311,7 @@ static struct aead_alg qat_aeads[] = { { .base = { .cra_name = "authenc(hmac(sha512),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha512", - .cra_priority = 4001, + .cra_priority = 100, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), @@ -1329,7 +1329,7 @@ static struct aead_alg qat_aeads[] = { { static struct skcipher_alg qat_skciphers[] = { { .base.cra_name = "cbc(aes)", .base.cra_driver_name = "qat_aes_cbc", - .base.cra_priority = 4001, + .base.cra_priority = 100, .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), @@ -1347,7 +1347,7 @@ static struct skcipher_alg qat_skciphers[] = { { }, { .base.cra_name = "ctr(aes)", .base.cra_driver_name = "qat_aes_ctr", - .base.cra_priority = 4001, + .base.cra_priority = 100, .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), @@ -1365,7 +1365,7 @@ static struct skcipher_alg qat_skciphers[] = { { }, { .base.cra_name = "xts(aes)", .base.cra_driver_name = "qat_aes_xts", - .base.cra_priority = 4001, + .base.cra_priority = 100, .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ALLOCATES_MEMORY, .base.cra_blocksize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/intel/qat/qat_common/qat_bl.c b/drivers/crypto/intel/qat/qat_common/qat_bl.c index 76baed0a76c0e..0d2ce20db6d86 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_bl.c +++ b/drivers/crypto/intel/qat/qat_common/qat_bl.c @@ -38,7 +38,7 @@ void qat_bl_free_bufl(struct adf_accel_dev *accel_dev, for (i = 0; i < blout->num_mapped_bufs; i++) { dma_unmap_single(dev, blout->buffers[i].addr, blout->buffers[i].len, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); } dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); @@ -160,7 +160,7 @@ static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, } buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, sg->length - left, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(dev, buffers[y].addr))) goto err_out; buffers[y].len = sg->length; @@ -202,7 +202,7 @@ static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, if (!dma_mapping_error(dev, buflout->buffers[i].addr)) dma_unmap_single(dev, buflout->buffers[i].addr, buflout->buffers[i].len, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); } if (!buf->sgl_dst_valid) diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 7842a9f22178c..cf94ba3011d51 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -197,7 +197,7 @@ static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) struct adf_dc_data *dc_data = NULL; u8 *obuff = NULL; - dc_data = devm_kzalloc(dev, sizeof(*dc_data), GFP_KERNEL); + dc_data = kzalloc_node(sizeof(*dc_data), GFP_KERNEL, dev_to_node(dev)); if (!dc_data) goto err; @@ -205,7 +205,7 @@ static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) if (!obuff) goto err; - obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_FROM_DEVICE); + obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(dev, obuff_p))) goto err; @@ -233,9 +233,9 @@ static void qat_free_dc_data(struct adf_accel_dev *accel_dev) return; dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); kfree_sensitive(dc_data->ovf_buff); - devm_kfree(dev, dc_data); + kfree(dc_data); accel_dev->dc_data = NULL; } diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 4bd150d1441a0..473e1ab9b8baa 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1745,7 +1745,7 @@ static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) if (sobj_hdr) sobj_chunk_num = sobj_hdr->num_chunks; - mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * + mobj_hdr = kcalloc(size_add(uobj_chunk_num, sobj_chunk_num), sizeof(*mobj_hdr), GFP_KERNEL); if (!mobj_hdr) return -ENOMEM; diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 1e748e8ce12d5..b68903212b640 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -27,12 +27,14 @@ MODULE_DEVICE_TABLE(pci, adf_pci_tbl); static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); static void adf_remove(struct pci_dev *dev); +static void adf_shutdown(struct pci_dev *dev); static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_DH895XCC_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; @@ -227,6 +229,13 @@ static void adf_remove(struct pci_dev *pdev) kfree(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev, false); +} + static int __init adfdrv_init(void) { request_module("intel_qat"); diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index b61e35b932e59..76f34b81258dc 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -94,7 +94,7 @@ static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) { - if (engine->chain.first && engine->chain.last) + if (engine->chain_hw.first && engine->chain_hw.last) return mv_cesa_tdma_process(engine, status); return mv_cesa_std_process(engine, status); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index d215a6bed6bc7..50ca1039fdaa7 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -440,8 +440,10 @@ struct mv_cesa_dev { * SRAM * @queue: fifo of the pending crypto requests * @load: engine load counter, useful for load balancing - * @chain: list of the current tdma descriptors being processed - * by this engine. + * @chain_hw: list of the current tdma descriptors being processed + * by the hardware. + * @chain_sw: list of the current tdma descriptors that will be + * submitted to the hardware. * @complete_queue: fifo of the processed requests by the engine * * Structure storing CESA engine information. @@ -463,7 +465,8 @@ struct mv_cesa_engine { struct gen_pool *pool; struct crypto_queue queue; atomic_t load; - struct mv_cesa_tdma_chain chain; + struct mv_cesa_tdma_chain chain_hw; + struct mv_cesa_tdma_chain chain_sw; struct list_head complete_queue; int irq; }; diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index 0f37dfd42d850..eabed9d977df6 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -75,9 +75,12 @@ mv_cesa_skcipher_dma_cleanup(struct skcipher_request *req) static inline void mv_cesa_skcipher_cleanup(struct skcipher_request *req) { struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_engine *engine = creq->base.engine; if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_skcipher_dma_cleanup(req); + + atomic_sub(req->cryptlen, &engine->load); } static void mv_cesa_skcipher_std_step(struct skcipher_request *req) @@ -212,7 +215,6 @@ mv_cesa_skcipher_complete(struct crypto_async_request *req) struct mv_cesa_engine *engine = creq->base.engine; unsigned int ivsize; - atomic_sub(skreq->cryptlen, &engine->load); ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(skreq)); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) { @@ -459,6 +461,9 @@ static int mv_cesa_skcipher_queue_req(struct skcipher_request *req, struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_engine *engine; + if (!req->cryptlen) + return 0; + ret = mv_cesa_skcipher_req_init(req, tmpl); if (ret) return ret; diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index f150861ceaf69..e339ce7ad5331 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -110,9 +110,12 @@ static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) static inline void mv_cesa_ahash_cleanup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_engine *engine = creq->base.engine; if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_cleanup(req); + + atomic_sub(req->nbytes, &engine->load); } static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) @@ -395,8 +398,6 @@ static void mv_cesa_ahash_complete(struct crypto_async_request *req) } } } - - atomic_sub(ahashreq->nbytes, &engine->load); } static void mv_cesa_ahash_prepare(struct crypto_async_request *req, @@ -663,7 +664,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (ret) goto err_free_tdma; - if (iter.src.sg) { + if (iter.base.len > iter.src.op_offset) { /* * Add all the new data, inserting an operation block and * launch command between each full SRAM block-worth of diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index 388a06e180d64..243305354420c 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -38,6 +38,15 @@ void mv_cesa_dma_step(struct mv_cesa_req *dreq) { struct mv_cesa_engine *engine = dreq->engine; + spin_lock_bh(&engine->lock); + if (engine->chain_sw.first == dreq->chain.first) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; + } + engine->chain_hw.first = dreq->chain.first; + engine->chain_hw.last = dreq->chain.last; + spin_unlock_bh(&engine->lock); + writel_relaxed(0, engine->regs + CESA_SA_CFG); mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); @@ -96,25 +105,27 @@ void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, struct mv_cesa_req *dreq) { - if (engine->chain.first == NULL && engine->chain.last == NULL) { - engine->chain.first = dreq->chain.first; - engine->chain.last = dreq->chain.last; - } else { - struct mv_cesa_tdma_desc *last; + struct mv_cesa_tdma_desc *last = engine->chain_sw.last; - last = engine->chain.last; + /* + * Break the DMA chain if the request being queued needs the IV + * regs to be set before lauching the request. + */ + if (!last || dreq->chain.first->flags & CESA_TDMA_SET_STATE) + engine->chain_sw.first = dreq->chain.first; + else { last->next = dreq->chain.first; - engine->chain.last = dreq->chain.last; - - /* - * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on - * the last element of the current chain, or if the request - * being queued needs the IV regs to be set before lauching - * the request. - */ - if (!(last->flags & CESA_TDMA_BREAK_CHAIN) && - !(dreq->chain.first->flags & CESA_TDMA_SET_STATE)) - last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + } + last = dreq->chain.last; + engine->chain_sw.last = last; + /* + * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on + * the last element of the current chain. + */ + if (last->flags & CESA_TDMA_BREAK_CHAIN) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; } } @@ -127,7 +138,7 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) tdma_cur = readl(engine->regs + CESA_TDMA_CUR); - for (tdma = engine->chain.first; tdma; tdma = next) { + for (tdma = engine->chain_hw.first; tdma; tdma = next) { spin_lock_bh(&engine->lock); next = tdma->next; spin_unlock_bh(&engine->lock); @@ -149,12 +160,12 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) &backlog); /* Re-chaining to the next request */ - engine->chain.first = tdma->next; + engine->chain_hw.first = tdma->next; tdma->next = NULL; /* If this is the last request, clear the chain */ - if (engine->chain.first == NULL) - engine->chain.last = NULL; + if (engine->chain_hw.first == NULL) + engine->chain_hw.last = NULL; spin_unlock_bh(&engine->lock); ctx = crypto_tfm_ctx(req->tfm); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 1958b797a4210..682e7d80adb8c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -1485,6 +1485,7 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) dma_addr_t rptr_baddr; struct pci_dev *pdev; u32 len, compl_rlen; + int timeout = 10000; int ret, etype; void *rptr; @@ -1547,16 +1548,27 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) etype); otx2_cpt_fill_inst(&inst, &iq_cmd, rptr_baddr); lfs->ops->send_cmd(&inst, 1, &cptpf->lfs.lf[0]); + timeout = 10000; while (lfs->ops->cpt_get_compcode(result) == - OTX2_CPT_COMPLETION_CODE_INIT) + OTX2_CPT_COMPLETION_CODE_INIT) { cpu_relax(); + udelay(1); + timeout--; + if (!timeout) { + ret = -ENODEV; + cptpf->is_eng_caps_discovered = false; + dev_warn(&pdev->dev, "Timeout on CPT load_fvc completion poll\n"); + goto error_no_response; + } + } cptpf->eng_caps[etype].u = be64_to_cpup(rptr); } - dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL); cptpf->is_eng_caps_discovered = true; +error_no_response: + dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL); free_result: kfree(result); lf_cleanup: diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c index 811ded72ce5fb..798bb40fed68d 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c @@ -410,9 +410,10 @@ static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, break; } - dev_err(&pdev->dev, - "Request failed with software error code 0x%x\n", - cpt_status->s.uc_compcode); + pr_debug("Request failed with software error code 0x%x: algo = %s driver = %s\n", + cpt_status->s.uc_compcode, + info->req->areq->tfm->__crt_alg->cra_name, + info->req->areq->tfm->__crt_alg->cra_driver_name); otx2_cpt_dump_sg_list(pdev, info->req); break; } diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index 29c9537216fa6..8f3406289df43 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -252,7 +252,7 @@ static void rk_hash_unprepare(struct crypto_engine *engine, void *breq) struct rk_ahash_rctx *rctx = ahash_request_ctx(areq); struct rk_crypto_info *rkc = rctx->dev; - dma_unmap_sg(rkc->dev, areq->src, rctx->nrsg, DMA_TO_DEVICE); + dma_unmap_sg(rkc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); } static int rk_hash_run(struct crypto_engine *engine, void *breq) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index d7f7f88009d7d..1728cae1e8409 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1653,6 +1653,13 @@ static int find_pos_and_ways(struct cxl_port *port, struct range *range, } put_device(dev); + if (rc) + dev_err(port->uport_dev, + "failed to find %s:%s in target list of %s\n", + dev_name(&port->dev), + dev_name(port->parent_dport->dport_dev), + dev_name(&cxlsd->cxld.dev)); + return rc; } diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index 49c542ecccde3..7b991bbef4895 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -1382,15 +1382,11 @@ int devfreq_remove_governor(struct devfreq_governor *governor) int ret; struct device *dev = devfreq->dev.parent; + if (!devfreq->governor) + continue; + if (!strncmp(devfreq->governor->name, governor->name, DEVFREQ_NAME_LEN)) { - /* we should have a devfreq governor! */ - if (!devfreq->governor) { - dev_warn(dev, "%s: Governor %s NOT present\n", - __func__, governor->name); - continue; - /* Fall through */ - } ret = devfreq->governor->event_handler(devfreq, DEVFREQ_GOV_STOP, NULL); if (ret) { diff --git a/drivers/devfreq/governor_userspace.c b/drivers/devfreq/governor_userspace.c index d69672ccacc49..8d057cea09d5b 100644 --- a/drivers/devfreq/governor_userspace.c +++ b/drivers/devfreq/governor_userspace.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -39,10 +40,13 @@ static ssize_t set_freq_store(struct device *dev, struct device_attribute *attr, unsigned long wanted; int err = 0; + err = kstrtoul(buf, 0, &wanted); + if (err) + return err; + mutex_lock(&devfreq->lock); data = devfreq->governor_data; - sscanf(buf, "%lu", &wanted); data->user_frequency = wanted; data->valid = true; err = update_devfreq(devfreq); diff --git a/drivers/devfreq/mtk-cci-devfreq.c b/drivers/devfreq/mtk-cci-devfreq.c index 83a73f0ccd803..eff9b2c06aef4 100644 --- a/drivers/devfreq/mtk-cci-devfreq.c +++ b/drivers/devfreq/mtk-cci-devfreq.c @@ -385,7 +385,8 @@ static int mtk_ccifreq_probe(struct platform_device *pdev) out_free_resources: if (regulator_is_enabled(drv->proc_reg)) regulator_disable(drv->proc_reg); - if (drv->sram_reg && regulator_is_enabled(drv->sram_reg)) + if (!IS_ERR_OR_NULL(drv->sram_reg) && + regulator_is_enabled(drv->sram_reg)) regulator_disable(drv->sram_reg); return ret; diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 0b02ced1eb336..d152c43fa9aec 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -19,7 +19,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -31,16 +34,35 @@ #include #include +#ifndef __GENKSYMS__ +#include +#include +#endif #include #include "dma-buf-sysfs-stats.h" -struct dma_buf_list { - struct list_head head; - struct mutex lock; -}; +DEFINE_STATIC_KEY_TRUE(dmabuf_accounting_key); + +static DEFINE_MUTEX(dmabuf_list_mutex); +static LIST_HEAD(dmabuf_list); + +static void __dma_buf_list_add(struct dma_buf *dmabuf) +{ + mutex_lock(&dmabuf_list_mutex); + list_add(&dmabuf->list_node, &dmabuf_list); + mutex_unlock(&dmabuf_list_mutex); +} + +static void __dma_buf_list_del(struct dma_buf *dmabuf) +{ + if (!dmabuf) + return; -static struct dma_buf_list db_list; + mutex_lock(&dmabuf_list_mutex); + list_del(&dmabuf->list_node); + mutex_unlock(&dmabuf_list_mutex); +} /** * dma_buf_get_each - Helps in traversing the db_list and calls the @@ -60,21 +82,87 @@ int dma_buf_get_each(int (*callback)(const struct dma_buf *dmabuf, void *private), void *private) { struct dma_buf *buf; - int ret = mutex_lock_interruptible(&db_list.lock); + int ret = mutex_lock_interruptible(&dmabuf_list_mutex); if (ret) return ret; - list_for_each_entry(buf, &db_list.head, list_node) { + list_for_each_entry(buf, &dmabuf_list, list_node) { ret = callback(buf, private); if (ret) break; } - mutex_unlock(&db_list.lock); + mutex_unlock(&dmabuf_list_mutex); return ret; } EXPORT_SYMBOL_NS_GPL(dma_buf_get_each, MINIDUMP); +/** + * dma_buf_iter_begin - begin iteration through global list of all DMA buffers + * + * Returns the first buffer in the global list of DMA-bufs that's not in the + * process of being destroyed. Increments that buffer's reference count to + * prevent buffer destruction. Callers must release the reference, either by + * continuing iteration with dma_buf_iter_next(), or with dma_buf_put(). + * + * Return: + * * First buffer from global list, with refcount elevated + * * NULL if no active buffers are present + */ +struct dma_buf *dma_buf_iter_begin(void) +{ + struct dma_buf *ret = NULL, *dmabuf; + + /* + * The list mutex does not protect a dmabuf's refcount, so it can be + * zeroed while we are iterating. We cannot call get_dma_buf() since the + * caller may not already own a reference to the buffer. + */ + mutex_lock(&dmabuf_list_mutex); + list_for_each_entry(dmabuf, &dmabuf_list, list_node) { + if (get_file_rcu(dmabuf->file)) { + ret = dmabuf; + break; + } + } + mutex_unlock(&dmabuf_list_mutex); + return ret; +} + +/** + * dma_buf_iter_next - continue iteration through global list of all DMA buffers + * @dmabuf: [in] pointer to dma_buf + * + * Decrements the reference count on the provided buffer. Returns the next + * buffer from the remainder of the global list of DMA-bufs with its reference + * count incremented. Callers must release the reference, either by continuing + * iteration with dma_buf_iter_next(), or with dma_buf_put(). + * + * Return: + * * Next buffer from global list, with refcount elevated + * * NULL if no additional active buffers are present + */ +struct dma_buf *dma_buf_iter_next(struct dma_buf *dmabuf) +{ + struct dma_buf *ret = NULL; + + /* + * The list mutex does not protect a dmabuf's refcount, so it can be + * zeroed while we are iterating. We cannot call get_dma_buf() since the + * caller may not already own a reference to the buffer. + */ + mutex_lock(&dmabuf_list_mutex); + dma_buf_put(dmabuf); + list_for_each_entry_continue(dmabuf, &dmabuf_list, list_node) { + if (get_file_rcu(dmabuf->file)) { + ret = dmabuf; + break; + } + } + mutex_unlock(&dmabuf_list_mutex); + return ret; +} + static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen) { struct dma_buf *dmabuf; @@ -115,6 +203,9 @@ static void dma_buf_release(struct dentry *dentry) if (dmabuf->resv == (struct dma_resv *)&dmabuf[1]) dma_resv_fini(dmabuf->resv); + if (atomic64_read(&dmabuf->nr_task_refs)) + pr_alert("destroying dmabuf with non-zero task refs\n"); + WARN_ON(!list_empty(&dmabuf->attachments)); module_put(dmabuf->owner); kfree(dmabuf->name); @@ -123,17 +214,10 @@ static void dma_buf_release(struct dentry *dentry) static int dma_buf_file_release(struct inode *inode, struct file *file) { - struct dma_buf *dmabuf; - if (!is_dma_buf_file(file)) return -EINVAL; - dmabuf = file->private_data; - if (dmabuf) { - mutex_lock(&db_list.lock); - list_del(&dmabuf->list_node); - mutex_unlock(&db_list.lock); - } + __dma_buf_list_del(file->private_data); return 0; } @@ -162,9 +246,549 @@ static struct file_system_type dma_buf_fs_type = { .kill_sb = kill_anon_super, }; +struct task_dma_buf_record_preload { + local_trylock_t lock; + size_t size; + struct list_head list; +}; + +static DEFINE_PER_CPU(struct task_dma_buf_record_preload, dmabuf_rec_reloads); + +static struct kmem_cache *task_dmabuf_record_cachep; + +static void __init init_task_dmabuf_record_pool(void) +{ + int cpu; + + task_dmabuf_record_cachep = kmem_cache_create("task_dmabuf_record", + sizeof(struct task_dma_buf_record), 0, + SLAB_PANIC | SLAB_ACCOUNT, NULL); + + for_each_possible_cpu(cpu) { + struct task_dma_buf_record_preload *preload; + + preload = &per_cpu(dmabuf_rec_reloads, cpu); + local_trylock_init(&preload->lock); + INIT_LIST_HEAD(&preload->list); + preload->size = 0; + } +} + +/* + * Load up this CPU's task_dma_buf_record_preload list with requested number of + * records. On success, returns true, with preemption disabled. On error, + * return false with preemption not disabled. + */ +static bool task_dmabuf_records_preload(size_t count) +{ + struct task_dma_buf_record_preload *preload; + + local_lock(&dmabuf_rec_reloads.lock); + preload = this_cpu_ptr(&dmabuf_rec_reloads); + while (preload->size < count) { + struct task_dma_buf_record *rec; + + local_unlock(&dmabuf_rec_reloads.lock); + rec = kmem_cache_alloc(task_dmabuf_record_cachep, GFP_KERNEL); + if (!rec) + return false; + + local_lock(&dmabuf_rec_reloads.lock); + preload = this_cpu_ptr(&dmabuf_rec_reloads); + if (preload->size < count) { + list_add(&rec->node, &preload->list); + preload->size++; + } else { + kmem_cache_free(task_dmabuf_record_cachep, rec); + } + } + + return true; +} + +static inline void task_dmabuf_records_preload_end(void) +{ + local_unlock(&dmabuf_rec_reloads.lock); +} + +static struct task_dma_buf_record *alloc_task_dmabuf_record(void) +{ + struct task_dma_buf_record_preload *preload; + struct task_dma_buf_record *rec = NULL; + + lockdep_assert_held(this_cpu_ptr(&dmabuf_rec_reloads.lock)); + preload = this_cpu_ptr(&dmabuf_rec_reloads); + if (preload->size > 0) { + rec = list_first_entry(&preload->list, typeof(*rec), node); + list_del(&rec->node); + preload->size--; + } + + return rec; +} + +#define MAX_PCP_POOL_SIZE 32 + +static void free_task_dmabuf_record(struct task_dma_buf_record *rec) +{ + if (local_trylock(&dmabuf_rec_reloads.lock)) { + struct task_dma_buf_record_preload *preload = this_cpu_ptr(&dmabuf_rec_reloads); + + if (preload->size < MAX_PCP_POOL_SIZE) { + list_add(&rec->node, &preload->list); + preload->size++; + } else { + kmem_cache_free(task_dmabuf_record_cachep, rec); + } + local_unlock(&dmabuf_rec_reloads.lock); + } else { + kmem_cache_free(task_dmabuf_record_cachep, rec); + } +} + +static void trim_task_dmabuf_records_locked(void) +{ + struct task_dma_buf_record_preload *preload; + + lockdep_assert_held(this_cpu_ptr(&dmabuf_rec_reloads.lock)); + preload = this_cpu_ptr(&dmabuf_rec_reloads); + while (preload->size > MAX_PCP_POOL_SIZE) { + struct task_dma_buf_record *rec; + + rec = list_first_entry(&preload->list, typeof(*rec), node); + list_del(&rec->node); + preload->size--; + kmem_cache_free(task_dmabuf_record_cachep, rec); + } +} + +static void trim_task_dmabuf_records(void) +{ + local_lock(&dmabuf_rec_reloads.lock); + trim_task_dmabuf_records_locked(); + local_unlock(&dmabuf_rec_reloads.lock); +} + +static struct task_dma_buf_record *find_task_dmabuf_record( + struct task_dma_buf_info *dmabuf_info, struct dma_buf *dmabuf) +{ + struct task_dma_buf_record *rec; + + lockdep_assert_held(&dmabuf_info->lock); + + list_for_each_entry(rec, &dmabuf_info->dmabufs, node) + if (dmabuf == rec->dmabuf) + return rec; + + return NULL; +} + +static void add_task_dmabuf_record(struct task_dma_buf_info *dmabuf_info, + struct dma_buf *dmabuf, + struct task_dma_buf_record *rec) +{ + lockdep_assert_held(&dmabuf_info->lock); + + rec->dmabuf = dmabuf; + rec->refcnt = 1; + list_add(&rec->node, &dmabuf_info->dmabufs); + dmabuf_info->dmabuf_count++; + dmabuf_info->rss += dmabuf->size; + if (dmabuf_info->rss > dmabuf_info->rss_hwm) + dmabuf_info->rss_hwm = dmabuf_info->rss; + trace_dmabuf_rss_stat(dmabuf_info->rss, dmabuf->size, dmabuf); + atomic64_inc(&dmabuf->nr_task_refs); +} + +static int __dma_buf_account_task(struct dma_buf *dmabuf, struct task_dma_buf_info *dmabuf_info, + bool preload) +{ + struct task_dma_buf_record *rec; + int ret = 0; + + if (!static_key_enabled(&dmabuf_accounting_key)) + return 0; + + if (!dmabuf_info) + return 0; + + if (preload && !task_dmabuf_records_preload(1)) + return -ENOMEM; + + spin_lock(&dmabuf_info->lock); + rec = find_task_dmabuf_record(dmabuf_info, dmabuf); + if (rec) { + ++rec->refcnt; + if (preload) + trim_task_dmabuf_records_locked(); + } else { + rec = alloc_task_dmabuf_record(); + if (WARN_ON(!rec)) { + ret = -ENOMEM; + goto err; + } + add_task_dmabuf_record(dmabuf_info, dmabuf, rec); + } +err: + spin_unlock(&dmabuf_info->lock); + if (preload) + task_dmabuf_records_preload_end(); + + return ret; +} + +/** + * dma_buf_account_task - Account a dmabuf to a task + * @dmabuf: [in] pointer to dma_buf + * @task: [in] pointer to task_struct + * + * When a process obtains a dmabuf file descriptor, or maps a dmabuf, this + * function attributes the provided @dmabuf to the @task. The first time @dmabuf + * is attributed to @task, the buffer's size is added to the @task's dmabuf RSS. + * + * Return: + * * 0 on success + * * A negative error code upon error + */ +int dma_buf_account_task(struct dma_buf *dmabuf, struct task_dma_buf_info *dmabuf_info) +{ + return __dma_buf_account_task(dmabuf, dmabuf_info, true); +} + +/** + * dma_buf_unaccount_task - Unaccount a dmabuf from a task + * @dmabuf: [in] pointer to dma_buf + * @task: [in] pointer to task_struct + * + * When a process closes a dmabuf file descriptor, or unmaps a dmabuf, this + * function removes the provided @dmabuf attribution from the @task. When all + * references to @dmabuf are removed from @task, the buffer's size is removed + * from the task's dmabuf RSS. + */ +void dma_buf_unaccount_task(struct dma_buf *dmabuf, struct task_dma_buf_info *dmabuf_info) +{ + struct task_dma_buf_record *rec; + + if (!static_key_enabled(&dmabuf_accounting_key)) + return; + + if (!dmabuf_info) + return; + + spin_lock(&dmabuf_info->lock); + rec = find_task_dmabuf_record(dmabuf_info, dmabuf); + if (rec) { + if (--rec->refcnt == 0) { + list_del(&rec->node); + dmabuf_info->dmabuf_count--; + dmabuf_info->rss -= dmabuf->size; + trace_dmabuf_rss_stat(dmabuf_info->rss, -dmabuf->size, dmabuf); + atomic64_dec(&dmabuf->nr_task_refs); + } else { + rec = NULL; + } + } else { + pr_err("Could not find dmabuf %lu in unaccount\n", + file_inode(dmabuf->file)->i_ino); + } + spin_unlock(&dmabuf_info->lock); + if (rec) + free_task_dmabuf_record(rec); +} + +static struct task_dma_buf_info *alloc_task_dma_buf_info(void) +{ + struct task_dma_buf_info *dmabuf_info; + + dmabuf_info = kzalloc(sizeof(*dmabuf_info), GFP_KERNEL); + if (!dmabuf_info) + return NULL; + + refcount_set(&dmabuf_info->refcnt, 1); + spin_lock_init(&dmabuf_info->lock); + INIT_LIST_HEAD(&dmabuf_info->dmabufs); + + return dmabuf_info; +} + +static struct task_dma_buf_info *dup_dma_buf_info(struct task_dma_buf_info *from) +{ + struct task_dma_buf_info *to; + struct task_dma_buf_record *from_rec, *to_rec; + unsigned int count; + int retries = 0; + + /* Allocate now before locked section below. */ + to = alloc_task_dma_buf_info(); + if (!to) + return NULL; + + /* Read required count racily, before obtaining dmabuf_info->lock */ + count = READ_ONCE(from->dmabuf_count); + if (!task_dmabuf_records_preload(count)) + goto err_list_copy; + +retry: + spin_lock(&from->lock); + if (from->dmabuf_count > count) { + /* We don't have enough reserved records, allocate more */ + count = from->dmabuf_count; + + spin_unlock(&from->lock); + task_dmabuf_records_preload_end(); + if (!task_dmabuf_records_preload(count)) + goto err_list_copy; + + /* Limit the number of retries to avoid live-lock */ + if (retries++ > 5) { + task_dmabuf_records_preload_end(); + goto err_list_copy; + } + + goto retry; + } + + /* All required records are reserved */ + list_for_each_entry(from_rec, &from->dmabufs, node) { + to_rec = alloc_task_dmabuf_record(); + WARN_ON(!to_rec); + to_rec->dmabuf = from_rec->dmabuf; + to_rec->refcnt = from_rec->refcnt; + list_add(&to_rec->node, &to->dmabufs); + atomic64_inc(&to_rec->dmabuf->nr_task_refs); + } + to->dmabuf_count = from->dmabuf_count; + to->rss = from->rss; + to->rss_hwm = to->rss; + spin_unlock(&from->lock); + + trim_task_dmabuf_records_locked(); + task_dmabuf_records_preload_end(); + + return to; + +err_list_copy: + trim_task_dmabuf_records(); + kfree(to); + + return NULL; +} + +int copy_dmabuf_info(u64 clone_flags, struct task_struct *task) +{ + struct task_dma_buf_info *parent_dmabuf_info = current->dmabuf_info; + struct task_dma_buf_info *child_dmabuf_info; + bool share_vm = clone_flags & CLONE_VM; + bool share_fs = clone_flags & CLONE_FILES; + + if (!static_key_enabled(&dmabuf_accounting_key)) + return 0; + + /* kthreads are not supported */ + if (task->flags & PF_KTHREAD) { + task->dmabuf_info = NULL; + return 0; + } + + /* + * Non-kthread direct descendants of pid 0 are roots of their own task_dma_buf_info trees, + * even if they want to partially share with pid 0. Init does this. We assume no dmabuf + * sharing will actually occur through pid 0. + */ + if (unlikely(!task_pid_nr(current))) { + task->dmabuf_info = alloc_task_dma_buf_info(); + if (!task->dmabuf_info) + return -ENOMEM; + + return 0; + } + + /* + * Partial sharing is not supported. + * Children of such tasks are also not supported. + */ + if (share_vm != share_fs || !parent_dmabuf_info) { + task->dmabuf_info = NULL; + return 0; + } + + /* + * Full sharing: Both MM and FD references to dmabufs are shared with + * the parent, so they can both share the same dmabuf_info. + */ + if (share_vm && share_fs) { + refcount_inc(&parent_dmabuf_info->refcnt); + task->dmabuf_info = parent_dmabuf_info; + + if (task->mm) { + refcount_inc(&task->dmabuf_info->refcnt); + task->mm->dmabuf_info = task->dmabuf_info; + } + + if (task->files) { + refcount_inc(&task->dmabuf_info->refcnt); + task->files->dmabuf_info = task->dmabuf_info; + } + return 0; + } + + /* + * No sharing: Both MM and FD references to dmabufs are duplicated in the child. We + * duplicate the dmabuf accounting info into the child as well here. + */ + child_dmabuf_info = dup_dma_buf_info(parent_dmabuf_info); + if (!child_dmabuf_info) + return -ENOMEM; + + task->dmabuf_info = child_dmabuf_info; + + if (task->mm) { + refcount_inc(&child_dmabuf_info->refcnt); + task->mm->dmabuf_info = child_dmabuf_info; + } + if (task->files) { + refcount_inc(&child_dmabuf_info->refcnt); + task->files->dmabuf_info = child_dmabuf_info; + } + + return 0; +} + +void put_dmabuf_info(struct task_dma_buf_info *dmabuf_info) +{ + if (!dmabuf_info) + return; + + if (!refcount_dec_and_test(&dmabuf_info->refcnt)) + return; + + if (dmabuf_info->rss) + pr_alert("destroying task_dma_buf_info with non-zero dmabuf rss %lu\n", + dmabuf_info->rss); + + if (!list_empty(&dmabuf_info->dmabufs) || dmabuf_info->dmabuf_count > 0) + pr_alert("destroying task with non-empty dmabuf list %zu %u\n", + list_count_nodes(&dmabuf_info->dmabufs), + dmabuf_info->dmabuf_count); + + kfree(dmabuf_info); +} + +#define COUNT_DMABUF_FDS(file_lookup_func) ({ \ + size_t count = 0; \ + for (unsigned int n = 0; n < files_fdtable(current->files)->max_fds; ++n) { \ + struct file *file = file_lookup_func(current->files, n); \ + if (file && is_dma_buf_file(file)) \ + ++count; \ + } \ + count; \ +}) + +/* + * begin_new_exec is the starting point for the execution of a new program. It involves unsharing + * files_struct (possibly creating a new one), and installs a new mm_struct. Since this modifies the + * existing (task, mm, files) accounting relationship a new task_dma_buf_info is required for use by + * the new files_struct and mm_struct that are about to be used by the current task. The MM will be + * empty of dmabufs, but any dmabufs already accounted via file descriptors need to be accounted to + * the new files_struct. + */ +int dma_buf_begin_new_exec(struct files_struct *old_files) +{ + struct task_dma_buf_info *new_dmabuf_info; + struct task_dma_buf_info *old_dmabuf_info = current->dmabuf_info; + struct files_struct *my_files = current->files; + + if (!static_key_enabled(&dmabuf_accounting_key)) + return 0; + + new_dmabuf_info = alloc_task_dma_buf_info(); + if (!new_dmabuf_info) + return -ENOMEM; + + /* Any dmabufs need to be accounted to new_dmabuf_info */ + if (my_files) { + size_t num_dmabuf_fds, num_dmabuf_fds_check; + unsigned int retries = 0; + + /* Attempt to count dmabuf FDs locklessly before allocating */ + rcu_read_lock(); + num_dmabuf_fds = COUNT_DMABUF_FDS(files_lookup_fd_rcu); + rcu_read_unlock(); +retry: + if (!task_dmabuf_records_preload(num_dmabuf_fds)) + goto err_prealloc; + + spin_lock(&my_files->file_lock); + + /* First make sure we have enough preallocated records */ + num_dmabuf_fds_check = COUNT_DMABUF_FDS(files_lookup_fd_locked); + + if (num_dmabuf_fds_check > num_dmabuf_fds) { + spin_unlock(&my_files->file_lock); + + if (retries++ > 5) { + trim_task_dmabuf_records_locked(); + task_dmabuf_records_preload_end(); + goto err_retries; + } + + task_dmabuf_records_preload_end(); + num_dmabuf_fds = num_dmabuf_fds_check; + goto retry; + } + + for (unsigned int n = 0; n < files_fdtable(my_files)->max_fds; n++) { + struct file *file = files_lookup_fd_locked(my_files, n); + int err; + + if (!file || !is_dma_buf_file(file)) + continue; + + err = __dma_buf_account_task(file->private_data, new_dmabuf_info, false); + if (err) + pr_err("dmabuf accounting failed during begin_new_exec, err %d\n", + err); + + /* + * No put_files_struct in this case, so buffers don't get closed and + * unaccounted from the old dmabuf_info. + */ + if (my_files == old_files) + dma_buf_unaccount_task(file->private_data, my_files->dmabuf_info); + } + + /* + * put_files_struct puts the dmabuf_info, but not if we're reusing the original + * files_struct. + */ + if (my_files == old_files) + put_dmabuf_info(my_files->dmabuf_info); + + /* Finally swap over to the new dmabuf info */ + refcount_inc(&new_dmabuf_info->refcnt); + my_files->dmabuf_info = new_dmabuf_info; + spin_unlock(&my_files->file_lock); + + trim_task_dmabuf_records_locked(); + task_dmabuf_records_preload_end(); + } + + current->dmabuf_info = new_dmabuf_info; // refcount from alloc_task_dma_buf_info + put_dmabuf_info(old_dmabuf_info); + + return 0; + +err_prealloc: + trim_task_dmabuf_records(); +err_retries: + kfree(new_dmabuf_info); + return -ENOMEM; +} + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; + int ret; if (!is_dma_buf_file(file)) return -EINVAL; @@ -180,7 +804,15 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) dmabuf->size >> PAGE_SHIFT) return -EINVAL; - return dmabuf->ops->mmap(dmabuf, vma); + ret = dmabuf->ops->mmap(dmabuf, vma); + if (!ret) { + int err = dma_buf_account_task(dmabuf, vma->vm_mm->dmabuf_info); + + if (err) + pr_err("dmabuf accounting failed during mmap operation, err %d\n", err); + } + + return ret; } static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence) @@ -556,7 +1188,6 @@ static void dma_buf_show_fdinfo(struct seq_file *m, struct file *file) seq_printf(m, "name:\t%s\n", dmabuf->name); spin_unlock(&dmabuf->name_lock); } - static const struct file_operations dma_buf_fops = { .release = dma_buf_file_release, .mmap = dma_buf_mmap_internal, @@ -716,13 +1347,13 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) dmabuf->resv = resv; } + atomic64_set(&dmabuf->nr_task_refs, 0); + file->private_data = dmabuf; file->f_path.dentry->d_fsdata = dmabuf; dmabuf->file = file; - mutex_lock(&db_list.lock); - list_add(&dmabuf->list_node, &db_list.head); - mutex_unlock(&db_list.lock); + __dma_buf_list_add(dmabuf); ret = dma_buf_stats_setup(dmabuf, file); if (ret) @@ -731,9 +1362,9 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) return dmabuf; err_sysfs: - mutex_lock(&db_list.lock); + mutex_lock(&dmabuf_list_mutex); list_del(&dmabuf->list_node); - mutex_unlock(&db_list.lock); + mutex_unlock(&dmabuf_list_mutex); dmabuf->file = NULL; file->f_path.dentry->d_fsdata = NULL; file->private_data = NULL; @@ -1555,6 +2186,8 @@ EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access_partial); int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, unsigned long pgoff) { + int ret; + if (WARN_ON(!dmabuf || !vma)) return -EINVAL; @@ -1575,7 +2208,15 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, vma_set_file(vma, dmabuf->file); vma->vm_pgoff = pgoff; - return dmabuf->ops->mmap(dmabuf, vma); + ret = dmabuf->ops->mmap(dmabuf, vma); + if (!ret) { + int err = dma_buf_account_task(dmabuf, vma->vm_mm->dmabuf_info); + + if (err) + pr_err("dmabuf accounting failed during mmap operation, err %d\n", err); + } + + return ret; } EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, DMA_BUF); @@ -1722,7 +2363,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) size_t size = 0; int ret; - ret = mutex_lock_interruptible(&db_list.lock); + ret = mutex_lock_interruptible(&dmabuf_list_mutex); if (ret) return ret; @@ -1731,7 +2372,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\tname\n", "size", "flags", "mode", "count", "ino"); - list_for_each_entry(buf_obj, &db_list.head, list_node) { + list_for_each_entry(buf_obj, &dmabuf_list, list_node) { ret = dma_resv_lock_interruptible(buf_obj->resv, NULL); if (ret) @@ -1768,11 +2409,11 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size); - mutex_unlock(&db_list.lock); + mutex_unlock(&dmabuf_list_mutex); return 0; error_unlock: - mutex_unlock(&db_list.lock); + mutex_unlock(&dmabuf_list_mutex); return ret; } @@ -1817,6 +2458,24 @@ static inline void dma_buf_uninit_debugfs(void) } #endif +static int __init setup_early_dmabuf_accounting(char *str) +{ + bool enable; + + if (kstrtobool(str, &enable)) + return -EINVAL; + + if (enable != static_key_enabled(&dmabuf_accounting_key)) { + if (enable) + static_branch_enable(&dmabuf_accounting_key); + else + static_branch_disable(&dmabuf_accounting_key); + } + + return 0; +} +early_param("dmabuf_accounting", setup_early_dmabuf_accounting); + static int __init dma_buf_init(void) { int ret; @@ -1829,8 +2488,7 @@ static int __init dma_buf_init(void) if (IS_ERR(dma_buf_mnt)) return PTR_ERR(dma_buf_mnt); - mutex_init(&db_list.lock); - INIT_LIST_HEAD(&db_list.head); + init_task_dmabuf_record_pool(); dma_buf_init_debugfs(); return 0; } diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c index 9093f751f1336..8f3fa149a76d9 100644 --- a/drivers/dma-buf/dma-resv.c +++ b/drivers/dma-buf/dma-resv.c @@ -678,11 +678,13 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage, dma_resv_iter_begin(&cursor, obj, usage); dma_resv_for_each_fence_unlocked(&cursor, fence) { - ret = dma_fence_wait_timeout(fence, intr, ret); - if (ret <= 0) { - dma_resv_iter_end(&cursor); - return ret; - } + ret = dma_fence_wait_timeout(fence, intr, timeout); + if (ret <= 0) + break; + + /* Even for zero timeout the return value is 1 */ + if (timeout) + timeout = ret; } dma_resv_iter_end(&cursor); diff --git a/drivers/dma-buf/heaps/deferred-free-helper.c b/drivers/dma-buf/heaps/deferred-free-helper.c index bae271720011b..bc42a1c9769ee 100644 --- a/drivers/dma-buf/heaps/deferred-free-helper.c +++ b/drivers/dma-buf/heaps/deferred-free-helper.c @@ -19,9 +19,11 @@ static LIST_HEAD(free_list); static size_t list_nr_pages; wait_queue_head_t freelist_waitqueue; -struct task_struct *freelist_task; static DEFINE_SPINLOCK(free_list_lock); +struct task_struct *freelist_task; +EXPORT_SYMBOL_GPL(freelist_task); + void deferred_free(struct deferred_freelist_item *item, void (*free)(struct deferred_freelist_item*, enum df_reason), diff --git a/drivers/dma-buf/heaps/deferred-free-helper.h b/drivers/dma-buf/heaps/deferred-free-helper.h index 11940328ce3f1..a2e563c359275 100644 --- a/drivers/dma-buf/heaps/deferred-free-helper.h +++ b/drivers/dma-buf/heaps/deferred-free-helper.h @@ -24,6 +24,8 @@ enum df_reason { DF_UNDER_PRESSURE, }; +extern struct task_struct *freelist_task; + /** * deferred_freelist_item - item structure for deferred freelist * diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index 373282beeb606..980975ffc55d3 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -161,8 +161,7 @@ static int begin_cpu_udmabuf(struct dma_buf *buf, ubuf->sg = NULL; } } else { - dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents, - direction); + dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction); } return ret; @@ -177,7 +176,7 @@ static int end_cpu_udmabuf(struct dma_buf *buf, if (!ubuf->sg) return -EINVAL; - dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction); + dma_sync_sgtable_for_device(dev, ubuf->sg, direction); return 0; } diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 78b8a97b23637..ffe621695e472 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -827,9 +827,9 @@ static int dmatest_func(void *data) } else { dma_async_issue_pending(chan); - wait_event_timeout(thread->done_wait, - done->done, - msecs_to_jiffies(params->timeout)); + wait_event_freezable_timeout(thread->done_wait, + done->done, + msecs_to_jiffies(params->timeout)); status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 68236247059d1..9ae789d4aca7b 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -595,6 +595,25 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, return dw_edma_device_transfer(&xfer); } +static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, + enum dmaengine_tx_result result) +{ + u32 residue = 0; + struct dw_edma_desc *desc; + struct dmaengine_result *res; + + if (!vd->tx.callback_result) + return; + + desc = vd2dw_edma_desc(vd); + if (desc) + residue = desc->alloc_sz - desc->xfer_sz; + + res = &vd->tx_result; + res->result = result; + res->residue = residue; +} + static void dw_edma_done_interrupt(struct dw_edma_chan *chan) { struct dw_edma_desc *desc; @@ -608,6 +627,8 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) case EDMA_REQ_NONE: desc = vd2dw_edma_desc(vd); if (!desc->chunks_alloc) { + dw_hdma_set_callback_result(vd, + DMA_TRANS_NOERROR); list_del(&vd->node); vchan_cookie_complete(vd); } @@ -644,6 +665,7 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); if (vd) { + dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); } diff --git a/drivers/dma/dw/rzn1-dmamux.c b/drivers/dma/dw/rzn1-dmamux.c index 4fb8508419dbd..deadf135681b6 100644 --- a/drivers/dma/dw/rzn1-dmamux.c +++ b/drivers/dma/dw/rzn1-dmamux.c @@ -48,12 +48,16 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, u32 mask; int ret; - if (dma_spec->args_count != RNZ1_DMAMUX_NCELLS) - return ERR_PTR(-EINVAL); + if (dma_spec->args_count != RNZ1_DMAMUX_NCELLS) { + ret = -EINVAL; + goto put_device; + } map = kzalloc(sizeof(*map), GFP_KERNEL); - if (!map) - return ERR_PTR(-ENOMEM); + if (!map) { + ret = -ENOMEM; + goto put_device; + } chan = dma_spec->args[0]; map->req_idx = dma_spec->args[4]; @@ -94,12 +98,15 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, if (ret) goto clear_bitmap; + put_device(&pdev->dev); return map; clear_bitmap: clear_bit(map->req_idx, dmamux->used_chans); free_map: kfree(map); +put_device: + put_device(&pdev->dev); return ERR_PTR(ret); } diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index cc9923ab686dc..eccbcf67951fb 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -58,7 +58,7 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) intr = edma_readl_chreg(fsl_chan, ch_int); if (!intr) - return IRQ_HANDLED; + return IRQ_NONE; edma_writel_chreg(fsl_chan, 1, ch_int); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index c18633ad8455f..aa39fcd389a94 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -225,7 +225,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) struct idxd_wq *wq; struct device *dev, *fdev; int rc = 0; - struct iommu_sva *sva; + struct iommu_sva *sva = NULL; unsigned int pasid; struct idxd_cdev *idxd_cdev; @@ -322,7 +322,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) if (device_user_pasid_enabled(idxd)) idxd_xa_pasid_remove(ctx); failed_get_pasid: - if (device_user_pasid_enabled(idxd)) + if (device_user_pasid_enabled(idxd) && !IS_ERR_OR_NULL(sva)) iommu_sva_unbind_device(sva); failed: mutex_unlock(&wq->wq_lock); @@ -354,7 +354,9 @@ static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) set_bit(h, evl->bmap); h = (h + 1) % size; } - drain_workqueue(wq->wq); + if (wq->wq) + drain_workqueue(wq->wq); + mutex_unlock(&evl->lock); } @@ -412,6 +414,9 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO)) return -EPERM; + if (current->mm != ctx->mm) + return -EPERM; + rc = check_vma(wq, vma, __func__); if (rc < 0) return rc; @@ -478,6 +483,9 @@ static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t ssize_t written = 0; int i; + if (current->mm != ctx->mm) + return -EPERM; + for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) { int rc = idxd_submit_user_descriptor(ctx, udesc + i); @@ -498,6 +506,9 @@ static __poll_t idxd_cdev_poll(struct file *filp, struct idxd_device *idxd = wq->idxd; __poll_t out = 0; + if (current->mm != ctx->mm) + return POLLNVAL; + poll_wait(filp, &wq->err_queue, wait); spin_lock(&idxd->dev_lock); if (idxd->sw_err.valid) @@ -584,6 +595,7 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) { + struct device *dev = &idxd_dev->conf_dev; struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); struct idxd_device *idxd = wq->idxd; int rc; @@ -611,6 +623,12 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) mutex_lock(&wq->wq_lock); + if (!idxd_wq_driver_name_match(wq, dev)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + rc = -ENODEV; + goto wq_err; + } + wq->wq = create_workqueue(dev_name(wq_confdev(wq))); if (!wq->wq) { rc = -ENOMEM; diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 07623fb0f52fc..47a01893cfdbf 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -306,6 +306,12 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) return -ENXIO; mutex_lock(&wq->wq_lock); + if (!idxd_wq_driver_name_match(wq, dev)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + rc = -ENODEV; + goto err; + } + wq->type = IDXD_WQT_KERNEL; rc = drv_enable_wq(wq); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index bea10c5cdb76b..fcbb8caea8995 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -159,6 +159,8 @@ struct idxd_cdev { int minor; }; +#define DRIVER_NAME_SIZE 128 + #define IDXD_ALLOCATED_BATCH_SIZE 128U #define WQ_NAME_SIZE 1024 #define WQ_TYPE_SIZE 10 @@ -227,6 +229,8 @@ struct idxd_wq { /* Lock to protect upasid_xa access. */ struct mutex uc_lock; struct xarray upasid_xa; + + char driver_name[DRIVER_NAME_SIZE + 1]; }; struct idxd_engine { @@ -648,6 +652,11 @@ static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wq wqcfg->max_batch_shift = max_batch_shift; } +static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev) +{ + return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); +} + int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *module, const char *mod_name); #define idxd_driver_register(driver) \ diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 786afb256b6e0..4b999c5802f4b 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -145,6 +145,25 @@ static void idxd_cleanup_interrupts(struct idxd_device *idxd) pci_free_irq_vectors(pdev); } +static void idxd_clean_wqs(struct idxd_device *idxd) +{ + struct idxd_wq *wq; + struct device *conf_dev; + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + wq = idxd->wqs[i]; + if (idxd->hw.wq_cap.op_config) + bitmap_free(wq->opcap_bmap); + kfree(wq->wqcfg); + conf_dev = wq_confdev(wq); + put_device(conf_dev); + kfree(wq); + } + bitmap_free(idxd->wq_enable_map); + kfree(idxd->wqs); +} + static int idxd_setup_wqs(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -159,29 +178,30 @@ static int idxd_setup_wqs(struct idxd_device *idxd) idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); if (!idxd->wq_enable_map) { - kfree(idxd->wqs); - return -ENOMEM; + rc = -ENOMEM; + goto err_free_wqs; } for (i = 0; i < idxd->max_wqs; i++) { wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); if (!wq) { rc = -ENOMEM; - goto err; + goto err_unwind; } idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); conf_dev = wq_confdev(wq); wq->id = i; wq->idxd = idxd; - device_initialize(wq_confdev(wq)); + device_initialize(conf_dev); conf_dev->parent = idxd_confdev(idxd); conf_dev->bus = &dsa_bus_type; conf_dev->type = &idxd_wq_device_type; rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); if (rc < 0) { put_device(conf_dev); - goto err; + kfree(wq); + goto err_unwind; } mutex_init(&wq->wq_lock); @@ -194,16 +214,19 @@ static int idxd_setup_wqs(struct idxd_device *idxd) wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); if (!wq->wqcfg) { put_device(conf_dev); + kfree(wq); rc = -ENOMEM; - goto err; + goto err_unwind; } if (idxd->hw.wq_cap.op_config) { wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); if (!wq->opcap_bmap) { + kfree(wq->wqcfg); put_device(conf_dev); + kfree(wq); rc = -ENOMEM; - goto err; + goto err_unwind; } bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); } @@ -214,15 +237,39 @@ static int idxd_setup_wqs(struct idxd_device *idxd) return 0; - err: +err_unwind: while (--i >= 0) { wq = idxd->wqs[i]; + if (idxd->hw.wq_cap.op_config) + bitmap_free(wq->opcap_bmap); + kfree(wq->wqcfg); conf_dev = wq_confdev(wq); put_device(conf_dev); + kfree(wq); } + bitmap_free(idxd->wq_enable_map); + +err_free_wqs: + kfree(idxd->wqs); + return rc; } +static void idxd_clean_engines(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct device *conf_dev; + int i; + + for (i = 0; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + conf_dev = engine_confdev(engine); + put_device(conf_dev); + kfree(engine); + } + kfree(idxd->engines); +} + static int idxd_setup_engines(struct idxd_device *idxd) { struct idxd_engine *engine; @@ -253,6 +300,7 @@ static int idxd_setup_engines(struct idxd_device *idxd) rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); if (rc < 0) { put_device(conf_dev); + kfree(engine); goto err; } @@ -266,10 +314,26 @@ static int idxd_setup_engines(struct idxd_device *idxd) engine = idxd->engines[i]; conf_dev = engine_confdev(engine); put_device(conf_dev); + kfree(engine); } + kfree(idxd->engines); + return rc; } +static void idxd_clean_groups(struct idxd_device *idxd) +{ + struct idxd_group *group; + int i; + + for (i = 0; i < idxd->max_groups; i++) { + group = idxd->groups[i]; + put_device(group_confdev(group)); + kfree(group); + } + kfree(idxd->groups); +} + static int idxd_setup_groups(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -300,6 +364,7 @@ static int idxd_setup_groups(struct idxd_device *idxd) rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); if (rc < 0) { put_device(conf_dev); + kfree(group); goto err; } @@ -324,20 +389,18 @@ static int idxd_setup_groups(struct idxd_device *idxd) while (--i >= 0) { group = idxd->groups[i]; put_device(group_confdev(group)); + kfree(group); } + kfree(idxd->groups); + return rc; } static void idxd_cleanup_internals(struct idxd_device *idxd) { - int i; - - for (i = 0; i < idxd->max_groups; i++) - put_device(group_confdev(idxd->groups[i])); - for (i = 0; i < idxd->max_engines; i++) - put_device(engine_confdev(idxd->engines[i])); - for (i = 0; i < idxd->max_wqs; i++) - put_device(wq_confdev(idxd->wqs[i])); + idxd_clean_groups(idxd); + idxd_clean_engines(idxd); + idxd_clean_wqs(idxd); destroy_workqueue(idxd->wq); } @@ -380,7 +443,7 @@ static int idxd_init_evl(struct idxd_device *idxd) static int idxd_setup_internals(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; - int rc, i; + int rc; init_waitqueue_head(&idxd->cmd_waitq); @@ -411,14 +474,11 @@ static int idxd_setup_internals(struct idxd_device *idxd) err_evl: destroy_workqueue(idxd->wq); err_wkq_create: - for (i = 0; i < idxd->max_groups; i++) - put_device(group_confdev(idxd->groups[i])); + idxd_clean_groups(idxd); err_group: - for (i = 0; i < idxd->max_engines; i++) - put_device(engine_confdev(idxd->engines[i])); + idxd_clean_engines(idxd); err_engine: - for (i = 0; i < idxd->max_wqs; i++) - put_device(wq_confdev(idxd->wqs[i])); + idxd_clean_wqs(idxd); err_wqs: return rc; } @@ -518,6 +578,17 @@ static void idxd_read_caps(struct idxd_device *idxd) idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); } +static void idxd_free(struct idxd_device *idxd) +{ + if (!idxd) + return; + + put_device(idxd_confdev(idxd)); + bitmap_free(idxd->opcap_bmap); + ida_free(&idxd_ida, idxd->id); + kfree(idxd); +} + static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) { struct device *dev = &pdev->dev; @@ -535,28 +606,34 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); if (idxd->id < 0) - return NULL; + goto err_ida; idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); - if (!idxd->opcap_bmap) { - ida_free(&idxd_ida, idxd->id); - return NULL; - } + if (!idxd->opcap_bmap) + goto err_opcap; device_initialize(conf_dev); conf_dev->parent = dev; conf_dev->bus = &dsa_bus_type; conf_dev->type = idxd->data->dev_type; rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); - if (rc < 0) { - put_device(conf_dev); - return NULL; - } + if (rc < 0) + goto err_name; spin_lock_init(&idxd->dev_lock); spin_lock_init(&idxd->cmd_lock); return idxd; + +err_name: + put_device(conf_dev); + bitmap_free(idxd->opcap_bmap); +err_opcap: + ida_free(&idxd_ida, idxd->id); +err_ida: + kfree(idxd); + + return NULL; } static int idxd_enable_system_pasid(struct idxd_device *idxd) @@ -778,7 +855,7 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err: pci_iounmap(pdev, idxd->reg_base); err_iomap: - put_device(idxd_confdev(idxd)); + idxd_free(idxd); err_idxd_alloc: pci_disable_device(pdev); return rc; @@ -815,7 +892,6 @@ static void idxd_shutdown(struct pci_dev *pdev) static void idxd_remove(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); - struct idxd_irq_entry *irq_entry; idxd_unregister_devices(idxd); /* @@ -828,20 +904,14 @@ static void idxd_remove(struct pci_dev *pdev) get_device(idxd_confdev(idxd)); device_unregister(idxd_confdev(idxd)); idxd_shutdown(pdev); + idxd_device_remove_debugfs(idxd); + perfmon_pmu_remove(idxd); + idxd_cleanup_interrupts(idxd); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - idxd_device_remove_debugfs(idxd); - - irq_entry = idxd_get_ie(idxd, 0); - free_irq(irq_entry->vector, irq_entry); - pci_free_irq_vectors(pdev); pci_iounmap(pdev, idxd->reg_base); - if (device_user_pasid_enabled(idxd)) - idxd_disable_sva(pdev); - pci_disable_device(pdev); - destroy_workqueue(idxd->wq); - perfmon_pmu_remove(idxd); put_device(idxd_confdev(idxd)); + pci_disable_device(pdev); } static struct pci_driver idxd_pci_driver = { diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 1fd5a93045f79..3a5ce477a81ad 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1282,6 +1282,39 @@ static ssize_t wq_op_config_store(struct device *dev, struct device_attribute *a static struct device_attribute dev_attr_wq_op_config = __ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store); +static ssize_t wq_driver_name_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + + return sysfs_emit(buf, "%s\n", wq->driver_name); +} + +static ssize_t wq_driver_name_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + char *input, *pos; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + if (strlen(buf) > DRIVER_NAME_SIZE || strlen(buf) == 0) + return -EINVAL; + + input = kstrndup(buf, count, GFP_KERNEL); + if (!input) + return -ENOMEM; + + pos = strim(input); + memset(wq->driver_name, 0, DRIVER_NAME_SIZE + 1); + sprintf(wq->driver_name, "%s", pos); + kfree(input); + return count; +} + +static struct device_attribute dev_attr_wq_driver_name = + __ATTR(driver_name, 0644, wq_driver_name_show, wq_driver_name_store); + static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_clients.attr, &dev_attr_wq_state.attr, @@ -1301,6 +1334,7 @@ static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_occupancy.attr, &dev_attr_wq_enqcmds_retries.attr, &dev_attr_wq_op_config.attr, + &dev_attr_wq_driver_name.attr, NULL, }; diff --git a/drivers/dma/mediatek/mtk-cqdma.c b/drivers/dma/mediatek/mtk-cqdma.c index 324b7387b1b92..525bb92ced8f8 100644 --- a/drivers/dma/mediatek/mtk-cqdma.c +++ b/drivers/dma/mediatek/mtk-cqdma.c @@ -420,15 +420,11 @@ static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, { struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); struct virt_dma_desc *vd; - unsigned long flags; - spin_lock_irqsave(&cvc->pc->lock, flags); list_for_each_entry(vd, &cvc->pc->queue, node) if (vd->tx.cookie == cookie) { - spin_unlock_irqrestore(&cvc->pc->lock, flags); return vd; } - spin_unlock_irqrestore(&cvc->pc->lock, flags); list_for_each_entry(vd, &cvc->vc.desc_issued, node) if (vd->tx.cookie == cookie) @@ -452,9 +448,11 @@ static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, if (ret == DMA_COMPLETE || !txstate) return ret; - spin_lock_irqsave(&cvc->vc.lock, flags); + spin_lock_irqsave(&cvc->pc->lock, flags); + spin_lock(&cvc->vc.lock); vd = mtk_cqdma_find_active_desc(c, cookie); - spin_unlock_irqrestore(&cvc->vc.lock, flags); + spin_unlock(&cvc->vc.lock); + spin_unlock_irqrestore(&cvc->pc->lock, flags); if (vd) { cvd = to_cqdma_vdesc(vd); diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index ea48661e87ea7..8b215cbca1186 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -1013,7 +1013,7 @@ static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan) dma_async_device_unregister(&mv_chan->dmadev); - dma_free_coherent(dev, MV_XOR_POOL_SIZE, + dma_free_wc(dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); dma_unmap_single(dev, mv_chan->dummy_src_addr, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); @@ -1061,8 +1061,16 @@ mv_xor_channel_add(struct mv_xor_device *xordev, */ mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev, mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); + if (dma_mapping_error(dma_dev->dev, mv_chan->dummy_src_addr)) + return ERR_PTR(-ENOMEM); + mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev, mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev->dev, mv_chan->dummy_dst_addr)) { + ret = -ENOMEM; + goto err_unmap_src; + } + /* allocate coherent memory for hardware descriptors * note: writecombine gives slightly better performance, but @@ -1071,8 +1079,10 @@ mv_xor_channel_add(struct mv_xor_device *xordev, mv_chan->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool, GFP_KERNEL); - if (!mv_chan->dma_desc_pool_virt) - return ERR_PTR(-ENOMEM); + if (!mv_chan->dma_desc_pool_virt) { + ret = -ENOMEM; + goto err_unmap_dst; + } /* discover transaction capabilites from the platform data */ dma_dev->cap_mask = cap_mask; @@ -1153,8 +1163,15 @@ mv_xor_channel_add(struct mv_xor_device *xordev, err_free_irq: free_irq(mv_chan->irq, mv_chan); err_free_dma: - dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE, + dma_free_wc(&pdev->dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); +err_unmap_dst: + dma_unmap_single(dma_dev->dev, mv_chan->dummy_dst_addr, + MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); +err_unmap_src: + dma_unmap_single(dma_dev->dev, mv_chan->dummy_src_addr, + MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); + return ERR_PTR(ret); } diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index 0b2f96fd8bf0c..69c80d9339033 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -711,6 +711,9 @@ static int nbpf_desc_page_alloc(struct nbpf_channel *chan) list_add_tail(&ldesc->node, &lhead); ldesc->hwdesc_dma_addr = dma_map_single(dchan->device->dev, hwdesc, sizeof(*hwdesc), DMA_TO_DEVICE); + if (dma_mapping_error(dchan->device->dev, + ldesc->hwdesc_dma_addr)) + goto unmap_error; dev_dbg(dev, "%s(): mapped 0x%p to %pad\n", __func__, hwdesc, &ldesc->hwdesc_dma_addr); @@ -737,6 +740,16 @@ static int nbpf_desc_page_alloc(struct nbpf_channel *chan) spin_unlock_irq(&chan->lock); return ARRAY_SIZE(dpage->desc); + +unmap_error: + while (i--) { + ldesc--; hwdesc--; + + dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr, + sizeof(hwdesc), DMA_TO_DEVICE); + } + + return -ENOMEM; } static void nbpf_desc_put(struct nbpf_desc *desc) @@ -1351,7 +1364,7 @@ static int nbpf_probe(struct platform_device *pdev) if (irqs == 1) { eirq = irqbuf[0]; - for (i = 0; i <= num_channels; i++) + for (i = 0; i < num_channels; i++) nbpf->chan[i].irq = irqbuf[0]; } else { eirq = platform_get_irq_byname(pdev, "error"); @@ -1361,16 +1374,15 @@ static int nbpf_probe(struct platform_device *pdev) if (irqs == num_channels + 1) { struct nbpf_channel *chan; - for (i = 0, chan = nbpf->chan; i <= num_channels; + for (i = 0, chan = nbpf->chan; i < num_channels; i++, chan++) { /* Skip the error IRQ */ if (irqbuf[i] == eirq) i++; + if (i >= ARRAY_SIZE(irqbuf)) + return -EINVAL; chan->irq = irqbuf[i]; } - - if (chan != nbpf->chan + num_channels) - return -EINVAL; } else { /* 2 IRQs and more than one channel */ if (irqbuf[0] == eirq) @@ -1378,7 +1390,7 @@ static int nbpf_probe(struct platform_device *pdev) else irq = irqbuf[0]; - for (i = 0; i <= num_channels; i++) + for (i = 0; i < num_channels; i++) nbpf->chan[i].irq = irq; } } diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 4c3eb972039d6..d5882c8537e52 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1283,13 +1283,17 @@ static int bam_dma_probe(struct platform_device *pdev) if (!bdev->bamclk) { ret = of_property_read_u32(pdev->dev.of_node, "num-channels", &bdev->num_channels); - if (ret) + if (ret) { dev_err(bdev->dev, "num-channels unspecified in dt\n"); + return ret; + } ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees", &bdev->num_ees); - if (ret) + if (ret) { dev_err(bdev->dev, "num-ees unspecified in dt\n"); + return ret; + } } ret = clk_prepare_enable(bdev->bamclk); diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index 588c5f409a808..8d796504cb7f1 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -129,12 +129,25 @@ static dma_cookie_t shdma_tx_submit(struct dma_async_tx_descriptor *tx) const struct shdma_ops *ops = sdev->ops; dev_dbg(schan->dev, "Bring up channel %d\n", schan->id); - /* - * TODO: .xfer_setup() might fail on some platforms. - * Make it int then, on error remove chunks from the - * queue again - */ - ops->setup_xfer(schan, schan->slave_id); + + ret = ops->setup_xfer(schan, schan->slave_id); + if (ret < 0) { + dev_err(schan->dev, "setup_xfer failed: %d\n", ret); + + /* Remove chunks from the queue and mark them as idle */ + list_for_each_entry_safe(chunk, c, &schan->ld_queue, node) { + if (chunk->cookie == cookie) { + chunk->mark = DESC_IDLE; + list_move(&chunk->node, &schan->ld_free); + } + } + + schan->pm_state = SHDMA_PM_ESTABLISHED; + ret = pm_runtime_put(schan->dev); + + spin_unlock_irq(&schan->chan_lock); + return ret; + } if (schan->pm_state == SHDMA_PM_PENDING) shdma_chan_xfer_ld_queue(schan); diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 00067b29e2322..d8210488dd40c 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -300,21 +300,30 @@ static bool sh_dmae_channel_busy(struct shdma_chan *schan) return dmae_is_busy(sh_chan); } -static void sh_dmae_setup_xfer(struct shdma_chan *schan, - int slave_id) +static int sh_dmae_setup_xfer(struct shdma_chan *schan, int slave_id) { struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan, shdma_chan); + int ret = 0; if (slave_id >= 0) { const struct sh_dmae_slave_config *cfg = sh_chan->config; - dmae_set_dmars(sh_chan, cfg->mid_rid); - dmae_set_chcr(sh_chan, cfg->chcr); + ret = dmae_set_dmars(sh_chan, cfg->mid_rid); + if (ret < 0) + goto END; + + ret = dmae_set_chcr(sh_chan, cfg->chcr); + if (ret < 0) + goto END; + } else { dmae_init(sh_chan); } + +END: + return ret; } /* diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 9840594a6aaa1..3882080cffa69 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -745,7 +745,7 @@ static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr) /* cyclic while CIRC/DBM disable => post resume reconfiguration needed */ if (!(scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM))) stm32_dma_post_resume_reconfigure(chan); - else if (scr & STM32_DMA_SCR_DBM) + else if (scr & STM32_DMA_SCR_DBM && chan->desc->num_sgs > 2) stm32_dma_configure_next_sg(chan); } else { chan->busy = false; diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index c0fa541324675..f7ddf588b7f9b 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -2063,8 +2063,8 @@ static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata, * priority. So Q0 is the highest priority queue and the last queue has * the lowest priority. */ - queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8), - GFP_KERNEL); + queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, + sizeof(*queue_priority_map), GFP_KERNEL); if (!queue_priority_map) return -ENOMEM; diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 02a1ab04f498e..d99cf81358c54 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -1091,8 +1091,11 @@ static void udma_check_tx_completion(struct work_struct *work) u32 residue_diff; ktime_t time_diff; unsigned long delay; + unsigned long flags; while (1) { + spin_lock_irqsave(&uc->vc.lock, flags); + if (uc->desc) { /* Get previous residue and time stamp */ residue_diff = uc->tx_drain.residue; @@ -1127,6 +1130,8 @@ static void udma_check_tx_completion(struct work_struct *work) break; } + spin_unlock_irqrestore(&uc->vc.lock, flags); + usleep_range(ktime_to_us(delay), ktime_to_us(delay) + 10); continue; @@ -1143,6 +1148,8 @@ static void udma_check_tx_completion(struct work_struct *work) break; } + + spin_unlock_irqrestore(&uc->vc.lock, flags); } static irqreturn_t udma_ring_irq_handler(int irq, void *data) @@ -4214,7 +4221,6 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct udma_dev *ud = ofdma->of_dma_data; - dma_cap_mask_t mask = ud->ddev.cap_mask; struct udma_filter_param filter_param; struct dma_chan *chan; @@ -4246,7 +4252,7 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, } } - chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, + chan = __dma_request_channel(&ud->ddev.cap_mask, udma_dma_filter_fn, &filter_param, ofdma->of_node); if (!chan) { dev_err(ud->dev, "get channel fail in %s.\n", __func__); @@ -5531,7 +5537,8 @@ static int udma_probe(struct platform_device *pdev) uc->config.dir = DMA_MEM_TO_MEM; uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", dev_name(dev), i); - + if (!uc->name) + return -ENOMEM; vchan_init(&uc->vc, &ud->ddev); /* Use custom vchan completion handling */ tasklet_setup(&uc->vc.task, udma_vchan_complete); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 0a3b2e22f23db..14c4c5031b556 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -2900,6 +2900,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, return -EINVAL; } + xdev->common.directions |= chan->direction; + /* Request the interrupt */ chan->irq = of_irq_get(node, chan->tdest); if (chan->irq < 0) diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 8b31cd54bdb6d..4f8f87207b67b 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -98,7 +98,7 @@ static irqreturn_t altr_sdram_mc_err_handler(int irq, void *dev_id) if (status & priv->ecc_stat_ce_mask) { regmap_read(drvdata->mc_vbase, priv->ecc_saddr_offset, &err_addr); - if (priv->ecc_uecnt_offset) + if (priv->ecc_cecnt_offset) regmap_read(drvdata->mc_vbase, priv->ecc_cecnt_offset, &err_count); edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, err_count, @@ -127,7 +127,6 @@ static ssize_t altr_sdr_mc_err_inject_write(struct file *file, ptemp = dma_alloc_coherent(mci->pdev, 16, &dma_handle, GFP_KERNEL); if (!ptemp) { - dma_free_coherent(mci->pdev, 16, ptemp, dma_handle); edac_printk(KERN_ERR, EDAC_MC, "Inject: Buffer Allocation error\n"); return -ENOMEM; @@ -1015,9 +1014,6 @@ altr_init_a10_ecc_block(struct device_node *np, u32 irq_mask, } } - /* Interrupt mode set to every SBERR */ - regmap_write(ecc_mgr_map, ALTR_A10_ECC_INTMODE_OFST, - ALTR_A10_ECC_INTMODE); /* Enable ECC */ ecc_set_bits(ecc_ctrl_en_mask, (ecc_block_base + ALTR_A10_ECC_CTRL_OFST)); @@ -1198,10 +1194,22 @@ altr_check_ocram_deps_init(struct altr_edac_device_dev *device) if (ret) return ret; - /* Verify OCRAM has been initialized */ + /* + * Verify that OCRAM has been initialized. + * During a warm reset, OCRAM contents are retained, but the control + * and status registers are reset to their default values. Therefore, + * ECC must be explicitly re-enabled in the control register. + * Error condition: if INITCOMPLETEA is clear and ECC_EN is already set. + */ if (!ecc_test_bits(ALTR_A10_ECC_INITCOMPLETEA, - (base + ALTR_A10_ECC_INITSTAT_OFST))) - return -ENODEV; + (base + ALTR_A10_ECC_INITSTAT_OFST))) { + if (!ecc_test_bits(ALTR_A10_ECC_EN, + (base + ALTR_A10_ECC_CTRL_OFST))) + ecc_set_bits(ALTR_A10_ECC_EN, + (base + ALTR_A10_ECC_CTRL_OFST)); + else + return -ENODEV; + } /* Enable IRQ on Single Bit Error */ writel(ALTR_A10_ECC_SERRINTEN, (base + ALTR_A10_ECC_ERRINTENS_OFST)); @@ -1371,7 +1379,7 @@ static const struct edac_device_prv_data a10_enetecc_data = { .ue_set_mask = ALTR_A10_ECC_TDERRA, .set_err_ofst = ALTR_A10_ECC_INTTEST_OFST, .ecc_irq_handler = altr_edac_a10_ecc_irq, - .inject_fops = &altr_edac_a10_device_inject2_fops, + .inject_fops = &altr_edac_a10_device_inject_fops, }; #endif /* CONFIG_EDAC_ALTERA_ETHERNET */ @@ -1461,7 +1469,7 @@ static const struct edac_device_prv_data a10_usbecc_data = { .ue_set_mask = ALTR_A10_ECC_TDERRA, .set_err_ofst = ALTR_A10_ECC_INTTEST_OFST, .ecc_irq_handler = altr_edac_a10_ecc_irq, - .inject_fops = &altr_edac_a10_device_inject2_fops, + .inject_fops = &altr_edac_a10_device_inject_fops, }; #endif /* CONFIG_EDAC_ALTERA_USB */ @@ -1759,9 +1767,9 @@ altr_edac_a10_device_trig(struct file *file, const char __user *user_buf, local_irq_save(flags); if (trig_type == ALTR_UE_TRIGGER_CHAR) - writel(priv->ue_set_mask, set_addr); + writew(priv->ue_set_mask, set_addr); else - writel(priv->ce_set_mask, set_addr); + writew(priv->ce_set_mask, set_addr); /* Ensure the interrupt test bits are set */ wmb(); @@ -1791,7 +1799,7 @@ altr_edac_a10_device_trig2(struct file *file, const char __user *user_buf, local_irq_save(flags); if (trig_type == ALTR_UE_TRIGGER_CHAR) { - writel(priv->ue_set_mask, set_addr); + writew(priv->ue_set_mask, set_addr); } else { /* Setup read/write of 4 bytes */ writel(ECC_WORD_WRITE, drvdata->base + ECC_BLK_DBYTECTRL_OFST); @@ -2138,6 +2146,10 @@ static int altr_edac_a10_probe(struct platform_device *pdev) return PTR_ERR(edac->ecc_mgr_map); } + /* Set irq mask for DDR SBE to avoid any pending irq before registration */ + regmap_write(edac->ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_SET_OFST, + (A10_SYSMGR_ECC_INTMASK_SDMMCB | A10_SYSMGR_ECC_INTMASK_DDR0)); + edac->irq_chip.name = pdev->dev.of_node->name; edac->irq_chip.irq_mask = a10_eccmgr_irq_mask; edac->irq_chip.irq_unmask = a10_eccmgr_irq_unmask; diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h index 3727e72c8c2e7..7248d24c4908d 100644 --- a/drivers/edac/altera_edac.h +++ b/drivers/edac/altera_edac.h @@ -249,6 +249,8 @@ struct altr_sdram_mc_data { #define A10_SYSMGR_ECC_INTMASK_SET_OFST 0x94 #define A10_SYSMGR_ECC_INTMASK_CLR_OFST 0x98 #define A10_SYSMGR_ECC_INTMASK_OCRAM BIT(1) +#define A10_SYSMGR_ECC_INTMASK_SDMMCB BIT(16) +#define A10_SYSMGR_ECC_INTMASK_DDR0 BIT(17) #define A10_SYSMGR_ECC_INTSTAT_SERR_OFST 0x9C #define A10_SYSMGR_ECC_INTSTAT_DERR_OFST 0xA0 diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 9cd86390a1674..a8db2bde63840 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1475,7 +1475,9 @@ static int umc_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt) if (csrow_enabled(2 * dimm + 1, ctrl, pvt)) cs_mode |= CS_ODD_PRIMARY; - /* Asymmetric dual-rank DIMM support. */ + if (csrow_sec_enabled(2 * dimm, ctrl, pvt)) + cs_mode |= CS_EVEN_SECONDARY; + if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt)) cs_mode |= CS_ODD_SECONDARY; @@ -1496,12 +1498,13 @@ static int umc_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt) return cs_mode; } -static int __addr_mask_to_cs_size(u32 addr_mask_orig, unsigned int cs_mode, - int csrow_nr, int dimm) +static int calculate_cs_size(u32 mask, unsigned int cs_mode) { - u32 msb, weight, num_zero_bits; - u32 addr_mask_deinterleaved; - int size = 0; + int msb, weight, num_zero_bits; + u32 deinterleaved_mask; + + if (!mask) + return 0; /* * The number of zero bits in the mask is equal to the number of bits @@ -1514,19 +1517,30 @@ static int __addr_mask_to_cs_size(u32 addr_mask_orig, unsigned int cs_mode, * without swapping with the most significant bit. This can be handled * by keeping the MSB where it is and ignoring the single zero bit. */ - msb = fls(addr_mask_orig) - 1; - weight = hweight_long(addr_mask_orig); + msb = fls(mask) - 1; + weight = hweight_long(mask); num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE); /* Take the number of zero bits off from the top of the mask. */ - addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1); + deinterleaved_mask = GENMASK(msb - num_zero_bits, 1); + edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", deinterleaved_mask); + + return (deinterleaved_mask >> 2) + 1; +} + +static int __addr_mask_to_cs_size(u32 addr_mask, u32 addr_mask_sec, + unsigned int cs_mode, int csrow_nr, int dimm) +{ + int size; edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm); - edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig); - edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved); + edac_dbg(1, " Primary AddrMask: 0x%x\n", addr_mask); /* Register [31:1] = Address [39:9]. Size is in kBs here. */ - size = (addr_mask_deinterleaved >> 2) + 1; + size = calculate_cs_size(addr_mask, cs_mode); + + edac_dbg(1, " Secondary AddrMask: 0x%x\n", addr_mask_sec); + size += calculate_cs_size(addr_mask_sec, cs_mode); /* Return size in MBs. */ return size >> 10; @@ -1535,8 +1549,8 @@ static int __addr_mask_to_cs_size(u32 addr_mask_orig, unsigned int cs_mode, static int umc_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc, unsigned int cs_mode, int csrow_nr) { + u32 addr_mask = 0, addr_mask_sec = 0; int cs_mask_nr = csrow_nr; - u32 addr_mask_orig; int dimm, size = 0; /* No Chip Selects are enabled. */ @@ -1574,13 +1588,13 @@ static int umc_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc, if (!pvt->flags.zn_regs_v2) cs_mask_nr >>= 1; - /* Asymmetric dual-rank DIMM support. */ - if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY)) - addr_mask_orig = pvt->csels[umc].csmasks_sec[cs_mask_nr]; - else - addr_mask_orig = pvt->csels[umc].csmasks[cs_mask_nr]; + if (cs_mode & (CS_EVEN_PRIMARY | CS_ODD_PRIMARY)) + addr_mask = pvt->csels[umc].csmasks[cs_mask_nr]; + + if (cs_mode & (CS_EVEN_SECONDARY | CS_ODD_SECONDARY)) + addr_mask_sec = pvt->csels[umc].csmasks_sec[cs_mask_nr]; - return __addr_mask_to_cs_size(addr_mask_orig, cs_mode, csrow_nr, dimm); + return __addr_mask_to_cs_size(addr_mask, addr_mask_sec, cs_mode, csrow_nr, dimm); } static void umc_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) @@ -3773,9 +3787,10 @@ static void gpu_get_err_info(struct mce *m, struct err_info *err) static int gpu_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc, unsigned int cs_mode, int csrow_nr) { - u32 addr_mask_orig = pvt->csels[umc].csmasks[csrow_nr]; + u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr]; + u32 addr_mask_sec = pvt->csels[umc].csmasks_sec[csrow_nr]; - return __addr_mask_to_cs_size(addr_mask_orig, cs_mode, csrow_nr, csrow_nr >> 1); + return __addr_mask_to_cs_size(addr_mask, addr_mask_sec, cs_mode, csrow_nr, csrow_nr >> 1); } static void gpu_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) @@ -4130,6 +4145,7 @@ static int per_family_init(struct amd64_pvt *pvt) break; case 0x70 ... 0x7f: pvt->ctl_name = "F19h_M70h"; + pvt->max_mcs = 4; pvt->flags.zn_regs_v2 = 1; break; case 0xa0 ... 0xaf: diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c index 15f63452a9bec..b01436d9ddaed 100644 --- a/drivers/edac/edac_mc_sysfs.c +++ b/drivers/edac/edac_mc_sysfs.c @@ -306,6 +306,14 @@ DEVICE_CHANNEL(ch10_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 10); DEVICE_CHANNEL(ch11_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 11); +DEVICE_CHANNEL(ch12_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 12); +DEVICE_CHANNEL(ch13_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 13); +DEVICE_CHANNEL(ch14_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 14); +DEVICE_CHANNEL(ch15_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 15); /* Total possible dynamic DIMM Label attribute file table */ static struct attribute *dynamic_csrow_dimm_attr[] = { @@ -321,6 +329,10 @@ static struct attribute *dynamic_csrow_dimm_attr[] = { &dev_attr_legacy_ch9_dimm_label.attr.attr, &dev_attr_legacy_ch10_dimm_label.attr.attr, &dev_attr_legacy_ch11_dimm_label.attr.attr, + &dev_attr_legacy_ch12_dimm_label.attr.attr, + &dev_attr_legacy_ch13_dimm_label.attr.attr, + &dev_attr_legacy_ch14_dimm_label.attr.attr, + &dev_attr_legacy_ch15_dimm_label.attr.attr, NULL }; @@ -349,6 +361,14 @@ DEVICE_CHANNEL(ch10_ce_count, S_IRUGO, channel_ce_count_show, NULL, 10); DEVICE_CHANNEL(ch11_ce_count, S_IRUGO, channel_ce_count_show, NULL, 11); +DEVICE_CHANNEL(ch12_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 12); +DEVICE_CHANNEL(ch13_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 13); +DEVICE_CHANNEL(ch14_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 14); +DEVICE_CHANNEL(ch15_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 15); /* Total possible dynamic ce_count attribute file table */ static struct attribute *dynamic_csrow_ce_count_attr[] = { @@ -364,6 +384,10 @@ static struct attribute *dynamic_csrow_ce_count_attr[] = { &dev_attr_legacy_ch9_ce_count.attr.attr, &dev_attr_legacy_ch10_ce_count.attr.attr, &dev_attr_legacy_ch11_ce_count.attr.attr, + &dev_attr_legacy_ch12_ce_count.attr.attr, + &dev_attr_legacy_ch13_ce_count.attr.attr, + &dev_attr_legacy_ch14_ce_count.attr.attr, + &dev_attr_legacy_ch15_ce_count.attr.attr, NULL }; diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 67a46abe07da9..3c70d86074975 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -99,7 +99,7 @@ static u32 offsets_demand2_spr[] = {0x22c70, 0x22d80, 0x22f18, 0x22d58, 0x22c64, static u32 offsets_demand_spr_hbm0[] = {0x2a54, 0x2a60, 0x2b10, 0x2a58, 0x2a5c, 0x0ee0}; static u32 offsets_demand_spr_hbm1[] = {0x2e54, 0x2e60, 0x2f10, 0x2e58, 0x2e5c, 0x0fb0}; -static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable, +static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable, u32 *rrl_ctl, u32 *offsets_scrub, u32 *offsets_demand, u32 *offsets_demand2) { @@ -112,10 +112,10 @@ static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable if (enable) { /* Save default configurations */ - imc->chan[chan].retry_rd_err_log_s = s; - imc->chan[chan].retry_rd_err_log_d = d; + rrl_ctl[0] = s; + rrl_ctl[1] = d; if (offsets_demand2) - imc->chan[chan].retry_rd_err_log_d2 = d2; + rrl_ctl[2] = d2; s &= ~RETRY_RD_ERR_LOG_NOOVER_UC; s |= RETRY_RD_ERR_LOG_EN; @@ -129,25 +129,25 @@ static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable } } else { /* Restore default configurations */ - if (imc->chan[chan].retry_rd_err_log_s & RETRY_RD_ERR_LOG_UC) + if (rrl_ctl[0] & RETRY_RD_ERR_LOG_UC) s |= RETRY_RD_ERR_LOG_UC; - if (imc->chan[chan].retry_rd_err_log_s & RETRY_RD_ERR_LOG_NOOVER) + if (rrl_ctl[0] & RETRY_RD_ERR_LOG_NOOVER) s |= RETRY_RD_ERR_LOG_NOOVER; - if (!(imc->chan[chan].retry_rd_err_log_s & RETRY_RD_ERR_LOG_EN)) + if (!(rrl_ctl[0] & RETRY_RD_ERR_LOG_EN)) s &= ~RETRY_RD_ERR_LOG_EN; - if (imc->chan[chan].retry_rd_err_log_d & RETRY_RD_ERR_LOG_UC) + if (rrl_ctl[1] & RETRY_RD_ERR_LOG_UC) d |= RETRY_RD_ERR_LOG_UC; - if (imc->chan[chan].retry_rd_err_log_d & RETRY_RD_ERR_LOG_NOOVER) + if (rrl_ctl[1] & RETRY_RD_ERR_LOG_NOOVER) d |= RETRY_RD_ERR_LOG_NOOVER; - if (!(imc->chan[chan].retry_rd_err_log_d & RETRY_RD_ERR_LOG_EN)) + if (!(rrl_ctl[1] & RETRY_RD_ERR_LOG_EN)) d &= ~RETRY_RD_ERR_LOG_EN; if (offsets_demand2) { - if (imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_UC) + if (rrl_ctl[2] & RETRY_RD_ERR_LOG_UC) d2 |= RETRY_RD_ERR_LOG_UC; - if (!(imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_NOOVER)) + if (!(rrl_ctl[2] & RETRY_RD_ERR_LOG_NOOVER)) d2 &= ~RETRY_RD_ERR_LOG_NOOVER; - if (!(imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_EN)) + if (!(rrl_ctl[2] & RETRY_RD_ERR_LOG_EN)) d2 &= ~RETRY_RD_ERR_LOG_EN; } } @@ -161,6 +161,7 @@ static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable static void enable_retry_rd_err_log(bool enable) { int i, j, imc_num, chan_num; + struct skx_channel *chan; struct skx_imc *imc; struct skx_dev *d; @@ -175,8 +176,9 @@ static void enable_retry_rd_err_log(bool enable) if (!imc->mbase) continue; + chan = d->imc[i].chan; for (j = 0; j < chan_num; j++) - __enable_retry_rd_err_log(imc, j, enable, + __enable_retry_rd_err_log(imc, j, enable, chan[j].rrl_ctl[0], res_cfg->offsets_scrub, res_cfg->offsets_demand, res_cfg->offsets_demand2); @@ -190,12 +192,13 @@ static void enable_retry_rd_err_log(bool enable) if (!imc->mbase || !imc->hbm_mc) continue; + chan = d->imc[i].chan; for (j = 0; j < chan_num; j++) { - __enable_retry_rd_err_log(imc, j, enable, + __enable_retry_rd_err_log(imc, j, enable, chan[j].rrl_ctl[0], res_cfg->offsets_scrub_hbm0, res_cfg->offsets_demand_hbm0, NULL); - __enable_retry_rd_err_log(imc, j, enable, + __enable_retry_rd_err_log(imc, j, enable, chan[j].rrl_ctl[1], res_cfg->offsets_scrub_hbm1, res_cfg->offsets_demand_hbm1, NULL); @@ -967,6 +970,15 @@ static bool i10nm_check_ecc(struct skx_imc *imc, int chan) return !!GET_BITFIELD(mcmtr, 2, 2); } +static bool i10nm_channel_disabled(struct skx_imc *imc, int chan) +{ + u32 mcmtr = I10NM_GET_MCMTR(imc, chan); + + edac_dbg(1, "mc%d ch%d mcmtr reg %x\n", imc->mc, chan, mcmtr); + + return (mcmtr == ~0 || GET_BITFIELD(mcmtr, 18, 18)); +} + static int i10nm_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg) { @@ -980,6 +992,11 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci, if (!imc->mbase) continue; + if (i10nm_channel_disabled(imc, i)) { + edac_dbg(1, "mc%d ch%d is disabled.\n", imc->mc, i); + continue; + } + ndimms = 0; amap = I10NM_GET_AMAP(imc, i); diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index 56be8ef40f376..e3635fba63b49 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -405,10 +405,9 @@ static int ie31200_probe1(struct pci_dev *pdev, int dev_idx) int i, j, ret; struct mem_ctl_info *mci = NULL; struct edac_mc_layer layers[2]; - struct dimm_data dimm_info[IE31200_CHANNELS][IE31200_DIMMS_PER_CHANNEL]; void __iomem *window; struct ie31200_priv *priv; - u32 addr_decode, mad_offset; + u32 addr_decode[IE31200_CHANNELS], mad_offset; /* * Kaby Lake, Coffee Lake seem to work like Skylake. Please re-visit @@ -466,19 +465,10 @@ static int ie31200_probe1(struct pci_dev *pdev, int dev_idx) mad_offset = IE31200_MAD_DIMM_0_OFFSET; } - /* populate DIMM info */ for (i = 0; i < IE31200_CHANNELS; i++) { - addr_decode = readl(window + mad_offset + + addr_decode[i] = readl(window + mad_offset + (i * 4)); - edac_dbg(0, "addr_decode: 0x%x\n", addr_decode); - for (j = 0; j < IE31200_DIMMS_PER_CHANNEL; j++) { - populate_dimm_info(&dimm_info[i][j], addr_decode, j, - skl); - edac_dbg(0, "size: 0x%x, rank: %d, width: %d\n", - dimm_info[i][j].size, - dimm_info[i][j].dual_rank, - dimm_info[i][j].x16_width); - } + edac_dbg(0, "addr_decode: 0x%x\n", addr_decode[i]); } /* @@ -489,14 +479,22 @@ static int ie31200_probe1(struct pci_dev *pdev, int dev_idx) */ for (i = 0; i < IE31200_DIMMS_PER_CHANNEL; i++) { for (j = 0; j < IE31200_CHANNELS; j++) { + struct dimm_data dimm_info; struct dimm_info *dimm; unsigned long nr_pages; - nr_pages = IE31200_PAGES(dimm_info[j][i].size, skl); + populate_dimm_info(&dimm_info, addr_decode[j], i, + skl); + edac_dbg(0, "size: 0x%x, rank: %d, width: %d\n", + dimm_info.size, + dimm_info.dual_rank, + dimm_info.x16_width); + + nr_pages = IE31200_PAGES(dimm_info.size, skl); if (nr_pages == 0) continue; - if (dimm_info[j][i].dual_rank) { + if (dimm_info.dual_rank) { nr_pages = nr_pages / 2; dimm = edac_get_dimm(mci, (i * 2) + 1, j, 0); dimm->nr_pages = nr_pages; diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c index 0c779a0326b62..6c3a40e6f4c63 100644 --- a/drivers/edac/sb_edac.c +++ b/drivers/edac/sb_edac.c @@ -109,8 +109,8 @@ static const u32 knl_interleave_list[] = { 0x104, 0x10c, 0x114, 0x11c, /* 20-23 */ }; #define MAX_INTERLEAVE \ - (max_t(unsigned int, ARRAY_SIZE(sbridge_interleave_list), \ - max_t(unsigned int, ARRAY_SIZE(ibridge_interleave_list), \ + (MAX_T(unsigned int, ARRAY_SIZE(sbridge_interleave_list), \ + MAX_T(unsigned int, ARRAY_SIZE(ibridge_interleave_list), \ ARRAY_SIZE(knl_interleave_list)))) struct interleave_pkg { diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index d47f0055217e4..9e43aed72bd9f 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -115,6 +115,7 @@ EXPORT_SYMBOL_GPL(skx_adxl_get); void skx_adxl_put(void) { + adxl_component_count = 0; kfree(adxl_values); kfree(adxl_msg); } diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 5acfef8fd3d36..1fa31a6fdfcd5 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -45,7 +45,6 @@ #define I10NM_NUM_CHANNELS MAX(I10NM_NUM_DDR_CHANNELS, I10NM_NUM_HBM_CHANNELS) #define I10NM_NUM_DIMMS MAX(I10NM_NUM_DDR_DIMMS, I10NM_NUM_HBM_DIMMS) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) #define NUM_IMC MAX(SKX_NUM_IMC, I10NM_NUM_IMC) #define NUM_CHANNELS MAX(SKX_NUM_CHANNELS, I10NM_NUM_CHANNELS) #define NUM_DIMMS MAX(SKX_NUM_DIMMS, I10NM_NUM_DIMMS) @@ -80,6 +79,9 @@ */ #define MCACOD_EXT_MEM_ERR 0x280 +/* Max RRL register sets per {,sub-,pseudo-}channel. */ +#define NUM_RRL_SET 3 + /* * Each cpu socket contains some pci devices that provide global * information, and also some that are local to each of the two @@ -118,9 +120,11 @@ struct skx_dev { struct skx_channel { struct pci_dev *cdev; struct pci_dev *edev; - u32 retry_rd_err_log_s; - u32 retry_rd_err_log_d; - u32 retry_rd_err_log_d2; + /* + * Two groups of RRL control registers per channel to save default RRL + * settings of two {sub-,pseudo-}channels in Linux RRL control mode. + */ + u32 rrl_ctl[2][NUM_RRL_SET]; struct skx_dimm { u8 close_pg; u8 bank_xor_enable; diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c index 6ddc90d7ba7c2..f8aaada42d3f2 100644 --- a/drivers/edac/synopsys_edac.c +++ b/drivers/edac/synopsys_edac.c @@ -332,20 +332,26 @@ struct synps_edac_priv { #endif }; +enum synps_platform_type { + ZYNQ, + ZYNQMP, + SYNPS, +}; + /** * struct synps_platform_data - synps platform data structure. + * @platform: Identifies the target hardware platform * @get_error_info: Get EDAC error info. * @get_mtype: Get mtype. * @get_dtype: Get dtype. - * @get_ecc_state: Get ECC state. * @get_mem_info: Get EDAC memory info * @quirks: To differentiate IPs. */ struct synps_platform_data { + enum synps_platform_type platform; int (*get_error_info)(struct synps_edac_priv *priv); enum mem_type (*get_mtype)(const void __iomem *base); enum dev_type (*get_dtype)(const void __iomem *base); - bool (*get_ecc_state)(void __iomem *base); #ifdef CONFIG_EDAC_DEBUG u64 (*get_mem_info)(struct synps_edac_priv *priv); #endif @@ -720,51 +726,38 @@ static enum dev_type zynqmp_get_dtype(const void __iomem *base) return dt; } -/** - * zynq_get_ecc_state - Return the controller ECC enable/disable status. - * @base: DDR memory controller base address. - * - * Get the ECC enable/disable status of the controller. - * - * Return: true if enabled, otherwise false. - */ -static bool zynq_get_ecc_state(void __iomem *base) +static bool get_ecc_state(struct synps_edac_priv *priv) { + u32 ecctype, clearval; enum dev_type dt; - u32 ecctype; - - dt = zynq_get_dtype(base); - if (dt == DEV_UNKNOWN) - return false; - ecctype = readl(base + SCRUB_OFST) & SCRUB_MODE_MASK; - if ((ecctype == SCRUB_MODE_SECDED) && (dt == DEV_X2)) - return true; - - return false; -} - -/** - * zynqmp_get_ecc_state - Return the controller ECC enable/disable status. - * @base: DDR memory controller base address. - * - * Get the ECC enable/disable status for the controller. - * - * Return: a ECC status boolean i.e true/false - enabled/disabled. - */ -static bool zynqmp_get_ecc_state(void __iomem *base) -{ - enum dev_type dt; - u32 ecctype; - - dt = zynqmp_get_dtype(base); - if (dt == DEV_UNKNOWN) - return false; - - ecctype = readl(base + ECC_CFG0_OFST) & SCRUB_MODE_MASK; - if ((ecctype == SCRUB_MODE_SECDED) && - ((dt == DEV_X2) || (dt == DEV_X4) || (dt == DEV_X8))) - return true; + if (priv->p_data->platform == ZYNQ) { + dt = zynq_get_dtype(priv->baseaddr); + if (dt == DEV_UNKNOWN) + return false; + + ecctype = readl(priv->baseaddr + SCRUB_OFST) & SCRUB_MODE_MASK; + if (ecctype == SCRUB_MODE_SECDED && dt == DEV_X2) { + clearval = ECC_CTRL_CLR_CE_ERR | ECC_CTRL_CLR_UE_ERR; + writel(clearval, priv->baseaddr + ECC_CTRL_OFST); + writel(0x0, priv->baseaddr + ECC_CTRL_OFST); + return true; + } + } else { + dt = zynqmp_get_dtype(priv->baseaddr); + if (dt == DEV_UNKNOWN) + return false; + + ecctype = readl(priv->baseaddr + ECC_CFG0_OFST) & SCRUB_MODE_MASK; + if (ecctype == SCRUB_MODE_SECDED && + (dt == DEV_X2 || dt == DEV_X4 || dt == DEV_X8)) { + clearval = readl(priv->baseaddr + ECC_CLR_OFST) | + ECC_CTRL_CLR_CE_ERR | ECC_CTRL_CLR_CE_ERRCNT | + ECC_CTRL_CLR_UE_ERR | ECC_CTRL_CLR_UE_ERRCNT; + writel(clearval, priv->baseaddr + ECC_CLR_OFST); + return true; + } + } return false; } @@ -934,18 +927,18 @@ static int setup_irq(struct mem_ctl_info *mci, } static const struct synps_platform_data zynq_edac_def = { + .platform = ZYNQ, .get_error_info = zynq_get_error_info, .get_mtype = zynq_get_mtype, .get_dtype = zynq_get_dtype, - .get_ecc_state = zynq_get_ecc_state, .quirks = 0, }; static const struct synps_platform_data zynqmp_edac_def = { + .platform = ZYNQMP, .get_error_info = zynqmp_get_error_info, .get_mtype = zynqmp_get_mtype, .get_dtype = zynqmp_get_dtype, - .get_ecc_state = zynqmp_get_ecc_state, #ifdef CONFIG_EDAC_DEBUG .get_mem_info = zynqmp_get_mem_info, #endif @@ -957,10 +950,10 @@ static const struct synps_platform_data zynqmp_edac_def = { }; static const struct synps_platform_data synopsys_edac_def = { + .platform = SYNPS, .get_error_info = zynqmp_get_error_info, .get_mtype = zynqmp_get_mtype, .get_dtype = zynqmp_get_dtype, - .get_ecc_state = zynqmp_get_ecc_state, .quirks = (DDR_ECC_INTR_SUPPORT | DDR_ECC_INTR_SELF_CLEAR #ifdef CONFIG_EDAC_DEBUG | DDR_ECC_DATA_POISON_SUPPORT @@ -1392,10 +1385,6 @@ static int mc_probe(struct platform_device *pdev) if (!p_data) return -ENODEV; - if (!p_data->get_ecc_state(baseaddr)) { - edac_printk(KERN_INFO, EDAC_MC, "ECC not enabled\n"); - return -ENXIO; - } layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; layers[0].size = SYNPS_EDAC_NR_CSROWS; @@ -1415,6 +1404,12 @@ static int mc_probe(struct platform_device *pdev) priv = mci->pvt_info; priv->baseaddr = baseaddr; priv->p_data = p_data; + if (!get_ecc_state(priv)) { + edac_printk(KERN_INFO, EDAC_MC, "ECC not enabled\n"); + rc = -ENODEV; + goto free_edac_mc; + } + spin_lock_init(&priv->reglock); mc_init(mci, pdev); diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c index 0317b614b6805..26b083ccc94b2 100644 --- a/drivers/extcon/extcon-adc-jack.c +++ b/drivers/extcon/extcon-adc-jack.c @@ -162,6 +162,8 @@ static int adc_jack_remove(struct platform_device *pdev) { struct adc_jack_data *data = platform_get_drvdata(pdev); + if (data->wakeup_source) + device_init_wakeup(&pdev->dev, false); free_irq(data->irq, data); cancel_work_sync(&data->handler.work); diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c index 73cc2f2dcbf92..bdf4b035325e9 100644 --- a/drivers/firewire/core-cdev.c +++ b/drivers/firewire/core-cdev.c @@ -39,7 +39,7 @@ /* * ABI version history is documented in linux/firewire-cdev.h. */ -#define FW_CDEV_KERNEL_VERSION 5 +#define FW_CDEV_KERNEL_VERSION 6 #define FW_CDEV_VERSION_EVENT_REQUEST2 4 #define FW_CDEV_VERSION_ALLOCATE_REGION_END 4 #define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW 5 diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 3f2f22e47bfa1..8ecffdce94b16 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -40,7 +40,6 @@ config ARM_SCPI_POWER_DOMAIN config ARM_SDE_INTERFACE bool "ARM Software Delegated Exception Interface (SDEI)" depends on ARM64 - depends on ACPI_APEI_GHES help The Software Delegated Exception Interface (SDEI) is an ARM standard for registering callbacks from the platform firmware diff --git a/drivers/firmware/arm_ffa/bus.c b/drivers/firmware/arm_ffa/bus.c index 7865438b36960..d885e1381072a 100644 --- a/drivers/firmware/arm_ffa/bus.c +++ b/drivers/firmware/arm_ffa/bus.c @@ -191,6 +191,7 @@ struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id, dev = &ffa_dev->dev; dev->bus = &ffa_bus_type; dev->release = ffa_release_device; + dev->dma_mask = &dev->coherent_dma_mask; dev_set_name(&ffa_dev->dev, "arm-ffa-%d", id); ffa_dev->id = id; diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index f871c1d0cdf75..7e5fbde69de3c 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -147,6 +147,14 @@ static int ffa_version_check(u32 *version) return -EOPNOTSUPP; } + if (FFA_MAJOR_VERSION(ver.a0) > FFA_MAJOR_VERSION(FFA_DRIVER_VERSION)) { + pr_err("Incompatible v%d.%d! Latest supported v%d.%d\n", + FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0), + FFA_MAJOR_VERSION(FFA_DRIVER_VERSION), + FFA_MINOR_VERSION(FFA_DRIVER_VERSION)); + return -EINVAL; + } + if (ver.a0 < FFA_MIN_VERSION) { pr_err("Incompatible v%d.%d! Earliest supported v%d.%d\n", FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0), @@ -251,7 +259,8 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, memcpy(buffer + idx, drv_info->rx_buffer + idx * sz, buf_sz); - ffa_rx_release(); + if (!(flags & PARTITION_INFO_GET_RETURN_COUNT_ONLY)) + ffa_rx_release(); mutex_unlock(&drv_info->rx_lock); diff --git a/drivers/firmware/arm_scmi/Kconfig b/drivers/firmware/arm_scmi/Kconfig index 706d1264d0389..d2542a415d66e 100644 --- a/drivers/firmware/arm_scmi/Kconfig +++ b/drivers/firmware/arm_scmi/Kconfig @@ -55,6 +55,20 @@ config ARM_SCMI_RAW_MODE_SUPPORT_COEX operate normally, thing which could make an SCMI test suite using the SCMI Raw mode support unreliable. If unsure, say N. +config ARM_SCMI_DEBUG_COUNTERS + bool "Enable SCMI communication debug metrics tracking" + select ARM_SCMI_NEED_DEBUGFS + depends on DEBUG_FS + default n + help + Enables tracking of some key communication metrics for debug + purposes. It may track metrics like how many messages were sent + or received, were there any failures, what kind of failures, ..etc. + + Enable this option to create a new debugfs directory which contains + such useful debug counters. This can be helpful for debugging and + SCMI monitoring. + config ARM_SCMI_HAVE_TRANSPORT bool help diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c index dcf774d3edfe4..e1b949aedf9e0 100644 --- a/drivers/firmware/arm_scmi/bus.c +++ b/drivers/firmware/arm_scmi/bus.c @@ -42,7 +42,7 @@ static atomic_t scmi_syspower_registered = ATOMIC_INIT(0); * This helper let an SCMI driver request specific devices identified by the * @id_table to be created for each active SCMI instance. * - * The requested device name MUST NOT be already existent for any protocol; + * The requested device name MUST NOT be already existent for this protocol; * at first the freshly requested @id_table is annotated in the IDR table * @scmi_requested_devices and then the requested device is advertised to any * registered party via the @scmi_requested_devices_nh notification chain. @@ -52,7 +52,6 @@ static atomic_t scmi_syspower_registered = ATOMIC_INIT(0); static int scmi_protocol_device_request(const struct scmi_device_id *id_table) { int ret = 0; - unsigned int id = 0; struct list_head *head, *phead = NULL; struct scmi_requested_dev *rdev; @@ -67,19 +66,13 @@ static int scmi_protocol_device_request(const struct scmi_device_id *id_table) } /* - * Search for the matching protocol rdev list and then search - * of any existent equally named device...fails if any duplicate found. + * Find the matching protocol rdev list and then search of any + * existent equally named device...fails if any duplicate found. */ mutex_lock(&scmi_requested_devices_mtx); - idr_for_each_entry(&scmi_requested_devices, head, id) { - if (!phead) { - /* A list found registered in the IDR is never empty */ - rdev = list_first_entry(head, struct scmi_requested_dev, - node); - if (rdev->id_table->protocol_id == - id_table->protocol_id) - phead = head; - } + phead = idr_find(&scmi_requested_devices, id_table->protocol_id); + if (phead) { + head = phead; list_for_each_entry(rdev, head, node) { if (!strcmp(rdev->id_table->name, id_table->name)) { pr_err("Ignoring duplicate request [%d] %s\n", @@ -240,6 +233,9 @@ static struct scmi_device *scmi_child_dev_find(struct device *parent, if (!dev) return NULL; + /* Drop the refcnt bumped implicitly by device_find_child */ + put_device(dev); + return to_scmi_dev(dev); } diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index 0456f81730c14..84cd0652e3c7e 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -303,6 +303,59 @@ extern const struct scmi_desc scmi_optee_desc; void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr, void *priv); +enum debug_counters { + SENT_OK, + SENT_FAIL, + SENT_FAIL_POLLING_UNSUPPORTED, + SENT_FAIL_CHANNEL_NOT_FOUND, + RESPONSE_OK, + NOTIFICATION_OK, + DELAYED_RESPONSE_OK, + XFERS_RESPONSE_TIMEOUT, + XFERS_RESPONSE_POLLED_TIMEOUT, + RESPONSE_POLLED_OK, + ERR_MSG_UNEXPECTED, + ERR_MSG_INVALID, + ERR_MSG_NOMEM, + ERR_PROTOCOL, + SCMI_DEBUG_COUNTERS_LAST +}; + +/** + * struct scmi_debug_info - Debug common info + * @top_dentry: A reference to the top debugfs dentry + * @name: Name of this SCMI instance + * @type: Type of this SCMI instance + * @is_atomic: Flag to state if the transport of this instance is atomic + * @counters: An array of atomic_c's used for tracking statistics (if enabled) + */ +struct scmi_debug_info { + struct dentry *top_dentry; + const char *name; + const char *type; + bool is_atomic; + atomic_t counters[SCMI_DEBUG_COUNTERS_LAST]; +}; + +static inline void scmi_inc_count(struct scmi_debug_info *dbg, int stat) +{ + if (IS_ENABLED(CONFIG_ARM_SCMI_DEBUG_COUNTERS)) { + if (dbg) + atomic_inc(&dbg->counters[stat]); + } +} + +enum scmi_bad_msg { + MSG_UNEXPECTED = -1, + MSG_INVALID = -2, + MSG_UNKNOWN = -3, + MSG_NOMEM = -4, + MSG_MBOX_SPURIOUS = -5, +}; + +void scmi_bad_message_trace(struct scmi_chan_info *cinfo, u32 msg_hdr, + enum scmi_bad_msg err); + /* shmem related declarations */ struct scmi_shared_mem; diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index da221ac1c9e77..be02e492daf5f 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -104,20 +104,6 @@ struct scmi_protocol_instance { #define ph_to_pi(h) container_of(h, struct scmi_protocol_instance, ph) -/** - * struct scmi_debug_info - Debug common info - * @top_dentry: A reference to the top debugfs dentry - * @name: Name of this SCMI instance - * @type: Type of this SCMI instance - * @is_atomic: Flag to state if the transport of this instance is atomic - */ -struct scmi_debug_info { - struct dentry *top_dentry; - const char *name; - const char *type; - bool is_atomic; -}; - /** * struct scmi_info - Structure representing a SCMI instance * @@ -645,6 +631,7 @@ __scmi_xfer_put(struct scmi_xfers_info *minfo, struct scmi_xfer *xfer) hash_del(&xfer->node); xfer->pending = false; } + xfer->flags = 0; hlist_add_head(&xfer->node, &minfo->free_xfers); } spin_unlock_irqrestore(&minfo->xfer_lock, flags); @@ -663,8 +650,6 @@ void scmi_xfer_raw_put(const struct scmi_handle *handle, struct scmi_xfer *xfer) { struct scmi_info *info = handle_to_scmi_info(handle); - xfer->flags &= ~SCMI_XFER_FLAG_IS_RAW; - xfer->flags &= ~SCMI_XFER_FLAG_CHAN_SET; return __scmi_xfer_put(&info->tx_minfo, xfer); } @@ -691,6 +676,45 @@ scmi_xfer_lookup_unlocked(struct scmi_xfers_info *minfo, u16 xfer_id) return xfer ?: ERR_PTR(-EINVAL); } +/** + * scmi_bad_message_trace - A helper to trace weird messages + * + * @cinfo: A reference to the channel descriptor on which the message was + * received + * @msg_hdr: Message header to track + * @err: A specific error code used as a status value in traces. + * + * This helper can be used to trace any kind of weird, incomplete, unexpected, + * timed-out message that arrives and as such, can be traced only referring to + * the header content, since the payload is missing/unreliable. + */ +void scmi_bad_message_trace(struct scmi_chan_info *cinfo, u32 msg_hdr, + enum scmi_bad_msg err) +{ + char *tag; + struct scmi_info *info = handle_to_scmi_info(cinfo->handle); + + switch (MSG_XTRACT_TYPE(msg_hdr)) { + case MSG_TYPE_COMMAND: + tag = "!RESP"; + break; + case MSG_TYPE_DELAYED_RESP: + tag = "!DLYD"; + break; + case MSG_TYPE_NOTIFICATION: + tag = "!NOTI"; + break; + default: + tag = "!UNKN"; + break; + } + + trace_scmi_msg_dump(info->id, cinfo->id, + MSG_XTRACT_PROT_ID(msg_hdr), + MSG_XTRACT_ID(msg_hdr), tag, + MSG_XTRACT_TOKEN(msg_hdr), err, NULL, 0); +} + /** * scmi_msg_response_validate - Validate message type against state of related * xfer @@ -817,6 +841,10 @@ scmi_xfer_command_acquire(struct scmi_chan_info *cinfo, u32 msg_hdr) "Message for %d type %d is not expected!\n", xfer_id, msg_type); spin_unlock_irqrestore(&minfo->xfer_lock, flags); + + scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNEXPECTED); + scmi_inc_count(info->dbg, ERR_MSG_UNEXPECTED); + return xfer; } refcount_inc(&xfer->users); @@ -841,6 +869,11 @@ scmi_xfer_command_acquire(struct scmi_chan_info *cinfo, u32 msg_hdr) dev_err(cinfo->dev, "Invalid message type:%d for %d - HDR:0x%X state:%d\n", msg_type, xfer_id, msg_hdr, xfer->state); + + scmi_bad_message_trace(cinfo, msg_hdr, MSG_INVALID); + scmi_inc_count(info->dbg, ERR_MSG_INVALID); + + /* On error the refcount incremented above has to be dropped */ __scmi_xfer_put(minfo, xfer); xfer = ERR_PTR(-EINVAL); @@ -882,6 +915,10 @@ static void scmi_handle_notification(struct scmi_chan_info *cinfo, if (IS_ERR(xfer)) { dev_err(dev, "failed to get free message slot (%ld)\n", PTR_ERR(xfer)); + + scmi_bad_message_trace(cinfo, msg_hdr, MSG_NOMEM); + scmi_inc_count(info->dbg, ERR_MSG_NOMEM); + scmi_clear_channel(info, cinfo); return; } @@ -896,6 +933,7 @@ static void scmi_handle_notification(struct scmi_chan_info *cinfo, trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id, xfer->hdr.id, "NOTI", xfer->hdr.seq, xfer->hdr.status, xfer->rx.buf, xfer->rx.len); + scmi_inc_count(info->dbg, NOTIFICATION_OK); scmi_notify(cinfo->handle, xfer->hdr.protocol_id, xfer->hdr.id, xfer->rx.buf, xfer->rx.len, ts); @@ -955,8 +993,10 @@ static void scmi_handle_response(struct scmi_chan_info *cinfo, if (xfer->hdr.type == MSG_TYPE_DELAYED_RESP) { scmi_clear_channel(info, cinfo); complete(xfer->async_done); + scmi_inc_count(info->dbg, DELAYED_RESPONSE_OK); } else { complete(&xfer->done); + scmi_inc_count(info->dbg, RESPONSE_OK); } if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) { @@ -1001,6 +1041,7 @@ void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr, void *priv) break; default: WARN_ONCE(1, "received unknown msg_type:%d\n", msg_type); + scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNKNOWN); break; } } @@ -1021,7 +1062,8 @@ static void xfer_put(const struct scmi_protocol_handle *ph, } static bool scmi_xfer_done_no_timeout(struct scmi_chan_info *cinfo, - struct scmi_xfer *xfer, ktime_t stop) + struct scmi_xfer *xfer, ktime_t stop, + bool *ooo) { struct scmi_info *info = handle_to_scmi_info(cinfo->handle); @@ -1030,7 +1072,7 @@ static bool scmi_xfer_done_no_timeout(struct scmi_chan_info *cinfo, * in case of out-of-order receptions of delayed responses */ return info->desc->ops->poll_done(cinfo, xfer) || - try_wait_for_completion(&xfer->done) || + (*ooo = try_wait_for_completion(&xfer->done)) || ktime_after(ktime_get(), stop); } @@ -1039,6 +1081,7 @@ static int scmi_wait_for_reply(struct device *dev, const struct scmi_desc *desc, struct scmi_xfer *xfer, unsigned int timeout_ms) { int ret = 0; + struct scmi_info *info = handle_to_scmi_info(cinfo->handle); if (xfer->hdr.poll_completion) { /* @@ -1046,26 +1089,27 @@ static int scmi_wait_for_reply(struct device *dev, const struct scmi_desc *desc, * itself to support synchronous commands replies. */ if (!desc->sync_cmds_completed_on_ret) { + bool ooo = false; + /* * Poll on xfer using transport provided .poll_done(); * assumes no completion interrupt was available. */ ktime_t stop = ktime_add_ms(ktime_get(), timeout_ms); - spin_until_cond(scmi_xfer_done_no_timeout(cinfo, - xfer, stop)); - if (ktime_after(ktime_get(), stop)) { + spin_until_cond(scmi_xfer_done_no_timeout(cinfo, xfer, + stop, &ooo)); + if (!ooo && !info->desc->ops->poll_done(cinfo, xfer)) { dev_err(dev, "timed out in resp(caller: %pS) - polling\n", (void *)_RET_IP_); ret = -ETIMEDOUT; + scmi_inc_count(info->dbg, XFERS_RESPONSE_POLLED_TIMEOUT); } } if (!ret) { unsigned long flags; - struct scmi_info *info = - handle_to_scmi_info(cinfo->handle); /* * Do not fetch_response if an out-of-order delayed @@ -1085,6 +1129,7 @@ static int scmi_wait_for_reply(struct device *dev, const struct scmi_desc *desc, "RESP" : "resp", xfer->hdr.seq, xfer->hdr.status, xfer->rx.buf, xfer->rx.len); + scmi_inc_count(info->dbg, RESPONSE_POLLED_OK); if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) { struct scmi_info *info = @@ -1102,6 +1147,7 @@ static int scmi_wait_for_reply(struct device *dev, const struct scmi_desc *desc, dev_err(dev, "timed out in resp(caller: %pS)\n", (void *)_RET_IP_); ret = -ETIMEDOUT; + scmi_inc_count(info->dbg, XFERS_RESPONSE_TIMEOUT); } } @@ -1185,13 +1231,15 @@ static int do_xfer(const struct scmi_protocol_handle *ph, !is_transport_polling_capable(info->desc)) { dev_warn_once(dev, "Polling mode is not supported by transport.\n"); + scmi_inc_count(info->dbg, SENT_FAIL_POLLING_UNSUPPORTED); return -EINVAL; } cinfo = idr_find(&info->tx_idr, pi->proto->id); - if (unlikely(!cinfo)) + if (unlikely(!cinfo)) { + scmi_inc_count(info->dbg, SENT_FAIL_CHANNEL_NOT_FOUND); return -EINVAL; - + } /* True ONLY if also supported by transport. */ if (is_polling_enabled(cinfo, info->desc)) xfer->hdr.poll_completion = true; @@ -1223,16 +1271,20 @@ static int do_xfer(const struct scmi_protocol_handle *ph, ret = info->desc->ops->send_message(cinfo, xfer); if (ret < 0) { dev_dbg(dev, "Failed to send message %d\n", ret); + scmi_inc_count(info->dbg, SENT_FAIL); return ret; } trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id, xfer->hdr.id, "CMND", xfer->hdr.seq, xfer->hdr.status, xfer->tx.buf, xfer->tx.len); + scmi_inc_count(info->dbg, SENT_OK); ret = scmi_wait_for_message_response(cinfo, xfer); - if (!ret && xfer->hdr.status) + if (!ret && xfer->hdr.status) { ret = scmi_to_linux_errno(xfer->hdr.status); + scmi_inc_count(info->dbg, ERR_PROTOCOL); + } if (info->desc->ops->mark_txdone) info->desc->ops->mark_txdone(cinfo, ret, xfer); @@ -1502,6 +1554,39 @@ static int scmi_common_get_max_msg_size(const struct scmi_protocol_handle *ph) return info->desc->max_msg_size; } +/** + * scmi_protocol_msg_check - Check protocol message attributes + * + * @ph: A reference to the protocol handle. + * @message_id: The ID of the message to check. + * @attributes: A parameter to optionally return the retrieved message + * attributes, in case of Success. + * + * An helper to check protocol message attributes for a specific protocol + * and message pair. + * + * Return: 0 on SUCCESS + */ +static int scmi_protocol_msg_check(const struct scmi_protocol_handle *ph, + u32 message_id, u32 *attributes) +{ + int ret; + struct scmi_xfer *t; + + ret = xfer_get_init(ph, PROTOCOL_MESSAGE_ATTRIBUTES, + sizeof(__le32), 0, &t); + if (ret) + return ret; + + put_unaligned_le32(message_id, t->tx.buf); + ret = do_xfer(ph, t); + if (!ret && attributes) + *attributes = get_unaligned_le32(t->rx.buf); + xfer_put(ph, t); + + return ret; +} + /** * struct scmi_iterator - Iterator descriptor * @msg: A reference to the message TX buffer; filled by @prepare_message with @@ -1643,6 +1728,7 @@ scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, int ret; u32 flags; u64 phys_addr; + u32 attributes; u8 size; void __iomem *addr; struct scmi_xfer *t; @@ -1651,6 +1737,15 @@ scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, struct scmi_msg_resp_desc_fc *resp; const struct scmi_protocol_instance *pi = ph_to_pi(ph); + /* Check if the MSG_ID supports fastchannel */ + ret = scmi_protocol_msg_check(ph, message_id, &attributes); + if (ret || !MSG_SUPPORTS_FASTCHANNEL(attributes)) { + dev_dbg(ph->dev, + "Skip FC init for 0x%02X/%d domain:%d - ret:%d\n", + pi->proto->id, message_id, domain, ret); + return; + } + if (!p_addr) { ret = -EINVAL; goto err_out; diff --git a/drivers/firmware/arm_scmi/mailbox.c b/drivers/firmware/arm_scmi/mailbox.c index 4417239561ba3..726982b7a2c76 100644 --- a/drivers/firmware/arm_scmi/mailbox.c +++ b/drivers/firmware/arm_scmi/mailbox.c @@ -60,6 +60,9 @@ static void rx_callback(struct mbox_client *cl, void *m) */ if (cl->knows_txdone && !shmem_channel_free(smbox->shmem)) { dev_warn(smbox->cinfo->dev, "Ignoring spurious A2P IRQ !\n"); + scmi_bad_message_trace(smbox->cinfo, + shmem_read_header(smbox->shmem), + MSG_MBOX_SPURIOUS); return; } diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h index 996410962b91f..26e24eca2d191 100644 --- a/drivers/firmware/arm_scmi/protocols.h +++ b/drivers/firmware/arm_scmi/protocols.h @@ -29,6 +29,8 @@ #define PROTOCOL_REV_MAJOR(x) ((u16)(FIELD_GET(PROTOCOL_REV_MAJOR_MASK, (x)))) #define PROTOCOL_REV_MINOR(x) ((u16)(FIELD_GET(PROTOCOL_REV_MINOR_MASK, (x)))) +#define MSG_SUPPORTS_FASTCHANNEL(x) ((x) & BIT(0)) + enum scmi_common_cmd { PROTOCOL_VERSION = 0x0, PROTOCOL_ATTRIBUTES = 0x1, diff --git a/drivers/firmware/arm_scmi/scmi_pm_domain.c b/drivers/firmware/arm_scmi/scmi_pm_domain.c index 0e05a79de82d8..82d923a9d75d7 100644 --- a/drivers/firmware/arm_scmi/scmi_pm_domain.c +++ b/drivers/firmware/arm_scmi/scmi_pm_domain.c @@ -54,7 +54,7 @@ static int scmi_pd_power_off(struct generic_pm_domain *domain) static int scmi_pm_domain_probe(struct scmi_device *sdev) { - int num_domains, i; + int num_domains, i, ret; struct device *dev = &sdev->dev; struct device_node *np = dev->of_node; struct scmi_pm_domain *scmi_pd; @@ -112,9 +112,18 @@ static int scmi_pm_domain_probe(struct scmi_device *sdev) scmi_pd_data->domains = domains; scmi_pd_data->num_domains = num_domains; + ret = of_genpd_add_provider_onecell(np, scmi_pd_data); + if (ret) + goto err_rm_genpds; + dev_set_drvdata(dev, scmi_pd_data); - return of_genpd_add_provider_onecell(np, scmi_pd_data); + return 0; +err_rm_genpds: + for (i = num_domains - 1; i >= 0; i--) + pm_genpd_remove(domains[i]); + + return ret; } static void scmi_pm_domain_remove(struct scmi_device *sdev) diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c index 3e8051fe82965..71e2a9a89f6ad 100644 --- a/drivers/firmware/arm_sdei.c +++ b/drivers/firmware/arm_sdei.c @@ -1062,13 +1062,12 @@ static bool __init sdei_present_acpi(void) return true; } -void __init sdei_init(void) +void __init acpi_sdei_init(void) { struct platform_device *pdev; int ret; - ret = platform_driver_register(&sdei_driver); - if (ret || !sdei_present_acpi()) + if (!sdei_present_acpi()) return; pdev = platform_device_register_simple(sdei_driver.driver.name, @@ -1081,6 +1080,12 @@ void __init sdei_init(void) } } +static int __init sdei_init(void) +{ + return platform_driver_register(&sdei_driver); +} +arch_initcall(sdei_init); + int sdei_event_handler(struct pt_regs *regs, struct sdei_registered_event *arg) { diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c index 3dc2f9aaf08db..492d09b6048bd 100644 --- a/drivers/firmware/efi/libstub/efi-stub-helper.c +++ b/drivers/firmware/efi/libstub/efi-stub-helper.c @@ -561,6 +561,7 @@ efi_status_t efi_load_initrd_cmdline(efi_loaded_image_t *image, * @image: EFI loaded image protocol * @soft_limit: preferred address for loading the initrd * @hard_limit: upper limit address for loading the initrd + * @out: pointer to store the address of the initrd table * * Return: status code */ diff --git a/drivers/firmware/meson/Kconfig b/drivers/firmware/meson/Kconfig index f2fdd37566482..179f5d46d8ddf 100644 --- a/drivers/firmware/meson/Kconfig +++ b/drivers/firmware/meson/Kconfig @@ -5,7 +5,7 @@ config MESON_SM tristate "Amlogic Secure Monitor driver" depends on ARCH_MESON || COMPILE_TEST - default y + default ARCH_MESON depends on ARM64_4K_PAGES help Say y here to enable the Amlogic secure monitor driver diff --git a/drivers/firmware/meson/meson_sm.c b/drivers/firmware/meson/meson_sm.c index 9a2656d73600b..b39f395c077ee 100644 --- a/drivers/firmware/meson/meson_sm.c +++ b/drivers/firmware/meson/meson_sm.c @@ -225,11 +225,16 @@ EXPORT_SYMBOL(meson_sm_call_write); struct meson_sm_firmware *meson_sm_get(struct device_node *sm_node) { struct platform_device *pdev = of_find_device_by_node(sm_node); + struct meson_sm_firmware *fw; if (!pdev) return NULL; - return platform_get_drvdata(pdev); + fw = platform_get_drvdata(pdev); + + put_device(&pdev->dev); + + return fw; } EXPORT_SYMBOL_GPL(meson_sm_get); diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c index e2125e7a53077..fe36c59e0ae37 100644 --- a/drivers/firmware/psci/psci.c +++ b/drivers/firmware/psci/psci.c @@ -771,8 +771,10 @@ int __init psci_dt_init(void) np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np); - if (!np || !of_device_is_available(np)) + if (!np || !of_device_is_available(np)) { + of_node_put(np); return -ENODEV; + } init_fn = (psci_initcall_t)matched_np->data; ret = init_fn(np); diff --git a/drivers/firmware/tegra/Kconfig b/drivers/firmware/tegra/Kconfig index cde1ab8bd9d1c..91f2320c0d0f8 100644 --- a/drivers/firmware/tegra/Kconfig +++ b/drivers/firmware/tegra/Kconfig @@ -2,7 +2,7 @@ menu "Tegra firmware driver" config TEGRA_IVC - bool "Tegra IVC protocol" + bool "Tegra IVC protocol" if COMPILE_TEST depends on ARCH_TEGRA help IVC (Inter-VM Communication) protocol is part of the IPC @@ -13,8 +13,9 @@ config TEGRA_IVC config TEGRA_BPMP bool "Tegra BPMP driver" - depends on ARCH_TEGRA && TEGRA_HSP_MBOX && TEGRA_IVC + depends on ARCH_TEGRA && TEGRA_HSP_MBOX depends on !CPU_BIG_ENDIAN + select TEGRA_IVC help BPMP (Boot and Power Management Processor) is designed to off-loading the PM functions which include clock/DVFS/thermal/power from the CPU. diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c index 4ffb9da537d82..5295ff90482bc 100644 --- a/drivers/fpga/altera-cvp.c +++ b/drivers/fpga/altera-cvp.c @@ -52,7 +52,7 @@ /* V2 Defines */ #define VSE_CVP_TX_CREDITS 0x49 /* 8bit */ -#define V2_CREDIT_TIMEOUT_US 20000 +#define V2_CREDIT_TIMEOUT_US 40000 #define V2_CHECK_CREDIT_US 10 #define V2_POLL_TIMEOUT_US 1000000 #define V2_USER_TIMEOUT_US 500000 diff --git a/drivers/fpga/tests/fpga-mgr-test.c b/drivers/fpga/tests/fpga-mgr-test.c index 6acec55b60ce9..bfe25c0c0c1c3 100644 --- a/drivers/fpga/tests/fpga-mgr-test.c +++ b/drivers/fpga/tests/fpga-mgr-test.c @@ -253,6 +253,7 @@ static void fpga_mgr_test_img_load_sgt(struct kunit *test) img_buf = init_test_buffer(test, IMAGE_SIZE); sgt = kunit_kzalloc(test, sizeof(*sgt), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sgt); ret = sg_alloc_table(sgt, 1, GFP_KERNEL); KUNIT_ASSERT_EQ(test, ret, 0); sg_init_one(sgt->sgl, img_buf, IMAGE_SIZE); diff --git a/drivers/fpga/zynq-fpga.c b/drivers/fpga/zynq-fpga.c index 96611d424a104..b77097c93973a 100644 --- a/drivers/fpga/zynq-fpga.c +++ b/drivers/fpga/zynq-fpga.c @@ -405,12 +405,12 @@ static int zynq_fpga_ops_write(struct fpga_manager *mgr, struct sg_table *sgt) } } - priv->dma_nelms = - dma_map_sg(mgr->dev.parent, sgt->sgl, sgt->nents, DMA_TO_DEVICE); - if (priv->dma_nelms == 0) { + err = dma_map_sgtable(mgr->dev.parent, sgt, DMA_TO_DEVICE, 0); + if (err) { dev_err(&mgr->dev, "Unable to DMA map (TO_DEVICE)\n"); - return -ENOMEM; + return err; } + priv->dma_nelms = sgt->nents; /* enable clock */ err = clk_enable(priv->clk); @@ -478,7 +478,7 @@ static int zynq_fpga_ops_write(struct fpga_manager *mgr, struct sg_table *sgt) clk_disable(priv->clk); out_free: - dma_unmap_sg(mgr->dev.parent, sgt->sgl, sgt->nents, DMA_TO_DEVICE); + dma_unmap_sgtable(mgr->dev.parent, sgt, DMA_TO_DEVICE, 0); return err; } diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 2c80f6aee214d..65d692011bf6a 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -1312,9 +1312,9 @@ config GPIO_KEMPLD config GPIO_LJCA tristate "INTEL La Jolla Cove Adapter GPIO support" - depends on MFD_LJCA + depends on USB_LJCA select GPIOLIB_IRQCHIP - default MFD_LJCA + default USB_LJCA help Select this option to enable GPIO driver for the INTEL La Jolla Cove Adapter (LJCA) board. diff --git a/drivers/gpio/gpio-104-idio-16.c b/drivers/gpio/gpio-104-idio-16.c index f03ccd0f534cf..f60872d1e8a58 100644 --- a/drivers/gpio/gpio-104-idio-16.c +++ b/drivers/gpio/gpio-104-idio-16.c @@ -59,6 +59,7 @@ static const struct regmap_config idio_16_regmap_config = { .reg_stride = 1, .val_bits = 8, .io_port = true, + .max_register = 0x5, .wr_table = &idio_16_wr_table, .rd_table = &idio_16_rd_table, .volatile_table = &idio_16_rd_table, diff --git a/drivers/gpio/gpio-idio-16.c b/drivers/gpio/gpio-idio-16.c index 53b1eb876a125..e978fd0898aaa 100644 --- a/drivers/gpio/gpio-idio-16.c +++ b/drivers/gpio/gpio-idio-16.c @@ -3,6 +3,7 @@ * GPIO library for the ACCES IDIO-16 family * Copyright (C) 2022 William Breathitt Gray */ +#include #include #include #include @@ -106,6 +107,7 @@ int devm_idio_16_regmap_register(struct device *const dev, struct idio_16_data *data; struct regmap_irq_chip *chip; struct regmap_irq_chip_data *chip_data; + DECLARE_BITMAP(fixed_direction_output, IDIO_16_NGPIO); if (!config->parent) return -EINVAL; @@ -163,6 +165,9 @@ int devm_idio_16_regmap_register(struct device *const dev, gpio_config.irq_domain = regmap_irq_get_domain(chip_data); gpio_config.reg_mask_xlate = idio_16_reg_mask_xlate; + bitmap_from_u64(fixed_direction_output, GENMASK_U64(15, 0)); + gpio_config.fixed_direction_output = fixed_direction_output; + return PTR_ERR_OR_ZERO(devm_gpio_regmap_register(dev, &gpio_config)); } EXPORT_SYMBOL_GPL(devm_idio_16_regmap_register); diff --git a/drivers/gpio/gpio-ljca.c b/drivers/gpio/gpio-ljca.c index 87863f0230f5c..c3a595c6f6c72 100644 --- a/drivers/gpio/gpio-ljca.c +++ b/drivers/gpio/gpio-ljca.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -13,19 +14,18 @@ #include #include #include -#include #include -#include #include #include +#include /* GPIO commands */ -#define LJCA_GPIO_CONFIG 1 -#define LJCA_GPIO_READ 2 -#define LJCA_GPIO_WRITE 3 -#define LJCA_GPIO_INT_EVENT 4 -#define LJCA_GPIO_INT_MASK 5 -#define LJCA_GPIO_INT_UNMASK 6 +#define LJCA_GPIO_CONFIG 1 +#define LJCA_GPIO_READ 2 +#define LJCA_GPIO_WRITE 3 +#define LJCA_GPIO_INT_EVENT 4 +#define LJCA_GPIO_INT_MASK 5 +#define LJCA_GPIO_INT_UNMASK 6 #define LJCA_GPIO_CONF_DISABLE BIT(0) #define LJCA_GPIO_CONF_INPUT BIT(1) @@ -36,89 +36,93 @@ #define LJCA_GPIO_CONF_INTERRUPT BIT(6) #define LJCA_GPIO_INT_TYPE BIT(7) -#define LJCA_GPIO_CONF_EDGE FIELD_PREP(LJCA_GPIO_INT_TYPE, 1) -#define LJCA_GPIO_CONF_LEVEL FIELD_PREP(LJCA_GPIO_INT_TYPE, 0) +#define LJCA_GPIO_CONF_EDGE FIELD_PREP(LJCA_GPIO_INT_TYPE, 1) +#define LJCA_GPIO_CONF_LEVEL FIELD_PREP(LJCA_GPIO_INT_TYPE, 0) /* Intentional overlap with PULLUP / PULLDOWN */ -#define LJCA_GPIO_CONF_SET BIT(3) -#define LJCA_GPIO_CONF_CLR BIT(4) +#define LJCA_GPIO_CONF_SET BIT(3) +#define LJCA_GPIO_CONF_CLR BIT(4) -struct gpio_op { +#define LJCA_GPIO_BUF_SIZE 60u + +struct ljca_gpio_op { u8 index; u8 value; } __packed; -struct gpio_packet { +struct ljca_gpio_packet { u8 num; - struct gpio_op item[]; + struct ljca_gpio_op item[] __counted_by(num); } __packed; -#define LJCA_GPIO_BUF_SIZE 60 struct ljca_gpio_dev { - struct platform_device *pdev; + struct ljca_client *ljca; struct gpio_chip gc; struct ljca_gpio_info *gpio_info; DECLARE_BITMAP(unmasked_irqs, LJCA_MAX_GPIO_NUM); DECLARE_BITMAP(enabled_irqs, LJCA_MAX_GPIO_NUM); DECLARE_BITMAP(reenable_irqs, LJCA_MAX_GPIO_NUM); + DECLARE_BITMAP(output_enabled, LJCA_MAX_GPIO_NUM); u8 *connect_mode; - /* mutex to protect irq bus */ + /* protect irq bus */ struct mutex irq_lock; struct work_struct work; - /* lock to protect package transfer to Hardware */ + /* protect package transfer to hardware */ struct mutex trans_lock; u8 obuf[LJCA_GPIO_BUF_SIZE]; u8 ibuf[LJCA_GPIO_BUF_SIZE]; }; -static int gpio_config(struct ljca_gpio_dev *ljca_gpio, u8 gpio_id, u8 config) +static int ljca_gpio_config(struct ljca_gpio_dev *ljca_gpio, u8 gpio_id, + u8 config) { - struct gpio_packet *packet = (struct gpio_packet *)ljca_gpio->obuf; + struct ljca_gpio_packet *packet = + (struct ljca_gpio_packet *)ljca_gpio->obuf; int ret; mutex_lock(&ljca_gpio->trans_lock); + packet->num = 1; packet->item[0].index = gpio_id; packet->item[0].value = config | ljca_gpio->connect_mode[gpio_id]; - packet->num = 1; - ret = ljca_transfer(ljca_gpio->gpio_info->ljca, LJCA_GPIO_CONFIG, packet, - struct_size(packet, item, packet->num), NULL, NULL); + ret = ljca_transfer(ljca_gpio->ljca, LJCA_GPIO_CONFIG, (u8 *)packet, + struct_size(packet, item, packet->num), NULL, 0); mutex_unlock(&ljca_gpio->trans_lock); - return ret; + + return ret < 0 ? ret : 0; } static int ljca_gpio_read(struct ljca_gpio_dev *ljca_gpio, u8 gpio_id) { - struct gpio_packet *packet = (struct gpio_packet *)ljca_gpio->obuf; - struct gpio_packet *ack_packet = (struct gpio_packet *)ljca_gpio->ibuf; - unsigned int ibuf_len = LJCA_GPIO_BUF_SIZE; + struct ljca_gpio_packet *ack_packet = + (struct ljca_gpio_packet *)ljca_gpio->ibuf; + struct ljca_gpio_packet *packet = + (struct ljca_gpio_packet *)ljca_gpio->obuf; int ret; mutex_lock(&ljca_gpio->trans_lock); packet->num = 1; packet->item[0].index = gpio_id; - ret = ljca_transfer(ljca_gpio->gpio_info->ljca, LJCA_GPIO_READ, packet, - struct_size(packet, item, packet->num), ljca_gpio->ibuf, &ibuf_len); - if (ret) - goto out_unlock; - - if (!ibuf_len || ack_packet->num != packet->num) { - dev_err(&ljca_gpio->pdev->dev, "failed gpio_id:%u %u", gpio_id, ack_packet->num); - ret = -EIO; + ret = ljca_transfer(ljca_gpio->ljca, LJCA_GPIO_READ, (u8 *)packet, + struct_size(packet, item, packet->num), + ljca_gpio->ibuf, LJCA_GPIO_BUF_SIZE); + + if (ret <= 0 || ack_packet->num != packet->num) { + dev_err(&ljca_gpio->ljca->auxdev.dev, + "read package error, gpio_id: %u num: %u ret: %d\n", + gpio_id, ack_packet->num, ret); + ret = ret < 0 ? ret : -EIO; } - -out_unlock: mutex_unlock(&ljca_gpio->trans_lock); - if (ret) - return ret; - return ack_packet->item[0].value > 0; + + return ret < 0 ? ret : ack_packet->item[0].value > 0; } -static int ljca_gpio_write(struct ljca_gpio_dev *ljca_gpio, u8 gpio_id, - int value) +static int ljca_gpio_write(struct ljca_gpio_dev *ljca_gpio, u8 gpio_id, int value) { - struct gpio_packet *packet = (struct gpio_packet *)ljca_gpio->obuf; + struct ljca_gpio_packet *packet = + (struct ljca_gpio_packet *)ljca_gpio->obuf; int ret; mutex_lock(&ljca_gpio->trans_lock); @@ -126,10 +130,11 @@ static int ljca_gpio_write(struct ljca_gpio_dev *ljca_gpio, u8 gpio_id, packet->item[0].index = gpio_id; packet->item[0].value = value & 1; - ret = ljca_transfer(ljca_gpio->gpio_info->ljca, LJCA_GPIO_WRITE, packet, - struct_size(packet, item, packet->num), NULL, NULL); + ret = ljca_transfer(ljca_gpio->ljca, LJCA_GPIO_WRITE, (u8 *)packet, + struct_size(packet, item, packet->num), NULL, 0); mutex_unlock(&ljca_gpio->trans_lock); - return ret; + + return ret < 0 ? ret : 0; } static int ljca_gpio_get_value(struct gpio_chip *chip, unsigned int offset) @@ -147,16 +152,24 @@ static void ljca_gpio_set_value(struct gpio_chip *chip, unsigned int offset, ret = ljca_gpio_write(ljca_gpio, offset, val); if (ret) - dev_err(chip->parent, "offset:%u val:%d set value failed %d\n", offset, val, ret); + dev_err(chip->parent, + "set value failed offset: %u val: %d ret: %d\n", + offset, val, ret); } -static int ljca_gpio_direction_input(struct gpio_chip *chip, - unsigned int offset) +static int ljca_gpio_direction_input(struct gpio_chip *chip, unsigned int offset) { struct ljca_gpio_dev *ljca_gpio = gpiochip_get_data(chip); u8 config = LJCA_GPIO_CONF_INPUT | LJCA_GPIO_CONF_CLR; + int ret; - return gpio_config(ljca_gpio, offset, config); + ret = ljca_gpio_config(ljca_gpio, offset, config); + if (ret) + return ret; + + clear_bit(offset, ljca_gpio->output_enabled); + + return 0; } static int ljca_gpio_direction_output(struct gpio_chip *chip, @@ -166,14 +179,26 @@ static int ljca_gpio_direction_output(struct gpio_chip *chip, u8 config = LJCA_GPIO_CONF_OUTPUT | LJCA_GPIO_CONF_CLR; int ret; - ret = gpio_config(ljca_gpio, offset, config); + ret = ljca_gpio_config(ljca_gpio, offset, config); if (ret) return ret; ljca_gpio_set_value(chip, offset, val); + set_bit(offset, ljca_gpio->output_enabled); + return 0; } +static int ljca_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) +{ + struct ljca_gpio_dev *ljca_gpio = gpiochip_get_data(chip); + + if (test_bit(offset, ljca_gpio->output_enabled)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; +} + static int ljca_gpio_set_config(struct gpio_chip *chip, unsigned int offset, unsigned long config) { @@ -197,7 +222,8 @@ static int ljca_gpio_set_config(struct gpio_chip *chip, unsigned int offset, return 0; } -static int ljca_gpio_init_valid_mask(struct gpio_chip *chip, unsigned long *valid_mask, +static int ljca_gpio_init_valid_mask(struct gpio_chip *chip, + unsigned long *valid_mask, unsigned int ngpios) { struct ljca_gpio_dev *ljca_gpio = gpiochip_get_data(chip); @@ -208,15 +234,18 @@ static int ljca_gpio_init_valid_mask(struct gpio_chip *chip, unsigned long *vali return 0; } -static void ljca_gpio_irq_init_valid_mask(struct gpio_chip *chip, unsigned long *valid_mask, +static void ljca_gpio_irq_init_valid_mask(struct gpio_chip *chip, + unsigned long *valid_mask, unsigned int ngpios) { ljca_gpio_init_valid_mask(chip, valid_mask, ngpios); } -static int ljca_enable_irq(struct ljca_gpio_dev *ljca_gpio, int gpio_id, bool enable) +static int ljca_enable_irq(struct ljca_gpio_dev *ljca_gpio, int gpio_id, + bool enable) { - struct gpio_packet *packet = (struct gpio_packet *)ljca_gpio->obuf; + struct ljca_gpio_packet *packet = + (struct ljca_gpio_packet *)ljca_gpio->obuf; int ret; mutex_lock(&ljca_gpio->trans_lock); @@ -224,18 +253,20 @@ static int ljca_enable_irq(struct ljca_gpio_dev *ljca_gpio, int gpio_id, bool en packet->item[0].index = gpio_id; packet->item[0].value = 0; - ret = ljca_transfer(ljca_gpio->gpio_info->ljca, - enable ? LJCA_GPIO_INT_UNMASK : LJCA_GPIO_INT_MASK, packet, - struct_size(packet, item, packet->num), NULL, NULL); + ret = ljca_transfer(ljca_gpio->ljca, + enable ? LJCA_GPIO_INT_UNMASK : LJCA_GPIO_INT_MASK, + (u8 *)packet, struct_size(packet, item, packet->num), + NULL, 0); mutex_unlock(&ljca_gpio->trans_lock); - return ret; + + return ret < 0 ? ret : 0; } static void ljca_gpio_async(struct work_struct *work) { - struct ljca_gpio_dev *ljca_gpio = container_of(work, struct ljca_gpio_dev, work); - int gpio_id; - int unmasked; + struct ljca_gpio_dev *ljca_gpio = + container_of(work, struct ljca_gpio_dev, work); + int gpio_id, unmasked; for_each_set_bit(gpio_id, ljca_gpio->reenable_irqs, ljca_gpio->gc.ngpio) { clear_bit(gpio_id, ljca_gpio->reenable_irqs); @@ -245,25 +276,19 @@ static void ljca_gpio_async(struct work_struct *work) } } -static void ljca_gpio_event_cb(void *context, u8 cmd, const void *evt_data, int len) +static void ljca_gpio_event_cb(void *context, u8 cmd, const void *evt_data, + int len) { - const struct gpio_packet *packet = evt_data; + const struct ljca_gpio_packet *packet = evt_data; struct ljca_gpio_dev *ljca_gpio = context; int i; - int irq; if (cmd != LJCA_GPIO_INT_EVENT) return; for (i = 0; i < packet->num; i++) { - irq = irq_find_mapping(ljca_gpio->gc.irq.domain, packet->item[i].index); - if (!irq) { - dev_err(ljca_gpio->gc.parent, "gpio_id %u does not mapped to IRQ yet\n", - packet->item[i].index); - return; - } - - generic_handle_domain_irq(ljca_gpio->gc.irq.domain, irq); + generic_handle_domain_irq(ljca_gpio->gc.irq.domain, + packet->item[i].index); set_bit(packet->item[i].index, ljca_gpio->reenable_irqs); } @@ -299,18 +324,22 @@ static int ljca_irq_set_type(struct irq_data *irqd, unsigned int type) ljca_gpio->connect_mode[gpio_id] = LJCA_GPIO_CONF_INTERRUPT; switch (type) { case IRQ_TYPE_LEVEL_HIGH: - ljca_gpio->connect_mode[gpio_id] |= (LJCA_GPIO_CONF_LEVEL | LJCA_GPIO_CONF_PULLUP); + ljca_gpio->connect_mode[gpio_id] |= + (LJCA_GPIO_CONF_LEVEL | LJCA_GPIO_CONF_PULLUP); break; case IRQ_TYPE_LEVEL_LOW: - ljca_gpio->connect_mode[gpio_id] |= (LJCA_GPIO_CONF_LEVEL | LJCA_GPIO_CONF_PULLDOWN); + ljca_gpio->connect_mode[gpio_id] |= + (LJCA_GPIO_CONF_LEVEL | LJCA_GPIO_CONF_PULLDOWN); break; case IRQ_TYPE_EDGE_BOTH: break; case IRQ_TYPE_EDGE_RISING: - ljca_gpio->connect_mode[gpio_id] |= (LJCA_GPIO_CONF_EDGE | LJCA_GPIO_CONF_PULLUP); + ljca_gpio->connect_mode[gpio_id] |= + (LJCA_GPIO_CONF_EDGE | LJCA_GPIO_CONF_PULLUP); break; case IRQ_TYPE_EDGE_FALLING: - ljca_gpio->connect_mode[gpio_id] |= (LJCA_GPIO_CONF_EDGE | LJCA_GPIO_CONF_PULLDOWN); + ljca_gpio->connect_mode[gpio_id] |= + (LJCA_GPIO_CONF_EDGE | LJCA_GPIO_CONF_PULLDOWN); break; default: return -EINVAL; @@ -332,15 +361,14 @@ static void ljca_irq_bus_unlock(struct irq_data *irqd) struct gpio_chip *gc = irq_data_get_irq_chip_data(irqd); struct ljca_gpio_dev *ljca_gpio = gpiochip_get_data(gc); int gpio_id = irqd_to_hwirq(irqd); - int enabled; - int unmasked; + int enabled, unmasked; enabled = test_bit(gpio_id, ljca_gpio->enabled_irqs); unmasked = test_bit(gpio_id, ljca_gpio->unmasked_irqs); if (enabled != unmasked) { if (unmasked) { - gpio_config(ljca_gpio, gpio_id, 0); + ljca_gpio_config(ljca_gpio, gpio_id, 0); ljca_enable_irq(ljca_gpio, gpio_id, true); set_bit(gpio_id, ljca_gpio->enabled_irqs); } else { @@ -363,43 +391,48 @@ static const struct irq_chip ljca_gpio_irqchip = { GPIOCHIP_IRQ_RESOURCE_HELPERS, }; -static int ljca_gpio_probe(struct platform_device *pdev) +static int ljca_gpio_probe(struct auxiliary_device *auxdev, + const struct auxiliary_device_id *aux_dev_id) { + struct ljca_client *ljca = auxiliary_dev_to_ljca_client(auxdev); struct ljca_gpio_dev *ljca_gpio; struct gpio_irq_chip *girq; int ret; - ljca_gpio = devm_kzalloc(&pdev->dev, sizeof(*ljca_gpio), GFP_KERNEL); + ljca_gpio = devm_kzalloc(&auxdev->dev, sizeof(*ljca_gpio), GFP_KERNEL); if (!ljca_gpio) return -ENOMEM; - ljca_gpio->gpio_info = dev_get_platdata(&pdev->dev); - ljca_gpio->connect_mode = devm_kcalloc(&pdev->dev, ljca_gpio->gpio_info->num, - sizeof(*ljca_gpio->connect_mode), GFP_KERNEL); + ljca_gpio->ljca = ljca; + ljca_gpio->gpio_info = dev_get_platdata(&auxdev->dev); + ljca_gpio->connect_mode = devm_kcalloc(&auxdev->dev, + ljca_gpio->gpio_info->num, + sizeof(*ljca_gpio->connect_mode), + GFP_KERNEL); if (!ljca_gpio->connect_mode) return -ENOMEM; mutex_init(&ljca_gpio->irq_lock); mutex_init(&ljca_gpio->trans_lock); - ljca_gpio->pdev = pdev; ljca_gpio->gc.direction_input = ljca_gpio_direction_input; ljca_gpio->gc.direction_output = ljca_gpio_direction_output; + ljca_gpio->gc.get_direction = ljca_gpio_get_direction; ljca_gpio->gc.get = ljca_gpio_get_value; ljca_gpio->gc.set = ljca_gpio_set_value; ljca_gpio->gc.set_config = ljca_gpio_set_config; ljca_gpio->gc.init_valid_mask = ljca_gpio_init_valid_mask; ljca_gpio->gc.can_sleep = true; - ljca_gpio->gc.parent = &pdev->dev; + ljca_gpio->gc.parent = &auxdev->dev; ljca_gpio->gc.base = -1; ljca_gpio->gc.ngpio = ljca_gpio->gpio_info->num; - ljca_gpio->gc.label = ACPI_COMPANION(&pdev->dev) ? - acpi_dev_name(ACPI_COMPANION(&pdev->dev)) : - dev_name(&pdev->dev); + ljca_gpio->gc.label = ACPI_COMPANION(&auxdev->dev) ? + acpi_dev_name(ACPI_COMPANION(&auxdev->dev)) : + dev_name(&auxdev->dev); ljca_gpio->gc.owner = THIS_MODULE; - platform_set_drvdata(pdev, ljca_gpio); - ljca_register_event_cb(ljca_gpio->gpio_info->ljca, ljca_gpio_event_cb, ljca_gpio); + auxiliary_set_drvdata(auxdev, ljca_gpio); + ljca_register_event_cb(ljca, ljca_gpio_event_cb, ljca_gpio); girq = &ljca_gpio->gc.irq; gpio_irq_chip_set_chip(girq, &ljca_gpio_irqchip); @@ -413,7 +446,7 @@ static int ljca_gpio_probe(struct platform_device *pdev) INIT_WORK(&ljca_gpio->work, ljca_gpio_async); ret = gpiochip_add_data(&ljca_gpio->gc, ljca_gpio); if (ret) { - ljca_unregister_event_cb(ljca_gpio->gpio_info->ljca); + ljca_unregister_event_cb(ljca); mutex_destroy(&ljca_gpio->irq_lock); mutex_destroy(&ljca_gpio->trans_lock); } @@ -421,34 +454,33 @@ static int ljca_gpio_probe(struct platform_device *pdev) return ret; } -static int ljca_gpio_remove(struct platform_device *pdev) +static void ljca_gpio_remove(struct auxiliary_device *auxdev) { - struct ljca_gpio_dev *ljca_gpio = platform_get_drvdata(pdev); + struct ljca_gpio_dev *ljca_gpio = auxiliary_get_drvdata(auxdev); gpiochip_remove(&ljca_gpio->gc); - ljca_unregister_event_cb(ljca_gpio->gpio_info->ljca); + ljca_unregister_event_cb(ljca_gpio->ljca); + cancel_work_sync(&ljca_gpio->work); mutex_destroy(&ljca_gpio->irq_lock); mutex_destroy(&ljca_gpio->trans_lock); - return 0; } -#define LJCA_GPIO_DRV_NAME "ljca-gpio" -static const struct platform_device_id ljca_gpio_id[] = { - { LJCA_GPIO_DRV_NAME, 0 }, - { /* sentinel */ } +static const struct auxiliary_device_id ljca_gpio_id_table[] = { + { "usb_ljca.ljca-gpio", 0 }, + { /* sentinel */ }, }; -MODULE_DEVICE_TABLE(platform, ljca_gpio_id); +MODULE_DEVICE_TABLE(auxiliary, ljca_gpio_id_table); -static struct platform_driver ljca_gpio_driver = { - .driver.name = LJCA_GPIO_DRV_NAME, +static struct auxiliary_driver ljca_gpio_driver = { .probe = ljca_gpio_probe, .remove = ljca_gpio_remove, + .id_table = ljca_gpio_id_table, }; -module_platform_driver(ljca_gpio_driver); +module_auxiliary_driver(ljca_gpio_driver); -MODULE_AUTHOR("Ye Xiang "); -MODULE_AUTHOR("Wang Zhifeng "); -MODULE_AUTHOR("Zhang Lixu "); +MODULE_AUTHOR("Wentong Wu "); +MODULE_AUTHOR("Zhifeng Wang "); +MODULE_AUTHOR("Lixu Zhang "); MODULE_DESCRIPTION("Intel La Jolla Cove Adapter USB-GPIO driver"); MODULE_LICENSE("GPL"); MODULE_IMPORT_NS(LJCA); diff --git a/drivers/gpio/gpio-mlxbf2.c b/drivers/gpio/gpio-mlxbf2.c index 6abe01bc39c3e..c03945af8538e 100644 --- a/drivers/gpio/gpio-mlxbf2.c +++ b/drivers/gpio/gpio-mlxbf2.c @@ -397,7 +397,7 @@ mlxbf2_gpio_probe(struct platform_device *pdev) gc->ngpio = npins; gc->owner = THIS_MODULE; - irq = platform_get_irq(pdev, 0); + irq = platform_get_irq_optional(pdev, 0); if (irq >= 0) { girq = &gs->gc.irq; gpio_irq_chip_set_chip(girq, &mlxbf2_gpio_irq_chip); diff --git a/drivers/gpio/gpio-mlxbf3.c b/drivers/gpio/gpio-mlxbf3.c index 10ea71273c891..ed29b07d16c19 100644 --- a/drivers/gpio/gpio-mlxbf3.c +++ b/drivers/gpio/gpio-mlxbf3.c @@ -227,7 +227,7 @@ static int mlxbf3_gpio_probe(struct platform_device *pdev) gc->owner = THIS_MODULE; gc->add_pin_ranges = mlxbf3_gpio_add_pin_ranges; - irq = platform_get_irq(pdev, 0); + irq = platform_get_irq_optional(pdev, 0); if (irq >= 0) { girq = &gs->gc.irq; gpio_irq_chip_set_chip(girq, &gpio_mlxbf3_irqchip); diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index b882b26ab5007..faadbe66b23e7 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -519,12 +520,10 @@ static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off) struct pca953x_chip *chip = gpiochip_get_data(gc); u8 dirreg = chip->recalc_addr(chip, chip->regs->direction, off); u8 bit = BIT(off % BANK_SZ); - int ret; - mutex_lock(&chip->i2c_lock); - ret = regmap_write_bits(chip->regmap, dirreg, bit, bit); - mutex_unlock(&chip->i2c_lock); - return ret; + guard(mutex)(&chip->i2c_lock); + + return regmap_write_bits(chip->regmap, dirreg, bit, bit); } static int pca953x_gpio_direction_output(struct gpio_chip *gc, @@ -536,17 +535,15 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc, u8 bit = BIT(off % BANK_SZ); int ret; - mutex_lock(&chip->i2c_lock); + guard(mutex)(&chip->i2c_lock); + /* set output level */ ret = regmap_write_bits(chip->regmap, outreg, bit, val ? bit : 0); if (ret) - goto exit; + return ret; /* then direction */ - ret = regmap_write_bits(chip->regmap, dirreg, bit, 0); -exit: - mutex_unlock(&chip->i2c_lock); - return ret; + return regmap_write_bits(chip->regmap, dirreg, bit, 0); } static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off) @@ -557,9 +554,8 @@ static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off) u32 reg_val; int ret; - mutex_lock(&chip->i2c_lock); - ret = regmap_read(chip->regmap, inreg, ®_val); - mutex_unlock(&chip->i2c_lock); + scoped_guard(mutex, &chip->i2c_lock) + ret = regmap_read(chip->regmap, inreg, ®_val); if (ret < 0) return ret; @@ -572,9 +568,9 @@ static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val) u8 outreg = chip->recalc_addr(chip, chip->regs->output, off); u8 bit = BIT(off % BANK_SZ); - mutex_lock(&chip->i2c_lock); + guard(mutex)(&chip->i2c_lock); + regmap_write_bits(chip->regmap, outreg, bit, val ? bit : 0); - mutex_unlock(&chip->i2c_lock); } static int pca953x_gpio_get_direction(struct gpio_chip *gc, unsigned off) @@ -585,9 +581,8 @@ static int pca953x_gpio_get_direction(struct gpio_chip *gc, unsigned off) u32 reg_val; int ret; - mutex_lock(&chip->i2c_lock); - ret = regmap_read(chip->regmap, dirreg, ®_val); - mutex_unlock(&chip->i2c_lock); + scoped_guard(mutex, &chip->i2c_lock) + ret = regmap_read(chip->regmap, dirreg, ®_val); if (ret < 0) return ret; @@ -604,9 +599,8 @@ static int pca953x_gpio_get_multiple(struct gpio_chip *gc, DECLARE_BITMAP(reg_val, MAX_LINE); int ret; - mutex_lock(&chip->i2c_lock); - ret = pca953x_read_regs(chip, chip->regs->input, reg_val); - mutex_unlock(&chip->i2c_lock); + scoped_guard(mutex, &chip->i2c_lock) + ret = pca953x_read_regs(chip, chip->regs->input, reg_val); if (ret) return ret; @@ -621,16 +615,15 @@ static void pca953x_gpio_set_multiple(struct gpio_chip *gc, DECLARE_BITMAP(reg_val, MAX_LINE); int ret; - mutex_lock(&chip->i2c_lock); + guard(mutex)(&chip->i2c_lock); + ret = pca953x_read_regs(chip, chip->regs->output, reg_val); if (ret) - goto exit; + return; bitmap_replace(reg_val, reg_val, bits, mask, gc->ngpio); pca953x_write_regs(chip, chip->regs->output, reg_val); -exit: - mutex_unlock(&chip->i2c_lock); } static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip, @@ -638,7 +631,6 @@ static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip, unsigned long config) { enum pin_config_param param = pinconf_to_config_param(config); - u8 pull_en_reg = chip->recalc_addr(chip, PCAL953X_PULL_EN, offset); u8 pull_sel_reg = chip->recalc_addr(chip, PCAL953X_PULL_SEL, offset); u8 bit = BIT(offset % BANK_SZ); @@ -651,7 +643,7 @@ static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip, if (!(chip->driver_data & PCA_PCAL)) return -ENOTSUPP; - mutex_lock(&chip->i2c_lock); + guard(mutex)(&chip->i2c_lock); /* Configure pull-up/pull-down */ if (param == PIN_CONFIG_BIAS_PULL_UP) @@ -661,17 +653,13 @@ static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip, else ret = 0; if (ret) - goto exit; + return ret; /* Disable/Enable pull-up/pull-down */ if (param == PIN_CONFIG_BIAS_DISABLE) - ret = regmap_write_bits(chip->regmap, pull_en_reg, bit, 0); + return regmap_write_bits(chip->regmap, pull_en_reg, bit, 0); else - ret = regmap_write_bits(chip->regmap, pull_en_reg, bit, bit); - -exit: - mutex_unlock(&chip->i2c_lock); - return ret; + return regmap_write_bits(chip->regmap, pull_en_reg, bit, bit); } static int pca953x_gpio_set_config(struct gpio_chip *gc, unsigned int offset, @@ -883,10 +871,8 @@ static irqreturn_t pca953x_irq_handler(int irq, void *devid) bitmap_zero(pending, MAX_LINE); - mutex_lock(&chip->i2c_lock); - ret = pca953x_irq_pending(chip, pending); - mutex_unlock(&chip->i2c_lock); - + scoped_guard(mutex, &chip->i2c_lock) + ret = pca953x_irq_pending(chip, pending); if (ret) { ret = 0; @@ -1168,9 +1154,9 @@ static int pca953x_probe(struct i2c_client *client) } #ifdef CONFIG_PM_SLEEP -static int pca953x_regcache_sync(struct device *dev) +static int pca953x_regcache_sync(struct pca953x_chip *chip) { - struct pca953x_chip *chip = dev_get_drvdata(dev); + struct device *dev = &chip->client->dev; int ret; u8 regaddr; @@ -1217,13 +1203,38 @@ static int pca953x_regcache_sync(struct device *dev) return 0; } +static int pca953x_restore_context(struct pca953x_chip *chip) +{ + int ret; + + guard(mutex)(&chip->i2c_lock); + + if (chip->client->irq > 0) + enable_irq(chip->client->irq); + regcache_cache_only(chip->regmap, false); + regcache_mark_dirty(chip->regmap); + ret = pca953x_regcache_sync(chip); + if (ret) + return ret; + + return regcache_sync(chip->regmap); +} + +static void pca953x_save_context(struct pca953x_chip *chip) +{ + guard(mutex)(&chip->i2c_lock); + + /* Disable IRQ to prevent early triggering while regmap "cache only" is on */ + if (chip->client->irq > 0) + disable_irq(chip->client->irq); + regcache_cache_only(chip->regmap, true); +} + static int pca953x_suspend(struct device *dev) { struct pca953x_chip *chip = dev_get_drvdata(dev); - mutex_lock(&chip->i2c_lock); - regcache_cache_only(chip->regmap, true); - mutex_unlock(&chip->i2c_lock); + pca953x_save_context(chip); if (atomic_read(&chip->wakeup_path)) device_set_wakeup_path(dev); @@ -1246,17 +1257,7 @@ static int pca953x_resume(struct device *dev) } } - mutex_lock(&chip->i2c_lock); - regcache_cache_only(chip->regmap, false); - regcache_mark_dirty(chip->regmap); - ret = pca953x_regcache_sync(dev); - if (ret) { - mutex_unlock(&chip->i2c_lock); - return ret; - } - - ret = regcache_sync(chip->regmap); - mutex_unlock(&chip->i2c_lock); + ret = pca953x_restore_context(chip); if (ret) { dev_err(dev, "Failed to restore register map: %d\n", ret); return ret; diff --git a/drivers/gpio/gpio-pci-idio-16.c b/drivers/gpio/gpio-pci-idio-16.c index 44c0a21b1d1d9..5827b18d190bd 100644 --- a/drivers/gpio/gpio-pci-idio-16.c +++ b/drivers/gpio/gpio-pci-idio-16.c @@ -41,6 +41,7 @@ static const struct regmap_config idio_16_regmap_config = { .reg_stride = 1, .val_bits = 8, .io_port = true, + .max_register = 0x7, .wr_table = &idio_16_wr_table, .rd_table = &idio_16_rd_table, .volatile_table = &idio_16_rd_table, diff --git a/drivers/gpio/gpio-regmap.c b/drivers/gpio/gpio-regmap.c index c08c8e528867e..fd986afa7db5f 100644 --- a/drivers/gpio/gpio-regmap.c +++ b/drivers/gpio/gpio-regmap.c @@ -29,6 +29,12 @@ struct gpio_regmap { unsigned int reg_clr_base; unsigned int reg_dir_in_base; unsigned int reg_dir_out_base; + unsigned long *fixed_direction_output; + +#ifdef CONFIG_REGMAP_IRQ + int regmap_irq_line; + struct regmap_irq_chip_data *irq_chip_data; +#endif int (*reg_mask_xlate)(struct gpio_regmap *gpio, unsigned int base, unsigned int offset, unsigned int *reg, @@ -117,6 +123,13 @@ static int gpio_regmap_get_direction(struct gpio_chip *chip, unsigned int base, val, reg, mask; int invert, ret; + if (gpio->fixed_direction_output) { + if (test_bit(offset, gpio->fixed_direction_output)) + return GPIO_LINE_DIRECTION_OUT; + else + return GPIO_LINE_DIRECTION_IN; + } + if (gpio->reg_dat_base && !gpio->reg_set_base) return GPIO_LINE_DIRECTION_IN; if (gpio->reg_set_base && !gpio->reg_dat_base) @@ -203,6 +216,7 @@ EXPORT_SYMBOL_GPL(gpio_regmap_get_drvdata); */ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config) { + struct irq_domain *irq_domain; struct gpio_regmap *gpio; struct gpio_chip *chip; int ret; @@ -274,12 +288,37 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config chip->direction_output = gpio_regmap_direction_output; } + if (config->fixed_direction_output) { + gpio->fixed_direction_output = bitmap_alloc(chip->ngpio, + GFP_KERNEL); + if (!gpio->fixed_direction_output) { + ret = -ENOMEM; + goto err_free_gpio; + } + bitmap_copy(gpio->fixed_direction_output, + config->fixed_direction_output, chip->ngpio); + } + ret = gpiochip_add_data(chip, gpio); if (ret < 0) - goto err_free_gpio; + goto err_free_bitmap; + +#ifdef CONFIG_REGMAP_IRQ + if (config->regmap_irq_chip) { + gpio->regmap_irq_line = config->regmap_irq_line; + ret = regmap_add_irq_chip_fwnode(dev_fwnode(config->parent), config->regmap, + config->regmap_irq_line, config->regmap_irq_flags, + 0, config->regmap_irq_chip, &gpio->irq_chip_data); + if (ret) + goto err_free_bitmap; - if (config->irq_domain) { - ret = gpiochip_irqchip_add_domain(chip, config->irq_domain); + irq_domain = regmap_irq_get_domain(gpio->irq_chip_data); + } else +#endif + irq_domain = config->irq_domain; + + if (irq_domain) { + ret = gpiochip_irqchip_add_domain(chip, irq_domain); if (ret) goto err_remove_gpiochip; } @@ -288,6 +327,8 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config err_remove_gpiochip: gpiochip_remove(chip); +err_free_bitmap: + bitmap_free(gpio->fixed_direction_output); err_free_gpio: kfree(gpio); return ERR_PTR(ret); @@ -300,7 +341,13 @@ EXPORT_SYMBOL_GPL(gpio_regmap_register); */ void gpio_regmap_unregister(struct gpio_regmap *gpio) { +#ifdef CONFIG_REGMAP_IRQ + if (gpio->irq_chip_data) + regmap_del_irq_chip(gpio->regmap_irq_line, gpio->irq_chip_data); +#endif + gpiochip_remove(&gpio->gpio_chip); + bitmap_free(gpio->fixed_direction_output); kfree(gpio); } EXPORT_SYMBOL_GPL(gpio_regmap_unregister); diff --git a/drivers/gpio/gpio-tps65912.c b/drivers/gpio/gpio-tps65912.c index fab771cb6a87b..bac757c191c2e 100644 --- a/drivers/gpio/gpio-tps65912.c +++ b/drivers/gpio/gpio-tps65912.c @@ -49,10 +49,13 @@ static int tps65912_gpio_direction_output(struct gpio_chip *gc, unsigned offset, int value) { struct tps65912_gpio *gpio = gpiochip_get_data(gc); + int ret; /* Set the initial value */ - regmap_update_bits(gpio->tps->regmap, TPS65912_GPIO1 + offset, - GPIO_SET_MASK, value ? GPIO_SET_MASK : 0); + ret = regmap_update_bits(gpio->tps->regmap, TPS65912_GPIO1 + offset, + GPIO_SET_MASK, value ? GPIO_SET_MASK : 0); + if (ret) + return ret; return regmap_update_bits(gpio->tps->regmap, TPS65912_GPIO1 + offset, GPIO_CFG_MASK, GPIO_CFG_MASK); diff --git a/drivers/gpio/gpio-virtio.c b/drivers/gpio/gpio-virtio.c index fcc5e8c08973b..1d0eb49eae3b4 100644 --- a/drivers/gpio/gpio-virtio.c +++ b/drivers/gpio/gpio-virtio.c @@ -539,7 +539,6 @@ static const char **virtio_gpio_get_names(struct virtio_gpio *vgpio, static int virtio_gpio_probe(struct virtio_device *vdev) { - struct virtio_gpio_config config; struct device *dev = &vdev->dev; struct virtio_gpio *vgpio; u32 gpio_names_size; @@ -551,9 +550,11 @@ static int virtio_gpio_probe(struct virtio_device *vdev) return -ENOMEM; /* Read configuration */ - virtio_cread_bytes(vdev, 0, &config, sizeof(config)); - gpio_names_size = le32_to_cpu(config.gpio_names_size); - ngpio = le16_to_cpu(config.ngpio); + gpio_names_size = + virtio_cread32(vdev, offsetof(struct virtio_gpio_config, + gpio_names_size)); + ngpio = virtio_cread16(vdev, offsetof(struct virtio_gpio_config, + ngpio)); if (!ngpio) { dev_err(dev, "Number of GPIOs can't be zero\n"); return -EINVAL; diff --git a/drivers/gpio/gpio-wcd934x.c b/drivers/gpio/gpio-wcd934x.c index 2bba27b13947f..03b16b8f639ad 100644 --- a/drivers/gpio/gpio-wcd934x.c +++ b/drivers/gpio/gpio-wcd934x.c @@ -46,9 +46,12 @@ static int wcd_gpio_direction_output(struct gpio_chip *chip, unsigned int pin, int val) { struct wcd_gpio_data *data = gpiochip_get_data(chip); + int ret; - regmap_update_bits(data->map, WCD_REG_DIR_CTL_OFFSET, - WCD_PIN_MASK(pin), WCD_PIN_MASK(pin)); + ret = regmap_update_bits(data->map, WCD_REG_DIR_CTL_OFFSET, + WCD_PIN_MASK(pin), WCD_PIN_MASK(pin)); + if (ret) + return ret; return regmap_update_bits(data->map, WCD_REG_VAL_CTL_OFFSET, WCD_PIN_MASK(pin), @@ -99,7 +102,7 @@ static int wcd_gpio_probe(struct platform_device *pdev) chip->base = -1; chip->ngpio = WCD934X_NPINS; chip->label = dev_name(dev); - chip->can_sleep = false; + chip->can_sleep = true; return devm_gpiochip_add_data(dev, chip, data); } diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 4a7218b83ccad..e3f81a440321c 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -203,6 +203,15 @@ static void of_gpio_try_fixup_polarity(const struct device_node *np, */ { "lantiq,pci-xway", "gpio-reset", false }, #endif +#if IS_ENABLED(CONFIG_REGULATOR_S5M8767) + /* + * According to S5M8767, the DVS and DS pin are + * active-high signals. However, exynos5250-spring.dts use + * active-low setting. + */ + { "samsung,s5m8767-pmic", "s5m8767,pmic-buck-dvs-gpios", true }, + { "samsung,s5m8767-pmic", "s5m8767,pmic-buck-ds-gpios", true }, +#endif #if IS_ENABLED(CONFIG_TOUCHSCREEN_TSC2005) /* * DTS for Nokia N900 incorrectly specified "active high" diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index efb592b6f6aa7..9b8a200423858 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -4022,6 +4022,23 @@ static struct gpio_desc *gpiod_find_by_fwnode(struct fwnode_handle *fwnode, return desc; } +static struct gpio_desc *gpiod_fwnode_lookup(struct fwnode_handle *fwnode, + struct device *consumer, + const char *con_id, + unsigned int idx, + enum gpiod_flags *flags, + unsigned long *lookupflags) +{ + struct gpio_desc *desc; + + desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx, flags, lookupflags); + if (gpiod_not_found(desc) && !IS_ERR_OR_NULL(fwnode)) + desc = gpiod_find_by_fwnode(fwnode->secondary, consumer, con_id, + idx, flags, lookupflags); + + return desc; +} + struct gpio_desc *gpiod_find_and_request(struct device *consumer, struct fwnode_handle *fwnode, const char *con_id, @@ -4034,7 +4051,7 @@ struct gpio_desc *gpiod_find_and_request(struct device *consumer, struct gpio_desc *desc; int ret; - desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx, &flags, &lookupflags); + desc = gpiod_fwnode_lookup(fwnode, consumer, con_id, idx, &flags, &lookupflags); if (gpiod_not_found(desc) && platform_lookup_allowed) { /* * Either we are not using DT or ACPI, or their lookup did not diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index d59e8536192ca..78426f8c5420d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -788,6 +788,7 @@ struct amdgpu_device { bool need_swiotlb; bool accel_working; struct notifier_block acpi_nb; + struct notifier_block pm_nb; struct amdgpu_i2c_chan *i2c_bus[AMDGPU_MAX_I2C_BUS]; struct debugfs_blob_wrapper debugfs_vbios_blob; struct debugfs_blob_wrapper debugfs_discovery_blob; @@ -1284,7 +1285,9 @@ int emu_soc_asic_init(struct amdgpu_device *adev); for (i = ffs(inst_mask); i-- != 0; \ i = ffs(inst_mask & BIT_MASK_UPPER(i + 1))) +#ifndef MIN #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#endif /* Common functions */ bool amdgpu_device_has_job_running(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index a1f35510d5395..c94e1cc7e3a73 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -2285,10 +2285,9 @@ void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem) int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev, struct kfd_vm_fault_info *mem) { - if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) { + if (atomic_read_acquire(&adev->gmc.vm_fault_info_updated) == 1) { *mem = *adev->gmc.vm_fault_info; - mb(); /* make sure read happened */ - atomic_set(&adev->gmc.vm_fault_info_updated, 0); + atomic_set_release(&adev->gmc.vm_fault_info_updated, 0); } return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c index d34037b85cf85..52c093e42531b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c @@ -1205,29 +1205,69 @@ static void amdgpu_connector_dvi_force(struct drm_connector *connector) amdgpu_connector->use_digital = true; } +/** + * amdgpu_max_hdmi_pixel_clock - Return max supported HDMI (TMDS) pixel clock + * @adev: pointer to amdgpu_device + * + * Return: maximum supported HDMI (TMDS) pixel clock in KHz. + */ +static int amdgpu_max_hdmi_pixel_clock(const struct amdgpu_device *adev) +{ + if (adev->asic_type >= CHIP_POLARIS10) + return 600000; + else if (adev->asic_type >= CHIP_TONGA) + return 300000; + else + return 297000; +} + +/** + * amdgpu_connector_dvi_mode_valid - Validate a mode on DVI/HDMI connectors + * @connector: DRM connector to validate the mode on + * @mode: display mode to validate + * + * Validate the given display mode on DVI and HDMI connectors, including + * analog signals on DVI-I. + * + * Return: drm_mode_status indicating whether the mode is valid. + */ static enum drm_mode_status amdgpu_connector_dvi_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { struct drm_device *dev = connector->dev; struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + const int max_hdmi_pixel_clock = amdgpu_max_hdmi_pixel_clock(adev); + const int max_dvi_single_link_pixel_clock = 165000; + int max_digital_pixel_clock_khz; /* XXX check mode bandwidth */ - if (amdgpu_connector->use_digital && (mode->clock > 165000)) { - if ((amdgpu_connector->connector_object_id == CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I) || - (amdgpu_connector->connector_object_id == CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D) || - (amdgpu_connector->connector_object_id == CONNECTOR_OBJECT_ID_HDMI_TYPE_B)) { - return MODE_OK; - } else if (connector->display_info.is_hdmi) { - /* HDMI 1.3+ supports max clock of 340 Mhz */ - if (mode->clock > 340000) - return MODE_CLOCK_HIGH; - else - return MODE_OK; - } else { - return MODE_CLOCK_HIGH; + if (amdgpu_connector->use_digital) { + switch (amdgpu_connector->connector_object_id) { + case CONNECTOR_OBJECT_ID_HDMI_TYPE_A: + max_digital_pixel_clock_khz = max_hdmi_pixel_clock; + break; + case CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_I: + case CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D: + max_digital_pixel_clock_khz = max_dvi_single_link_pixel_clock; + break; + case CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I: + case CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D: + case CONNECTOR_OBJECT_ID_HDMI_TYPE_B: + max_digital_pixel_clock_khz = max_dvi_single_link_pixel_clock * 2; + break; } + + /* When the display EDID claims that it's an HDMI display, + * we use the HDMI encoder mode of the display HW, + * so we should verify against the max HDMI clock here. + */ + if (connector->display_info.is_hdmi) + max_digital_pixel_clock_khz = max_hdmi_pixel_clock; + + if (mode->clock > max_digital_pixel_clock_khz) + return MODE_CLOCK_HIGH; } /* check against the max pixel clock */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 13c97ba7a820b..05712d322024a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -286,7 +286,7 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, } } - if (!p->gang_size) { + if (!p->gang_size || (amdgpu_sriov_vf(p->adev) && p->gang_size > 1)) { ret = -EINVAL; goto free_all_kdata; } @@ -690,7 +690,7 @@ static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, */ const s64 us_upper_bound = 200000; - if (!adev->mm_stats.log2_max_MBps) { + if ((!adev->mm_stats.log2_max_MBps) || !ttm_resource_manager_used(&adev->mman.vram_mgr.manager)) { *max_bytes = 0; *max_vis_bytes = 0; return; @@ -1732,30 +1732,21 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, { struct amdgpu_device *adev = drm_to_adev(dev); union drm_amdgpu_wait_fences *wait = data; - uint32_t fence_count = wait->in.fence_count; - struct drm_amdgpu_fence *fences_user; struct drm_amdgpu_fence *fences; int r; /* Get the fences from userspace */ - fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence), - GFP_KERNEL); - if (fences == NULL) - return -ENOMEM; - - fences_user = u64_to_user_ptr(wait->in.fences); - if (copy_from_user(fences, fences_user, - sizeof(struct drm_amdgpu_fence) * fence_count)) { - r = -EFAULT; - goto err_free_fences; - } + fences = memdup_array_user(u64_to_user_ptr(wait->in.fences), + wait->in.fence_count, + sizeof(struct drm_amdgpu_fence)); + if (IS_ERR(fences)) + return PTR_ERR(fences); if (wait->in.wait_all) r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); else r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); -err_free_fences: kfree(fences); return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 963e106d32eed..256cc15fc9b5a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -1890,7 +1890,7 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring) continue; } job = to_amdgpu_job(s_job); - if (preempted && (&job->hw_fence) == fence) + if (preempted && (&job->hw_fence.base) == fence) /* mark the job as preempted */ job->preemption_status |= AMDGPU_IB_PREEMPTED; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 10f5a3d0f5916..b2a1dc193cb8f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -93,6 +93,7 @@ MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); +MODULE_FIRMWARE("amdgpu/cyan_skillfish_gpu_info.bin"); #define AMDGPU_RESUME_MS 2000 #define AMDGPU_MAX_RETRY_LIMIT 2 @@ -141,6 +142,10 @@ const char *amdgpu_asic_name[] = { "LAST", }; +static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev); +static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, + void *data); + /** * DOC: pcie_replay_count * @@ -1935,6 +1940,9 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) case CHIP_NAVI12: chip_name = "navi12"; break; + case CHIP_CYAN_SKILLFISH: + chip_name = "cyan_skillfish"; + break; } snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name); @@ -3920,6 +3928,11 @@ int amdgpu_device_init(struct amdgpu_device *adev, amdgpu_device_check_iommu_direct_map(adev); + adev->pm_nb.notifier_call = amdgpu_device_pm_notifier; + r = register_pm_notifier(&adev->pm_nb); + if (r) + goto failed; + return 0; release_ras_con: @@ -3981,6 +3994,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) flush_delayed_work(&adev->delayed_init_work); adev->shutdown = true; + unregister_pm_notifier(&adev->pm_nb); + /* make sure IB test finished before entering exclusive mode * to avoid preemption on IB test */ @@ -4098,6 +4113,10 @@ static int amdgpu_device_evict_resources(struct amdgpu_device *adev) if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU)) return 0; + /* No need to evict when going to S5 through S4 callbacks */ + if (system_state == SYSTEM_POWER_OFF) + return 0; + ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); if (ret) DRM_WARN("evicting device resources failed\n"); @@ -4107,6 +4126,33 @@ static int amdgpu_device_evict_resources(struct amdgpu_device *adev) /* * Suspend & resume. */ +/** + * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events + * @nb: notifier block + * @mode: suspend mode + * @data: data + * + * This function is called when the system is about to suspend or hibernate. + * It is used to set the appropriate flags so that eviction can be optimized + * in the pm prepare callback. + */ +static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, + void *data) +{ + struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb); + + switch (mode) { + case PM_HIBERNATION_PREPARE: + adev->in_s4 = true; + break; + case PM_POST_HIBERNATION: + adev->in_s4 = false; + break; + } + + return NOTIFY_DONE; +} + /** * amdgpu_device_prepare - prepare for device suspend * @@ -4551,6 +4597,8 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, retry: amdgpu_amdkfd_pre_reset(adev); + amdgpu_device_stop_pending_resets(adev); + if (from_hypervisor) r = amdgpu_virt_request_full_gpu(adev, true); else @@ -5327,7 +5375,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, * * job->base holds a reference to parent fence */ - if (job && dma_fence_is_signaled(&job->hw_fence)) { + if (job && dma_fence_is_signaled(&job->hw_fence.base)) { job_signaled = true; dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); goto skip_hw_reset; @@ -5347,11 +5395,12 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, tmp_adev->asic_reset_res = r; } - /* - * Drop all pending non scheduler resets. Scheduler resets - * were already dropped during drm_sched_stop - */ - amdgpu_device_stop_pending_resets(tmp_adev); + if (!amdgpu_sriov_vf(tmp_adev)) + /* + * Drop all pending non scheduler resets. Scheduler resets + * were already dropped during drm_sched_stop + */ + amdgpu_device_stop_pending_resets(tmp_adev); } /* Actual ASIC resets if needed.*/ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index b04d789bfd100..2e492f779b54c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -1819,13 +1819,16 @@ static int amdgpu_discovery_set_smu_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 0, 5): case IP_VERSION(11, 0, 9): case IP_VERSION(11, 0, 7): - case IP_VERSION(11, 0, 8): case IP_VERSION(11, 0, 11): case IP_VERSION(11, 0, 12): case IP_VERSION(11, 0, 13): case IP_VERSION(11, 5, 0): amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); break; + case IP_VERSION(11, 0, 8): + if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) + amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); + break; case IP_VERSION(12, 0, 0): case IP_VERSION(12, 0, 1): amdgpu_device_ip_block_add(adev, &smu_v12_0_ip_block); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index be4cc4868a748..493e18bcea069 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -43,6 +43,29 @@ #include #include +static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops; + +/** + * dma_buf_attach_adev - Helper to get adev of an attachment + * + * @attach: attachment + * + * Returns: + * A struct amdgpu_device * if the attaching device is an amdgpu device or + * partition, NULL otherwise. + */ +static struct amdgpu_device *dma_buf_attach_adev(struct dma_buf_attachment *attach) +{ + if (attach->importer_ops == &amdgpu_dma_buf_attach_ops) { + struct drm_gem_object *obj = attach->importer_priv; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + + return amdgpu_ttm_adev(bo->tbo.bdev); + } + + return NULL; +} + /** * amdgpu_dma_buf_attach - &dma_buf_ops.attach implementation * @@ -54,12 +77,14 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) { + struct amdgpu_device *attach_adev = dma_buf_attach_adev(attach); struct drm_gem_object *obj = dmabuf->priv; struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); int r; - if (pci_p2pdma_distance(adev->pdev, attach->dev, false) < 0) + if (!amdgpu_dmabuf_is_xgmi_accessible(attach_adev, bo) && + pci_p2pdma_distance(adev->pdev, attach->dev, false) < 0) attach->peer2peer = false; r = pm_runtime_get_sync(adev_to_drm(adev)->dev); @@ -482,6 +507,9 @@ bool amdgpu_dmabuf_is_xgmi_accessible(struct amdgpu_device *adev, struct drm_gem_object *obj = &bo->tbo.base; struct drm_gem_object *gobj; + if (!adev) + return false; + if (obj->import_attach) { struct dma_buf *dma_buf = obj->import_attach->dmabuf; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index bacf2e5de2abc..b15ce4df74798 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -1983,6 +1983,11 @@ static const struct pci_device_id pciidlist[] = { {0x1002, 0x7410, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN}, /* CYAN_SKILLFISH */ + {0x1002, 0x13DB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, + {0x1002, 0x13F9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, + {0x1002, 0x13FA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, + {0x1002, 0x13FB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, + {0x1002, 0x13FC, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, {0x1002, 0x13FE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, {0x1002, 0x143F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, @@ -2463,7 +2468,6 @@ static int amdgpu_pmops_freeze(struct device *dev) struct amdgpu_device *adev = drm_to_adev(drm_dev); int r; - adev->in_s4 = true; r = amdgpu_device_suspend(drm_dev, true); if (r) return r; @@ -2476,13 +2480,8 @@ static int amdgpu_pmops_freeze(struct device *dev) static int amdgpu_pmops_thaw(struct device *dev) { struct drm_device *drm_dev = dev_get_drvdata(dev); - struct amdgpu_device *adev = drm_to_adev(drm_dev); - int r; - - r = amdgpu_device_resume(drm_dev, true); - adev->in_s4 = false; - return r; + return amdgpu_device_resume(drm_dev, true); } static int amdgpu_pmops_poweroff(struct device *dev) @@ -2495,9 +2494,6 @@ static int amdgpu_pmops_poweroff(struct device *dev) static int amdgpu_pmops_restore(struct device *dev) { struct drm_device *drm_dev = dev_get_drvdata(dev); - struct amdgpu_device *adev = drm_to_adev(drm_dev); - - adev->in_s4 = false; return amdgpu_device_resume(drm_dev, true); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 7537f5aa76f0c..017dd494d0a2f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -41,22 +41,6 @@ #include "amdgpu_trace.h" #include "amdgpu_reset.h" -/* - * Fences mark an event in the GPUs pipeline and are used - * for GPU/CPU synchronization. When the fence is written, - * it is expected that all buffers associated with that fence - * are no longer in use by the associated ring on the GPU and - * that the relevant GPU caches have been flushed. - */ - -struct amdgpu_fence { - struct dma_fence base; - - /* RB, DMA, etc. */ - struct amdgpu_ring *ring; - ktime_t start_timestamp; -}; - static struct kmem_cache *amdgpu_fence_slab; int amdgpu_fence_slab_init(void) @@ -153,12 +137,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC); if (am_fence == NULL) return -ENOMEM; - fence = &am_fence->base; - am_fence->ring = ring; } else { /* take use of job-embedded fence */ - fence = &job->hw_fence; + am_fence = &job->hw_fence; } + fence = &am_fence->base; + am_fence->ring = ring; seq = ++ring->fence_drv.sync_seq; if (job && job->job_run_counter) { @@ -719,7 +703,7 @@ void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring) * it right here or we won't be able to track them in fence_drv * and they will remain unsignaled during sa_bo free. */ - job = container_of(old, struct amdgpu_job, hw_fence); + job = container_of(old, struct amdgpu_job, hw_fence.base); if (!job->base.s_fence && !dma_fence_is_signaled(old)) dma_fence_signal(old); RCU_INIT_POINTER(*ptr, NULL); @@ -781,7 +765,7 @@ static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f) static const char *amdgpu_job_fence_get_timeline_name(struct dma_fence *f) { - struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence); + struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence.base); return (const char *)to_amdgpu_ring(job->base.sched)->name; } @@ -811,7 +795,7 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f) */ static bool amdgpu_job_fence_enable_signaling(struct dma_fence *f) { - struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence); + struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence.base); if (!timer_pending(&to_amdgpu_ring(job->base.sched)->fence_drv.fallback_timer)) amdgpu_fence_schedule_fallback(to_amdgpu_ring(job->base.sched)); @@ -846,7 +830,7 @@ static void amdgpu_job_fence_free(struct rcu_head *rcu) struct dma_fence *f = container_of(rcu, struct dma_fence, rcu); /* free job if fence has a parent job */ - kfree(container_of(f, struct amdgpu_job, hw_fence)); + kfree(container_of(f, struct amdgpu_job, hw_fence.base)); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 49a6b6b88843d..e9adfc88a54ab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -165,8 +165,8 @@ void amdgpu_job_free_resources(struct amdgpu_job *job) /* Check if any fences where initialized */ if (job->base.s_fence && job->base.s_fence->finished.ops) f = &job->base.s_fence->finished; - else if (job->hw_fence.ops) - f = &job->hw_fence; + else if (job->hw_fence.base.ops) + f = &job->hw_fence.base; else f = NULL; @@ -183,10 +183,10 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job) amdgpu_sync_free(&job->explicit_sync); /* only put the hw fence if has embedded fence */ - if (!job->hw_fence.ops) + if (!job->hw_fence.base.ops) kfree(job); else - dma_fence_put(&job->hw_fence); + dma_fence_put(&job->hw_fence.base); } void amdgpu_job_set_gang_leader(struct amdgpu_job *job, @@ -215,10 +215,10 @@ void amdgpu_job_free(struct amdgpu_job *job) if (job->gang_submit != &job->base.s_fence->scheduled) dma_fence_put(job->gang_submit); - if (!job->hw_fence.ops) + if (!job->hw_fence.base.ops) kfree(job); else - dma_fence_put(&job->hw_fence); + dma_fence_put(&job->hw_fence.base); } struct dma_fence *amdgpu_job_submit(struct amdgpu_job *job) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index a963a25ddd620..65b6fbab544e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h @@ -48,7 +48,7 @@ struct amdgpu_job { struct drm_sched_job base; struct amdgpu_vm *vm; struct amdgpu_sync explicit_sync; - struct dma_fence hw_fence; + struct amdgpu_fence hw_fence; struct dma_fence *gang_submit; uint32_t preamble_status; uint32_t preemption_status; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c index 2ff2897fd1db6..cd8fa1164d540 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c @@ -87,10 +87,12 @@ static void amdgpu_jpeg_idle_work_handler(struct work_struct *work) fences += amdgpu_fence_count_emitted(&adev->jpeg.inst[i].ring_dec[j]); } - if (!fences && !atomic_read(&adev->jpeg.total_submission_cnt)) + if (!fences && !atomic_read(&adev->jpeg.total_submission_cnt)) { + mutex_lock(&adev->jpeg.jpeg_pg_lock); amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG, AMD_PG_STATE_GATE); - else + mutex_unlock(&adev->jpeg.jpeg_pg_lock); + } else schedule_delayed_work(&adev->jpeg.idle_work, JPEG_IDLE_TIMEOUT); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 5797055b1148f..1f0de6e717112 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -651,7 +651,8 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) ui64 = atomic64_read(&adev->num_vram_cpu_page_faults); return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; case AMDGPU_INFO_VRAM_USAGE: - ui64 = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager); + ui64 = ttm_resource_manager_used(&adev->mman.vram_mgr.manager) ? + ttm_resource_manager_usage(&adev->mman.vram_mgr.manager) : 0; return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; case AMDGPU_INFO_VIS_VRAM_USAGE: ui64 = amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); @@ -697,8 +698,8 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) mem.vram.usable_heap_size = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size) - AMDGPU_VM_RESERVED_VRAM; - mem.vram.heap_usage = - ttm_resource_manager_usage(vram_man); + mem.vram.heap_usage = ttm_resource_manager_used(&adev->mman.vram_mgr.manager) ? + ttm_resource_manager_usage(vram_man) : 0; mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4; mem.cpu_accessible_vram.total_heap_size = diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 6a24e8ceb9449..08886e0ee6428 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -43,7 +43,7 @@ #include "amdgpu_securedisplay.h" #include "amdgpu_atomfirmware.h" -#define AMD_VBIOS_FILE_MAX_SIZE_B (1024*1024*3) +#define AMD_VBIOS_FILE_MAX_SIZE_B (1024*1024*16) static int psp_load_smu_fw(struct psp_context *psp); static int psp_rap_terminate(struct psp_context *psp); @@ -291,21 +291,22 @@ static int psp_memory_training_init(struct psp_context *psp) struct psp_memory_training_context *ctx = &psp->mem_train_ctx; if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) { - DRM_DEBUG("memory training is not supported!\n"); + dev_dbg(psp->adev->dev, "memory training is not supported!\n"); return 0; } ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL); if (ctx->sys_cache == NULL) { - DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n"); + dev_err(psp->adev->dev, "alloc mem_train_ctx.sys_cache failed!\n"); ret = -ENOMEM; goto Err_out; } - DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n", - ctx->train_data_size, - ctx->p2c_train_data_offset, - ctx->c2p_train_data_offset); + dev_dbg(psp->adev->dev, + "train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n", + ctx->train_data_size, + ctx->p2c_train_data_offset, + ctx->c2p_train_data_offset); ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS; return 0; @@ -407,8 +408,8 @@ static int psp_sw_init(void *handle) psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); if (!psp->cmd) { - DRM_ERROR("Failed to allocate memory to command buffer!\n"); - ret = -ENOMEM; + dev_err(adev->dev, "Failed to allocate memory to command buffer!\n"); + return -ENOMEM; } adev->psp.xgmi_context.supports_extended_data = @@ -454,13 +455,13 @@ static int psp_sw_init(void *handle) if (mem_training_ctx->enable_mem_training) { ret = psp_memory_training_init(psp); if (ret) { - DRM_ERROR("Failed to initialize memory training!\n"); + dev_err(adev->dev, "Failed to initialize memory training!\n"); return ret; } ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT); if (ret) { - DRM_ERROR("Failed to process memory training!\n"); + dev_err(adev->dev, "Failed to process memory training!\n"); return ret; } } @@ -506,7 +507,6 @@ static int psp_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; struct psp_context *psp = &adev->psp; - struct psp_gfx_cmd_resp *cmd = psp->cmd; psp_memory_training_fini(psp); @@ -516,8 +516,8 @@ static int psp_sw_fini(void *handle) amdgpu_ucode_release(&psp->cap_fw); amdgpu_ucode_release(&psp->toc_fw); - kfree(cmd); - cmd = NULL; + kfree(psp->cmd); + psp->cmd = NULL; psp_free_shared_bufs(psp); @@ -675,9 +675,11 @@ psp_cmd_submit_buf(struct psp_context *psp, */ if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) { if (ucode) - DRM_WARN("failed to load ucode %s(0x%X) ", - amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id); - DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n", + dev_warn(psp->adev->dev, + "failed to load ucode %s(0x%X) ", + amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id); + dev_warn(psp->adev->dev, + "psp gfx command %s(0x%X) failed and response status is (0x%X)\n", psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id, psp->cmd_buf_mem->resp.status); /* If any firmware (including CAP) load fails under SRIOV, it should @@ -807,7 +809,7 @@ static int psp_tmr_init(struct psp_context *psp) psp->fw_pri_buf) { ret = psp_load_toc(psp, &tmr_size); if (ret) { - DRM_ERROR("Failed to load toc\n"); + dev_err(psp->adev->dev, "Failed to load toc\n"); return ret; } } @@ -855,7 +857,7 @@ static int psp_tmr_load(struct psp_context *psp) psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo); if (psp->tmr_bo) - DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n", + dev_info(psp->adev->dev, "reserve 0x%lx from 0x%llx for PSP TMR\n", amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr); ret = psp_cmd_submit_buf(psp, NULL, cmd, @@ -1113,7 +1115,7 @@ int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg, psp_prep_reg_prog_cmd_buf(cmd, reg, value); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); if (ret) - DRM_ERROR("PSP failed to program reg id %d", reg); + dev_err(psp->adev->dev, "PSP failed to program reg id %d\n", reg); release_psp_cmd_buf(psp); @@ -1493,22 +1495,22 @@ static void psp_ras_ta_check_status(struct psp_context *psp) switch (ras_cmd->ras_status) { case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP: dev_warn(psp->adev->dev, - "RAS WARNING: cmd failed due to unsupported ip\n"); + "RAS WARNING: cmd failed due to unsupported ip\n"); break; case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ: dev_warn(psp->adev->dev, - "RAS WARNING: cmd failed due to unsupported error injection\n"); + "RAS WARNING: cmd failed due to unsupported error injection\n"); break; case TA_RAS_STATUS__SUCCESS: break; case TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED: if (ras_cmd->cmd_id == TA_RAS_COMMAND__TRIGGER_ERROR) dev_warn(psp->adev->dev, - "RAS WARNING: Inject error to critical region is not allowed\n"); + "RAS WARNING: Inject error to critical region is not allowed\n"); break; default: dev_warn(psp->adev->dev, - "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status); + "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status); break; } } @@ -1532,7 +1534,7 @@ int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id) return ret; if (ras_cmd->if_version > RAS_TA_HOST_IF_VER) { - DRM_WARN("RAS: Unsupported Interface"); + dev_warn(psp->adev->dev, "RAS: Unsupported Interface\n"); return -EINVAL; } @@ -1682,7 +1684,7 @@ int psp_ras_initialize(struct psp_context *psp) psp->ras_context.context.initialized = true; else { if (ras_cmd->ras_status) - dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status); + dev_warn(adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status); /* fail to load RAS TA */ psp->ras_context.context.initialized = false; @@ -2010,11 +2012,14 @@ static int psp_securedisplay_initialize(struct psp_context *psp) } ret = psp_ta_load(psp, &psp->securedisplay_context.context); - if (!ret) { + if (!ret && !psp->securedisplay_context.context.resp_status) { psp->securedisplay_context.context.initialized = true; mutex_init(&psp->securedisplay_context.mutex); - } else + } else { + /* don't try again */ + psp->securedisplay_context.context.bin_desc.size_bytes = 0; return ret; + } mutex_lock(&psp->securedisplay_context.mutex); @@ -2102,7 +2107,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_kdb != NULL)) { ret = psp_bootloader_load_kdb(psp); if (ret) { - DRM_ERROR("PSP load kdb failed!\n"); + dev_err(adev->dev, "PSP load kdb failed!\n"); return ret; } } @@ -2111,7 +2116,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_spl != NULL)) { ret = psp_bootloader_load_spl(psp); if (ret) { - DRM_ERROR("PSP load spl failed!\n"); + dev_err(adev->dev, "PSP load spl failed!\n"); return ret; } } @@ -2120,7 +2125,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_sysdrv != NULL)) { ret = psp_bootloader_load_sysdrv(psp); if (ret) { - DRM_ERROR("PSP load sys drv failed!\n"); + dev_err(adev->dev, "PSP load sys drv failed!\n"); return ret; } } @@ -2129,7 +2134,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_soc_drv != NULL)) { ret = psp_bootloader_load_soc_drv(psp); if (ret) { - DRM_ERROR("PSP load soc drv failed!\n"); + dev_err(adev->dev, "PSP load soc drv failed!\n"); return ret; } } @@ -2138,7 +2143,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_intf_drv != NULL)) { ret = psp_bootloader_load_intf_drv(psp); if (ret) { - DRM_ERROR("PSP load intf drv failed!\n"); + dev_err(adev->dev, "PSP load intf drv failed!\n"); return ret; } } @@ -2147,7 +2152,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_dbg_drv != NULL)) { ret = psp_bootloader_load_dbg_drv(psp); if (ret) { - DRM_ERROR("PSP load dbg drv failed!\n"); + dev_err(adev->dev, "PSP load dbg drv failed!\n"); return ret; } } @@ -2156,7 +2161,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_ras_drv != NULL)) { ret = psp_bootloader_load_ras_drv(psp); if (ret) { - DRM_ERROR("PSP load ras_drv failed!\n"); + dev_err(adev->dev, "PSP load ras_drv failed!\n"); return ret; } } @@ -2165,7 +2170,7 @@ static int psp_hw_start(struct psp_context *psp) (psp->funcs->bootloader_load_sos != NULL)) { ret = psp_bootloader_load_sos(psp); if (ret) { - DRM_ERROR("PSP load sos failed!\n"); + dev_err(adev->dev, "PSP load sos failed!\n"); return ret; } } @@ -2173,7 +2178,7 @@ static int psp_hw_start(struct psp_context *psp) ret = psp_ring_create(psp, PSP_RING_TYPE__KM); if (ret) { - DRM_ERROR("PSP create ring failed!\n"); + dev_err(adev->dev, "PSP create ring failed!\n"); return ret; } @@ -2183,7 +2188,7 @@ static int psp_hw_start(struct psp_context *psp) if (!psp_boottime_tmr(psp)) { ret = psp_tmr_init(psp); if (ret) { - DRM_ERROR("PSP tmr init failed!\n"); + dev_err(adev->dev, "PSP tmr init failed!\n"); return ret; } } @@ -2202,7 +2207,7 @@ static int psp_hw_start(struct psp_context *psp) ret = psp_tmr_load(psp); if (ret) { - DRM_ERROR("PSP load tmr failed!\n"); + dev_err(adev->dev, "PSP load tmr failed!\n"); return ret; } @@ -2449,7 +2454,8 @@ static void psp_print_fw_hdr(struct psp_context *psp, } } -static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode, +static int psp_prep_load_ip_fw_cmd_buf(struct psp_context *psp, + struct amdgpu_firmware_info *ucode, struct psp_gfx_cmd_resp *cmd) { int ret; @@ -2462,7 +2468,7 @@ static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode, ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type); if (ret) - DRM_ERROR("Unknown firmware type\n"); + dev_err(psp->adev->dev, "Unknown firmware type\n"); return ret; } @@ -2473,7 +2479,7 @@ int psp_execute_ip_fw_load(struct psp_context *psp, int ret = 0; struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); - ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd); + ret = psp_prep_load_ip_fw_cmd_buf(psp, ucode, cmd); if (!ret) { ret = psp_cmd_submit_buf(psp, ucode, cmd, psp->fence_buf_mc_addr); @@ -2508,13 +2514,13 @@ static int psp_load_smu_fw(struct psp_context *psp) adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 2)))) { ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD); if (ret) - DRM_WARN("Failed to set MP1 state prepare for reload\n"); + dev_err(adev->dev, "Failed to set MP1 state prepare for reload\n"); } ret = psp_execute_ip_fw_load(psp, ucode); if (ret) - DRM_ERROR("PSP load smu failed!\n"); + dev_err(adev->dev, "PSP load smu failed!\n"); return ret; } @@ -2610,7 +2616,7 @@ static int psp_load_non_psp_fw(struct psp_context *psp) adev->virt.autoload_ucode_id : AMDGPU_UCODE_ID_RLC_G)) { ret = psp_rlc_autoload_start(psp); if (ret) { - DRM_ERROR("Failed to start rlc autoload\n"); + dev_err(adev->dev, "Failed to start rlc autoload\n"); return ret; } } @@ -2632,7 +2638,7 @@ static int psp_load_fw(struct amdgpu_device *adev) ret = psp_ring_init(psp, PSP_RING_TYPE__KM); if (ret) { - DRM_ERROR("PSP ring init failed!\n"); + dev_err(adev->dev, "PSP ring init failed!\n"); goto failed; } } @@ -2647,13 +2653,13 @@ static int psp_load_fw(struct amdgpu_device *adev) ret = psp_asd_initialize(psp); if (ret) { - DRM_ERROR("PSP load asd failed!\n"); + dev_err(adev->dev, "PSP load asd failed!\n"); goto failed1; } ret = psp_rl_load(adev); if (ret) { - DRM_ERROR("PSP load RL failed!\n"); + dev_err(adev->dev, "PSP load RL failed!\n"); goto failed1; } @@ -2673,7 +2679,7 @@ static int psp_load_fw(struct amdgpu_device *adev) ret = psp_ras_initialize(psp); if (ret) dev_err(psp->adev->dev, - "RAS: Failed to initialize RAS\n"); + "RAS: Failed to initialize RAS\n"); ret = psp_hdcp_initialize(psp); if (ret) @@ -2726,7 +2732,7 @@ static int psp_hw_init(void *handle) ret = psp_load_fw(adev); if (ret) { - DRM_ERROR("PSP firmware loading failed\n"); + dev_err(adev->dev, "PSP firmware loading failed\n"); goto failed; } @@ -2773,7 +2779,7 @@ static int psp_suspend(void *handle) psp->xgmi_context.context.initialized) { ret = psp_xgmi_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate xgmi ta\n"); + dev_err(adev->dev, "Failed to terminate xgmi ta\n"); goto out; } } @@ -2781,46 +2787,46 @@ static int psp_suspend(void *handle) if (psp->ta_fw) { ret = psp_ras_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate ras ta\n"); + dev_err(adev->dev, "Failed to terminate ras ta\n"); goto out; } ret = psp_hdcp_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate hdcp ta\n"); + dev_err(adev->dev, "Failed to terminate hdcp ta\n"); goto out; } ret = psp_dtm_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate dtm ta\n"); + dev_err(adev->dev, "Failed to terminate dtm ta\n"); goto out; } ret = psp_rap_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate rap ta\n"); + dev_err(adev->dev, "Failed to terminate rap ta\n"); goto out; } ret = psp_securedisplay_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate securedisplay ta\n"); + dev_err(adev->dev, "Failed to terminate securedisplay ta\n"); goto out; } } ret = psp_asd_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate asd\n"); + dev_err(adev->dev, "Failed to terminate asd\n"); goto out; } ret = psp_tmr_terminate(psp); if (ret) { - DRM_ERROR("Failed to terminate tmr\n"); + dev_err(adev->dev, "Failed to terminate tmr\n"); goto out; } ret = psp_ring_stop(psp, PSP_RING_TYPE__KM); if (ret) - DRM_ERROR("PSP ring stop failed\n"); + dev_err(adev->dev, "PSP ring stop failed\n"); out: return ret; @@ -2832,12 +2838,12 @@ static int psp_resume(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; struct psp_context *psp = &adev->psp; - DRM_INFO("PSP is resuming...\n"); + dev_info(adev->dev, "PSP is resuming...\n"); if (psp->mem_train_ctx.enable_mem_training) { ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME); if (ret) { - DRM_ERROR("Failed to process memory training!\n"); + dev_err(adev->dev, "Failed to process memory training!\n"); return ret; } } @@ -2854,7 +2860,7 @@ static int psp_resume(void *handle) ret = psp_asd_initialize(psp); if (ret) { - DRM_ERROR("PSP load asd failed!\n"); + dev_err(adev->dev, "PSP load asd failed!\n"); goto failed; } @@ -2878,7 +2884,7 @@ static int psp_resume(void *handle) ret = psp_ras_initialize(psp); if (ret) dev_err(psp->adev->dev, - "RAS: Failed to initialize RAS\n"); + "RAS: Failed to initialize RAS\n"); ret = psp_hdcp_initialize(psp); if (ret) @@ -2906,7 +2912,7 @@ static int psp_resume(void *handle) return 0; failed: - DRM_ERROR("PSP resume failed\n"); + dev_err(adev->dev, "PSP resume failed\n"); mutex_unlock(&adev->firmware.mutex); return ret; } @@ -2967,9 +2973,11 @@ int psp_ring_cmd_submit(struct psp_context *psp, write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw); /* Check invalid write_frame ptr address */ if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) { - DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n", - ring_buffer_start, ring_buffer_end, write_frame); - DRM_ERROR("write_frame is pointing to address out of bounds\n"); + dev_err(adev->dev, + "ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n", + ring_buffer_start, ring_buffer_end, write_frame); + dev_err(adev->dev, + "write_frame is pointing to address out of bounds\n"); return -EINVAL; } @@ -3496,7 +3504,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev, int ret; if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) { - DRM_INFO("PSP block is not ready yet."); + dev_info(adev->dev, "PSP block is not ready yet\n."); return -EBUSY; } @@ -3505,7 +3513,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev, mutex_unlock(&adev->psp.mutex); if (ret) { - DRM_ERROR("Failed to read USBC PD FW, err = %d", ret); + dev_err(adev->dev, "Failed to read USBC PD FW, err = %d\n", ret); return ret; } @@ -3527,7 +3535,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, void *fw_pri_cpu_addr; if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) { - DRM_INFO("PSP block is not ready yet."); + dev_err(adev->dev, "PSP block is not ready yet."); return -EBUSY; } @@ -3560,7 +3568,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, release_firmware(usbc_pd_fw); fail: if (ret) { - DRM_ERROR("Failed to load USBC PD FW, err = %d", ret); + dev_err(adev->dev, "Failed to load USBC PD FW, err = %d", ret); count = ret; } @@ -3607,7 +3615,7 @@ static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj, /* Safeguard against memory drain */ if (adev->psp.vbflash_image_size > AMD_VBIOS_FILE_MAX_SIZE_B) { - dev_err(adev->dev, "File size cannot exceed %u", AMD_VBIOS_FILE_MAX_SIZE_B); + dev_err(adev->dev, "File size cannot exceed %u\n", AMD_VBIOS_FILE_MAX_SIZE_B); kvfree(adev->psp.vbflash_tmp_buf); adev->psp.vbflash_tmp_buf = NULL; adev->psp.vbflash_image_size = 0; @@ -3626,7 +3634,7 @@ static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj, adev->psp.vbflash_image_size += count; mutex_unlock(&adev->psp.mutex); - dev_dbg(adev->dev, "IFWI staged for update"); + dev_dbg(adev->dev, "IFWI staged for update\n"); return count; } @@ -3646,7 +3654,7 @@ static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj, if (adev->psp.vbflash_image_size == 0) return -EINVAL; - dev_dbg(adev->dev, "PSP IFWI flash process initiated"); + dev_dbg(adev->dev, "PSP IFWI flash process initiated\n"); ret = amdgpu_bo_create_kernel(adev, adev->psp.vbflash_image_size, AMDGPU_GPU_PAGE_SIZE, @@ -3671,11 +3679,11 @@ static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj, adev->psp.vbflash_image_size = 0; if (ret) { - dev_err(adev->dev, "Failed to load IFWI, err = %d", ret); + dev_err(adev->dev, "Failed to load IFWI, err = %d\n", ret); return ret; } - dev_dbg(adev->dev, "PSP IFWI flash process done"); + dev_dbg(adev->dev, "PSP IFWI flash process done\n"); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index f44b303ae287a..eebac2e1a6c75 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -396,9 +396,6 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring) dma_fence_put(ring->vmid_wait); ring->vmid_wait = NULL; ring->me = 0; - - if (!ring->is_mes_queue) - ring->adev->rings[ring->idx] = NULL; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index e2ab303ad2708..60f770b99c2c5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -123,6 +123,22 @@ struct amdgpu_fence_driver { struct dma_fence **fences; }; +/* + * Fences mark an event in the GPUs pipeline and are used + * for GPU/CPU synchronization. When the fence is written, + * it is expected that all buffers associated with that fence + * are no longer in use by the associated ring on the GPU and + * that the relevant GPU caches have been flushed. + */ + +struct amdgpu_fence { + struct dma_fence base; + + /* RB, DMA, etc. */ + struct amdgpu_ring *ring; + ktime_t start_timestamp; +}; + extern const struct drm_sched_backend_ops amdgpu_sched_ops; void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c index bef7541770641..e9d2fcdde0e1c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c @@ -28,6 +28,10 @@ #include "amdgpu.h" #include "amdgpu_ucode.h" +static const struct kicker_device kicker_device_list[] = { + {0x744B, 0x00}, +}; + static void amdgpu_ucode_print_common_hdr(const struct common_firmware_header *hdr) { DRM_DEBUG("size_bytes: %u\n", le32_to_cpu(hdr->size_bytes)); @@ -1268,6 +1272,19 @@ static const char *amdgpu_ucode_legacy_naming(struct amdgpu_device *adev, int bl return NULL; } +bool amdgpu_is_kicker_fw(struct amdgpu_device *adev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kicker_device_list); i++) { + if (adev->pdev->device == kicker_device_list[i].device && + adev->pdev->revision == kicker_device_list[i].revision) + return true; + } + + return false; +} + void amdgpu_ucode_ip_version_decode(struct amdgpu_device *adev, int block_type, char *ucode_prefix, int len) { int maj, min, rev; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index b03321e7d2d89..4760092aafd72 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -536,6 +536,11 @@ struct amdgpu_firmware { uint64_t fw_buf_mc; }; +struct kicker_device{ + unsigned short device; + u8 revision; +}; + void amdgpu_ucode_print_mc_hdr(const struct common_firmware_header *hdr); void amdgpu_ucode_print_smc_hdr(const struct common_firmware_header *hdr); void amdgpu_ucode_print_imu_hdr(const struct common_firmware_header *hdr); @@ -562,5 +567,6 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type); const char *amdgpu_ucode_name(enum AMDGPU_UCODE_ID ucode_id); void amdgpu_ucode_ip_version_decode(struct amdgpu_device *adev, int block_type, char *ucode_prefix, int len); +bool amdgpu_is_kicker_fw(struct amdgpu_device *adev); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 22575422ca7ec..5a4b1b625f037 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -32,6 +32,7 @@ #include "amdgpu.h" #include "amdgpu_ras.h" +#include "amdgpu_reset.h" #include "vi.h" #include "soc15.h" #include "nv.h" @@ -468,7 +469,7 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) return -EINVAL; if (pf2vf_info->size > 1024) { - DRM_ERROR("invalid pf2vf message size\n"); + dev_err(adev->dev, "invalid pf2vf message size: 0x%x\n", pf2vf_info->size); return -EINVAL; } @@ -479,7 +480,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, adev->virt.fw_reserve.checksum_key, checksum); if (checksum != checkval) { - DRM_ERROR("invalid pf2vf message\n"); + dev_err(adev->dev, + "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n", + checksum, checkval); return -EINVAL; } @@ -493,7 +496,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, 0, checksum); if (checksum != checkval) { - DRM_ERROR("invalid pf2vf message\n"); + dev_err(adev->dev, + "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n", + checksum, checkval); return -EINVAL; } @@ -529,7 +534,7 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid; break; default: - DRM_ERROR("invalid pf2vf version\n"); + dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version); return -EINVAL; } @@ -599,8 +604,8 @@ static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev) vf2pf_info->driver_cert = 0; vf2pf_info->os_info.all = 0; - vf2pf_info->fb_usage = - ttm_resource_manager_usage(&adev->mman.vram_mgr.manager) >> 20; + vf2pf_info->fb_usage = ttm_resource_manager_used(&adev->mman.vram_mgr.manager) ? + ttm_resource_manager_usage(&adev->mman.vram_mgr.manager) >> 20 : 0; vf2pf_info->fb_vis_usage = amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr) >> 20; vf2pf_info->fb_size = adev->gmc.real_vram_size >> 20; @@ -628,8 +633,21 @@ static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work) int ret; ret = amdgpu_virt_read_pf2vf_data(adev); - if (ret) + if (ret) { + adev->virt.vf2pf_update_retry_cnt++; + if ((adev->virt.vf2pf_update_retry_cnt >= AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT) && + amdgpu_sriov_runtime(adev) && !amdgpu_in_reset(adev)) { + if (amdgpu_reset_domain_schedule(adev->reset_domain, + &adev->virt.flr_work)) + return; + else + dev_err(adev->dev, "Failed to queue work! at %s", __func__); + } + goto out; + } + + adev->virt.vf2pf_update_retry_cnt = 0; amdgpu_virt_write_vf2pf_data(adev); out: @@ -650,6 +668,7 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) adev->virt.fw_reserve.p_pf2vf = NULL; adev->virt.fw_reserve.p_vf2pf = NULL; adev->virt.vf2pf_update_interval_ms = 0; + adev->virt.vf2pf_update_retry_cnt = 0; if (adev->mman.fw_vram_usage_va && adev->mman.drv_vram_usage_va) { DRM_WARN("Currently fw_vram and drv_vram should not have values at the same time!"); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h index 23b6efa9d25df..891713757a8f5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h @@ -51,6 +51,8 @@ /* tonga/fiji use this offset */ #define mmBIF_IOV_FUNC_IDENTIFIER 0x1503 +#define AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT 30 + enum amdgpu_sriov_vf_mode { SRIOV_VF_MODE_BARE_METAL = 0, SRIOV_VF_MODE_ONE_VF, @@ -253,6 +255,7 @@ struct amdgpu_virt { /* vf2pf message */ struct delayed_work vf2pf_work; uint32_t vf2pf_update_interval_ms; + int vf2pf_update_retry_cnt; /* multimedia bandwidth config */ bool is_mm_bw_enabled; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index c7085a747b03b..451c37d04e456 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -435,7 +435,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, int r; lpfn = (u64)place->lpfn << PAGE_SHIFT; - if (!lpfn) + if (!lpfn || lpfn > man->size) lpfn = man->size; fpfn = (u64)place->fpfn << PAGE_SHIFT; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 584cd5277f927..e2dd7d4361cf3 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -1459,17 +1459,12 @@ static int dce_v10_0_audio_init(struct amdgpu_device *adev) static void dce_v10_0_audio_fini(struct amdgpu_device *adev) { - int i; - if (!amdgpu_audio) return; if (!adev->mode_info.audio.enabled) return; - for (i = 0; i < adev->mode_info.audio.num_pins; i++) - dce_v10_0_audio_enable(adev, &adev->mode_info.audio.pin[i], false); - adev->mode_info.audio.enabled = false; } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index c14b70350a51a..7ce89654e12b4 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -1508,17 +1508,12 @@ static int dce_v11_0_audio_init(struct amdgpu_device *adev) static void dce_v11_0_audio_fini(struct amdgpu_device *adev) { - int i; - if (!amdgpu_audio) return; if (!adev->mode_info.audio.enabled) return; - for (i = 0; i < adev->mode_info.audio.num_pins; i++) - dce_v11_0_audio_enable(adev, &adev->mode_info.audio.pin[i], false); - adev->mode_info.audio.enabled = false; } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c index 7f85ba5b726f6..c3d05ab7b12ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c @@ -1377,17 +1377,12 @@ static int dce_v6_0_audio_init(struct amdgpu_device *adev) static void dce_v6_0_audio_fini(struct amdgpu_device *adev) { - int i; - if (!amdgpu_audio) return; if (!adev->mode_info.audio.enabled) return; - for (i = 0; i < adev->mode_info.audio.num_pins; i++) - dce_v6_0_audio_enable(adev, &adev->mode_info.audio.pin[i], false); - adev->mode_info.audio.enabled = false; } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index f2b3cb5ed6bec..ce2300c3c36b4 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -1426,17 +1426,12 @@ static int dce_v8_0_audio_init(struct amdgpu_device *adev) static void dce_v8_0_audio_fini(struct amdgpu_device *adev) { - int i; - if (!amdgpu_audio) return; if (!adev->mode_info.audio.enabled) return; - for (i = 0; i < adev->mode_info.audio.num_pins; i++) - dce_v8_0_audio_enable(adev, &adev->mode_info.audio.pin[i], false); - adev->mode_info.audio.enabled = false; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 480d718d09cb6..7591a2803ae12 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -5296,9 +5296,9 @@ static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, if (flags & AMDGPU_IB_PREEMPTED) control |= INDIRECT_BUFFER_PRE_RESUME(1); - if (vmid) + if (vmid && !ring->adev->gfx.rs64_enable) gfx_v11_0_ring_emit_de_meta(ring, - (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false); + !amdgpu_sriov_vf(ring->adev) && (flags & AMDGPU_IB_PREEMPTED)); } if (ring->is_mes_queue) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 1943beb135c4c..472cb0f9e8f6c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -4656,6 +4656,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); /* reset ring buffer */ ring->wptr = 0; + atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); amdgpu_ring_clear_ring(ring); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c index 66c6bab75f8a5..0d3d00681edac 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c @@ -92,12 +92,12 @@ static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev) { uint64_t value; - /* Program the AGP BAR */ - WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_BASE, 0); - WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 24); - WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 24); - if (!amdgpu_sriov_vf(adev) || adev->asic_type <= CHIP_VEGA10) { + /* Program the AGP BAR */ + WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_BASE, 0); + WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 24); + WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 24); + /* Program the system aperture low logical page number. */ WREG32_SOC15_RLC(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index fd905889a4c63..e2ee10a98640a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -1061,7 +1061,7 @@ static int gmc_v7_0_sw_init(void *handle) GFP_KERNEL); if (!adev->gmc.vm_fault_info) return -ENOMEM; - atomic_set(&adev->gmc.vm_fault_info_updated, 0); + atomic_set_release(&adev->gmc.vm_fault_info_updated, 0); return 0; } @@ -1290,7 +1290,7 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev, vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID); if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid) - && !atomic_read(&adev->gmc.vm_fault_info_updated)) { + && !atomic_read_acquire(&adev->gmc.vm_fault_info_updated)) { struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info; u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, @@ -1306,8 +1306,7 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev, info->prot_read = protections & 0x8 ? true : false; info->prot_write = protections & 0x10 ? true : false; info->prot_exec = protections & 0x20 ? true : false; - mb(); - atomic_set(&adev->gmc.vm_fault_info_updated, 1); + atomic_set_release(&adev->gmc.vm_fault_info_updated, 1); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index 0bebcdbb26580..ed268c5739eb8 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -1174,7 +1174,7 @@ static int gmc_v8_0_sw_init(void *handle) GFP_KERNEL); if (!adev->gmc.vm_fault_info) return -ENOMEM; - atomic_set(&adev->gmc.vm_fault_info_updated, 0); + atomic_set_release(&adev->gmc.vm_fault_info_updated, 0); return 0; } @@ -1465,7 +1465,7 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev, vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID); if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid) - && !atomic_read(&adev->gmc.vm_fault_info_updated)) { + && !atomic_read_acquire(&adev->gmc.vm_fault_info_updated)) { struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info; u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, @@ -1481,8 +1481,7 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev, info->prot_read = protections & 0x8 ? true : false; info->prot_write = protections & 0x10 ? true : false; info->prot_exec = protections & 0x20 ? true : false; - mb(); - atomic_set(&adev->gmc.vm_fault_info_updated, 1); + atomic_set_release(&adev->gmc.vm_fault_info_updated, 1); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c index 30210613dc5c4..2f3054ed7b1b5 100644 --- a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c @@ -42,7 +42,12 @@ static void hdp_v4_0_flush_hdp(struct amdgpu_device *adev, { if (!ring || !ring->funcs->emit_wreg) { WREG32((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - RREG32((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2); + /* We just need to read back a register to post the write. + * Reading back the remapped register causes problems on + * some platforms so just read back the memory size register. + */ + if (adev->nbio.funcs->get_memsize) + adev->nbio.funcs->get_memsize(adev); } else { amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c index d3962d4690881..40705e13ca567 100644 --- a/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c @@ -33,7 +33,12 @@ static void hdp_v5_0_flush_hdp(struct amdgpu_device *adev, { if (!ring || !ring->funcs->emit_wreg) { WREG32((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - RREG32((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2); + /* We just need to read back a register to post the write. + * Reading back the remapped register causes problems on + * some platforms so just read back the memory size register. + */ + if (adev->nbio.funcs->get_memsize) + adev->nbio.funcs->get_memsize(adev); } else { amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v5_2.c b/drivers/gpu/drm/amd/amdgpu/hdp_v5_2.c index f52552c5fa27b..6b9f2e1d9d690 100644 --- a/drivers/gpu/drm/amd/amdgpu/hdp_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v5_2.c @@ -34,7 +34,17 @@ static void hdp_v5_2_flush_hdp(struct amdgpu_device *adev, if (!ring || !ring->funcs->emit_wreg) { WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - RREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2); + if (amdgpu_sriov_vf(adev)) { + /* this is fine because SR_IOV doesn't remap the register */ + RREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2); + } else { + /* We just need to read back a register to post the write. + * Reading back the remapped register causes problems on + * some platforms so just read back the memory size register. + */ + if (adev->nbio.funcs->get_memsize) + adev->nbio.funcs->get_memsize(adev); + } } else { amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v6_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v6_0.c index b6d71ec1debf9..0d0c568f38393 100644 --- a/drivers/gpu/drm/amd/amdgpu/hdp_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v6_0.c @@ -33,7 +33,12 @@ static void hdp_v6_0_flush_hdp(struct amdgpu_device *adev, { if (!ring || !ring->funcs->emit_wreg) { WREG32((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - RREG32((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2); + /* We just need to read back a register to post the write. + * Reading back the remapped register causes problems on + * some platforms so just read back the memory size register. + */ + if (adev->nbio.funcs->get_memsize) + adev->nbio.funcs->get_memsize(adev); } else { amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c index 9086f2fdfaf42..553f4f24f5ade 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c @@ -172,6 +172,30 @@ static void mmhub_v1_7_init_tlb_regs(struct amdgpu_device *adev) WREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL, tmp); } +/* Set snoop bit for SDMA so that SDMA writes probe-invalidates RW lines */ +static void mmhub_v1_7_init_snoop_override_regs(struct amdgpu_device *adev) +{ + uint32_t tmp; + int i; + uint32_t distance = regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE - + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE; + + for (i = 0; i < 5; i++) { /* DAGB instances */ + tmp = RREG32_SOC15_OFFSET(MMHUB, 0, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, i * distance); + tmp |= (1 << 15); /* SDMA client is BIT15 */ + WREG32_SOC15_OFFSET(MMHUB, 0, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, i * distance, tmp); + + tmp = RREG32_SOC15_OFFSET(MMHUB, 0, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, i * distance); + tmp |= (1 << 15); + WREG32_SOC15_OFFSET(MMHUB, 0, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, i * distance, tmp); + } + +} + static void mmhub_v1_7_init_cache_regs(struct amdgpu_device *adev) { uint32_t tmp; @@ -337,6 +361,7 @@ static int mmhub_v1_7_gart_enable(struct amdgpu_device *adev) mmhub_v1_7_init_system_aperture_regs(adev); mmhub_v1_7_init_tlb_regs(adev); mmhub_v1_7_init_cache_regs(adev); + mmhub_v1_7_init_snoop_override_regs(adev); mmhub_v1_7_enable_system_domain(adev); mmhub_v1_7_disable_identity_aperture(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c index 3d8e579d5c4e8..c7bdccff785b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c @@ -213,6 +213,32 @@ static void mmhub_v1_8_init_tlb_regs(struct amdgpu_device *adev) } } +/* Set snoop bit for SDMA so that SDMA writes probe-invalidates RW lines */ +static void mmhub_v1_8_init_snoop_override_regs(struct amdgpu_device *adev) +{ + uint32_t tmp, inst_mask; + int i, j; + uint32_t distance = regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE - + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE; + + inst_mask = adev->aid_mask; + for_each_inst(i, inst_mask) { + for (j = 0; j < 5; j++) { /* DAGB instances */ + tmp = RREG32_SOC15_OFFSET(MMHUB, i, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance); + tmp |= (1 << 15); /* SDMA client is BIT15 */ + WREG32_SOC15_OFFSET(MMHUB, i, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance, tmp); + + tmp = RREG32_SOC15_OFFSET(MMHUB, i, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance); + tmp |= (1 << 15); + WREG32_SOC15_OFFSET(MMHUB, i, + regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance, tmp); + } + } +} + static void mmhub_v1_8_init_cache_regs(struct amdgpu_device *adev) { uint32_t tmp, inst_mask; @@ -418,6 +444,7 @@ static int mmhub_v1_8_gart_enable(struct amdgpu_device *adev) mmhub_v1_8_init_system_aperture_regs(adev); mmhub_v1_8_init_tlb_regs(adev); mmhub_v1_8_init_cache_regs(adev); + mmhub_v1_8_init_snoop_override_regs(adev); mmhub_v1_8_enable_system_domain(adev); mmhub_v1_8_disable_identity_aperture(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c index 12c7f4b46ea94..b0c003da52fc1 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c @@ -36,40 +36,47 @@ static const char *mmhub_client_ids_v3_0_1[][2] = { [0][0] = "VMC", + [1][0] = "ISPXT", + [2][0] = "ISPIXT", [4][0] = "DCEDMC", [5][0] = "DCEVGA", [6][0] = "MP0", [7][0] = "MP1", - [8][0] = "MPIO", - [16][0] = "HDP", - [17][0] = "LSDMA", - [18][0] = "JPEG", - [19][0] = "VCNU0", - [21][0] = "VSCH", - [22][0] = "VCNU1", - [23][0] = "VCN1", - [32+20][0] = "VCN0", - [2][1] = "DBGUNBIO", + [8][0] = "MPM", + [12][0] = "ISPTNR", + [14][0] = "ISPCRD0", + [15][0] = "ISPCRD1", + [16][0] = "ISPCRD2", + [22][0] = "HDP", + [23][0] = "LSDMA", + [24][0] = "JPEG", + [27][0] = "VSCH", + [28][0] = "VCNU", + [29][0] = "VCN", + [1][1] = "ISPXT", + [2][1] = "ISPIXT", [3][1] = "DCEDWB", [4][1] = "DCEDMC", [5][1] = "DCEVGA", [6][1] = "MP0", [7][1] = "MP1", - [8][1] = "MPIO", - [10][1] = "DBGU0", - [11][1] = "DBGU1", - [12][1] = "DBGU2", - [13][1] = "DBGU3", - [14][1] = "XDP", - [15][1] = "OSSSYS", - [16][1] = "HDP", - [17][1] = "LSDMA", - [18][1] = "JPEG", - [19][1] = "VCNU0", - [20][1] = "VCN0", - [21][1] = "VSCH", - [22][1] = "VCNU1", - [23][1] = "VCN1", + [8][1] = "MPM", + [10][1] = "ISPMWR0", + [11][1] = "ISPMWR1", + [12][1] = "ISPTNR", + [13][1] = "ISPSWR", + [14][1] = "ISPCWR0", + [15][1] = "ISPCWR1", + [16][1] = "ISPCWR2", + [17][1] = "ISPCWR3", + [18][1] = "XDP", + [21][1] = "OSSSYS", + [22][1] = "HDP", + [23][1] = "LSDMA", + [24][1] = "JPEG", + [27][1] = "VSCH", + [28][1] = "VCNU", + [29][1] = "VCN", }; static uint32_t mmhub_v3_0_1_get_invalidate_req(unsigned int vmid, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c index 5718e4d40e666..9713cb59d1c14 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c @@ -198,6 +198,36 @@ static void mmhub_v9_4_init_tlb_regs(struct amdgpu_device *adev, int hubid) hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp); } +/* Set snoop bit for SDMA so that SDMA writes probe-invalidates RW lines */ +static void mmhub_v9_4_init_snoop_override_regs(struct amdgpu_device *adev, int hubid) +{ + uint32_t tmp; + int i; + uint32_t distance = mmDAGB1_WRCLI_GPU_SNOOP_OVERRIDE - + mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE; + uint32_t huboffset = hubid * MMHUB_INSTANCE_REGISTER_OFFSET; + + for (i = 0; i < 5 - (2 * hubid); i++) { + /* DAGB instances 0 to 4 are in hub0 and 5 to 7 are in hub1 */ + tmp = RREG32_SOC15_OFFSET(MMHUB, 0, + mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, + huboffset + i * distance); + tmp |= (1 << 15); /* SDMA client is BIT15 */ + WREG32_SOC15_OFFSET(MMHUB, 0, + mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, + huboffset + i * distance, tmp); + + tmp = RREG32_SOC15_OFFSET(MMHUB, 0, + mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, + huboffset + i * distance); + tmp |= (1 << 15); + WREG32_SOC15_OFFSET(MMHUB, 0, + mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, + huboffset + i * distance, tmp); + } + +} + static void mmhub_v9_4_init_cache_regs(struct amdgpu_device *adev, int hubid) { uint32_t tmp; @@ -392,6 +422,7 @@ static int mmhub_v9_4_gart_enable(struct amdgpu_device *adev) if (!amdgpu_sriov_vf(adev)) mmhub_v9_4_init_cache_regs(adev, i); + mmhub_v9_4_init_snoop_override_regs(adev, i); mmhub_v9_4_enable_system_domain(adev, i); if (!amdgpu_sriov_vf(adev)) mmhub_v9_4_disable_identity_aperture(adev, i); diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index 63725b2ebc037..37ac6d8ff8136 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -276,6 +276,8 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) timeout -= 10; } while (timeout > 1); + dev_warn(adev->dev, "waiting IDH_FLR_NOTIFICATION_CMPL timeout\n"); + flr_done: atomic_set(&adev->reset_domain->in_gpu_reset, 0); up_write(&adev->reset_domain->sem); diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c index 6a68ee946f1cc..96edd5d11326d 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c @@ -298,6 +298,8 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) timeout -= 10; } while (timeout > 1); + dev_warn(adev->dev, "waiting IDH_FLR_NOTIFICATION_CMPL timeout\n"); + flr_done: atomic_set(&adev->reset_domain->in_gpu_reset, 0); up_write(&adev->reset_domain->sem); diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index 7910c463ae385..82709e692f4cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -142,23 +142,23 @@ static struct amdgpu_video_codec_info sriov_sc_video_codecs_encode_array[] = { }; static struct amdgpu_video_codec_info sriov_sc_video_codecs_decode_array_vcn0[] = { - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 4096, 4096, 3)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 4096, 4096, 5)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 1920, 1088, 3)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 1920, 1088, 5)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4096, 4)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 1920, 1088, 4)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1, 8192, 4352, 0)}, }; static struct amdgpu_video_codec_info sriov_sc_video_codecs_decode_array_vcn1[] = { - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 4096, 4096, 3)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 4096, 4096, 5)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 1920, 1088, 3)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 1920, 1088, 5)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4096, 4)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 1920, 1088, 4)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)}, }; diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index 4712ffc0a482c..7819f5d584f59 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -117,23 +117,17 @@ static struct amdgpu_video_codecs sriov_vcn_4_0_0_video_codecs_encode_vcn1 = { }; static struct amdgpu_video_codec_info sriov_vcn_4_0_0_video_codecs_decode_array_vcn0[] = { - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 4096, 4096, 3)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 4096, 4096, 5)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4096, 4)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1, 8192, 4352, 0)}, }; static struct amdgpu_video_codec_info sriov_vcn_4_0_0_video_codecs_decode_array_vcn1[] = { - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 4096, 4096, 3)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 4096, 4096, 5)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4096, 4)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)}, - {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)}, + {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)}, {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)}, }; diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c index 5534c769b6550..1e0189f90fd6a 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c @@ -625,7 +625,22 @@ static void uvd_v3_1_enable_mgcg(struct amdgpu_device *adev, * * @handle: handle used to pass amdgpu_device pointer * - * Initialize the hardware, boot up the VCPU and do some testing + * Initialize the hardware, boot up the VCPU and do some testing. + * + * On SI, the UVD is meant to be used in a specific power state, + * or alternatively the driver can manually enable its clock. + * In amdgpu we use the dedicated UVD power state when DPM is enabled. + * Calling amdgpu_dpm_enable_uvd makes DPM select the UVD power state + * for the SMU and afterwards enables the UVD clock. + * This is automatically done by amdgpu_uvd_ring_begin_use when work + * is submitted to the UVD ring. Here, we have to call it manually + * in order to power up UVD before firmware validation. + * + * Note that we must not disable the UVD clock here, as that would + * cause the ring test to fail. However, UVD is powered off + * automatically after the ring test: amdgpu_uvd_ring_end_use calls + * the UVD idle work handler which will disable the UVD clock when + * all fences are signalled. */ static int uvd_v3_1_hw_init(void *handle) { @@ -635,6 +650,15 @@ static int uvd_v3_1_hw_init(void *handle) int r; uvd_v3_1_mc_resume(adev); + uvd_v3_1_enable_mgcg(adev, true); + + /* Make sure UVD is powered during FW validation. + * It's going to be automatically powered off after the ring test. + */ + if (adev->pm.dpm_enabled) + amdgpu_dpm_enable_uvd(adev, true); + else + amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); r = uvd_v3_1_fw_validate(adev); if (r) { @@ -642,9 +666,6 @@ static int uvd_v3_1_hw_init(void *handle) return r; } - uvd_v3_1_enable_mgcg(adev, true); - amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); - uvd_v3_1_start(adev); r = amdgpu_ring_test_helper(ring); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index a61ecefdafc51..710328f12194d 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -1765,15 +1765,19 @@ static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p, struct amdgpu_job *job) { struct drm_gpu_scheduler **scheds; - - /* The create msg must be in the first IB submitted */ - if (atomic_read(&job->base.entity->fence_seq)) - return -EINVAL; + struct dma_fence *fence; /* if VCN0 is harvested, we can't support AV1 */ if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0) return -EINVAL; + /* wait for all jobs to finish before switching to instance 0 */ + fence = amdgpu_ctx_get_fence(p->ctx, job->base.entity, ~0ull); + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC] [AMDGPU_RING_PRIO_DEFAULT].sched; drm_sched_entity_modify_sched(job->base.entity, scheds, 1); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c index 29164289c5f3e..43249e9f66d74 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c @@ -1644,15 +1644,19 @@ static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p, struct amdgpu_job *job) { struct drm_gpu_scheduler **scheds; - - /* The create msg must be in the first IB submitted */ - if (atomic_read(&job->base.entity->fence_seq)) - return -EINVAL; + struct dma_fence *fence; /* if VCN0 is harvested, we can't support AV1 */ if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0) return -EINVAL; + /* wait for all jobs to finish before switching to instance 0 */ + fence = amdgpu_ctx_get_fence(p->ctx, job->base.entity, ~0ull); + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC] [AMDGPU_RING_PRIO_0].sched; drm_sched_entity_modify_sched(job->base.entity, scheds, 1); @@ -1743,22 +1747,16 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, #define RADEON_VCN_ENGINE_TYPE_ENCODE (0x00000002) #define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003) - #define RADEON_VCN_ENGINE_INFO (0x30000001) -#define RADEON_VCN_ENGINE_INFO_MAX_OFFSET 16 - #define RENCODE_ENCODE_STANDARD_AV1 2 #define RENCODE_IB_PARAM_SESSION_INIT 0x00000003 -#define RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET 64 -/* return the offset in ib if id is found, -1 otherwise - * to speed up the searching we only search upto max_offset - */ -static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int max_offset) +/* return the offset in ib if id is found, -1 otherwise */ +static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start) { int i; - for (i = 0; i < ib->length_dw && i < max_offset && ib->ptr[i] >= 8; i += ib->ptr[i]/4) { + for (i = start; i < ib->length_dw && ib->ptr[i] >= 8; i += ib->ptr[i] / 4) { if (ib->ptr[i + 1] == id) return i; } @@ -1773,33 +1771,29 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, struct amdgpu_vcn_decode_buffer *decode_buffer; uint64_t addr; uint32_t val; - int idx; + int idx = 0, sidx; /* The first instance can decode anything */ if (!ring->me) return 0; - /* RADEON_VCN_ENGINE_INFO is at the top of ib block */ - idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, - RADEON_VCN_ENGINE_INFO_MAX_OFFSET); - if (idx < 0) /* engine info is missing */ - return 0; - - val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */ - if (val == RADEON_VCN_ENGINE_TYPE_DECODE) { - decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6]; - - if (!(decode_buffer->valid_buf_flag & 0x1)) - return 0; - - addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 | - decode_buffer->msg_buffer_address_lo; - return vcn_v4_0_dec_msg(p, job, addr); - } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) { - idx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, - RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET); - if (idx >= 0 && ib->ptr[idx + 2] == RENCODE_ENCODE_STANDARD_AV1) - return vcn_v4_0_limit_sched(p, job); + while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx)) >= 0) { + val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */ + if (val == RADEON_VCN_ENGINE_TYPE_DECODE) { + decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6]; + + if (!(decode_buffer->valid_buf_flag & 0x1)) + return 0; + + addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 | + decode_buffer->msg_buffer_address_lo; + return vcn_v4_0_dec_msg(p, job, addr); + } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) { + sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx); + if (sidx >= 0 && ib->ptr[sidx + 2] == RENCODE_ENCODE_STANDARD_AV1) + return vcn_v4_0_limit_sched(p, job); + } + idx += ib->ptr[idx] / 4; } return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 35dc926f234e3..2e194aa608489 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1085,7 +1085,12 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, svm_range_list_lock_and_flush_work(&p->svms, current->mm); mutex_lock(&p->svms.lock); mmap_write_unlock(current->mm); - if (interval_tree_iter_first(&p->svms.objects, + + /* Skip a special case that allocates VRAM without VA, + * VA will be invalid of 0. + */ + if (!(!args->va_addr && (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) && + interval_tree_iter_first(&p->svms.objects, args->va_addr >> PAGE_SHIFT, (args->va_addr + args->size - 1) >> PAGE_SHIFT)) { pr_err("Address: 0x%llx already allocated by SVM\n", @@ -2567,8 +2572,8 @@ static int criu_restore(struct file *filep, pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n", args->num_devices, args->num_bos, args->num_objects, args->priv_data_size); - if (!args->bos || !args->devices || !args->priv_data || !args->priv_data_size || - !args->num_devices || !args->num_bos) + if ((args->num_bos > 0 && !args->bos) || !args->devices || !args->priv_data || + !args->priv_data_size || !args->num_devices) return -EINVAL; mutex_lock(&p->mutex); @@ -3252,8 +3257,10 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) int retcode = -EINVAL; bool ptrace_attached = false; - if (nr >= AMDKFD_CORE_IOCTL_COUNT) + if (nr >= AMDKFD_CORE_IOCTL_COUNT) { + retcode = -ENOTTY; goto err_i1; + } if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { u32 amdkfd_size; @@ -3266,8 +3273,10 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) asize = amdkfd_size; cmd = ioctl->cmd; - } else + } else { + retcode = -ENOTTY; goto err_i1; + } dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 2786d47961e07..6af65db4de947 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -1017,7 +1017,15 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) } for (i = 0; i < kfd->num_nodes; i++) { - node = kfd->nodes[i]; + /* Race if another thread in b/w + * kfd_cleanup_nodes and kfree(kfd), + * when kfd->nodes[i] = NULL + */ + if (kfd->nodes[i]) + node = kfd->nodes[i]; + else + return; + spin_lock_irqsave(&node->interrupt_lock, flags); if (node->interrupts_active diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 4d9a406925e18..fd4a75073364c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -2147,14 +2147,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, return retval; } -/* - * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to - * stay in user mode. - */ -#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL -/* APE1 limit is inclusive and 64K aligned. */ -#define APE1_LIMIT_ALIGNMENT 0xFFFF - static bool set_cache_memory_policy(struct device_queue_manager *dqm, struct qcm_process_device *qpd, enum cache_policy default_policy, @@ -2169,34 +2161,6 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm, dqm_lock(dqm); - if (alternate_aperture_size == 0) { - /* base > limit disables APE1 */ - qpd->sh_mem_ape1_base = 1; - qpd->sh_mem_ape1_limit = 0; - } else { - /* - * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, - * SH_MEM_APE1_BASE[31:0], 0x0000 } - * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, - * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } - * Verify that the base and size parameters can be - * represented in this format and convert them. - * Additionally restrict APE1 to user-mode addresses. - */ - - uint64_t base = (uintptr_t)alternate_aperture_base; - uint64_t limit = base + alternate_aperture_size - 1; - - if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 || - (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) { - retval = false; - goto out; - } - - qpd->sh_mem_ape1_base = base >> 16; - qpd->sh_mem_ape1_limit = limit >> 16; - } - retval = dqm->asic_ops.set_cache_memory_policy( dqm, qpd, @@ -2205,6 +2169,9 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm, alternate_aperture_base, alternate_aperture_size); + if (retval) + goto out; + if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) program_sh_mem_settings(dqm, qpd); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c index d4d95c7f2e5d4..32bedef912b3b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c @@ -27,6 +27,14 @@ #include "oss/oss_2_4_sh_mask.h" #include "gca/gfx_7_2_sh_mask.h" +/* + * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to + * stay in user mode. + */ +#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL +/* APE1 limit is inclusive and 64K aligned. */ +#define APE1_LIMIT_ALIGNMENT 0xFFFF + static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm, struct qcm_process_device *qpd, enum cache_policy default_policy, @@ -84,6 +92,36 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm, { uint32_t default_mtype; uint32_t ape1_mtype; + unsigned int temp; + bool retval = true; + + if (alternate_aperture_size == 0) { + /* base > limit disables APE1 */ + qpd->sh_mem_ape1_base = 1; + qpd->sh_mem_ape1_limit = 0; + } else { + /* + * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, + * SH_MEM_APE1_BASE[31:0], 0x0000 } + * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, + * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } + * Verify that the base and size parameters can be + * represented in this format and convert them. + * Additionally restrict APE1 to user-mode addresses. + */ + + uint64_t base = (uintptr_t)alternate_aperture_base; + uint64_t limit = base + alternate_aperture_size - 1; + + if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 || + (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) { + retval = false; + goto out; + } + + qpd->sh_mem_ape1_base = base >> 16; + qpd->sh_mem_ape1_limit = limit >> 16; + } default_mtype = (default_policy == cache_policy_coherent) ? MTYPE_NONCACHED : @@ -97,37 +135,22 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm, | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) | DEFAULT_MTYPE(default_mtype) | APE1_MTYPE(ape1_mtype); - - return true; -} - -static int update_qpd_cik(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) -{ - struct kfd_process_device *pdd; - unsigned int temp; - - pdd = qpd_to_pdd(qpd); - - /* check if sh_mem_config register already configured */ - if (qpd->sh_mem_config == 0) { - qpd->sh_mem_config = - ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) | - DEFAULT_MTYPE(MTYPE_NONCACHED) | - APE1_MTYPE(MTYPE_NONCACHED); - qpd->sh_mem_ape1_limit = 0; - qpd->sh_mem_ape1_base = 0; - } - /* On dGPU we're always in GPUVM64 addressing mode with 64-bit * aperture addresses. */ - temp = get_sh_mem_bases_nybble_64(pdd); + temp = get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd)); qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp); pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n", qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases); +out: + return retval; +} + +static int update_qpd_cik(struct device_queue_manager *dqm, + struct qcm_process_device *qpd) +{ return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c index b291ee0fab943..320518f418903 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c @@ -27,6 +27,14 @@ #include "gca/gfx_8_0_sh_mask.h" #include "oss/oss_3_0_sh_mask.h" +/* + * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to + * stay in user mode. + */ +#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL +/* APE1 limit is inclusive and 64K aligned. */ +#define APE1_LIMIT_ALIGNMENT 0xFFFF + static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, struct qcm_process_device *qpd, enum cache_policy default_policy, @@ -85,6 +93,36 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, { uint32_t default_mtype; uint32_t ape1_mtype; + unsigned int temp; + bool retval = true; + + if (alternate_aperture_size == 0) { + /* base > limit disables APE1 */ + qpd->sh_mem_ape1_base = 1; + qpd->sh_mem_ape1_limit = 0; + } else { + /* + * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, + * SH_MEM_APE1_BASE[31:0], 0x0000 } + * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, + * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } + * Verify that the base and size parameters can be + * represented in this format and convert them. + * Additionally restrict APE1 to user-mode addresses. + */ + + uint64_t base = (uintptr_t)alternate_aperture_base; + uint64_t limit = base + alternate_aperture_size - 1; + + if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 || + (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) { + retval = false; + goto out; + } + + qpd->sh_mem_ape1_base = base >> 16; + qpd->sh_mem_ape1_limit = limit >> 16; + } default_mtype = (default_policy == cache_policy_coherent) ? MTYPE_UC : @@ -100,40 +138,21 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, default_mtype << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | ape1_mtype << SH_MEM_CONFIG__APE1_MTYPE__SHIFT; - return true; -} - -static int update_qpd_vi(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) -{ - struct kfd_process_device *pdd; - unsigned int temp; - - pdd = qpd_to_pdd(qpd); - - /* check if sh_mem_config register already configured */ - if (qpd->sh_mem_config == 0) { - qpd->sh_mem_config = - SH_MEM_ALIGNMENT_MODE_UNALIGNED << - SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT | - MTYPE_UC << - SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | - MTYPE_UC << - SH_MEM_CONFIG__APE1_MTYPE__SHIFT; - - qpd->sh_mem_ape1_limit = 0; - qpd->sh_mem_ape1_base = 0; - } - /* On dGPU we're always in GPUVM64 addressing mode with 64-bit * aperture addresses. */ - temp = get_sh_mem_bases_nybble_64(pdd); + temp = get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd)); qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp); pr_debug("sh_mem_bases nybble: 0x%X and register 0x%X\n", temp, qpd->sh_mem_bases); +out: + return retval; +} +static int update_qpd_vi(struct device_queue_manager *dqm, + struct qcm_process_device *qpd) +{ return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 0f58be65132fc..2b07c0000df6e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -1287,6 +1287,7 @@ void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid) user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id); if (unlikely(user_gpu_id == -EINVAL)) { WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id); + kfd_unref_process(p); return; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c index aee2212e52f69..33aa23450b3f7 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c @@ -78,8 +78,8 @@ static int kfd_init(void) static void kfd_exit(void) { kfd_cleanup_processes(); - kfd_debugfs_fini(); kfd_process_destroy_wq(); + kfd_debugfs_fini(); kfd_procfs_shutdown(); kfd_topology_shutdown(); kfd_chardev_exit(); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c index 1a03173e23133..18e82d0da75bc 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c @@ -225,7 +225,7 @@ static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer, packet->bitfields2.engine_sel = engine_sel__mes_map_queues__compute_vi; - packet->bitfields2.gws_control_queue = q->gws ? 1 : 0; + packet->bitfields2.gws_control_queue = q->properties.is_gws ? 1 : 0; packet->bitfields2.extended_engine_sel = extended_engine_sel__mes_map_queues__legacy_engine_sel; packet->bitfields2.queue_type = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 27c9d5c43765a..b475c2ab9768a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -111,7 +111,14 @@ #define KFD_KERNEL_QUEUE_SIZE 2048 -#define KFD_UNMAP_LATENCY_MS (4000) +/* KFD_UNMAP_LATENCY_MS is the timeout CP waiting for SDMA preemption. One XCC + * can be associated to 2 SDMA engines. queue_preemption_timeout_ms is the time + * driver waiting for CP returning the UNMAP_QUEUE fence. Thus the math is + * queue_preemption_timeout_ms = sdma_preemption_time * 2 + cp workload + * The format here makes CP workload 10% of total timeout + */ +#define KFD_UNMAP_LATENCY_MS \ + ((queue_preemption_timeout_ms - queue_preemption_timeout_ms / 10) >> 1) #define KFD_MAX_SDMA_QUEUES 128 diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index a6d08dee74f6e..93740b8fc3f44 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -812,6 +812,14 @@ struct kfd_process *kfd_create_process(struct task_struct *thread) return ERR_PTR(-EINVAL); } + /* If the process just called exec(3), it is possible that the + * cleanup of the kfd_process (following the release of the mm + * of the old process image) is still in the cleanup work queue. + * Make sure to drain any job before trying to recreate any + * resource for this process. + */ + flush_workqueue(kfd_process_wq); + /* * take kfd processes mutex before starting of process creation * so there won't be a case where two threads of the same process @@ -830,14 +838,6 @@ struct kfd_process *kfd_create_process(struct task_struct *thread) if (process) { pr_debug("Process already found\n"); } else { - /* If the process just called exec(3), it is possible that the - * cleanup of the kfd_process (following the release of the mm - * of the old process image) is still in the cleanup work queue. - * Make sure to drain any job before trying to recreate any - * resource for this process. - */ - flush_workqueue(kfd_process_wq); - process = create_process(thread); if (IS_ERR(process)) goto out; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 6b7c6f45a80a8..9ba9732f0172a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -1130,13 +1130,12 @@ svm_range_split_head(struct svm_range *prange, } static void -svm_range_add_child(struct svm_range *prange, struct mm_struct *mm, - struct svm_range *pchild, enum svm_work_list_ops op) +svm_range_add_child(struct svm_range *prange, struct svm_range *pchild, enum svm_work_list_ops op) { pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n", pchild, pchild->start, pchild->last, prange, op); - pchild->work_item.mm = mm; + pchild->work_item.mm = NULL; pchild->work_item.op = op; list_add_tail(&pchild->child_list, &prange->child_list); } @@ -1182,14 +1181,14 @@ svm_range_split_by_granularity(struct kfd_process *p, struct mm_struct *mm, r = svm_range_split(prange, start, prange->last, &head); if (r) return r; - svm_range_add_child(parent, mm, head, SVM_OP_ADD_RANGE); + svm_range_add_child(parent, head, SVM_OP_ADD_RANGE); } if (last < prange->last) { r = svm_range_split(prange, prange->start, last, &tail); if (r) return r; - svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE); + svm_range_add_child(parent, tail, SVM_OP_ADD_RANGE); } /* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */ @@ -1710,6 +1709,29 @@ static int svm_range_validate_and_map(struct mm_struct *mm, next = min(vma->vm_end, end); npages = (next - addr) >> PAGE_SHIFT; + /* HMM requires at least READ permissions. If provided with PROT_NONE, + * unmap the memory. If it's not already mapped, this is a no-op + * If PROT_WRITE is provided without READ, warn first then unmap + */ + if (!(vma->vm_flags & VM_READ)) { + unsigned long e, s; + + svm_range_lock(prange); + if (vma->vm_flags & VM_WRITE) + pr_debug("VM_WRITE without VM_READ is not supported"); + s = max(start, prange->start); + e = min(end, prange->last); + if (e >= s) + r = svm_range_unmap_from_gpus(prange, s, e, + KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU); + svm_range_unlock(prange); + /* If unmap returns non-zero, we'll bail on the next for loop + * iteration, so just leave r and continue + */ + addr = next; + continue; + } + WRITE_ONCE(p->svms.faulting_task, current); r = amdgpu_hmm_range_get_pages(&prange->notifier, addr, npages, readonly, owner, NULL, @@ -2393,15 +2415,17 @@ svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange, prange->work_item.op != SVM_OP_UNMAP_RANGE) prange->work_item.op = op; } else { - prange->work_item.op = op; - - /* Pairs with mmput in deferred_list_work */ - mmget(mm); - prange->work_item.mm = mm; - list_add_tail(&prange->deferred_list, - &prange->svms->deferred_range_list); - pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n", - prange, prange->start, prange->last, op); + /* Pairs with mmput in deferred_list_work. + * If process is exiting and mm is gone, don't update mmu notifier. + */ + if (mmget_not_zero(mm)) { + prange->work_item.mm = mm; + prange->work_item.op = op; + list_add_tail(&prange->deferred_list, + &prange->svms->deferred_range_list); + pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n", + prange, prange->start, prange->last, op); + } } spin_unlock(&svms->deferred_list_lock); } @@ -2415,8 +2439,7 @@ void schedule_deferred_list_work(struct svm_range_list *svms) } static void -svm_range_unmap_split(struct mm_struct *mm, struct svm_range *parent, - struct svm_range *prange, unsigned long start, +svm_range_unmap_split(struct svm_range *parent, struct svm_range *prange, unsigned long start, unsigned long last) { struct svm_range *head; @@ -2437,12 +2460,12 @@ svm_range_unmap_split(struct mm_struct *mm, struct svm_range *parent, svm_range_split(tail, last + 1, tail->last, &head); if (head != prange && tail != prange) { - svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE); - svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE); + svm_range_add_child(parent, head, SVM_OP_UNMAP_RANGE); + svm_range_add_child(parent, tail, SVM_OP_ADD_RANGE); } else if (tail != prange) { - svm_range_add_child(parent, mm, tail, SVM_OP_UNMAP_RANGE); + svm_range_add_child(parent, tail, SVM_OP_UNMAP_RANGE); } else if (head != prange) { - svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE); + svm_range_add_child(parent, head, SVM_OP_UNMAP_RANGE); } else if (parent != prange) { prange->work_item.op = SVM_OP_UNMAP_RANGE; } @@ -2481,14 +2504,14 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, l = min(last, pchild->last); if (l >= s) svm_range_unmap_from_gpus(pchild, s, l, trigger); - svm_range_unmap_split(mm, prange, pchild, start, last); + svm_range_unmap_split(prange, pchild, start, last); mutex_unlock(&pchild->lock); } s = max(start, prange->start); l = min(last, prange->last); if (l >= s) svm_range_unmap_from_gpus(prange, s, l, trigger); - svm_range_unmap_split(mm, prange, prange, start, last); + svm_range_unmap_split(prange, prange, start, last); if (unmap_parent) svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE); @@ -2531,8 +2554,6 @@ svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, if (range->event == MMU_NOTIFY_RELEASE) return true; - if (!mmget_not_zero(mni->mm)) - return true; start = mni->interval_tree.start; last = mni->interval_tree.last; @@ -2559,7 +2580,6 @@ svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, } svm_range_unlock(prange); - mmput(mni->mm); return true; } @@ -4145,7 +4165,7 @@ svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, r = svm_range_get_attr(p, mm, start, size, nattrs, attrs); break; default: - r = EINVAL; + r = -EINVAL; break; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 2ab4958cae834..38caa02ffa5e0 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -2888,11 +2888,6 @@ static int dm_resume(void *handle) return 0; } - - /* leave display off for S4 sequence */ - if (adev->in_s4) - return 0; - /* Recreate dc_state - DC invalidates it when setting power state to S3. */ dc_release_state(dm_state->context); dm_state->context = dc_create_state(dm->dc); @@ -2996,6 +2991,7 @@ static int dm_resume(void *handle) /* Do mst topology probing after resuming cached state*/ drm_connector_list_iter_begin(ddev, &iter); drm_for_each_connector_iter(connector, &iter) { + bool init = false; if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) continue; @@ -3005,7 +3001,14 @@ static int dm_resume(void *handle) aconnector->mst_root) continue; - drm_dp_mst_topology_queue_probe(&aconnector->mst_mgr); + scoped_guard(mutex, &aconnector->mst_mgr.lock) { + init = !aconnector->mst_mgr.mst_primary; + } + if (init) + dm_helpers_dp_mst_start_top_mgr(aconnector->dc_link->ctx, + aconnector->dc_link, false); + else + drm_dp_mst_topology_queue_probe(&aconnector->mst_mgr); } drm_connector_list_iter_end(&iter); @@ -4623,7 +4626,8 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev) static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm) { - drm_atomic_private_obj_fini(&dm->atomic_obj); + if (dm->atomic_obj.state) + drm_atomic_private_obj_fini(&dm->atomic_obj); } /****************************************************************************** @@ -6781,6 +6785,9 @@ amdgpu_dm_connector_atomic_check(struct drm_connector *conn, struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(conn); int ret; + if (WARN_ON(unlikely(!old_con_state || !new_con_state))) + return -EINVAL; + trace_amdgpu_dm_connector_atomic_check(new_con_state); if (conn->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { @@ -7466,7 +7473,7 @@ static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap, int i; int result = -EIO; - if (!ddc_service->ddc_pin || !ddc_service->ddc_pin->hw_info.hw_supported) + if (!ddc_service->ddc_pin) return result; cmd.payloads = kcalloc(num, sizeof(struct i2c_payload), GFP_KERNEL); @@ -9242,6 +9249,8 @@ static void get_freesync_config_for_crtc( } else { config.state = VRR_STATE_INACTIVE; } + } else { + config.state = VRR_STATE_UNSUPPORTED; } out: new_crtc_state->freesync_config = config; @@ -10837,7 +10846,7 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, dm_con_state = to_dm_connector_state(connector->state); - if (!adev->dm.freesync_module) + if (!adev->dm.freesync_module || !dc_supports_vrr(sink->ctx->dce_version)) goto update; /* Some eDP panels only have the refresh rate range info in DisplayID */ @@ -11047,7 +11056,7 @@ int amdgpu_dm_process_dmub_aux_transfer_sync( * Transient states before tunneling is enabled could * lead to this error. We can ignore this for now. */ - if (p_notify->result != AUX_RET_ERROR_PROTOCOL_ERROR) { + if (p_notify->result == AUX_RET_ERROR_PROTOCOL_ERROR) { DRM_WARN("DPIA AUX failed on 0x%x(%d), error %d\n", payload->address, payload->length, p_notify->result); @@ -11056,22 +11065,15 @@ int amdgpu_dm_process_dmub_aux_transfer_sync( goto out; } + payload->reply[0] = adev->dm.dmub_notify->aux_reply.command & 0xF; + if (adev->dm.dmub_notify->aux_reply.command & 0xF0) + /* The reply is stored in the top nibble of the command. */ + payload->reply[0] = (adev->dm.dmub_notify->aux_reply.command >> 4) & 0xF; - payload->reply[0] = adev->dm.dmub_notify->aux_reply.command; - if (!payload->write && p_notify->aux_reply.length && - (payload->reply[0] == AUX_TRANSACTION_REPLY_AUX_ACK)) { - - if (payload->length != p_notify->aux_reply.length) { - DRM_WARN("invalid read length %d from DPIA AUX 0x%x(%d)!\n", - p_notify->aux_reply.length, - payload->address, payload->length); - *operation_result = AUX_RET_ERROR_INVALID_REPLY; - goto out; - } - + /*write req may receive a byte indicating partially written number as well*/ + if (p_notify->aux_reply.length) memcpy(payload->data, p_notify->aux_reply.data, p_notify->aux_reply.length); - } /* success */ ret = p_notify->aux_reply.length; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 30d4c6fd95f53..dab732c6c6c7c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -410,6 +410,15 @@ static int dm_crtc_helper_atomic_check(struct drm_crtc *crtc, return -EINVAL; } + if (!state->legacy_cursor_update && amdgpu_dm_crtc_vrr_active(dm_crtc_state)) { + struct drm_plane_state *primary_state; + + /* Pull in primary plane for correct VRR handling */ + primary_state = drm_atomic_get_plane_state(state, crtc->primary); + if (IS_ERR(primary_state)) + return PTR_ERR(primary_state); + } + /* In some use cases, like reset, no stream is attached */ if (!dm_crtc_state->stream) return 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 2ad9f900a8574..a048022d9865a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -172,7 +172,10 @@ void hdcp_update_display(struct hdcp_workqueue *hdcp_work, struct mod_hdcp_display_adjustment display_adjust; unsigned int conn_index = aconnector->base.index; - mutex_lock(&hdcp_w->mutex); + guard(mutex)(&hdcp_w->mutex); + drm_connector_get(&aconnector->base); + if (hdcp_w->aconnector[conn_index]) + drm_connector_put(&hdcp_w->aconnector[conn_index]->base); hdcp_w->aconnector[conn_index] = aconnector; memset(&link_adjust, 0, sizeof(link_adjust)); @@ -209,7 +212,6 @@ void hdcp_update_display(struct hdcp_workqueue *hdcp_work, mod_hdcp_update_display(&hdcp_w->hdcp, conn_index, &link_adjust, &display_adjust, &hdcp_w->output); process_output(hdcp_w); - mutex_unlock(&hdcp_w->mutex); } static void hdcp_remove_display(struct hdcp_workqueue *hdcp_work, @@ -220,8 +222,7 @@ static void hdcp_remove_display(struct hdcp_workqueue *hdcp_work, struct drm_connector_state *conn_state = aconnector->base.state; unsigned int conn_index = aconnector->base.index; - mutex_lock(&hdcp_w->mutex); - hdcp_w->aconnector[conn_index] = aconnector; + guard(mutex)(&hdcp_w->mutex); /* the removal of display will invoke auth reset -> hdcp destroy and * we'd expect the Content Protection (CP) property changed back to @@ -237,9 +238,11 @@ static void hdcp_remove_display(struct hdcp_workqueue *hdcp_work, } mod_hdcp_remove_display(&hdcp_w->hdcp, aconnector->base.index, &hdcp_w->output); - + if (hdcp_w->aconnector[conn_index]) { + drm_connector_put(&hdcp_w->aconnector[conn_index]->base); + hdcp_w->aconnector[conn_index] = NULL; + } process_output(hdcp_w); - mutex_unlock(&hdcp_w->mutex); } void hdcp_reset_display(struct hdcp_workqueue *hdcp_work, unsigned int link_index) @@ -247,7 +250,7 @@ void hdcp_reset_display(struct hdcp_workqueue *hdcp_work, unsigned int link_inde struct hdcp_workqueue *hdcp_w = &hdcp_work[link_index]; unsigned int conn_index; - mutex_lock(&hdcp_w->mutex); + guard(mutex)(&hdcp_w->mutex); mod_hdcp_reset_connection(&hdcp_w->hdcp, &hdcp_w->output); @@ -256,11 +259,13 @@ void hdcp_reset_display(struct hdcp_workqueue *hdcp_work, unsigned int link_inde for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; conn_index++) { hdcp_w->encryption_status[conn_index] = MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF; + if (hdcp_w->aconnector[conn_index]) { + drm_connector_put(&hdcp_w->aconnector[conn_index]->base); + hdcp_w->aconnector[conn_index] = NULL; + } } process_output(hdcp_w); - - mutex_unlock(&hdcp_w->mutex); } void hdcp_handle_cpirq(struct hdcp_workqueue *hdcp_work, unsigned int link_index) @@ -277,7 +282,7 @@ static void event_callback(struct work_struct *work) hdcp_work = container_of(to_delayed_work(work), struct hdcp_workqueue, callback_dwork); - mutex_lock(&hdcp_work->mutex); + guard(mutex)(&hdcp_work->mutex); cancel_delayed_work(&hdcp_work->callback_dwork); @@ -285,8 +290,6 @@ static void event_callback(struct work_struct *work) &hdcp_work->output); process_output(hdcp_work); - - mutex_unlock(&hdcp_work->mutex); } static void event_property_update(struct work_struct *work) @@ -323,7 +326,7 @@ static void event_property_update(struct work_struct *work) continue; drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - mutex_lock(&hdcp_work->mutex); + guard(mutex)(&hdcp_work->mutex); if (conn_state->commit) { ret = wait_for_completion_interruptible_timeout(&conn_state->commit->hw_done, @@ -355,7 +358,6 @@ static void event_property_update(struct work_struct *work) drm_hdcp_update_content_protection(connector, DRM_MODE_CONTENT_PROTECTION_DESIRED); } - mutex_unlock(&hdcp_work->mutex); drm_modeset_unlock(&dev->mode_config.connection_mutex); } } @@ -368,7 +370,7 @@ static void event_property_validate(struct work_struct *work) struct amdgpu_dm_connector *aconnector; unsigned int conn_index; - mutex_lock(&hdcp_work->mutex); + guard(mutex)(&hdcp_work->mutex); for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; conn_index++) { @@ -408,8 +410,6 @@ static void event_property_validate(struct work_struct *work) schedule_work(&hdcp_work->property_update_work); } } - - mutex_unlock(&hdcp_work->mutex); } static void event_watchdog_timer(struct work_struct *work) @@ -420,7 +420,7 @@ static void event_watchdog_timer(struct work_struct *work) struct hdcp_workqueue, watchdog_timer_dwork); - mutex_lock(&hdcp_work->mutex); + guard(mutex)(&hdcp_work->mutex); cancel_delayed_work(&hdcp_work->watchdog_timer_dwork); @@ -429,8 +429,6 @@ static void event_watchdog_timer(struct work_struct *work) &hdcp_work->output); process_output(hdcp_work); - - mutex_unlock(&hdcp_work->mutex); } static void event_cpirq(struct work_struct *work) @@ -439,13 +437,11 @@ static void event_cpirq(struct work_struct *work) hdcp_work = container_of(work, struct hdcp_workqueue, cpirq_work); - mutex_lock(&hdcp_work->mutex); + guard(mutex)(&hdcp_work->mutex); mod_hdcp_process_event(&hdcp_work->hdcp, MOD_HDCP_EVENT_CPIRQ, &hdcp_work->output); process_output(hdcp_work); - - mutex_unlock(&hdcp_work->mutex); } void hdcp_destroy(struct kobject *kobj, struct hdcp_workqueue *hdcp_work) @@ -479,7 +475,7 @@ static bool enable_assr(void *handle, struct dc_link *link) dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.context.mem_context.shared_buf; - mutex_lock(&psp->dtm_context.mutex); + guard(mutex)(&psp->dtm_context.mutex); memset(dtm_cmd, 0, sizeof(struct ta_dtm_shared_memory)); dtm_cmd->cmd_id = TA_DTM_COMMAND__TOPOLOGY_ASSR_ENABLE; @@ -494,8 +490,6 @@ static bool enable_assr(void *handle, struct dc_link *link) res = false; } - mutex_unlock(&psp->dtm_context.mutex); - return res; } @@ -504,6 +498,7 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) struct hdcp_workqueue *hdcp_work = handle; struct amdgpu_dm_connector *aconnector = config->dm_stream_ctx; int link_index = aconnector->dc_link->link_index; + unsigned int conn_index = aconnector->base.index; struct mod_hdcp_display *display = &hdcp_work[link_index].display; struct mod_hdcp_link *link = &hdcp_work[link_index].link; struct hdcp_workqueue *hdcp_w = &hdcp_work[link_index]; @@ -557,13 +552,14 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) (!!aconnector->base.state) ? aconnector->base.state->hdcp_content_type : -1); - mutex_lock(&hdcp_w->mutex); + guard(mutex)(&hdcp_w->mutex); mod_hdcp_add_display(&hdcp_w->hdcp, link, display, &hdcp_w->output); - + drm_connector_get(&aconnector->base); + if (hdcp_w->aconnector[conn_index]) + drm_connector_put(&hdcp_w->aconnector[conn_index]->base); + hdcp_w->aconnector[conn_index] = aconnector; process_output(hdcp_w); - mutex_unlock(&hdcp_w->mutex); - } /** diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 5858e288b3fd6..2698e5c74ddfd 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -48,6 +48,9 @@ #define PEAK_FACTOR_X1000 1006 +/* + * This function handles both native AUX and I2C-Over-AUX transactions. + */ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) { @@ -56,6 +59,7 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, enum aux_return_code_type operation_result; struct amdgpu_device *adev; struct ddc_service *ddc; + uint8_t copy[16]; if (WARN_ON(msg->size > 16)) return -E2BIG; @@ -71,6 +75,11 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, (msg->request & DP_AUX_I2C_WRITE_STATUS_UPDATE) != 0; payload.defer_delay = 0; + if (payload.write) { + memcpy(copy, msg->buffer, msg->size); + payload.data = copy; + } + result = dc_link_aux_transfer_raw(TO_DM_AUX(aux)->ddc_service, &payload, &operation_result); @@ -84,15 +93,25 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, if (adev->dm.aux_hpd_discon_quirk) { if (msg->address == DP_SIDEBAND_MSG_DOWN_REQ_BASE && operation_result == AUX_RET_ERROR_HPD_DISCON) { - result = 0; + result = msg->size; operation_result = AUX_RET_SUCCESS; } } - if (payload.write && result >= 0) - result = msg->size; + /* + * result equals to 0 includes the cases of AUX_DEFER/I2C_DEFER + */ + if (payload.write && result >= 0) { + if (result) { + /*one byte indicating partially written bytes*/ + drm_dbg_dp(adev_to_drm(adev), "amdgpu: AUX partially written\n"); + result = payload.data[0]; + } else if (!payload.reply[0]) + /*I2C_ACK|AUX_ACK*/ + result = msg->size; + } - if (result < 0) + if (result < 0) { switch (operation_result) { case AUX_RET_SUCCESS: break; @@ -111,6 +130,13 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, break; } + drm_dbg_dp(adev_to_drm(adev), "amdgpu: DP AUX transfer fail:%d\n", operation_result); + } + + if (payload.reply[0]) + drm_dbg_dp(adev_to_drm(adev), "amdgpu: AUX reply command not ACK: 0x%02x.", + payload.reply[0]); + return result; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c index fe96bab7d05d7..67972d25366e5 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c @@ -124,8 +124,10 @@ bool amdgpu_dm_link_setup_psr(struct dc_stream_state *stream) psr_config.allow_multi_disp_optimizations = (amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT); - if (!psr_su_set_dsc_slice_height(dc, link, stream, &psr_config)) - return false; + if (link->psr_settings.psr_version == DC_PSR_VERSION_SU_1) { + if (!psr_su_set_dsc_slice_height(dc, link, stream, &psr_config)) + return false; + } ret = dc_link_setup_psr(link, stream, &psr_config, &psr_context); diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c index 818a529cacc37..12a54fabd80ec 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c @@ -993,7 +993,7 @@ static enum bp_result set_pixel_clock_v3( allocation.sPCLKInput.usFbDiv = cpu_to_le16((uint16_t)bp_params->feedback_divider); allocation.sPCLKInput.ucFracFbDiv = - (uint8_t)bp_params->fractional_feedback_divider; + (uint8_t)(bp_params->fractional_feedback_divider / 100000); allocation.sPCLKInput.ucPostDiv = (uint8_t)bp_params->pixel_clock_post_divider; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c index dcedf96451615..719881406f18e 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c @@ -162,7 +162,6 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p return NULL; } dce60_clk_mgr_construct(ctx, clk_mgr); - dce_clk_mgr_construct(ctx, clk_mgr); return &clk_mgr->base; } #endif diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c index 26feefbb8990a..b268c367c27cc 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c @@ -386,8 +386,6 @@ static void dce_pplib_apply_display_requirements( { struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg; - pp_display_cfg->avail_mclk_switch_time_us = dce110_get_min_vblank_time_us(context); - dce110_fill_display_configs(context, pp_display_cfg); if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) != 0) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c index 78df96882d6ec..fb2f154f4fda1 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c @@ -120,9 +120,15 @@ void dce110_fill_display_configs( const struct dc_state *context, struct dm_pp_display_configuration *pp_display_cfg) { + struct dc *dc = context->clk_mgr->ctx->dc; int j; int num_cfgs = 0; + pp_display_cfg->avail_mclk_switch_time_us = dce110_get_min_vblank_time_us(context); + pp_display_cfg->disp_clk_khz = dc->clk_mgr->clks.dispclk_khz; + pp_display_cfg->avail_mclk_switch_time_in_disp_active_us = 0; + pp_display_cfg->crtc_index = dc->res_pool->res_cap->num_timing_generator; + for (j = 0; j < context->stream_count; j++) { int k; @@ -164,6 +170,23 @@ void dce110_fill_display_configs( cfg->v_refresh /= stream->timing.h_total; cfg->v_refresh = (cfg->v_refresh + stream->timing.v_total / 2) / stream->timing.v_total; + + /* Find first CRTC index and calculate its line time. + * This is necessary for DPM on SI GPUs. + */ + if (cfg->pipe_idx < pp_display_cfg->crtc_index) { + const struct dc_crtc_timing *timing = + &context->streams[0]->timing; + + pp_display_cfg->crtc_index = cfg->pipe_idx; + pp_display_cfg->line_time_in_us = + timing->h_total * 10000 / timing->pix_clk_100hz; + } + } + + if (!num_cfgs) { + pp_display_cfg->crtc_index = 0; + pp_display_cfg->line_time_in_us = 0; } pp_display_cfg->display_count = num_cfgs; @@ -223,25 +246,8 @@ void dce11_pplib_apply_display_requirements( pp_display_cfg->min_engine_clock_deep_sleep_khz = context->bw_ctx.bw.dce.sclk_deep_sleep_khz; - pp_display_cfg->avail_mclk_switch_time_us = - dce110_get_min_vblank_time_us(context); - /* TODO: dce11.2*/ - pp_display_cfg->avail_mclk_switch_time_in_disp_active_us = 0; - - pp_display_cfg->disp_clk_khz = dc->clk_mgr->clks.dispclk_khz; - dce110_fill_display_configs(context, pp_display_cfg); - /* TODO: is this still applicable?*/ - if (pp_display_cfg->display_count == 1) { - const struct dc_crtc_timing *timing = - &context->streams[0]->timing; - - pp_display_cfg->crtc_index = - pp_display_cfg->disp_configs[0].pipe_idx; - pp_display_cfg->line_time_in_us = timing->h_total * 10000 / timing->pix_clk_100hz; - } - if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) != 0) dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg); } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c index 0267644717b27..ffd0f4a763102 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c @@ -83,22 +83,13 @@ static const struct state_dependent_clocks dce60_max_clks_by_state[] = { static int dce60_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base) { struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); - int dprefclk_wdivider; - int dp_ref_clk_khz; - int target_div; + struct dc_context *ctx = clk_mgr_base->ctx; + int dp_ref_clk_khz = 0; - /* DCE6 has no DPREFCLK_CNTL to read DP Reference Clock source */ - - /* Read the mmDENTIST_DISPCLK_CNTL to get the currently - * programmed DID DENTIST_DPREFCLK_WDIVIDER*/ - REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, &dprefclk_wdivider); - - /* Convert DENTIST_DPREFCLK_WDIVIDERto actual divider*/ - target_div = dentist_get_divider_from_did(dprefclk_wdivider); - - /* Calculate the current DFS clock, in kHz.*/ - dp_ref_clk_khz = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR - * clk_mgr->base.dentist_vco_freq_khz) / target_div; + if (ASIC_REV_IS_TAHITI_P(ctx->asic_id.hw_internal_rev)) + dp_ref_clk_khz = ctx->dc_bios->fw_info.default_display_engine_pll_frequency; + else + dp_ref_clk_khz = clk_mgr_base->clks.dispclk_khz; return dce_adjust_dp_ref_freq_for_ss(clk_mgr, dp_ref_clk_khz); } @@ -109,8 +100,6 @@ static void dce60_pplib_apply_display_requirements( { struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg; - pp_display_cfg->avail_mclk_switch_time_us = dce110_get_min_vblank_time_us(context); - dce110_fill_display_configs(context, pp_display_cfg); if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) != 0) @@ -123,11 +112,9 @@ static void dce60_update_clocks(struct clk_mgr *clk_mgr_base, { struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base); struct dm_pp_power_level_change_request level_change_req; - int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz; - - /*TODO: W/A for dal3 linux, investigate why this works */ - if (!clk_mgr_dce->dfs_bypass_active) - patched_disp_clk = patched_disp_clk * 115 / 100; + const int max_disp_clk = + clk_mgr_dce->max_clks_by_state[DM_PP_CLOCKS_STATE_PERFORMANCE].display_clk_khz; + int patched_disp_clk = min(max_disp_clk, context->bw_ctx.bw.dce.dispclk_khz); level_change_req.power_level = dce_get_required_clocks_state(clk_mgr_base, context); /* get max clock state from PPLIB */ diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c index a5489fe6875f4..086f60e1dd173 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c @@ -561,6 +561,7 @@ static void vg_clk_mgr_helper_populate_bw_params( { int i, j; struct clk_bw_params *bw_params = clk_mgr->base.bw_params; + uint32_t max_dispclk = 0, max_dppclk = 0; j = -1; @@ -581,6 +582,15 @@ static void vg_clk_mgr_helper_populate_bw_params( return; } + /* dispclk and dppclk can be max at any voltage, same number of levels for both */ + if (clock_table->NumDispClkLevelsEnabled <= VG_NUM_DISPCLK_DPM_LEVELS && + clock_table->NumDispClkLevelsEnabled <= VG_NUM_DPPCLK_DPM_LEVELS) { + max_dispclk = find_max_clk_value(clock_table->DispClocks, clock_table->NumDispClkLevelsEnabled); + max_dppclk = find_max_clk_value(clock_table->DppClocks, clock_table->NumDispClkLevelsEnabled); + } else { + ASSERT(0); + } + bw_params->clk_table.num_entries = j + 1; for (i = 0; i < bw_params->clk_table.num_entries - 1; i++, j--) { @@ -588,11 +598,17 @@ static void vg_clk_mgr_helper_populate_bw_params( bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[j].memclk; bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[j].voltage; bw_params->clk_table.entries[i].dcfclk_mhz = find_dcfclk_for_voltage(clock_table, clock_table->DfPstateTable[j].voltage); + + /* Now update clocks we do read */ + bw_params->clk_table.entries[i].dispclk_mhz = max_dispclk; + bw_params->clk_table.entries[i].dppclk_mhz = max_dppclk; } bw_params->clk_table.entries[i].fclk_mhz = clock_table->DfPstateTable[j].fclk; bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[j].memclk; bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[j].voltage; bw_params->clk_table.entries[i].dcfclk_mhz = find_max_clk_value(clock_table->DcfClocks, VG_NUM_DCFCLK_DPM_LEVELS); + bw_params->clk_table.entries[i].dispclk_mhz = find_max_clk_value(clock_table->DispClocks, VG_NUM_DISPCLK_DPM_LEVELS); + bw_params->clk_table.entries[i].dppclk_mhz = find_max_clk_value(clock_table->DppClocks, VG_NUM_DPPCLK_DPM_LEVELS); bw_params->vram_type = bios_info->memory_type; bw_params->num_channels = bios_info->ma_channel_number; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c index d4d3f58a613f7..327776eeb9f3e 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c @@ -130,7 +130,7 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk; struct dc *dc = clk_mgr_base->ctx->dc; - int display_count; + int display_count = 0; bool update_dppclk = false; bool update_dispclk = false; bool dpp_clock_lowered = false; @@ -194,8 +194,6 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, // workaround: Limit dppclk to 100Mhz to avoid lower eDP panel switch to plus 4K monitor underflow. if (new_clocks->dppclk_khz < MIN_DPP_DISP_CLK) new_clocks->dppclk_khz = MIN_DPP_DISP_CLK; - if (new_clocks->dispclk_khz < MIN_DPP_DISP_CLK) - new_clocks->dispclk_khz = MIN_DPP_DISP_CLK; if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->base.clks.dppclk_khz)) { if (clk_mgr->base.clks.dppclk_khz > new_clocks->dppclk_khz) @@ -204,15 +202,19 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, update_dppclk = true; } - if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) { - /* No need to apply the w/a if we haven't taken over from bios yet */ - if (clk_mgr_base->clks.dispclk_khz) - dcn315_disable_otg_wa(clk_mgr_base, context, true); + if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) && + (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) { + int requested_dispclk_khz = new_clocks->dispclk_khz; + dcn315_disable_otg_wa(clk_mgr_base, context, true); + + /* Clamp the requested clock to PMFW based on their limit. */ + if (dc->debug.min_disp_clk_khz > 0 && requested_dispclk_khz < dc->debug.min_disp_clk_khz) + requested_dispclk_khz = dc->debug.min_disp_clk_khz; + + dcn315_smu_set_dispclk(clk_mgr, requested_dispclk_khz); clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz; - dcn315_smu_set_dispclk(clk_mgr, clk_mgr_base->clks.dispclk_khz); - if (clk_mgr_base->clks.dispclk_khz) - dcn315_disable_otg_wa(clk_mgr_base, context, false); + dcn315_disable_otg_wa(clk_mgr_base, context, false); update_dispclk = true; } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c index a13ead3d21e31..f95e5e767eb1a 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c @@ -140,7 +140,7 @@ static void dcn316_update_clocks(struct clk_mgr *clk_mgr_base, struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk; struct dc *dc = clk_mgr_base->ctx->dc; - int display_count; + int display_count = 0; bool update_dppclk = false; bool update_dispclk = false; bool dpp_clock_lowered = false; @@ -201,8 +201,6 @@ static void dcn316_update_clocks(struct clk_mgr *clk_mgr_base, // workaround: Limit dppclk to 100Mhz to avoid lower eDP panel switch to plus 4K monitor underflow. if (new_clocks->dppclk_khz < 100000) new_clocks->dppclk_khz = 100000; - if (new_clocks->dispclk_khz < 100000) - new_clocks->dispclk_khz = 100000; if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->base.clks.dppclk_khz)) { if (clk_mgr->base.clks.dppclk_khz > new_clocks->dppclk_khz) @@ -211,11 +209,18 @@ static void dcn316_update_clocks(struct clk_mgr *clk_mgr_base, update_dppclk = true; } - if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) { + if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) && + (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) { + int requested_dispclk_khz = new_clocks->dispclk_khz; + dcn316_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true); + /* Clamp the requested clock to PMFW based on their limit. */ + if (dc->debug.min_disp_clk_khz > 0 && requested_dispclk_khz < dc->debug.min_disp_clk_khz) + requested_dispclk_khz = dc->debug.min_disp_clk_khz; + + dcn316_smu_set_dispclk(clk_mgr, requested_dispclk_khz); clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz; - dcn316_smu_set_dispclk(clk_mgr, clk_mgr_base->clks.dispclk_khz); dcn316_disable_otg_wa(clk_mgr_base, context, safe_to_lower, false); update_dispclk = true; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index c2efe18ceacd0..31c53491b8374 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -266,6 +266,7 @@ static bool create_links( link->link_id.type = OBJECT_TYPE_CONNECTOR; link->link_id.id = CONNECTOR_ID_VIRTUAL; link->link_id.enum_id = ENUM_ID_1; + link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED; link->link_enc = kzalloc(sizeof(*link->link_enc), GFP_KERNEL); if (!link->link_enc) { @@ -2067,6 +2068,18 @@ enum dc_status dc_commit_streams(struct dc *dc, goto fail; } + /* + * If not already seamless, make transition seamless by inserting intermediate minimal transition + */ + if (dc->hwss.is_pipe_topology_transition_seamless && + !dc->hwss.is_pipe_topology_transition_seamless(dc, dc->current_state, context)) { + res = commit_minimal_transition_state(dc, context); + if (res != DC_OK) { + BREAK_TO_DEBUGGER(); + goto fail; + } + } + res = dc_commit_state_no_check(dc, context); for (i = 0; i < stream_count; i++) { @@ -2939,6 +2952,9 @@ static void copy_stream_update_to_stream(struct dc *dc, if (update->adaptive_sync_infopacket) stream->adaptive_sync_infopacket = *update->adaptive_sync_infopacket; + if (update->avi_infopacket) + stream->avi_infopacket = *update->avi_infopacket; + if (update->dither_option) stream->dither_option = *update->dither_option; @@ -3145,7 +3161,8 @@ static void commit_planes_do_stream_update(struct dc *dc, stream_update->vsp_infopacket || stream_update->hfvsif_infopacket || stream_update->adaptive_sync_infopacket || - stream_update->vtem_infopacket) { + stream_update->vtem_infopacket || + stream_update->avi_infopacket) { resource_build_info_frame(pipe_ctx); dc->hwss.update_info_frame(pipe_ctx); @@ -4228,6 +4245,7 @@ static bool full_update_required(struct dc *dc, stream_update->hfvsif_infopacket || stream_update->vtem_infopacket || stream_update->adaptive_sync_infopacket || + stream_update->avi_infopacket || stream_update->dpms_off || stream_update->allow_freesync || stream_update->vrr_active_variable || diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 2f25f7096c4d6..802c0e19d03b3 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -150,7 +150,13 @@ enum dce_version resource_parse_asic_id(struct hw_asic_id asic_id) case FAMILY_NV: dc_version = DCN_VERSION_2_0; - if (asic_id.chip_id == DEVICE_ID_NV_13FE || asic_id.chip_id == DEVICE_ID_NV_143F) { + if (asic_id.chip_id == DEVICE_ID_NV_13FE || + asic_id.chip_id == DEVICE_ID_NV_143F || + asic_id.chip_id == DEVICE_ID_NV_13F9 || + asic_id.chip_id == DEVICE_ID_NV_13FA || + asic_id.chip_id == DEVICE_ID_NV_13FB || + asic_id.chip_id == DEVICE_ID_NV_13FC || + asic_id.chip_id == DEVICE_ID_NV_13DB) { dc_version = DCN_VERSION_2_01; break; } @@ -3219,8 +3225,14 @@ static void set_avi_info_frame( unsigned int fr_ind = pipe_ctx->stream->timing.fr_index; enum dc_timing_3d_format format; + if (stream->avi_infopacket.valid) { + *info_packet = stream->avi_infopacket; + return; + } + memset(&hdmi_info, 0, sizeof(union hdmi_info_packet)); + color_space = pipe_ctx->stream->output_color_space; if (color_space == COLOR_SPACE_UNKNOWN) color_space = (stream->timing.pixel_encoding == PIXEL_ENCODING_RGB) ? diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c index 3907eeff560ce..0713a503f7f6d 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_helper.c +++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c @@ -744,3 +744,8 @@ char *dce_version_to_string(const int version) return "Unknown"; } } + +bool dc_supports_vrr(const enum dce_version v) +{ + return v >= DCE_VERSION_8_0; +} diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index d5b3e3a32cc6d..ad020cc246376 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -197,6 +197,7 @@ struct dc_stream_state { struct dc_info_packet hfvsif_infopacket; struct dc_info_packet vtem_infopacket; struct dc_info_packet adaptive_sync_infopacket; + struct dc_info_packet avi_infopacket; uint8_t dsc_packed_pps[128]; struct rect src; /* composition area */ struct rect dst; /* stream addressable area */ @@ -323,6 +324,8 @@ struct dc_stream_update { struct dc_info_packet *hfvsif_infopacket; struct dc_info_packet *vtem_infopacket; struct dc_info_packet *adaptive_sync_infopacket; + struct dc_info_packet *avi_infopacket; + bool *dpms_off; bool integer_scaling_update; bool *allow_freesync; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 136bd93c3b655..0a33f8f117e92 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -896,13 +896,13 @@ void dce110_link_encoder_construct( enc110->base.id, &bp_cap_info); /* Override features with DCE-specific values */ - if (BP_RESULT_OK == result) { + if (result == BP_RESULT_OK) { enc110->base.features.flags.bits.IS_HBR2_CAPABLE = bp_cap_info.DP_HBR2_EN; enc110->base.features.flags.bits.IS_HBR3_CAPABLE = bp_cap_info.DP_HBR3_EN; enc110->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN; - } else { + } else if (result != BP_RESULT_NORECORD) { DC_LOG_WARNING("%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n", __func__, result); @@ -1795,13 +1795,13 @@ void dce60_link_encoder_construct( enc110->base.id, &bp_cap_info); /* Override features with DCE-specific values */ - if (BP_RESULT_OK == result) { + if (result == BP_RESULT_OK) { enc110->base.features.flags.bits.IS_HBR2_CAPABLE = bp_cap_info.DP_HBR2_EN; enc110->base.features.flags.bits.IS_HBR3_CAPABLE = bp_cap_info.DP_HBR3_EN; enc110->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN; - } else { + } else if (result != BP_RESULT_NORECORD) { DC_LOG_WARNING("%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n", __func__, result); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c index 670d5ab9d9984..f97c182677082 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c @@ -154,10 +154,13 @@ static bool dce60_setup_scaling_configuration( REG_SET(SCL_BYPASS_CONTROL, 0, SCL_BYPASS_MODE, 0); if (data->taps.h_taps + data->taps.v_taps <= 2) { - /* Set bypass */ - - /* DCE6 has no SCL_MODE register, skip scale mode programming */ + /* Disable scaler functionality */ + REG_WRITE(SCL_SCALER_ENABLE, 0); + /* Clear registers that can cause glitches even when the scaler is off */ + REG_WRITE(SCL_TAP_CONTROL, 0); + REG_WRITE(SCL_AUTOMATIC_MODE_CONTROL, 0); + REG_WRITE(SCL_F_SHARP_CONTROL, 0); return false; } @@ -165,7 +168,7 @@ static bool dce60_setup_scaling_configuration( SCL_H_NUM_OF_TAPS, data->taps.h_taps - 1, SCL_V_NUM_OF_TAPS, data->taps.v_taps - 1); - /* DCE6 has no SCL_MODE register, skip scale mode programming */ + REG_WRITE(SCL_SCALER_ENABLE, 1); /* DCE6 has no SCL_BOUNDARY_MODE bit, skip replace out of bound pixels */ @@ -502,6 +505,8 @@ static void dce60_transform_set_scaler( REG_SET(DC_LB_MEM_SIZE, 0, DC_LB_MEM_SIZE, xfm_dce->lb_memory_size); + REG_WRITE(SCL_UPDATE, 0x00010000); + /* Clear SCL_F_SHARP_CONTROL value to 0 */ REG_WRITE(SCL_F_SHARP_CONTROL, 0); @@ -527,8 +532,7 @@ static void dce60_transform_set_scaler( if (coeffs_v != xfm_dce->filter_v || coeffs_h != xfm_dce->filter_h) { /* 4. Program vertical filters */ if (xfm_dce->filter_v == NULL) - REG_SET(SCL_VERT_FILTER_CONTROL, 0, - SCL_V_2TAP_HARDCODE_COEF_EN, 0); + REG_WRITE(SCL_VERT_FILTER_CONTROL, 0); program_multi_taps_filter( xfm_dce, data->taps.v_taps, @@ -542,8 +546,7 @@ static void dce60_transform_set_scaler( /* 5. Program horizontal filters */ if (xfm_dce->filter_h == NULL) - REG_SET(SCL_HORZ_FILTER_CONTROL, 0, - SCL_H_2TAP_HARDCODE_COEF_EN, 0); + REG_WRITE(SCL_HORZ_FILTER_CONTROL, 0); program_multi_taps_filter( xfm_dce, data->taps.h_taps, @@ -566,6 +569,8 @@ static void dce60_transform_set_scaler( /* DCE6 has no SCL_COEF_UPDATE_COMPLETE bit to flip to new coefficient memory */ /* DCE6 DATA_FORMAT register does not support ALPHA_EN */ + + REG_WRITE(SCL_UPDATE, 0); } #endif diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h index cbce194ec7b82..eb716e8337e23 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h @@ -155,6 +155,9 @@ SRI(SCL_COEF_RAM_TAP_DATA, SCL, id), \ SRI(VIEWPORT_START, SCL, id), \ SRI(VIEWPORT_SIZE, SCL, id), \ + SRI(SCL_SCALER_ENABLE, SCL, id), \ + SRI(SCL_HORZ_FILTER_INIT_RGB_LUMA, SCL, id), \ + SRI(SCL_HORZ_FILTER_INIT_CHROMA, SCL, id), \ SRI(SCL_HORZ_FILTER_SCALE_RATIO, SCL, id), \ SRI(SCL_VERT_FILTER_SCALE_RATIO, SCL, id), \ SRI(SCL_VERT_FILTER_INIT, SCL, id), \ @@ -590,6 +593,7 @@ struct dce_transform_registers { uint32_t SCL_VERT_FILTER_SCALE_RATIO; uint32_t SCL_HORZ_FILTER_INIT; #if defined(CONFIG_DRM_AMD_DC_SI) + uint32_t SCL_SCALER_ENABLE; uint32_t SCL_HORZ_FILTER_INIT_RGB_LUMA; uint32_t SCL_HORZ_FILTER_INIT_CHROMA; #endif diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index 7b5c1498941dd..d389eeb264a79 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -1066,7 +1066,8 @@ void dce110_edp_backlight_control( DC_LOG_DC("edp_receiver_ready_T9 skipped\n"); } - if (!enable && link->dpcd_sink_ext_caps.bits.oled) { + if (!enable) { + /*follow oem panel config's requirement*/ pre_T11_delay += link->panel_config.pps.extra_pre_t11_ms; msleep(pre_T11_delay); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index a825fd6c7fa66..f0b472e84a53d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -163,14 +163,13 @@ static void dcn20_setup_gsl_group_as_lock( } /* at this point we want to program whether it's to enable or disable */ - if (pipe_ctx->stream_res.tg->funcs->set_gsl != NULL && - pipe_ctx->stream_res.tg->funcs->set_gsl_source_select != NULL) { + if (pipe_ctx->stream_res.tg->funcs->set_gsl != NULL) { pipe_ctx->stream_res.tg->funcs->set_gsl( pipe_ctx->stream_res.tg, &gsl); - - pipe_ctx->stream_res.tg->funcs->set_gsl_source_select( - pipe_ctx->stream_res.tg, group_idx, enable ? 4 : 0); + if (pipe_ctx->stream_res.tg->funcs->set_gsl_source_select != NULL) + pipe_ctx->stream_res.tg->funcs->set_gsl_source_select( + pipe_ctx->stream_res.tg, group_idx, enable ? 4 : 0); } else BREAK_TO_DEBUGGER(); } @@ -782,7 +781,7 @@ enum dc_status dcn20_enable_stream_timing( return DC_ERROR_UNEXPECTED; } - hws->funcs.wait_for_blank_complete(pipe_ctx->stream_res.opp); + fsleep(stream->timing.v_total * (stream->timing.h_total * 10000u / stream->timing.pix_clk_100hz)); params.vertical_total_min = stream->adjust.v_total_min; params.vertical_total_max = stream->adjust.v_total_max; diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c index 50dc834046446..4ce45f1bdac0f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c @@ -392,11 +392,6 @@ bool dpp3_get_optimal_number_of_taps( int min_taps_y, min_taps_c; enum lb_memory_config lb_config; - if (scl_data->viewport.width > scl_data->h_active && - dpp->ctx->dc->debug.max_downscale_src_width != 0 && - scl_data->viewport.width > dpp->ctx->dc->debug.max_downscale_src_width) - return false; - /* * Set default taps if none are provided * From programming guide: taps = min{ ceil(2*H_RATIO,1), 8} for downscaling @@ -434,6 +429,12 @@ bool dpp3_get_optimal_number_of_taps( else scl_data->taps.h_taps_c = in_taps->h_taps_c; + // Avoid null data in the scl data with this early return, proceed non-adaptive calcualtion first + if (scl_data->viewport.width > scl_data->h_active && + dpp->ctx->dc->debug.max_downscale_src_width != 0 && + scl_data->viewport.width > dpp->ctx->dc->debug.max_downscale_src_width) + return false; + /*Ensure we can support the requested number of vtaps*/ min_taps_y = dc_fixpt_ceil(scl_data->ratios.vert); min_taps_c = dc_fixpt_ceil(scl_data->ratios.vert_c); diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c index 597fa0364a3a9..d1601d61f05ad 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c @@ -1692,7 +1692,7 @@ static int dcn315_populate_dml_pipes_from_context( pipes[pipe_cnt].dout.dsc_input_bpc = 0; DC_FP_START(); dcn31_zero_pipe_dcc_fraction(pipes, pipe_cnt); - if (pixel_rate_crb && !pipe->top_pipe && !pipe->prev_odm_pipe) { + if (pixel_rate_crb) { int bpp = source_format_to_bpp(pipes[pipe_cnt].pipe.src.source_format); /* Ceil to crb segment size */ int approx_det_segs_required_for_pstate = dcn_get_approx_det_segs_required_for_pstate( @@ -1749,28 +1749,26 @@ static int dcn315_populate_dml_pipes_from_context( continue; } - if (!pipe->top_pipe && !pipe->prev_odm_pipe) { - bool split_required = pipe->stream->timing.pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc) - || (pipe->plane_state && pipe->plane_state->src_rect.width > 5120); - - if (remaining_det_segs > MIN_RESERVED_DET_SEGS && crb_pipes != 0) - pipes[pipe_cnt].pipe.src.det_size_override += (remaining_det_segs - MIN_RESERVED_DET_SEGS) / crb_pipes + - (crb_idx < (remaining_det_segs - MIN_RESERVED_DET_SEGS) % crb_pipes ? 1 : 0); - if (pipes[pipe_cnt].pipe.src.det_size_override > 2 * DCN3_15_MAX_DET_SEGS) { - /* Clamp to 2 pipe split max det segments */ - remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override - 2 * (DCN3_15_MAX_DET_SEGS); - pipes[pipe_cnt].pipe.src.det_size_override = 2 * DCN3_15_MAX_DET_SEGS; - } - if (pipes[pipe_cnt].pipe.src.det_size_override > DCN3_15_MAX_DET_SEGS || split_required) { - /* If we are splitting we must have an even number of segments */ - remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override % 2; - pipes[pipe_cnt].pipe.src.det_size_override -= pipes[pipe_cnt].pipe.src.det_size_override % 2; - } - /* Convert segments into size for DML use */ - pipes[pipe_cnt].pipe.src.det_size_override *= DCN3_15_CRB_SEGMENT_SIZE_KB; - - crb_idx++; + bool split_required = pipe->stream->timing.pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc) + || (pipe->plane_state && pipe->plane_state->src_rect.width > 5120); + + if (remaining_det_segs > MIN_RESERVED_DET_SEGS && crb_pipes != 0) + pipes[pipe_cnt].pipe.src.det_size_override += (remaining_det_segs - MIN_RESERVED_DET_SEGS) / crb_pipes + + (crb_idx < (remaining_det_segs - MIN_RESERVED_DET_SEGS) % crb_pipes ? 1 : 0); + if (pipes[pipe_cnt].pipe.src.det_size_override > 2 * DCN3_15_MAX_DET_SEGS) { + /* Clamp to 2 pipe split max det segments */ + remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override - 2 * (DCN3_15_MAX_DET_SEGS); + pipes[pipe_cnt].pipe.src.det_size_override = 2 * DCN3_15_MAX_DET_SEGS; + } + if (pipes[pipe_cnt].pipe.src.det_size_override > DCN3_15_MAX_DET_SEGS || split_required) { + /* If we are splitting we must have an even number of segments */ + remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override % 2; + pipes[pipe_cnt].pipe.src.det_size_override -= pipes[pipe_cnt].pipe.src.det_size_override % 2; } + /* Convert segments into size for DML use */ + pipes[pipe_cnt].pipe.src.det_size_override *= DCN3_15_CRB_SEGMENT_SIZE_KB; + + crb_idx++; pipe_cnt++; } } diff --git a/drivers/gpu/drm/amd/display/dc/dm_services.h b/drivers/gpu/drm/amd/display/dc/dm_services.h index d0eed3b4771e6..f2ab2c42781a4 100644 --- a/drivers/gpu/drm/amd/display/dc/dm_services.h +++ b/drivers/gpu/drm/amd/display/dc/dm_services.h @@ -294,4 +294,6 @@ void dm_dtn_log_end(struct dc_context *ctx, char *dce_version_to_string(const int version); +bool dc_supports_vrr(const enum dce_version v); + #endif /* __DM_SERVICES_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c index 6ce90678b33c0..dc7435d5ef4df 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c @@ -326,7 +326,7 @@ void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param struct dcn301_resource_pool *pool = TO_DCN301_RES_POOL(dc->res_pool); struct clk_limit_table *clk_table = &bw_params->clk_table; unsigned int i, closest_clk_lvl; - int j; + int j = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0; dc_assert_fp_enabled(); @@ -338,6 +338,15 @@ void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param dcn3_01_soc.num_chans = bw_params->num_channels; ASSERT(clk_table->num_entries); + + /* Prepass to find max clocks independent of voltage level. */ + for (i = 0; i < clk_table->num_entries; ++i) { + if (clk_table->entries[i].dispclk_mhz > max_dispclk_mhz) + max_dispclk_mhz = clk_table->entries[i].dispclk_mhz; + if (clk_table->entries[i].dppclk_mhz > max_dppclk_mhz) + max_dppclk_mhz = clk_table->entries[i].dppclk_mhz; + } + for (i = 0; i < clk_table->num_entries; i++) { /* loop backwards*/ for (closest_clk_lvl = 0, j = dcn3_01_soc.num_states - 1; j >= 0; j--) { @@ -353,8 +362,13 @@ void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param s[i].socclk_mhz = clk_table->entries[i].socclk_mhz; s[i].dram_speed_mts = clk_table->entries[i].memclk_mhz * 2; - s[i].dispclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dispclk_mhz; - s[i].dppclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dppclk_mhz; + /* Clocks independent of voltage level. */ + s[i].dispclk_mhz = max_dispclk_mhz ? max_dispclk_mhz : + dcn3_01_soc.clock_limits[closest_clk_lvl].dispclk_mhz; + + s[i].dppclk_mhz = max_dppclk_mhz ? max_dppclk_mhz : + dcn3_01_soc.clock_limits[closest_clk_lvl].dppclk_mhz; + s[i].dram_bw_per_chan_gbps = dcn3_01_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps; s[i].dscclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dscclk_mhz; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c index 9ba6cb67655f4..6c75aa82327ac 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c @@ -139,7 +139,6 @@ void dml32_rq_dlg_get_rq_reg(display_rq_regs_st *rq_regs, if (dual_plane) { unsigned int p1_pte_row_height_linear = get_dpte_row_height_linear_c(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); - ; if (src->sw_mode == dm_sw_linear) ASSERT(p1_pte_row_height_linear >= 8); diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h index eaad1260bfd18..4b284ce669ae5 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h +++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h @@ -532,7 +532,7 @@ struct dc_state { */ struct bw_context bw_ctx; - struct block_sequence block_sequence[50]; + struct block_sequence block_sequence[100]; unsigned int block_sequence_steps; struct dc_dmub_cmd dc_dmub_cmd[10]; unsigned int dmub_cmd_count; diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c index c7a9e286a5d4d..2d98f539a100a 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c @@ -1113,6 +1113,11 @@ static bool detect_link_and_local_sink(struct dc_link *link, if (sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A && !sink->edid_caps.edid_hdmi) sink->sink_signal = SIGNAL_TYPE_DVI_SINGLE_LINK; + else if (dc_is_dvi_signal(sink->sink_signal) && + dc_is_dvi_signal(link->connector_signal) && + aud_support->hdmi_audio_native && + sink->edid_caps.edid_hdmi) + sink->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A; if (link->local_sink && dc_is_dp_signal(sink_caps.signal)) dp_trace_init(link); diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index 4901e27f678bc..2ce2d9ff7568a 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -137,7 +137,8 @@ void link_blank_dp_stream(struct dc_link *link, bool hw_init) } } - if ((!link->wa_flags.dp_keep_receiver_powered) || hw_init) + if (((!link->wa_flags.dp_keep_receiver_powered) || hw_init) && + (link->type != dc_connection_none)) dpcd_write_rx_power_ctrl(link, false); } } @@ -145,6 +146,7 @@ void link_blank_dp_stream(struct dc_link *link, bool hw_init) void link_set_all_streams_dpms_off_for_link(struct dc_link *link) { struct pipe_ctx *pipes[MAX_PIPES]; + struct dc_stream_state *streams[MAX_PIPES]; struct dc_state *state = link->dc->current_state; uint8_t count; int i; @@ -157,10 +159,18 @@ void link_set_all_streams_dpms_off_for_link(struct dc_link *link) link_get_master_pipes_with_dpms_on(link, state, &count, pipes); + /* The subsequent call to dc_commit_updates_for_stream for a full update + * will release the current state and swap to a new state. Releasing the + * current state results in the stream pointers in the pipe_ctx structs + * to be zero'd. Hence, cache all streams prior to dc_commit_updates_for_stream. + */ + for (i = 0; i < count; i++) + streams[i] = pipes[i]->stream; + for (i = 0; i < count; i++) { - stream_update.stream = pipes[i]->stream; + stream_update.stream = streams[i]; dc_commit_updates_for_stream(link->ctx->dc, NULL, 0, - pipes[i]->stream, &stream_update, + streams[i], &stream_update, state); } diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c index 3d589072fe307..56999657f4d86 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c @@ -239,21 +239,21 @@ static uint32_t intersect_frl_link_bw_support( { uint32_t supported_bw_in_kbps = max_supported_frl_bw_in_kbps; - // HDMI_ENCODED_LINK_BW bits are only valid if HDMI Link Configuration bit is 1 (FRL mode) - if (hdmi_encoded_link_bw.bits.FRL_MODE) { - if (hdmi_encoded_link_bw.bits.BW_48Gbps) - supported_bw_in_kbps = 48000000; - else if (hdmi_encoded_link_bw.bits.BW_40Gbps) - supported_bw_in_kbps = 40000000; - else if (hdmi_encoded_link_bw.bits.BW_32Gbps) - supported_bw_in_kbps = 32000000; - else if (hdmi_encoded_link_bw.bits.BW_24Gbps) - supported_bw_in_kbps = 24000000; - else if (hdmi_encoded_link_bw.bits.BW_18Gbps) - supported_bw_in_kbps = 18000000; - else if (hdmi_encoded_link_bw.bits.BW_9Gbps) - supported_bw_in_kbps = 9000000; - } + /* Skip checking FRL_MODE bit, as certain PCON will clear + * it despite supporting the link BW indicated in the other bits. + */ + if (hdmi_encoded_link_bw.bits.BW_48Gbps) + supported_bw_in_kbps = 48000000; + else if (hdmi_encoded_link_bw.bits.BW_40Gbps) + supported_bw_in_kbps = 40000000; + else if (hdmi_encoded_link_bw.bits.BW_32Gbps) + supported_bw_in_kbps = 32000000; + else if (hdmi_encoded_link_bw.bits.BW_24Gbps) + supported_bw_in_kbps = 24000000; + else if (hdmi_encoded_link_bw.bits.BW_18Gbps) + supported_bw_in_kbps = 18000000; + else if (hdmi_encoded_link_bw.bits.BW_9Gbps) + supported_bw_in_kbps = 9000000; return supported_bw_in_kbps; } @@ -920,6 +920,9 @@ bool link_decide_link_settings(struct dc_stream_state *stream, * TODO: add MST specific link training routine */ decide_mst_link_settings(link, link_setting); + } else if (stream->signal == SIGNAL_TYPE_VIRTUAL) { + link_setting->lane_count = LANE_COUNT_FOUR; + link_setting->link_rate = LINK_RATE_HIGH3; } else if (link->connector_signal == SIGNAL_TYPE_EDP) { /* enable edp link optimization for DSC eDP case */ if (stream->timing.flags.DSC) { @@ -1549,7 +1552,7 @@ static bool retrieve_link_cap(struct dc_link *link) union edp_configuration_cap edp_config_cap; union dp_downstream_port_present ds_port = { 0 }; enum dc_status status = DC_ERROR_UNEXPECTED; - uint32_t read_dpcd_retry_cnt = 3; + uint32_t read_dpcd_retry_cnt = 20; int i; struct dp_sink_hw_fw_revision dp_hw_fw_revision; const uint32_t post_oui_delay = 30; // 30ms @@ -1595,12 +1598,13 @@ static bool retrieve_link_cap(struct dc_link *link) status = dpcd_get_tunneling_device_data(link); dpcd_set_source_specific_data(link); - /* Sink may need to configure internals based on vendor, so allow some - * time before proceeding with possibly vendor specific transactions - */ - msleep(post_oui_delay); for (i = 0; i < read_dpcd_retry_cnt; i++) { + /* + * Sink may need to configure internals based on vendor, so allow some + * time before proceeding with possibly vendor specific transactions + */ + msleep(post_oui_delay); status = core_link_read_dpcd( link, DP_DPCD_REV, diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c index 9bde0c8bf914a..f01a3df584552 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c @@ -74,7 +74,8 @@ void dp_disable_link_phy(struct dc_link *link, struct dc *dc = link->ctx->dc; if (!link->wa_flags.dp_keep_receiver_powered && - !link->skip_implict_edp_power_control) + !link->skip_implict_edp_power_control && + link->type != dc_connection_none) dpcd_write_rx_power_ctrl(link, false); dc->hwss.disable_link_output(link, link_res, signal); @@ -159,8 +160,9 @@ enum dc_status dp_set_fec_ready(struct dc_link *link, const struct link_resource } else { if (link->fec_state == dc_link_fec_ready) { fec_config = 0; - core_link_write_dpcd(link, DP_FEC_CONFIGURATION, - &fec_config, sizeof(fec_config)); + if (link->type != dc_connection_none) + core_link_write_dpcd(link, DP_FEC_CONFIGURATION, + &fec_config, sizeof(fec_config)); link_enc->funcs->fec_set_ready(link_enc, false); link->fec_state = dc_link_fec_not_ready; diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c index 51e88efee11e4..08c2f11724140 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c @@ -974,14 +974,19 @@ void repeater_training_done(struct dc_link *link, uint32_t offset) static void dpcd_exit_training_mode(struct dc_link *link, enum dp_link_encoding encoding) { uint8_t sink_status = 0; - uint8_t i; + uint32_t i; + uint8_t lttpr_count = dp_parse_lttpr_repeater_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt); + uint32_t intra_hop_disable_time_ms = (lttpr_count > 0 ? lttpr_count * 300 : 10); + + // Each hop could theoretically take over 256ms (max 128b/132b AUX RD INTERVAL) + // To be safe, allow 300ms per LTTPR and 10ms for no LTTPR case /* clear training pattern set */ dpcd_set_training_pattern(link, DP_TRAINING_PATTERN_VIDEOIDLE); if (encoding == DP_128b_132b_ENCODING) { /* poll for intra-hop disable */ - for (i = 0; i < 10; i++) { + for (i = 0; i < intra_hop_disable_time_ms; i++) { if ((core_link_read_dpcd(link, DP_SINK_STATUS, &sink_status, 1) == DC_OK) && (sink_status & DP_INTRA_HOP_AUX_REPLY_INDICATION) == 0) break; diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c index 2b4c15b0b4070..52261e7c11c0b 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c @@ -36,7 +36,8 @@ link->ctx->logger static int32_t get_cr_training_aux_rd_interval(struct dc_link *link, - const struct dc_link_settings *link_settings) + const struct dc_link_settings *link_settings, + enum lttpr_mode lttpr_mode) { union training_aux_rd_interval training_rd_interval; uint32_t wait_in_micro_secs = 100; @@ -49,6 +50,8 @@ static int32_t get_cr_training_aux_rd_interval(struct dc_link *link, DP_TRAINING_AUX_RD_INTERVAL, (uint8_t *)&training_rd_interval, sizeof(training_rd_interval)); + if (lttpr_mode != LTTPR_MODE_NON_TRANSPARENT) + wait_in_micro_secs = 400; if (training_rd_interval.bits.TRAINIG_AUX_RD_INTERVAL) wait_in_micro_secs = training_rd_interval.bits.TRAINIG_AUX_RD_INTERVAL * 4000; } @@ -110,7 +113,6 @@ void decide_8b_10b_training_settings( */ lt_settings->link_settings.link_spread = link->dp_ss_off ? LINK_SPREAD_DISABLED : LINK_SPREAD_05_DOWNSPREAD_30KHZ; - lt_settings->cr_pattern_time = get_cr_training_aux_rd_interval(link, link_setting); lt_settings->eq_pattern_time = get_eq_training_aux_rd_interval(link, link_setting); lt_settings->pattern_for_cr = decide_cr_training_pattern(link_setting); lt_settings->pattern_for_eq = decide_eq_training_pattern(link, link_setting); @@ -119,6 +121,7 @@ void decide_8b_10b_training_settings( lt_settings->disallow_per_lane_settings = true; lt_settings->always_match_dpcd_with_hw_lane_settings = true; lt_settings->lttpr_mode = dp_decide_8b_10b_lttpr_mode(link); + lt_settings->cr_pattern_time = get_cr_training_aux_rd_interval(link, link_setting, lt_settings->lttpr_mode); dp_hw_to_dpcd_lane_settings(lt_settings, lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings); } diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c index 13104d000b9e0..d4d92da153ece 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c @@ -662,6 +662,18 @@ bool edp_setup_psr(struct dc_link *link, if (!link) return false; + //Clear PSR cfg + memset(&psr_configuration, 0, sizeof(psr_configuration)); + dm_helpers_dp_write_dpcd( + link->ctx, + link, + DP_PSR_EN_CFG, + &psr_configuration.raw, + sizeof(psr_configuration.raw)); + + if (link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED) + return false; + dc = link->ctx->dc; dmcu = dc->res_pool->dmcu; psr = dc->res_pool->psr; @@ -672,9 +684,6 @@ bool edp_setup_psr(struct dc_link *link, if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst)) return false; - - memset(&psr_configuration, 0, sizeof(psr_configuration)); - psr_configuration.bits.ENABLE = 1; psr_configuration.bits.CRC_VERIFICATION = 1; psr_configuration.bits.FRAME_CAPTURE_INDICATION = @@ -938,6 +947,16 @@ bool edp_setup_replay(struct dc_link *link, const struct dc_stream_state *stream if (!link) return false; + //Clear Replay config + dm_helpers_dp_write_dpcd(link->ctx, link, + DP_SINK_PR_ENABLE_AND_CONFIGURATION, + (uint8_t *)&(replay_config.raw), sizeof(uint8_t)); + + if (!(link->replay_settings.config.replay_supported)) + return false; + + link->replay_settings.config.replay_error_status.raw = 0; + dc = link->ctx->dc; replay = dc->res_pool->replay; diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h index e317089cf6ee7..b913d08a5e038 100644 --- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h +++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h @@ -213,6 +213,11 @@ enum { #endif #define DEVICE_ID_NV_13FE 0x13FE // CYAN_SKILLFISH #define DEVICE_ID_NV_143F 0x143F +#define DEVICE_ID_NV_13F9 0x13F9 +#define DEVICE_ID_NV_13FA 0x13FA +#define DEVICE_ID_NV_13FB 0x13FB +#define DEVICE_ID_NV_13FC 0x13FC +#define DEVICE_ID_NV_13DB 0x13DB #define FAMILY_VGH 144 #define DEVICE_ID_VGH_163F 0x163F #define DEVICE_ID_VGH_1435 0x1435 diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c index 1b2df97226a3f..40286e8dd4e1a 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c @@ -25,7 +25,9 @@ #include "hdcp.h" +#ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif #define HDCP_I2C_ADDR 0x3a /* 0x74 >> 1*/ #define KSV_READ_SIZE 0xf /* 0x6803b - 0x6802c */ #define HDCP_MAX_AUX_TRANSACTION_SIZE 16 diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c index ff930a71e496a..ab6964ca1c2b4 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c @@ -260,6 +260,9 @@ enum mod_hdcp_status mod_hdcp_hdcp1_create_session(struct mod_hdcp *hdcp) return MOD_HDCP_STATUS_FAILURE; } + if (!display) + return MOD_HDCP_STATUS_DISPLAY_NOT_FOUND; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; mutex_lock(&psp->hdcp_context.mutex); @@ -368,6 +371,9 @@ enum mod_hdcp_status mod_hdcp_hdcp1_enable_encryption(struct mod_hdcp *hdcp) struct mod_hdcp_display *display = get_first_active_display(hdcp); enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; + if (!display) + return MOD_HDCP_STATUS_DISPLAY_NOT_FOUND; + mutex_lock(&psp->hdcp_context.mutex); hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h index 9de01ae574c03..067eddd9c62d8 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h @@ -4115,6 +4115,7 @@ #define mmSCL0_SCL_COEF_RAM_CONFLICT_STATUS 0x1B55 #define mmSCL0_SCL_COEF_RAM_SELECT 0x1B40 #define mmSCL0_SCL_COEF_RAM_TAP_DATA 0x1B41 +#define mmSCL0_SCL_SCALER_ENABLE 0x1B42 #define mmSCL0_SCL_CONTROL 0x1B44 #define mmSCL0_SCL_DEBUG 0x1B6A #define mmSCL0_SCL_DEBUG2 0x1B69 @@ -4144,6 +4145,7 @@ #define mmSCL1_SCL_COEF_RAM_CONFLICT_STATUS 0x1E55 #define mmSCL1_SCL_COEF_RAM_SELECT 0x1E40 #define mmSCL1_SCL_COEF_RAM_TAP_DATA 0x1E41 +#define mmSCL1_SCL_SCALER_ENABLE 0x1E42 #define mmSCL1_SCL_CONTROL 0x1E44 #define mmSCL1_SCL_DEBUG 0x1E6A #define mmSCL1_SCL_DEBUG2 0x1E69 @@ -4173,6 +4175,7 @@ #define mmSCL2_SCL_COEF_RAM_CONFLICT_STATUS 0x4155 #define mmSCL2_SCL_COEF_RAM_SELECT 0x4140 #define mmSCL2_SCL_COEF_RAM_TAP_DATA 0x4141 +#define mmSCL2_SCL_SCALER_ENABLE 0x4142 #define mmSCL2_SCL_CONTROL 0x4144 #define mmSCL2_SCL_DEBUG 0x416A #define mmSCL2_SCL_DEBUG2 0x4169 @@ -4202,6 +4205,7 @@ #define mmSCL3_SCL_COEF_RAM_CONFLICT_STATUS 0x4455 #define mmSCL3_SCL_COEF_RAM_SELECT 0x4440 #define mmSCL3_SCL_COEF_RAM_TAP_DATA 0x4441 +#define mmSCL3_SCL_SCALER_ENABLE 0x4442 #define mmSCL3_SCL_CONTROL 0x4444 #define mmSCL3_SCL_DEBUG 0x446A #define mmSCL3_SCL_DEBUG2 0x4469 @@ -4231,6 +4235,7 @@ #define mmSCL4_SCL_COEF_RAM_CONFLICT_STATUS 0x4755 #define mmSCL4_SCL_COEF_RAM_SELECT 0x4740 #define mmSCL4_SCL_COEF_RAM_TAP_DATA 0x4741 +#define mmSCL4_SCL_SCALER_ENABLE 0x4742 #define mmSCL4_SCL_CONTROL 0x4744 #define mmSCL4_SCL_DEBUG 0x476A #define mmSCL4_SCL_DEBUG2 0x4769 @@ -4260,6 +4265,7 @@ #define mmSCL5_SCL_COEF_RAM_CONFLICT_STATUS 0x4A55 #define mmSCL5_SCL_COEF_RAM_SELECT 0x4A40 #define mmSCL5_SCL_COEF_RAM_TAP_DATA 0x4A41 +#define mmSCL5_SCL_SCALER_ENABLE 0x4A42 #define mmSCL5_SCL_CONTROL 0x4A44 #define mmSCL5_SCL_DEBUG 0x4A6A #define mmSCL5_SCL_DEBUG2 0x4A69 @@ -4287,6 +4293,7 @@ #define mmSCL_COEF_RAM_CONFLICT_STATUS 0x1B55 #define mmSCL_COEF_RAM_SELECT 0x1B40 #define mmSCL_COEF_RAM_TAP_DATA 0x1B41 +#define mmSCL_SCALER_ENABLE 0x1B42 #define mmSCL_CONTROL 0x1B44 #define mmSCL_DEBUG 0x1B6A #define mmSCL_DEBUG2 0x1B69 diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h index bd8085ec54ed5..da5596fbfdcb3 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h @@ -8648,6 +8648,8 @@ #define REGAMMA_LUT_INDEX__REGAMMA_LUT_INDEX__SHIFT 0x00000000 #define REGAMMA_LUT_WRITE_EN_MASK__REGAMMA_LUT_WRITE_EN_MASK_MASK 0x00000007L #define REGAMMA_LUT_WRITE_EN_MASK__REGAMMA_LUT_WRITE_EN_MASK__SHIFT 0x00000000 +#define SCL_SCALER_ENABLE__SCL_SCALE_EN_MASK 0x00000001L +#define SCL_SCALER_ENABLE__SCL_SCALE_EN__SHIFT 0x00000000 #define SCL_ALU_CONTROL__SCL_ALU_DISABLE_MASK 0x00000001L #define SCL_ALU_CONTROL__SCL_ALU_DISABLE__SHIFT 0x00000000 #define SCL_BYPASS_CONTROL__SCL_BYPASS_MODE_MASK 0x00000003L diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h index c488d4a50cf46..b2252deabc17a 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h @@ -203,6 +203,10 @@ #define mmDAGB0_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB0_WR_MISC_CREDIT 0x0058 #define mmDAGB0_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE 0x005b +#define mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x005c +#define mmDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB0_WRCLI_ASK_PENDING 0x005d #define mmDAGB0_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB0_WRCLI_GO_PENDING 0x005e @@ -455,6 +459,10 @@ #define mmDAGB1_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB1_WR_MISC_CREDIT 0x00d8 #define mmDAGB1_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB1_WRCLI_GPU_SNOOP_OVERRIDE 0x00db +#define mmDAGB1_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x00dc +#define mmDAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB1_WRCLI_ASK_PENDING 0x00dd #define mmDAGB1_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB1_WRCLI_GO_PENDING 0x00de @@ -707,6 +715,10 @@ #define mmDAGB2_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB2_WR_MISC_CREDIT 0x0158 #define mmDAGB2_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB2_WRCLI_GPU_SNOOP_OVERRIDE 0x015b +#define mmDAGB2_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x015c +#define mmDAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB2_WRCLI_ASK_PENDING 0x015d #define mmDAGB2_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB2_WRCLI_GO_PENDING 0x015e @@ -959,6 +971,10 @@ #define mmDAGB3_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB3_WR_MISC_CREDIT 0x01d8 #define mmDAGB3_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB3_WRCLI_GPU_SNOOP_OVERRIDE 0x01db +#define mmDAGB3_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x01dc +#define mmDAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB3_WRCLI_ASK_PENDING 0x01dd #define mmDAGB3_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB3_WRCLI_GO_PENDING 0x01de @@ -1211,6 +1227,10 @@ #define mmDAGB4_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB4_WR_MISC_CREDIT 0x0258 #define mmDAGB4_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB4_WRCLI_GPU_SNOOP_OVERRIDE 0x025b +#define mmDAGB4_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x025c +#define mmDAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB4_WRCLI_ASK_PENDING 0x025d #define mmDAGB4_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB4_WRCLI_GO_PENDING 0x025e @@ -4793,6 +4813,10 @@ #define mmDAGB5_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB5_WR_MISC_CREDIT 0x3058 #define mmDAGB5_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB5_WRCLI_GPU_SNOOP_OVERRIDE 0x305b +#define mmDAGB5_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB5_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x305c +#define mmDAGB5_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB5_WRCLI_ASK_PENDING 0x305d #define mmDAGB5_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB5_WRCLI_GO_PENDING 0x305e @@ -5045,6 +5069,10 @@ #define mmDAGB6_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB6_WR_MISC_CREDIT 0x30d8 #define mmDAGB6_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB6_WRCLI_GPU_SNOOP_OVERRIDE 0x30db +#define mmDAGB6_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB6_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x30dc +#define mmDAGB6_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB6_WRCLI_ASK_PENDING 0x30dd #define mmDAGB6_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB6_WRCLI_GO_PENDING 0x30de @@ -5297,6 +5325,10 @@ #define mmDAGB7_WR_DATA_CREDIT_BASE_IDX 1 #define mmDAGB7_WR_MISC_CREDIT 0x3158 #define mmDAGB7_WR_MISC_CREDIT_BASE_IDX 1 +#define mmDAGB7_WRCLI_GPU_SNOOP_OVERRIDE 0x315b +#define mmDAGB7_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX 1 +#define mmDAGB7_WRCLI_GPU_SNOOP_OVERRIDE_VALUE 0x315c +#define mmDAGB7_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX 1 #define mmDAGB7_WRCLI_ASK_PENDING 0x315d #define mmDAGB7_WRCLI_ASK_PENDING_BASE_IDX 1 #define mmDAGB7_WRCLI_GO_PENDING 0x315e diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h index 2969fbf282b7d..5069d2fd467f2 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h @@ -1532,6 +1532,12 @@ //DAGB0_WRCLI_DBUS_GO_PENDING #define DAGB0_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB0_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB0_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB0_DAGB_DLY #define DAGB0_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB0_DAGB_DLY__CLI__SHIFT 0x8 @@ -3207,6 +3213,12 @@ //DAGB1_WRCLI_DBUS_GO_PENDING #define DAGB1_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB1_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB1_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB1_DAGB_DLY #define DAGB1_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB1_DAGB_DLY__CLI__SHIFT 0x8 @@ -4882,6 +4894,12 @@ //DAGB2_WRCLI_DBUS_GO_PENDING #define DAGB2_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB2_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB2_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB2_DAGB_DLY #define DAGB2_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB2_DAGB_DLY__CLI__SHIFT 0x8 @@ -6557,6 +6575,12 @@ //DAGB3_WRCLI_DBUS_GO_PENDING #define DAGB3_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB3_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB3_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB3_DAGB_DLY #define DAGB3_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB3_DAGB_DLY__CLI__SHIFT 0x8 @@ -8232,6 +8256,12 @@ //DAGB4_WRCLI_DBUS_GO_PENDING #define DAGB4_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB4_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB4_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB4_DAGB_DLY #define DAGB4_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB4_DAGB_DLY__CLI__SHIFT 0x8 @@ -28737,6 +28767,12 @@ //DAGB5_WRCLI_DBUS_GO_PENDING #define DAGB5_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB5_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB5_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB5_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB5_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB5_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB5_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB5_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB5_DAGB_DLY #define DAGB5_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB5_DAGB_DLY__CLI__SHIFT 0x8 @@ -30412,6 +30448,12 @@ //DAGB6_WRCLI_DBUS_GO_PENDING #define DAGB6_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB6_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB6_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB6_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB6_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB6_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB6_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB6_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB6_DAGB_DLY #define DAGB6_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB6_DAGB_DLY__CLI__SHIFT 0x8 @@ -32087,6 +32129,12 @@ //DAGB7_WRCLI_DBUS_GO_PENDING #define DAGB7_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT 0x0 #define DAGB7_WRCLI_DBUS_GO_PENDING__BUSY_MASK 0xFFFFFFFFL +//DAGB7_WRCLI_GPU_SNOOP_OVERRIDE +#define DAGB7_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT 0x0 +#define DAGB7_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK 0xFFFFFFFFL +//DAGB7_WRCLI_GPU_SNOOP_OVERRIDE_VALUE +#define DAGB7_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT 0x0 +#define DAGB7_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK 0xFFFFFFFFL //DAGB7_DAGB_DLY #define DAGB7_DAGB_DLY__DLY__SHIFT 0x0 #define DAGB7_DAGB_DLY__CLI__SHIFT 0x8 diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm_internal.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm_internal.c index 42efe838fa85c..2d2d2d5e67634 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_dpm_internal.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm_internal.c @@ -66,6 +66,13 @@ u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev) (amdgpu_crtc->v_border * 2)); vblank_time_us = vblank_in_pixels * 1000 / amdgpu_crtc->hw_mode.clock; + + /* we have issues with mclk switching with + * refresh rates over 120 hz on the non-DC code. + */ + if (drm_mode_vrefresh(&amdgpu_crtc->hw_mode) > 120) + vblank_time_us = 0; + break; } } diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c index e7b1fa2feb928..e5f68b2b8def3 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c @@ -3066,7 +3066,13 @@ static bool si_dpm_vblank_too_short(void *handle) /* we never hit the non-gddr5 limit so disable it */ u32 switch_limit = adev->gmc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 0; - if (vblank_time < switch_limit) + /* Consider zero vblank time too short and disable MCLK switching. + * Note that the vblank time is set to maximum when no displays are attached, + * so we'll still enable MCLK switching in that case. + */ + if (vblank_time == 0) + return true; + else if (vblank_time < switch_limit) return true; else return false; @@ -3424,12 +3430,14 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, { struct si_ps *ps = si_get_ps(rps); struct amdgpu_clock_and_voltage_limits *max_limits; + struct amdgpu_connector *conn; bool disable_mclk_switching = false; bool disable_sclk_switching = false; u32 mclk, sclk; u16 vddc, vddci, min_vce_voltage = 0; u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc; u32 max_sclk = 0, max_mclk = 0; + u32 high_pixelclock_count = 0; int i; if (adev->asic_type == CHIP_HAINAN) { @@ -3457,6 +3465,40 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, } } + /* We define "high pixelclock" for SI as higher than necessary for 4K 30Hz. + * For example, 4K 60Hz and 1080p 144Hz fall into this category. + * Find number of such displays connected. + */ + for (i = 0; i < adev->mode_info.num_crtc; i++) { + if (!(adev->pm.dpm.new_active_crtcs & (1 << i)) || + !adev->mode_info.crtcs[i]->enabled) + continue; + + conn = to_amdgpu_connector(adev->mode_info.crtcs[i]->connector); + + if (conn->pixelclock_for_modeset > 297000) + high_pixelclock_count++; + } + + /* These are some ad-hoc fixes to some issues observed with SI GPUs. + * They are necessary because we don't have something like dce_calcs + * for these GPUs to calculate bandwidth requirements. + */ + if (high_pixelclock_count) { + /* Work around flickering lines at the bottom edge + * of the screen when using a single 4K 60Hz monitor. + */ + disable_mclk_switching = true; + + /* On Oland, we observe some flickering when two 4K 60Hz + * displays are connected, possibly because voltage is too low. + * Raise the voltage by requiring a higher SCLK. + * (Voltage cannot be adjusted independently without also SCLK.) + */ + if (high_pixelclock_count > 1 && adev->asic_type == CHIP_OLAND) + disable_sclk_switching = true; + } + if (rps->vce_active) { rps->evclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].evclk; rps->ecclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].ecclk; @@ -5617,14 +5659,10 @@ static int si_populate_smc_t(struct amdgpu_device *adev, static int si_disable_ulv(struct amdgpu_device *adev) { - struct si_power_info *si_pi = si_get_pi(adev); - struct si_ulv_param *ulv = &si_pi->ulv; + PPSMC_Result r; - if (ulv->supported) - return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ? - 0 : -EINVAL; - - return 0; + r = amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_DisableULV); + return (r == PPSMC_Result_OK) ? 0 : -EINVAL; } static bool si_is_state_ulv_compatible(struct amdgpu_device *adev, @@ -5797,9 +5835,9 @@ static int si_upload_smc_data(struct amdgpu_device *adev) { struct amdgpu_crtc *amdgpu_crtc = NULL; int i; - - if (adev->pm.dpm.new_active_crtc_count == 0) - return 0; + u32 crtc_index = 0; + u32 mclk_change_block_cp_min = 0; + u32 mclk_change_block_cp_max = 0; for (i = 0; i < adev->mode_info.num_crtc; i++) { if (adev->pm.dpm.new_active_crtcs & (1 << i)) { @@ -5808,26 +5846,31 @@ static int si_upload_smc_data(struct amdgpu_device *adev) } } - if (amdgpu_crtc == NULL) - return 0; + /* When a display is plugged in, program these so that the SMC + * performs MCLK switching when it doesn't cause flickering. + * When no display is plugged in, there is no need to restrict + * MCLK switching, so program them to zero. + */ + if (adev->pm.dpm.new_active_crtc_count && amdgpu_crtc) { + crtc_index = amdgpu_crtc->crtc_id; - if (amdgpu_crtc->line_time <= 0) - return 0; + if (amdgpu_crtc->line_time) { + mclk_change_block_cp_min = 200 / amdgpu_crtc->line_time; + mclk_change_block_cp_max = 100 / amdgpu_crtc->line_time; + } + } - if (si_write_smc_soft_register(adev, - SI_SMC_SOFT_REGISTER_crtc_index, - amdgpu_crtc->crtc_id) != PPSMC_Result_OK) - return 0; + si_write_smc_soft_register(adev, + SI_SMC_SOFT_REGISTER_crtc_index, + crtc_index); - if (si_write_smc_soft_register(adev, - SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min, - amdgpu_crtc->wm_high / amdgpu_crtc->line_time) != PPSMC_Result_OK) - return 0; + si_write_smc_soft_register(adev, + SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min, + mclk_change_block_cp_min); - if (si_write_smc_soft_register(adev, - SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max, - amdgpu_crtc->wm_low / amdgpu_crtc->line_time) != PPSMC_Result_OK) - return 0; + si_write_smc_soft_register(adev, + SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max, + mclk_change_block_cp_max); return 0; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c index 1fbd23922082a..7e37354a03411 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c @@ -144,6 +144,10 @@ int atomctrl_initialize_mc_reg_table( vram_info = (ATOM_VRAM_INFO_HEADER_V2_1 *) smu_atom_get_data_table(hwmgr->adev, GetIndexIntoMasterTable(DATA, VRAM_Info), &size, &frev, &crev); + if (!vram_info) { + pr_err("Could not retrieve the VramInfo table!"); + return -EINVAL; + } if (module_index >= vram_info->ucNumOfVRAMModule) { pr_err("Invalid VramInfo table."); @@ -181,6 +185,10 @@ int atomctrl_initialize_mc_reg_table_v2_2( vram_info = (ATOM_VRAM_INFO_HEADER_V2_2 *) smu_atom_get_data_table(hwmgr->adev, GetIndexIntoMasterTable(DATA, VRAM_Info), &size, &frev, &crev); + if (!vram_info) { + pr_err("Could not retrieve the VramInfo table!"); + return -EINVAL; + } if (module_index >= vram_info->ucNumOfVRAMModule) { pr_err("Invalid VramInfo table."); diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppevvmath.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppevvmath.h index 6f54c410c2f98..409aeec6baa92 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppevvmath.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppevvmath.h @@ -22,12 +22,18 @@ */ #include -#define SHIFT_AMOUNT 16 /* We multiply all original integers with 2^SHIFT_AMOUNT to get the fInt representation */ +enum ppevvmath_constants { + /* We multiply all original integers with 2^SHIFT_AMOUNT to get the fInt representation */ + SHIFT_AMOUNT = 16, -#define PRECISION 5 /* Change this value to change the number of decimal places in the final output - 5 is a good default */ + /* Change this value to change the number of decimal places in the final output - 5 is a good default */ + PRECISION = 5, -#define SHIFTED_2 (2 << SHIFT_AMOUNT) -#define MAX (1 << (SHIFT_AMOUNT - 1)) - 1 /* 32767 - Might change in the future */ + SHIFTED_2 = (2 << SHIFT_AMOUNT), + + /* 32767 - Might change in the future */ + MAX = (1 << (SHIFT_AMOUNT - 1)) - 1, +}; /* ------------------------------------------------------------------------------- * NEW TYPE - fINT diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index 53849fd3615f6..965ffcac17f86 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -5437,8 +5437,7 @@ static int smu7_get_thermal_temperature_range(struct pp_hwmgr *hwmgr, thermal_data->max = table_info->cac_dtp_table->usSoftwareShutdownTemp * PP_TEMPERATURE_UNITS_PER_CENTIGRADES; else if (hwmgr->pp_table_version == PP_TABLE_V0) - thermal_data->max = data->thermal_temp_setting.temperature_shutdown * - PP_TEMPERATURE_UNITS_PER_CENTIGRADES; + thermal_data->max = data->thermal_temp_setting.temperature_shutdown; thermal_data->sw_ctf_threshold = thermal_data->max; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c index 79a566f3564a5..c305ea4ec17d2 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c @@ -149,7 +149,7 @@ int phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr, } cgs_write_register(hwmgr->device, indirect_port, index); - return phm_wait_on_register(hwmgr, indirect_port + 1, mask, value); + return phm_wait_on_register(hwmgr, indirect_port + 1, value, mask); } int phm_wait_for_register_unequal(struct pp_hwmgr *hwmgr, diff --git a/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.c index 5e43ad2b29564..e7e497b166b3e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.c @@ -2024,7 +2024,7 @@ static int fiji_init_smc_table(struct pp_hwmgr *hwmgr) table->VoltageResponseTime = 0; table->PhaseResponseTime = 0; table->MemoryThermThrottleEnable = 1; - table->PCIeBootLinkLevel = 0; /* 0:Gen1 1:Gen2 2:Gen3*/ + table->PCIeBootLinkLevel = (uint8_t) (data->dpm_table.pcie_speed_table.count); table->PCIeGenInterval = 1; table->VRConfig = 0; diff --git a/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.c index 97d9802fe6731..43458f1b0077d 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.c @@ -2028,7 +2028,7 @@ static int iceland_init_smc_table(struct pp_hwmgr *hwmgr) table->VoltageResponseTime = 0; table->PhaseResponseTime = 0; table->MemoryThermThrottleEnable = 1; - table->PCIeBootLinkLevel = 0; + table->PCIeBootLinkLevel = (uint8_t) (data->dpm_table.pcie_speed_table.count); table->PCIeGenInterval = 1; result = iceland_populate_smc_svi2_config(hwmgr, table); diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 2997aeed63408..632fc8aed6536 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -1757,6 +1757,12 @@ static int smu_resume(void *handle) adev->pm.dpm_enabled = true; + if (smu->current_power_limit) { + ret = smu_set_power_limit(smu, smu->current_power_limit); + if (ret && ret != -EOPNOTSUPP) + return ret; + } + dev_info(adev->dev, "SMU is resumed successfully!\n"); return 0; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index 0cdf3257b19b3..a47898487de09 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -2376,7 +2376,7 @@ static ssize_t arcturus_get_gpu_metrics(struct smu_context *smu, ret = smu_cmn_get_metrics_table(smu, &metrics, - true); + false); if (ret) return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index a7f4f82d23b4b..2fdb982e70ef7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -2082,7 +2082,9 @@ static int sienna_cichlid_display_disable_memory_clock_switch(struct smu_context return ret; } +#ifndef MAX #define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif static int sienna_cichlid_update_pcie_parameters(struct smu_context *smu, uint8_t pcie_gen_cap, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 454216bd6f1dd..4fabecaa2b419 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -686,7 +686,6 @@ static int vangogh_print_clk_levels(struct smu_context *smu, { DpmClocks_t *clk_table = smu->smu_table.clocks_table; SmuMetrics_t metrics; - struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); int i, idx, size = 0, ret = 0; uint32_t cur_value = 0, value = 0, count = 0; bool cur_value_match_level = false; @@ -702,31 +701,25 @@ static int vangogh_print_clk_levels(struct smu_context *smu, switch (clk_type) { case SMU_OD_SCLK: - if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK"); - size += sysfs_emit_at(buf, size, "0: %10uMhz\n", - (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq); - size += sysfs_emit_at(buf, size, "1: %10uMhz\n", - (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq); - } + size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK"); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", + (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq); + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", + (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq); break; case SMU_OD_CCLK: - if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size += sysfs_emit_at(buf, size, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); - size += sysfs_emit_at(buf, size, "0: %10uMhz\n", - (smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq); - size += sysfs_emit_at(buf, size, "1: %10uMhz\n", - (smu->cpu_actual_soft_max_freq > 0) ? smu->cpu_actual_soft_max_freq : smu->cpu_default_soft_max_freq); - } + size += sysfs_emit_at(buf, size, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", + (smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq); + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", + (smu->cpu_actual_soft_max_freq > 0) ? smu->cpu_actual_soft_max_freq : smu->cpu_default_soft_max_freq); break; case SMU_OD_RANGE: - if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", - smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq); - size += sysfs_emit_at(buf, size, "CCLK: %7uMhz %10uMhz\n", - smu->cpu_default_soft_min_freq, smu->cpu_default_soft_max_freq); - } + size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); + size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", + smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq); + size += sysfs_emit_at(buf, size, "CCLK: %7uMhz %10uMhz\n", + smu->cpu_default_soft_min_freq, smu->cpu_default_soft_max_freq); break; case SMU_SOCCLK: /* the level 3 ~ 6 of socclk use the same frequency for vangogh */ diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c index b225617801499..d1fd643e7a48c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c @@ -1737,7 +1737,7 @@ static ssize_t aldebaran_get_gpu_metrics(struct smu_context *smu, ret = smu_cmn_get_metrics_table(smu, &metrics, - true); + false); if (ret) return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 4022dd44ebb2b..fe6c42a736d08 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -1696,7 +1696,10 @@ static int smu_v13_0_0_get_thermal_temperature_range(struct smu_context *smu, return 0; } +#ifndef MAX #define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + static ssize_t smu_v13_0_0_get_gpu_metrics(struct smu_context *smu, void **table) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index e1521d3a5e0ca..df99d185cf8e0 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -1674,7 +1674,10 @@ static int smu_v13_0_7_get_thermal_temperature_range(struct smu_context *smu, return 0; } +#ifndef MAX #define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + static ssize_t smu_v13_0_7_get_gpu_metrics(struct smu_context *smu, void **table) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 12618a583e97c..2c9612b5f1568 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -708,7 +708,7 @@ static const char *smu_get_feature_name(struct smu_context *smu, size_t smu_cmn_get_pp_feature_mask(struct smu_context *smu, char *buf) { - int8_t sort_feature[max(SMU_FEATURE_COUNT, SMU_FEATURE_MAX)]; + int8_t sort_feature[MAX(SMU_FEATURE_COUNT, SMU_FEATURE_MAX)]; uint64_t feature_mask; int i, feature_index; uint32_t count = 0; @@ -870,7 +870,7 @@ int smu_cmn_update_table(struct smu_context *smu, table_index); uint32_t table_size; int ret = 0; - if (!table_data || table_id >= SMU_TABLE_COUNT || table_id < 0) + if (!table_data || table_index >= SMU_TABLE_COUNT || table_id < 0) return -EINVAL; table_size = smu_table->tables[table_index].size; diff --git a/drivers/gpu/drm/ast/ast_dp.c b/drivers/gpu/drm/ast/ast_dp.c index c6f226b6f0813..fc9c36c8b81b9 100644 --- a/drivers/gpu/drm/ast/ast_dp.c +++ b/drivers/gpu/drm/ast/ast_dp.c @@ -62,7 +62,7 @@ int ast_astdp_read_edid(struct drm_device *dev, u8 *ediddata) * of right-click of mouse. * 2. The Delays are often longer a lot when system resume from S3/S4. */ - mdelay(j+1); + msleep(j + 1); if (!(ast_get_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xD1, ASTDP_MCU_FW_EXECUTING) && diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c index 3de0f457fff6a..27e660e92489f 100644 --- a/drivers/gpu/drm/ast/ast_mode.c +++ b/drivers/gpu/drm/ast/ast_mode.c @@ -132,7 +132,7 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format, return false; } - switch (mode->crtc_hdisplay) { + switch (mode->hdisplay) { case 640: vbios_mode->enh_table = &res_640x480[refresh_rate_index]; break; @@ -146,7 +146,7 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format, vbios_mode->enh_table = &res_1152x864[refresh_rate_index]; break; case 1280: - if (mode->crtc_vdisplay == 800) + if (mode->vdisplay == 800) vbios_mode->enh_table = &res_1280x800[refresh_rate_index]; else vbios_mode->enh_table = &res_1280x1024[refresh_rate_index]; @@ -158,7 +158,7 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format, vbios_mode->enh_table = &res_1440x900[refresh_rate_index]; break; case 1600: - if (mode->crtc_vdisplay == 900) + if (mode->vdisplay == 900) vbios_mode->enh_table = &res_1600x900[refresh_rate_index]; else vbios_mode->enh_table = &res_1600x1200[refresh_rate_index]; @@ -167,7 +167,7 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format, vbios_mode->enh_table = &res_1680x1050[refresh_rate_index]; break; case 1920: - if (mode->crtc_vdisplay == 1080) + if (mode->vdisplay == 1080) vbios_mode->enh_table = &res_1920x1080[refresh_rate_index]; else vbios_mode->enh_table = &res_1920x1200[refresh_rate_index]; @@ -211,6 +211,7 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format, hborder = (vbios_mode->enh_table->flags & HBorder) ? 8 : 0; vborder = (vbios_mode->enh_table->flags & VBorder) ? 8 : 0; + adjusted_mode->crtc_hdisplay = vbios_mode->enh_table->hde; adjusted_mode->crtc_htotal = vbios_mode->enh_table->ht; adjusted_mode->crtc_hblank_start = vbios_mode->enh_table->hde + hborder; adjusted_mode->crtc_hblank_end = vbios_mode->enh_table->ht - hborder; @@ -220,6 +221,7 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format, vbios_mode->enh_table->hfp + vbios_mode->enh_table->hsync); + adjusted_mode->crtc_vdisplay = vbios_mode->enh_table->vde; adjusted_mode->crtc_vtotal = vbios_mode->enh_table->vt; adjusted_mode->crtc_vblank_start = vbios_mode->enh_table->vde + vborder; adjusted_mode->crtc_vblank_end = vbios_mode->enh_table->vt - vborder; @@ -1866,9 +1868,9 @@ static void ast_mode_config_helper_atomic_commit_tail(struct drm_atomic_state *s /* * Concurrent operations could possibly trigger a call to - * drm_connector_helper_funcs.get_modes by trying to read the - * display modes. Protect access to I/O registers by acquiring - * the I/O-register lock. Released in atomic_flush(). + * drm_connector_helper_funcs.get_modes by reading the display + * modes. Protect access to registers by acquiring the modeset + * lock. */ mutex_lock(&ast->ioregs_lock); drm_atomic_helper_commit_tail_rpm(state); diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig index 3e6a4e2044c0e..5a4d574b37b89 100644 --- a/drivers/gpu/drm/bridge/Kconfig +++ b/drivers/gpu/drm/bridge/Kconfig @@ -85,6 +85,7 @@ config DRM_ITE_IT6505 select EXTCON select CRYPTO select CRYPTO_HASH + select REGMAP_I2C help ITE IT6505 DisplayPort bridge chip driver. diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c b/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c index 8f786592143b6..24e1e11acf697 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c +++ b/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c @@ -244,7 +244,9 @@ static const struct hdmi_codec_pdata codec_data = { .ops = &adv7511_codec_ops, .max_i2s_channels = 2, .i2s = 1, + .no_i2s_capture = 1, .spdif = 1, + .no_spdif_capture = 1, }; int adv7511_audio_init(struct device *dev, struct adv7511 *adv7511) diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c index 412c6575e87b7..4f19f3ff12fd2 100644 --- a/drivers/gpu/drm/bridge/analogix/anx7625.c +++ b/drivers/gpu/drm/bridge/analogix/anx7625.c @@ -2703,7 +2703,7 @@ static int anx7625_i2c_probe(struct i2c_client *client) ret = devm_request_threaded_irq(dev, platform->pdata.intp_irq, NULL, anx7625_intr_hpd_isr, IRQF_TRIGGER_FALLING | - IRQF_ONESHOT, + IRQF_ONESHOT | IRQF_NO_AUTOEN, "anx7625-intp", platform); if (ret) { DRM_DEV_ERROR(dev, "fail to request irq\n"); @@ -2773,8 +2773,10 @@ static int anx7625_i2c_probe(struct i2c_client *client) } /* Add work function */ - if (platform->pdata.intp_irq) + if (platform->pdata.intp_irq) { + enable_irq(platform->pdata.intp_irq); queue_work(platform->workqueue, &platform->work); + } if (platform->pdata.audio_en) anx7625_register_audio(dev, platform); diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c index 7457d38622b0c..ddfbb2009c8d3 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c @@ -568,15 +568,18 @@ static int cdns_dsi_check_conf(struct cdns_dsi *dsi, struct phy_configure_opts_mipi_dphy *phy_cfg = &output->phy_opts.mipi_dphy; unsigned long dsi_hss_hsa_hse_hbp; unsigned int nlanes = output->dev->lanes; + int mode_clock = (mode_valid_check ? mode->clock : mode->crtc_clock); int ret; ret = cdns_dsi_mode2cfg(dsi, mode, dsi_cfg, mode_valid_check); if (ret) return ret; - phy_mipi_dphy_get_default_config(mode->crtc_clock * 1000, - mipi_dsi_pixel_format_to_bpp(output->dev->format), - nlanes, phy_cfg); + ret = phy_mipi_dphy_get_default_config(mode_clock * 1000, + mipi_dsi_pixel_format_to_bpp(output->dev->format), + nlanes, phy_cfg); + if (ret) + return ret; ret = cdns_dsi_adjust_phy_config(dsi, dsi_cfg, phy_cfg, mode, mode_valid_check); if (ret) @@ -680,6 +683,11 @@ static void cdns_dsi_bridge_post_disable(struct drm_bridge *bridge) struct cdns_dsi_input *input = bridge_to_cdns_dsi_input(bridge); struct cdns_dsi *dsi = input_to_dsi(input); + dsi->phy_initialized = false; + dsi->link_initialized = false; + phy_power_off(dsi->dphy); + phy_exit(dsi->dphy); + pm_runtime_put(dsi->base.dev); } @@ -761,7 +769,7 @@ static void cdns_dsi_bridge_enable(struct drm_bridge *bridge) struct phy_configure_opts_mipi_dphy *phy_cfg = &output->phy_opts.mipi_dphy; unsigned long tx_byte_period; struct cdns_dsi_cfg dsi_cfg; - u32 tmp, reg_wakeup, div; + u32 tmp, reg_wakeup, div, status; int nlanes; if (WARN_ON(pm_runtime_get_sync(dsi->base.dev) < 0)) @@ -778,6 +786,19 @@ static void cdns_dsi_bridge_enable(struct drm_bridge *bridge) cdns_dsi_hs_init(dsi); cdns_dsi_init_link(dsi); + /* + * Now that the DSI Link and DSI Phy are initialized, + * wait for the CLK and Data Lanes to be ready. + */ + tmp = CLK_LANE_RDY; + for (int i = 0; i < nlanes; i++) + tmp |= DATA_LANE_RDY(i); + + if (readl_poll_timeout(dsi->regs + MCTL_MAIN_STS, status, + (tmp == (status & tmp)), 100, 500000)) + dev_err(dsi->base.dev, + "Timed Out: DSI-DPhy Clock and Data Lanes not ready.\n"); + writel(HBP_LEN(dsi_cfg.hbp) | HSA_LEN(dsi_cfg.hsa), dsi->regs + VID_HSIZE1); writel(HFP_LEN(dsi_cfg.hfp) | HACT_LEN(dsi_cfg.hact), @@ -812,7 +833,13 @@ static void cdns_dsi_bridge_enable(struct drm_bridge *bridge) tx_byte_period = DIV_ROUND_DOWN_ULL((u64)NSEC_PER_SEC * 8, phy_cfg->hs_clk_rate); - reg_wakeup = (phy_cfg->hs_prepare + phy_cfg->hs_zero) / tx_byte_period; + + /* + * Estimated time [in clock cycles] to perform LP->HS on D-PHY. + * It is not clear how to calculate this, so for now, + * set it to 1/10 of the total number of clocks in a line. + */ + reg_wakeup = dsi_cfg.htotal / nlanes / 10; writel(REG_WAKEUP_TIME(reg_wakeup) | REG_LINE_DURATION(tmp), dsi->regs + VID_DPHY_TIME); @@ -932,10 +959,6 @@ static int cdns_dsi_attach(struct mipi_dsi_host *host, if (output->dev) return -EBUSY; - /* We do not support burst mode yet. */ - if (dev->mode_flags & MIPI_DSI_MODE_VIDEO_BURST) - return -ENOTSUPP; - /* * The host <-> device link might be described using an OF-graph * representation, in this case we extract the device of_node from @@ -952,7 +975,7 @@ static int cdns_dsi_attach(struct mipi_dsi_host *host, bridge = drm_panel_bridge_add_typed(panel, DRM_MODE_CONNECTOR_DSI); } else { - bridge = of_drm_find_bridge(dev->dev.of_node); + bridge = of_drm_find_bridge(np); if (!bridge) bridge = ERR_PTR(-EINVAL); } @@ -1152,7 +1175,6 @@ static int __maybe_unused cdns_dsi_suspend(struct device *dev) clk_disable_unprepare(dsi->dsi_sys_clk); clk_disable_unprepare(dsi->dsi_p_clk); reset_control_assert(dsi->dsi_p_rst); - dsi->link_initialized = false; return 0; } diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c index 858f5b6508491..bdb9fc00c776b 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c @@ -2057,8 +2057,10 @@ static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge, mhdp_state = to_cdns_mhdp_bridge_state(new_state); mhdp_state->current_mode = drm_mode_duplicate(bridge->dev, mode); - if (!mhdp_state->current_mode) - return; + if (!mhdp_state->current_mode) { + ret = -EINVAL; + goto out; + } drm_mode_set_name(mhdp_state->current_mode); diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c index 08bd5695ddae0..0f3714aae6089 100644 --- a/drivers/gpu/drm/bridge/display-connector.c +++ b/drivers/gpu/drm/bridge/display-connector.c @@ -363,7 +363,8 @@ static int display_connector_probe(struct platform_device *pdev) if (conn->bridge.ddc) conn->bridge.ops |= DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT; - if (conn->hpd_gpio) + /* Detecting the monitor requires reading DPCD */ + if (conn->hpd_gpio && type != DRM_MODE_CONNECTOR_DisplayPort) conn->bridge.ops |= DRM_BRIDGE_OP_DETECT; if (conn->hpd_irq >= 0) conn->bridge.ops |= DRM_BRIDGE_OP_HPD; diff --git a/drivers/gpu/drm/bridge/lontium-lt9211.c b/drivers/gpu/drm/bridge/lontium-lt9211.c index 4d404f5ef87eb..ea192c90b543e 100644 --- a/drivers/gpu/drm/bridge/lontium-lt9211.c +++ b/drivers/gpu/drm/bridge/lontium-lt9211.c @@ -120,8 +120,7 @@ static int lt9211_read_chipid(struct lt9211 *ctx) } /* Test for known Chip ID. */ - if (chipid[0] != REG_CHIPID0_VALUE || chipid[1] != REG_CHIPID1_VALUE || - chipid[2] != REG_CHIPID2_VALUE) { + if (chipid[0] != REG_CHIPID0_VALUE || chipid[1] != REG_CHIPID1_VALUE) { dev_err(ctx->dev, "Unknown Chip ID: 0x%02x 0x%02x 0x%02x\n", chipid[0], chipid[1], chipid[2]); return -EINVAL; diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c index c41ffd0bc0494..d458a4f37ac8f 100644 --- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c +++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c @@ -962,7 +962,11 @@ static int lt9611uxc_probe(struct i2c_client *client) } } - return lt9611uxc_audio_init(dev, lt9611uxc); + ret = lt9611uxc_audio_init(dev, lt9611uxc); + if (ret) + goto err_remove_bridge; + + return 0; err_remove_bridge: free_irq(client->irq, lt9611uxc); diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index bfbd3fee12567..560935f2e8cbe 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -331,12 +331,18 @@ static void ti_sn65dsi86_enable_comms(struct ti_sn65dsi86 *pdata) * 200 ms. We'll assume that the panel driver will have the hardcoded * delay in its prepare and always disable HPD. * - * If HPD somehow makes sense on some future panel we'll have to - * change this to be conditional on someone specifying that HPD should - * be used. + * For DisplayPort bridge type, we need HPD. So we use the bridge type + * to conditionally disable HPD. + * NOTE: The bridge type is set in ti_sn_bridge_probe() but enable_comms() + * can be called before. So for DisplayPort, HPD will be enabled once + * bridge type is set. We are using bridge type instead of "no-hpd" + * property because it is not used properly in devicetree description + * and hence is unreliable. */ - regmap_update_bits(pdata->regmap, SN_HPD_DISABLE_REG, HPD_DISABLE, - HPD_DISABLE); + + if (pdata->bridge.type != DRM_MODE_CONNECTOR_DisplayPort) + regmap_update_bits(pdata->regmap, SN_HPD_DISABLE_REG, HPD_DISABLE, + HPD_DISABLE); pdata->comms_enabled = true; @@ -369,6 +375,17 @@ static int __maybe_unused ti_sn65dsi86_resume(struct device *dev) gpiod_set_value_cansleep(pdata->enable_gpio, 1); + /* + * After EN is deasserted and an external clock is detected, the bridge + * will sample GPIO3:1 to determine its frequency. The driver will + * overwrite this setting in ti_sn_bridge_set_refclk_freq(). But this is + * racy. Thus we have to wait a couple of us. According to the datasheet + * the GPIO lines has to be stable at least 5 us (td5) but it seems that + * is not enough and the refclk frequency value is still lost or + * overwritten by the bridge itself. Waiting for 20us seems to work. + */ + usleep_range(20, 30); + /* * If we have a reference clock we can enable communication w/ the * panel (including the aux channel) w/out any need for an input clock @@ -424,36 +441,8 @@ static int status_show(struct seq_file *s, void *data) return 0; } - DEFINE_SHOW_ATTRIBUTE(status); -static void ti_sn65dsi86_debugfs_remove(void *data) -{ - debugfs_remove_recursive(data); -} - -static void ti_sn65dsi86_debugfs_init(struct ti_sn65dsi86 *pdata) -{ - struct device *dev = pdata->dev; - struct dentry *debugfs; - int ret; - - debugfs = debugfs_create_dir(dev_name(dev), NULL); - - /* - * We might get an error back if debugfs wasn't enabled in the kernel - * so let's just silently return upon failure. - */ - if (IS_ERR_OR_NULL(debugfs)) - return; - - ret = devm_add_action_or_reset(dev, ti_sn65dsi86_debugfs_remove, debugfs); - if (ret) - return; - - debugfs_create_file("status", 0600, debugfs, pdata, &status_fops); -} - /* ----------------------------------------------------------------------------- * Auxiliary Devices (*not* AUX) */ @@ -1201,9 +1190,14 @@ static enum drm_connector_status ti_sn_bridge_detect(struct drm_bridge *bridge) struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge); int val = 0; - pm_runtime_get_sync(pdata->dev); + /* + * Runtime reference is grabbed in ti_sn_bridge_hpd_enable() + * as the chip won't report HPD just after being powered on. + * HPD_DEBOUNCED_STATE reflects correct state only after the + * debounce time (~100-400 ms). + */ + regmap_read(pdata->regmap, SN_HPD_DISABLE_REG, &val); - pm_runtime_put_autosuspend(pdata->dev); return val & HPD_DEBOUNCED_STATE ? connector_status_connected : connector_status_disconnected; @@ -1217,6 +1211,35 @@ static struct edid *ti_sn_bridge_get_edid(struct drm_bridge *bridge, return drm_get_edid(connector, &pdata->aux.ddc); } +static void ti_sn65dsi86_debugfs_init(struct drm_bridge *bridge, struct dentry *root) +{ + struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge); + struct dentry *debugfs; + + debugfs = debugfs_create_dir(dev_name(pdata->dev), root); + debugfs_create_file("status", 0600, debugfs, pdata, &status_fops); +} + +static void ti_sn_bridge_hpd_enable(struct drm_bridge *bridge) +{ + struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge); + + /* + * Device needs to be powered on before reading the HPD state + * for reliable hpd detection in ti_sn_bridge_detect() due to + * the high debounce time. + */ + + pm_runtime_get_sync(pdata->dev); +} + +static void ti_sn_bridge_hpd_disable(struct drm_bridge *bridge) +{ + struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge); + + pm_runtime_put_autosuspend(pdata->dev); +} + static const struct drm_bridge_funcs ti_sn_bridge_funcs = { .attach = ti_sn_bridge_attach, .detach = ti_sn_bridge_detach, @@ -1230,6 +1253,9 @@ static const struct drm_bridge_funcs ti_sn_bridge_funcs = { .atomic_reset = drm_atomic_helper_bridge_reset, .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, + .debugfs_init = ti_sn65dsi86_debugfs_init, + .hpd_enable = ti_sn_bridge_hpd_enable, + .hpd_disable = ti_sn_bridge_hpd_disable, }; static void ti_sn_bridge_parse_lanes(struct ti_sn65dsi86 *pdata, @@ -1318,8 +1344,26 @@ static int ti_sn_bridge_probe(struct auxiliary_device *adev, pdata->bridge.type = pdata->next_bridge->type == DRM_MODE_CONNECTOR_DisplayPort ? DRM_MODE_CONNECTOR_DisplayPort : DRM_MODE_CONNECTOR_eDP; - if (pdata->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) - pdata->bridge.ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT; + if (pdata->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) { + pdata->bridge.ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT | + DRM_BRIDGE_OP_HPD; + /* + * If comms were already enabled they would have been enabled + * with the wrong value of HPD_DISABLE. Update it now. Comms + * could be enabled if anyone is holding a pm_runtime reference + * (like if a GPIO is in use). Note that in most cases nobody + * is doing AUX channel xfers before the bridge is added so + * HPD doesn't _really_ matter then. The only exception is in + * the eDP case where the panel wants to read the EDID before + * the bridge is added. We always consistently have HPD disabled + * for eDP. + */ + mutex_lock(&pdata->comms_mutex); + if (pdata->comms_enabled) + regmap_update_bits(pdata->regmap, SN_HPD_DISABLE_REG, + HPD_DISABLE, 0); + mutex_unlock(&pdata->comms_mutex); + } drm_bridge_add(&pdata->bridge); @@ -1935,8 +1979,6 @@ static int ti_sn65dsi86_probe(struct i2c_client *client) if (ret) return ret; - ti_sn65dsi86_debugfs_init(pdata); - /* * Break ourselves up into a collection of aux devices. The only real * motiviation here is to solve the chicken-and-egg problem of probe diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index f3681970887cc..1aa59586c8f81 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -573,6 +573,30 @@ mode_valid(struct drm_atomic_state *state) return 0; } +static int drm_atomic_check_valid_clones(struct drm_atomic_state *state, + struct drm_crtc *crtc) +{ + struct drm_encoder *drm_enc; + struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, + crtc); + + drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc_state->encoder_mask) { + if (!drm_enc->possible_clones) { + DRM_DEBUG("enc%d possible_clones is 0\n", drm_enc->base.id); + continue; + } + + if ((crtc_state->encoder_mask & drm_enc->possible_clones) != + crtc_state->encoder_mask) { + DRM_DEBUG("crtc%d failed valid clone check for mask 0x%x\n", + crtc->base.id, crtc_state->encoder_mask); + return -EINVAL; + } + } + + return 0; +} + /** * drm_atomic_helper_check_modeset - validate state object for modeset changes * @dev: DRM device @@ -744,6 +768,10 @@ drm_atomic_helper_check_modeset(struct drm_device *dev, ret = drm_atomic_add_affected_planes(state, crtc); if (ret != 0) return ret; + + ret = drm_atomic_check_valid_clones(state, crtc); + if (ret != 0) + return ret; } /* diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c index d021497841b84..3969dc548cff6 100644 --- a/drivers/gpu/drm/drm_color_mgmt.c +++ b/drivers/gpu/drm/drm_color_mgmt.c @@ -532,7 +532,7 @@ int drm_plane_create_color_properties(struct drm_plane *plane, { struct drm_device *dev = plane->dev; struct drm_property *prop; - struct drm_prop_enum_list enum_list[max_t(int, DRM_COLOR_ENCODING_MAX, + struct drm_prop_enum_list enum_list[MAX_T(int, DRM_COLOR_ENCODING_MAX, DRM_COLOR_RANGE_MAX)]; int i, len; diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index ee3fab115c4b5..2daa1fac56809 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -6465,6 +6465,7 @@ static void drm_reset_display_info(struct drm_connector *connector) info->has_hdmi_infoframe = false; info->rgb_quant_range_selectable = false; memset(&info->hdmi, 0, sizeof(info->hdmi)); + memset(&connector->hdr_sink_metadata, 0, sizeof(connector->hdr_sink_metadata)); info->edid_hdmi_rgb444_dc_modes = 0; info->edid_hdmi_ycbcr444_dc_modes = 0; diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index 1d22dba69b275..b943221b238f8 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -1015,6 +1015,10 @@ void drm_show_fdinfo(struct seq_file *m, struct file *f) struct drm_file *file = f->private_data; struct drm_device *dev = file->minor->dev; struct drm_printer p = drm_seq_file_printer(m); + int idx; + + if (!drm_dev_enter(dev, &idx)) + return; drm_printf(&p, "drm-driver:\t%s\n", dev->driver->name); drm_printf(&p, "drm-client-id:\t%llu\n", file->client_id); @@ -1029,6 +1033,8 @@ void drm_show_fdinfo(struct seq_file *m, struct file *f) if (dev->driver->show_fdinfo) dev->driver->show_fdinfo(&p, file); + + drm_dev_exit(idx); } EXPORT_SYMBOL(drm_show_fdinfo); diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c index 1955eaeba0ab7..4ef05c8c75e65 100644 --- a/drivers/gpu/drm/drm_framebuffer.c +++ b/drivers/gpu/drm/drm_framebuffer.c @@ -844,11 +844,23 @@ void drm_framebuffer_free(struct kref *kref) int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb, const struct drm_framebuffer_funcs *funcs) { + unsigned int i; int ret; + bool exists; if (WARN_ON_ONCE(fb->dev != dev || !fb->format)) return -EINVAL; + for (i = 0; i < fb->format->num_planes; i++) { + if (drm_WARN_ON_ONCE(dev, fb->internal_flags & DRM_FRAMEBUFFER_HAS_HANDLE_REF(i))) + fb->internal_flags &= ~DRM_FRAMEBUFFER_HAS_HANDLE_REF(i); + if (fb->obj[i]) { + exists = drm_gem_object_handle_get_if_exists_unlocked(fb->obj[i]); + if (exists) + fb->internal_flags |= DRM_FRAMEBUFFER_HAS_HANDLE_REF(i); + } + } + INIT_LIST_HEAD(&fb->filp_head); fb->funcs = funcs; @@ -857,7 +869,7 @@ int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb, ret = __drm_mode_object_add(dev, &fb->base, DRM_MODE_OBJECT_FB, false, drm_framebuffer_free); if (ret) - goto out; + goto err; mutex_lock(&dev->mode_config.fb_lock); dev->mode_config.num_fb++; @@ -865,7 +877,16 @@ int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb, mutex_unlock(&dev->mode_config.fb_lock); drm_mode_object_register(dev, &fb->base); -out: + + return 0; + +err: + for (i = 0; i < fb->format->num_planes; i++) { + if (fb->internal_flags & DRM_FRAMEBUFFER_HAS_HANDLE_REF(i)) { + drm_gem_object_handle_put_unlocked(fb->obj[i]); + fb->internal_flags &= ~DRM_FRAMEBUFFER_HAS_HANDLE_REF(i); + } + } return ret; } EXPORT_SYMBOL(drm_framebuffer_init); @@ -942,6 +963,12 @@ EXPORT_SYMBOL(drm_framebuffer_unregister_private); void drm_framebuffer_cleanup(struct drm_framebuffer *fb) { struct drm_device *dev = fb->dev; + unsigned int i; + + for (i = 0; i < fb->format->num_planes; i++) { + if (fb->internal_flags & DRM_FRAMEBUFFER_HAS_HANDLE_REF(i)) + drm_gem_object_handle_put_unlocked(fb->obj[i]); + } mutex_lock(&dev->mode_config.fb_lock); list_del(&fb->head); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 44a948b80ee14..a3370c77e949d 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -186,6 +186,46 @@ void drm_gem_private_object_fini(struct drm_gem_object *obj) } EXPORT_SYMBOL(drm_gem_private_object_fini); +static void drm_gem_object_handle_get(struct drm_gem_object *obj) +{ + struct drm_device *dev = obj->dev; + + drm_WARN_ON(dev, !mutex_is_locked(&dev->object_name_lock)); + + if (obj->handle_count++ == 0) + drm_gem_object_get(obj); +} + +/** + * drm_gem_object_handle_get_if_exists_unlocked - acquire reference on user-space handle, if any + * @obj: GEM object + * + * Acquires a reference on the GEM buffer object's handle. Required to keep + * the GEM object alive. Call drm_gem_object_handle_put_if_exists_unlocked() + * to release the reference. Does nothing if the buffer object has no handle. + * + * Returns: + * True if a handle exists, or false otherwise + */ +bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj) +{ + struct drm_device *dev = obj->dev; + + guard(mutex)(&dev->object_name_lock); + + /* + * First ref taken during GEM object creation, if any. Some + * drivers set up internal framebuffers with GEM objects that + * do not have a GEM handle. Hence, this counter can be zero. + */ + if (!obj->handle_count) + return false; + + drm_gem_object_handle_get(obj); + + return true; +} + /** * drm_gem_object_handle_free - release resources bound to userspace handles * @obj: GEM object to clean up. @@ -216,20 +256,26 @@ static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj) } } -static void -drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj) +/** + * drm_gem_object_handle_put_unlocked - releases reference on user-space handle + * @obj: GEM object + * + * Releases a reference on the GEM buffer object's handle. Possibly releases + * the GEM buffer object and associated dma-buf objects. + */ +void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj) { struct drm_device *dev = obj->dev; bool final = false; - if (WARN_ON(READ_ONCE(obj->handle_count) == 0)) + if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0)) return; /* - * Must bump handle count first as this may be the last - * ref, in which case the object would disappear before we - * checked for a name - */ + * Must bump handle count first as this may be the last + * ref, in which case the object would disappear before + * we checked for a name. + */ mutex_lock(&dev->object_name_lock); if (--obj->handle_count == 0) { @@ -253,6 +299,9 @@ drm_gem_object_release_handle(int id, void *ptr, void *data) struct drm_file *file_priv = data; struct drm_gem_object *obj = ptr; + if (drm_WARN_ON(obj->dev, !data)) + return 0; + if (obj->funcs->close) obj->funcs->close(obj, file_priv); @@ -322,7 +371,7 @@ int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, return -ENOENT; /* Don't allow imported objects to be mapped */ - if (obj->import_attach) { + if (drm_gem_is_imported(obj)) { ret = -EINVAL; goto out; } @@ -363,8 +412,8 @@ drm_gem_handle_create_tail(struct drm_file *file_priv, int ret; WARN_ON(!mutex_is_locked(&dev->object_name_lock)); - if (obj->handle_count++ == 0) - drm_gem_object_get(obj); + + drm_gem_object_handle_get(obj); /* * Get the user-visible handle using idr. Preload and perform @@ -373,7 +422,7 @@ drm_gem_handle_create_tail(struct drm_file *file_priv, idr_preload(GFP_KERNEL); spin_lock(&file_priv->table_lock); - ret = idr_alloc(&file_priv->object_idr, obj, 1, 0, GFP_NOWAIT); + ret = idr_alloc(&file_priv->object_idr, NULL, 1, 0, GFP_NOWAIT); spin_unlock(&file_priv->table_lock); idr_preload_end(); @@ -394,6 +443,11 @@ drm_gem_handle_create_tail(struct drm_file *file_priv, goto err_revoke; } + /* mirrors drm_gem_handle_delete to avoid races */ + spin_lock(&file_priv->table_lock); + obj = idr_replace(&file_priv->object_idr, obj, handle); + WARN_ON(obj != NULL); + spin_unlock(&file_priv->table_lock); *handlep = handle; return 0; @@ -1155,7 +1209,7 @@ void drm_gem_print_info(struct drm_printer *p, unsigned int indent, drm_vma_node_start(&obj->vma_node)); drm_printf_indent(p, indent, "size=%zu\n", obj->size); drm_printf_indent(p, indent, "imported=%s\n", - str_yes_no(obj->import_attach)); + str_yes_no(drm_gem_is_imported(obj))); if (obj->funcs->print_info) obj->funcs->print_info(p, indent, obj); diff --git a/drivers/gpu/drm/drm_gem_atomic_helper.c b/drivers/gpu/drm/drm_gem_atomic_helper.c index 5d4b9cd077f7a..e0ea3c661cb77 100644 --- a/drivers/gpu/drm/drm_gem_atomic_helper.c +++ b/drivers/gpu/drm/drm_gem_atomic_helper.c @@ -301,7 +301,11 @@ EXPORT_SYMBOL(drm_gem_destroy_shadow_plane_state); void __drm_gem_reset_shadow_plane(struct drm_plane *plane, struct drm_shadow_plane_state *shadow_plane_state) { - __drm_atomic_helper_plane_reset(plane, &shadow_plane_state->base); + if (shadow_plane_state) { + __drm_atomic_helper_plane_reset(plane, &shadow_plane_state->base); + } else { + __drm_atomic_helper_plane_reset(plane, NULL); + } } EXPORT_SYMBOL(__drm_gem_reset_shadow_plane); diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h index 0ef5fc2a61f19..8d433fe37e8fa 100644 --- a/drivers/gpu/drm/drm_internal.h +++ b/drivers/gpu/drm/drm_internal.h @@ -155,6 +155,8 @@ void drm_sysfs_lease_event(struct drm_device *dev); /* drm_gem.c */ int drm_gem_init(struct drm_device *dev); +bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj); +void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj); int drm_gem_handle_create_tail(struct drm_file *file_priv, struct drm_gem_object *obj, u32 *handlep); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c index b13a17276d07c..88385dc3b30d8 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c @@ -347,7 +347,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, u32 link_target, link_dwords; bool switch_context = gpu->exec_state != exec_state; bool switch_mmu_context = gpu->mmu_context != mmu_context; - unsigned int new_flush_seq = READ_ONCE(gpu->mmu_context->flush_seq); + unsigned int new_flush_seq = READ_ONCE(mmu_context->flush_seq); bool need_flush = switch_mmu_context || gpu->flush_seq != new_flush_seq; bool has_blt = !!(gpu->identity.minor_features5 & chipMinorFeatures5_BLT_ENGINE); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 97e406d9ac06f..a3bd396c9d829 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) { struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); + struct drm_gpu_scheduler *sched = sched_job->sched; struct etnaviv_gpu *gpu = submit->gpu; u32 dma_addr; int change; @@ -76,7 +77,9 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job return DRM_GPU_SCHED_STAT_NOMINAL; out_no_timeout: - list_add(&sched_job->list, &sched_job->sched->pending_list); + spin_lock(&sched->job_list_lock); + list_add(&sched_job->list, &sched->pending_list); + spin_unlock(&sched->job_list_lock); return DRM_GPU_SCHED_STAT_NOMINAL; } diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c index 0156a5e944359..0ed62cd51fb51 100644 --- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c +++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c @@ -51,7 +51,6 @@ struct decon_context { void __iomem *regs; unsigned long irq_flags; bool i80_if; - bool suspended; wait_queue_head_t wait_vsync_queue; atomic_t wait_vsync_event; @@ -81,13 +80,30 @@ static const enum drm_plane_type decon_win_types[WINDOWS_NR] = { DRM_PLANE_TYPE_CURSOR, }; -static void decon_wait_for_vblank(struct exynos_drm_crtc *crtc) +/** + * decon_shadow_protect_win() - disable updating values from shadow registers at vsync + * + * @ctx: display and enhancement controller context + * @win: window to protect registers for + * @protect: 1 to protect (disable updates) + */ +static void decon_shadow_protect_win(struct decon_context *ctx, + unsigned int win, bool protect) { - struct decon_context *ctx = crtc->ctx; + u32 bits, val; - if (ctx->suspended) - return; + bits = SHADOWCON_WINx_PROTECT(win); + val = readl(ctx->regs + SHADOWCON); + if (protect) + val |= bits; + else + val &= ~bits; + writel(val, ctx->regs + SHADOWCON); +} + +static void decon_wait_for_vblank(struct decon_context *ctx) +{ atomic_set(&ctx->wait_vsync_event, 1); /* @@ -100,25 +116,33 @@ static void decon_wait_for_vblank(struct exynos_drm_crtc *crtc) DRM_DEV_DEBUG_KMS(ctx->dev, "vblank wait timed out.\n"); } -static void decon_clear_channels(struct exynos_drm_crtc *crtc) +static void decon_clear_channels(struct decon_context *ctx) { - struct decon_context *ctx = crtc->ctx; unsigned int win, ch_enabled = 0; + u32 val; /* Check if any channel is enabled. */ for (win = 0; win < WINDOWS_NR; win++) { - u32 val = readl(ctx->regs + WINCON(win)); + val = readl(ctx->regs + WINCON(win)); if (val & WINCONx_ENWIN) { + decon_shadow_protect_win(ctx, win, true); + val &= ~WINCONx_ENWIN; writel(val, ctx->regs + WINCON(win)); ch_enabled = 1; + + decon_shadow_protect_win(ctx, win, false); } } + val = readl(ctx->regs + DECON_UPDATE); + val |= DECON_UPDATE_STANDALONE_F; + writel(val, ctx->regs + DECON_UPDATE); + /* Wait for vsync, as disable channel takes effect at next vsync */ if (ch_enabled) - decon_wait_for_vblank(ctx->crtc); + decon_wait_for_vblank(ctx); } static int decon_ctx_initialize(struct decon_context *ctx, @@ -126,7 +150,7 @@ static int decon_ctx_initialize(struct decon_context *ctx, { ctx->drm_dev = drm_dev; - decon_clear_channels(ctx->crtc); + decon_clear_channels(ctx); return exynos_drm_register_dma(drm_dev, ctx->dev, &ctx->dma_priv); } @@ -155,9 +179,6 @@ static void decon_commit(struct exynos_drm_crtc *crtc) struct drm_display_mode *mode = &crtc->base.state->adjusted_mode; u32 val, clkdiv; - if (ctx->suspended) - return; - /* nothing to do if we haven't set the mode yet */ if (mode->htotal == 0 || mode->vtotal == 0) return; @@ -219,9 +240,6 @@ static int decon_enable_vblank(struct exynos_drm_crtc *crtc) struct decon_context *ctx = crtc->ctx; u32 val; - if (ctx->suspended) - return -EPERM; - if (!test_and_set_bit(0, &ctx->irq_flags)) { val = readl(ctx->regs + VIDINTCON0); @@ -244,9 +262,6 @@ static void decon_disable_vblank(struct exynos_drm_crtc *crtc) struct decon_context *ctx = crtc->ctx; u32 val; - if (ctx->suspended) - return; - if (test_and_clear_bit(0, &ctx->irq_flags)) { val = readl(ctx->regs + VIDINTCON0); @@ -343,36 +358,11 @@ static void decon_win_set_colkey(struct decon_context *ctx, unsigned int win) writel(keycon1, ctx->regs + WKEYCON1_BASE(win)); } -/** - * decon_shadow_protect_win() - disable updating values from shadow registers at vsync - * - * @ctx: display and enhancement controller context - * @win: window to protect registers for - * @protect: 1 to protect (disable updates) - */ -static void decon_shadow_protect_win(struct decon_context *ctx, - unsigned int win, bool protect) -{ - u32 bits, val; - - bits = SHADOWCON_WINx_PROTECT(win); - - val = readl(ctx->regs + SHADOWCON); - if (protect) - val |= bits; - else - val &= ~bits; - writel(val, ctx->regs + SHADOWCON); -} - static void decon_atomic_begin(struct exynos_drm_crtc *crtc) { struct decon_context *ctx = crtc->ctx; int i; - if (ctx->suspended) - return; - for (i = 0; i < WINDOWS_NR; i++) decon_shadow_protect_win(ctx, i, true); } @@ -392,9 +382,6 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc, unsigned int cpp = fb->format->cpp[0]; unsigned int pitch = fb->pitches[0]; - if (ctx->suspended) - return; - /* * SHADOWCON/PRTCON register is used for enabling timing. * @@ -482,9 +469,6 @@ static void decon_disable_plane(struct exynos_drm_crtc *crtc, unsigned int win = plane->index; u32 val; - if (ctx->suspended) - return; - /* protect windows */ decon_shadow_protect_win(ctx, win, true); @@ -503,9 +487,6 @@ static void decon_atomic_flush(struct exynos_drm_crtc *crtc) struct decon_context *ctx = crtc->ctx; int i; - if (ctx->suspended) - return; - for (i = 0; i < WINDOWS_NR; i++) decon_shadow_protect_win(ctx, i, false); exynos_crtc_handle_event(crtc); @@ -533,9 +514,6 @@ static void decon_atomic_enable(struct exynos_drm_crtc *crtc) struct decon_context *ctx = crtc->ctx; int ret; - if (!ctx->suspended) - return; - ret = pm_runtime_resume_and_get(ctx->dev); if (ret < 0) { DRM_DEV_ERROR(ctx->dev, "failed to enable DECON device.\n"); @@ -549,8 +527,6 @@ static void decon_atomic_enable(struct exynos_drm_crtc *crtc) decon_enable_vblank(ctx->crtc); decon_commit(ctx->crtc); - - ctx->suspended = false; } static void decon_atomic_disable(struct exynos_drm_crtc *crtc) @@ -558,9 +534,6 @@ static void decon_atomic_disable(struct exynos_drm_crtc *crtc) struct decon_context *ctx = crtc->ctx; int i; - if (ctx->suspended) - return; - /* * We need to make sure that all windows are disabled before we * suspend that connector. Otherwise we might try to scan from @@ -570,8 +543,6 @@ static void decon_atomic_disable(struct exynos_drm_crtc *crtc) decon_disable_plane(crtc, &ctx->planes[i]); pm_runtime_put_sync(ctx->dev); - - ctx->suspended = true; } static const struct exynos_drm_crtc_ops decon_crtc_ops = { @@ -601,6 +572,10 @@ static irqreturn_t decon_irq_handler(int irq, void *dev_id) if (!ctx->drm_dev) goto out; + /* check if crtc and vblank have been initialized properly */ + if (!drm_dev_has_vblank(ctx->drm_dev)) + goto out; + if (!ctx->i80_if) { drm_crtc_handle_vblank(&ctx->crtc->base); @@ -688,7 +663,6 @@ static int decon_probe(struct platform_device *pdev) return -ENOMEM; ctx->dev = dev; - ctx->suspended = true; i80_if_timings = of_get_child_by_name(dev->of_node, "i80-if-timings"); if (i80_if_timings) diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c index 5bdc246f5fad0..341e95269836e 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c @@ -187,6 +187,7 @@ struct fimd_context { u32 i80ifcon; bool i80_if; bool suspended; + bool dp_clk_enabled; wait_queue_head_t wait_vsync_queue; atomic_t wait_vsync_event; atomic_t win_updated; @@ -1047,7 +1048,18 @@ static void fimd_dp_clock_enable(struct exynos_drm_clk *clk, bool enable) struct fimd_context *ctx = container_of(clk, struct fimd_context, dp_clk); u32 val = enable ? DP_MIE_CLK_DP_ENABLE : DP_MIE_CLK_DISABLE; + + if (enable == ctx->dp_clk_enabled) + return; + + if (enable) + pm_runtime_resume_and_get(ctx->dev); + + ctx->dp_clk_enabled = enable; writel(val, ctx->regs + DP_MIE_CLKCON); + + if (!enable) + pm_runtime_put(ctx->dev); } static const struct exynos_drm_crtc_ops fimd_crtc_ops = { diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c index ed8626c73541c..f0ae675581d9a 100644 --- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c +++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c @@ -726,8 +726,8 @@ void oaktrail_hdmi_teardown(struct drm_device *dev) if (hdmi_dev) { pdev = hdmi_dev->dev; - pci_set_drvdata(pdev, NULL); oaktrail_hdmi_i2c_exit(pdev); + pci_set_drvdata(pdev, NULL); iounmap(hdmi_dev->regs); kfree(hdmi_dev); pci_dev_put(pdev); diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c index 8a98fa276e8a9..96f960bcfd82d 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c @@ -258,13 +258,13 @@ static int hibmc_load(struct drm_device *dev) ret = hibmc_hw_init(priv); if (ret) - goto err; + return ret; ret = drmm_vram_helper_init(dev, pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); if (ret) { drm_err(dev, "Error initializing VRAM MM; %d\n", ret); - goto err; + return ret; } ret = hibmc_kms_init(priv); diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c index ff9b9918b0a13..d414627813aab 100644 --- a/drivers/gpu/drm/i915/display/intel_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_backlight.c @@ -40,8 +40,9 @@ static u32 scale(u32 source_val, { u64 target_val; - WARN_ON(source_min > source_max); - WARN_ON(target_min > target_max); + if (WARN_ON(source_min >= source_max) || + WARN_ON(target_min > target_max)) + return target_min; /* defensive */ source_val = clamp(source_val, source_min, source_max); diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 9e01054c24300..8beeda3439818 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -1170,7 +1170,7 @@ static void icl_mbus_init(struct drm_i915_private *dev_priv) if (DISPLAY_VER(dev_priv) == 12) abox_regs |= BIT(0); - for_each_set_bit(i, &abox_regs, sizeof(abox_regs)) + for_each_set_bit(i, &abox_regs, BITS_PER_TYPE(abox_regs)) intel_de_rmw(dev_priv, MBUS_ABOX_CTL(i), mask, val); } @@ -1623,11 +1623,11 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv) if (table[config].page_mask == 0) { drm_dbg(&dev_priv->drm, "Unknown memory configuration; disabling address buddy logic.\n"); - for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) + for_each_set_bit(i, &abox_mask, BITS_PER_TYPE(abox_mask)) intel_de_write(dev_priv, BW_BUDDY_CTL(i), BW_BUDDY_DISABLE); } else { - for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) { + for_each_set_bit(i, &abox_mask, BITS_PER_TYPE(abox_mask)) { intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i), table[config].page_mask); diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index c8b6d0f79c9b4..9a894e234f623 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -1293,6 +1293,12 @@ int intel_dp_rate_select(struct intel_dp *intel_dp, int rate) void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, u8 *link_bw, u8 *rate_select) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + + /* FIXME g4x can't generate an exact 2.7GHz with the 96MHz non-SSC refclk */ + if (IS_G4X(i915) && port_clock == 268800) + port_clock = 270000; + /* eDP 1.4 rate select method. */ if (intel_dp->use_rate_select) { *link_bw = 0; diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.c b/drivers/gpu/drm/i915/gt/intel_gsc.c index bcc3605158dbd..27420ed631d85 100644 --- a/drivers/gpu/drm/i915/gt/intel_gsc.c +++ b/drivers/gpu/drm/i915/gt/intel_gsc.c @@ -298,7 +298,7 @@ static void gsc_irq_handler(struct intel_gt *gt, unsigned int intf_id) if (gt->gsc.intf[intf_id].irq < 0) return; - ret = generic_handle_irq(gt->gsc.intf[intf_id].irq); + ret = generic_handle_irq_safe(gt->gsc.intf[intf_id].irq); if (ret) drm_err_ratelimited(>->i915->drm, "error handling GSC irq: %d\n", ret); } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c index 7c9be4fd1c8c4..7a950c1502b6e 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c @@ -208,7 +208,7 @@ static u64 div_u64_roundup(u64 nom, u32 den) u64 intel_gt_clock_interval_to_ns(const struct intel_gt *gt, u64 count) { - return div_u64_roundup(count * NSEC_PER_SEC, gt->clock_frequency); + return mul_u64_u32_div(count, NSEC_PER_SEC, gt->clock_frequency); } u64 intel_gt_pm_interval_to_ns(const struct intel_gt *gt, u64 count) @@ -218,7 +218,7 @@ u64 intel_gt_pm_interval_to_ns(const struct intel_gt *gt, u64 count) u64 intel_gt_ns_to_clock_interval(const struct intel_gt *gt, u64 ns) { - return div_u64_roundup(gt->clock_frequency * ns, NSEC_PER_SEC); + return mul_u64_u32_div(ns, gt->clock_frequency, NSEC_PER_SEC); } u64 intel_gt_ns_to_pm_interval(const struct intel_gt *gt, u64 ns) diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 92085ffd23de0..4eb78895773f6 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -573,7 +573,6 @@ static int ring_context_alloc(struct intel_context *ce) /* One ringbuffer to rule them all */ GEM_BUG_ON(!engine->legacy.ring); ce->ring = engine->legacy.ring; - ce->timeline = intel_timeline_get(engine->legacy.timeline); GEM_BUG_ON(ce->state); if (engine->context_size) { @@ -586,6 +585,8 @@ static int ring_context_alloc(struct intel_context *ce) ce->state = vma; } + ce->timeline = intel_timeline_get(engine->legacy.timeline); + return 0; } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c index 97eadd08181d6..38fad14ffd435 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c @@ -1281,9 +1281,16 @@ static int ct_receive(struct intel_guc_ct *ct) static void ct_try_receive_message(struct intel_guc_ct *ct) { + struct intel_guc *guc = ct_to_guc(ct); int ret; - if (GEM_WARN_ON(!ct->enabled)) + if (!ct->enabled) { + GEM_WARN_ON(!guc_to_gt(guc)->uc.reset_in_progress); + return; + } + + /* When interrupt disabled, message handling is not expected */ + if (!guc->interrupts.enabled) return; ret = ct_receive(ct); diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index 7b1076b5e748c..461aafc2ae9af 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -101,11 +101,11 @@ static unsigned int config_bit(const u64 config) return other_bit(config); } -static u32 config_mask(const u64 config) +static __always_inline u32 config_mask(const u64 config) { unsigned int bit = config_bit(config); - if (__builtin_constant_p(config)) + if (__builtin_constant_p(bit)) BUILD_BUG_ON(bit > BITS_PER_TYPE(typeof_member(struct i915_pmu, enable)) - 1); @@ -114,7 +114,7 @@ static u32 config_mask(const u64 config) BITS_PER_TYPE(typeof_member(struct i915_pmu, enable)) - 1); - return BIT(config_bit(config)); + return BIT(bit); } static bool is_engine_event(struct perf_event *event) diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 46e4a45e3c72a..772594360d57e 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -1586,8 +1586,20 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, err_vma_res: i915_vma_resource_free(vma_res); err_fence: - if (work) - dma_fence_work_commit_imm(&work->base); + if (work) { + /* + * When pinning VMA to GGTT on CHV or BXT with VTD enabled, + * commit VMA binding asynchronously to avoid risk of lock + * inversion among reservation_ww locks held here and + * cpu_hotplug_lock acquired from stop_machine(), which we + * wrap around GGTT updates when running in those environments. + */ + if (i915_vma_is_ggtt(vma) && + intel_vm_no_concurrent_access_wa(vma->vm->i915)) + dma_fence_work_commit(&work->base); + else + dma_fence_work_commit_imm(&work->base); + } err_rpm: intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref); diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_gsccs.h b/drivers/gpu/drm/i915/pxp/intel_pxp_gsccs.h index 298ad38e6c7df..c36d956b9b824 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp_gsccs.h +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_gsccs.h @@ -25,6 +25,7 @@ int intel_pxp_gsccs_init(struct intel_pxp *pxp); int intel_pxp_gsccs_create_session(struct intel_pxp *pxp, int arb_session_id); void intel_pxp_gsccs_end_arb_fw_session(struct intel_pxp *pxp, u32 arb_session_id); +bool intel_pxp_gsccs_is_ready_for_sessions(struct intel_pxp *pxp); #else static inline void intel_pxp_gsccs_fini(struct intel_pxp *pxp) @@ -36,8 +37,11 @@ static inline int intel_pxp_gsccs_init(struct intel_pxp *pxp) return 0; } -#endif +static inline bool intel_pxp_gsccs_is_ready_for_sessions(struct intel_pxp *pxp) +{ + return false; +} -bool intel_pxp_gsccs_is_ready_for_sessions(struct intel_pxp *pxp); +#endif #endif /*__INTEL_PXP_GSCCS_H__ */ diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index a9b79888c1931..c7ce2c570ad1f 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -73,8 +73,8 @@ static int igt_add_request(void *arg) /* Basic preliminary test to create a request and let it loose! */ request = mock_request(rcs0(i915)->kernel_context, HZ / 10); - if (!request) - return -ENOMEM; + if (IS_ERR(request)) + return PTR_ERR(request); i915_request_add(request); @@ -91,8 +91,8 @@ static int igt_wait_request(void *arg) /* Submit a request, then wait upon it */ request = mock_request(rcs0(i915)->kernel_context, T); - if (!request) - return -ENOMEM; + if (IS_ERR(request)) + return PTR_ERR(request); i915_request_get(request); @@ -160,8 +160,8 @@ static int igt_fence_wait(void *arg) /* Submit a request, treat it as a fence and wait upon it */ request = mock_request(rcs0(i915)->kernel_context, T); - if (!request) - return -ENOMEM; + if (IS_ERR(request)) + return PTR_ERR(request); if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) { pr_err("fence wait success before submit (expected timeout)!\n"); @@ -219,8 +219,8 @@ static int igt_request_rewind(void *arg) GEM_BUG_ON(IS_ERR(ce)); request = mock_request(ce, 2 * HZ); intel_context_put(ce); - if (!request) { - err = -ENOMEM; + if (IS_ERR(request)) { + err = PTR_ERR(request); goto err_context_0; } @@ -237,8 +237,8 @@ static int igt_request_rewind(void *arg) GEM_BUG_ON(IS_ERR(ce)); vip = mock_request(ce, 0); intel_context_put(ce); - if (!vip) { - err = -ENOMEM; + if (IS_ERR(vip)) { + err = PTR_ERR(vip); goto err_context_1; } diff --git a/drivers/gpu/drm/i915/selftests/mock_request.c b/drivers/gpu/drm/i915/selftests/mock_request.c index 09f747228dff5..1b0cf073e9643 100644 --- a/drivers/gpu/drm/i915/selftests/mock_request.c +++ b/drivers/gpu/drm/i915/selftests/mock_request.c @@ -35,7 +35,7 @@ mock_request(struct intel_context *ce, unsigned long delay) /* NB the i915->requests slab cache is enlarged to fit mock_request */ request = intel_context_create_request(ce); if (IS_ERR(request)) - return NULL; + return request; request->mock.delay = delay; return request; diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c index 54fc3f819577e..6391afdf202e2 100644 --- a/drivers/gpu/drm/mediatek/mtk_dpi.c +++ b/drivers/gpu/drm/mediatek/mtk_dpi.c @@ -410,12 +410,13 @@ static void mtk_dpi_config_swap_input(struct mtk_dpi *dpi, bool enable) static void mtk_dpi_config_2n_h_fre(struct mtk_dpi *dpi) { - mtk_dpi_mask(dpi, dpi->conf->reg_h_fre_con, H_FRE_2N, H_FRE_2N); + if (dpi->conf->reg_h_fre_con) + mtk_dpi_mask(dpi, dpi->conf->reg_h_fre_con, H_FRE_2N, H_FRE_2N); } static void mtk_dpi_config_disable_edge(struct mtk_dpi *dpi) { - if (dpi->conf->edge_sel_en) + if (dpi->conf->edge_sel_en && dpi->conf->reg_h_fre_con) mtk_dpi_mask(dpi, dpi->conf->reg_h_fre_con, 0, EDGE_SEL_EN); } diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c index 8b41a07c3641f..83dcc475769b1 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c @@ -352,6 +352,7 @@ static bool mtk_drm_get_all_drm_priv(struct device *dev) { struct mtk_drm_private *drm_priv = dev_get_drvdata(dev); struct mtk_drm_private *all_drm_priv[MAX_CRTC]; + struct mtk_drm_private *temp_drm_priv; struct device_node *phandle = dev->parent->of_node; const struct of_device_id *of_id; struct device_node *node; @@ -371,13 +372,29 @@ static bool mtk_drm_get_all_drm_priv(struct device *dev) continue; drm_dev = device_find_child(&pdev->dev, NULL, mtk_drm_match); - if (!drm_dev || !dev_get_drvdata(drm_dev)) - continue; + if (!drm_dev) + goto next_put_device_pdev_dev; + + temp_drm_priv = dev_get_drvdata(drm_dev); + if (!temp_drm_priv) + goto next_put_device_drm_dev; + + if (temp_drm_priv->data->main_len) + all_drm_priv[CRTC_MAIN] = temp_drm_priv; + else if (temp_drm_priv->data->ext_len) + all_drm_priv[CRTC_EXT] = temp_drm_priv; + else if (temp_drm_priv->data->third_len) + all_drm_priv[CRTC_THIRD] = temp_drm_priv; - all_drm_priv[cnt] = dev_get_drvdata(drm_dev); - if (all_drm_priv[cnt] && all_drm_priv[cnt]->mtk_drm_bound) + if (temp_drm_priv->mtk_drm_bound) cnt++; +next_put_device_drm_dev: + put_device(drm_dev); + +next_put_device_pdev_dev: + put_device(&pdev->dev); + if (cnt == MAX_CRTC) { of_node_put(node); break; @@ -431,7 +448,7 @@ static int mtk_drm_kms_init(struct drm_device *drm) ret = drmm_mode_config_init(drm); if (ret) - goto put_mutex_dev; + return ret; drm->mode_config.min_width = 64; drm->mode_config.min_height = 64; @@ -449,8 +466,11 @@ static int mtk_drm_kms_init(struct drm_device *drm) for (i = 0; i < private->data->mmsys_dev_num; i++) { drm->dev_private = private->all_drm_private[i]; ret = component_bind_all(private->all_drm_private[i]->dev, drm); - if (ret) - goto put_mutex_dev; + if (ret) { + while (--i >= 0) + component_unbind_all(private->all_drm_private[i]->dev, drm); + return ret; + } } /* @@ -472,21 +492,21 @@ static int mtk_drm_kms_init(struct drm_device *drm) for (j = 0; j < private->data->mmsys_dev_num; j++) { priv_n = private->all_drm_private[j]; - if (i == 0 && priv_n->data->main_len) { + if (i == CRTC_MAIN && priv_n->data->main_len) { ret = mtk_drm_crtc_create(drm, priv_n->data->main_path, priv_n->data->main_len, j); if (ret) goto err_component_unbind; continue; - } else if (i == 1 && priv_n->data->ext_len) { + } else if (i == CRTC_EXT && priv_n->data->ext_len) { ret = mtk_drm_crtc_create(drm, priv_n->data->ext_path, priv_n->data->ext_len, j); if (ret) goto err_component_unbind; continue; - } else if (i == 2 && priv_n->data->third_len) { + } else if (i == CRTC_THIRD && priv_n->data->third_len) { ret = mtk_drm_crtc_create(drm, priv_n->data->third_path, priv_n->data->third_len, j); if (ret) @@ -532,9 +552,6 @@ static int mtk_drm_kms_init(struct drm_device *drm) err_component_unbind: for (i = 0; i < private->data->mmsys_dev_num; i++) component_unbind_all(private->all_drm_private[i]->dev, drm); -put_mutex_dev: - for (i = 0; i < private->data->mmsys_dev_num; i++) - put_device(private->all_drm_private[i]->mutex_dev); return ret; } @@ -608,8 +625,10 @@ static int mtk_drm_bind(struct device *dev) return 0; drm = drm_dev_alloc(&mtk_drm_driver, dev); - if (IS_ERR(drm)) - return PTR_ERR(drm); + if (IS_ERR(drm)) { + ret = PTR_ERR(drm); + goto err_put_dev; + } private->drm_master = true; drm->dev_private = private; @@ -635,6 +654,8 @@ static int mtk_drm_bind(struct device *dev) drm_dev_put(drm); for (i = 0; i < private->data->mmsys_dev_num; i++) private->all_drm_private[i]->drm = NULL; +err_put_dev: + put_device(private->mutex_dev); return ret; } @@ -647,6 +668,7 @@ static void mtk_drm_unbind(struct device *dev) drm_dev_unregister(private->drm); mtk_drm_kms_deinit(private->drm); drm_dev_put(private->drm); + put_device(private->mutex_dev); } private->mtk_drm_bound = false; private->drm_master = false; diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h index eb2fd45941f09..f4de8bb276850 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h @@ -9,11 +9,17 @@ #include #include "mtk_drm_ddp_comp.h" -#define MAX_CRTC 3 #define MAX_CONNECTOR 2 #define DDP_COMPONENT_DRM_OVL_ADAPTOR (DDP_COMPONENT_ID_MAX + 1) #define DDP_COMPONENT_DRM_ID_MAX (DDP_COMPONENT_DRM_OVL_ADAPTOR + 1) +enum mtk_drm_crtc_path { + CRTC_MAIN, + CRTC_EXT, + CRTC_THIRD, + MAX_CRTC, +}; + struct device; struct device_node; struct drm_crtc; diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c index f10d4cc6c2234..32038cff27308 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c @@ -21,9 +21,6 @@ static const u64 modifiers[] = { DRM_FORMAT_MOD_LINEAR, - DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 | - AFBC_FORMAT_MOD_SPLIT | - AFBC_FORMAT_MOD_SPARSE), DRM_FORMAT_MOD_INVALID, }; @@ -71,26 +68,7 @@ static bool mtk_plane_format_mod_supported(struct drm_plane *plane, uint32_t format, uint64_t modifier) { - if (modifier == DRM_FORMAT_MOD_LINEAR) - return true; - - if (modifier != DRM_FORMAT_MOD_ARM_AFBC( - AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 | - AFBC_FORMAT_MOD_SPLIT | - AFBC_FORMAT_MOD_SPARSE)) - return false; - - if (format != DRM_FORMAT_XRGB8888 && - format != DRM_FORMAT_ARGB8888 && - format != DRM_FORMAT_BGRX8888 && - format != DRM_FORMAT_BGRA8888 && - format != DRM_FORMAT_ABGR8888 && - format != DRM_FORMAT_XBGR8888 && - format != DRM_FORMAT_RGB888 && - format != DRM_FORMAT_BGR888) - return false; - - return true; + return modifier == DRM_FORMAT_MOD_LINEAR; } static void mtk_drm_plane_destroy_state(struct drm_plane *plane, diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c index 095f634ff7c79..27441e5716300 100644 --- a/drivers/gpu/drm/meson/meson_drv.c +++ b/drivers/gpu/drm/meson/meson_drv.c @@ -168,7 +168,7 @@ static const struct meson_drm_soc_attr meson_drm_soc_attrs[] = { /* S805X/S805Y HDMI PLL won't lock for HDMI PHY freq > 1,65GHz */ { .limits = { - .max_hdmi_phy_freq = 1650000, + .max_hdmi_phy_freq = 1650000000, }, .attrs = (const struct soc_device_attribute []) { { .soc_id = "GXL (S805*)", }, diff --git a/drivers/gpu/drm/meson/meson_drv.h b/drivers/gpu/drm/meson/meson_drv.h index 3f9345c14f31c..be4b0e4df6e13 100644 --- a/drivers/gpu/drm/meson/meson_drv.h +++ b/drivers/gpu/drm/meson/meson_drv.h @@ -37,7 +37,7 @@ struct meson_drm_match_data { }; struct meson_drm_soc_limits { - unsigned int max_hdmi_phy_freq; + unsigned long long max_hdmi_phy_freq; }; struct meson_drm { diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.c b/drivers/gpu/drm/meson/meson_encoder_hdmi.c index c4686568c9ca5..0b7e8e0ba4fc1 100644 --- a/drivers/gpu/drm/meson/meson_encoder_hdmi.c +++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.c @@ -70,12 +70,12 @@ static void meson_encoder_hdmi_set_vclk(struct meson_encoder_hdmi *encoder_hdmi, { struct meson_drm *priv = encoder_hdmi->priv; int vic = drm_match_cea_mode(mode); - unsigned int phy_freq; - unsigned int vclk_freq; - unsigned int venc_freq; - unsigned int hdmi_freq; + unsigned long long phy_freq; + unsigned long long vclk_freq; + unsigned long long venc_freq; + unsigned long long hdmi_freq; - vclk_freq = mode->clock; + vclk_freq = mode->clock * 1000ULL; /* For 420, pixel clock is half unlike venc clock */ if (encoder_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24) @@ -107,7 +107,8 @@ static void meson_encoder_hdmi_set_vclk(struct meson_encoder_hdmi *encoder_hdmi, if (mode->flags & DRM_MODE_FLAG_DBLCLK) venc_freq /= 2; - dev_dbg(priv->dev, "vclk:%d phy=%d venc=%d hdmi=%d enci=%d\n", + dev_dbg(priv->dev, + "phy:%lluHz vclk=%lluHz venc=%lluHz hdmi=%lluHz enci=%d\n", phy_freq, vclk_freq, venc_freq, hdmi_freq, priv->venc.hdmi_use_enci); @@ -122,10 +123,11 @@ static enum drm_mode_status meson_encoder_hdmi_mode_valid(struct drm_bridge *bri struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge); struct meson_drm *priv = encoder_hdmi->priv; bool is_hdmi2_sink = display_info->hdmi.scdc.supported; - unsigned int phy_freq; - unsigned int vclk_freq; - unsigned int venc_freq; - unsigned int hdmi_freq; + unsigned long long clock = mode->clock * 1000ULL; + unsigned long long phy_freq; + unsigned long long vclk_freq; + unsigned long long venc_freq; + unsigned long long hdmi_freq; int vic = drm_match_cea_mode(mode); enum drm_mode_status status; @@ -144,12 +146,12 @@ static enum drm_mode_status meson_encoder_hdmi_mode_valid(struct drm_bridge *bri if (status != MODE_OK) return status; - return meson_vclk_dmt_supported_freq(priv, mode->clock); + return meson_vclk_dmt_supported_freq(priv, clock); /* Check against supported VIC modes */ } else if (!meson_venc_hdmi_supported_vic(vic)) return MODE_BAD; - vclk_freq = mode->clock; + vclk_freq = clock; /* For 420, pixel clock is half unlike venc clock */ if (drm_mode_is_420_only(display_info, mode) || @@ -179,7 +181,8 @@ static enum drm_mode_status meson_encoder_hdmi_mode_valid(struct drm_bridge *bri if (mode->flags & DRM_MODE_FLAG_DBLCLK) venc_freq /= 2; - dev_dbg(priv->dev, "%s: vclk:%d phy=%d venc=%d hdmi=%d\n", + dev_dbg(priv->dev, + "%s: vclk:%lluHz phy=%lluHz venc=%lluHz hdmi=%lluHz\n", __func__, phy_freq, vclk_freq, venc_freq, hdmi_freq); return meson_vclk_vic_supported_freq(priv, phy_freq, vclk_freq); diff --git a/drivers/gpu/drm/meson/meson_vclk.c b/drivers/gpu/drm/meson/meson_vclk.c index 2a942dc6a6dc2..dfe0c28a0f054 100644 --- a/drivers/gpu/drm/meson/meson_vclk.c +++ b/drivers/gpu/drm/meson/meson_vclk.c @@ -110,7 +110,7 @@ #define HDMI_PLL_LOCK BIT(31) #define HDMI_PLL_LOCK_G12A (3 << 30) -#define FREQ_1000_1001(_freq) DIV_ROUND_CLOSEST(_freq * 1000, 1001) +#define FREQ_1000_1001(_freq) DIV_ROUND_CLOSEST_ULL((_freq) * 1000ULL, 1001ULL) /* VID PLL Dividers */ enum { @@ -360,11 +360,11 @@ enum { }; struct meson_vclk_params { - unsigned int pll_freq; - unsigned int phy_freq; - unsigned int vclk_freq; - unsigned int venc_freq; - unsigned int pixel_freq; + unsigned long long pll_freq; + unsigned long long phy_freq; + unsigned long long vclk_freq; + unsigned long long venc_freq; + unsigned long long pixel_freq; unsigned int pll_od1; unsigned int pll_od2; unsigned int pll_od3; @@ -372,11 +372,11 @@ struct meson_vclk_params { unsigned int vclk_div; } params[] = { [MESON_VCLK_HDMI_ENCI_54000] = { - .pll_freq = 4320000, - .phy_freq = 270000, - .vclk_freq = 54000, - .venc_freq = 54000, - .pixel_freq = 54000, + .pll_freq = 4320000000, + .phy_freq = 270000000, + .vclk_freq = 54000000, + .venc_freq = 54000000, + .pixel_freq = 54000000, .pll_od1 = 4, .pll_od2 = 4, .pll_od3 = 1, @@ -384,11 +384,11 @@ struct meson_vclk_params { .vclk_div = 1, }, [MESON_VCLK_HDMI_DDR_54000] = { - .pll_freq = 4320000, - .phy_freq = 270000, - .vclk_freq = 54000, - .venc_freq = 54000, - .pixel_freq = 27000, + .pll_freq = 4320000000, + .phy_freq = 270000000, + .vclk_freq = 54000000, + .venc_freq = 54000000, + .pixel_freq = 27000000, .pll_od1 = 4, .pll_od2 = 4, .pll_od3 = 1, @@ -396,11 +396,11 @@ struct meson_vclk_params { .vclk_div = 1, }, [MESON_VCLK_HDMI_DDR_148500] = { - .pll_freq = 2970000, - .phy_freq = 742500, - .vclk_freq = 148500, - .venc_freq = 148500, - .pixel_freq = 74250, + .pll_freq = 2970000000, + .phy_freq = 742500000, + .vclk_freq = 148500000, + .venc_freq = 148500000, + .pixel_freq = 74250000, .pll_od1 = 4, .pll_od2 = 1, .pll_od3 = 1, @@ -408,11 +408,11 @@ struct meson_vclk_params { .vclk_div = 1, }, [MESON_VCLK_HDMI_74250] = { - .pll_freq = 2970000, - .phy_freq = 742500, - .vclk_freq = 74250, - .venc_freq = 74250, - .pixel_freq = 74250, + .pll_freq = 2970000000, + .phy_freq = 742500000, + .vclk_freq = 74250000, + .venc_freq = 74250000, + .pixel_freq = 74250000, .pll_od1 = 2, .pll_od2 = 2, .pll_od3 = 2, @@ -420,11 +420,11 @@ struct meson_vclk_params { .vclk_div = 1, }, [MESON_VCLK_HDMI_148500] = { - .pll_freq = 2970000, - .phy_freq = 1485000, - .vclk_freq = 148500, - .venc_freq = 148500, - .pixel_freq = 148500, + .pll_freq = 2970000000, + .phy_freq = 1485000000, + .vclk_freq = 148500000, + .venc_freq = 148500000, + .pixel_freq = 148500000, .pll_od1 = 1, .pll_od2 = 2, .pll_od3 = 2, @@ -432,11 +432,11 @@ struct meson_vclk_params { .vclk_div = 1, }, [MESON_VCLK_HDMI_297000] = { - .pll_freq = 5940000, - .phy_freq = 2970000, - .venc_freq = 297000, - .vclk_freq = 297000, - .pixel_freq = 297000, + .pll_freq = 5940000000, + .phy_freq = 2970000000, + .venc_freq = 297000000, + .vclk_freq = 297000000, + .pixel_freq = 297000000, .pll_od1 = 2, .pll_od2 = 1, .pll_od3 = 1, @@ -444,11 +444,11 @@ struct meson_vclk_params { .vclk_div = 2, }, [MESON_VCLK_HDMI_594000] = { - .pll_freq = 5940000, - .phy_freq = 5940000, - .venc_freq = 594000, - .vclk_freq = 594000, - .pixel_freq = 594000, + .pll_freq = 5940000000, + .phy_freq = 5940000000, + .venc_freq = 594000000, + .vclk_freq = 594000000, + .pixel_freq = 594000000, .pll_od1 = 1, .pll_od2 = 1, .pll_od3 = 2, @@ -456,11 +456,11 @@ struct meson_vclk_params { .vclk_div = 1, }, [MESON_VCLK_HDMI_594000_YUV420] = { - .pll_freq = 5940000, - .phy_freq = 2970000, - .venc_freq = 594000, - .vclk_freq = 594000, - .pixel_freq = 297000, + .pll_freq = 5940000000, + .phy_freq = 2970000000, + .venc_freq = 594000000, + .vclk_freq = 594000000, + .pixel_freq = 297000000, .pll_od1 = 2, .pll_od2 = 1, .pll_od3 = 1, @@ -617,16 +617,16 @@ static void meson_hdmi_pll_set_params(struct meson_drm *priv, unsigned int m, 3 << 20, pll_od_to_reg(od3) << 20); } -#define XTAL_FREQ 24000 +#define XTAL_FREQ (24 * 1000 * 1000) static unsigned int meson_hdmi_pll_get_m(struct meson_drm *priv, - unsigned int pll_freq) + unsigned long long pll_freq) { /* The GXBB PLL has a /2 pre-multiplier */ if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXBB)) - pll_freq /= 2; + pll_freq = DIV_ROUND_DOWN_ULL(pll_freq, 2); - return pll_freq / XTAL_FREQ; + return DIV_ROUND_DOWN_ULL(pll_freq, XTAL_FREQ); } #define HDMI_FRAC_MAX_GXBB 4096 @@ -635,12 +635,13 @@ static unsigned int meson_hdmi_pll_get_m(struct meson_drm *priv, static unsigned int meson_hdmi_pll_get_frac(struct meson_drm *priv, unsigned int m, - unsigned int pll_freq) + unsigned long long pll_freq) { - unsigned int parent_freq = XTAL_FREQ; + unsigned long long parent_freq = XTAL_FREQ; unsigned int frac_max = HDMI_FRAC_MAX_GXL; unsigned int frac_m; unsigned int frac; + u32 remainder; /* The GXBB PLL has a /2 pre-multiplier and a larger FRAC width */ if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXBB)) { @@ -652,11 +653,11 @@ static unsigned int meson_hdmi_pll_get_frac(struct meson_drm *priv, frac_max = HDMI_FRAC_MAX_G12A; /* We can have a perfect match !*/ - if (pll_freq / m == parent_freq && - pll_freq % m == 0) + if (div_u64_rem(pll_freq, m, &remainder) == parent_freq && + remainder == 0) return 0; - frac = div_u64((u64)pll_freq * (u64)frac_max, parent_freq); + frac = mul_u64_u64_div_u64(pll_freq, frac_max, parent_freq); frac_m = m * frac_max; if (frac_m > frac) return frac_max; @@ -666,7 +667,7 @@ static unsigned int meson_hdmi_pll_get_frac(struct meson_drm *priv, } static bool meson_hdmi_pll_validate_params(struct meson_drm *priv, - unsigned int m, + unsigned long long m, unsigned int frac) { if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXBB)) { @@ -694,7 +695,7 @@ static bool meson_hdmi_pll_validate_params(struct meson_drm *priv, } static bool meson_hdmi_pll_find_params(struct meson_drm *priv, - unsigned int freq, + unsigned long long freq, unsigned int *m, unsigned int *frac, unsigned int *od) @@ -706,7 +707,7 @@ static bool meson_hdmi_pll_find_params(struct meson_drm *priv, continue; *frac = meson_hdmi_pll_get_frac(priv, *m, freq * *od); - DRM_DEBUG_DRIVER("PLL params for %dkHz: m=%x frac=%x od=%d\n", + DRM_DEBUG_DRIVER("PLL params for %lluHz: m=%x frac=%x od=%d\n", freq, *m, *frac, *od); if (meson_hdmi_pll_validate_params(priv, *m, *frac)) @@ -718,7 +719,7 @@ static bool meson_hdmi_pll_find_params(struct meson_drm *priv, /* pll_freq is the frequency after the OD dividers */ enum drm_mode_status -meson_vclk_dmt_supported_freq(struct meson_drm *priv, unsigned int freq) +meson_vclk_dmt_supported_freq(struct meson_drm *priv, unsigned long long freq) { unsigned int od, m, frac; @@ -741,7 +742,7 @@ EXPORT_SYMBOL_GPL(meson_vclk_dmt_supported_freq); /* pll_freq is the frequency after the OD dividers */ static void meson_hdmi_pll_generic_set(struct meson_drm *priv, - unsigned int pll_freq) + unsigned long long pll_freq) { unsigned int od, m, frac, od1, od2, od3; @@ -756,7 +757,7 @@ static void meson_hdmi_pll_generic_set(struct meson_drm *priv, od1 = od / od2; } - DRM_DEBUG_DRIVER("PLL params for %dkHz: m=%x frac=%x od=%d/%d/%d\n", + DRM_DEBUG_DRIVER("PLL params for %lluHz: m=%x frac=%x od=%d/%d/%d\n", pll_freq, m, frac, od1, od2, od3); meson_hdmi_pll_set_params(priv, m, frac, od1, od2, od3); @@ -764,17 +765,48 @@ static void meson_hdmi_pll_generic_set(struct meson_drm *priv, return; } - DRM_ERROR("Fatal, unable to find parameters for PLL freq %d\n", + DRM_ERROR("Fatal, unable to find parameters for PLL freq %lluHz\n", pll_freq); } +static bool meson_vclk_freqs_are_matching_param(unsigned int idx, + unsigned long long phy_freq, + unsigned long long vclk_freq) +{ + DRM_DEBUG_DRIVER("i = %d vclk_freq = %lluHz alt = %lluHz\n", + idx, params[idx].vclk_freq, + FREQ_1000_1001(params[idx].vclk_freq)); + DRM_DEBUG_DRIVER("i = %d phy_freq = %lluHz alt = %lluHz\n", + idx, params[idx].phy_freq, + FREQ_1000_1001(params[idx].phy_freq)); + + /* Match strict frequency */ + if (phy_freq == params[idx].phy_freq && + vclk_freq == params[idx].vclk_freq) + return true; + + /* Match 1000/1001 variant: vclk deviation has to be less than 1kHz + * (drm EDID is defined in 1kHz steps, so everything smaller must be + * rounding error) and the PHY freq deviation has to be less than + * 10kHz (as the TMDS clock is 10 times the pixel clock, so anything + * smaller must be rounding error as well). + */ + if (abs(vclk_freq - FREQ_1000_1001(params[idx].vclk_freq)) < 1000 && + abs(phy_freq - FREQ_1000_1001(params[idx].phy_freq)) < 10000) + return true; + + /* no match */ + return false; +} + enum drm_mode_status -meson_vclk_vic_supported_freq(struct meson_drm *priv, unsigned int phy_freq, - unsigned int vclk_freq) +meson_vclk_vic_supported_freq(struct meson_drm *priv, + unsigned long long phy_freq, + unsigned long long vclk_freq) { int i; - DRM_DEBUG_DRIVER("phy_freq = %d vclk_freq = %d\n", + DRM_DEBUG_DRIVER("phy_freq = %lluHz vclk_freq = %lluHz\n", phy_freq, vclk_freq); /* Check against soc revision/package limits */ @@ -785,19 +817,7 @@ meson_vclk_vic_supported_freq(struct meson_drm *priv, unsigned int phy_freq, } for (i = 0 ; params[i].pixel_freq ; ++i) { - DRM_DEBUG_DRIVER("i = %d pixel_freq = %d alt = %d\n", - i, params[i].pixel_freq, - FREQ_1000_1001(params[i].pixel_freq)); - DRM_DEBUG_DRIVER("i = %d phy_freq = %d alt = %d\n", - i, params[i].phy_freq, - FREQ_1000_1001(params[i].phy_freq/1000)*1000); - /* Match strict frequency */ - if (phy_freq == params[i].phy_freq && - vclk_freq == params[i].vclk_freq) - return MODE_OK; - /* Match 1000/1001 variant */ - if (phy_freq == (FREQ_1000_1001(params[i].phy_freq/1000)*1000) && - vclk_freq == FREQ_1000_1001(params[i].vclk_freq)) + if (meson_vclk_freqs_are_matching_param(i, phy_freq, vclk_freq)) return MODE_OK; } @@ -805,8 +825,9 @@ meson_vclk_vic_supported_freq(struct meson_drm *priv, unsigned int phy_freq, } EXPORT_SYMBOL_GPL(meson_vclk_vic_supported_freq); -static void meson_vclk_set(struct meson_drm *priv, unsigned int pll_base_freq, - unsigned int od1, unsigned int od2, unsigned int od3, +static void meson_vclk_set(struct meson_drm *priv, + unsigned long long pll_base_freq, unsigned int od1, + unsigned int od2, unsigned int od3, unsigned int vid_pll_div, unsigned int vclk_div, unsigned int hdmi_tx_div, unsigned int venc_div, bool hdmi_use_enci, bool vic_alternate_clock) @@ -826,15 +847,15 @@ static void meson_vclk_set(struct meson_drm *priv, unsigned int pll_base_freq, meson_hdmi_pll_generic_set(priv, pll_base_freq); } else if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXBB)) { switch (pll_base_freq) { - case 2970000: + case 2970000000: m = 0x3d; frac = vic_alternate_clock ? 0xd02 : 0xe00; break; - case 4320000: + case 4320000000: m = vic_alternate_clock ? 0x59 : 0x5a; frac = vic_alternate_clock ? 0xe8f : 0; break; - case 5940000: + case 5940000000: m = 0x7b; frac = vic_alternate_clock ? 0xa05 : 0xc00; break; @@ -844,15 +865,15 @@ static void meson_vclk_set(struct meson_drm *priv, unsigned int pll_base_freq, } else if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXM) || meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXL)) { switch (pll_base_freq) { - case 2970000: + case 2970000000: m = 0x7b; frac = vic_alternate_clock ? 0x281 : 0x300; break; - case 4320000: + case 4320000000: m = vic_alternate_clock ? 0xb3 : 0xb4; frac = vic_alternate_clock ? 0x347 : 0; break; - case 5940000: + case 5940000000: m = 0xf7; frac = vic_alternate_clock ? 0x102 : 0x200; break; @@ -861,15 +882,15 @@ static void meson_vclk_set(struct meson_drm *priv, unsigned int pll_base_freq, meson_hdmi_pll_set_params(priv, m, frac, od1, od2, od3); } else if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_G12A)) { switch (pll_base_freq) { - case 2970000: + case 2970000000: m = 0x7b; frac = vic_alternate_clock ? 0x140b4 : 0x18000; break; - case 4320000: + case 4320000000: m = vic_alternate_clock ? 0xb3 : 0xb4; frac = vic_alternate_clock ? 0x1a3ee : 0; break; - case 5940000: + case 5940000000: m = 0xf7; frac = vic_alternate_clock ? 0x8148 : 0x10000; break; @@ -1025,14 +1046,14 @@ static void meson_vclk_set(struct meson_drm *priv, unsigned int pll_base_freq, } void meson_vclk_setup(struct meson_drm *priv, unsigned int target, - unsigned int phy_freq, unsigned int vclk_freq, - unsigned int venc_freq, unsigned int dac_freq, + unsigned long long phy_freq, unsigned long long vclk_freq, + unsigned long long venc_freq, unsigned long long dac_freq, bool hdmi_use_enci) { bool vic_alternate_clock = false; - unsigned int freq; - unsigned int hdmi_tx_div; - unsigned int venc_div; + unsigned long long freq; + unsigned long long hdmi_tx_div; + unsigned long long venc_div; if (target == MESON_VCLK_TARGET_CVBS) { meson_venci_cvbs_clock_config(priv); @@ -1052,27 +1073,25 @@ void meson_vclk_setup(struct meson_drm *priv, unsigned int target, return; } - hdmi_tx_div = vclk_freq / dac_freq; + hdmi_tx_div = DIV_ROUND_DOWN_ULL(vclk_freq, dac_freq); if (hdmi_tx_div == 0) { - pr_err("Fatal Error, invalid HDMI-TX freq %d\n", + pr_err("Fatal Error, invalid HDMI-TX freq %lluHz\n", dac_freq); return; } - venc_div = vclk_freq / venc_freq; + venc_div = DIV_ROUND_DOWN_ULL(vclk_freq, venc_freq); if (venc_div == 0) { - pr_err("Fatal Error, invalid HDMI venc freq %d\n", + pr_err("Fatal Error, invalid HDMI venc freq %lluHz\n", venc_freq); return; } for (freq = 0 ; params[freq].pixel_freq ; ++freq) { - if ((phy_freq == params[freq].phy_freq || - phy_freq == FREQ_1000_1001(params[freq].phy_freq/1000)*1000) && - (vclk_freq == params[freq].vclk_freq || - vclk_freq == FREQ_1000_1001(params[freq].vclk_freq))) { + if (meson_vclk_freqs_are_matching_param(freq, phy_freq, + vclk_freq)) { if (vclk_freq != params[freq].vclk_freq) vic_alternate_clock = true; else @@ -1098,7 +1117,8 @@ void meson_vclk_setup(struct meson_drm *priv, unsigned int target, } if (!params[freq].pixel_freq) { - pr_err("Fatal Error, invalid HDMI vclk freq %d\n", vclk_freq); + pr_err("Fatal Error, invalid HDMI vclk freq %lluHz\n", + vclk_freq); return; } diff --git a/drivers/gpu/drm/meson/meson_vclk.h b/drivers/gpu/drm/meson/meson_vclk.h index 60617aaf18dd1..7ac55744e5749 100644 --- a/drivers/gpu/drm/meson/meson_vclk.h +++ b/drivers/gpu/drm/meson/meson_vclk.h @@ -20,17 +20,18 @@ enum { }; /* 27MHz is the CVBS Pixel Clock */ -#define MESON_VCLK_CVBS 27000 +#define MESON_VCLK_CVBS (27 * 1000 * 1000) enum drm_mode_status -meson_vclk_dmt_supported_freq(struct meson_drm *priv, unsigned int freq); +meson_vclk_dmt_supported_freq(struct meson_drm *priv, unsigned long long freq); enum drm_mode_status -meson_vclk_vic_supported_freq(struct meson_drm *priv, unsigned int phy_freq, - unsigned int vclk_freq); +meson_vclk_vic_supported_freq(struct meson_drm *priv, + unsigned long long phy_freq, + unsigned long long vclk_freq); void meson_vclk_setup(struct meson_drm *priv, unsigned int target, - unsigned int phy_freq, unsigned int vclk_freq, - unsigned int venc_freq, unsigned int dac_freq, + unsigned long long phy_freq, unsigned long long vclk_freq, + unsigned long long venc_freq, unsigned long long dac_freq, bool hdmi_use_enci); #endif /* __MESON_VCLK_H */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index e7136b7759cb3..e816ddcac2f8d 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -230,6 +230,8 @@ static int a6xx_gmu_start(struct a6xx_gmu *gmu) if (ret) DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n"); + set_bit(GMU_STATUS_FW_START, &gmu->status); + return ret; } @@ -460,9 +462,10 @@ static int a6xx_rpmh_start(struct a6xx_gmu *gmu) int ret; u32 val; - gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1 << 1); - /* Wait for the register to finish posting */ - wmb(); + if (!test_and_clear_bit(GMU_STATUS_PDC_SLEEP, &gmu->status)) + return 0; + + gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, BIT(1)); ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val, val & (1 << 1), 100, 10000); @@ -489,6 +492,9 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) int ret; u32 val; + if (test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) + return; + gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1); ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, @@ -497,6 +503,8 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n"); gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0); + + set_bit(GMU_STATUS_PDC_SLEEP, &gmu->status); } static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value) @@ -617,8 +625,6 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) /* ensure no writes happen before the uCode is fully written */ wmb(); - a6xx_rpmh_stop(gmu); - err: if (!IS_ERR_OR_NULL(pdcptr)) iounmap(pdcptr); @@ -687,6 +693,9 @@ static bool fw_block_mem(struct a6xx_gmu_bo *bo, const struct block_header *blk) return true; } +#define NEXT_BLK(blk) \ + ((const struct block_header *)((const char *)(blk) + sizeof(*(blk)) + (blk)->size)) + static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu) { struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); @@ -717,7 +726,7 @@ static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu) for (blk = (const struct block_header *) fw_image->data; (const u8*) blk < fw_image->data + fw_image->size; - blk = (const struct block_header *) &blk->data[blk->size >> 2]) { + blk = NEXT_BLK(blk)) { if (blk->size == 0) continue; @@ -755,22 +764,18 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1); } - if (state == GMU_WARM_BOOT) { - ret = a6xx_rpmh_start(gmu); - if (ret) - return ret; - } else { + /* Turn on register retention */ + gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1); + + ret = a6xx_rpmh_start(gmu); + if (ret) + return ret; + + if (state == GMU_COLD_BOOT) { if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU], "GMU firmware is not loaded\n")) return -ENOENT; - /* Turn on register retention */ - gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1); - - ret = a6xx_rpmh_start(gmu); - if (ret) - return ret; - ret = a6xx_gmu_fw_load(gmu); if (ret) return ret; @@ -909,6 +914,8 @@ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) /* Reset GPU core blocks */ a6xx_gpu_sw_reset(gpu, true); + + a6xx_rpmh_stop(gmu); } static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h index 236f81a43caa6..6a28ecaf8594e 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -96,6 +96,12 @@ struct a6xx_gmu { /* For power domain callback */ struct notifier_block pd_nb; struct completion pd_gate; + +/* To check if we can trigger sleep seq at PDC. Cleared in a6xx_rpmh_stop() */ +#define GMU_STATUS_FW_START 0 +/* To track if PDC sleep seq was done */ +#define GMU_STATUS_PDC_SLEEP 1 + unsigned long status; }; static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 3664c1476a83a..4654c0f362c7d 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -1209,14 +1209,16 @@ static int hw_init(struct msm_gpu *gpu) /* Clear GBIF halt in case GX domain was not collapsed */ if (adreno_is_a619_holi(adreno_gpu)) { gpu_write(gpu, REG_A6XX_GBIF_HALT, 0); + gpu_read(gpu, REG_A6XX_GBIF_HALT); + gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, 0); - /* Let's make extra sure that the GPU can access the memory.. */ - mb(); + gpu_read(gpu, REG_A6XX_RBBM_GPR0_CNTL); } else if (a6xx_has_gbif(adreno_gpu)) { gpu_write(gpu, REG_A6XX_GBIF_HALT, 0); + gpu_read(gpu, REG_A6XX_GBIF_HALT); + gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0); - /* Let's make extra sure that the GPU can access the memory.. */ - mb(); + gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT); } gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); @@ -1700,6 +1702,9 @@ static void a6xx_fault_detect_irq(struct msm_gpu *gpu) /* Turn off the hangcheck timer to keep it from bothering us */ del_timer(&gpu->hangcheck_timer); + /* Turn off interrupts to avoid triggering recovery again */ + gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, 0); + kthread_queue_work(gpu->worker, &gpu->recover_work); } diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_1_sc8180x.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_1_sc8180x.h index 427dec0cd1d36..b77ecec50733b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_1_sc8180x.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_1_sc8180x.h @@ -378,6 +378,7 @@ static const struct dpu_perf_cfg sc8180x_perf_data = { .min_core_ib = 2400000, .min_llcc_ib = 800000, .min_dram_ib = 800000, + .min_prefill_lines = 24, .danger_lut_tbl = {0xf, 0xffff, 0x0}, .safe_lut_tbl = {0xfff0, 0xf000, 0xffff}, .qos_lut_tbl = { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c index 20c8b9af7a219..2cda9bbf68f96 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c @@ -93,17 +93,21 @@ static void drm_mode_to_intf_timing_params( timing->vsync_polarity = 0; } - /* for DP/EDP, Shift timings to align it to bottom right */ - if (phys_enc->hw_intf->cap->type == INTF_DP) { + timing->wide_bus_en = dpu_encoder_is_widebus_enabled(phys_enc->parent); + timing->compression_en = dpu_encoder_is_dsc_enabled(phys_enc->parent); + + /* + * For DP/EDP, Shift timings to align it to bottom right. + * wide_bus_en is set for everything excluding SDM845 & + * porch changes cause DisplayPort failure and HDMI tearing. + */ + if (phys_enc->hw_intf->cap->type == INTF_DP && timing->wide_bus_en) { timing->h_back_porch += timing->h_front_porch; timing->h_front_porch = 0; timing->v_back_porch += timing->v_front_porch; timing->v_front_porch = 0; } - timing->wide_bus_en = dpu_encoder_is_widebus_enabled(phys_enc->parent); - timing->compression_en = dpu_encoder_is_dsc_enabled(phys_enc->parent); - /* * for DP, divide the horizonal parameters by 2 when * widebus is enabled diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c index 0a45c546b03f2..a5cd41c9224d7 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c @@ -433,7 +433,7 @@ static void _dpu_encoder_phys_wb_handle_wbdone_timeout( static int dpu_encoder_phys_wb_wait_for_commit_done( struct dpu_encoder_phys *phys_enc) { - unsigned long ret; + int ret; struct dpu_encoder_wait_info wait_info; struct dpu_encoder_phys_wb *wb_enc = to_dpu_encoder_phys_wb(phys_enc); diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c index 27b592c776a30..1c111969342a7 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c @@ -716,6 +716,13 @@ static int dsi_pll_10nm_init(struct msm_dsi_phy *phy) /* TODO: Remove this when we have proper display handover support */ msm_dsi_phy_pll_save_state(phy); + /* + * Store also proper vco_current_rate, because its value will be used in + * dsi_10nm_pll_restore_state(). + */ + if (!dsi_pll_10nm_vco_recalc_rate(&pll_10nm->clk_hw, VCO_REF_CLK_RATE)) + pll_10nm->vco_current_rate = pll_10nm->phy->cfg->min_pll_rate; + return 0; } diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c index f72ce6a3c456d..bed9867ced6d4 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c @@ -445,6 +445,10 @@ static int dsi_pll_7nm_vco_prepare(struct clk_hw *hw) if (pll_7nm->slave) dsi_pll_enable_global_clk(pll_7nm->slave); + writel(0x1, pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_RBUF_CTRL); + if (pll_7nm->slave) + writel(0x1, pll_7nm->slave->phy->base + REG_DSI_7nm_PHY_CMN_RBUF_CTRL); + error: return rc; } @@ -793,6 +797,12 @@ static int dsi_pll_7nm_init(struct msm_dsi_phy *phy) /* TODO: Remove this when we have proper display handover support */ msm_dsi_phy_pll_save_state(phy); + /* + * Store also proper vco_current_rate, because its value will be used in + * dsi_7nm_pll_restore_state(). + */ + if (!dsi_pll_7nm_vco_recalc_rate(&pll_7nm->clk_hw, VCO_REF_CLK_RATE)) + pll_7nm->vco_current_rate = pll_7nm->phy->cfg->min_pll_rate; return 0; } diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c old mode 100644 new mode 100755 diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 1113e6b2ec8ec..aaf7c338eb96d 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -928,7 +928,8 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, uint64_t off = drm_vma_node_start(&obj->vma_node); const char *madv; - msm_gem_lock(obj); + if (!msm_gem_trylock(obj)) + return; stats->all.count++; stats->all.size += obj->size; diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 8ddef54431408..631a9aa129bd5 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -183,6 +183,12 @@ msm_gem_lock(struct drm_gem_object *obj) dma_resv_lock(obj->resv, NULL); } +static inline bool __must_check +msm_gem_trylock(struct drm_gem_object *obj) +{ + return dma_resv_trylock(obj->resv); +} + static inline int msm_gem_lock_interruptible(struct drm_gem_object *obj) { diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c index 018b39546fc1d..dc142f9e4f602 100644 --- a/drivers/gpu/drm/msm/msm_gem_submit.c +++ b/drivers/gpu/drm/msm/msm_gem_submit.c @@ -85,6 +85,15 @@ void __msm_gem_submit_destroy(struct kref *kref) container_of(kref, struct msm_gem_submit, ref); unsigned i; + /* + * In error paths, we could unref the submit without calling + * drm_sched_entity_push_job(), so msm_job_free() will never + * get called. Since drm_sched_job_cleanup() will NULL out + * s_fence, we can use that to detect this case. + */ + if (submit->base.s_fence) + drm_sched_job_cleanup(&submit->base); + if (submit->fence_id) { spin_lock(&submit->queue->idr_lock); idr_remove(&submit->queue->fence_idr, submit->fence_id); @@ -754,6 +763,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, struct msm_ringbuffer *ring; struct msm_submit_post_dep *post_deps = NULL; struct drm_syncobj **syncobjs_to_reset = NULL; + struct sync_file *sync_file = NULL; int out_fence_fd = -1; bool has_ww_ticket = false; unsigned i; @@ -970,13 +980,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, } if (ret == 0 && args->flags & MSM_SUBMIT_FENCE_FD_OUT) { - struct sync_file *sync_file = sync_file_create(submit->user_fence); - if (!sync_file) { + sync_file = sync_file_create(submit->user_fence); + if (!sync_file) ret = -ENOMEM; - } else { - fd_install(out_fence_fd, sync_file->file); - args->fence_fd = out_fence_fd; - } } submit_attach_object_fences(submit); @@ -1003,8 +1009,15 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, out_unlock: mutex_unlock(&queue->lock); out_post_unlock: - if (ret && (out_fence_fd >= 0)) - put_unused_fd(out_fence_fd); + if (ret) { + if (out_fence_fd >= 0) + put_unused_fd(out_fence_fd); + if (sync_file) + fput(sync_file->file); + } else if (sync_file) { + fd_install(out_fence_fd, sync_file->file); + args->fence_fd = out_fence_fd; + } if (!IS_ERR_OR_NULL(submit)) { msm_gem_submit_put(submit); diff --git a/drivers/gpu/drm/msm/msm_gpu_devfreq.c b/drivers/gpu/drm/msm/msm_gpu_devfreq.c index 6970b0f7f457c..2e1d5c3432728 100644 --- a/drivers/gpu/drm/msm/msm_gpu_devfreq.c +++ b/drivers/gpu/drm/msm/msm_gpu_devfreq.c @@ -156,6 +156,7 @@ void msm_devfreq_init(struct msm_gpu *gpu) priv->gpu_devfreq_config.downdifferential = 10; mutex_init(&df->lock); + df->suspended = true; ret = dev_pm_qos_add_request(&gpu->pdev->dev, &df->boost_freq, DEV_PM_QOS_MIN_FREQUENCY, 0); diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c index 7a2cceaee6e97..1199dfc1194c8 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c +++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c @@ -663,6 +663,10 @@ static bool nv50_plane_format_mod_supported(struct drm_plane *plane, struct nouveau_drm *drm = nouveau_drm(plane->dev); uint8_t i; + /* All chipsets can display all formats in linear layout */ + if (modifier == DRM_FORMAT_MOD_LINEAR) + return true; + if (drm->client.device.info.chipset < 0xc0) { const struct drm_format_info *info = drm_format_info(format); const uint8_t kind = (modifier >> 12) & 0xff; diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c index 91b5ecc575380..4e6f305c9e504 100644 --- a/drivers/gpu/drm/nouveau/nouveau_backlight.c +++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c @@ -42,7 +42,7 @@ #include "nouveau_acpi.h" static struct ida bl_ida; -#define BL_NAME_SIZE 15 // 12 for name + 2 for digits + 1 for '\0' +#define BL_NAME_SIZE 24 // 12 for name + 11 for digits + 1 for '\0' static bool nouveau_get_backlight_name(char backlight_name[BL_NAME_SIZE], diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 036ee03439728..11f164a15d7c2 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -836,7 +836,7 @@ nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, nvif_vmm_put(vmm, &old_mem->vma[1]); nvif_vmm_put(vmm, &old_mem->vma[0]); } - return 0; + return ret; } static int diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index 03eacb22648ef..1bfa312d6fb85 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -90,7 +90,7 @@ nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error) while (!list_empty(&fctx->pending)) { fence = list_entry(fctx->pending.next, typeof(*fence), head); - if (error) + if (error && !dma_fence_is_signaled_locked(&fence->base)) dma_fence_set_error(&fence->base, error); if (nouveau_fence_signal(fence)) diff --git a/drivers/gpu/drm/nouveau/nvif/vmm.c b/drivers/gpu/drm/nouveau/nvif/vmm.c index 99296f03371ae..07c1ebc2a9414 100644 --- a/drivers/gpu/drm/nouveau/nvif/vmm.c +++ b/drivers/gpu/drm/nouveau/nvif/vmm.c @@ -219,7 +219,8 @@ nvif_vmm_ctor(struct nvif_mmu *mmu, const char *name, s32 oclass, case RAW: args->type = NVIF_VMM_V0_TYPE_RAW; break; default: WARN_ON(1); - return -EINVAL; + ret = -EINVAL; + goto done; } memcpy(args->data, argv, argc); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/enum.c b/drivers/gpu/drm/nouveau/nvkm/core/enum.c index b9581feb24ccb..a23b40b27b81b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/enum.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/enum.c @@ -44,7 +44,7 @@ nvkm_snprintbf(char *data, int size, const struct nvkm_bitfield *bf, u32 value) bool space = false; while (size >= 1 && bf->name) { if (value & bf->mask) { - int this = snprintf(data, size, "%s%s", + int this = scnprintf(data, size, "%s%s", space ? " " : "", bf->name); size -= this; data += this; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 5db37247dc29b..572c54a370913 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -348,6 +348,8 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) nvkm_chid_unref(&fifo->chid); nvkm_event_fini(&fifo->nonstall.event); + if (fifo->func->nonstall_dtor) + fifo->func->nonstall_dtor(fifo); mutex_destroy(&fifo->mutex); return fifo; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c index c56d2a839efba..686a2c9fec46d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c @@ -516,19 +516,11 @@ ga100_fifo_nonstall_intr(struct nvkm_inth *inth) static void ga100_fifo_nonstall_block(struct nvkm_event *event, int type, int index) { - struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); - struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0); - - nvkm_inth_block(&runl->nonstall.inth); } static void ga100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index) { - struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); - struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0); - - nvkm_inth_allow(&runl->nonstall.inth); } const struct nvkm_event_func @@ -559,12 +551,26 @@ ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo) if (ret) return ret; + nvkm_inth_allow(&runl->nonstall.inth); + nr = max(nr, runl->id + 1); } return nr; } +void +ga100_fifo_nonstall_dtor(struct nvkm_fifo *fifo) +{ + struct nvkm_runl *runl; + + nvkm_runl_foreach(runl, fifo) { + if (runl->nonstall.vector < 0) + continue; + nvkm_inth_block(&runl->nonstall.inth); + } +} + int ga100_fifo_runl_ctor(struct nvkm_fifo *fifo) { @@ -594,6 +600,7 @@ ga100_fifo = { .runl_ctor = ga100_fifo_runl_ctor, .mmu_fault = &tu102_fifo_mmu_fault, .nonstall_ctor = ga100_fifo_nonstall_ctor, + .nonstall_dtor = ga100_fifo_nonstall_dtor, .nonstall = &ga100_fifo_nonstall, .runl = &ga100_runl, .runq = &ga100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c index 2cdf5da339b60..dccf38101fd9e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c @@ -28,6 +28,7 @@ ga102_fifo = { .runl_ctor = ga100_fifo_runl_ctor, .mmu_fault = &tu102_fifo_mmu_fault, .nonstall_ctor = ga100_fifo_nonstall_ctor, + .nonstall_dtor = ga100_fifo_nonstall_dtor, .nonstall = &ga100_fifo_nonstall, .runl = &ga100_runl, .runq = &ga100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 4d448be19224a..b4ccf6b8bd21a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -38,6 +38,7 @@ struct nvkm_fifo_func { void (*start)(struct nvkm_fifo *, unsigned long *); int (*nonstall_ctor)(struct nvkm_fifo *); + void (*nonstall_dtor)(struct nvkm_fifo *); const struct nvkm_event_func *nonstall; const struct nvkm_runl_func *runl; @@ -194,6 +195,7 @@ extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; int ga100_fifo_runl_ctor(struct nvkm_fifo *); int ga100_fifo_nonstall_ctor(struct nvkm_fifo *); +void ga100_fifo_nonstall_dtor(struct nvkm_fifo *); extern const struct nvkm_event_func ga100_fifo_nonstall; extern const struct nvkm_runl_func ga100_runl; extern const struct nvkm_runq_func ga100_runq; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c index cac6d64ab67d1..4e8b3f1c7e25d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c @@ -159,6 +159,8 @@ nvkm_falcon_fw_dtor(struct nvkm_falcon_fw *fw) nvkm_memory_unref(&fw->inst); nvkm_falcon_fw_dtor_sigs(fw); nvkm_firmware_dtor(&fw->fw); + kfree(fw->boot); + fw->boot = NULL; } static const struct nvkm_firmware_func diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c index b7da3ab44c277..7c43397c19e61 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c @@ -103,7 +103,7 @@ gm200_flcn_pio_imem_wr_init(struct nvkm_falcon *falcon, u8 port, bool sec, u32 i static void gm200_flcn_pio_imem_wr(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len, u16 tag) { - nvkm_falcon_wr32(falcon, 0x188 + (port * 0x10), tag++); + nvkm_falcon_wr32(falcon, 0x188 + (port * 0x10), tag); while (len >= 4) { nvkm_falcon_wr32(falcon, 0x184 + (port * 0x10), *(u32 *)img); img += 4; @@ -249,9 +249,11 @@ int gm200_flcn_fw_load(struct nvkm_falcon_fw *fw) { struct nvkm_falcon *falcon = fw->falcon; - int target, ret; + int ret; if (fw->inst) { + int target; + nvkm_falcon_mask(falcon, 0x048, 0x00000001, 0x00000001); switch (nvkm_memory_target(fw->inst)) { @@ -285,15 +287,6 @@ gm200_flcn_fw_load(struct nvkm_falcon_fw *fw) } if (fw->boot) { - switch (nvkm_memory_target(&fw->fw.mem.memory)) { - case NVKM_MEM_TARGET_VRAM: target = 4; break; - case NVKM_MEM_TARGET_HOST: target = 5; break; - case NVKM_MEM_TARGET_NCOH: target = 6; break; - default: - WARN_ON(1); - return -EINVAL; - } - ret = nvkm_falcon_pio_wr(falcon, fw->boot, 0, 0, IMEM, falcon->code.limit - fw->boot_size, fw->boot_size, fw->boot_addr >> 8, false); diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index 94fe2f3836a9a..53b3b24d7d7c0 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -1923,6 +1923,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('S', 'H', 'P', 0x1523, &sharp_lq140m1jw46.delay, "LQ140M1JW46"), EDP_PANEL_ENTRY('S', 'H', 'P', 0x154c, &delay_200_500_p2e100, "LQ116M1JW10"), + EDP_PANEL_ENTRY('S', 'T', 'A', 0x0004, &delay_200_500_e200, "116KHD024006"), EDP_PANEL_ENTRY('S', 'T', 'A', 0x0100, &delay_100_500_e200, "2081116HHD028001-51D"), { /* sentinal */ } diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35560.c b/drivers/gpu/drm/panel/panel-novatek-nt35560.c index 5bbea734123bc..ee04c55175bb8 100644 --- a/drivers/gpu/drm/panel/panel-novatek-nt35560.c +++ b/drivers/gpu/drm/panel/panel-novatek-nt35560.c @@ -161,7 +161,7 @@ static int nt35560_set_brightness(struct backlight_device *bl) par = 0x00; ret = mipi_dsi_dcs_write(dsi, MIPI_DCS_WRITE_CONTROL_DISPLAY, &par, 1); - if (ret) { + if (ret < 0) { dev_err(nt->dev, "failed to disable display backlight (%d)\n", ret); return ret; } diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 37fe54c34b141..0d69098eddd90 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -979,27 +979,28 @@ static const struct panel_desc auo_g070vvn01 = { }, }; -static const struct drm_display_mode auo_g101evn010_mode = { - .clock = 68930, - .hdisplay = 1280, - .hsync_start = 1280 + 82, - .hsync_end = 1280 + 82 + 2, - .htotal = 1280 + 82 + 2 + 84, - .vdisplay = 800, - .vsync_start = 800 + 8, - .vsync_end = 800 + 8 + 2, - .vtotal = 800 + 8 + 2 + 6, +static const struct display_timing auo_g101evn010_timing = { + .pixelclock = { 64000000, 68930000, 85000000 }, + .hactive = { 1280, 1280, 1280 }, + .hfront_porch = { 8, 64, 256 }, + .hback_porch = { 8, 64, 256 }, + .hsync_len = { 40, 168, 767 }, + .vactive = { 800, 800, 800 }, + .vfront_porch = { 4, 8, 100 }, + .vback_porch = { 4, 8, 100 }, + .vsync_len = { 8, 16, 223 }, }; static const struct panel_desc auo_g101evn010 = { - .modes = &auo_g101evn010_mode, - .num_modes = 1, + .timings = &auo_g101evn010_timing, + .num_timings = 1, .bpc = 6, .size = { .width = 216, .height = 135, }, .bus_format = MEDIA_BUS_FMT_RGB666_1X7X3_SPWG, + .bus_flags = DRM_BUS_FLAG_DE_HIGH, .connector_type = DRM_MODE_CONNECTOR_LVDS, }; diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c index 820c2c3641d38..1311f10fad660 100644 --- a/drivers/gpu/drm/radeon/evergreen_cs.c +++ b/drivers/gpu/drm/radeon/evergreen_cs.c @@ -33,8 +33,10 @@ #include "evergreen_reg_safe.h" #include "cayman_reg_safe.h" +#ifndef MIN #define MAX(a,b) (((a)>(b))?(a):(b)) #define MIN(a,b) (((a)<(b))?(a):(b)) +#endif #define REG_SAFE_BM_SIZE ARRAY_SIZE(evergreen_reg_safe_bm) diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c index ac77d1246b945..811265648a582 100644 --- a/drivers/gpu/drm/radeon/r600_cs.c +++ b/drivers/gpu/drm/radeon/r600_cs.c @@ -1408,7 +1408,7 @@ static void r600_texture_size(unsigned nfaces, unsigned blevel, unsigned llevel, unsigned block_align, unsigned height_align, unsigned base_align, unsigned *l0_size, unsigned *mipmap_size) { - unsigned offset, i, level; + unsigned offset, i; unsigned width, height, depth, size; unsigned blocksize; unsigned nbx, nby; @@ -1420,7 +1420,7 @@ static void r600_texture_size(unsigned nfaces, unsigned blevel, unsigned llevel, w0 = r600_mip_minify(w0, 0); h0 = r600_mip_minify(h0, 0); d0 = r600_mip_minify(d0, 0); - for(i = 0, offset = 0, level = blevel; i < nlevels; i++, level++) { + for (i = 0, offset = 0; i < nlevels; i++) { width = r600_mip_minify(w0, i); nbx = r600_fmt_get_nblocksx(format, width); diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c index 70d8ad065bfa1..4c8fe83dd6101 100644 --- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c +++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c @@ -705,7 +705,7 @@ static int rcar_du_vsps_init(struct rcar_du_device *rcdu) ret = of_parse_phandle_with_fixed_args(np, vsps_prop_name, cells, i, &args); if (ret < 0) - goto error; + goto done; /* * Add the VSP to the list or update the corresponding existing @@ -743,13 +743,11 @@ static int rcar_du_vsps_init(struct rcar_du_device *rcdu) vsp->dev = rcdu; ret = rcar_du_vsp_init(vsp, vsps[i].np, vsps[i].crtcs_mask); - if (ret < 0) - goto error; + if (ret) + goto done; } - return 0; - -error: +done: for (i = 0; i < ARRAY_SIZE(vsps); ++i) of_node_put(vsps[i].np); diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c index 92f4261305bd9..f2ae5d17ea601 100644 --- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c +++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c @@ -576,7 +576,10 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi, udelay(10); rcar_mipi_dsi_clr(dsi, CLOCKSET1, CLOCKSET1_UPDATEPLL); - ppisetr = PPISETR_DLEN_3 | PPISETR_CLEN; + rcar_mipi_dsi_clr(dsi, TXSETR, TXSETR_LANECNT_MASK); + rcar_mipi_dsi_set(dsi, TXSETR, dsi->lanes - 1); + + ppisetr = ((BIT(dsi->lanes) - 1) & PPISETR_DLEN_MASK) | PPISETR_CLEN; rcar_mipi_dsi_write(dsi, PPISETR, ppisetr); rcar_mipi_dsi_set(dsi, PHYSETUP, PHYSETUP_SHUTDOWNZ); diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h index a6b276f1d6ee1..a54c7eb4113b9 100644 --- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h +++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi_regs.h @@ -12,6 +12,9 @@ #define LINKSR_LPBUSY (1 << 1) #define LINKSR_HSBUSY (1 << 0) +#define TXSETR 0x100 +#define TXSETR_LANECNT_MASK (0x3 << 0) + /* * Video Mode Register */ @@ -80,10 +83,7 @@ * PHY-Protocol Interface (PPI) Registers */ #define PPISETR 0x700 -#define PPISETR_DLEN_0 (0x1 << 0) -#define PPISETR_DLEN_1 (0x3 << 0) -#define PPISETR_DLEN_2 (0x7 << 0) -#define PPISETR_DLEN_3 (0xf << 0) +#define PPISETR_DLEN_MASK (0xf << 0) #define PPISETR_CLEN (1 << 8) #define PPICLCR 0x710 diff --git a/drivers/gpu/drm/renesas/rcar-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rcar-du/rzg2l_mipi_dsi.c index 10febea473cde..6cec796dd463f 100644 --- a/drivers/gpu/drm/renesas/rcar-du/rzg2l_mipi_dsi.c +++ b/drivers/gpu/drm/renesas/rcar-du/rzg2l_mipi_dsi.c @@ -585,6 +585,9 @@ rzg2l_mipi_dsi_bridge_mode_valid(struct drm_bridge *bridge, if (mode->clock > 148500) return MODE_CLOCK_HIGH; + if (mode->clock < 5803) + return MODE_CLOCK_LOW; + return MODE_OK; } diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c index cfe8b793d3446..69ab8d4f289cd 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c @@ -52,16 +52,9 @@ rockchip_fb_create(struct drm_device *dev, struct drm_file *file, } if (drm_is_afbc(mode_cmd->modifier[0])) { - int ret, i; - ret = drm_gem_fb_afbc_init(dev, mode_cmd, afbc_fb); if (ret) { - struct drm_gem_object **obj = afbc_fb->base.obj; - - for (i = 0; i < info->num_planes; ++i) - drm_gem_object_put(obj[i]); - - kfree(afbc_fb); + drm_framebuffer_put(&afbc_fb->base); return ERR_PTR(ret); } } diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index d8f8c37c326c4..97486eba01b7b 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -984,7 +984,7 @@ static int vop2_plane_atomic_check(struct drm_plane *plane, return format; if (drm_rect_width(src) >> 16 < 4 || drm_rect_height(src) >> 16 < 4 || - drm_rect_width(dest) < 4 || drm_rect_width(dest) < 4) { + drm_rect_width(dest) < 4 || drm_rect_height(dest) < 4) { drm_err(vop2->drm, "Invalid size: %dx%d->%dx%d, min size is 4x4\n", drm_rect_width(src) >> 16, drm_rect_height(src) >> 16, drm_rect_width(dest), drm_rect_height(dest)); @@ -1290,10 +1290,8 @@ static void vop2_plane_atomic_update(struct drm_plane *plane, rb_swap = vop2_win_rb_swap(fb->format->format); vop2_win_write(win, VOP2_WIN_RB_SWAP, rb_swap); - if (!vop2_cluster_window(win)) { - uv_swap = vop2_win_uv_swap(fb->format->format); - vop2_win_write(win, VOP2_WIN_UV_SWAP, uv_swap); - } + uv_swap = vop2_win_uv_swap(fb->format->format); + vop2_win_write(win, VOP2_WIN_UV_SWAP, uv_swap); if (fb->format->is_yuv) { vop2_win_write(win, VOP2_WIN_UV_VIR, DIV_ROUND_UP(fb->pitches[1], 4)); diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 53130a50584ca..6855fbb7e6a3b 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -163,25 +163,15 @@ int drm_sched_entity_error(struct drm_sched_entity *entity) } EXPORT_SYMBOL(drm_sched_entity_error); +static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, + struct dma_fence_cb *cb); + static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) { struct drm_sched_job *job = container_of(wrk, typeof(*job), work); - - drm_sched_fence_finished(job->s_fence, -ESRCH); - WARN_ON(job->s_fence->parent); - job->sched->ops->free_job(job); -} - -/* Signal the scheduler finished fence when the entity in question is killed. */ -static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, - struct dma_fence_cb *cb) -{ - struct drm_sched_job *job = container_of(cb, struct drm_sched_job, - finish_cb); + struct dma_fence *f; unsigned long index; - dma_fence_put(f); - /* Wait for all dependencies to avoid data corruptions */ xa_for_each(&job->dependencies, index, f) { struct drm_sched_fence *s_fence = to_drm_sched_fence(f); @@ -209,6 +199,21 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, dma_fence_put(f); } + drm_sched_fence_scheduled(job->s_fence, NULL); + drm_sched_fence_finished(job->s_fence, -ESRCH); + WARN_ON(job->s_fence->parent); + job->sched->ops->free_job(job); +} + +/* Signal the scheduler finished fence when the entity in question is killed. */ +static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, + struct dma_fence_cb *cb) +{ + struct drm_sched_job *job = container_of(cb, struct drm_sched_job, + finish_cb); + + dma_fence_put(f); + INIT_WORK(&job->work, drm_sched_entity_kill_jobs_work); schedule_work(&job->work); } @@ -345,20 +350,9 @@ void drm_sched_entity_destroy(struct drm_sched_entity *entity) } EXPORT_SYMBOL(drm_sched_entity_destroy); -/* drm_sched_entity_clear_dep - callback to clear the entities dependency */ -static void drm_sched_entity_clear_dep(struct dma_fence *f, - struct dma_fence_cb *cb) -{ - struct drm_sched_entity *entity = - container_of(cb, struct drm_sched_entity, cb); - - entity->dependency = NULL; - dma_fence_put(f); -} - /* - * drm_sched_entity_clear_dep - callback to clear the entities dependency and - * wake up scheduler + * drm_sched_entity_wakeup - callback to clear the entity's dependency and + * wake up the scheduler */ static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb) @@ -366,7 +360,8 @@ static void drm_sched_entity_wakeup(struct dma_fence *f, struct drm_sched_entity *entity = container_of(cb, struct drm_sched_entity, cb); - drm_sched_entity_clear_dep(f, cb); + entity->dependency = NULL; + dma_fence_put(f); drm_sched_wakeup_if_can_queue(entity->rq->sched); } @@ -419,13 +414,6 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity) fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(entity->dependency); entity->dependency = fence; - if (!dma_fence_add_callback(fence, &entity->cb, - drm_sched_entity_clear_dep)) - return true; - - /* Ignore it when it is already scheduled */ - dma_fence_put(fence); - return false; } if (!dma_fence_add_callback(entity->dependency, &entity->cb, @@ -547,10 +535,11 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity) drm_sched_rq_remove_entity(entity->rq, entity); entity->rq = rq; } - spin_unlock(&entity->rq_lock); if (entity->num_sched_list == 1) entity->sched_list = NULL; + + spin_unlock(&entity->rq_lock); } /** diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index fa4652f234718..4faa2108c0a73 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -783,13 +783,14 @@ int drm_sched_job_add_resv_dependencies(struct drm_sched_job *job, dma_resv_assert_held(resv); dma_resv_for_each_fence(&cursor, resv, usage, fence) { - /* Make sure to grab an additional ref on the added fence */ - dma_fence_get(fence); - ret = drm_sched_job_add_dependency(job, fence); - if (ret) { - dma_fence_put(fence); + /* + * As drm_sched_job_add_dependency always consumes the fence + * reference (even when it fails), and dma_resv_for_each_fence + * is not obtaining one, we need to grab one before calling. + */ + ret = drm_sched_job_add_dependency(job, dma_fence_get(fence)); + if (ret) return ret; - } } return 0; } diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c index 13b182ab905fb..caa1fe6cc386e 100644 --- a/drivers/gpu/drm/tegra/dc.c +++ b/drivers/gpu/drm/tegra/dc.c @@ -1320,10 +1320,16 @@ static struct drm_plane *tegra_dc_add_shared_planes(struct drm_device *drm, if (wgrp->dc == dc->pipe) { for (j = 0; j < wgrp->num_windows; j++) { unsigned int index = wgrp->windows[j]; + enum drm_plane_type type; + + if (primary) + type = DRM_PLANE_TYPE_OVERLAY; + else + type = DRM_PLANE_TYPE_PRIMARY; plane = tegra_shared_plane_create(drm, dc, wgrp->index, - index); + index, type); if (IS_ERR(plane)) return plane; @@ -1331,10 +1337,8 @@ static struct drm_plane *tegra_dc_add_shared_planes(struct drm_device *drm, * Choose the first shared plane owned by this * head as the primary plane. */ - if (!primary) { - plane->type = DRM_PLANE_TYPE_PRIMARY; + if (!primary) primary = plane; - } } } } @@ -1388,7 +1392,10 @@ static void tegra_crtc_reset(struct drm_crtc *crtc) if (crtc->state) tegra_crtc_atomic_destroy_state(crtc, crtc->state); - __drm_atomic_helper_crtc_reset(crtc, &state->base); + if (state) + __drm_atomic_helper_crtc_reset(crtc, &state->base); + else + __drm_atomic_helper_crtc_reset(crtc, NULL); } static struct drm_crtc_state * @@ -3133,6 +3140,7 @@ static int tegra_dc_couple(struct tegra_dc *dc) dc->client.parent = &parent->client; dev_dbg(dc->dev, "coupled to %s\n", dev_name(companion)); + put_device(companion); } return 0; diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c index 839dbad9bc483..49fc4690c63af 100644 --- a/drivers/gpu/drm/tegra/dsi.c +++ b/drivers/gpu/drm/tegra/dsi.c @@ -912,15 +912,6 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder) u32 value; int err; - /* If the bootloader enabled DSI it needs to be disabled - * in order for the panel initialization commands to be - * properly sent. - */ - value = tegra_dsi_readl(dsi, DSI_POWER_CONTROL); - - if (value & DSI_POWER_CONTROL_ENABLE) - tegra_dsi_disable(dsi); - err = tegra_dsi_prepare(dsi); if (err < 0) { dev_err(dsi->dev, "failed to prepare: %d\n", err); diff --git a/drivers/gpu/drm/tegra/hub.c b/drivers/gpu/drm/tegra/hub.c index 1af5f8318d914..0f88cbb333170 100644 --- a/drivers/gpu/drm/tegra/hub.c +++ b/drivers/gpu/drm/tegra/hub.c @@ -756,9 +756,9 @@ static const struct drm_plane_helper_funcs tegra_shared_plane_helper_funcs = { struct drm_plane *tegra_shared_plane_create(struct drm_device *drm, struct tegra_dc *dc, unsigned int wgrp, - unsigned int index) + unsigned int index, + enum drm_plane_type type) { - enum drm_plane_type type = DRM_PLANE_TYPE_OVERLAY; struct tegra_drm *tegra = drm->dev_private; struct tegra_display_hub *hub = tegra->hub; struct tegra_shared_plane *plane; diff --git a/drivers/gpu/drm/tegra/hub.h b/drivers/gpu/drm/tegra/hub.h index 23c4b2115ed1e..a66f18c4facc9 100644 --- a/drivers/gpu/drm/tegra/hub.h +++ b/drivers/gpu/drm/tegra/hub.h @@ -80,7 +80,8 @@ void tegra_display_hub_cleanup(struct tegra_display_hub *hub); struct drm_plane *tegra_shared_plane_create(struct drm_device *drm, struct tegra_dc *dc, unsigned int wgrp, - unsigned int index); + unsigned int index, + enum drm_plane_type type); int tegra_display_hub_atomic_check(struct drm_device *drm, struct drm_atomic_state *state); diff --git a/drivers/gpu/drm/tegra/nvdec.c b/drivers/gpu/drm/tegra/nvdec.c index 4860790666af5..14ef61b44f47c 100644 --- a/drivers/gpu/drm/tegra/nvdec.c +++ b/drivers/gpu/drm/tegra/nvdec.c @@ -261,10 +261,8 @@ static int nvdec_load_falcon_firmware(struct nvdec *nvdec) if (!client->group) { virt = dma_alloc_coherent(nvdec->dev, size, &iova, GFP_KERNEL); - - err = dma_mapping_error(nvdec->dev, iova); - if (err < 0) - return err; + if (!virt) + return -ENOMEM; } else { virt = tegra_drm_alloc(tegra, size, &iova); if (IS_ERR(virt)) diff --git a/drivers/gpu/drm/tegra/rgb.c b/drivers/gpu/drm/tegra/rgb.c index d6424abd3c45d..8930460ba6282 100644 --- a/drivers/gpu/drm/tegra/rgb.c +++ b/drivers/gpu/drm/tegra/rgb.c @@ -190,6 +190,11 @@ static const struct drm_encoder_helper_funcs tegra_rgb_encoder_helper_funcs = { .atomic_check = tegra_rgb_encoder_atomic_check, }; +static void tegra_dc_of_node_put(void *data) +{ + of_node_put(data); +} + int tegra_dc_rgb_probe(struct tegra_dc *dc) { struct device_node *np; @@ -197,7 +202,14 @@ int tegra_dc_rgb_probe(struct tegra_dc *dc) int err; np = of_get_child_by_name(dc->dev->of_node, "rgb"); - if (!np || !of_device_is_available(np)) + if (!np) + return -ENODEV; + + err = devm_add_action_or_reset(dc->dev, tegra_dc_of_node_put, np); + if (err < 0) + return err; + + if (!of_device_is_available(np)) return -ENODEV; rgb = devm_kzalloc(dc->dev, sizeof(*rgb), GFP_KERNEL); diff --git a/drivers/gpu/drm/tegra/uapi.c b/drivers/gpu/drm/tegra/uapi.c index 5adab6b229164..d0b6a1fa6efad 100644 --- a/drivers/gpu/drm/tegra/uapi.c +++ b/drivers/gpu/drm/tegra/uapi.c @@ -114,9 +114,12 @@ int tegra_drm_ioctl_channel_open(struct drm_device *drm, void *data, struct drm_ if (err) goto put_channel; - if (supported) + if (supported) { + struct pid *pid = get_task_pid(current, PIDTYPE_TGID); context->memory_context = host1x_memory_context_alloc( - host, client->base.dev, get_task_pid(current, PIDTYPE_TGID)); + host, client->base.dev, pid); + put_pid(pid); + } if (IS_ERR(context->memory_context)) { if (PTR_ERR(context->memory_context) != -EOPNOTSUPP) { diff --git a/drivers/gpu/drm/tidss/tidss_crtc.c b/drivers/gpu/drm/tidss/tidss_crtc.c index 1baa4ace12e15..356b7566013b1 100644 --- a/drivers/gpu/drm/tidss/tidss_crtc.c +++ b/drivers/gpu/drm/tidss/tidss_crtc.c @@ -91,7 +91,7 @@ static int tidss_crtc_atomic_check(struct drm_crtc *crtc, struct dispc_device *dispc = tidss->dispc; struct tidss_crtc *tcrtc = to_tidss_crtc(crtc); u32 hw_videoport = tcrtc->hw_videoport; - const struct drm_display_mode *mode; + struct drm_display_mode *mode; enum drm_mode_status ok; dev_dbg(ddev->dev, "%s\n", __func__); @@ -108,6 +108,9 @@ static int tidss_crtc_atomic_check(struct drm_crtc *crtc, return -EINVAL; } + if (drm_atomic_crtc_needs_modeset(crtc_state)) + drm_mode_set_crtcinfo(mode, 0); + return dispc_vp_bus_check(dispc, hw_videoport, crtc_state); } @@ -229,7 +232,7 @@ static void tidss_crtc_atomic_enable(struct drm_crtc *crtc, tidss_runtime_get(tidss); r = dispc_vp_set_clk_rate(tidss->dispc, tcrtc->hw_videoport, - mode->clock * 1000); + mode->crtc_clock * 1000); if (r != 0) return; diff --git a/drivers/gpu/drm/tidss/tidss_dispc.c b/drivers/gpu/drm/tidss/tidss_dispc.c index 355c64bafb82b..8a702f4059b95 100644 --- a/drivers/gpu/drm/tidss/tidss_dispc.c +++ b/drivers/gpu/drm/tidss/tidss_dispc.c @@ -1030,13 +1030,13 @@ void dispc_vp_enable(struct dispc_device *dispc, u32 hw_videoport, dispc_set_num_datalines(dispc, hw_videoport, fmt->data_width); - hfp = mode->hsync_start - mode->hdisplay; - hsw = mode->hsync_end - mode->hsync_start; - hbp = mode->htotal - mode->hsync_end; + hfp = mode->crtc_hsync_start - mode->crtc_hdisplay; + hsw = mode->crtc_hsync_end - mode->crtc_hsync_start; + hbp = mode->crtc_htotal - mode->crtc_hsync_end; - vfp = mode->vsync_start - mode->vdisplay; - vsw = mode->vsync_end - mode->vsync_start; - vbp = mode->vtotal - mode->vsync_end; + vfp = mode->crtc_vsync_start - mode->crtc_vdisplay; + vsw = mode->crtc_vsync_end - mode->crtc_vsync_start; + vbp = mode->crtc_vtotal - mode->crtc_vsync_end; dispc_vp_write(dispc, hw_videoport, DISPC_VP_TIMING_H, FLD_VAL(hsw - 1, 7, 0) | @@ -1078,8 +1078,8 @@ void dispc_vp_enable(struct dispc_device *dispc, u32 hw_videoport, FLD_VAL(ivs, 12, 12)); dispc_vp_write(dispc, hw_videoport, DISPC_VP_SIZE_SCREEN, - FLD_VAL(mode->hdisplay - 1, 11, 0) | - FLD_VAL(mode->vdisplay - 1, 27, 16)); + FLD_VAL(mode->crtc_hdisplay - 1, 11, 0) | + FLD_VAL(mode->crtc_vdisplay - 1, 27, 16)); VP_REG_FLD_MOD(dispc, hw_videoport, DISPC_VP_CONTROL, 1, 0, 0); } diff --git a/drivers/gpu/drm/tiny/cirrus.c b/drivers/gpu/drm/tiny/cirrus.c index 594bc472862fe..b0361ef53c463 100644 --- a/drivers/gpu/drm/tiny/cirrus.c +++ b/drivers/gpu/drm/tiny/cirrus.c @@ -318,7 +318,6 @@ static void cirrus_pitch_set(struct cirrus_device *cirrus, unsigned int pitch) /* Enable extended blanking and pitch bits, and enable full memory */ cr1b = 0x22; cr1b |= (pitch >> 7) & 0x10; - cr1b |= (pitch >> 6) & 0x40; wreg_crt(cirrus, 0x1b, cr1b); cirrus_set_start_address(cirrus, 0); diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c index 8ea120eb8674b..30676b1073034 100644 --- a/drivers/gpu/drm/tiny/simpledrm.c +++ b/drivers/gpu/drm/tiny/simpledrm.c @@ -276,7 +276,7 @@ static struct simpledrm_device *simpledrm_device_of_dev(struct drm_device *dev) static void simpledrm_device_release_clocks(void *res) { - struct simpledrm_device *sdev = simpledrm_device_of_dev(res); + struct simpledrm_device *sdev = res; unsigned int i; for (i = 0; i < sdev->clk_count; ++i) { @@ -374,7 +374,7 @@ static int simpledrm_device_init_clocks(struct simpledrm_device *sdev) static void simpledrm_device_release_regulators(void *res) { - struct simpledrm_device *sdev = simpledrm_device_of_dev(res); + struct simpledrm_device *sdev = res; unsigned int i; for (i = 0; i < sdev->regulator_count; ++i) { diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 0b3f4267130c4..64606104551cd 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -254,6 +254,13 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, ret = dma_resv_trylock(&fbo->base.base._resv); WARN_ON(!ret); + ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); + if (ret) { + dma_resv_unlock(&fbo->base.base._resv); + kfree(fbo); + return ret; + } + if (fbo->base.resource) { ttm_resource_set_bo(fbo->base.resource, &fbo->base); bo->resource = NULL; @@ -262,12 +269,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->base.bulk_move = NULL; } - ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); - if (ret) { - kfree(fbo); - return ret; - } - ttm_bo_get(bo); fbo->bo = bo; diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index 37c08fac7e7d0..80ba34cabca36 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -615,7 +615,6 @@ void ttm_pool_fini(struct ttm_pool *pool) } EXPORT_SYMBOL(ttm_pool_fini); -/* As long as pages are available make sure to release at least one */ static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { @@ -623,9 +622,12 @@ static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, do num_freed += ttm_pool_shrink(); - while (!num_freed && atomic_long_read(&allocated_pages)); + while (num_freed < sc->nr_to_scan && + atomic_long_read(&allocated_pages)); - return num_freed; + sc->nr_scanned = num_freed; + + return num_freed ?: SHRINK_STOP; } /* Return the number of pages available or SHRINK_EMPTY if we have none */ diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 46ff9c75bb124..8f2423a15c71c 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -437,6 +437,9 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, } spin_unlock(&bdev->lru_lock); + if (ret && ret != -ENOENT) + return ret; + spin_lock(&man->move_lock); fence = dma_fence_get(man->move); spin_unlock(&man->move_lock); diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c index 1506094a80098..ac357ab1aee54 100644 --- a/drivers/gpu/drm/udl/udl_drv.c +++ b/drivers/gpu/drm/udl/udl_drv.c @@ -126,9 +126,9 @@ static void udl_usb_disconnect(struct usb_interface *interface) { struct drm_device *dev = usb_get_intfdata(interface); + drm_dev_unplug(dev); drm_kms_helper_poll_fini(dev); udl_drop_usb(dev); - drm_dev_unplug(dev); } /* diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c index ffbbe9d527d32..0e8ea99011884 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.c +++ b/drivers/gpu/drm/v3d/v3d_drv.c @@ -226,11 +226,21 @@ static int v3d_platform_drm_probe(struct platform_device *pdev) if (ret) return ret; + v3d->clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(v3d->clk)) + return dev_err_probe(dev, PTR_ERR(v3d->clk), "Failed to get V3D clock\n"); + + ret = clk_prepare_enable(v3d->clk); + if (ret) { + dev_err(&pdev->dev, "Couldn't enable the V3D clock\n"); + return ret; + } + mmu_debug = V3D_READ(V3D_MMU_DEBUG_INFO); mask = DMA_BIT_MASK(30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_PA_WIDTH)); ret = dma_set_mask_and_coherent(dev, mask); if (ret) - return ret; + goto clk_disable; v3d->va_width = 30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_VA_WIDTH); @@ -245,28 +255,29 @@ static int v3d_platform_drm_probe(struct platform_device *pdev) ret = PTR_ERR(v3d->reset); if (ret == -EPROBE_DEFER) - return ret; + goto clk_disable; v3d->reset = NULL; ret = map_regs(v3d, &v3d->bridge_regs, "bridge"); if (ret) { dev_err(dev, "Failed to get reset control or bridge regs\n"); - return ret; + goto clk_disable; } } if (v3d->ver < 41) { ret = map_regs(v3d, &v3d->gca_regs, "gca"); if (ret) - return ret; + goto clk_disable; } v3d->mmu_scratch = dma_alloc_wc(dev, 4096, &v3d->mmu_scratch_paddr, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); if (!v3d->mmu_scratch) { dev_err(dev, "Failed to allocate MMU scratch page\n"); - return -ENOMEM; + ret = -ENOMEM; + goto clk_disable; } ret = v3d_gem_init(drm); @@ -289,6 +300,8 @@ static int v3d_platform_drm_probe(struct platform_device *pdev) v3d_gem_destroy(drm); dma_free: dma_free_wc(dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr); +clk_disable: + clk_disable_unprepare(v3d->clk); return ret; } @@ -303,6 +316,8 @@ static void v3d_platform_drm_remove(struct platform_device *pdev) dma_free_wc(v3d->drm.dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr); + + clk_disable_unprepare(v3d->clk); } static struct platform_driver v3d_platform_driver = { diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 7f664a4b2a752..bcef978ba9c4c 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -62,6 +62,12 @@ struct v3d_perfmon { u64 values[]; }; +enum v3d_irq { + V3D_CORE_IRQ, + V3D_HUB_IRQ, + V3D_MAX_IRQS, +}; + struct v3d_dev { struct drm_device drm; @@ -71,6 +77,8 @@ struct v3d_dev { int ver; bool single_irq_line; + int irq[V3D_MAX_IRQS]; + void __iomem *hub_regs; void __iomem *core_regs[3]; void __iomem *bridge_regs; diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c index 2e94ce788c714..ef991a9b1c6c4 100644 --- a/drivers/gpu/drm/v3d/v3d_gem.c +++ b/drivers/gpu/drm/v3d/v3d_gem.c @@ -120,6 +120,8 @@ v3d_reset(struct v3d_dev *v3d) if (false) v3d_idle_axi(v3d, 0); + v3d_irq_disable(v3d); + v3d_idle_gca(v3d); v3d_reset_v3d(v3d); diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c index b2d59a1686972..641315dbee8b2 100644 --- a/drivers/gpu/drm/v3d/v3d_irq.c +++ b/drivers/gpu/drm/v3d/v3d_irq.c @@ -215,7 +215,7 @@ v3d_hub_irq(int irq, void *arg) int v3d_irq_init(struct v3d_dev *v3d) { - int irq1, ret, core; + int irq, ret, core; INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work); @@ -226,17 +226,24 @@ v3d_irq_init(struct v3d_dev *v3d) V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS); V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS); - irq1 = platform_get_irq_optional(v3d_to_pdev(v3d), 1); - if (irq1 == -EPROBE_DEFER) - return irq1; - if (irq1 > 0) { - ret = devm_request_irq(v3d->drm.dev, irq1, + irq = platform_get_irq_optional(v3d_to_pdev(v3d), 1); + if (irq == -EPROBE_DEFER) + return irq; + if (irq > 0) { + v3d->irq[V3D_CORE_IRQ] = irq; + + ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], v3d_irq, IRQF_SHARED, "v3d_core0", v3d); if (ret) goto fail; - ret = devm_request_irq(v3d->drm.dev, - platform_get_irq(v3d_to_pdev(v3d), 0), + + irq = platform_get_irq(v3d_to_pdev(v3d), 0); + if (irq < 0) + return irq; + v3d->irq[V3D_HUB_IRQ] = irq; + + ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_HUB_IRQ], v3d_hub_irq, IRQF_SHARED, "v3d_hub", v3d); if (ret) @@ -244,8 +251,12 @@ v3d_irq_init(struct v3d_dev *v3d) } else { v3d->single_irq_line = true; - ret = devm_request_irq(v3d->drm.dev, - platform_get_irq(v3d_to_pdev(v3d), 0), + irq = platform_get_irq(v3d_to_pdev(v3d), 0); + if (irq < 0) + return irq; + v3d->irq[V3D_CORE_IRQ] = irq; + + ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], v3d_irq, IRQF_SHARED, "v3d", v3d); if (ret) @@ -286,6 +297,12 @@ v3d_irq_disable(struct v3d_dev *v3d) V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0); V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0); + /* Finish any interrupt handler still in flight. */ + for (int i = 0; i < V3D_MAX_IRQS; i++) { + if (v3d->irq[i]) + synchronize_irq(v3d->irq[i]); + } + /* Clear any pending interrupts we might have left. */ for (core = 0; core < v3d->cores; core++) V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS); diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 5b729013fd26f..41493cf3d03b8 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -289,11 +289,16 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job) return DRM_GPU_SCHED_STAT_NOMINAL; } -/* If the current address or return address have changed, then the GPU - * has probably made progress and we should delay the reset. This - * could fail if the GPU got in an infinite loop in the CL, but that - * is pretty unlikely outside of an i-g-t testcase. - */ +static void +v3d_sched_skip_reset(struct drm_sched_job *sched_job) +{ + struct drm_gpu_scheduler *sched = sched_job->sched; + + spin_lock(&sched->job_list_lock); + list_add(&sched_job->list, &sched->pending_list); + spin_unlock(&sched->job_list_lock); +} + static enum drm_gpu_sched_stat v3d_cl_job_timedout(struct drm_sched_job *sched_job, enum v3d_queue q, u32 *timedout_ctca, u32 *timedout_ctra) @@ -303,9 +308,16 @@ v3d_cl_job_timedout(struct drm_sched_job *sched_job, enum v3d_queue q, u32 ctca = V3D_CORE_READ(0, V3D_CLE_CTNCA(q)); u32 ctra = V3D_CORE_READ(0, V3D_CLE_CTNRA(q)); + /* If the current address or return address have changed, then the GPU + * has probably made progress and we should delay the reset. This + * could fail if the GPU got in an infinite loop in the CL, but that + * is pretty unlikely outside of an i-g-t testcase. + */ if (*timedout_ctca != ctca || *timedout_ctra != ctra) { *timedout_ctca = ctca; *timedout_ctra = ctra; + + v3d_sched_skip_reset(sched_job); return DRM_GPU_SCHED_STAT_NOMINAL; } @@ -345,11 +357,13 @@ v3d_csd_job_timedout(struct drm_sched_job *sched_job) struct v3d_dev *v3d = job->base.v3d; u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4); - /* If we've made progress, skip reset and let the timer get - * rearmed. + /* If we've made progress, skip reset, add the job to the pending + * list, and let the timer get rearmed. */ if (job->timedout_batches != batches) { job->timedout_batches = batches; + + v3d_sched_skip_reset(sched_job); return DRM_GPU_SCHED_STAT_NOMINAL; } diff --git a/drivers/gpu/drm/vc4/tests/vc4_mock_output.c b/drivers/gpu/drm/vc4/tests/vc4_mock_output.c index e70d7c3076acf..f0ddc223c1f83 100644 --- a/drivers/gpu/drm/vc4/tests/vc4_mock_output.c +++ b/drivers/gpu/drm/vc4/tests/vc4_mock_output.c @@ -75,24 +75,30 @@ int vc4_mock_atomic_add_output(struct kunit *test, int ret; encoder = vc4_find_encoder_by_type(drm, type); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, encoder); + if (!encoder) + return -ENODEV; crtc = vc4_find_crtc_for_encoder(test, drm, encoder); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc); + if (!crtc) + return -ENODEV; output = encoder_to_vc4_dummy_output(encoder); conn = &output->connector; conn_state = drm_atomic_get_connector_state(state, conn); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state); + if (IS_ERR(conn_state)) + return PTR_ERR(conn_state); ret = drm_atomic_set_crtc_for_connector(conn_state, crtc); - KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + return ret; crtc_state = drm_atomic_get_crtc_state(state, crtc); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); ret = drm_atomic_set_mode_for_crtc(crtc_state, &default_mode); - KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + return ret; crtc_state->active = true; @@ -113,26 +119,32 @@ int vc4_mock_atomic_del_output(struct kunit *test, int ret; encoder = vc4_find_encoder_by_type(drm, type); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, encoder); + if (!encoder) + return -ENODEV; crtc = vc4_find_crtc_for_encoder(test, drm, encoder); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc); + if (!crtc) + return -ENODEV; crtc_state = drm_atomic_get_crtc_state(state, crtc); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); crtc_state->active = false; ret = drm_atomic_set_mode_for_crtc(crtc_state, NULL); - KUNIT_ASSERT_EQ(test, ret, 0); + if (ret) + return ret; output = encoder_to_vc4_dummy_output(encoder); conn = &output->connector; conn_state = drm_atomic_get_connector_state(state, conn); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state); + if (IS_ERR(conn_state)) + return PTR_ERR(conn_state); ret = drm_atomic_set_crtc_for_connector(conn_state, NULL); - KUNIT_ASSERT_EQ(test, ret, 0); + if (ret) + return ret; return 0; } diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c index 61e500b8c9da2..894c484f99e95 100644 --- a/drivers/gpu/drm/vkms/vkms_crtc.c +++ b/drivers/gpu/drm/vkms/vkms_crtc.c @@ -201,7 +201,7 @@ static int vkms_crtc_atomic_check(struct drm_crtc *crtc, i++; } - vkms_state->active_planes = kcalloc(i, sizeof(plane), GFP_KERNEL); + vkms_state->active_planes = kcalloc(i, sizeof(*vkms_state->active_planes), GFP_KERNEL); if (!vkms_state->active_planes) return -ENOMEM; vkms_state->num_active_planes = i; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index 5fef0b31c1179..92b3e44d022fe 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -1514,6 +1514,7 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv, SVGA3dCmdHeader *header) { struct vmw_bo *vmw_bo = NULL; + struct vmw_resource *res; struct vmw_surface *srf = NULL; VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSurfaceDMA); int ret; @@ -1549,18 +1550,24 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv, dirty = (cmd->body.transfer == SVGA3D_WRITE_HOST_VRAM) ? VMW_RES_DIRTY_SET : 0; - ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface, - dirty, user_surface_converter, - &cmd->body.host.sid, NULL); + ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface, dirty, + user_surface_converter, &cmd->body.host.sid, + NULL); if (unlikely(ret != 0)) { if (unlikely(ret != -ERESTARTSYS)) VMW_DEBUG_USER("could not find surface for DMA.\n"); return ret; } - srf = vmw_res_to_srf(sw_context->res_cache[vmw_res_surface].res); + res = sw_context->res_cache[vmw_res_surface].res; + if (!res) { + VMW_DEBUG_USER("Invalid DMA surface.\n"); + return -EINVAL; + } - vmw_kms_cursor_snoop(srf, sw_context->fp->tfile, &vmw_bo->tbo, header); + srf = vmw_res_to_srf(res); + vmw_kms_cursor_snoop(srf, sw_context->fp->tfile, &vmw_bo->tbo, + header); return 0; } @@ -3676,6 +3683,11 @@ static int vmw_cmd_check(struct vmw_private *dev_priv, cmd_id = header->id; + if (header->size > SVGA_CMD_MAX_DATASIZE) { + VMW_DEBUG_USER("SVGA3D command: %d is too big.\n", + cmd_id + SVGA_3D_CMD_BASE); + return -E2BIG; + } *size = header->size + sizeof(SVGA3dCmdHeader); cmd_id -= SVGA_3D_CMD_BASE; @@ -4083,6 +4095,23 @@ static int vmw_execbuf_tie_context(struct vmw_private *dev_priv, return 0; } +/* + * DMA fence callback to remove a seqno_waiter + */ +struct seqno_waiter_rm_context { + struct dma_fence_cb base; + struct vmw_private *dev_priv; +}; + +static void seqno_waiter_rm_cb(struct dma_fence *f, struct dma_fence_cb *cb) +{ + struct seqno_waiter_rm_context *ctx = + container_of(cb, struct seqno_waiter_rm_context, base); + + vmw_seqno_waiter_remove(ctx->dev_priv); + kfree(ctx); +} + int vmw_execbuf_process(struct drm_file *file_priv, struct vmw_private *dev_priv, void __user *user_commands, void *kernel_commands, @@ -4263,6 +4292,15 @@ int vmw_execbuf_process(struct drm_file *file_priv, } else { /* Link the fence with the FD created earlier */ fd_install(out_fence_fd, sync_file->file); + struct seqno_waiter_rm_context *ctx = + kmalloc(sizeof(*ctx), GFP_KERNEL); + ctx->dev_priv = dev_priv; + vmw_seqno_waiter_add(dev_priv); + if (dma_fence_add_callback(&fence->base, &ctx->base, + seqno_waiter_rm_cb) < 0) { + vmw_seqno_waiter_remove(dev_priv); + kfree(ctx); + } } } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c index 7fb1c88bcc475..69dfe69ce0f87 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c @@ -896,7 +896,7 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv, .busy_domain = VMW_BO_DOMAIN_SYS, .bo_type = ttm_bo_type_device, .size = size, - .pin = true, + .pin = false, .keep_resv = true, }; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c index aaacbdcbd742f..946f166d6fc76 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c @@ -326,8 +326,10 @@ int vmw_validation_add_resource(struct vmw_validation_context *ctx, hash_add_rcu(ctx->sw_context->res_ht, &node->hash.head, node->hash.key); } node->res = vmw_resource_reference_unless_doomed(res); - if (!node->res) + if (!node->res) { + hash_del_rcu(&node->hash.head); return -ESRCH; + } node->first_usage = 1; if (!res->dev_priv->has_mob) { @@ -654,7 +656,7 @@ void vmw_validation_drop_ht(struct vmw_validation_context *ctx) hash_del_rcu(&val->hash.head); list_for_each_entry(val, &ctx->resource_ctx_list, head) - hash_del_rcu(&entry->hash.head); + hash_del_rcu(&val->hash.head); ctx->sw_context = NULL; } diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c index 862ca8d072326..e2047ef864ebf 100644 --- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c +++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c @@ -163,6 +163,8 @@ static int amd_sfh1_1_hid_client_init(struct amd_mp2_dev *privdata) if (rc) goto cleanup; + mp2_ops->stop(privdata, cl_data->sensor_idx[i]); + amd_sfh_wait_for_response(privdata, cl_data->sensor_idx[i], DISABLE_SENSOR); writel(0, privdata->mmio + AMD_P2C_MSG(0)); mp2_ops->start(privdata, info); status = amd_sfh_wait_for_response diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 0b561c1eb59e2..7cf17c671da48 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -858,10 +858,12 @@ static int apple_probe(struct hid_device *hdev, return ret; } - timer_setup(&asc->battery_timer, apple_battery_timer_tick, 0); - mod_timer(&asc->battery_timer, - jiffies + msecs_to_jiffies(APPLE_BATTERY_TIMEOUT_MS)); - apple_fetch_battery(hdev); + if (quirks & APPLE_RDESC_BATTERY) { + timer_setup(&asc->battery_timer, apple_battery_timer_tick, 0); + mod_timer(&asc->battery_timer, + jiffies + msecs_to_jiffies(APPLE_BATTERY_TIMEOUT_MS)); + apple_fetch_battery(hdev); + } if (quirks & APPLE_BACKLIGHT_CTL) apple_backlight_init(hdev); @@ -873,7 +875,8 @@ static void apple_remove(struct hid_device *hdev) { struct apple_sc *asc = hid_get_drvdata(hdev); - del_timer_sync(&asc->battery_timer); + if (asc->quirks & APPLE_RDESC_BATTERY) + del_timer_sync(&asc->battery_timer); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index 84625e817ce95..d971e339b5eac 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -869,7 +869,10 @@ static int asus_input_mapping(struct hid_device *hdev, case 0xc4: asus_map_key_clear(KEY_KBDILLUMUP); break; case 0xc5: asus_map_key_clear(KEY_KBDILLUMDOWN); break; case 0xc7: asus_map_key_clear(KEY_KBDILLUMTOGGLE); break; + case 0x4e: asus_map_key_clear(KEY_FN_ESC); break; + case 0x7e: asus_map_key_clear(KEY_EMOJI_PICKER); break; + case 0x8b: asus_map_key_clear(KEY_PROG1); break; /* ProArt Creator Hub key */ case 0x6b: asus_map_key_clear(KEY_F21); break; /* ASUS touchpad toggle */ case 0x38: asus_map_key_clear(KEY_PROG1); break; /* ROG key */ case 0xba: asus_map_key_clear(KEY_PROG2); break; /* Fn+C ASUS Splendid */ @@ -1108,7 +1111,13 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) return ret; } - if (!drvdata->input) { + /* + * Check that input registration succeeded. Checking that + * HID_CLAIMED_INPUT is set prevents a UAF when all input devices + * were freed during registration due to no usages being mapped, + * leaving drvdata->input pointing to freed memory. + */ + if (!drvdata->input || !(hdev->claimed & HID_CLAIMED_INPUT)) { hid_err(hdev, "Asus input not registered\n"); ret = -ENOMEM; goto err_stop_hw; @@ -1273,9 +1282,6 @@ static const struct hid_device_id asus_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2), QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, - { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, - USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR), QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, @@ -1305,6 +1311,9 @@ static const struct hid_device_id asus_devices[] = { * Note bind to the HID_GROUP_GENERIC group, so that we only bind to the keyboard * part, while letting hid-multitouch.c handle the touchpad. */ + { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, + USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO), + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T101HA_KEYBOARD) }, { } diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 8f156a69912c8..266cd56dec505 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -1873,9 +1873,12 @@ u8 *hid_alloc_report_buf(struct hid_report *report, gfp_t flags) /* * 7 extra bytes are necessary to achieve proper functionality * of implement() working on 8 byte chunks + * 1 extra byte for the report ID if it is null (not used) so + * we can reserve that extra byte in the first position of the buffer + * when sending it to .raw_request() */ - u32 len = hid_report_len(report) + 7; + u32 len = hid_report_len(report) + 7 + (report->id == 0); return kzalloc(len, flags); } @@ -1938,7 +1941,7 @@ static struct hid_report *hid_get_report(struct hid_report_enum *report_enum, int __hid_request(struct hid_device *hid, struct hid_report *report, enum hid_class_request reqtype) { - char *buf; + char *buf, *data_buf; int ret; u32 len; @@ -1946,13 +1949,19 @@ int __hid_request(struct hid_device *hid, struct hid_report *report, if (!buf) return -ENOMEM; + data_buf = buf; len = hid_report_len(report); + if (report->id == 0) { + /* reserve the first byte for the report ID */ + data_buf++; + len++; + } + if (reqtype == HID_REQ_SET_REPORT) - hid_output_report(report, buf); + hid_output_report(report, data_buf); - ret = hid->ll_driver->raw_request(hid, report->id, buf, len, - report->type, reqtype); + ret = hid_hw_raw_request(hid, report->id, buf, len, report->type, reqtype); if (ret < 0) { dbg_hid("unable to complete request: %d\n", ret); goto out; diff --git a/drivers/hid/hid-hyperv.c b/drivers/hid/hid-hyperv.c index 0fb210e40a412..9eafff0b6ea4c 100644 --- a/drivers/hid/hid-hyperv.c +++ b/drivers/hid/hid-hyperv.c @@ -192,7 +192,7 @@ static void mousevsc_on_receive_device_info(struct mousevsc_dev *input_device, goto cleanup; input_device->report_desc_size = le16_to_cpu( - desc->desc[0].wDescriptorLength); + desc->rpt_desc.wDescriptorLength); if (input_device->report_desc_size == 0) { input_device->dev_info_status = -EINVAL; goto cleanup; @@ -210,7 +210,7 @@ static void mousevsc_on_receive_device_info(struct mousevsc_dev *input_device, memcpy(input_device->report_desc, ((unsigned char *)desc) + desc->bLength, - le16_to_cpu(desc->desc[0].wDescriptorLength)); + le16_to_cpu(desc->rpt_desc.wDescriptorLength)); /* Send the ack */ memset(&ack, 0, sizeof(struct mousevsc_prt_msg)); diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 8e721ec3faaff..6a538dca21a9a 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -41,6 +41,10 @@ #define USB_VENDOR_ID_ACTIONSTAR 0x2101 #define USB_DEVICE_ID_ACTIONSTAR_1011 0x1011 +#define USB_VENDOR_ID_ADATA_XPG 0x125f +#define USB_VENDOR_ID_ADATA_XPG_WL_GAMING_MOUSE 0x7505 +#define USB_VENDOR_ID_ADATA_XPG_WL_GAMING_MOUSE_DONGLE 0x7506 + #define USB_VENDOR_ID_ADS_TECH 0x06e1 #define USB_DEVICE_ID_ADS_TECH_RADIO_SI470X 0xa155 @@ -209,7 +213,7 @@ #define USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD3 0x1822 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD 0x1866 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2 0x19b6 -#define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3 0x1a30 +#define USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO 0x1a30 #define USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR 0x18c6 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY 0x1abe #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X 0x1b4c @@ -301,6 +305,8 @@ #define USB_DEVICE_ID_ASUS_AK1D 0x1125 #define USB_DEVICE_ID_CHICONY_TOSHIBA_WT10A 0x1408 #define USB_DEVICE_ID_CHICONY_ACER_SWITCH12 0x1421 +#define USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA 0xb824 +#define USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA2 0xb82c #define USB_VENDOR_ID_CHUNGHWAT 0x2247 #define USB_DEVICE_ID_CHUNGHWAT_MULTITOUCH 0x0001 @@ -326,6 +332,9 @@ #define USB_DEVICE_ID_CODEMERCS_IOW_FIRST 0x1500 #define USB_DEVICE_ID_CODEMERCS_IOW_LAST 0x15ff +#define USB_VENDOR_ID_COOLER_MASTER 0x2516 +#define USB_DEVICE_ID_COOLER_MASTER_MICE_DONGLE 0x01b7 + #define USB_VENDOR_ID_CORSAIR 0x1b1c #define USB_DEVICE_ID_CORSAIR_K90 0x1b02 #define USB_DEVICE_ID_CORSAIR_K70R 0x1b09 @@ -803,6 +812,7 @@ #define USB_DEVICE_ID_LENOVO_TPPRODOCK 0x6067 #define USB_DEVICE_ID_LENOVO_X1_COVER 0x6085 #define USB_DEVICE_ID_LENOVO_X1_TAB 0x60a3 +#define USB_DEVICE_ID_LENOVO_X1_TAB2 0x60a4 #define USB_DEVICE_ID_LENOVO_X1_TAB3 0x60b5 #define USB_DEVICE_ID_LENOVO_X12_TAB 0x60fe #define USB_DEVICE_ID_LENOVO_X12_TAB2 0x61ae @@ -811,6 +821,8 @@ #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6019 0x6019 #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_602E 0x602e #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6093 0x6093 +#define USB_DEVICE_ID_LENOVO_LEGION_GO_DUAL_DINPUT 0x6184 +#define USB_DEVICE_ID_LENOVO_LEGION_GO2_DUAL_DINPUT 0x61ed #define USB_VENDOR_ID_LETSKETCH 0x6161 #define USB_DEVICE_ID_WP9620N 0x4d15 @@ -884,6 +896,7 @@ #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_2 0xc534 #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1 0xc539 #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_1 0xc53f +#define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_2 0xc543 #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_POWERPLAY 0xc53a #define USB_DEVICE_ID_LOGITECH_BOLT_RECEIVER 0xc548 #define USB_DEVICE_ID_SPACETRAVELLER 0xc623 @@ -1390,6 +1403,7 @@ #define USB_VENDOR_ID_VRS 0x0483 #define USB_DEVICE_ID_VRS_DFP 0xa355 +#define USB_DEVICE_ID_VRS_R295 0xa44c #define USB_VENDOR_ID_VTL 0x0306 #define USB_DEVICE_ID_VTL_MULTITOUCH_FF3F 0xff3f @@ -1497,4 +1511,7 @@ #define USB_VENDOR_ID_SIGNOTEC 0x2133 #define USB_DEVICE_ID_SIGNOTEC_VIEWSONIC_PD1011 0x0018 +#define USB_VENDOR_ID_JIELI_SDK_DEFAULT 0x4c4a +#define USB_DEVICE_ID_JIELI_SDK_4155 0x4155 + #endif diff --git a/drivers/hid/hid-input-test.c b/drivers/hid/hid-input-test.c index 77c2d45ac62a7..6f5c71660d823 100644 --- a/drivers/hid/hid-input-test.c +++ b/drivers/hid/hid-input-test.c @@ -7,7 +7,7 @@ #include -static void hid_test_input_set_battery_charge_status(struct kunit *test) +static void hid_test_input_update_battery_charge_status(struct kunit *test) { struct hid_device *dev; bool handled; @@ -15,15 +15,15 @@ static void hid_test_input_set_battery_charge_status(struct kunit *test) dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); - handled = hidinput_set_battery_charge_status(dev, HID_DG_HEIGHT, 0); + handled = hidinput_update_battery_charge_status(dev, HID_DG_HEIGHT, 0); KUNIT_EXPECT_FALSE(test, handled); KUNIT_EXPECT_EQ(test, dev->battery_charge_status, POWER_SUPPLY_STATUS_UNKNOWN); - handled = hidinput_set_battery_charge_status(dev, HID_BAT_CHARGING, 0); + handled = hidinput_update_battery_charge_status(dev, HID_BAT_CHARGING, 0); KUNIT_EXPECT_TRUE(test, handled); KUNIT_EXPECT_EQ(test, dev->battery_charge_status, POWER_SUPPLY_STATUS_DISCHARGING); - handled = hidinput_set_battery_charge_status(dev, HID_BAT_CHARGING, 1); + handled = hidinput_update_battery_charge_status(dev, HID_BAT_CHARGING, 1); KUNIT_EXPECT_TRUE(test, handled); KUNIT_EXPECT_EQ(test, dev->battery_charge_status, POWER_SUPPLY_STATUS_CHARGING); } @@ -63,7 +63,7 @@ static void hid_test_input_get_battery_property(struct kunit *test) } static struct kunit_case hid_input_tests[] = { - KUNIT_CASE(hid_test_input_set_battery_charge_status), + KUNIT_CASE(hid_test_input_update_battery_charge_status), KUNIT_CASE(hid_test_input_get_battery_property), { } }; diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 6acc048204312..f073d5621050a 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -595,14 +595,37 @@ static void hidinput_cleanup_battery(struct hid_device *dev) dev->battery = NULL; } -static void hidinput_update_battery(struct hid_device *dev, int value) +static bool hidinput_update_battery_charge_status(struct hid_device *dev, + unsigned int usage, int value) +{ + switch (usage) { + case HID_BAT_CHARGING: + dev->battery_charge_status = value ? + POWER_SUPPLY_STATUS_CHARGING : + POWER_SUPPLY_STATUS_DISCHARGING; + return true; + } + + return false; +} + +static void hidinput_update_battery(struct hid_device *dev, unsigned int usage, + int value) { int capacity; if (!dev->battery) return; - if (value == 0 || value < dev->battery_min || value > dev->battery_max) + if (hidinput_update_battery_charge_status(dev, usage, value)) { + power_supply_changed(dev->battery); + return; + } + + if ((usage & HID_USAGE_PAGE) == HID_UP_DIGITIZER && value == 0) + return; + + if (value < dev->battery_min || value > dev->battery_max) return; capacity = hidinput_scale_battery_capacity(dev, value); @@ -617,21 +640,6 @@ static void hidinput_update_battery(struct hid_device *dev, int value) power_supply_changed(dev->battery); } } - -static bool hidinput_set_battery_charge_status(struct hid_device *dev, - unsigned int usage, int value) -{ - switch (usage) { - case HID_BAT_CHARGING: - dev->battery_charge_status = value ? - POWER_SUPPLY_STATUS_CHARGING : - POWER_SUPPLY_STATUS_DISCHARGING; - power_supply_changed(dev->battery); - return true; - } - - return false; -} #else /* !CONFIG_HID_BATTERY_STRENGTH */ static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type, struct hid_field *field, bool is_percentage) @@ -643,15 +651,10 @@ static void hidinput_cleanup_battery(struct hid_device *dev) { } -static void hidinput_update_battery(struct hid_device *dev, int value) +static void hidinput_update_battery(struct hid_device *dev, unsigned int usage, + int value) { } - -static bool hidinput_set_battery_charge_status(struct hid_device *dev, - unsigned int usage, int value) -{ - return false; -} #endif /* CONFIG_HID_BATTERY_STRENGTH */ static bool hidinput_field_in_collection(struct hid_device *device, struct hid_field *field, @@ -1516,11 +1519,7 @@ void hidinput_hid_event(struct hid_device *hid, struct hid_field *field, struct return; if (usage->type == EV_PWR) { - bool handled = hidinput_set_battery_charge_status(hid, usage->hid, value); - - if (!handled) - hidinput_update_battery(hid, value); - + hidinput_update_battery(hid, usage->hid, value); return; } diff --git a/drivers/hid/hid-lenovo.c b/drivers/hid/hid-lenovo.c index a4062f617ba20..32cb2e75228c4 100644 --- a/drivers/hid/hid-lenovo.c +++ b/drivers/hid/hid-lenovo.c @@ -473,6 +473,7 @@ static int lenovo_input_mapping(struct hid_device *hdev, return lenovo_input_mapping_tp10_ultrabook_kbd(hdev, hi, field, usage, bit, max); case USB_DEVICE_ID_LENOVO_X1_TAB: + case USB_DEVICE_ID_LENOVO_X1_TAB2: case USB_DEVICE_ID_LENOVO_X1_TAB3: return lenovo_input_mapping_x1_tab_kbd(hdev, hi, field, usage, bit, max); default: @@ -529,11 +530,14 @@ static void lenovo_features_set_cptkbd(struct hid_device *hdev) /* * Tell the keyboard a driver understands it, and turn F7, F9, F11 into - * regular keys + * regular keys (Compact only) */ - ret = lenovo_send_cmd_cptkbd(hdev, 0x01, 0x03); - if (ret) - hid_warn(hdev, "Failed to switch F7/9/11 mode: %d\n", ret); + if (hdev->product == USB_DEVICE_ID_LENOVO_CUSBKBD || + hdev->product == USB_DEVICE_ID_LENOVO_CBTKBD) { + ret = lenovo_send_cmd_cptkbd(hdev, 0x01, 0x03); + if (ret) + hid_warn(hdev, "Failed to switch F7/9/11 mode: %d\n", ret); + } /* Switch middle button to native mode */ ret = lenovo_send_cmd_cptkbd(hdev, 0x09, 0x01); @@ -584,6 +588,7 @@ static ssize_t attr_fn_lock_store(struct device *dev, break; case USB_DEVICE_ID_LENOVO_TP10UBKBD: case USB_DEVICE_ID_LENOVO_X1_TAB: + case USB_DEVICE_ID_LENOVO_X1_TAB2: case USB_DEVICE_ID_LENOVO_X1_TAB3: ret = lenovo_led_set_tp10ubkbd(hdev, TP10UBKBD_FN_LOCK_LED, value); if (ret) @@ -779,6 +784,7 @@ static int lenovo_event(struct hid_device *hdev, struct hid_field *field, return lenovo_event_cptkbd(hdev, field, usage, value); case USB_DEVICE_ID_LENOVO_TP10UBKBD: case USB_DEVICE_ID_LENOVO_X1_TAB: + case USB_DEVICE_ID_LENOVO_X1_TAB2: case USB_DEVICE_ID_LENOVO_X1_TAB3: return lenovo_event_tp10ubkbd(hdev, field, usage, value); default: @@ -1062,6 +1068,7 @@ static int lenovo_led_brightness_set(struct led_classdev *led_cdev, break; case USB_DEVICE_ID_LENOVO_TP10UBKBD: case USB_DEVICE_ID_LENOVO_X1_TAB: + case USB_DEVICE_ID_LENOVO_X1_TAB2: case USB_DEVICE_ID_LENOVO_X1_TAB3: ret = lenovo_led_set_tp10ubkbd(hdev, tp10ubkbd_led[led_nr], value); break; @@ -1293,6 +1300,7 @@ static int lenovo_probe(struct hid_device *hdev, break; case USB_DEVICE_ID_LENOVO_TP10UBKBD: case USB_DEVICE_ID_LENOVO_X1_TAB: + case USB_DEVICE_ID_LENOVO_X1_TAB2: case USB_DEVICE_ID_LENOVO_X1_TAB3: ret = lenovo_probe_tp10ubkbd(hdev); break; @@ -1380,6 +1388,7 @@ static void lenovo_remove(struct hid_device *hdev) break; case USB_DEVICE_ID_LENOVO_TP10UBKBD: case USB_DEVICE_ID_LENOVO_X1_TAB: + case USB_DEVICE_ID_LENOVO_X1_TAB2: case USB_DEVICE_ID_LENOVO_X1_TAB3: lenovo_remove_tp10ubkbd(hdev); break; @@ -1430,6 +1439,8 @@ static const struct hid_device_id lenovo_devices[] = { */ { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X1_TAB) }, + { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, + USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X1_TAB2) }, { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X1_TAB3) }, { } diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c index 37958edec55f5..e2d5b3f699146 100644 --- a/drivers/hid/hid-logitech-dj.c +++ b/drivers/hid/hid-logitech-dj.c @@ -1983,6 +1983,10 @@ static const struct hid_device_id logi_dj_receivers[] = { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_1), .driver_data = recvr_type_gaming_hidpp}, + { /* Logitech lightspeed receiver (0xc543) */ + HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, + USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_2), + .driver_data = recvr_type_gaming_hidpp}, { /* Logitech 27 MHz HID++ 1.0 receiver (0xc513) */ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_MX3000_RECEIVER), diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 4519ee377aa76..68bb521a574bd 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -4524,10 +4524,12 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id) if (hidpp->quirks & HIDPP_QUIRK_CLASS_G920) { ret = hidpp_ff_init(hidpp, &data); - if (ret) + if (ret) { hid_warn(hidpp->hid_dev, "Unable to initialize force feedback support, errno %d\n", ret); + ret = 0; + } } /* @@ -4652,6 +4654,8 @@ static const struct hid_device_id hidpp_devices[] = { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC094) }, { /* Logitech G Pro X Superlight 2 Gaming Mouse over USB */ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC09b) }, + { /* Logitech G PRO 2 LIGHTSPEED Wireless Mouse over USB */ + HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xc09a) }, { /* G935 Gaming Headset */ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0x0a87), diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 9bb8daf7f7860..4fe1e0bc24493 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -772,16 +772,30 @@ static void magicmouse_enable_mt_work(struct work_struct *work) hid_err(msc->hdev, "unable to request touch data (%d)\n", ret); } +static bool is_usb_magicmouse2(__u32 vendor, __u32 product) +{ + if (vendor != USB_VENDOR_ID_APPLE) + return false; + return product == USB_DEVICE_ID_APPLE_MAGICMOUSE2; +} + +static bool is_usb_magictrackpad2(__u32 vendor, __u32 product) +{ + if (vendor != USB_VENDOR_ID_APPLE) + return false; + return product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || + product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC; +} + static int magicmouse_fetch_battery(struct hid_device *hdev) { #ifdef CONFIG_HID_BATTERY_STRENGTH struct hid_report_enum *report_enum; struct hid_report *report; - if (!hdev->battery || hdev->vendor != USB_VENDOR_ID_APPLE || - (hdev->product != USB_DEVICE_ID_APPLE_MAGICMOUSE2 && - hdev->product != USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 && - hdev->product != USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC)) + if (!hdev->battery || + (!is_usb_magicmouse2(hdev->vendor, hdev->product) && + !is_usb_magictrackpad2(hdev->vendor, hdev->product))) return -1; report_enum = &hdev->report_enum[hdev->battery_report_type]; @@ -843,16 +857,17 @@ static int magicmouse_probe(struct hid_device *hdev, return ret; } - timer_setup(&msc->battery_timer, magicmouse_battery_timer_tick, 0); - mod_timer(&msc->battery_timer, - jiffies + msecs_to_jiffies(USB_BATTERY_TIMEOUT_MS)); - magicmouse_fetch_battery(hdev); + if (is_usb_magicmouse2(id->vendor, id->product) || + is_usb_magictrackpad2(id->vendor, id->product)) { + timer_setup(&msc->battery_timer, magicmouse_battery_timer_tick, 0); + mod_timer(&msc->battery_timer, + jiffies + msecs_to_jiffies(USB_BATTERY_TIMEOUT_MS)); + magicmouse_fetch_battery(hdev); + } - if (id->vendor == USB_VENDOR_ID_APPLE && - (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || - ((id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || - id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC) && - hdev->type != HID_TYPE_USBMOUSE))) + if (is_usb_magicmouse2(id->vendor, id->product) || + (is_usb_magictrackpad2(id->vendor, id->product) && + hdev->type != HID_TYPE_USBMOUSE)) return 0; if (!msc->input) { @@ -908,7 +923,10 @@ static int magicmouse_probe(struct hid_device *hdev, return 0; err_stop_hw: - del_timer_sync(&msc->battery_timer); + if (is_usb_magicmouse2(id->vendor, id->product) || + is_usb_magictrackpad2(id->vendor, id->product)) + del_timer_sync(&msc->battery_timer); + hid_hw_stop(hdev); return ret; } @@ -919,7 +937,9 @@ static void magicmouse_remove(struct hid_device *hdev) if (msc) { cancel_delayed_work_sync(&msc->work); - del_timer_sync(&msc->battery_timer); + if (is_usb_magicmouse2(hdev->vendor, hdev->product) || + is_usb_magictrackpad2(hdev->vendor, hdev->product)) + del_timer_sync(&msc->battery_timer); } hid_hw_stop(hdev); @@ -936,10 +956,8 @@ static __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc, * 0x05, 0x01, // Usage Page (Generic Desktop) 0 * 0x09, 0x02, // Usage (Mouse) 2 */ - if (hdev->vendor == USB_VENDOR_ID_APPLE && - (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || - hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || - hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC) && + if ((is_usb_magicmouse2(hdev->vendor, hdev->product) || + is_usb_magictrackpad2(hdev->vendor, hdev->product)) && *rsize == 83 && rdesc[46] == 0x84 && rdesc[58] == 0x85) { hid_info(hdev, "fixing up magicmouse battery report descriptor\n"); diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c index c5bfca8ac5e6e..d2d413c829f44 100644 --- a/drivers/hid/hid-mcp2221.c +++ b/drivers/hid/hid-mcp2221.c @@ -49,6 +49,7 @@ enum { MCP2221_I2C_MASK_ADDR_NACK = 0x40, MCP2221_I2C_WRADDRL_SEND = 0x21, MCP2221_I2C_ADDR_NACK = 0x25, + MCP2221_I2C_READ_PARTIAL = 0x54, MCP2221_I2C_READ_COMPL = 0x55, MCP2221_ALT_F_NOT_GPIOV = 0xEE, MCP2221_ALT_F_NOT_GPIOD = 0xEF, @@ -187,6 +188,25 @@ static int mcp_cancel_last_cmd(struct mcp2221 *mcp) return mcp_send_data_req_status(mcp, mcp->txbuf, 8); } +/* Check if the last command succeeded or failed and return the result. + * If the command did fail, cancel that command which will free the i2c bus. + */ +static int mcp_chk_last_cmd_status_free_bus(struct mcp2221 *mcp) +{ + int ret; + + ret = mcp_chk_last_cmd_status(mcp); + if (ret) { + /* The last command was a failure. + * Send a cancel which will also free the bus. + */ + usleep_range(980, 1000); + mcp_cancel_last_cmd(mcp); + } + + return ret; +} + static int mcp_set_i2c_speed(struct mcp2221 *mcp) { int ret; @@ -241,7 +261,7 @@ static int mcp_i2c_write(struct mcp2221 *mcp, usleep_range(980, 1000); if (last_status) { - ret = mcp_chk_last_cmd_status(mcp); + ret = mcp_chk_last_cmd_status_free_bus(mcp); if (ret) return ret; } @@ -278,6 +298,7 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, { int ret; u16 total_len; + int retries = 0; mcp->txbuf[0] = type; if (msg) { @@ -301,20 +322,31 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, mcp->rxbuf_idx = 0; do { + /* Wait for the data to be read by the device */ + usleep_range(980, 1000); + memset(mcp->txbuf, 0, 4); mcp->txbuf[0] = MCP2221_I2C_GET_DATA; ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1); - if (ret) - return ret; - - ret = mcp_chk_last_cmd_status(mcp); - if (ret) - return ret; - - usleep_range(980, 1000); + if (ret) { + if (retries < 5) { + /* The data wasn't ready to read. + * Wait a bit longer and try again. + */ + usleep_range(90, 100); + retries++; + } else { + return ret; + } + } else { + retries = 0; + } } while (mcp->rxbuf_idx < total_len); + usleep_range(980, 1000); + ret = mcp_chk_last_cmd_status_free_bus(mcp); + return ret; } @@ -328,11 +360,6 @@ static int mcp_i2c_xfer(struct i2c_adapter *adapter, mutex_lock(&mcp->lock); - /* Setting speed before every transaction is required for mcp2221 */ - ret = mcp_set_i2c_speed(mcp); - if (ret) - goto exit; - if (num == 1) { if (msgs->flags & I2C_M_RD) { ret = mcp_i2c_smbus_read(mcp, msgs, MCP2221_I2C_RD_DATA, @@ -417,9 +444,7 @@ static int mcp_smbus_write(struct mcp2221 *mcp, u16 addr, if (last_status) { usleep_range(980, 1000); - ret = mcp_chk_last_cmd_status(mcp); - if (ret) - return ret; + ret = mcp_chk_last_cmd_status_free_bus(mcp); } return ret; @@ -437,10 +462,6 @@ static int mcp_smbus_xfer(struct i2c_adapter *adapter, u16 addr, mutex_lock(&mcp->lock); - ret = mcp_set_i2c_speed(mcp); - if (ret) - goto exit; - switch (size) { case I2C_SMBUS_QUICK: @@ -791,7 +812,12 @@ static int mcp2221_raw_event(struct hid_device *hdev, mcp->status = -EIO; break; } - if (data[2] == MCP2221_I2C_READ_COMPL) { + if (data[2] == MCP2221_I2C_READ_COMPL || + data[2] == MCP2221_I2C_READ_PARTIAL) { + if (!mcp->rxbuf || mcp->rxbuf_idx < 0 || data[3] > 60) { + mcp->status = -EINVAL; + break; + } buf = mcp->rxbuf; memcpy(&buf[mcp->rxbuf_idx], &data[4], data[3]); mcp->rxbuf_idx = mcp->rxbuf_idx + data[3]; @@ -1152,6 +1178,11 @@ static int mcp2221_probe(struct hid_device *hdev, if (i2c_clk_freq < 50) i2c_clk_freq = 50; mcp->cur_i2c_clk_div = (12000000 / (i2c_clk_freq * 1000)) - 3; + ret = mcp_set_i2c_speed(mcp); + if (ret) { + hid_err(hdev, "can't set i2c speed: %d\n", ret); + return ret; + } mcp->adapter.owner = THIS_MODULE; mcp->adapter.class = I2C_CLASS_HWMON; diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index 6386043aab0bb..b9e67b408a4b9 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -83,9 +83,8 @@ enum latency_mode { HID_LATENCY_HIGH = 1, }; -#define MT_IO_FLAGS_RUNNING 0 -#define MT_IO_FLAGS_ACTIVE_SLOTS 1 -#define MT_IO_FLAGS_PENDING_SLOTS 2 +#define MT_IO_SLOTS_MASK GENMASK(7, 0) /* reserve first 8 bits for slot tracking */ +#define MT_IO_FLAGS_RUNNING 32 static const bool mtrue = true; /* default for true */ static const bool mfalse; /* default for false */ @@ -161,7 +160,11 @@ struct mt_device { struct mt_class mtclass; /* our mt device class */ struct timer_list release_timer; /* to release sticky fingers */ struct hid_device *hdev; /* hid_device we're attached to */ - unsigned long mt_io_flags; /* mt flags (MT_IO_FLAGS_*) */ + unsigned long mt_io_flags; /* mt flags (MT_IO_FLAGS_RUNNING) + * first 8 bits are reserved for keeping the slot + * states, this is fine because we only support up + * to 250 slots (MT_MAX_MAXCONTACT) + */ __u8 inputmode_value; /* InputMode HID feature value */ __u8 maxcontacts; bool is_buttonpad; /* is this device a button pad? */ @@ -936,6 +939,7 @@ static void mt_release_pending_palms(struct mt_device *td, for_each_set_bit(slotnum, app->pending_palm_slots, td->maxcontacts) { clear_bit(slotnum, app->pending_palm_slots); + clear_bit(slotnum, &td->mt_io_flags); input_mt_slot(input, slotnum); input_mt_report_slot_inactive(input); @@ -967,12 +971,6 @@ static void mt_sync_frame(struct mt_device *td, struct mt_application *app, app->num_received = 0; app->left_button_state = 0; - - if (test_bit(MT_IO_FLAGS_ACTIVE_SLOTS, &td->mt_io_flags)) - set_bit(MT_IO_FLAGS_PENDING_SLOTS, &td->mt_io_flags); - else - clear_bit(MT_IO_FLAGS_PENDING_SLOTS, &td->mt_io_flags); - clear_bit(MT_IO_FLAGS_ACTIVE_SLOTS, &td->mt_io_flags); } static int mt_compute_timestamp(struct mt_application *app, __s32 value) @@ -1147,7 +1145,9 @@ static int mt_process_slot(struct mt_device *td, struct input_dev *input, input_event(input, EV_ABS, ABS_MT_TOUCH_MAJOR, major); input_event(input, EV_ABS, ABS_MT_TOUCH_MINOR, minor); - set_bit(MT_IO_FLAGS_ACTIVE_SLOTS, &td->mt_io_flags); + set_bit(slotnum, &td->mt_io_flags); + } else { + clear_bit(slotnum, &td->mt_io_flags); } return 0; @@ -1282,7 +1282,7 @@ static void mt_touch_report(struct hid_device *hid, * defect. */ if (app->quirks & MT_QUIRK_STICKY_FINGERS) { - if (test_bit(MT_IO_FLAGS_PENDING_SLOTS, &td->mt_io_flags)) + if (td->mt_io_flags & MT_IO_SLOTS_MASK) mod_timer(&td->release_timer, jiffies + msecs_to_jiffies(100)); else @@ -1448,6 +1448,14 @@ static __u8 *mt_report_fixup(struct hid_device *hdev, __u8 *rdesc, if (hdev->vendor == I2C_VENDOR_ID_GOODIX && (hdev->product == I2C_DEVICE_ID_GOODIX_01E8 || hdev->product == I2C_DEVICE_ID_GOODIX_01E9)) { + if (*size < 608) { + dev_info( + &hdev->dev, + "GT7868Q fixup: report descriptor is only %u bytes, skipping\n", + *size); + return rdesc; + } + if (rdesc[607] == 0x15) { rdesc[607] = 0x25; dev_info( @@ -1650,6 +1658,7 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi) case HID_CP_CONSUMER_CONTROL: case HID_GD_WIRELESS_RADIO_CTLS: case HID_GD_SYSTEM_MULTIAXIS: + case HID_DG_PEN: /* already handled by hid core */ break; case HID_DG_TOUCHSCREEN: @@ -1721,6 +1730,7 @@ static void mt_release_contacts(struct hid_device *hid) for (i = 0; i < mt->num_slots; i++) { input_mt_slot(input_dev, i); input_mt_report_slot_inactive(input_dev); + clear_bit(i, &td->mt_io_flags); } input_mt_sync_frame(input_dev); input_sync(input_dev); @@ -1743,7 +1753,7 @@ static void mt_expired_timeout(struct timer_list *t) */ if (test_and_set_bit_lock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags)) return; - if (test_bit(MT_IO_FLAGS_PENDING_SLOTS, &td->mt_io_flags)) + if (td->mt_io_flags & MT_IO_SLOTS_MASK) mt_release_contacts(hdev); clear_bit_unlock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags); } @@ -2110,12 +2120,18 @@ static const struct hid_device_id mt_devices[] = { HID_DEVICE(BUS_I2C, HID_GROUP_GENERIC, USB_VENDOR_ID_LG, I2C_DEVICE_ID_LG_7010) }, - /* Lenovo X1 TAB Gen 2 */ + /* Lenovo X1 TAB Gen 1 */ { .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT, HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X1_TAB) }, + /* Lenovo X1 TAB Gen 2 */ + { .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT, + HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_LENOVO, + USB_DEVICE_ID_LENOVO_X1_TAB2) }, + /* Lenovo X1 TAB Gen 3 */ { .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT, HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, diff --git a/drivers/hid/hid-ntrig.c b/drivers/hid/hid-ntrig.c index b5d26f03fe6bb..3c41f6841f775 100644 --- a/drivers/hid/hid-ntrig.c +++ b/drivers/hid/hid-ntrig.c @@ -142,10 +142,13 @@ static void ntrig_report_version(struct hid_device *hdev) int ret; char buf[20]; struct usb_device *usb_dev = hid_to_usb_dev(hdev); - unsigned char *data = kmalloc(8, GFP_KERNEL); + unsigned char *data __free(kfree) = kmalloc(8, GFP_KERNEL); + + if (!hid_is_usb(hdev)) + return; if (!data) - goto err_free; + return; ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0), USB_REQ_CLEAR_FEATURE, @@ -160,9 +163,6 @@ static void ntrig_report_version(struct hid_device *hdev) hid_info(hdev, "Firmware version: %s (%02x%02x %02x%02x)\n", buf, data[2], data[3], data[4], data[5]); } - -err_free: - kfree(data); } static ssize_t show_phys_width(struct device *dev, diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 5d7a418ccdbec..fa946666969b8 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -27,6 +27,8 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_GAMEPAD), HID_QUIRK_BADPAD }, { HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_PREDATOR), HID_QUIRK_BADPAD }, + { HID_USB_DEVICE(USB_VENDOR_ID_ADATA_XPG, USB_VENDOR_ID_ADATA_XPG_WL_GAMING_MOUSE), HID_QUIRK_ALWAYS_POLL }, + { HID_USB_DEVICE(USB_VENDOR_ID_ADATA_XPG, USB_VENDOR_ID_ADATA_XPG_WL_GAMING_MOUSE_DONGLE), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_AFATECH, USB_DEVICE_ID_AFATECH_AF9016), HID_QUIRK_FULLSPEED_INTERVAL }, { HID_USB_DEVICE(USB_VENDOR_ID_AIREN, USB_DEVICE_ID_AIREN_SLIMPLUS), HID_QUIRK_NOGET }, { HID_USB_DEVICE(USB_VENDOR_ID_AKAI_09E8, USB_DEVICE_ID_AKAI_09E8_MIDIMIX), HID_QUIRK_NO_INIT_REPORTS }, @@ -55,6 +57,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FLIGHT_SIM_YOKE), HID_QUIRK_NOGET }, { HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_PEDALS), HID_QUIRK_NOGET }, { HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_THROTTLE), HID_QUIRK_NOGET }, + { HID_USB_DEVICE(USB_VENDOR_ID_COOLER_MASTER, USB_DEVICE_ID_COOLER_MASTER_MICE_DONGLE), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB_RAPIDFIRE), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70RGB), HID_QUIRK_NO_INIT_REPORTS }, @@ -122,6 +125,8 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_T609A), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_LABTEC, USB_DEVICE_ID_LABTEC_ODDOR_HANDBRAKE), HID_QUIRK_ALWAYS_POLL }, + { HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_LEGION_GO_DUAL_DINPUT), HID_QUIRK_MULTI_INPUT }, + { HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_LEGION_GO2_DUAL_DINPUT), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_OPTICAL_USB_MOUSE_600E), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_608D), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6019), HID_QUIRK_ALWAYS_POLL }, @@ -202,6 +207,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_KNA5), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWA60), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_UGTIZER, USB_DEVICE_ID_UGTIZER_TABLET_WP5540), HID_QUIRK_MULTI_INPUT }, + { HID_USB_DEVICE(USB_VENDOR_ID_VRS, USB_DEVICE_ID_VRS_R295), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_10_6_INCH), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_14_1_INCH), HID_QUIRK_MULTI_INPUT }, { HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SIRIUS_BATTERY_FREE_TABLET), HID_QUIRK_MULTI_INPUT }, @@ -745,6 +751,8 @@ static const struct hid_device_id hid_ignore_list[] = { { HID_USB_DEVICE(USB_VENDOR_ID_AVERMEDIA, USB_DEVICE_ID_AVER_FM_MR800) }, { HID_USB_DEVICE(USB_VENDOR_ID_AXENTIA, USB_DEVICE_ID_AXENTIA_FM_RADIO) }, { HID_USB_DEVICE(USB_VENDOR_ID_BERKSHIRE, USB_DEVICE_ID_BERKSHIRE_PCWD) }, + { HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA) }, + { HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_HP_5MP_CAMERA2) }, { HID_USB_DEVICE(USB_VENDOR_ID_CIDC, 0x0103) }, { HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI470X) }, { HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI4713) }, @@ -1048,6 +1056,18 @@ bool hid_ignore(struct hid_device *hdev) strlen(elan_acpi_id[i].id))) return true; break; + case USB_VENDOR_ID_JIELI_SDK_DEFAULT: + /* + * Multiple USB devices with identical IDs (mic & touchscreen). + * The touch screen requires hid core processing, but the + * microphone does not. They can be distinguished by manufacturer + * and serial number. + */ + if (hdev->product == USB_DEVICE_ID_JIELI_SDK_4155 && + strncmp(hdev->name, "SmartlinkTechnology", 19) == 0 && + strncmp(hdev->uniq, "20201111000001", 14) == 0) + return true; + break; } if (hdev->type == HID_TYPE_USBMOUSE && diff --git a/drivers/hid/hid-thrustmaster.c b/drivers/hid/hid-thrustmaster.c index 3b81468a1df29..0bf70664c35ee 100644 --- a/drivers/hid/hid-thrustmaster.c +++ b/drivers/hid/hid-thrustmaster.c @@ -174,6 +174,7 @@ static void thrustmaster_interrupts(struct hid_device *hdev) u8 ep_addr[2] = {b_ep, 0}; if (!usb_check_int_endpoints(usbif, ep_addr)) { + kfree(send_buf); hid_err(hdev, "Unexpected non-int endpoint\n"); return; } diff --git a/drivers/hid/hid-uclogic-core.c b/drivers/hid/hid-uclogic-core.c index ad74cbc9a0aa5..45de01dea4b1c 100644 --- a/drivers/hid/hid-uclogic-core.c +++ b/drivers/hid/hid-uclogic-core.c @@ -142,11 +142,12 @@ static int uclogic_input_configured(struct hid_device *hdev, suffix = "System Control"; break; } - } - - if (suffix) + } else { hi->input->name = devm_kasprintf(&hdev->dev, GFP_KERNEL, "%s %s", hdev->name, suffix); + if (!hi->input->name) + return -ENOMEM; + } return 0; } diff --git a/drivers/hid/hid-uclogic-params.c b/drivers/hid/hid-uclogic-params.c index 9859dad36495a..eee05d668e361 100644 --- a/drivers/hid/hid-uclogic-params.c +++ b/drivers/hid/hid-uclogic-params.c @@ -1364,8 +1364,10 @@ static int uclogic_params_ugee_v2_init_event_hooks(struct hid_device *hdev, event_hook->hdev = hdev; event_hook->size = ARRAY_SIZE(reconnect_event); event_hook->event = kmemdup(reconnect_event, event_hook->size, GFP_KERNEL); - if (!event_hook->event) + if (!event_hook->event) { + kfree(event_hook); return -ENOMEM; + } list_add_tail(&event_hook->list, &p->event_hooks->list); diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.c b/drivers/hid/i2c-hid/i2c-hid-acpi.c index 1b49243adb16a..abd700a101f46 100644 --- a/drivers/hid/i2c-hid/i2c-hid-acpi.c +++ b/drivers/hid/i2c-hid/i2c-hid-acpi.c @@ -76,6 +76,13 @@ static int i2c_hid_acpi_get_descriptor(struct i2c_hid_acpi *ihid_acpi) return hid_descriptor_address; } +static void i2c_hid_acpi_restore_sequence(struct i2chid_ops *ops) +{ + struct i2c_hid_acpi *ihid_acpi = container_of(ops, struct i2c_hid_acpi, ops); + + i2c_hid_acpi_get_descriptor(ihid_acpi); +} + static void i2c_hid_acpi_shutdown_tail(struct i2chid_ops *ops) { struct i2c_hid_acpi *ihid_acpi = container_of(ops, struct i2c_hid_acpi, ops); @@ -96,6 +103,7 @@ static int i2c_hid_acpi_probe(struct i2c_client *client) ihid_acpi->adev = ACPI_COMPANION(dev); ihid_acpi->ops.shutdown_tail = i2c_hid_acpi_shutdown_tail; + ihid_acpi->ops.restore_sequence = i2c_hid_acpi_restore_sequence; ret = i2c_hid_acpi_get_descriptor(ihid_acpi); if (ret < 0) diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index 3dcdd3368b463..172b783274201 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -937,6 +937,14 @@ static void i2c_hid_core_shutdown_tail(struct i2c_hid *ihid) ihid->ops->shutdown_tail(ihid->ops); } +static void i2c_hid_core_restore_sequence(struct i2c_hid *ihid) +{ + if (!ihid->ops->restore_sequence) + return; + + ihid->ops->restore_sequence(ihid->ops); +} + static int i2c_hid_core_suspend(struct i2c_hid *ihid, bool force_poweroff) { struct i2c_client *client = ihid->client; @@ -1320,8 +1328,26 @@ static int i2c_hid_core_pm_resume(struct device *dev) return i2c_hid_core_resume(ihid); } +static int i2c_hid_core_pm_restore(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct i2c_hid *ihid = i2c_get_clientdata(client); + + if (ihid->is_panel_follower) + return 0; + + i2c_hid_core_restore_sequence(ihid); + + return i2c_hid_core_resume(ihid); +} + const struct dev_pm_ops i2c_hid_core_pm = { - SYSTEM_SLEEP_PM_OPS(i2c_hid_core_pm_suspend, i2c_hid_core_pm_resume) + .suspend = pm_sleep_ptr(i2c_hid_core_pm_suspend), + .resume = pm_sleep_ptr(i2c_hid_core_pm_resume), + .freeze = pm_sleep_ptr(i2c_hid_core_pm_suspend), + .thaw = pm_sleep_ptr(i2c_hid_core_pm_resume), + .poweroff = pm_sleep_ptr(i2c_hid_core_pm_suspend), + .restore = pm_sleep_ptr(i2c_hid_core_pm_restore), }; EXPORT_SYMBOL_GPL(i2c_hid_core_pm); diff --git a/drivers/hid/i2c-hid/i2c-hid.h b/drivers/hid/i2c-hid/i2c-hid.h index 2c7b66d5caa0f..1724a435c783a 100644 --- a/drivers/hid/i2c-hid/i2c-hid.h +++ b/drivers/hid/i2c-hid/i2c-hid.h @@ -27,11 +27,13 @@ static inline u32 i2c_hid_get_dmi_quirks(const u16 vendor, const u16 product) * @power_up: do sequencing to power up the device. * @power_down: do sequencing to power down the device. * @shutdown_tail: called at the end of shutdown. + * @restore_sequence: hibernation restore sequence. */ struct i2chid_ops { int (*power_up)(struct i2chid_ops *ops); void (*power_down)(struct i2chid_ops *ops); void (*shutdown_tail)(struct i2chid_ops *ops); + void (*restore_sequence)(struct i2chid_ops *ops); }; int i2c_hid_core_probe(struct i2c_client *client, struct i2chid_ops *ops, diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c index 257dd73e37bf5..a8e8da002b9bb 100644 --- a/drivers/hid/usbhid/hid-core.c +++ b/drivers/hid/usbhid/hid-core.c @@ -982,12 +982,11 @@ static int usbhid_parse(struct hid_device *hid) struct usb_host_interface *interface = intf->cur_altsetting; struct usb_device *dev = interface_to_usbdev (intf); struct hid_descriptor *hdesc; + struct hid_class_descriptor *hcdesc; u32 quirks = 0; unsigned int rsize = 0; char *rdesc; - int ret, n; - int num_descriptors; - size_t offset = offsetof(struct hid_descriptor, desc); + int ret; quirks = hid_lookup_quirk(hid); @@ -1009,20 +1008,19 @@ static int usbhid_parse(struct hid_device *hid) return -ENODEV; } - if (hdesc->bLength < sizeof(struct hid_descriptor)) { - dbg_hid("hid descriptor is too short\n"); + if (!hdesc->bNumDescriptors || + hdesc->bLength != sizeof(*hdesc) + + (hdesc->bNumDescriptors - 1) * sizeof(*hcdesc)) { + dbg_hid("hid descriptor invalid, bLen=%hhu bNum=%hhu\n", + hdesc->bLength, hdesc->bNumDescriptors); return -EINVAL; } hid->version = le16_to_cpu(hdesc->bcdHID); hid->country = hdesc->bCountryCode; - num_descriptors = min_t(int, hdesc->bNumDescriptors, - (hdesc->bLength - offset) / sizeof(struct hid_class_descriptor)); - - for (n = 0; n < num_descriptors; n++) - if (hdesc->desc[n].bDescriptorType == HID_DT_REPORT) - rsize = le16_to_cpu(hdesc->desc[n].wDescriptorLength); + if (hdesc->rpt_desc.bDescriptorType == HID_DT_REPORT) + rsize = le16_to_cpu(hdesc->rpt_desc.wDescriptorLength); if (!rsize || rsize > HID_MAX_DESCRIPTOR_SIZE) { dbg_hid("weird size of report descriptor (%u)\n", rsize); @@ -1050,6 +1048,11 @@ static int usbhid_parse(struct hid_device *hid) goto err; } + if (hdesc->bNumDescriptors > 1) + hid_warn(intf, + "%u unsupported optional hid class descriptors\n", + (int)(hdesc->bNumDescriptors - 1)); + hid->quirks |= quirks; return 0; diff --git a/drivers/hid/usbhid/usbkbd.c b/drivers/hid/usbhid/usbkbd.c index c439ed2f16dbc..af6bc76dbf649 100644 --- a/drivers/hid/usbhid/usbkbd.c +++ b/drivers/hid/usbhid/usbkbd.c @@ -160,7 +160,7 @@ static int usb_kbd_event(struct input_dev *dev, unsigned int type, return -1; spin_lock_irqsave(&kbd->leds_lock, flags); - kbd->newleds = (!!test_bit(LED_KANA, dev->led) << 3) | (!!test_bit(LED_COMPOSE, dev->led) << 3) | + kbd->newleds = (!!test_bit(LED_KANA, dev->led) << 4) | (!!test_bit(LED_COMPOSE, dev->led) << 3) | (!!test_bit(LED_SCROLLL, dev->led) << 2) | (!!test_bit(LED_CAPSL, dev->led) << 1) | (!!test_bit(LED_NUML, dev->led)); diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c index 5a72cf8d6944f..52011503ff3ba 100644 --- a/drivers/hid/wacom_sys.c +++ b/drivers/hid/wacom_sys.c @@ -2012,14 +2012,18 @@ static int wacom_initialize_remotes(struct wacom *wacom) remote->remote_dir = kobject_create_and_add("wacom_remote", &wacom->hdev->dev.kobj); - if (!remote->remote_dir) + if (!remote->remote_dir) { + kfifo_free(&remote->remote_fifo); return -ENOMEM; + } error = sysfs_create_files(remote->remote_dir, remote_unpair_attrs); if (error) { hid_err(wacom->hdev, "cannot create sysfs group err: %d\n", error); + kfifo_free(&remote->remote_fifo); + kobject_put(remote->remote_dir); return error; } diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index dd44373ba930e..42bc8f05e2635 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -684,6 +684,7 @@ static bool wacom_is_art_pen(int tool_id) case 0x885: /* Intuos3 Marker Pen */ case 0x804: /* Intuos4/5 13HD/24HD Marker Pen */ case 0x10804: /* Intuos4/5 13HD/24HD Art Pen */ + case 0x204: /* Art Pen 2 */ is_art_pen = true; break; } diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index fdb72345c8505..f3139d7f3f974 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -1114,68 +1114,10 @@ int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer, EXPORT_SYMBOL(vmbus_sendpacket); /* - * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer - * packets using a GPADL Direct packet type. This interface allows you - * to control notifying the host. This will be useful for sending - * batched data. Also the sender can control the send flags - * explicitly. - */ -int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel, - struct hv_page_buffer pagebuffers[], - u32 pagecount, void *buffer, u32 bufferlen, - u64 requestid) -{ - int i; - struct vmbus_channel_packet_page_buffer desc; - u32 descsize; - u32 packetlen; - u32 packetlen_aligned; - struct kvec bufferlist[3]; - u64 aligned_data = 0; - - if (pagecount > MAX_PAGE_BUFFER_COUNT) - return -EINVAL; - - /* - * Adjust the size down since vmbus_channel_packet_page_buffer is the - * largest size we support - */ - descsize = sizeof(struct vmbus_channel_packet_page_buffer) - - ((MAX_PAGE_BUFFER_COUNT - pagecount) * - sizeof(struct hv_page_buffer)); - packetlen = descsize + bufferlen; - packetlen_aligned = ALIGN(packetlen, sizeof(u64)); - - /* Setup the descriptor */ - desc.type = VM_PKT_DATA_USING_GPA_DIRECT; - desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; - desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */ - desc.length8 = (u16)(packetlen_aligned >> 3); - desc.transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */ - desc.reserved = 0; - desc.rangecount = pagecount; - - for (i = 0; i < pagecount; i++) { - desc.range[i].len = pagebuffers[i].len; - desc.range[i].offset = pagebuffers[i].offset; - desc.range[i].pfn = pagebuffers[i].pfn; - } - - bufferlist[0].iov_base = &desc; - bufferlist[0].iov_len = descsize; - bufferlist[1].iov_base = buffer; - bufferlist[1].iov_len = bufferlen; - bufferlist[2].iov_base = &aligned_data; - bufferlist[2].iov_len = (packetlen_aligned - packetlen); - - return hv_ringbuffer_write(channel, bufferlist, 3, requestid, NULL); -} -EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer); - -/* - * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet + * vmbus_sendpacket_mpb_desc - Send one or more multi-page buffer packets * using a GPADL Direct packet type. - * The buffer includes the vmbus descriptor. + * The desc argument must include space for the VMBus descriptor. The + * rangecount field must already be set. */ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, struct vmbus_packet_mpb_array *desc, @@ -1197,7 +1139,6 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, desc->length8 = (u16)(packetlen_aligned >> 3); desc->transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */ desc->reserved = 0; - desc->rangecount = 1; bufferlist[0].iov_base = desc; bufferlist[0].iov_len = desc_size; diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 2f4d09ce027a3..3c6011a48dabe 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -120,7 +120,9 @@ const struct vmbus_device vmbus_devs[] = { }, /* File copy */ - { .dev_type = HV_FCOPY, + /* fcopy always uses 16KB ring buffer size and is working well for last many years */ + { .pref_ring_size = 0x4000, + .dev_type = HV_FCOPY, HV_FCOPY_GUID, .perf_device = false, .allowed_in_isolated = false, @@ -140,12 +142,19 @@ const struct vmbus_device vmbus_devs[] = { .allowed_in_isolated = false, }, - /* Unknown GUID */ - { .dev_type = HV_UNKNOWN, + /* + * Unknown GUID + * 64 KB ring buffer + 4 KB header should be sufficient size for any Hyper-V device apart + * from HV_NIC and HV_SCSI. This case avoid the fallback for unknown devices to allocate + * much bigger (2 MB) of ring size. + */ + { .pref_ring_size = 0x11000, + .dev_type = HV_UNKNOWN, .perf_device = false, .allowed_in_isolated = false, }, }; +EXPORT_SYMBOL_GPL(vmbus_devs); static const struct { guid_t guid; diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index 7bafa3c6ddd89..653f586c3b352 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -207,11 +207,20 @@ int vmbus_connect(void) INIT_LIST_HEAD(&vmbus_connection.chn_list); mutex_init(&vmbus_connection.channel_mutex); + /* + * The following Hyper-V interrupt and monitor pages can be used by + * UIO for mapping to user-space, so they should always be allocated on + * system page boundaries. The system page size must be >= the Hyper-V + * page size. + */ + BUILD_BUG_ON(PAGE_SIZE < HV_HYP_PAGE_SIZE); + /* * Setup the vmbus event connection for channel interrupt * abstraction stuff */ - vmbus_connection.int_page = hv_alloc_hyperv_zeroed_page(); + vmbus_connection.int_page = + (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); if (vmbus_connection.int_page == NULL) { ret = -ENOMEM; goto cleanup; @@ -226,8 +235,8 @@ int vmbus_connect(void) * Setup the monitor notification facility. The 1st page for * parent->child and the 2nd page for child->parent */ - vmbus_connection.monitor_pages[0] = hv_alloc_hyperv_page(); - vmbus_connection.monitor_pages[1] = hv_alloc_hyperv_page(); + vmbus_connection.monitor_pages[0] = (void *)__get_free_page(GFP_KERNEL); + vmbus_connection.monitor_pages[1] = (void *)__get_free_page(GFP_KERNEL); if ((vmbus_connection.monitor_pages[0] == NULL) || (vmbus_connection.monitor_pages[1] == NULL)) { ret = -ENOMEM; @@ -343,21 +352,23 @@ void vmbus_disconnect(void) destroy_workqueue(vmbus_connection.work_queue); if (vmbus_connection.int_page) { - hv_free_hyperv_page(vmbus_connection.int_page); + free_page((unsigned long)vmbus_connection.int_page); vmbus_connection.int_page = NULL; } if (vmbus_connection.monitor_pages[0]) { if (!set_memory_encrypted( (unsigned long)vmbus_connection.monitor_pages[0], 1)) - hv_free_hyperv_page(vmbus_connection.monitor_pages[0]); + free_page((unsigned long) + vmbus_connection.monitor_pages[0]); vmbus_connection.monitor_pages[0] = NULL; } if (vmbus_connection.monitor_pages[1]) { if (!set_memory_encrypted( (unsigned long)vmbus_connection.monitor_pages[1], 1)) - hv_free_hyperv_page(vmbus_connection.monitor_pages[1]); + free_page((unsigned long) + vmbus_connection.monitor_pages[1]); vmbus_connection.monitor_pages[1] = NULL; } } diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 787b150686418..34b60009114a6 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -419,6 +419,11 @@ static inline bool hv_is_perf_channel(struct vmbus_channel *channel) return vmbus_devs[channel->device_id].perf_device; } +static inline size_t hv_dev_ring_size(struct vmbus_channel *channel) +{ + return vmbus_devs[channel->device_id].pref_ring_size; +} + static inline bool hv_is_allocated_cpu(unsigned int cpu) { struct vmbus_channel *channel, *sc; diff --git a/drivers/hwmon/adt7475.c b/drivers/hwmon/adt7475.c index 14b2547adae8d..ce0ab4e4e41a7 100644 --- a/drivers/hwmon/adt7475.c +++ b/drivers/hwmon/adt7475.c @@ -22,23 +22,23 @@ #include /* Indexes for the sysfs hooks */ - -#define INPUT 0 -#define MIN 1 -#define MAX 2 -#define CONTROL 3 -#define OFFSET 3 -#define AUTOMIN 4 -#define THERM 5 -#define HYSTERSIS 6 - +enum adt_sysfs_id { + INPUT = 0, + MIN = 1, + MAX = 2, + CONTROL = 3, + OFFSET = 3, // Dup + AUTOMIN = 4, + THERM = 5, + HYSTERSIS = 6, /* * These are unique identifiers for the sysfs functions - unlike the * numbers above, these are not also indexes into an array */ + ALARM = 9, + FAULT = 10, +}; -#define ALARM 9 -#define FAULT 10 /* 7475 Common Registers */ diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c index f20b864c1bb20..bc2197f1dfb7f 100644 --- a/drivers/hwmon/asus-ec-sensors.c +++ b/drivers/hwmon/asus-ec-sensors.c @@ -49,7 +49,7 @@ static char *mutex_path_override; */ #define ASUS_EC_MAX_BANK 3 -#define ACPI_LOCK_DELAY_MS 500 +#define ACPI_LOCK_DELAY_MS 800 /* ACPI mutex for locking access to the EC for the firmware */ #define ASUS_HW_ACCESS_MUTEX_ASMX "\\AMW0.ASMX" @@ -888,6 +888,10 @@ static int asus_ec_hwmon_read_string(struct device *dev, { struct ec_sensors_data *state = dev_get_drvdata(dev); int sensor_index = find_ec_sensor_index(state, type, channel); + + if (sensor_index < 0) + return sensor_index; + *str = get_sensor_info(state, sensor_index)->label; return 0; diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c index 280b90646a873..b37f36f55f88a 100644 --- a/drivers/hwmon/corsair-cpro.c +++ b/drivers/hwmon/corsair-cpro.c @@ -84,6 +84,7 @@ struct ccp_device { struct mutex mutex; /* whenever buffer is used, lock before send_usb_cmd */ u8 *cmd_buffer; u8 *buffer; + int buffer_recv_size; /* number of received bytes in buffer */ int target[6]; DECLARE_BITMAP(temp_cnct, NUM_TEMP_SENSORS); DECLARE_BITMAP(fan_cnct, NUM_FANS); @@ -139,6 +140,9 @@ static int send_usb_cmd(struct ccp_device *ccp, u8 command, u8 byte1, u8 byte2, if (!t) return -ETIMEDOUT; + if (ccp->buffer_recv_size != IN_BUFFER_SIZE) + return -EPROTO; + return ccp_get_errno(ccp); } @@ -150,6 +154,7 @@ static int ccp_raw_event(struct hid_device *hdev, struct hid_report *report, u8 spin_lock(&ccp->wait_input_report_lock); if (!completion_done(&ccp->wait_input_report)) { memcpy(ccp->buffer, data, min(IN_BUFFER_SIZE, size)); + ccp->buffer_recv_size = size; complete_all(&ccp->wait_input_report); } spin_unlock(&ccp->wait_input_report_lock); diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 44aaf9b9191d4..9844843bdfadc 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -67,7 +67,7 @@ #define I8K_POWER_BATTERY 0x01 #define DELL_SMM_NO_TEMP 10 -#define DELL_SMM_NO_FANS 3 +#define DELL_SMM_NO_FANS 4 struct dell_smm_data { struct mutex i8k_mutex; /* lock for sensors writes */ @@ -940,11 +940,14 @@ static const struct hwmon_channel_info * const dell_smm_info[] = { HWMON_F_INPUT | HWMON_F_LABEL | HWMON_F_MIN | HWMON_F_MAX | HWMON_F_TARGET, HWMON_F_INPUT | HWMON_F_LABEL | HWMON_F_MIN | HWMON_F_MAX | + HWMON_F_TARGET, + HWMON_F_INPUT | HWMON_F_LABEL | HWMON_F_MIN | HWMON_F_MAX | HWMON_F_TARGET ), HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT | HWMON_PWM_ENABLE, HWMON_PWM_INPUT, + HWMON_PWM_INPUT, HWMON_PWM_INPUT ), NULL @@ -1155,6 +1158,13 @@ static const struct dmi_system_id i8k_dmi_table[] __initconst = { }, .driver_data = (void *)&i8k_config_data[DELL_PRECISION_490], }, + { + .ident = "Dell OptiPlex 7040", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7040"), + }, + }, { .ident = "Dell Precision", .matches = { diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c index 29f0e4945f192..840acd5260f4e 100644 --- a/drivers/hwmon/emc2305.c +++ b/drivers/hwmon/emc2305.c @@ -303,6 +303,12 @@ static int emc2305_set_single_tz(struct device *dev, int idx) dev_err(dev, "Failed to register cooling device %s\n", emc2305_fan_name[idx]); return PTR_ERR(data->cdev_data[cdev_idx].cdev); } + + if (data->cdev_data[cdev_idx].cur_state > 0) + /* Update pwm when temperature is above trips */ + pwm = EMC2305_PWM_STATE2DUTY(data->cdev_data[cdev_idx].cur_state, + data->max_state, EMC2305_FAN_MAX); + /* Set minimal PWM speed. */ if (data->pwm_separate) { ret = emc2305_set_pwm(dev, pwm, cdev_idx); @@ -316,10 +322,10 @@ static int emc2305_set_single_tz(struct device *dev, int idx) } } data->cdev_data[cdev_idx].cur_state = - EMC2305_PWM_DUTY2STATE(data->pwm_min[cdev_idx], data->max_state, + EMC2305_PWM_DUTY2STATE(pwm, data->max_state, EMC2305_FAN_MAX); data->cdev_data[cdev_idx].last_hwmon_state = - EMC2305_PWM_DUTY2STATE(data->pwm_min[cdev_idx], data->max_state, + EMC2305_PWM_DUTY2STATE(pwm, data->max_state, EMC2305_FAN_MAX); return 0; } diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c index b74a2665e7332..27e5dd94b04f2 100644 --- a/drivers/hwmon/ftsteutates.c +++ b/drivers/hwmon/ftsteutates.c @@ -423,13 +423,16 @@ static int fts_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, break; case hwmon_pwm: switch (attr) { - case hwmon_pwm_auto_channels_temp: - if (data->fan_source[channel] == FTS_FAN_SOURCE_INVALID) + case hwmon_pwm_auto_channels_temp: { + u8 fan_source = data->fan_source[channel]; + + if (fan_source == FTS_FAN_SOURCE_INVALID || fan_source >= BITS_PER_LONG) *val = 0; else - *val = BIT(data->fan_source[channel]); + *val = BIT(fan_source); return 0; + } default: break; } diff --git a/drivers/hwmon/gpio-fan.c b/drivers/hwmon/gpio-fan.c index d92c536be9af7..b779240328d59 100644 --- a/drivers/hwmon/gpio-fan.c +++ b/drivers/hwmon/gpio-fan.c @@ -393,7 +393,12 @@ static int gpio_fan_set_cur_state(struct thermal_cooling_device *cdev, if (state >= fan_data->num_speed) return -EINVAL; + mutex_lock(&fan_data->lock); + set_fan_speed(fan_data, state); + + mutex_unlock(&fan_data->lock); + return 0; } @@ -489,7 +494,11 @@ MODULE_DEVICE_TABLE(of, of_gpio_fan_match); static void gpio_fan_stop(void *data) { + struct gpio_fan_data *fan_data = data; + + mutex_lock(&fan_data->lock); set_fan_speed(data, 0); + mutex_unlock(&fan_data->lock); } static int gpio_fan_probe(struct platform_device *pdev) @@ -562,7 +571,9 @@ static int gpio_fan_suspend(struct device *dev) if (fan_data->gpios) { fan_data->resume_speed = fan_data->speed_index; + mutex_lock(&fan_data->lock); set_fan_speed(fan_data, 0); + mutex_unlock(&fan_data->lock); } return 0; @@ -572,8 +583,11 @@ static int gpio_fan_resume(struct device *dev) { struct gpio_fan_data *fan_data = dev_get_drvdata(dev); - if (fan_data->gpios) + if (fan_data->gpios) { + mutex_lock(&fan_data->lock); set_fan_speed(fan_data, fan_data->resume_speed); + mutex_unlock(&fan_data->lock); + } return 0; } diff --git a/drivers/hwmon/gsc-hwmon.c b/drivers/hwmon/gsc-hwmon.c index 1501ceb551e79..23238a80c000d 100644 --- a/drivers/hwmon/gsc-hwmon.c +++ b/drivers/hwmon/gsc-hwmon.c @@ -65,7 +65,7 @@ static ssize_t pwm_auto_point_temp_show(struct device *dev, return ret; ret = regs[0] | regs[1] << 8; - return sprintf(buf, "%d\n", ret * 10); + return sprintf(buf, "%d\n", ret * 100); } static ssize_t pwm_auto_point_temp_store(struct device *dev, @@ -100,7 +100,7 @@ static ssize_t pwm_auto_point_pwm_show(struct device *dev, { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - return sprintf(buf, "%d\n", 255 * (50 + (attr->index * 10))); + return sprintf(buf, "%d\n", 255 * (50 + (attr->index * 10)) / 100); } static SENSOR_DEVICE_ATTR_RO(pwm1_auto_point1_pwm, pwm_auto_point_pwm, 0); diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c index c906731c6c2d3..dc82e33d59c5f 100644 --- a/drivers/hwmon/k10temp.c +++ b/drivers/hwmon/k10temp.c @@ -84,6 +84,13 @@ static DEFINE_MUTEX(nb_smu_ind_mutex); */ #define AMD_I3255_STR "3255" +/* + * PCI Device IDs for AMD's Family 1Ah-based SOCs. + * Defining locally as IDs are not shared. + */ +#define PCI_DEVICE_ID_AMD_1AH_M50H_DF_F3 0x12cb +#define PCI_DEVICE_ID_AMD_1AH_M90H_DF_F3 0x127b + struct k10temp_data { struct pci_dev *pdev; void (*read_htcreg)(struct pci_dev *pdev, u32 *regval); @@ -545,7 +552,10 @@ static const struct pci_device_id k10temp_id_table[] = { { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_19H_M78H_DF_F3) }, { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M00H_DF_F3) }, { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M20H_DF_F3) }, + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M50H_DF_F3) }, { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M60H_DF_F3) }, + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M70H_DF_F3) }, + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M90H_DF_F3) }, { PCI_VDEVICE(HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) }, {} }; diff --git a/drivers/hwmon/mlxreg-fan.c b/drivers/hwmon/mlxreg-fan.c index a5f89aab3fb4d..0ba9195c9d713 100644 --- a/drivers/hwmon/mlxreg-fan.c +++ b/drivers/hwmon/mlxreg-fan.c @@ -113,8 +113,8 @@ struct mlxreg_fan { int divider; }; -static int mlxreg_fan_set_cur_state(struct thermal_cooling_device *cdev, - unsigned long state); +static int _mlxreg_fan_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state, bool thermal); static int mlxreg_fan_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, @@ -224,8 +224,9 @@ mlxreg_fan_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, * last thermal state. */ if (pwm->last_hwmon_state >= pwm->last_thermal_state) - return mlxreg_fan_set_cur_state(pwm->cdev, - pwm->last_hwmon_state); + return _mlxreg_fan_set_cur_state(pwm->cdev, + pwm->last_hwmon_state, + false); return 0; } return regmap_write(fan->regmap, pwm->reg, val); @@ -357,9 +358,8 @@ static int mlxreg_fan_get_cur_state(struct thermal_cooling_device *cdev, return 0; } -static int mlxreg_fan_set_cur_state(struct thermal_cooling_device *cdev, - unsigned long state) - +static int _mlxreg_fan_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state, bool thermal) { struct mlxreg_fan_pwm *pwm = cdev->devdata; struct mlxreg_fan *fan = pwm->fan; @@ -369,7 +369,8 @@ static int mlxreg_fan_set_cur_state(struct thermal_cooling_device *cdev, return -EINVAL; /* Save thermal state. */ - pwm->last_thermal_state = state; + if (thermal) + pwm->last_thermal_state = state; state = max_t(unsigned long, state, pwm->last_hwmon_state); err = regmap_write(fan->regmap, pwm->reg, @@ -381,6 +382,13 @@ static int mlxreg_fan_set_cur_state(struct thermal_cooling_device *cdev, return 0; } +static int mlxreg_fan_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state) + +{ + return _mlxreg_fan_set_cur_state(cdev, state, true); +} + static const struct thermal_cooling_device_ops mlxreg_fan_cooling_ops = { .get_max_state = mlxreg_fan_get_max_state, .get_cur_state = mlxreg_fan_get_cur_state, @@ -561,15 +569,14 @@ static int mlxreg_fan_cooling_config(struct device *dev, struct mlxreg_fan *fan) if (!pwm->connected) continue; pwm->fan = fan; + /* Set minimal PWM speed. */ + pwm->last_hwmon_state = MLXREG_FAN_PWM_DUTY2STATE(MLXREG_FAN_MIN_DUTY); pwm->cdev = devm_thermal_of_cooling_device_register(dev, NULL, mlxreg_fan_name[i], pwm, &mlxreg_fan_cooling_ops); if (IS_ERR(pwm->cdev)) { dev_err(dev, "Failed to register cooling device\n"); return PTR_ERR(pwm->cdev); } - - /* Set minimal PWM speed. */ - pwm->last_hwmon_state = MLXREG_FAN_PWM_DUTY2STATE(MLXREG_FAN_MIN_DUTY); } return 0; diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index dd690f700d499..483f79b394298 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -459,12 +459,10 @@ static ssize_t occ_show_power_1(struct device *dev, return sysfs_emit(buf, "%llu\n", val); } -static u64 occ_get_powr_avg(u64 *accum, u32 *samples) +static u64 occ_get_powr_avg(u64 accum, u32 samples) { - u64 divisor = get_unaligned_be32(samples); - - return (divisor == 0) ? 0 : - div64_u64(get_unaligned_be64(accum) * 1000000ULL, divisor); + return (samples == 0) ? 0 : + mul_u64_u32_div(accum, 1000000UL, samples); } static ssize_t occ_show_power_2(struct device *dev, @@ -489,8 +487,8 @@ static ssize_t occ_show_power_2(struct device *dev, get_unaligned_be32(&power->sensor_id), power->function_id, power->apss_channel); case 1: - val = occ_get_powr_avg(&power->accumulator, - &power->update_tag); + val = occ_get_powr_avg(get_unaligned_be64(&power->accumulator), + get_unaligned_be32(&power->update_tag)); break; case 2: val = (u64)get_unaligned_be32(&power->update_tag) * @@ -527,8 +525,8 @@ static ssize_t occ_show_power_a0(struct device *dev, return sysfs_emit(buf, "%u_system\n", get_unaligned_be32(&power->sensor_id)); case 1: - val = occ_get_powr_avg(&power->system.accumulator, - &power->system.update_tag); + val = occ_get_powr_avg(get_unaligned_be64(&power->system.accumulator), + get_unaligned_be32(&power->system.update_tag)); break; case 2: val = (u64)get_unaligned_be32(&power->system.update_tag) * @@ -541,8 +539,8 @@ static ssize_t occ_show_power_a0(struct device *dev, return sysfs_emit(buf, "%u_proc\n", get_unaligned_be32(&power->sensor_id)); case 5: - val = occ_get_powr_avg(&power->proc.accumulator, - &power->proc.update_tag); + val = occ_get_powr_avg(get_unaligned_be64(&power->proc.accumulator), + get_unaligned_be32(&power->proc.update_tag)); break; case 6: val = (u64)get_unaligned_be32(&power->proc.update_tag) * @@ -555,8 +553,8 @@ static ssize_t occ_show_power_a0(struct device *dev, return sysfs_emit(buf, "%u_vdd\n", get_unaligned_be32(&power->sensor_id)); case 9: - val = occ_get_powr_avg(&power->vdd.accumulator, - &power->vdd.update_tag); + val = occ_get_powr_avg(get_unaligned_be64(&power->vdd.accumulator), + get_unaligned_be32(&power->vdd.update_tag)); break; case 10: val = (u64)get_unaligned_be32(&power->vdd.update_tag) * @@ -569,8 +567,8 @@ static ssize_t occ_show_power_a0(struct device *dev, return sysfs_emit(buf, "%u_vdn\n", get_unaligned_be32(&power->sensor_id)); case 13: - val = occ_get_powr_avg(&power->vdn.accumulator, - &power->vdn.update_tag); + val = occ_get_powr_avg(get_unaligned_be64(&power->vdn.accumulator), + get_unaligned_be32(&power->vdn.update_tag)); break; case 14: val = (u64)get_unaligned_be32(&power->vdn.update_tag) * @@ -747,29 +745,30 @@ static ssize_t occ_show_extended(struct device *dev, } /* - * Some helper macros to make it easier to define an occ_attribute. Since these - * are dynamically allocated, we shouldn't use the existing kernel macros which + * A helper to make it easier to define an occ_attribute. Since these + * are dynamically allocated, we cannot use the existing kernel macros which * stringify the name argument. */ -#define ATTR_OCC(_name, _mode, _show, _store) { \ - .attr = { \ - .name = _name, \ - .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \ - }, \ - .show = _show, \ - .store = _store, \ -} - -#define SENSOR_ATTR_OCC(_name, _mode, _show, _store, _nr, _index) { \ - .dev_attr = ATTR_OCC(_name, _mode, _show, _store), \ - .index = _index, \ - .nr = _nr, \ +static void occ_init_attribute(struct occ_attribute *attr, int mode, + ssize_t (*show)(struct device *dev, struct device_attribute *attr, char *buf), + ssize_t (*store)(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count), + int nr, int index, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + vsnprintf(attr->name, sizeof(attr->name), fmt, args); + va_end(args); + + attr->sensor.dev_attr.attr.name = attr->name; + attr->sensor.dev_attr.attr.mode = mode; + attr->sensor.dev_attr.show = show; + attr->sensor.dev_attr.store = store; + attr->sensor.index = index; + attr->sensor.nr = nr; } -#define OCC_INIT_ATTR(_name, _mode, _show, _store, _nr, _index) \ - ((struct sensor_device_attribute_2) \ - SENSOR_ATTR_OCC(_name, _mode, _show, _store, _nr, _index)) - /* * Allocate and instatiate sensor_device_attribute_2s. It's most efficient to * use our own instead of the built-in hwmon attribute types. @@ -855,14 +854,15 @@ static int occ_setup_sensor_attrs(struct occ *occ) sensors->extended.num_sensors = 0; } - occ->attrs = devm_kzalloc(dev, sizeof(*occ->attrs) * num_attrs, + occ->attrs = devm_kcalloc(dev, num_attrs, sizeof(*occ->attrs), GFP_KERNEL); if (!occ->attrs) return -ENOMEM; /* null-terminated list */ - occ->group.attrs = devm_kzalloc(dev, sizeof(*occ->group.attrs) * - num_attrs + 1, GFP_KERNEL); + occ->group.attrs = devm_kcalloc(dev, num_attrs + 1, + sizeof(*occ->group.attrs), + GFP_KERNEL); if (!occ->group.attrs) return -ENOMEM; @@ -872,43 +872,33 @@ static int occ_setup_sensor_attrs(struct occ *occ) s = i + 1; temp = ((struct temp_sensor_2 *)sensors->temp.data) + i; - snprintf(attr->name, sizeof(attr->name), "temp%d_label", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_temp, NULL, - 0, i); + occ_init_attribute(attr, 0444, show_temp, NULL, + 0, i, "temp%d_label", s); attr++; if (sensors->temp.version == 2 && temp->fru_type == OCC_FRU_TYPE_VRM) { - snprintf(attr->name, sizeof(attr->name), - "temp%d_alarm", s); + occ_init_attribute(attr, 0444, show_temp, NULL, + 1, i, "temp%d_alarm", s); } else { - snprintf(attr->name, sizeof(attr->name), - "temp%d_input", s); + occ_init_attribute(attr, 0444, show_temp, NULL, + 1, i, "temp%d_input", s); } - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_temp, NULL, - 1, i); attr++; if (sensors->temp.version > 1) { - snprintf(attr->name, sizeof(attr->name), - "temp%d_fru_type", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_temp, NULL, 2, i); + occ_init_attribute(attr, 0444, show_temp, NULL, + 2, i, "temp%d_fru_type", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "temp%d_fault", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_temp, NULL, 3, i); + occ_init_attribute(attr, 0444, show_temp, NULL, + 3, i, "temp%d_fault", s); attr++; if (sensors->temp.version == 0x10) { - snprintf(attr->name, sizeof(attr->name), - "temp%d_max", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_temp, NULL, - 4, i); + occ_init_attribute(attr, 0444, show_temp, NULL, + 4, i, "temp%d_max", s); attr++; } } @@ -917,14 +907,12 @@ static int occ_setup_sensor_attrs(struct occ *occ) for (i = 0; i < sensors->freq.num_sensors; ++i) { s = i + 1; - snprintf(attr->name, sizeof(attr->name), "freq%d_label", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_freq, NULL, - 0, i); + occ_init_attribute(attr, 0444, show_freq, NULL, + 0, i, "freq%d_label", s); attr++; - snprintf(attr->name, sizeof(attr->name), "freq%d_input", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_freq, NULL, - 1, i); + occ_init_attribute(attr, 0444, show_freq, NULL, + 1, i, "freq%d_input", s); attr++; } @@ -940,32 +928,24 @@ static int occ_setup_sensor_attrs(struct occ *occ) s = (i * 4) + 1; for (j = 0; j < 4; ++j) { - snprintf(attr->name, sizeof(attr->name), - "power%d_label", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, - nr++, i); + occ_init_attribute(attr, 0444, show_power, + NULL, nr++, i, + "power%d_label", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "power%d_average", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, - nr++, i); + occ_init_attribute(attr, 0444, show_power, + NULL, nr++, i, + "power%d_average", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "power%d_average_interval", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, - nr++, i); + occ_init_attribute(attr, 0444, show_power, + NULL, nr++, i, + "power%d_average_interval", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "power%d_input", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, - nr++, i); + occ_init_attribute(attr, 0444, show_power, + NULL, nr++, i, + "power%d_input", s); attr++; s++; @@ -977,28 +957,20 @@ static int occ_setup_sensor_attrs(struct occ *occ) for (i = 0; i < sensors->power.num_sensors; ++i) { s = i + 1; - snprintf(attr->name, sizeof(attr->name), - "power%d_label", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, 0, i); + occ_init_attribute(attr, 0444, show_power, NULL, + 0, i, "power%d_label", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "power%d_average", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, 1, i); + occ_init_attribute(attr, 0444, show_power, NULL, + 1, i, "power%d_average", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "power%d_average_interval", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, 2, i); + occ_init_attribute(attr, 0444, show_power, NULL, + 2, i, "power%d_average_interval", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "power%d_input", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_power, NULL, 3, i); + occ_init_attribute(attr, 0444, show_power, NULL, + 3, i, "power%d_input", s); attr++; } @@ -1006,56 +978,43 @@ static int occ_setup_sensor_attrs(struct occ *occ) } if (sensors->caps.num_sensors >= 1) { - snprintf(attr->name, sizeof(attr->name), "power%d_label", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_caps, NULL, - 0, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 0, 0, "power%d_label", s); attr++; - snprintf(attr->name, sizeof(attr->name), "power%d_cap", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_caps, NULL, - 1, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 1, 0, "power%d_cap", s); attr++; - snprintf(attr->name, sizeof(attr->name), "power%d_input", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_caps, NULL, - 2, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 2, 0, "power%d_input", s); attr++; - snprintf(attr->name, sizeof(attr->name), - "power%d_cap_not_redundant", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_caps, NULL, - 3, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 3, 0, "power%d_cap_not_redundant", s); attr++; - snprintf(attr->name, sizeof(attr->name), "power%d_cap_max", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_caps, NULL, - 4, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 4, 0, "power%d_cap_max", s); attr++; - snprintf(attr->name, sizeof(attr->name), "power%d_cap_min", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, show_caps, NULL, - 5, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 5, 0, "power%d_cap_min", s); attr++; - snprintf(attr->name, sizeof(attr->name), "power%d_cap_user", - s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0644, show_caps, - occ_store_caps_user, 6, 0); + occ_init_attribute(attr, 0644, show_caps, occ_store_caps_user, + 6, 0, "power%d_cap_user", s); attr++; if (sensors->caps.version > 1) { - snprintf(attr->name, sizeof(attr->name), - "power%d_cap_user_source", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_caps, NULL, 7, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 7, 0, "power%d_cap_user_source", s); attr++; if (sensors->caps.version > 2) { - snprintf(attr->name, sizeof(attr->name), - "power%d_cap_min_soft", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - show_caps, NULL, - 8, 0); + occ_init_attribute(attr, 0444, show_caps, NULL, + 8, 0, + "power%d_cap_min_soft", s); attr++; } } @@ -1064,19 +1023,16 @@ static int occ_setup_sensor_attrs(struct occ *occ) for (i = 0; i < sensors->extended.num_sensors; ++i) { s = i + 1; - snprintf(attr->name, sizeof(attr->name), "extn%d_label", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - occ_show_extended, NULL, 0, i); + occ_init_attribute(attr, 0444, occ_show_extended, NULL, + 0, i, "extn%d_label", s); attr++; - snprintf(attr->name, sizeof(attr->name), "extn%d_flags", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - occ_show_extended, NULL, 1, i); + occ_init_attribute(attr, 0444, occ_show_extended, NULL, + 1, i, "extn%d_flags", s); attr++; - snprintf(attr->name, sizeof(attr->name), "extn%d_input", s); - attr->sensor = OCC_INIT_ATTR(attr->name, 0444, - occ_show_extended, NULL, 2, i); + occ_init_attribute(attr, 0444, occ_show_extended, NULL, + 2, i, "extn%d_input", s); attr++; } diff --git a/drivers/hwmon/pmbus/max34440.c b/drivers/hwmon/pmbus/max34440.c index fe7f6b1b09851..e14be8ebaad30 100644 --- a/drivers/hwmon/pmbus/max34440.c +++ b/drivers/hwmon/pmbus/max34440.c @@ -34,16 +34,21 @@ enum chips { max34440, max34441, max34446, max34451, max34460, max34461 }; /* * The whole max344* family have IOUT_OC_WARN_LIMIT and IOUT_OC_FAULT_LIMIT * swapped from the standard pmbus spec addresses. + * For max34451, version MAX34451ETNA6+ and later has this issue fixed. */ #define MAX34440_IOUT_OC_WARN_LIMIT 0x46 #define MAX34440_IOUT_OC_FAULT_LIMIT 0x4A +#define MAX34451ETNA6_MFR_REV 0x0012 + #define MAX34451_MFR_CHANNEL_CONFIG 0xe4 #define MAX34451_MFR_CHANNEL_CONFIG_SEL_MASK 0x3f struct max34440_data { int id; struct pmbus_driver_info info; + u8 iout_oc_warn_limit; + u8 iout_oc_fault_limit; }; #define to_max34440_data(x) container_of(x, struct max34440_data, info) @@ -60,11 +65,11 @@ static int max34440_read_word_data(struct i2c_client *client, int page, switch (reg) { case PMBUS_IOUT_OC_FAULT_LIMIT: ret = pmbus_read_word_data(client, page, phase, - MAX34440_IOUT_OC_FAULT_LIMIT); + data->iout_oc_fault_limit); break; case PMBUS_IOUT_OC_WARN_LIMIT: ret = pmbus_read_word_data(client, page, phase, - MAX34440_IOUT_OC_WARN_LIMIT); + data->iout_oc_warn_limit); break; case PMBUS_VIRT_READ_VOUT_MIN: ret = pmbus_read_word_data(client, page, phase, @@ -133,11 +138,11 @@ static int max34440_write_word_data(struct i2c_client *client, int page, switch (reg) { case PMBUS_IOUT_OC_FAULT_LIMIT: - ret = pmbus_write_word_data(client, page, MAX34440_IOUT_OC_FAULT_LIMIT, + ret = pmbus_write_word_data(client, page, data->iout_oc_fault_limit, word); break; case PMBUS_IOUT_OC_WARN_LIMIT: - ret = pmbus_write_word_data(client, page, MAX34440_IOUT_OC_WARN_LIMIT, + ret = pmbus_write_word_data(client, page, data->iout_oc_warn_limit, word); break; case PMBUS_VIRT_RESET_POUT_HISTORY: @@ -235,6 +240,25 @@ static int max34451_set_supported_funcs(struct i2c_client *client, */ int page, rv; + bool max34451_na6 = false; + + rv = i2c_smbus_read_word_data(client, PMBUS_MFR_REVISION); + if (rv < 0) + return rv; + + if (rv >= MAX34451ETNA6_MFR_REV) { + max34451_na6 = true; + data->info.format[PSC_VOLTAGE_IN] = direct; + data->info.format[PSC_CURRENT_IN] = direct; + data->info.m[PSC_VOLTAGE_IN] = 1; + data->info.b[PSC_VOLTAGE_IN] = 0; + data->info.R[PSC_VOLTAGE_IN] = 3; + data->info.m[PSC_CURRENT_IN] = 1; + data->info.b[PSC_CURRENT_IN] = 0; + data->info.R[PSC_CURRENT_IN] = 2; + data->iout_oc_fault_limit = PMBUS_IOUT_OC_FAULT_LIMIT; + data->iout_oc_warn_limit = PMBUS_IOUT_OC_WARN_LIMIT; + } for (page = 0; page < 16; page++) { rv = i2c_smbus_write_byte_data(client, PMBUS_PAGE, page); @@ -251,16 +275,30 @@ static int max34451_set_supported_funcs(struct i2c_client *client, case 0x20: data->info.func[page] = PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT; + + if (max34451_na6) + data->info.func[page] |= PMBUS_HAVE_VIN | + PMBUS_HAVE_STATUS_INPUT; break; case 0x21: data->info.func[page] = PMBUS_HAVE_VOUT; + + if (max34451_na6) + data->info.func[page] |= PMBUS_HAVE_VIN; break; case 0x22: data->info.func[page] = PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT; + + if (max34451_na6) + data->info.func[page] |= PMBUS_HAVE_IIN | + PMBUS_HAVE_STATUS_INPUT; break; case 0x23: data->info.func[page] = PMBUS_HAVE_IOUT; + + if (max34451_na6) + data->info.func[page] |= PMBUS_HAVE_IIN; break; default: break; @@ -494,6 +532,8 @@ static int max34440_probe(struct i2c_client *client) return -ENOMEM; data->id = i2c_match_id(max34440_id, client)->driver_data; data->info = max34440_info[data->id]; + data->iout_oc_fault_limit = MAX34440_IOUT_OC_FAULT_LIMIT; + data->iout_oc_warn_limit = MAX34440_IOUT_OC_WARN_LIMIT; if (data->id == max34451) { rv = max34451_set_supported_funcs(client, data); diff --git a/drivers/hwmon/sbtsi_temp.c b/drivers/hwmon/sbtsi_temp.c index dd85cf89f008a..7c49fcf864143 100644 --- a/drivers/hwmon/sbtsi_temp.c +++ b/drivers/hwmon/sbtsi_temp.c @@ -14,6 +14,7 @@ #include #include #include +#include /* * SB-TSI registers only support SMBus byte data access. "_INT" registers are @@ -29,8 +30,22 @@ #define SBTSI_REG_TEMP_HIGH_DEC 0x13 /* RW */ #define SBTSI_REG_TEMP_LOW_DEC 0x14 /* RW */ +/* + * Bit for reporting value with temperature measurement range. + * bit == 0: Use default temperature range (0C to 255.875C). + * bit == 1: Use extended temperature range (-49C to +206.875C). + */ +#define SBTSI_CONFIG_EXT_RANGE_SHIFT 2 +/* + * ReadOrder bit specifies the reading order of integer and decimal part of + * CPU temperature for atomic reads. If bit == 0, reading integer part triggers + * latching of the decimal part, so integer part should be read first. + * If bit == 1, read order should be reversed. + */ #define SBTSI_CONFIG_READ_ORDER_SHIFT 5 +#define SBTSI_TEMP_EXT_RANGE_ADJ 49000 + #define SBTSI_TEMP_MIN 0 #define SBTSI_TEMP_MAX 255875 @@ -38,6 +53,8 @@ struct sbtsi_data { struct i2c_client *client; struct mutex lock; + bool ext_range_mode; + bool read_order; }; /* @@ -74,23 +91,11 @@ static int sbtsi_read(struct device *dev, enum hwmon_sensor_types type, { struct sbtsi_data *data = dev_get_drvdata(dev); s32 temp_int, temp_dec; - int err; switch (attr) { case hwmon_temp_input: - /* - * ReadOrder bit specifies the reading order of integer and - * decimal part of CPU temp for atomic reads. If bit == 0, - * reading integer part triggers latching of the decimal part, - * so integer part should be read first. If bit == 1, read - * order should be reversed. - */ - err = i2c_smbus_read_byte_data(data->client, SBTSI_REG_CONFIG); - if (err < 0) - return err; - mutex_lock(&data->lock); - if (err & BIT(SBTSI_CONFIG_READ_ORDER_SHIFT)) { + if (data->read_order) { temp_dec = i2c_smbus_read_byte_data(data->client, SBTSI_REG_TEMP_DEC); temp_int = i2c_smbus_read_byte_data(data->client, SBTSI_REG_TEMP_INT); } else { @@ -122,6 +127,8 @@ static int sbtsi_read(struct device *dev, enum hwmon_sensor_types type, return temp_dec; *val = sbtsi_reg_to_mc(temp_int, temp_dec); + if (data->ext_range_mode) + *val -= SBTSI_TEMP_EXT_RANGE_ADJ; return 0; } @@ -146,6 +153,8 @@ static int sbtsi_write(struct device *dev, enum hwmon_sensor_types type, return -EINVAL; } + if (data->ext_range_mode) + val += SBTSI_TEMP_EXT_RANGE_ADJ; val = clamp_val(val, SBTSI_TEMP_MIN, SBTSI_TEMP_MAX); sbtsi_mc_to_reg(val, &temp_int, &temp_dec); @@ -203,6 +212,7 @@ static int sbtsi_probe(struct i2c_client *client) struct device *dev = &client->dev; struct device *hwmon_dev; struct sbtsi_data *data; + int err; data = devm_kzalloc(dev, sizeof(struct sbtsi_data), GFP_KERNEL); if (!data) @@ -211,8 +221,14 @@ static int sbtsi_probe(struct i2c_client *client) data->client = client; mutex_init(&data->lock); - hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, data, &sbtsi_chip_info, - NULL); + err = i2c_smbus_read_byte_data(data->client, SBTSI_REG_CONFIG); + if (err < 0) + return err; + data->ext_range_mode = FIELD_GET(BIT(SBTSI_CONFIG_EXT_RANGE_SHIFT), err); + data->read_order = FIELD_GET(BIT(SBTSI_CONFIG_READ_ORDER_SHIFT), err); + + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, data, + &sbtsi_chip_info, NULL); return PTR_ERR_OR_ZERO(hwmon_dev); } diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c index 79657910b79e6..d8a86e60cf8c1 100644 --- a/drivers/hwmon/sht3x.c +++ b/drivers/hwmon/sht3x.c @@ -288,24 +288,26 @@ static struct sht3x_data *sht3x_update_client(struct device *dev) return data; } -static int temp1_input_read(struct device *dev) +static int temp1_input_read(struct device *dev, long *temp) { struct sht3x_data *data = sht3x_update_client(dev); if (IS_ERR(data)) return PTR_ERR(data); - return data->temperature; + *temp = data->temperature; + return 0; } -static int humidity1_input_read(struct device *dev) +static int humidity1_input_read(struct device *dev, long *humidity) { struct sht3x_data *data = sht3x_update_client(dev); if (IS_ERR(data)) return PTR_ERR(data); - return data->humidity; + *humidity = data->humidity; + return 0; } /* @@ -703,6 +705,7 @@ static int sht3x_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { enum sht3x_limits index; + int ret; switch (type) { case hwmon_chip: @@ -717,10 +720,12 @@ static int sht3x_read(struct device *dev, enum hwmon_sensor_types type, case hwmon_temp: switch (attr) { case hwmon_temp_input: - *val = temp1_input_read(dev); - break; + return temp1_input_read(dev, val); case hwmon_temp_alarm: - *val = temp1_alarm_read(dev); + ret = temp1_alarm_read(dev); + if (ret < 0) + return ret; + *val = ret; break; case hwmon_temp_max: index = limit_max; @@ -745,10 +750,12 @@ static int sht3x_read(struct device *dev, enum hwmon_sensor_types type, case hwmon_humidity: switch (attr) { case hwmon_humidity_input: - *val = humidity1_input_read(dev); - break; + return humidity1_input_read(dev, val); case hwmon_humidity_alarm: - *val = humidity1_alarm_read(dev); + ret = humidity1_alarm_read(dev); + if (ret < 0) + return ret; + *val = ret; break; case hwmon_humidity_max: index = limit_max; diff --git a/drivers/hwmon/sy7636a-hwmon.c b/drivers/hwmon/sy7636a-hwmon.c index ed110884786b4..a12fc0ce70e76 100644 --- a/drivers/hwmon/sy7636a-hwmon.c +++ b/drivers/hwmon/sy7636a-hwmon.c @@ -104,3 +104,4 @@ module_platform_driver(sy7636a_sensor_driver); MODULE_DESCRIPTION("SY7636A sensor driver"); MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:sy7636a-temperature"); diff --git a/drivers/hwmon/xgene-hwmon.c b/drivers/hwmon/xgene-hwmon.c index 207084d55044a..6768dbf390390 100644 --- a/drivers/hwmon/xgene-hwmon.c +++ b/drivers/hwmon/xgene-hwmon.c @@ -111,7 +111,7 @@ struct xgene_hwmon_dev { phys_addr_t comm_base_addr; void *pcc_comm_addr; - u64 usecs_lat; + unsigned int usecs_lat; }; /* diff --git a/drivers/hwtracing/coresight/coresight-config.h b/drivers/hwtracing/coresight/coresight-config.h index 6ba0139757418..84cdde6f0e4db 100644 --- a/drivers/hwtracing/coresight/coresight-config.h +++ b/drivers/hwtracing/coresight/coresight-config.h @@ -228,7 +228,7 @@ struct cscfg_feature_csdev { * @feats_csdev:references to the device features to enable. */ struct cscfg_config_csdev { - const struct cscfg_config_desc *config_desc; + struct cscfg_config_desc *config_desc; struct coresight_device *csdev; bool enabled; struct list_head node; diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 6725239e0a5ce..7ee2ed571b2d9 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -135,7 +135,8 @@ coresight_find_out_connection(struct coresight_device *src_dev, static inline u32 coresight_read_claim_tags(struct coresight_device *csdev) { - return csdev_access_relaxed_read32(&csdev->access, CORESIGHT_CLAIMCLR); + return FIELD_GET(CORESIGHT_CLAIM_MASK, + csdev_access_relaxed_read32(&csdev->access, CORESIGHT_CLAIMCLR)); } static inline bool coresight_is_claimed_self_hosted(struct coresight_device *csdev) @@ -1642,8 +1643,9 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) goto out_unlock; } - if (csdev->type == CORESIGHT_DEV_TYPE_SINK || - csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) { + if ((csdev->type == CORESIGHT_DEV_TYPE_SINK || + csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) && + sink_ops(csdev)->alloc_buffer) { ret = etm_perf_add_symlink_sink(csdev); if (ret) { diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 05d9f87e35333..e4d8d446ea4d3 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -481,7 +481,8 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); } - etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); + if (drvdata->numextinsel) + etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); @@ -1323,6 +1324,7 @@ static void etm4_init_arch_data(void *info) etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); /* NUMEXTIN, bits[8:0] number of external inputs implemented */ drvdata->nr_ext_inp = FIELD_GET(TRCIDR5_NUMEXTIN_MASK, etmidr5); + drvdata->numextinsel = FIELD_GET(TRCIDR5_NUMEXTINSEL_MASK, etmidr5); /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ drvdata->trcid_size = FIELD_GET(TRCIDR5_TRACEIDSIZE_MASK, etmidr5); /* ATBTRIG, bit[22] implementation can support ATB triggers? */ @@ -1750,7 +1752,9 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); } - state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); + + if (drvdata->numextinsel) + state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); @@ -1882,7 +1886,8 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); } - etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); + if (drvdata->numextinsel) + etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); @@ -2113,6 +2118,10 @@ static int etm4_probe(struct device *dev) if (WARN_ON(!drvdata)) return -ENOMEM; + drvdata->atclk = devm_clk_get_optional_enabled(dev, "atclk"); + if (IS_ERR(drvdata->atclk)) + return PTR_ERR(drvdata->atclk); + if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) pm_save_enable = coresight_loses_context_with_cpu(dev) ? PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER; @@ -2364,8 +2373,8 @@ static int etm4_runtime_suspend(struct device *dev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); - if (drvdata->pclk && !IS_ERR(drvdata->pclk)) - clk_disable_unprepare(drvdata->pclk); + clk_disable_unprepare(drvdata->atclk); + clk_disable_unprepare(drvdata->pclk); return 0; } @@ -2373,11 +2382,17 @@ static int etm4_runtime_suspend(struct device *dev) static int etm4_runtime_resume(struct device *dev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); + int ret; - if (drvdata->pclk && !IS_ERR(drvdata->pclk)) - clk_prepare_enable(drvdata->pclk); + ret = clk_prepare_enable(drvdata->pclk); + if (ret) + return ret; - return 0; + ret = clk_prepare_enable(drvdata->atclk); + if (ret) + clk_disable_unprepare(drvdata->pclk); + + return ret; } #endif diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 6b6760e49ed35..d8b8102d790cc 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -162,6 +162,7 @@ #define TRCIDR4_NUMVMIDC_MASK GENMASK(31, 28) #define TRCIDR5_NUMEXTIN_MASK GENMASK(8, 0) +#define TRCIDR5_NUMEXTINSEL_MASK GENMASK(11, 9) #define TRCIDR5_TRACEIDSIZE_MASK GENMASK(21, 16) #define TRCIDR5_ATBTRIG BIT(22) #define TRCIDR5_LPOVERRIDE BIT(23) @@ -919,7 +920,8 @@ struct etmv4_save_state { /** * struct etm4_drvdata - specifics associated to an ETM component - * @pclk APB clock if present, otherwise NULL + * @pclk: APB clock if present, otherwise NULL + * @atclk: Optional clock for the core parts of the ETMv4. * @base: Memory mapped base address for this component. * @csdev: Component vitals needed by the framework. * @spinlock: Only one at a time pls. @@ -987,6 +989,7 @@ struct etmv4_save_state { */ struct etmv4_drvdata { struct clk *pclk; + struct clk *atclk; void __iomem *base; struct coresight_device *csdev; spinlock_t spinlock; @@ -999,6 +1002,7 @@ struct etmv4_drvdata { u8 nr_cntr; u8 nr_ext_inp; u8 numcidc; + u8 numextinsel; u8 numvmidc; u8 nrseqstate; u8 nr_event; diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index 30c051055e54b..b758a42ed8c73 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -32,6 +32,7 @@ * Coresight device CLAIM protocol. * See PSCI - ARM DEN 0022D, Section: 6.8.1 Debug and Trace save and restore. */ +#define CORESIGHT_CLAIM_MASK GENMASK(1, 0) #define CORESIGHT_CLAIM_SELF_HOSTED BIT(1) #define TIMEOUT_US 100 diff --git a/drivers/hwtracing/coresight/coresight-syscfg.c b/drivers/hwtracing/coresight/coresight-syscfg.c index 11138a9762b01..30a561d874819 100644 --- a/drivers/hwtracing/coresight/coresight-syscfg.c +++ b/drivers/hwtracing/coresight/coresight-syscfg.c @@ -867,6 +867,25 @@ void cscfg_csdev_reset_feats(struct coresight_device *csdev) } EXPORT_SYMBOL_GPL(cscfg_csdev_reset_feats); +static bool cscfg_config_desc_get(struct cscfg_config_desc *config_desc) +{ + if (!atomic_fetch_inc(&config_desc->active_cnt)) { + /* must ensure that config cannot be unloaded in use */ + if (unlikely(cscfg_owner_get(config_desc->load_owner))) { + atomic_dec(&config_desc->active_cnt); + return false; + } + } + + return true; +} + +static void cscfg_config_desc_put(struct cscfg_config_desc *config_desc) +{ + if (!atomic_dec_return(&config_desc->active_cnt)) + cscfg_owner_put(config_desc->load_owner); +} + /* * This activate configuration for either perf or sysfs. Perf can have multiple * active configs, selected per event, sysfs is limited to one. @@ -890,22 +909,17 @@ static int _cscfg_activate_config(unsigned long cfg_hash) if (config_desc->available == false) return -EBUSY; - /* must ensure that config cannot be unloaded in use */ - err = cscfg_owner_get(config_desc->load_owner); - if (err) + if (!cscfg_config_desc_get(config_desc)) { + err = -EINVAL; break; + } + /* * increment the global active count - control changes to * active configurations */ atomic_inc(&cscfg_mgr->sys_active_cnt); - /* - * mark the descriptor as active so enable config on a - * device instance will use it - */ - atomic_inc(&config_desc->active_cnt); - err = 0; dev_dbg(cscfg_device(), "Activate config %s.\n", config_desc->name); break; @@ -920,9 +934,8 @@ static void _cscfg_deactivate_config(unsigned long cfg_hash) list_for_each_entry(config_desc, &cscfg_mgr->config_desc_list, item) { if ((unsigned long)config_desc->event_ea->var == cfg_hash) { - atomic_dec(&config_desc->active_cnt); atomic_dec(&cscfg_mgr->sys_active_cnt); - cscfg_owner_put(config_desc->load_owner); + cscfg_config_desc_put(config_desc); dev_dbg(cscfg_device(), "Deactivate config %s.\n", config_desc->name); break; } @@ -1047,7 +1060,7 @@ int cscfg_csdev_enable_active_config(struct coresight_device *csdev, unsigned long cfg_hash, int preset) { struct cscfg_config_csdev *config_csdev_active = NULL, *config_csdev_item; - const struct cscfg_config_desc *config_desc; + struct cscfg_config_desc *config_desc; unsigned long flags; int err = 0; @@ -1062,8 +1075,8 @@ int cscfg_csdev_enable_active_config(struct coresight_device *csdev, spin_lock_irqsave(&csdev->cscfg_csdev_lock, flags); list_for_each_entry(config_csdev_item, &csdev->config_csdev_list, node) { config_desc = config_csdev_item->config_desc; - if ((atomic_read(&config_desc->active_cnt)) && - ((unsigned long)config_desc->event_ea->var == cfg_hash)) { + if (((unsigned long)config_desc->event_ea->var == cfg_hash) && + cscfg_config_desc_get(config_desc)) { config_csdev_active = config_csdev_item; csdev->active_cscfg_ctxt = (void *)config_csdev_active; break; @@ -1097,7 +1110,11 @@ int cscfg_csdev_enable_active_config(struct coresight_device *csdev, err = -EBUSY; spin_unlock_irqrestore(&csdev->cscfg_csdev_lock, flags); } + + if (err) + cscfg_config_desc_put(config_desc); } + return err; } EXPORT_SYMBOL_GPL(cscfg_csdev_enable_active_config); @@ -1136,8 +1153,10 @@ void cscfg_csdev_disable_active_config(struct coresight_device *csdev) spin_unlock_irqrestore(&csdev->cscfg_csdev_lock, flags); /* true if there was an enabled active config */ - if (config_csdev) + if (config_csdev) { cscfg_csdev_disable_config(config_csdev); + cscfg_config_desc_put(config_csdev->config_desc); + } } EXPORT_SYMBOL_GPL(cscfg_csdev_disable_active_config); diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index e20c1c6acc731..a564bd5c8b62d 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -21,7 +21,8 @@ #include "coresight-self-hosted-trace.h" #include "coresight-trbe.h" -#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) +#define PERF_IDX2OFF(idx, buf) \ + ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) /* * A padding packet that will help the user space tools @@ -743,12 +744,12 @@ static void *arm_trbe_alloc_buffer(struct coresight_device *csdev, buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event)); if (!buf) - return ERR_PTR(-ENOMEM); + return NULL; pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL); if (!pglist) { kfree(buf); - return ERR_PTR(-ENOMEM); + return NULL; } for (i = 0; i < nr_pages; i++) @@ -758,7 +759,7 @@ static void *arm_trbe_alloc_buffer(struct coresight_device *csdev, if (!buf->trbe_base) { kfree(pglist); kfree(buf); - return ERR_PTR(-ENOMEM); + return NULL; } buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE; buf->trbe_write = buf->trbe_base; diff --git a/drivers/hwtracing/intel_th/Kconfig b/drivers/hwtracing/intel_th/Kconfig index 4b6359326ede9..4f7d2b6d79e29 100644 --- a/drivers/hwtracing/intel_th/Kconfig +++ b/drivers/hwtracing/intel_th/Kconfig @@ -60,6 +60,7 @@ config INTEL_TH_STH config INTEL_TH_MSU tristate "Intel(R) Trace Hub Memory Storage Unit" + depends on MMU help Memory Storage Unit (MSU) trace output device enables storing STP traces to system memory. It supports single diff --git a/drivers/hwtracing/intel_th/msu.c b/drivers/hwtracing/intel_th/msu.c index 9621efe0e95c4..54629458fb710 100644 --- a/drivers/hwtracing/intel_th/msu.c +++ b/drivers/hwtracing/intel_th/msu.c @@ -19,6 +19,7 @@ #include #include #include +#include #ifdef CONFIG_X86 #include @@ -965,7 +966,6 @@ static void msc_buffer_contig_free(struct msc *msc) for (off = 0; off < msc->nr_pages << PAGE_SHIFT; off += PAGE_SIZE) { struct page *page = virt_to_page(msc->base + off); - page->mapping = NULL; __free_page(page); } @@ -1147,9 +1147,6 @@ static void __msc_buffer_win_free(struct msc *msc, struct msc_window *win) int i; for_each_sg(win->sgt->sgl, sg, win->nr_segs, i) { - struct page *page = msc_sg_page(sg); - - page->mapping = NULL; dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE, sg_virt(sg), sg_dma_address(sg)); } @@ -1584,22 +1581,10 @@ static void msc_mmap_close(struct vm_area_struct *vma) { struct msc_iter *iter = vma->vm_file->private_data; struct msc *msc = iter->msc; - unsigned long pg; if (!atomic_dec_and_mutex_lock(&msc->mmap_count, &msc->buf_mutex)) return; - /* drop page _refcounts */ - for (pg = 0; pg < msc->nr_pages; pg++) { - struct page *page = msc_buffer_get_page(msc, pg); - - if (WARN_ON_ONCE(!page)) - continue; - - if (page->mapping) - page->mapping = NULL; - } - /* last mapping -- drop user_count */ atomic_dec(&msc->user_count); mutex_unlock(&msc->buf_mutex); @@ -1609,16 +1594,14 @@ static vm_fault_t msc_mmap_fault(struct vm_fault *vmf) { struct msc_iter *iter = vmf->vma->vm_file->private_data; struct msc *msc = iter->msc; + struct page *page; - vmf->page = msc_buffer_get_page(msc, vmf->pgoff); - if (!vmf->page) + page = msc_buffer_get_page(msc, vmf->pgoff); + if (!page) return VM_FAULT_SIGBUS; - get_page(vmf->page); - vmf->page->mapping = vmf->vma->vm_file->f_mapping; - vmf->page->index = vmf->pgoff; - - return 0; + get_page(page); + return vmf_insert_mixed(vmf->vma, vmf->address, page_to_pfn_t(page)); } static const struct vm_operations_struct msc_mmap_ops = { @@ -1659,7 +1642,7 @@ static int intel_th_msc_mmap(struct file *file, struct vm_area_struct *vma) atomic_dec(&msc->user_count); vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY); + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY | VM_MIXEDMAP); vma->vm_ops = &msc_mmap_ops; return ret; } diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 13aa6a88aed0d..4e7b9464828f2 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -899,6 +899,7 @@ config I2C_OMAP tristate "OMAP I2C adapter" depends on ARCH_OMAP || ARCH_K3 || COMPILE_TEST default MACH_OMAP_OSK + select MULTIPLEXER help If you say yes to this option, support will be included for the I2C interface on the Texas Instruments OMAP1/2 family of processors. diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index ced2fb4aeda8d..79e083aab08e0 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -669,6 +669,7 @@ void i2c_dw_disable(struct dw_i2c_dev *dev) i2c_dw_release_lock(dev); } +EXPORT_SYMBOL_GPL(i2c_dw_disable); MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter core"); MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index 5eb130c1d6719..e93870a0f9a45 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -238,7 +238,6 @@ struct reset_control; * @semaphore_idx: Index of table with semaphore type attached to the bus. It's * -1 if there is no semaphore. * @shared_with_punit: true if this bus is shared with the SoCs PUNIT - * @disable: function to disable the controller * @init: function to initialize the I2C hardware * @set_sda_hold_time: callback to retrieve IP specific SDA hold timing * @mode: operation mode - DW_IC_MASTER or DW_IC_SLAVE @@ -295,7 +294,6 @@ struct dw_i2c_dev { void (*release_lock)(void); int semaphore_idx; bool shared_with_punit; - void (*disable)(struct dw_i2c_dev *dev); int (*init)(struct dw_i2c_dev *dev); int (*set_sda_hold_time)(struct dw_i2c_dev *dev); int mode; @@ -305,6 +303,7 @@ struct dw_i2c_dev { #define ACCESS_INTR_MASK BIT(0) #define ACCESS_NO_IRQ_SUSPEND BIT(1) #define ARBITRATION_SEMAPHORE BIT(2) +#define ACCESS_POLLING BIT(3) #define MODEL_MSCC_OCELOT BIT(8) #define MODEL_BAIKAL_BT1 BIT(9) @@ -339,7 +338,6 @@ int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev); int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev); int i2c_dw_set_fifo_size(struct dw_i2c_dev *dev); u32 i2c_dw_func(struct i2c_adapter *adap); -void i2c_dw_disable(struct dw_i2c_dev *dev); static inline void __i2c_dw_enable(struct dw_i2c_dev *dev) { @@ -354,6 +352,7 @@ static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev) } void __i2c_dw_disable(struct dw_i2c_dev *dev); +void i2c_dw_disable(struct dw_i2c_dev *dev); extern void i2c_dw_configure_master(struct dw_i2c_dev *dev); extern int i2c_dw_probe_master(struct dw_i2c_dev *dev); diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index 579c668cb78a6..e865869ccc50e 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -327,6 +327,7 @@ static int amd_i2c_dw_xfer_quirk(struct i2c_adapter *adap, struct i2c_msg *msgs, dev->msgs = msgs; dev->msgs_num = num_msgs; + dev->msg_write_idx = 0; i2c_dw_xfer_init(dev); regmap_write(dev->map, DW_IC_INTR_MASK, 0); @@ -991,31 +992,6 @@ static int i2c_dw_init_recovery_info(struct dw_i2c_dev *dev) return 0; } -static int i2c_dw_poll_adap_quirk(struct dw_i2c_dev *dev) -{ - struct i2c_adapter *adap = &dev->adapter; - int ret; - - pm_runtime_get_noresume(dev->dev); - ret = i2c_add_numbered_adapter(adap); - if (ret) - dev_err(dev->dev, "Failed to add adapter: %d\n", ret); - pm_runtime_put_noidle(dev->dev); - - return ret; -} - -static bool i2c_dw_is_model_poll(struct dw_i2c_dev *dev) -{ - switch (dev->flags & MODEL_MASK) { - case MODEL_AMD_NAVI_GPU: - case MODEL_WANGXUN_SP: - return true; - default: - return false; - } -} - int i2c_dw_probe_master(struct dw_i2c_dev *dev) { struct i2c_adapter *adap = &dev->adapter; @@ -1026,7 +1002,6 @@ int i2c_dw_probe_master(struct dw_i2c_dev *dev) init_completion(&dev->cmd_complete); dev->init = i2c_dw_init_master; - dev->disable = i2c_dw_disable; ret = i2c_dw_init_regmap(dev); if (ret) @@ -1071,9 +1046,6 @@ int i2c_dw_probe_master(struct dw_i2c_dev *dev) adap->dev.parent = dev->dev; i2c_set_adapdata(adap, dev); - if (i2c_dw_is_model_poll(dev)) - return i2c_dw_poll_adap_quirk(dev); - if (dev->flags & ACCESS_NO_IRQ_SUSPEND) { irq_flags = IRQF_NO_SUSPEND; } else { @@ -1087,12 +1059,14 @@ int i2c_dw_probe_master(struct dw_i2c_dev *dev) regmap_write(dev->map, DW_IC_INTR_MASK, 0); i2c_dw_release_lock(dev); - ret = devm_request_irq(dev->dev, dev->irq, i2c_dw_isr, irq_flags, - dev_name(dev->dev), dev); - if (ret) { - dev_err(dev->dev, "failure requesting irq %i: %d\n", - dev->irq, ret); - return ret; + if (!(dev->flags & ACCESS_POLLING)) { + ret = devm_request_irq(dev->dev, dev->irq, i2c_dw_isr, + irq_flags, dev_name(dev->dev), dev); + if (ret) { + dev_err(dev->dev, "failure requesting irq %i: %d\n", + dev->irq, ret); + return ret; + } } ret = i2c_dw_init_recovery_info(dev); diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c index 61d7a27aa0701..b85f1e4ed13bc 100644 --- a/drivers/i2c/busses/i2c-designware-pcidrv.c +++ b/drivers/i2c/busses/i2c-designware-pcidrv.c @@ -154,7 +154,7 @@ static int navi_amd_setup(struct pci_dev *pdev, struct dw_pci_controller *c) { struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev); - dev->flags |= MODEL_AMD_NAVI_GPU; + dev->flags |= MODEL_AMD_NAVI_GPU | ACCESS_POLLING; dev->timings.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; return 0; } @@ -198,7 +198,7 @@ static int __maybe_unused i2c_dw_pci_runtime_suspend(struct device *dev) { struct dw_i2c_dev *i_dev = dev_get_drvdata(dev); - i_dev->disable(i_dev); + i2c_dw_disable(i_dev); return 0; } @@ -248,6 +248,7 @@ static const struct software_node dgpu_node = { static int i2c_dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct device *device = &pdev->dev; struct dw_i2c_dev *dev; struct i2c_adapter *adap; int r; @@ -256,25 +257,22 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, struct i2c_timings *t; if (id->driver_data >= ARRAY_SIZE(dw_pci_controllers)) - return dev_err_probe(&pdev->dev, -EINVAL, - "Invalid driver data %ld\n", + return dev_err_probe(device, -EINVAL, "Invalid driver data %ld\n", id->driver_data); controller = &dw_pci_controllers[id->driver_data]; r = pcim_enable_device(pdev); if (r) - return dev_err_probe(&pdev->dev, r, - "Failed to enable I2C PCI device\n"); + return dev_err_probe(device, r, "Failed to enable I2C PCI device\n"); pci_set_master(pdev); r = pcim_iomap_regions(pdev, 1 << 0, pci_name(pdev)); if (r) - return dev_err_probe(&pdev->dev, r, - "I/O memory remapping failed\n"); + return dev_err_probe(device, r, "I/O memory remapping failed\n"); - dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); + dev = devm_kzalloc(device, sizeof(*dev), GFP_KERNEL); if (!dev) return -ENOMEM; @@ -284,7 +282,7 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, dev->get_clk_rate_khz = controller->get_clk_rate_khz; dev->base = pcim_iomap_table(pdev)[0]; - dev->dev = &pdev->dev; + dev->dev = device; dev->irq = pci_irq_vector(pdev, 0); dev->flags |= controller->flags; @@ -337,15 +335,17 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, if ((dev->flags & MODEL_MASK) == MODEL_AMD_NAVI_GPU) { dev->slave = i2c_new_ccgx_ucsi(&dev->adapter, dev->irq, &dgpu_node); - if (IS_ERR(dev->slave)) - return dev_err_probe(dev->dev, PTR_ERR(dev->slave), + if (IS_ERR(dev->slave)) { + i2c_del_adapter(&dev->adapter); + return dev_err_probe(device, PTR_ERR(dev->slave), "register UCSI failed\n"); + } } - pm_runtime_set_autosuspend_delay(&pdev->dev, 1000); - pm_runtime_use_autosuspend(&pdev->dev); - pm_runtime_put_autosuspend(&pdev->dev); - pm_runtime_allow(&pdev->dev); + pm_runtime_set_autosuspend_delay(device, 1000); + pm_runtime_use_autosuspend(device); + pm_runtime_put_autosuspend(device); + pm_runtime_allow(device); return 0; } @@ -353,10 +353,12 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, static void i2c_dw_pci_remove(struct pci_dev *pdev) { struct dw_i2c_dev *dev = pci_get_drvdata(pdev); + struct device *device = &pdev->dev; + + i2c_dw_disable(dev); - dev->disable(dev); - pm_runtime_forbid(&pdev->dev); - pm_runtime_get_noresume(&pdev->dev); + pm_runtime_forbid(device); + pm_runtime_get_noresume(device); i2c_del_adapter(&dev->adapter); devm_free_irq(&pdev->dev, dev->irq, dev); diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 855b698e99c08..818650bbac7a8 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -275,6 +275,7 @@ static void i2c_dw_remove_lock_support(struct dw_i2c_dev *dev) static int dw_i2c_plat_probe(struct platform_device *pdev) { + struct device *device = &pdev->dev; struct i2c_adapter *adap; struct dw_i2c_dev *dev; struct i2c_timings *t; @@ -284,15 +285,15 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) if (irq < 0) return irq; - dev = devm_kzalloc(&pdev->dev, sizeof(struct dw_i2c_dev), GFP_KERNEL); + dev = devm_kzalloc(device, sizeof(*dev), GFP_KERNEL); if (!dev) return -ENOMEM; - dev->flags = (uintptr_t)device_get_match_data(&pdev->dev); - if (device_property_present(&pdev->dev, "wx,i2c-snps-model")) - dev->flags = MODEL_WANGXUN_SP; + dev->flags = (uintptr_t)device_get_match_data(device); + if (device_property_present(device, "wx,i2c-snps-model")) + dev->flags = MODEL_WANGXUN_SP | ACCESS_POLLING; - dev->dev = &pdev->dev; + dev->dev = device; dev->irq = irq; platform_set_drvdata(pdev, dev); @@ -300,7 +301,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) if (ret) return ret; - dev->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); + dev->rst = devm_reset_control_get_optional_exclusive(device, NULL); if (IS_ERR(dev->rst)) return PTR_ERR(dev->rst); @@ -328,13 +329,13 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) i2c_dw_configure(dev); /* Optional interface clock */ - dev->pclk = devm_clk_get_optional(&pdev->dev, "pclk"); + dev->pclk = devm_clk_get_optional(device, "pclk"); if (IS_ERR(dev->pclk)) { ret = PTR_ERR(dev->pclk); goto exit_reset; } - dev->clk = devm_clk_get_optional(&pdev->dev, NULL); + dev->clk = devm_clk_get_optional(device, NULL); if (IS_ERR(dev->clk)) { ret = PTR_ERR(dev->clk); goto exit_reset; @@ -363,28 +364,24 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) adap->dev.of_node = pdev->dev.of_node; adap->nr = -1; - if (dev->flags & ACCESS_NO_IRQ_SUSPEND) { - dev_pm_set_driver_flags(&pdev->dev, - DPM_FLAG_SMART_PREPARE); - } else { - dev_pm_set_driver_flags(&pdev->dev, - DPM_FLAG_SMART_PREPARE | - DPM_FLAG_SMART_SUSPEND); - } + if (dev->flags & ACCESS_NO_IRQ_SUSPEND) + dev_pm_set_driver_flags(device, DPM_FLAG_SMART_PREPARE); + else + dev_pm_set_driver_flags(device, DPM_FLAG_SMART_PREPARE | DPM_FLAG_SMART_SUSPEND); - device_enable_async_suspend(&pdev->dev); + device_enable_async_suspend(device); /* The code below assumes runtime PM to be disabled. */ - WARN_ON(pm_runtime_enabled(&pdev->dev)); + WARN_ON(pm_runtime_enabled(device)); - pm_runtime_set_autosuspend_delay(&pdev->dev, 1000); - pm_runtime_use_autosuspend(&pdev->dev); - pm_runtime_set_active(&pdev->dev); + pm_runtime_set_autosuspend_delay(device, 1000); + pm_runtime_use_autosuspend(device); + pm_runtime_set_active(device); if (dev->shared_with_punit) - pm_runtime_get_noresume(&pdev->dev); + pm_runtime_get_noresume(device); - pm_runtime_enable(&pdev->dev); + pm_runtime_enable(device); ret = i2c_dw_probe(dev); if (ret) @@ -394,6 +391,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) exit_probe: dw_i2c_plat_pm_cleanup(dev); + i2c_dw_prepare_clk(dev, false); exit_reset: reset_control_assert(dev->rst); return ret; @@ -402,17 +400,20 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) static void dw_i2c_plat_remove(struct platform_device *pdev) { struct dw_i2c_dev *dev = platform_get_drvdata(pdev); + struct device *device = &pdev->dev; - pm_runtime_get_sync(&pdev->dev); + pm_runtime_get_sync(device); i2c_del_adapter(&dev->adapter); - dev->disable(dev); + i2c_dw_disable(dev); - pm_runtime_dont_use_autosuspend(&pdev->dev); - pm_runtime_put_sync(&pdev->dev); + pm_runtime_dont_use_autosuspend(device); + pm_runtime_put_noidle(device); dw_i2c_plat_pm_cleanup(dev); + i2c_dw_prepare_clk(dev, false); + i2c_dw_remove_lock_support(dev); reset_control_assert(dev->rst); @@ -436,7 +437,7 @@ static int dw_i2c_plat_runtime_suspend(struct device *dev) if (i_dev->shared_with_punit) return 0; - i_dev->disable(i_dev); + i2c_dw_disable(i_dev); i2c_dw_prepare_clk(i_dev, false); return 0; diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c index 78e2c47e3d7da..ea4c4955fe264 100644 --- a/drivers/i2c/busses/i2c-designware-slave.c +++ b/drivers/i2c/busses/i2c-designware-slave.c @@ -88,10 +88,10 @@ static int i2c_dw_unreg_slave(struct i2c_client *slave) struct dw_i2c_dev *dev = i2c_get_adapdata(slave->adapter); regmap_write(dev->map, DW_IC_INTR_MASK, 0); - dev->disable(dev); + i2c_dw_disable(dev); synchronize_irq(dev->irq); dev->slave = NULL; - pm_runtime_put(dev->dev); + pm_runtime_put_sync_suspend(dev->dev); return 0; } @@ -235,7 +235,6 @@ int i2c_dw_probe_slave(struct dw_i2c_dev *dev) int ret; dev->init = i2c_dw_init_slave; - dev->disable = i2c_dw_disable; ret = i2c_dw_init_regmap(dev); if (ret) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 18c04f5e41d9c..89fdc75cdcfa5 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -1051,7 +1051,7 @@ static const struct pci_device_id i801_ids[] = { { PCI_DEVICE_DATA(INTEL, METEOR_LAKE_P_SMBUS, FEATURES_ICH5 | FEATURE_TCO_CNL) }, { PCI_DEVICE_DATA(INTEL, METEOR_LAKE_SOC_S_SMBUS, FEATURES_ICH5 | FEATURE_TCO_CNL) }, { PCI_DEVICE_DATA(INTEL, METEOR_LAKE_PCH_S_SMBUS, FEATURES_ICH5 | FEATURE_TCO_CNL) }, - { PCI_DEVICE_DATA(INTEL, BIRCH_STREAM_SMBUS, FEATURES_ICH5 | FEATURE_TCO_CNL) }, + { PCI_DEVICE_DATA(INTEL, BIRCH_STREAM_SMBUS, FEATURES_ICH5) }, { PCI_DEVICE_DATA(INTEL, ARROW_LAKE_H_SMBUS, FEATURES_ICH5 | FEATURE_TCO_CNL) }, { PCI_DEVICE_DATA(INTEL, PANTHER_LAKE_H_SMBUS, FEATURES_ICH5 | FEATURE_TCO_CNL) }, { PCI_DEVICE_DATA(INTEL, PANTHER_LAKE_P_SMBUS, FEATURES_ICH5 | FEATURE_TCO_CNL) }, diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index 5d4f04a3c6d32..b44b36bd565ea 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -616,9 +616,9 @@ static int lpi2c_imx_probe(struct platform_device *pdev) return 0; rpm_disable: - pm_runtime_put(&pdev->dev); - pm_runtime_disable(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); return ret; } diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c index 1a9b5a068ef1b..6b788bca64d54 100644 --- a/drivers/i2c/busses/i2c-mt65xx.c +++ b/drivers/i2c/busses/i2c-mt65xx.c @@ -1243,6 +1243,7 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap, { int ret; int left_num = num; + bool write_then_read_en = false; struct mtk_i2c *i2c = i2c_get_adapdata(adap); ret = clk_bulk_enable(I2C_MT65XX_CLK_MAX, i2c->clocks); @@ -1256,6 +1257,7 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap, if (!(msgs[0].flags & I2C_M_RD) && (msgs[1].flags & I2C_M_RD) && msgs[0].addr == msgs[1].addr) { i2c->auto_restart = 0; + write_then_read_en = true; } } @@ -1280,12 +1282,10 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap, else i2c->op = I2C_MASTER_WR; - if (!i2c->auto_restart) { - if (num > 1) { - /* combined two messages into one transaction */ - i2c->op = I2C_MASTER_WRRD; - left_num--; - } + if (write_then_read_en) { + /* combined two messages into one transaction */ + i2c->op = I2C_MASTER_WRRD; + left_num--; } /* always use DMA mode. */ @@ -1293,7 +1293,10 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap, if (ret < 0) goto err_exit; - msgs++; + if (i2c->op == I2C_MASTER_WRRD) + msgs += 2; + else + msgs++; } /* the return value is number of executed messages */ ret = num; diff --git a/drivers/i2c/busses/i2c-npcm7xx.c b/drivers/i2c/busses/i2c-npcm7xx.c index 91f508d50e7ab..5b3987460976e 100644 --- a/drivers/i2c/busses/i2c-npcm7xx.c +++ b/drivers/i2c/busses/i2c-npcm7xx.c @@ -1971,10 +1971,14 @@ static int npcm_i2c_init_module(struct npcm_i2c *bus, enum i2c_mode mode, /* Check HW is OK: SDA and SCL should be high at this point. */ if ((npcm_i2c_get_SDA(&bus->adap) == 0) || (npcm_i2c_get_SCL(&bus->adap) == 0)) { - dev_err(bus->dev, "I2C%d init fail: lines are low\n", bus->num); - dev_err(bus->dev, "SDA=%d SCL=%d\n", npcm_i2c_get_SDA(&bus->adap), - npcm_i2c_get_SCL(&bus->adap)); - return -ENXIO; + dev_warn(bus->dev, " I2C%d SDA=%d SCL=%d, attempting to recover\n", bus->num, + npcm_i2c_get_SDA(&bus->adap), npcm_i2c_get_SCL(&bus->adap)); + if (npcm_i2c_recovery_tgclk(&bus->adap)) { + dev_err(bus->dev, "I2C%d init fail: SDA=%d SCL=%d\n", + bus->num, npcm_i2c_get_SDA(&bus->adap), + npcm_i2c_get_SCL(&bus->adap)); + return -ENXIO; + } } npcm_i2c_int_enable(bus, true); diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c index 22975bfd6b252..fad88ab32716f 100644 --- a/drivers/i2c/busses/i2c-omap.c +++ b/drivers/i2c/busses/i2c-omap.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -211,6 +212,7 @@ struct omap_i2c_dev { u16 syscstate; u16 westate; u16 errata; + struct mux_state *mux_state; }; static const u8 reg_map_ip_v1[] = { @@ -1455,8 +1457,27 @@ omap_i2c_probe(struct platform_device *pdev) (1000 * omap->speed / 8); } + if (of_property_present(node, "mux-states")) { + struct mux_state *mux_state; + + mux_state = devm_mux_state_get(&pdev->dev, NULL); + if (IS_ERR(mux_state)) { + r = PTR_ERR(mux_state); + dev_dbg(&pdev->dev, "failed to get I2C mux: %d\n", r); + goto err_put_pm; + } + omap->mux_state = mux_state; + r = mux_state_select(omap->mux_state); + if (r) { + dev_err(&pdev->dev, "failed to select I2C mux: %d\n", r); + goto err_put_pm; + } + } + /* reset ASAP, clearing any IRQs */ - omap_i2c_init(omap); + r = omap_i2c_init(omap); + if (r) + goto err_mux_state_deselect; if (omap->rev < OMAP_I2C_OMAP1_REV_2) r = devm_request_irq(&pdev->dev, omap->irq, omap_i2c_omap1_isr, @@ -1499,6 +1520,10 @@ omap_i2c_probe(struct platform_device *pdev) err_unuse_clocks: omap_i2c_write_reg(omap, OMAP_I2C_CON_REG, 0); +err_mux_state_deselect: + if (omap->mux_state) + mux_state_deselect(omap->mux_state); +err_put_pm: pm_runtime_dont_use_autosuspend(omap->dev); pm_runtime_put_sync(omap->dev); err_disable_pm: @@ -1514,6 +1539,9 @@ static void omap_i2c_remove(struct platform_device *pdev) i2c_del_adapter(&omap->adapter); + if (omap->mux_state) + mux_state_deselect(omap->mux_state); + ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) dev_err(omap->dev, "Failed to resume hardware, skip disable\n"); diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c index 3bd406470940f..affdd94f06aaf 100644 --- a/drivers/i2c/busses/i2c-pxa.c +++ b/drivers/i2c/busses/i2c-pxa.c @@ -1504,7 +1504,10 @@ static int i2c_pxa_probe(struct platform_device *dev) i2c->adap.name); } - clk_prepare_enable(i2c->clk); + ret = clk_prepare_enable(i2c->clk); + if (ret) + return dev_err_probe(&dev->dev, ret, + "failed to enable clock\n"); if (i2c->use_pio) { i2c->adap.algo = &i2c_pxa_pio_algorithm; diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c index 598102d16677a..f324f397356c2 100644 --- a/drivers/i2c/busses/i2c-qup.c +++ b/drivers/i2c/busses/i2c-qup.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -150,6 +151,8 @@ /* TAG length for DATA READ in RX FIFO */ #define READ_RX_TAGS_LEN 2 +#define QUP_BUS_WIDTH 8 + static unsigned int scl_freq; module_param_named(scl_freq, scl_freq, uint, 0444); MODULE_PARM_DESC(scl_freq, "SCL frequency override"); @@ -227,6 +230,7 @@ struct qup_i2c_dev { int irq; struct clk *clk; struct clk *pclk; + struct icc_path *icc_path; struct i2c_adapter adap; int clk_ctl; @@ -255,6 +259,10 @@ struct qup_i2c_dev { /* To configure when bus is in run state */ u32 config_run; + /* bandwidth votes */ + u32 src_clk_freq; + u32 cur_bw_clk_freq; + /* dma parameters */ bool is_dma; /* To check if the current transfer is using DMA */ @@ -444,8 +452,10 @@ static int qup_i2c_bus_active(struct qup_i2c_dev *qup, int len) if (!(status & I2C_STATUS_BUS_ACTIVE)) break; - if (time_after(jiffies, timeout)) + if (time_after(jiffies, timeout)) { ret = -ETIMEDOUT; + break; + } usleep_range(len, len * 2); } @@ -453,6 +463,23 @@ static int qup_i2c_bus_active(struct qup_i2c_dev *qup, int len) return ret; } +static int qup_i2c_vote_bw(struct qup_i2c_dev *qup, u32 clk_freq) +{ + u32 needed_peak_bw; + int ret; + + if (qup->cur_bw_clk_freq == clk_freq) + return 0; + + needed_peak_bw = Bps_to_icc(clk_freq * QUP_BUS_WIDTH); + ret = icc_set_bw(qup->icc_path, 0, needed_peak_bw); + if (ret) + return ret; + + qup->cur_bw_clk_freq = clk_freq; + return 0; +} + static void qup_i2c_write_tx_fifo_v1(struct qup_i2c_dev *qup) { struct qup_i2c_block *blk = &qup->blk; @@ -840,6 +867,10 @@ static int qup_i2c_bam_xfer(struct i2c_adapter *adap, struct i2c_msg *msg, int ret = 0; int idx = 0; + ret = qup_i2c_vote_bw(qup, qup->src_clk_freq); + if (ret) + return ret; + enable_irq(qup->irq); ret = qup_i2c_req_dma(qup); @@ -1645,6 +1676,7 @@ static void qup_i2c_disable_clocks(struct qup_i2c_dev *qup) config = readl(qup->base + QUP_CONFIG); config |= QUP_CLOCK_AUTO_GATE; writel(config, qup->base + QUP_CONFIG); + qup_i2c_vote_bw(qup, 0); clk_disable_unprepare(qup->pclk); } @@ -1745,6 +1777,11 @@ static int qup_i2c_probe(struct platform_device *pdev) goto fail_dma; } qup->is_dma = true; + + qup->icc_path = devm_of_icc_get(&pdev->dev, NULL); + if (IS_ERR(qup->icc_path)) + return dev_err_probe(&pdev->dev, PTR_ERR(qup->icc_path), + "failed to get interconnect path\n"); } nodma: @@ -1793,6 +1830,7 @@ static int qup_i2c_probe(struct platform_device *pdev) qup_i2c_enable_clocks(qup); src_clk_freq = clk_get_rate(qup->clk); } + qup->src_clk_freq = src_clk_freq; /* * Bootloaders might leave a pending interrupt on certain QUP's, diff --git a/drivers/i2c/busses/i2c-robotfuzz-osif.c b/drivers/i2c/busses/i2c-robotfuzz-osif.c index 66dfa211e736b..8e4cf9028b234 100644 --- a/drivers/i2c/busses/i2c-robotfuzz-osif.c +++ b/drivers/i2c/busses/i2c-robotfuzz-osif.c @@ -111,6 +111,11 @@ static u32 osif_func(struct i2c_adapter *adapter) return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; } +/* prevent invalid 0-length usb_control_msg */ +static const struct i2c_adapter_quirks osif_quirks = { + .flags = I2C_AQ_NO_ZERO_LEN_READ, +}; + static const struct i2c_algorithm osif_algorithm = { .master_xfer = osif_xfer, .functionality = osif_func, @@ -143,6 +148,7 @@ static int osif_probe(struct usb_interface *interface, priv->adapter.owner = THIS_MODULE; priv->adapter.class = I2C_CLASS_HWMON; + priv->adapter.quirks = &osif_quirks; priv->adapter.algo = &osif_algorithm; priv->adapter.algo_data = priv; snprintf(priv->adapter.name, sizeof(priv->adapter.name), diff --git a/drivers/i2c/busses/i2c-stm32.c b/drivers/i2c/busses/i2c-stm32.c index 157c64e27d0bd..f84ec056e36df 100644 --- a/drivers/i2c/busses/i2c-stm32.c +++ b/drivers/i2c/busses/i2c-stm32.c @@ -102,7 +102,6 @@ int stm32_i2c_prep_dma_xfer(struct device *dev, struct stm32_i2c_dma *dma, void *dma_async_param) { struct dma_async_tx_descriptor *txdesc; - struct device *chan_dev; int ret; if (rd_wr) { @@ -116,11 +115,10 @@ int stm32_i2c_prep_dma_xfer(struct device *dev, struct stm32_i2c_dma *dma, } dma->dma_len = len; - chan_dev = dma->chan_using->device->dev; - dma->dma_buf = dma_map_single(chan_dev, buf, dma->dma_len, + dma->dma_buf = dma_map_single(dev, buf, dma->dma_len, dma->dma_data_dir); - if (dma_mapping_error(chan_dev, dma->dma_buf)) { + if (dma_mapping_error(dev, dma->dma_buf)) { dev_err(dev, "DMA mapping failed\n"); return -EINVAL; } @@ -150,7 +148,7 @@ int stm32_i2c_prep_dma_xfer(struct device *dev, struct stm32_i2c_dma *dma, return 0; err: - dma_unmap_single(chan_dev, dma->dma_buf, dma->dma_len, + dma_unmap_single(dev, dma->dma_buf, dma->dma_len, dma->dma_data_dir); return ret; } diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index b4f10ff31102b..956803ba6c1e1 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -726,12 +726,13 @@ static void stm32f7_i2c_disable_dma_req(struct stm32f7_i2c_dev *i2c_dev) static void stm32f7_i2c_dma_callback(void *arg) { - struct stm32f7_i2c_dev *i2c_dev = (struct stm32f7_i2c_dev *)arg; + struct stm32f7_i2c_dev *i2c_dev = arg; struct stm32_i2c_dma *dma = i2c_dev->dma; - struct device *dev = dma->chan_using->device->dev; stm32f7_i2c_disable_dma_req(i2c_dev); - dma_unmap_single(dev, dma->dma_buf, dma->dma_len, dma->dma_data_dir); + dmaengine_terminate_async(dma->chan_using); + dma_unmap_single(i2c_dev->dev, dma->dma_buf, dma->dma_len, + dma->dma_data_dir); complete(&dma->dma_complete); } @@ -1496,17 +1497,11 @@ static irqreturn_t stm32f7_i2c_slave_isr_event(struct stm32f7_i2c_dev *i2c_dev) static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data) { struct stm32f7_i2c_dev *i2c_dev = data; - struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg; - struct stm32_i2c_dma *dma = i2c_dev->dma; - void __iomem *base = i2c_dev->base; - u32 status, mask; - int ret = IRQ_HANDLED; + u32 status; - /* Check if the interrupt if for a slave device */ - if (!i2c_dev->master_mode) { - ret = stm32f7_i2c_slave_isr_event(i2c_dev); - return ret; - } + /* Check if the interrupt is for a slave device */ + if (!i2c_dev->master_mode) + return IRQ_WAKE_THREAD; status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR); @@ -1518,45 +1513,59 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data) if (status & STM32F7_I2C_ISR_RXNE) stm32f7_i2c_read_rx_data(i2c_dev); + /* Wake up the thread if other flags are raised */ + if (status & + (STM32F7_I2C_ISR_NACKF | STM32F7_I2C_ISR_STOPF | + STM32F7_I2C_ISR_TC | STM32F7_I2C_ISR_TCR)) + return IRQ_WAKE_THREAD; + + return IRQ_HANDLED; +} + +static irqreturn_t stm32f7_i2c_isr_event_thread(int irq, void *data) +{ + struct stm32f7_i2c_dev *i2c_dev = data; + struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg; + void __iomem *base = i2c_dev->base; + u32 status, mask; + int ret; + + if (!i2c_dev->master_mode) + return stm32f7_i2c_slave_isr_event(i2c_dev); + + status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR); + /* NACK received */ if (status & STM32F7_I2C_ISR_NACKF) { dev_dbg(i2c_dev->dev, "<%s>: Receive NACK (addr %x)\n", __func__, f7_msg->addr); writel_relaxed(STM32F7_I2C_ICR_NACKCF, base + STM32F7_I2C_ICR); - if (i2c_dev->use_dma) { - stm32f7_i2c_disable_dma_req(i2c_dev); - dmaengine_terminate_async(dma->chan_using); - } + if (i2c_dev->use_dma) + stm32f7_i2c_dma_callback(i2c_dev); f7_msg->result = -ENXIO; } - /* STOP detection flag */ - if (status & STM32F7_I2C_ISR_STOPF) { - /* Disable interrupts */ - if (stm32f7_i2c_is_slave_registered(i2c_dev)) - mask = STM32F7_I2C_XFER_IRQ_MASK; + if (status & STM32F7_I2C_ISR_TCR) { + if (f7_msg->smbus) + stm32f7_i2c_smbus_reload(i2c_dev); else - mask = STM32F7_I2C_ALL_IRQ_MASK; - stm32f7_i2c_disable_irq(i2c_dev, mask); - - /* Clear STOP flag */ - writel_relaxed(STM32F7_I2C_ICR_STOPCF, base + STM32F7_I2C_ICR); - - if (i2c_dev->use_dma && !f7_msg->result) { - ret = IRQ_WAKE_THREAD; - } else { - i2c_dev->master_mode = false; - complete(&i2c_dev->complete); - } + stm32f7_i2c_reload(i2c_dev); } /* Transfer complete */ if (status & STM32F7_I2C_ISR_TC) { + /* Wait for dma transfer completion before sending next message */ + if (i2c_dev->use_dma && !f7_msg->result) { + ret = wait_for_completion_timeout(&i2c_dev->dma->dma_complete, HZ); + if (!ret) { + dev_dbg(i2c_dev->dev, "<%s>: Timed out\n", __func__); + stm32f7_i2c_dma_callback(i2c_dev); + f7_msg->result = -ETIMEDOUT; + } + } if (f7_msg->stop) { mask = STM32F7_I2C_CR2_STOP; stm32f7_i2c_set_bits(base + STM32F7_I2C_CR2, mask); - } else if (i2c_dev->use_dma && !f7_msg->result) { - ret = IRQ_WAKE_THREAD; } else if (f7_msg->smbus) { stm32f7_i2c_smbus_rep_start(i2c_dev); } else { @@ -1566,47 +1575,18 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data) } } - if (status & STM32F7_I2C_ISR_TCR) { - if (f7_msg->smbus) - stm32f7_i2c_smbus_reload(i2c_dev); + /* STOP detection flag */ + if (status & STM32F7_I2C_ISR_STOPF) { + /* Disable interrupts */ + if (stm32f7_i2c_is_slave_registered(i2c_dev)) + mask = STM32F7_I2C_XFER_IRQ_MASK; else - stm32f7_i2c_reload(i2c_dev); - } - - return ret; -} - -static irqreturn_t stm32f7_i2c_isr_event_thread(int irq, void *data) -{ - struct stm32f7_i2c_dev *i2c_dev = data; - struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg; - struct stm32_i2c_dma *dma = i2c_dev->dma; - u32 status; - int ret; - - /* - * Wait for dma transfer completion before sending next message or - * notity the end of xfer to the client - */ - ret = wait_for_completion_timeout(&i2c_dev->dma->dma_complete, HZ); - if (!ret) { - dev_dbg(i2c_dev->dev, "<%s>: Timed out\n", __func__); - stm32f7_i2c_disable_dma_req(i2c_dev); - dmaengine_terminate_async(dma->chan_using); - f7_msg->result = -ETIMEDOUT; - } + mask = STM32F7_I2C_ALL_IRQ_MASK; + stm32f7_i2c_disable_irq(i2c_dev, mask); - status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR); + /* Clear STOP flag */ + writel_relaxed(STM32F7_I2C_ICR_STOPCF, base + STM32F7_I2C_ICR); - if (status & STM32F7_I2C_ISR_TC) { - if (f7_msg->smbus) { - stm32f7_i2c_smbus_rep_start(i2c_dev); - } else { - i2c_dev->msg_id++; - i2c_dev->msg++; - stm32f7_i2c_xfer_msg(i2c_dev, i2c_dev->msg); - } - } else { i2c_dev->master_mode = false; complete(&i2c_dev->complete); } @@ -1614,21 +1594,20 @@ static irqreturn_t stm32f7_i2c_isr_event_thread(int irq, void *data) return IRQ_HANDLED; } -static irqreturn_t stm32f7_i2c_isr_error(int irq, void *data) +static irqreturn_t stm32f7_i2c_isr_error_thread(int irq, void *data) { struct stm32f7_i2c_dev *i2c_dev = data; struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg; + u16 addr = f7_msg->addr; void __iomem *base = i2c_dev->base; struct device *dev = i2c_dev->dev; - struct stm32_i2c_dma *dma = i2c_dev->dma; u32 status; status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR); /* Bus error */ if (status & STM32F7_I2C_ISR_BERR) { - dev_err(dev, "<%s>: Bus error accessing addr 0x%x\n", - __func__, f7_msg->addr); + dev_err(dev, "Bus error accessing addr 0x%x\n", addr); writel_relaxed(STM32F7_I2C_ICR_BERRCF, base + STM32F7_I2C_ICR); stm32f7_i2c_release_bus(&i2c_dev->adap); f7_msg->result = -EIO; @@ -1636,21 +1615,19 @@ static irqreturn_t stm32f7_i2c_isr_error(int irq, void *data) /* Arbitration loss */ if (status & STM32F7_I2C_ISR_ARLO) { - dev_dbg(dev, "<%s>: Arbitration loss accessing addr 0x%x\n", - __func__, f7_msg->addr); + dev_dbg(dev, "Arbitration loss accessing addr 0x%x\n", addr); writel_relaxed(STM32F7_I2C_ICR_ARLOCF, base + STM32F7_I2C_ICR); f7_msg->result = -EAGAIN; } if (status & STM32F7_I2C_ISR_PECERR) { - dev_err(dev, "<%s>: PEC error in reception accessing addr 0x%x\n", - __func__, f7_msg->addr); + dev_err(dev, "PEC error in reception accessing addr 0x%x\n", addr); writel_relaxed(STM32F7_I2C_ICR_PECCF, base + STM32F7_I2C_ICR); f7_msg->result = -EINVAL; } if (status & STM32F7_I2C_ISR_ALERT) { - dev_dbg(dev, "<%s>: SMBus alert received\n", __func__); + dev_dbg(dev, "SMBus alert received\n"); writel_relaxed(STM32F7_I2C_ICR_ALERTCF, base + STM32F7_I2C_ICR); i2c_handle_smbus_alert(i2c_dev->alert->ara); return IRQ_HANDLED; @@ -1667,10 +1644,8 @@ static irqreturn_t stm32f7_i2c_isr_error(int irq, void *data) } /* Disable dma */ - if (i2c_dev->use_dma) { - stm32f7_i2c_disable_dma_req(i2c_dev); - dmaengine_terminate_async(dma->chan_using); - } + if (i2c_dev->use_dma) + stm32f7_i2c_dma_callback(i2c_dev); i2c_dev->master_mode = false; complete(&i2c_dev->complete); @@ -2177,23 +2152,16 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) i2c_dev->wakeup_src = of_property_read_bool(pdev->dev.of_node, "wakeup-source"); - i2c_dev->clk = devm_clk_get(&pdev->dev, NULL); + i2c_dev->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(i2c_dev->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(i2c_dev->clk), - "Failed to get controller clock\n"); - - ret = clk_prepare_enable(i2c_dev->clk); - if (ret) { - dev_err(&pdev->dev, "Failed to prepare_enable clock\n"); - return ret; - } + "Failed to enable controller clock\n"); rst = devm_reset_control_get(&pdev->dev, NULL); - if (IS_ERR(rst)) { - ret = dev_err_probe(&pdev->dev, PTR_ERR(rst), - "Error: Missing reset ctrl\n"); - goto clk_free; - } + if (IS_ERR(rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(rst), + "Error: Missing reset ctrl\n"); + reset_control_assert(rst); udelay(2); reset_control_deassert(rst); @@ -2205,40 +2173,36 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) stm32f7_i2c_isr_event_thread, IRQF_ONESHOT, pdev->name, i2c_dev); - if (ret) { - dev_err(&pdev->dev, "Failed to request irq event %i\n", - irq_event); - goto clk_free; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to request irq event\n"); - ret = devm_request_irq(&pdev->dev, irq_error, stm32f7_i2c_isr_error, 0, - pdev->name, i2c_dev); - if (ret) { - dev_err(&pdev->dev, "Failed to request irq error %i\n", - irq_error); - goto clk_free; - } + ret = devm_request_threaded_irq(&pdev->dev, irq_error, + NULL, + stm32f7_i2c_isr_error_thread, + IRQF_ONESHOT, + pdev->name, i2c_dev); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to request irq error\n"); setup = of_device_get_match_data(&pdev->dev); if (!setup) { dev_err(&pdev->dev, "Can't get device data\n"); - ret = -ENODEV; - goto clk_free; + return -ENODEV; } i2c_dev->setup = *setup; ret = stm32f7_i2c_setup_timing(i2c_dev, &i2c_dev->setup); if (ret) - goto clk_free; + return ret; /* Setup Fast mode plus if necessary */ if (i2c_dev->bus_rate > I2C_MAX_FAST_MODE_FREQ) { ret = stm32f7_i2c_setup_fm_plus_bits(pdev, i2c_dev); if (ret) - goto clk_free; + return ret; ret = stm32f7_i2c_write_fm_plus_bits(i2c_dev, true); if (ret) - goto clk_free; + return ret; } adap = &i2c_dev->adap; @@ -2349,9 +2313,6 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) fmp_clear: stm32f7_i2c_write_fm_plus_bits(i2c_dev, false); -clk_free: - clk_disable_unprepare(i2c_dev->clk); - return ret; } @@ -2385,8 +2346,6 @@ static void stm32f7_i2c_remove(struct platform_device *pdev) } stm32f7_i2c_write_fm_plus_bits(i2c_dev, false); - - clk_disable_unprepare(i2c_dev->clk); } static int __maybe_unused stm32f7_i2c_runtime_suspend(struct device *dev) diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 91be04b534fe6..5766231b13cd1 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -607,7 +607,6 @@ static int tegra_i2c_wait_for_config_load(struct tegra_i2c_dev *i2c_dev) static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev) { u32 val, clk_divisor, clk_multiplier, tsu_thd, tlow, thigh, non_hs_mode; - acpi_handle handle = ACPI_HANDLE(i2c_dev->dev); struct i2c_timings *t = &i2c_dev->timings; int err; @@ -619,11 +618,7 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev) * emit a noisy warning on error, which won't stay unnoticed and * won't hose machine entirely. */ - if (handle) - err = acpi_evaluate_object(handle, "_RST", NULL, NULL); - else - err = reset_control_reset(i2c_dev->rst); - + err = device_reset(i2c_dev->dev); WARN_ON_ONCE(err); if (IS_DVC(i2c_dev)) @@ -1397,6 +1392,11 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], ret = tegra_i2c_xfer_msg(i2c_dev, &msgs[i], MSG_END_CONTINUE); if (ret) break; + + /* Validate message length before proceeding */ + if (msgs[i].buf[0] == 0 || msgs[i].buf[0] > I2C_SMBUS_BLOCK_MAX) + break; + /* Set the msg length from first byte */ msgs[i].len += msgs[i].buf[0]; dev_dbg(i2c_dev->dev, "reading %d bytes\n", msgs[i].len); @@ -1663,19 +1663,6 @@ static void tegra_i2c_parse_dt(struct tegra_i2c_dev *i2c_dev) i2c_dev->is_vi = true; } -static int tegra_i2c_init_reset(struct tegra_i2c_dev *i2c_dev) -{ - if (ACPI_HANDLE(i2c_dev->dev)) - return 0; - - i2c_dev->rst = devm_reset_control_get_exclusive(i2c_dev->dev, "i2c"); - if (IS_ERR(i2c_dev->rst)) - return dev_err_probe(i2c_dev->dev, PTR_ERR(i2c_dev->rst), - "failed to get reset control\n"); - - return 0; -} - static int tegra_i2c_init_clocks(struct tegra_i2c_dev *i2c_dev) { int err; @@ -1785,10 +1772,6 @@ static int tegra_i2c_probe(struct platform_device *pdev) tegra_i2c_parse_dt(i2c_dev); - err = tegra_i2c_init_reset(i2c_dev); - if (err) - return err; - err = tegra_i2c_init_clocks(i2c_dev); if (err) return err; diff --git a/drivers/i2c/busses/i2c-tiny-usb.c b/drivers/i2c/busses/i2c-tiny-usb.c index 1bffe36c40ad8..d984b90b35270 100644 --- a/drivers/i2c/busses/i2c-tiny-usb.c +++ b/drivers/i2c/busses/i2c-tiny-usb.c @@ -140,6 +140,11 @@ static u32 usb_func(struct i2c_adapter *adapter) return ret; } +/* prevent invalid 0-length usb_control_msg */ +static const struct i2c_adapter_quirks usb_quirks = { + .flags = I2C_AQ_NO_ZERO_LEN_READ, +}; + /* This is the actual algorithm we define */ static const struct i2c_algorithm usb_algorithm = { .master_xfer = usb_xfer, @@ -248,6 +253,7 @@ static int i2c_tiny_usb_probe(struct usb_interface *interface, /* setup i2c adapter description */ dev->adapter.owner = THIS_MODULE; dev->adapter.class = I2C_CLASS_HWMON; + dev->adapter.quirks = &usb_quirks; dev->adapter.algo = &usb_algorithm; dev->adapter.algo_data = dev; snprintf(dev->adapter.name, sizeof(dev->adapter.name), diff --git a/drivers/i2c/busses/i2c-virtio.c b/drivers/i2c/busses/i2c-virtio.c index c60ae531ba57c..726c162cabd86 100644 --- a/drivers/i2c/busses/i2c-virtio.c +++ b/drivers/i2c/busses/i2c-virtio.c @@ -116,15 +116,16 @@ static int virtio_i2c_complete_reqs(struct virtqueue *vq, for (i = 0; i < num; i++) { struct virtio_i2c_req *req = &reqs[i]; - wait_for_completion(&req->completion); - - if (!failed && req->in_hdr.status != VIRTIO_I2C_MSG_OK) - failed = true; + if (!failed) { + if (wait_for_completion_interruptible(&req->completion)) + failed = true; + else if (req->in_hdr.status != VIRTIO_I2C_MSG_OK) + failed = true; + else + j++; + } i2c_put_dma_safe_msg_buf(reqs[i].buf, &msgs[i], !failed); - - if (!failed) - j++; } return j; diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index d2499f302b508..f43067f6797e9 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -370,6 +370,7 @@ static const struct acpi_device_id i2c_acpi_force_100khz_device_ids[] = { * the device works without issues on Windows at what is expected to be * a 400KHz frequency. The root cause of the issue is not known. */ + { "DLL0945", 0 }, { "ELAN06FA", 0 }, {} }; diff --git a/drivers/i3c/internals.h b/drivers/i3c/internals.h index 908a807badaf9..e267ea5ec5b95 100644 --- a/drivers/i3c/internals.h +++ b/drivers/i3c/internals.h @@ -9,6 +9,7 @@ #define I3C_INTERNALS_H #include +#include extern struct bus_type i3c_bus_type; diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 15f24bbc35f2a..f5fdccbd2cbc9 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -1274,7 +1274,7 @@ static int i3c_master_retrieve_dev_info(struct i3c_dev_desc *dev) if (dev->info.bcr & I3C_BCR_HDR_CAP) { ret = i3c_master_gethdrcap_locked(master, &dev->info); - if (ret) + if (ret && ret != -ENOTSUPP) return ret; } @@ -2298,6 +2298,8 @@ static int i3c_i2c_notifier_call(struct notifier_block *nb, unsigned long action case BUS_NOTIFY_DEL_DEVICE: ret = i3c_master_i2c_detach(adap, client); break; + default: + ret = -EINVAL; } i3c_bus_maintenance_unlock(&master->bus); diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index b90347d6b6357..e604d542d9583 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -356,6 +356,7 @@ static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master, SVC_I3C_MSTATUS_COMPLETE(val), 0, 1000); if (ret) { dev_err(master->dev, "Timeout when polling for COMPLETE\n"); + i3c_generic_ibi_recycle_slot(data->ibi_pool, slot); return ret; } @@ -417,9 +418,24 @@ static void svc_i3c_master_ibi_work(struct work_struct *work) */ writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS); - /* Acknowledge the incoming interrupt with the AUTOIBI mechanism */ - writel(SVC_I3C_MCTRL_REQUEST_AUTO_IBI | - SVC_I3C_MCTRL_IBIRESP_AUTO, + /* + * Write REQUEST_START_ADDR request to emit broadcast address for arbitration, + * instend of using AUTO_IBI. + * + * Using AutoIBI request may cause controller to remain in AutoIBI state when + * there is a glitch on SDA line (high->low->high). + * 1. SDA high->low, raising an interrupt to execute IBI isr. + * 2. SDA low->high. + * 3. IBI isr writes an AutoIBI request. + * 4. The controller will not start AutoIBI process because SDA is not low. + * 5. IBIWON polling times out. + * 6. Controller reamins in AutoIBI state and doesn't accept EmitStop request. + */ + writel(SVC_I3C_MCTRL_REQUEST_START_ADDR | + SVC_I3C_MCTRL_TYPE_I3C | + SVC_I3C_MCTRL_IBIRESP_MANUAL | + SVC_I3C_MCTRL_DIR(SVC_I3C_MCTRL_DIR_WRITE) | + SVC_I3C_MCTRL_ADDR(I3C_BROADCAST_ADDR), master->regs + SVC_I3C_MCTRL); /* Wait for IBIWON, should take approximately 100us */ @@ -439,10 +455,15 @@ static void svc_i3c_master_ibi_work(struct work_struct *work) switch (ibitype) { case SVC_I3C_MSTATUS_IBITYPE_IBI: dev = svc_i3c_master_dev_from_addr(master, ibiaddr); - if (!dev) + if (!dev) { svc_i3c_master_nack_ibi(master); - else + } else { + if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD) + svc_i3c_master_ack_ibi(master, true); + else + svc_i3c_master_ack_ibi(master, false); svc_i3c_master_handle_ibi(master, dev); + } break; case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN: svc_i3c_master_ack_ibi(master, false); @@ -485,6 +506,8 @@ static void svc_i3c_master_ibi_work(struct work_struct *work) queue_work(master->base.wq, &master->hj_work); break; case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST: + svc_i3c_master_emit_stop(master); + break; default: break; } @@ -773,6 +796,8 @@ static int svc_i3c_master_do_daa_locked(struct svc_i3c_master *master, u32 reg; int ret, i; + svc_i3c_master_flush_fifo(master); + while (true) { /* Enter/proceed with DAA */ writel(SVC_I3C_MCTRL_REQUEST_PROC_DAA | diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 44842f243f40b..6908052dea778 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1432,7 +1432,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = { }; static const struct x86_cpu_id intel_mwait_ids[] __initconst = { - X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), + X86_MATCH_VENDOR_FAM_FEATURE(INTEL, X86_FAMILY_ANY, X86_FEATURE_MWAIT, NULL), {} }; diff --git a/drivers/iio/accel/adis16201.c b/drivers/iio/accel/adis16201.c index d054721859b3b..99b05548b7bdb 100644 --- a/drivers/iio/accel/adis16201.c +++ b/drivers/iio/accel/adis16201.c @@ -211,9 +211,9 @@ static const struct iio_chan_spec adis16201_channels[] = { BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14), ADIS_AUX_ADC_CHAN(ADIS16201_AUX_ADC_REG, ADIS16201_SCAN_AUX_ADC, 0, 12), ADIS_INCLI_CHAN(X, ADIS16201_XINCL_OUT_REG, ADIS16201_SCAN_INCLI_X, - BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14), + BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 12), ADIS_INCLI_CHAN(Y, ADIS16201_YINCL_OUT_REG, ADIS16201_SCAN_INCLI_Y, - BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14), + BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 12), IIO_CHAN_SOFT_TIMESTAMP(7) }; diff --git a/drivers/iio/accel/adxl355_core.c b/drivers/iio/accel/adxl355_core.c index 0c9225d18fb29..4973e8da5399d 100644 --- a/drivers/iio/accel/adxl355_core.c +++ b/drivers/iio/accel/adxl355_core.c @@ -231,7 +231,7 @@ struct adxl355_data { u8 transf_buf[3]; struct { u8 buf[14]; - s64 ts; + aligned_s64 ts; } buffer; } __aligned(IIO_DMA_MINALIGN); }; diff --git a/drivers/iio/accel/adxl367.c b/drivers/iio/accel/adxl367.c index 484fe2e9fb174..3a55475691de3 100644 --- a/drivers/iio/accel/adxl367.c +++ b/drivers/iio/accel/adxl367.c @@ -620,18 +620,14 @@ static int _adxl367_set_odr(struct adxl367_state *st, enum adxl367_odr odr) if (ret) return ret; + st->odr = odr; + /* Activity timers depend on ODR */ ret = _adxl367_set_act_time_ms(st, st->act_time_ms); if (ret) return ret; - ret = _adxl367_set_inact_time_ms(st, st->inact_time_ms); - if (ret) - return ret; - - st->odr = odr; - - return 0; + return _adxl367_set_inact_time_ms(st, st->inact_time_ms); } static int adxl367_set_odr(struct iio_dev *indio_dev, enum adxl367_odr odr) diff --git a/drivers/iio/accel/fxls8962af-core.c b/drivers/iio/accel/fxls8962af-core.c index be8a15cb945fd..f4e253c6c7d70 100644 --- a/drivers/iio/accel/fxls8962af-core.c +++ b/drivers/iio/accel/fxls8962af-core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -434,8 +435,16 @@ static int fxls8962af_read_raw(struct iio_dev *indio_dev, *val = FXLS8962AF_TEMP_CENTER_VAL; return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: - *val = 0; - return fxls8962af_read_full_scale(data, val2); + switch (chan->type) { + case IIO_TEMP: + *val = MILLIDEGREE_PER_DEGREE; + return IIO_VAL_INT; + case IIO_ACCEL: + *val = 0; + return fxls8962af_read_full_scale(data, val2); + default: + return -EINVAL; + } case IIO_CHAN_INFO_SAMP_FREQ: return fxls8962af_read_samp_freq(data, val, val2); default: @@ -734,9 +743,11 @@ static const struct iio_event_spec fxls8962af_event[] = { .type = IIO_TEMP, \ .address = FXLS8962AF_TEMP_OUT, \ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ + BIT(IIO_CHAN_INFO_SCALE) | \ BIT(IIO_CHAN_INFO_OFFSET),\ .scan_index = -1, \ .scan_type = { \ + .sign = 's', \ .realbits = 8, \ .storagebits = 8, \ }, \ @@ -854,6 +865,8 @@ static int fxls8962af_buffer_predisable(struct iio_dev *indio_dev) if (ret) return ret; + synchronize_irq(data->irq); + ret = __fxls8962af_fifo_set_mode(data, false); if (data->enable_event) diff --git a/drivers/iio/adc/ad7124.c b/drivers/iio/adc/ad7124.c index 0e6baf017bfd1..f631351eef97b 100644 --- a/drivers/iio/adc/ad7124.c +++ b/drivers/iio/adc/ad7124.c @@ -300,9 +300,9 @@ static int ad7124_get_3db_filter_freq(struct ad7124_state *st, switch (st->channels[channel].cfg.filter_type) { case AD7124_SINC3_FILTER: - return DIV_ROUND_CLOSEST(fadc * 230, 1000); + return DIV_ROUND_CLOSEST(fadc * 272, 1000); case AD7124_SINC4_FILTER: - return DIV_ROUND_CLOSEST(fadc * 262, 1000); + return DIV_ROUND_CLOSEST(fadc * 230, 1000); default: return -EINVAL; } diff --git a/drivers/iio/adc/ad7266.c b/drivers/iio/adc/ad7266.c index 98648c679a55c..2ace3aafe4978 100644 --- a/drivers/iio/adc/ad7266.c +++ b/drivers/iio/adc/ad7266.c @@ -44,7 +44,7 @@ struct ad7266_state { */ struct { __be16 sample[2]; - s64 timestamp; + aligned_s64 timestamp; } data __aligned(IIO_DMA_MINALIGN); }; diff --git a/drivers/iio/adc/ad7606_spi.c b/drivers/iio/adc/ad7606_spi.c index 287a0591533b6..df48464a115d0 100644 --- a/drivers/iio/adc/ad7606_spi.c +++ b/drivers/iio/adc/ad7606_spi.c @@ -127,7 +127,7 @@ static int ad7606_spi_reg_read(struct ad7606_state *st, unsigned int addr) { .tx_buf = &st->d16[0], .len = 2, - .cs_change = 0, + .cs_change = 1, }, { .rx_buf = &st->d16[1], .len = 2, @@ -151,7 +151,7 @@ static int ad7606_spi_reg_write(struct ad7606_state *st, struct spi_device *spi = to_spi_device(st->dev); st->d16[0] = cpu_to_be16((st->bops->rd_wr_cmd(addr, 1) << 8) | - (val & 0x1FF)); + (val & 0xFF)); return spi_write(spi, &st->d16[0], sizeof(st->d16[0])); } diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c index 74b0c85944bd6..e147eaf1a3b15 100644 --- a/drivers/iio/adc/ad7768-1.c +++ b/drivers/iio/adc/ad7768-1.c @@ -169,7 +169,7 @@ struct ad7768_state { union { struct { __be32 chan; - s64 timestamp; + aligned_s64 timestamp; } scan; __be32 d32; u8 d8[2]; @@ -203,6 +203,24 @@ static int ad7768_spi_reg_write(struct ad7768_state *st, return spi_write(st->spi, st->data.d8, 2); } +static int ad7768_send_sync_pulse(struct ad7768_state *st) +{ + /* + * The datasheet specifies a minimum SYNC_IN pulse width of 1.5 × Tmclk, + * where Tmclk is the MCLK period. The supported MCLK frequencies range + * from 0.6 MHz to 17 MHz, which corresponds to a minimum SYNC_IN pulse + * width of approximately 2.5 µs in the worst-case scenario (0.6 MHz). + * + * Add a delay to ensure the pulse width is always sufficient to + * trigger synchronization. + */ + gpiod_set_value_cansleep(st->gpio_sync_in, 1); + fsleep(3); + gpiod_set_value_cansleep(st->gpio_sync_in, 0); + + return 0; +} + static int ad7768_set_mode(struct ad7768_state *st, enum ad7768_conv_mode mode) { @@ -288,10 +306,7 @@ static int ad7768_set_dig_fil(struct ad7768_state *st, return ret; /* A sync-in pulse is required every time the filter dec rate changes */ - gpiod_set_value(st->gpio_sync_in, 1); - gpiod_set_value(st->gpio_sync_in, 0); - - return 0; + return ad7768_send_sync_pulse(st); } static int ad7768_set_freq(struct ad7768_state *st, diff --git a/drivers/iio/adc/ad7949.c b/drivers/iio/adc/ad7949.c index edd0c3a35ab73..202561cad4012 100644 --- a/drivers/iio/adc/ad7949.c +++ b/drivers/iio/adc/ad7949.c @@ -308,7 +308,6 @@ static void ad7949_disable_reg(void *reg) static int ad7949_spi_probe(struct spi_device *spi) { - u32 spi_ctrl_mask = spi->controller->bits_per_word_mask; struct device *dev = &spi->dev; const struct ad7949_adc_spec *spec; struct ad7949_adc_chip *ad7949_adc; @@ -337,11 +336,11 @@ static int ad7949_spi_probe(struct spi_device *spi) ad7949_adc->resolution = spec->resolution; /* Set SPI bits per word */ - if (spi_ctrl_mask & SPI_BPW_MASK(ad7949_adc->resolution)) { + if (spi_is_bpw_supported(spi, ad7949_adc->resolution)) { spi->bits_per_word = ad7949_adc->resolution; - } else if (spi_ctrl_mask == SPI_BPW_MASK(16)) { + } else if (spi_is_bpw_supported(spi, 16)) { spi->bits_per_word = 16; - } else if (spi_ctrl_mask == SPI_BPW_MASK(8)) { + } else if (spi_is_bpw_supported(spi, 8)) { spi->bits_per_word = 8; } else { dev_err(dev, "unable to find common BPW with spi controller\n"); diff --git a/drivers/iio/adc/ad_sigma_delta.c b/drivers/iio/adc/ad_sigma_delta.c index 7e21928707437..914274ed899ec 100644 --- a/drivers/iio/adc/ad_sigma_delta.c +++ b/drivers/iio/adc/ad_sigma_delta.c @@ -378,7 +378,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) return ret; } - samples_buf_size = ALIGN(slot * indio_dev->channels[0].scan_type.storagebits, 8); + samples_buf_size = ALIGN(slot * indio_dev->channels[0].scan_type.storagebits / 8, 8); samples_buf_size += sizeof(int64_t); samples_buf = devm_krealloc(&sigma_delta->spi->dev, sigma_delta->samples_buf, samples_buf_size, GFP_KERNEL); @@ -406,7 +406,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) return ret; } -static int ad_sd_buffer_postdisable(struct iio_dev *indio_dev) +static int ad_sd_buffer_predisable(struct iio_dev *indio_dev) { struct ad_sigma_delta *sigma_delta = iio_device_get_drvdata(indio_dev); @@ -476,6 +476,10 @@ static irqreturn_t ad_sd_trigger_handler(int irq, void *p) * byte set to zero. */ ad_sd_read_reg_raw(sigma_delta, data_reg, transfer_size, &data[1]); break; + + default: + dev_err_ratelimited(&indio_dev->dev, "Unsupported reg_size: %u\n", reg_size); + goto irq_handled; } /* @@ -530,7 +534,7 @@ static bool ad_sd_validate_scan_mask(struct iio_dev *indio_dev, const unsigned l static const struct iio_buffer_setup_ops ad_sd_buffer_setup_ops = { .postenable = &ad_sd_buffer_postenable, - .postdisable = &ad_sd_buffer_postdisable, + .predisable = &ad_sd_buffer_predisable, .validate_scan_mask = &ad_sd_validate_scan_mask, }; diff --git a/drivers/iio/adc/dln2-adc.c b/drivers/iio/adc/dln2-adc.c index 97d162a3cba4e..49a2588e7431e 100644 --- a/drivers/iio/adc/dln2-adc.c +++ b/drivers/iio/adc/dln2-adc.c @@ -483,7 +483,7 @@ static irqreturn_t dln2_adc_trigger_h(int irq, void *p) struct iio_dev *indio_dev = pf->indio_dev; struct { __le16 values[DLN2_ADC_MAX_CHANNELS]; - int64_t timestamp_space; + aligned_s64 timestamp_space; } data; struct dln2_adc_get_all_vals dev_data; struct dln2_adc *dln2 = iio_priv(indio_dev); diff --git a/drivers/iio/adc/imx93_adc.c b/drivers/iio/adc/imx93_adc.c index 512d7b95b08e6..2303ae19c602e 100644 --- a/drivers/iio/adc/imx93_adc.c +++ b/drivers/iio/adc/imx93_adc.c @@ -38,6 +38,7 @@ #define IMX93_ADC_PCDR6 0x118 #define IMX93_ADC_PCDR7 0x11c #define IMX93_ADC_CALSTAT 0x39C +#define IMX93_ADC_CALCFG0 0x3A0 /* ADC bit shift */ #define IMX93_ADC_MCR_MODE_MASK BIT(29) @@ -58,6 +59,8 @@ #define IMX93_ADC_IMR_ECH_MASK BIT(0) #define IMX93_ADC_PCDR_CDATA_MASK GENMASK(11, 0) +#define IMX93_ADC_CALCFG0_LDFAIL_MASK BIT(4) + /* ADC status */ #define IMX93_ADC_MSR_ADCSTATUS_IDLE 0 #define IMX93_ADC_MSR_ADCSTATUS_POWER_DOWN 1 @@ -145,7 +148,7 @@ static void imx93_adc_config_ad_clk(struct imx93_adc *adc) static int imx93_adc_calibration(struct imx93_adc *adc) { - u32 mcr, msr; + u32 mcr, msr, calcfg; int ret; /* make sure ADC in power down mode */ @@ -158,6 +161,11 @@ static int imx93_adc_calibration(struct imx93_adc *adc) imx93_adc_power_up(adc); + /* Enable loading of calibrated values even in fail condition */ + calcfg = readl(adc->regs + IMX93_ADC_CALCFG0); + calcfg |= IMX93_ADC_CALCFG0_LDFAIL_MASK; + writel(calcfg, adc->regs + IMX93_ADC_CALCFG0); + /* * TODO: we use the default TSAMP/NRSMPL/AVGEN in MCR, * can add the setting of these bit if need in future. @@ -180,9 +188,13 @@ static int imx93_adc_calibration(struct imx93_adc *adc) /* check whether calbration is success or not */ msr = readl(adc->regs + IMX93_ADC_MSR); if (msr & IMX93_ADC_MSR_CALFAIL_MASK) { + /* + * Only give warning here, this means the noise of the + * reference voltage do not meet the requirement: + * ADC reference voltage Noise < 1.8V * 1/2^ENOB + * And the resault of ADC is not that accurate. + */ dev_warn(adc->dev, "ADC calibration failed!\n"); - imx93_adc_power_down(adc); - return -EAGAIN; } return 0; diff --git a/drivers/iio/adc/max1363.c b/drivers/iio/adc/max1363.c index b31581616ce34..5304235d733f1 100644 --- a/drivers/iio/adc/max1363.c +++ b/drivers/iio/adc/max1363.c @@ -510,10 +510,10 @@ static const struct iio_event_spec max1363_events[] = { MAX1363_CHAN_U(1, _s1, 1, bits, ev_spec, num_ev_spec), \ MAX1363_CHAN_U(2, _s2, 2, bits, ev_spec, num_ev_spec), \ MAX1363_CHAN_U(3, _s3, 3, bits, ev_spec, num_ev_spec), \ - MAX1363_CHAN_B(0, 1, d0m1, 4, bits, ev_spec, num_ev_spec), \ - MAX1363_CHAN_B(2, 3, d2m3, 5, bits, ev_spec, num_ev_spec), \ - MAX1363_CHAN_B(1, 0, d1m0, 6, bits, ev_spec, num_ev_spec), \ - MAX1363_CHAN_B(3, 2, d3m2, 7, bits, ev_spec, num_ev_spec), \ + MAX1363_CHAN_B(0, 1, d0m1, 12, bits, ev_spec, num_ev_spec), \ + MAX1363_CHAN_B(2, 3, d2m3, 13, bits, ev_spec, num_ev_spec), \ + MAX1363_CHAN_B(1, 0, d1m0, 18, bits, ev_spec, num_ev_spec), \ + MAX1363_CHAN_B(3, 2, d3m2, 19, bits, ev_spec, num_ev_spec), \ IIO_CHAN_SOFT_TIMESTAMP(8) \ } @@ -531,23 +531,23 @@ static const struct iio_chan_spec max1363_channels[] = /* Applies to max1236, max1237 */ static const enum max1363_modes max1236_mode_list[] = { _s0, _s1, _s2, _s3, - s0to1, s0to2, s0to3, + s0to1, s0to2, s2to3, s0to3, d0m1, d2m3, d1m0, d3m2, d0m1to2m3, d1m0to3m2, - s2to3, }; /* Applies to max1238, max1239 */ static const enum max1363_modes max1238_mode_list[] = { _s0, _s1, _s2, _s3, _s4, _s5, _s6, _s7, _s8, _s9, _s10, _s11, s0to1, s0to2, s0to3, s0to4, s0to5, s0to6, + s6to7, s6to8, s6to9, s6to10, s6to11, s0to7, s0to8, s0to9, s0to10, s0to11, d0m1, d2m3, d4m5, d6m7, d8m9, d10m11, d1m0, d3m2, d5m4, d7m6, d9m8, d11m10, - d0m1to2m3, d0m1to4m5, d0m1to6m7, d0m1to8m9, d0m1to10m11, - d1m0to3m2, d1m0to5m4, d1m0to7m6, d1m0to9m8, d1m0to11m10, - s6to7, s6to8, s6to9, s6to10, s6to11, - d6m7to8m9, d6m7to10m11, d7m6to9m8, d7m6to11m10, + d0m1to2m3, d0m1to4m5, d0m1to6m7, d6m7to8m9, + d0m1to8m9, d6m7to10m11, d0m1to10m11, d1m0to3m2, + d1m0to5m4, d1m0to7m6, d7m6to9m8, d1m0to9m8, + d7m6to11m10, d1m0to11m10, }; #define MAX1363_12X_CHANS(bits) { \ @@ -583,16 +583,15 @@ static const struct iio_chan_spec max1238_channels[] = MAX1363_12X_CHANS(12); static const enum max1363_modes max11607_mode_list[] = { _s0, _s1, _s2, _s3, - s0to1, s0to2, s0to3, - s2to3, + s0to1, s0to2, s2to3, + s0to3, d0m1, d2m3, d1m0, d3m2, d0m1to2m3, d1m0to3m2, }; static const enum max1363_modes max11608_mode_list[] = { _s0, _s1, _s2, _s3, _s4, _s5, _s6, _s7, - s0to1, s0to2, s0to3, s0to4, s0to5, s0to6, s0to7, - s6to7, + s0to1, s0to2, s0to3, s0to4, s0to5, s0to6, s6to7, s0to7, d0m1, d2m3, d4m5, d6m7, d1m0, d3m2, d5m4, d7m6, d0m1to2m3, d0m1to4m5, d0m1to6m7, @@ -608,14 +607,14 @@ static const enum max1363_modes max11608_mode_list[] = { MAX1363_CHAN_U(5, _s5, 5, bits, NULL, 0), \ MAX1363_CHAN_U(6, _s6, 6, bits, NULL, 0), \ MAX1363_CHAN_U(7, _s7, 7, bits, NULL, 0), \ - MAX1363_CHAN_B(0, 1, d0m1, 8, bits, NULL, 0), \ - MAX1363_CHAN_B(2, 3, d2m3, 9, bits, NULL, 0), \ - MAX1363_CHAN_B(4, 5, d4m5, 10, bits, NULL, 0), \ - MAX1363_CHAN_B(6, 7, d6m7, 11, bits, NULL, 0), \ - MAX1363_CHAN_B(1, 0, d1m0, 12, bits, NULL, 0), \ - MAX1363_CHAN_B(3, 2, d3m2, 13, bits, NULL, 0), \ - MAX1363_CHAN_B(5, 4, d5m4, 14, bits, NULL, 0), \ - MAX1363_CHAN_B(7, 6, d7m6, 15, bits, NULL, 0), \ + MAX1363_CHAN_B(0, 1, d0m1, 12, bits, NULL, 0), \ + MAX1363_CHAN_B(2, 3, d2m3, 13, bits, NULL, 0), \ + MAX1363_CHAN_B(4, 5, d4m5, 14, bits, NULL, 0), \ + MAX1363_CHAN_B(6, 7, d6m7, 15, bits, NULL, 0), \ + MAX1363_CHAN_B(1, 0, d1m0, 18, bits, NULL, 0), \ + MAX1363_CHAN_B(3, 2, d3m2, 19, bits, NULL, 0), \ + MAX1363_CHAN_B(5, 4, d5m4, 20, bits, NULL, 0), \ + MAX1363_CHAN_B(7, 6, d7m6, 21, bits, NULL, 0), \ IIO_CHAN_SOFT_TIMESTAMP(16) \ } static const struct iio_chan_spec max11602_channels[] = MAX1363_8X_CHANS(8); diff --git a/drivers/iio/adc/rockchip_saradc.c b/drivers/iio/adc/rockchip_saradc.c index 929cba215d99a..7b4eb4a200df8 100644 --- a/drivers/iio/adc/rockchip_saradc.c +++ b/drivers/iio/adc/rockchip_saradc.c @@ -485,15 +485,6 @@ static int rockchip_saradc_probe(struct platform_device *pdev) if (info->reset) rockchip_saradc_reset_controller(info->reset); - /* - * Use a default value for the converter clock. - * This may become user-configurable in the future. - */ - ret = clk_set_rate(info->clk, info->data->clk_rate); - if (ret < 0) - return dev_err_probe(&pdev->dev, ret, - "failed to set adc clk rate\n"); - ret = regulator_enable(info->vref); if (ret < 0) return dev_err_probe(&pdev->dev, ret, @@ -520,6 +511,14 @@ static int rockchip_saradc_probe(struct platform_device *pdev) if (IS_ERR(info->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(info->clk), "failed to get adc clock\n"); + /* + * Use a default value for the converter clock. + * This may become user-configurable in the future. + */ + ret = clk_set_rate(info->clk, info->data->clk_rate); + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "failed to set adc clk rate\n"); platform_set_drvdata(pdev, indio_dev); diff --git a/drivers/iio/adc/spear_adc.c b/drivers/iio/adc/spear_adc.c index ad54ef7981090..602ed05552bfd 100644 --- a/drivers/iio/adc/spear_adc.c +++ b/drivers/iio/adc/spear_adc.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -29,9 +30,9 @@ /* Bit definitions for SPEAR_ADC_STATUS */ #define SPEAR_ADC_STATUS_START_CONVERSION BIT(0) -#define SPEAR_ADC_STATUS_CHANNEL_NUM(x) ((x) << 1) +#define SPEAR_ADC_STATUS_CHANNEL_NUM_MASK GENMASK(3, 1) #define SPEAR_ADC_STATUS_ADC_ENABLE BIT(4) -#define SPEAR_ADC_STATUS_AVG_SAMPLE(x) ((x) << 5) +#define SPEAR_ADC_STATUS_AVG_SAMPLE_MASK GENMASK(8, 5) #define SPEAR_ADC_STATUS_VREF_INTERNAL BIT(9) #define SPEAR_ADC_DATA_MASK 0x03ff @@ -157,8 +158,8 @@ static int spear_adc_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_RAW: mutex_lock(&st->lock); - status = SPEAR_ADC_STATUS_CHANNEL_NUM(chan->channel) | - SPEAR_ADC_STATUS_AVG_SAMPLE(st->avg_samples) | + status = FIELD_PREP(SPEAR_ADC_STATUS_CHANNEL_NUM_MASK, chan->channel) | + FIELD_PREP(SPEAR_ADC_STATUS_AVG_SAMPLE_MASK, st->avg_samples) | SPEAR_ADC_STATUS_START_CONVERSION | SPEAR_ADC_STATUS_ADC_ENABLE; if (st->vref_external == 0) diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c index bbd5bdd732f01..9f1566a378de4 100644 --- a/drivers/iio/adc/stm32-adc-core.c +++ b/drivers/iio/adc/stm32-adc-core.c @@ -428,10 +428,9 @@ static int stm32_adc_irq_probe(struct platform_device *pdev, return -ENOMEM; } - for (i = 0; i < priv->cfg->num_irqs; i++) { - irq_set_chained_handler(priv->irq[i], stm32_adc_irq_handler); - irq_set_handler_data(priv->irq[i], priv); - } + for (i = 0; i < priv->cfg->num_irqs; i++) + irq_set_chained_handler_and_data(priv->irq[i], + stm32_adc_irq_handler, priv); return 0; } diff --git a/drivers/iio/adc/xilinx-ams.c b/drivers/iio/adc/xilinx-ams.c index f52abf759260f..5de86a953d418 100644 --- a/drivers/iio/adc/xilinx-ams.c +++ b/drivers/iio/adc/xilinx-ams.c @@ -118,7 +118,7 @@ #define AMS_ALARM_THRESHOLD_OFF_10 0x10 #define AMS_ALARM_THRESHOLD_OFF_20 0x20 -#define AMS_ALARM_THR_DIRECT_MASK BIT(1) +#define AMS_ALARM_THR_DIRECT_MASK BIT(0) #define AMS_ALARM_THR_MIN 0x0000 #define AMS_ALARM_THR_MAX (BIT(16) - 1) @@ -385,6 +385,29 @@ static void ams_update_pl_alarm(struct ams *ams, unsigned long alarm_mask) ams_pl_update_reg(ams, AMS_REG_CONFIG3, AMS_REGCFG3_ALARM_MASK, cfg); } +static void ams_unmask(struct ams *ams) +{ + unsigned int status, unmask; + + status = readl(ams->base + AMS_ISR_0); + + /* Clear those bits which are not active anymore */ + unmask = (ams->current_masked_alarm ^ status) & ams->current_masked_alarm; + + /* Clear status of disabled alarm */ + unmask |= ams->intr_mask; + + ams->current_masked_alarm &= status; + + /* Also clear those which are masked out anyway */ + ams->current_masked_alarm &= ~ams->intr_mask; + + /* Clear the interrupts before we unmask them */ + writel(unmask, ams->base + AMS_ISR_0); + + ams_update_intrmask(ams, ~AMS_ALARM_MASK, ~AMS_ALARM_MASK); +} + static void ams_update_alarm(struct ams *ams, unsigned long alarm_mask) { unsigned long flags; @@ -397,6 +420,7 @@ static void ams_update_alarm(struct ams *ams, unsigned long alarm_mask) spin_lock_irqsave(&ams->intr_lock, flags); ams_update_intrmask(ams, AMS_ISR0_ALARM_MASK, ~alarm_mask); + ams_unmask(ams); spin_unlock_irqrestore(&ams->intr_lock, flags); } @@ -1025,28 +1049,9 @@ static void ams_handle_events(struct iio_dev *indio_dev, unsigned long events) static void ams_unmask_worker(struct work_struct *work) { struct ams *ams = container_of(work, struct ams, ams_unmask_work.work); - unsigned int status, unmask; spin_lock_irq(&ams->intr_lock); - - status = readl(ams->base + AMS_ISR_0); - - /* Clear those bits which are not active anymore */ - unmask = (ams->current_masked_alarm ^ status) & ams->current_masked_alarm; - - /* Clear status of disabled alarm */ - unmask |= ams->intr_mask; - - ams->current_masked_alarm &= status; - - /* Also clear those which are masked out anyway */ - ams->current_masked_alarm &= ~ams->intr_mask; - - /* Clear the interrupts before we unmask them */ - writel(unmask, ams->base + AMS_ISR_0); - - ams_update_intrmask(ams, ~AMS_ALARM_MASK, ~AMS_ALARM_MASK); - + ams_unmask(ams); spin_unlock_irq(&ams->intr_lock); /* If still pending some alarm re-trigger the timer */ diff --git a/drivers/iio/chemical/pms7003.c b/drivers/iio/chemical/pms7003.c index e9857d93b307e..70c92cbfc9f14 100644 --- a/drivers/iio/chemical/pms7003.c +++ b/drivers/iio/chemical/pms7003.c @@ -5,7 +5,6 @@ * Copyright (c) Tomasz Duszynski */ -#include #include #include #include @@ -19,6 +18,8 @@ #include #include #include +#include +#include #define PMS7003_DRIVER_NAME "pms7003" @@ -76,7 +77,7 @@ struct pms7003_state { /* Used to construct scan to push to the IIO buffer */ struct { u16 data[3]; /* PM1, PM2P5, PM10 */ - s64 ts; + aligned_s64 ts; } scan; }; diff --git a/drivers/iio/chemical/sps30.c b/drivers/iio/chemical/sps30.c index 814ce0aad1ccc..4085a36cd1db7 100644 --- a/drivers/iio/chemical/sps30.c +++ b/drivers/iio/chemical/sps30.c @@ -108,7 +108,7 @@ static irqreturn_t sps30_trigger_handler(int irq, void *p) int ret; struct { s32 data[4]; /* PM1, PM2P5, PM4, PM10 */ - s64 ts; + aligned_s64 ts; } scan; mutex_lock(&state->lock); diff --git a/drivers/iio/dac/ad5360.c b/drivers/iio/dac/ad5360.c index e0b7f658d6119..cf9cf90cd6e27 100644 --- a/drivers/iio/dac/ad5360.c +++ b/drivers/iio/dac/ad5360.c @@ -262,7 +262,7 @@ static int ad5360_update_ctrl(struct iio_dev *indio_dev, unsigned int set, unsigned int clr) { struct ad5360_state *st = iio_priv(indio_dev); - unsigned int ret; + int ret; mutex_lock(&st->lock); diff --git a/drivers/iio/dac/ad5421.c b/drivers/iio/dac/ad5421.c index 7644acfd879e0..9228e3cee1b85 100644 --- a/drivers/iio/dac/ad5421.c +++ b/drivers/iio/dac/ad5421.c @@ -186,7 +186,7 @@ static int ad5421_update_ctrl(struct iio_dev *indio_dev, unsigned int set, unsigned int clr) { struct ad5421_state *st = iio_priv(indio_dev); - unsigned int ret; + int ret; mutex_lock(&st->lock); diff --git a/drivers/iio/filter/admv8818.c b/drivers/iio/filter/admv8818.c index d85b7d3de8660..cc8ce0fe74e7c 100644 --- a/drivers/iio/filter/admv8818.c +++ b/drivers/iio/filter/admv8818.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,16 @@ #define ADMV8818_HPF_WR0_MSK GENMASK(7, 4) #define ADMV8818_LPF_WR0_MSK GENMASK(3, 0) +#define ADMV8818_BAND_BYPASS 0 +#define ADMV8818_BAND_MIN 1 +#define ADMV8818_BAND_MAX 4 +#define ADMV8818_BAND_CORNER_LOW 0 +#define ADMV8818_BAND_CORNER_HIGH 1 + +#define ADMV8818_STATE_MIN 0 +#define ADMV8818_STATE_MAX 15 +#define ADMV8818_NUM_STATES 16 + enum { ADMV8818_BW_FREQ, ADMV8818_CENTER_FREQ @@ -90,20 +101,24 @@ struct admv8818_state { struct mutex lock; unsigned int filter_mode; u64 cf_hz; + u64 lpf_margin_hz; + u64 hpf_margin_hz; }; -static const unsigned long long freq_range_hpf[4][2] = { +static const unsigned long long freq_range_hpf[5][2] = { + {0ULL, 0ULL}, /* bypass */ {1750000000ULL, 3550000000ULL}, {3400000000ULL, 7250000000ULL}, {6600000000, 12000000000}, {12500000000, 19900000000} }; -static const unsigned long long freq_range_lpf[4][2] = { +static const unsigned long long freq_range_lpf[5][2] = { + {U64_MAX, U64_MAX}, /* bypass */ {2050000000ULL, 3850000000ULL}, {3350000000ULL, 7250000000ULL}, {7000000000, 13000000000}, - {12550000000, 18500000000} + {12550000000, 18850000000} }; static const struct regmap_config admv8818_regmap_config = { @@ -121,44 +136,59 @@ static const char * const admv8818_modes[] = { static int __admv8818_hpf_select(struct admv8818_state *st, u64 freq) { - unsigned int hpf_step = 0, hpf_band = 0, i, j; - u64 freq_step; - int ret; + int band, state, ret; + unsigned int hpf_state = ADMV8818_STATE_MIN, hpf_band = ADMV8818_BAND_BYPASS; + u64 freq_error, min_freq_error, freq_corner, freq_step; - if (freq < freq_range_hpf[0][0]) + if (freq < freq_range_hpf[ADMV8818_BAND_MIN][ADMV8818_BAND_CORNER_LOW]) goto hpf_write; - if (freq > freq_range_hpf[3][1]) { - hpf_step = 15; - hpf_band = 4; - + if (freq >= freq_range_hpf[ADMV8818_BAND_MAX][ADMV8818_BAND_CORNER_HIGH]) { + hpf_state = ADMV8818_STATE_MAX; + hpf_band = ADMV8818_BAND_MAX; goto hpf_write; } - for (i = 0; i < 4; i++) { - freq_step = div_u64((freq_range_hpf[i][1] - - freq_range_hpf[i][0]), 15); + /* Close HPF frequency gap between 12 and 12.5 GHz */ + if (freq >= 12000ULL * HZ_PER_MHZ && freq < 12500ULL * HZ_PER_MHZ) { + hpf_state = ADMV8818_STATE_MAX; + hpf_band = 3; + goto hpf_write; + } - if (freq > freq_range_hpf[i][0] && - (freq < freq_range_hpf[i][1] + freq_step)) { - hpf_band = i + 1; + min_freq_error = U64_MAX; + for (band = ADMV8818_BAND_MIN; band <= ADMV8818_BAND_MAX; band++) { + /* + * This (and therefore all other ranges) have a corner + * frequency higher than the target frequency. + */ + if (freq_range_hpf[band][ADMV8818_BAND_CORNER_LOW] > freq) + break; - for (j = 1; j <= 16; j++) { - if (freq < (freq_range_hpf[i][0] + (freq_step * j))) { - hpf_step = j - 1; - break; - } + freq_step = freq_range_hpf[band][ADMV8818_BAND_CORNER_HIGH] - + freq_range_hpf[band][ADMV8818_BAND_CORNER_LOW]; + freq_step = div_u64(freq_step, ADMV8818_NUM_STATES - 1); + + for (state = ADMV8818_STATE_MIN; state <= ADMV8818_STATE_MAX; state++) { + freq_corner = freq_range_hpf[band][ADMV8818_BAND_CORNER_LOW] + + freq_step * state; + + /* + * This (and therefore all other states) have a corner + * frequency higher than the target frequency. + */ + if (freq_corner > freq) + break; + + freq_error = freq - freq_corner; + if (freq_error < min_freq_error) { + min_freq_error = freq_error; + hpf_state = state; + hpf_band = band; } - break; } } - /* Close HPF frequency gap between 12 and 12.5 GHz */ - if (freq >= 12000 * HZ_PER_MHZ && freq <= 12500 * HZ_PER_MHZ) { - hpf_band = 3; - hpf_step = 15; - } - hpf_write: ret = regmap_update_bits(st->regmap, ADMV8818_REG_WR0_SW, ADMV8818_SW_IN_SET_WR0_MSK | @@ -170,7 +200,7 @@ static int __admv8818_hpf_select(struct admv8818_state *st, u64 freq) return regmap_update_bits(st->regmap, ADMV8818_REG_WR0_FILTER, ADMV8818_HPF_WR0_MSK, - FIELD_PREP(ADMV8818_HPF_WR0_MSK, hpf_step)); + FIELD_PREP(ADMV8818_HPF_WR0_MSK, hpf_state)); } static int admv8818_hpf_select(struct admv8818_state *st, u64 freq) @@ -186,31 +216,52 @@ static int admv8818_hpf_select(struct admv8818_state *st, u64 freq) static int __admv8818_lpf_select(struct admv8818_state *st, u64 freq) { - unsigned int lpf_step = 0, lpf_band = 0, i, j; - u64 freq_step; - int ret; + int band, state, ret; + unsigned int lpf_state = ADMV8818_STATE_MIN, lpf_band = ADMV8818_BAND_BYPASS; + u64 freq_error, min_freq_error, freq_corner, freq_step; - if (freq > freq_range_lpf[3][1]) + if (freq > freq_range_lpf[ADMV8818_BAND_MAX][ADMV8818_BAND_CORNER_HIGH]) goto lpf_write; - if (freq < freq_range_lpf[0][0]) { - lpf_band = 1; - + if (freq < freq_range_lpf[ADMV8818_BAND_MIN][ADMV8818_BAND_CORNER_LOW]) { + lpf_state = ADMV8818_STATE_MIN; + lpf_band = ADMV8818_BAND_MIN; goto lpf_write; } - for (i = 0; i < 4; i++) { - if (freq > freq_range_lpf[i][0] && freq < freq_range_lpf[i][1]) { - lpf_band = i + 1; - freq_step = div_u64((freq_range_lpf[i][1] - freq_range_lpf[i][0]), 15); + min_freq_error = U64_MAX; + for (band = ADMV8818_BAND_MAX; band >= ADMV8818_BAND_MIN; --band) { + /* + * At this point the highest corner frequency of + * all remaining ranges is below the target. + * LPF corner should be >= the target. + */ + if (freq > freq_range_lpf[band][ADMV8818_BAND_CORNER_HIGH]) + break; + + freq_step = freq_range_lpf[band][ADMV8818_BAND_CORNER_HIGH] - + freq_range_lpf[band][ADMV8818_BAND_CORNER_LOW]; + freq_step = div_u64(freq_step, ADMV8818_NUM_STATES - 1); + + for (state = ADMV8818_STATE_MAX; state >= ADMV8818_STATE_MIN; --state) { + + freq_corner = freq_range_lpf[band][ADMV8818_BAND_CORNER_LOW] + + state * freq_step; - for (j = 0; j <= 15; j++) { - if (freq < (freq_range_lpf[i][0] + (freq_step * j))) { - lpf_step = j; - break; - } + /* + * At this point all other states in range will + * place the corner frequency below the target + * LPF corner should >= the target. + */ + if (freq > freq_corner) + break; + + freq_error = freq_corner - freq; + if (freq_error < min_freq_error) { + min_freq_error = freq_error; + lpf_state = state; + lpf_band = band; } - break; } } @@ -225,7 +276,7 @@ static int __admv8818_lpf_select(struct admv8818_state *st, u64 freq) return regmap_update_bits(st->regmap, ADMV8818_REG_WR0_FILTER, ADMV8818_LPF_WR0_MSK, - FIELD_PREP(ADMV8818_LPF_WR0_MSK, lpf_step)); + FIELD_PREP(ADMV8818_LPF_WR0_MSK, lpf_state)); } static int admv8818_lpf_select(struct admv8818_state *st, u64 freq) @@ -242,16 +293,28 @@ static int admv8818_lpf_select(struct admv8818_state *st, u64 freq) static int admv8818_rfin_band_select(struct admv8818_state *st) { int ret; + u64 hpf_corner_target, lpf_corner_target; st->cf_hz = clk_get_rate(st->clkin); + /* Check for underflow */ + if (st->cf_hz > st->hpf_margin_hz) + hpf_corner_target = st->cf_hz - st->hpf_margin_hz; + else + hpf_corner_target = 0; + + /* Check for overflow */ + lpf_corner_target = st->cf_hz + st->lpf_margin_hz; + if (lpf_corner_target < st->cf_hz) + lpf_corner_target = U64_MAX; + mutex_lock(&st->lock); - ret = __admv8818_hpf_select(st, st->cf_hz); + ret = __admv8818_hpf_select(st, hpf_corner_target); if (ret) goto exit; - ret = __admv8818_lpf_select(st, st->cf_hz); + ret = __admv8818_lpf_select(st, lpf_corner_target); exit: mutex_unlock(&st->lock); return ret; @@ -278,8 +341,11 @@ static int __admv8818_read_hpf_freq(struct admv8818_state *st, u64 *hpf_freq) hpf_state = FIELD_GET(ADMV8818_HPF_WR0_MSK, data); - *hpf_freq = div_u64(freq_range_hpf[hpf_band - 1][1] - freq_range_hpf[hpf_band - 1][0], 15); - *hpf_freq = freq_range_hpf[hpf_band - 1][0] + (*hpf_freq * hpf_state); + *hpf_freq = freq_range_hpf[hpf_band][ADMV8818_BAND_CORNER_HIGH] - + freq_range_hpf[hpf_band][ADMV8818_BAND_CORNER_LOW]; + *hpf_freq = div_u64(*hpf_freq, ADMV8818_NUM_STATES - 1); + *hpf_freq = freq_range_hpf[hpf_band][ADMV8818_BAND_CORNER_LOW] + + (*hpf_freq * hpf_state); return ret; } @@ -316,8 +382,11 @@ static int __admv8818_read_lpf_freq(struct admv8818_state *st, u64 *lpf_freq) lpf_state = FIELD_GET(ADMV8818_LPF_WR0_MSK, data); - *lpf_freq = div_u64(freq_range_lpf[lpf_band - 1][1] - freq_range_lpf[lpf_band - 1][0], 15); - *lpf_freq = freq_range_lpf[lpf_band - 1][0] + (*lpf_freq * lpf_state); + *lpf_freq = freq_range_lpf[lpf_band][ADMV8818_BAND_CORNER_HIGH] - + freq_range_lpf[lpf_band][ADMV8818_BAND_CORNER_LOW]; + *lpf_freq = div_u64(*lpf_freq, ADMV8818_NUM_STATES - 1); + *lpf_freq = freq_range_lpf[lpf_band][ADMV8818_BAND_CORNER_LOW] + + (*lpf_freq * lpf_state); return ret; } @@ -333,6 +402,19 @@ static int admv8818_read_lpf_freq(struct admv8818_state *st, u64 *lpf_freq) return ret; } +static int admv8818_write_raw_get_fmt(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + case IIO_CHAN_INFO_HIGH_PASS_FILTER_3DB_FREQUENCY: + return IIO_VAL_INT_64; + default: + return -EINVAL; + } +} + static int admv8818_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int val, int val2, long info) @@ -341,6 +423,9 @@ static int admv8818_write_raw(struct iio_dev *indio_dev, u64 freq = ((u64)val2 << 32 | (u32)val); + if ((s64)freq < 0) + return -EINVAL; + switch (info) { case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: return admv8818_lpf_select(st, freq); @@ -502,6 +587,7 @@ static int admv8818_set_mode(struct iio_dev *indio_dev, static const struct iio_info admv8818_info = { .write_raw = admv8818_write_raw, + .write_raw_get_fmt = admv8818_write_raw_get_fmt, .read_raw = admv8818_read_raw, .debugfs_reg_access = &admv8818_reg_access, }; @@ -641,6 +727,32 @@ static int admv8818_clk_setup(struct admv8818_state *st) return devm_add_action_or_reset(&spi->dev, admv8818_clk_notifier_unreg, st); } +static int admv8818_read_properties(struct admv8818_state *st) +{ + struct spi_device *spi = st->spi; + u32 mhz; + int ret; + + ret = device_property_read_u32(&spi->dev, "adi,lpf-margin-mhz", &mhz); + if (ret == 0) + st->lpf_margin_hz = (u64)mhz * HZ_PER_MHZ; + else if (ret == -EINVAL) + st->lpf_margin_hz = 0; + else + return ret; + + + ret = device_property_read_u32(&spi->dev, "adi,hpf-margin-mhz", &mhz); + if (ret == 0) + st->hpf_margin_hz = (u64)mhz * HZ_PER_MHZ; + else if (ret == -EINVAL) + st->hpf_margin_hz = 0; + else if (ret < 0) + return ret; + + return 0; +} + static int admv8818_probe(struct spi_device *spi) { struct iio_dev *indio_dev; @@ -672,6 +784,10 @@ static int admv8818_probe(struct spi_device *spi) mutex_init(&st->lock); + ret = admv8818_read_properties(st); + if (ret) + return ret; + ret = admv8818_init(st); if (ret) return ret; diff --git a/drivers/iio/frequency/adf4350.c b/drivers/iio/frequency/adf4350.c index 4abf80f75ef5d..03e75261e891f 100644 --- a/drivers/iio/frequency/adf4350.c +++ b/drivers/iio/frequency/adf4350.c @@ -143,6 +143,19 @@ static int adf4350_set_freq(struct adf4350_state *st, unsigned long long freq) if (freq > ADF4350_MAX_OUT_FREQ || freq < st->min_out_freq) return -EINVAL; + st->r4_rf_div_sel = 0; + + /* + * !\TODO: The below computation is making sure we get a power of 2 + * shift (st->r4_rf_div_sel) so that freq becomes higher or equal to + * ADF4350_MIN_VCO_FREQ. This might be simplified with fls()/fls_long() + * and friends. + */ + while (freq < ADF4350_MIN_VCO_FREQ) { + freq <<= 1; + st->r4_rf_div_sel++; + } + if (freq > ADF4350_MAX_FREQ_45_PRESC) { prescaler = ADF4350_REG1_PRESCALER; mdiv = 75; @@ -151,13 +164,6 @@ static int adf4350_set_freq(struct adf4350_state *st, unsigned long long freq) mdiv = 23; } - st->r4_rf_div_sel = 0; - - while (freq < ADF4350_MIN_VCO_FREQ) { - freq <<= 1; - st->r4_rf_div_sel++; - } - /* * Allow a predefined reference division factor * if not set, compute our own diff --git a/drivers/iio/imu/bno055/bno055.c b/drivers/iio/imu/bno055/bno055.c index 52744dd98e65b..98f17c29da69b 100644 --- a/drivers/iio/imu/bno055/bno055.c +++ b/drivers/iio/imu/bno055/bno055.c @@ -118,6 +118,7 @@ struct bno055_sysfs_attr { int len; int *fusion_vals; int *hw_xlate; + int hw_xlate_len; int type; }; @@ -170,20 +171,24 @@ static int bno055_gyr_scale_vals[] = { 1000, 1877467, 2000, 1877467, }; +static int bno055_gyr_scale_hw_xlate[] = {0, 1, 2, 3, 4}; static struct bno055_sysfs_attr bno055_gyr_scale = { .vals = bno055_gyr_scale_vals, .len = ARRAY_SIZE(bno055_gyr_scale_vals), .fusion_vals = (int[]){1, 900}, - .hw_xlate = (int[]){4, 3, 2, 1, 0}, + .hw_xlate = bno055_gyr_scale_hw_xlate, + .hw_xlate_len = ARRAY_SIZE(bno055_gyr_scale_hw_xlate), .type = IIO_VAL_FRACTIONAL, }; static int bno055_gyr_lpf_vals[] = {12, 23, 32, 47, 64, 116, 230, 523}; +static int bno055_gyr_lpf_hw_xlate[] = {5, 4, 7, 3, 6, 2, 1, 0}; static struct bno055_sysfs_attr bno055_gyr_lpf = { .vals = bno055_gyr_lpf_vals, .len = ARRAY_SIZE(bno055_gyr_lpf_vals), .fusion_vals = (int[]){32}, - .hw_xlate = (int[]){5, 4, 7, 3, 6, 2, 1, 0}, + .hw_xlate = bno055_gyr_lpf_hw_xlate, + .hw_xlate_len = ARRAY_SIZE(bno055_gyr_lpf_hw_xlate), .type = IIO_VAL_INT, }; @@ -561,7 +566,7 @@ static int bno055_get_regmask(struct bno055_priv *priv, int *val, int *val2, idx = (hwval & mask) >> shift; if (attr->hw_xlate) - for (i = 0; i < attr->len; i++) + for (i = 0; i < attr->hw_xlate_len; i++) if (attr->hw_xlate[i] == idx) { idx = i; break; diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600.h b/drivers/iio/imu/inv_icm42600/inv_icm42600.h index 94c0eb0bf8748..e289afcb43e3c 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600.h +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600.h @@ -126,9 +126,9 @@ struct inv_icm42600_suspended { * @suspended: suspended sensors configuration. * @indio_gyro: gyroscope IIO device. * @indio_accel: accelerometer IIO device. - * @buffer: data transfer buffer aligned for DMA. - * @fifo: FIFO management structure. * @timestamp: interrupt timestamps. + * @fifo: FIFO management structure. + * @buffer: data transfer buffer aligned for DMA. */ struct inv_icm42600_state { struct mutex lock; @@ -142,12 +142,12 @@ struct inv_icm42600_state { struct inv_icm42600_suspended suspended; struct iio_dev *indio_gyro; struct iio_dev *indio_accel; - uint8_t buffer[2] __aligned(IIO_DMA_MINALIGN); - struct inv_icm42600_fifo fifo; struct { - int64_t gyro; - int64_t accel; + s64 gyro; + s64 accel; } timestamp; + struct inv_icm42600_fifo fifo; + u8 buffer[2] __aligned(IIO_DMA_MINALIGN); }; /* Virtual register addresses: @bank on MSB (4 upper bits), @address on LSB */ @@ -369,7 +369,7 @@ const struct iio_mount_matrix * inv_icm42600_get_mount_matrix(const struct iio_dev *indio_dev, const struct iio_chan_spec *chan); -uint32_t inv_icm42600_odr_to_period(enum inv_icm42600_odr odr); +u32 inv_icm42600_odr_to_period(enum inv_icm42600_odr odr); int inv_icm42600_set_accel_conf(struct inv_icm42600_state *st, struct inv_icm42600_sensor_conf *conf, diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c index 47720560de6e0..a4155939e9567 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c @@ -77,8 +77,8 @@ static const struct iio_chan_spec inv_icm42600_accel_channels[] = { */ struct inv_icm42600_accel_buffer { struct inv_icm42600_fifo_sensor_data accel; - int16_t temp; - int64_t timestamp __aligned(8); + s16 temp; + aligned_s64 timestamp; }; #define INV_ICM42600_SCAN_MASK_ACCEL_3AXIS \ @@ -142,7 +142,7 @@ static int inv_icm42600_accel_update_scan_mode(struct iio_dev *indio_dev, static int inv_icm42600_accel_read_sensor(struct inv_icm42600_state *st, struct iio_chan_spec const *chan, - int16_t *val) + s16 *val) { struct device *dev = regmap_get_device(st->map); struct inv_icm42600_sensor_conf conf = INV_ICM42600_SENSOR_CONF_INIT; @@ -182,7 +182,7 @@ static int inv_icm42600_accel_read_sensor(struct inv_icm42600_state *st, if (ret) goto exit; - *val = (int16_t)be16_to_cpup(data); + *val = (s16)be16_to_cpup(data); if (*val == INV_ICM42600_DATA_INVALID) ret = -EINVAL; exit: @@ -359,11 +359,11 @@ static int inv_icm42600_accel_read_offset(struct inv_icm42600_state *st, int *val, int *val2) { struct device *dev = regmap_get_device(st->map); - int64_t val64; - int32_t bias; + s64 val64; + s32 bias; unsigned int reg; - int16_t offset; - uint8_t data[2]; + s16 offset; + u8 data[2]; int ret; if (chan->type != IIO_ACCEL) @@ -417,7 +417,7 @@ static int inv_icm42600_accel_read_offset(struct inv_icm42600_state *st, * result in micro (1000000) * (offset * 5 * 9.806650 * 1000000) / 10000 */ - val64 = (int64_t)offset * 5LL * 9806650LL; + val64 = (s64)offset * 5LL * 9806650LL; /* for rounding, add + or - divisor (10000) divided by 2 */ if (val64 >= 0) val64 += 10000LL / 2LL; @@ -435,10 +435,10 @@ static int inv_icm42600_accel_write_offset(struct inv_icm42600_state *st, int val, int val2) { struct device *dev = regmap_get_device(st->map); - int64_t val64; - int32_t min, max; + s64 val64; + s32 min, max; unsigned int reg, regval; - int16_t offset; + s16 offset; int ret; if (chan->type != IIO_ACCEL) @@ -463,7 +463,7 @@ static int inv_icm42600_accel_write_offset(struct inv_icm42600_state *st, inv_icm42600_accel_calibbias[1]; max = inv_icm42600_accel_calibbias[4] * 1000000L + inv_icm42600_accel_calibbias[5]; - val64 = (int64_t)val * 1000000LL + (int64_t)val2; + val64 = (s64)val * 1000000LL + (s64)val2; if (val64 < min || val64 > max) return -EINVAL; @@ -538,7 +538,7 @@ static int inv_icm42600_accel_read_raw(struct iio_dev *indio_dev, int *val, int *val2, long mask) { struct inv_icm42600_state *st = iio_device_get_drvdata(indio_dev); - int16_t data; + s16 data; int ret; switch (chan->type) { @@ -755,7 +755,8 @@ int inv_icm42600_accel_parse_fifo(struct iio_dev *indio_dev) const int8_t *temp; unsigned int odr; int64_t ts_val; - struct inv_icm42600_accel_buffer buffer; + /* buffer is copied to userspace, zeroing it to avoid any data leak */ + struct inv_icm42600_accel_buffer buffer = { }; /* parse all fifo packets */ for (i = 0, no = 0; i < st->fifo.count; i += size, ++no) { @@ -774,8 +775,6 @@ int inv_icm42600_accel_parse_fifo(struct iio_dev *indio_dev) inv_sensors_timestamp_apply_odr(ts, st->fifo.period, st->fifo.nb.total, no); - /* buffer is copied to userspace, zeroing it to avoid any data leak */ - memset(&buffer, 0, sizeof(buffer)); memcpy(&buffer.accel, accel, sizeof(buffer.accel)); /* convert 8 bits FIFO temperature in high resolution format */ buffer.temp = temp ? (*temp * 64) : 0; diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c index 6ef1df9d60b77..aca6ce7588905 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c @@ -26,28 +26,28 @@ #define INV_ICM42600_FIFO_HEADER_ODR_GYRO BIT(0) struct inv_icm42600_fifo_1sensor_packet { - uint8_t header; + u8 header; struct inv_icm42600_fifo_sensor_data data; - int8_t temp; + s8 temp; } __packed; #define INV_ICM42600_FIFO_1SENSOR_PACKET_SIZE 8 struct inv_icm42600_fifo_2sensors_packet { - uint8_t header; + u8 header; struct inv_icm42600_fifo_sensor_data accel; struct inv_icm42600_fifo_sensor_data gyro; - int8_t temp; + s8 temp; __be16 timestamp; } __packed; #define INV_ICM42600_FIFO_2SENSORS_PACKET_SIZE 16 ssize_t inv_icm42600_fifo_decode_packet(const void *packet, const void **accel, - const void **gyro, const int8_t **temp, + const void **gyro, const s8 **temp, const void **timestamp, unsigned int *odr) { const struct inv_icm42600_fifo_1sensor_packet *pack1 = packet; const struct inv_icm42600_fifo_2sensors_packet *pack2 = packet; - uint8_t header = *((const uint8_t *)packet); + u8 header = *((const u8 *)packet); /* FIFO empty */ if (header & INV_ICM42600_FIFO_HEADER_MSG) { @@ -100,7 +100,7 @@ ssize_t inv_icm42600_fifo_decode_packet(const void *packet, const void **accel, void inv_icm42600_buffer_update_fifo_period(struct inv_icm42600_state *st) { - uint32_t period_gyro, period_accel, period; + u32 period_gyro, period_accel, period; if (st->fifo.en & INV_ICM42600_SENSOR_GYRO) period_gyro = inv_icm42600_odr_to_period(st->conf.gyro.odr); @@ -204,8 +204,8 @@ int inv_icm42600_buffer_update_watermark(struct inv_icm42600_state *st) { size_t packet_size, wm_size; unsigned int wm_gyro, wm_accel, watermark; - uint32_t period_gyro, period_accel, period; - uint32_t latency_gyro, latency_accel, latency; + u32 period_gyro, period_accel, period; + u32 latency_gyro, latency_accel, latency; bool restore; __le16 raw_wm; int ret; @@ -451,7 +451,7 @@ int inv_icm42600_buffer_fifo_read(struct inv_icm42600_state *st, __be16 *raw_fifo_count; ssize_t i, size; const void *accel, *gyro, *timestamp; - const int8_t *temp; + const s8 *temp; unsigned int odr; int ret; @@ -538,7 +538,7 @@ int inv_icm42600_buffer_hwfifo_flush(struct inv_icm42600_state *st, unsigned int count) { struct inv_sensors_timestamp *ts; - int64_t gyro_ts, accel_ts; + s64 gyro_ts, accel_ts; int ret; gyro_ts = iio_get_time_ns(st->indio_gyro); diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h index 8b85ee333bf8f..eed6a3152acf4 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h @@ -28,7 +28,7 @@ struct inv_icm42600_state; struct inv_icm42600_fifo { unsigned int on; unsigned int en; - uint32_t period; + u32 period; struct { unsigned int gyro; unsigned int accel; @@ -39,7 +39,7 @@ struct inv_icm42600_fifo { size_t accel; size_t total; } nb; - uint8_t data[2080] __aligned(IIO_DMA_MINALIGN); + u8 data[2080] __aligned(IIO_DMA_MINALIGN); }; /* FIFO data packet */ @@ -50,7 +50,7 @@ struct inv_icm42600_fifo_sensor_data { } __packed; #define INV_ICM42600_FIFO_DATA_INVALID -32768 -static inline int16_t inv_icm42600_fifo_get_sensor_data(__be16 d) +static inline s16 inv_icm42600_fifo_get_sensor_data(__be16 d) { return be16_to_cpu(d); } @@ -58,7 +58,7 @@ static inline int16_t inv_icm42600_fifo_get_sensor_data(__be16 d) static inline bool inv_icm42600_fifo_is_data_valid(const struct inv_icm42600_fifo_sensor_data *s) { - int16_t x, y, z; + s16 x, y, z; x = inv_icm42600_fifo_get_sensor_data(s->x); y = inv_icm42600_fifo_get_sensor_data(s->y); @@ -73,7 +73,7 @@ inv_icm42600_fifo_is_data_valid(const struct inv_icm42600_fifo_sensor_data *s) } ssize_t inv_icm42600_fifo_decode_packet(const void *packet, const void **accel, - const void **gyro, const int8_t **temp, + const void **gyro, const s8 **temp, const void **timestamp, unsigned int *odr); extern const struct iio_buffer_setup_ops inv_icm42600_buffer_ops; diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c index da65aa4e27242..99eb651743f80 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c @@ -56,7 +56,7 @@ const struct regmap_config inv_icm42600_spi_regmap_config = { EXPORT_SYMBOL_NS_GPL(inv_icm42600_spi_regmap_config, IIO_ICM42600); struct inv_icm42600_hw { - uint8_t whoami; + u8 whoami; const char *name; const struct inv_icm42600_conf *conf; }; @@ -115,9 +115,9 @@ inv_icm42600_get_mount_matrix(const struct iio_dev *indio_dev, return &st->orientation; } -uint32_t inv_icm42600_odr_to_period(enum inv_icm42600_odr odr) +u32 inv_icm42600_odr_to_period(enum inv_icm42600_odr odr) { - static uint32_t odr_periods[INV_ICM42600_ODR_NB] = { + static u32 odr_periods[INV_ICM42600_ODR_NB] = { /* reserved values */ 0, 0, 0, /* 8kHz */ @@ -567,20 +567,12 @@ static void inv_icm42600_disable_vdd_reg(void *_data) static void inv_icm42600_disable_vddio_reg(void *_data) { struct inv_icm42600_state *st = _data; - const struct device *dev = regmap_get_device(st->map); - int ret; - - ret = regulator_disable(st->vddio_supply); - if (ret) - dev_err(dev, "failed to disable vddio error %d\n", ret); -} + struct device *dev = regmap_get_device(st->map); -static void inv_icm42600_disable_pm(void *_data) -{ - struct device *dev = _data; + if (pm_runtime_status_suspended(dev)) + return; - pm_runtime_put_sync(dev); - pm_runtime_disable(dev); + regulator_disable(st->vddio_supply); } int inv_icm42600_core_probe(struct regmap *regmap, int chip, int irq, @@ -677,16 +669,14 @@ int inv_icm42600_core_probe(struct regmap *regmap, int chip, int irq, return ret; /* setup runtime power management */ - ret = pm_runtime_set_active(dev); + ret = devm_pm_runtime_set_active_enabled(dev); if (ret) return ret; - pm_runtime_get_noresume(dev); - pm_runtime_enable(dev); + pm_runtime_set_autosuspend_delay(dev, INV_ICM42600_SUSPEND_DELAY_MS); pm_runtime_use_autosuspend(dev); - pm_runtime_put(dev); - return devm_add_action_or_reset(dev, inv_icm42600_disable_pm, dev); + return ret; } EXPORT_SYMBOL_NS_GPL(inv_icm42600_core_probe, IIO_ICM42600); @@ -697,17 +687,15 @@ EXPORT_SYMBOL_NS_GPL(inv_icm42600_core_probe, IIO_ICM42600); static int inv_icm42600_suspend(struct device *dev) { struct inv_icm42600_state *st = dev_get_drvdata(dev); - int ret; + int ret = 0; mutex_lock(&st->lock); st->suspended.gyro = st->conf.gyro.mode; st->suspended.accel = st->conf.accel.mode; st->suspended.temp = st->conf.temp_en; - if (pm_runtime_suspended(dev)) { - ret = 0; + if (pm_runtime_suspended(dev)) goto out_unlock; - } /* disable FIFO data streaming */ if (st->fifo.on) { @@ -739,18 +727,17 @@ static int inv_icm42600_resume(struct device *dev) struct inv_icm42600_state *st = dev_get_drvdata(dev); struct inv_sensors_timestamp *gyro_ts = iio_priv(st->indio_gyro); struct inv_sensors_timestamp *accel_ts = iio_priv(st->indio_accel); - int ret; + int ret = 0; mutex_lock(&st->lock); + if (pm_runtime_suspended(dev)) + goto out_unlock; + ret = inv_icm42600_enable_regulator_vddio(st); if (ret) goto out_unlock; - pm_runtime_disable(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - /* restore sensors state */ ret = inv_icm42600_set_pwr_mgmt0(st, st->suspended.gyro, st->suspended.accel, diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c index d08cd6839a3a6..9ee26478b666b 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c @@ -77,8 +77,8 @@ static const struct iio_chan_spec inv_icm42600_gyro_channels[] = { */ struct inv_icm42600_gyro_buffer { struct inv_icm42600_fifo_sensor_data gyro; - int16_t temp; - int64_t timestamp __aligned(8); + s16 temp; + aligned_s64 timestamp; }; #define INV_ICM42600_SCAN_MASK_GYRO_3AXIS \ @@ -142,7 +142,7 @@ static int inv_icm42600_gyro_update_scan_mode(struct iio_dev *indio_dev, static int inv_icm42600_gyro_read_sensor(struct inv_icm42600_state *st, struct iio_chan_spec const *chan, - int16_t *val) + s16 *val) { struct device *dev = regmap_get_device(st->map); struct inv_icm42600_sensor_conf conf = INV_ICM42600_SENSOR_CONF_INIT; @@ -182,7 +182,7 @@ static int inv_icm42600_gyro_read_sensor(struct inv_icm42600_state *st, if (ret) goto exit; - *val = (int16_t)be16_to_cpup(data); + *val = (s16)be16_to_cpup(data); if (*val == INV_ICM42600_DATA_INVALID) ret = -EINVAL; exit: @@ -371,11 +371,11 @@ static int inv_icm42600_gyro_read_offset(struct inv_icm42600_state *st, int *val, int *val2) { struct device *dev = regmap_get_device(st->map); - int64_t val64; - int32_t bias; + s64 val64; + s32 bias; unsigned int reg; - int16_t offset; - uint8_t data[2]; + s16 offset; + u8 data[2]; int ret; if (chan->type != IIO_ANGL_VEL) @@ -429,7 +429,7 @@ static int inv_icm42600_gyro_read_offset(struct inv_icm42600_state *st, * result in nano (1000000000) * (offset * 64 * Pi * 1000000000) / (2048 * 180) */ - val64 = (int64_t)offset * 64LL * 3141592653LL; + val64 = (s64)offset * 64LL * 3141592653LL; /* for rounding, add + or - divisor (2048 * 180) divided by 2 */ if (val64 >= 0) val64 += 2048 * 180 / 2; @@ -447,9 +447,9 @@ static int inv_icm42600_gyro_write_offset(struct inv_icm42600_state *st, int val, int val2) { struct device *dev = regmap_get_device(st->map); - int64_t val64, min, max; + s64 val64, min, max; unsigned int reg, regval; - int16_t offset; + s16 offset; int ret; if (chan->type != IIO_ANGL_VEL) @@ -470,11 +470,11 @@ static int inv_icm42600_gyro_write_offset(struct inv_icm42600_state *st, } /* inv_icm42600_gyro_calibbias: min - step - max in nano */ - min = (int64_t)inv_icm42600_gyro_calibbias[0] * 1000000000LL + - (int64_t)inv_icm42600_gyro_calibbias[1]; - max = (int64_t)inv_icm42600_gyro_calibbias[4] * 1000000000LL + - (int64_t)inv_icm42600_gyro_calibbias[5]; - val64 = (int64_t)val * 1000000000LL + (int64_t)val2; + min = (s64)inv_icm42600_gyro_calibbias[0] * 1000000000LL + + (s64)inv_icm42600_gyro_calibbias[1]; + max = (s64)inv_icm42600_gyro_calibbias[4] * 1000000000LL + + (s64)inv_icm42600_gyro_calibbias[5]; + val64 = (s64)val * 1000000000LL + (s64)val2; if (val64 < min || val64 > max) return -EINVAL; @@ -549,7 +549,7 @@ static int inv_icm42600_gyro_read_raw(struct iio_dev *indio_dev, int *val, int *val2, long mask) { struct inv_icm42600_state *st = iio_device_get_drvdata(indio_dev); - int16_t data; + s16 data; int ret; switch (chan->type) { @@ -764,10 +764,11 @@ int inv_icm42600_gyro_parse_fifo(struct iio_dev *indio_dev) ssize_t i, size; unsigned int no; const void *accel, *gyro, *timestamp; - const int8_t *temp; + const s8 *temp; unsigned int odr; - int64_t ts_val; - struct inv_icm42600_gyro_buffer buffer; + s64 ts_val; + /* buffer is copied to userspace, zeroing it to avoid any data leak */ + struct inv_icm42600_gyro_buffer buffer = { }; /* parse all fifo packets */ for (i = 0, no = 0; i < st->fifo.count; i += size, ++no) { @@ -786,8 +787,6 @@ int inv_icm42600_gyro_parse_fifo(struct iio_dev *indio_dev) inv_sensors_timestamp_apply_odr(ts, st->fifo.period, st->fifo.nb.total, no); - /* buffer is copied to userspace, zeroing it to avoid any data leak */ - memset(&buffer, 0, sizeof(buffer)); memcpy(&buffer.gyro, gyro, sizeof(buffer.gyro)); /* convert 8 bits FIFO temperature in high resolution format */ buffer.temp = temp ? (*temp * 64) : 0; diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c index 213cce1c31110..51430b4f5e51b 100644 --- a/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c +++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c @@ -13,7 +13,7 @@ #include "inv_icm42600.h" #include "inv_icm42600_temp.h" -static int inv_icm42600_temp_read(struct inv_icm42600_state *st, int16_t *temp) +static int inv_icm42600_temp_read(struct inv_icm42600_state *st, s16 *temp) { struct device *dev = regmap_get_device(st->map); __be16 *raw; @@ -31,9 +31,13 @@ static int inv_icm42600_temp_read(struct inv_icm42600_state *st, int16_t *temp) if (ret) goto exit; - *temp = (int16_t)be16_to_cpup(raw); + *temp = (s16)be16_to_cpup(raw); + /* + * Temperature data is invalid if both accel and gyro are off. + * Return -EBUSY in this case. + */ if (*temp == INV_ICM42600_DATA_INVALID) - ret = -EINVAL; + ret = -EBUSY; exit: mutex_unlock(&st->lock); @@ -48,7 +52,7 @@ int inv_icm42600_temp_read_raw(struct iio_dev *indio_dev, int *val, int *val2, long mask) { struct inv_icm42600_state *st = iio_device_get_drvdata(indio_dev); - int16_t temp; + s16 temp; int ret; if (chan->type != IIO_TEMP) @@ -67,16 +71,18 @@ int inv_icm42600_temp_read_raw(struct iio_dev *indio_dev, return IIO_VAL_INT; /* * T°C = (temp / 132.48) + 25 - * Tm°C = 1000 * ((temp * 100 / 13248) + 25) + * Tm°C = 1000 * ((temp / 132.48) + 25) + * Tm°C = 7.548309 * temp + 25000 + * Tm°C = (temp + 3312) * 7.548309 * scale: 100000 / 13248 ~= 7.548309 - * offset: 25000 + * offset: 3312 */ case IIO_CHAN_INFO_SCALE: *val = 7; *val2 = 548309; return IIO_VAL_INT_PLUS_MICRO; case IIO_CHAN_INFO_OFFSET: - *val = 25000; + *val = 3312; return IIO_VAL_INT; default: return -EINVAL; diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c index d4f9b5d8d28d6..ace3ce4faea73 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c @@ -52,7 +52,7 @@ irqreturn_t inv_mpu6050_read_fifo(int irq, void *p) u16 fifo_count; u32 fifo_period; s64 timestamp; - u8 data[INV_MPU6050_OUTPUT_DATA_SIZE]; + u8 data[INV_MPU6050_OUTPUT_DATA_SIZE] __aligned(8); int int_status; size_t i, nb; diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c index 066fe561c5e88..b8119fa4768eb 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c @@ -370,6 +370,9 @@ int st_lsm6dsx_read_fifo(struct st_lsm6dsx_hw *hw) if (fifo_status & cpu_to_le16(ST_LSM6DSX_FIFO_EMPTY_MASK)) return 0; + if (!pattern_len) + pattern_len = ST_LSM6DSX_SAMPLE_SIZE; + fifo_len = (le16_to_cpu(fifo_status) & fifo_diff_mask) * ST_LSM6DSX_CHAN_SIZE; fifo_len = (fifo_len / pattern_len) * pattern_len; @@ -601,6 +604,9 @@ int st_lsm6dsx_read_tagged_fifo(struct st_lsm6dsx_hw *hw) if (!fifo_len) return 0; + if (!pattern_len) + pattern_len = ST_LSM6DSX_TAGGED_SAMPLE_SIZE; + for (read_len = 0; read_len < fifo_len; read_len += pattern_len) { err = st_lsm6dsx_read_block(hw, ST_LSM6DSX_REG_FIFO_OUT_TAG_ADDR, diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c index 8255035ff124f..c7b2ab7870993 100644 --- a/drivers/iio/inkern.c +++ b/drivers/iio/inkern.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -635,7 +636,7 @@ static int iio_convert_raw_to_processed_unlocked(struct iio_channel *chan, { int scale_type, scale_val, scale_val2; int offset_type, offset_val, offset_val2; - s64 raw64 = raw; + s64 denominator, raw64 = raw; offset_type = iio_channel_read(chan, &offset_val, &offset_val2, IIO_CHAN_INFO_OFFSET); @@ -670,7 +671,7 @@ static int iio_convert_raw_to_processed_unlocked(struct iio_channel *chan, * If no channel scaling is available apply consumer scale to * raw value and return. */ - *processed = raw * scale; + *processed = raw64 * scale; return 0; } @@ -679,20 +680,19 @@ static int iio_convert_raw_to_processed_unlocked(struct iio_channel *chan, *processed = raw64 * scale_val * scale; break; case IIO_VAL_INT_PLUS_MICRO: - if (scale_val2 < 0) - *processed = -raw64 * scale_val * scale; - else - *processed = raw64 * scale_val * scale; - *processed += div_s64(raw64 * (s64)scale_val2 * scale, - 1000000LL); - break; case IIO_VAL_INT_PLUS_NANO: - if (scale_val2 < 0) - *processed = -raw64 * scale_val * scale; - else - *processed = raw64 * scale_val * scale; - *processed += div_s64(raw64 * (s64)scale_val2 * scale, - 1000000000LL); + switch (scale_type) { + case IIO_VAL_INT_PLUS_MICRO: + denominator = MICRO; + break; + case IIO_VAL_INT_PLUS_NANO: + denominator = NANO; + break; + } + *processed = raw64 * scale * abs(scale_val); + *processed += div_s64(raw64 * scale * abs(scale_val2), denominator); + if (scale_val < 0 || scale_val2 < 0) + *processed *= -1; break; case IIO_VAL_FRACTIONAL: *processed = div_s64(raw64 * (s64)scale_val * scale, diff --git a/drivers/iio/light/as73211.c b/drivers/iio/light/as73211.c index c1f9604c27141..166874c68ff63 100644 --- a/drivers/iio/light/as73211.c +++ b/drivers/iio/light/as73211.c @@ -573,7 +573,7 @@ static irqreturn_t as73211_trigger_handler(int irq __always_unused, void *p) struct { __le16 chan[4]; s64 ts __aligned(8); - } scan; + } scan = { }; int data_result, ret; mutex_lock(&data->mutex); diff --git a/drivers/iio/light/hid-sensor-prox.c b/drivers/iio/light/hid-sensor-prox.c index a47591e1bad9d..24f7327907539 100644 --- a/drivers/iio/light/hid-sensor-prox.c +++ b/drivers/iio/light/hid-sensor-prox.c @@ -102,8 +102,7 @@ static int prox_read_raw(struct iio_dev *indio_dev, ret_type = prox_state->scale_precision; break; case IIO_CHAN_INFO_OFFSET: - *val = hid_sensor_convert_exponent( - prox_state->prox_attr.unit_expo); + *val = 0; ret_type = IIO_VAL_INT; break; case IIO_CHAN_INFO_SAMP_FREQ: @@ -227,6 +226,11 @@ static int prox_parse_report(struct platform_device *pdev, dev_dbg(&pdev->dev, "prox %x:%x\n", st->prox_attr.index, st->prox_attr.report_id); + st->scale_precision = hid_sensor_format_scale(hsdev->usage, + &st->prox_attr, + &st->scale_pre_decml, + &st->scale_post_decml); + return ret; } diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c index dc529cbe3805e..25a45c4251fbd 100644 --- a/drivers/iio/light/opt3001.c +++ b/drivers/iio/light/opt3001.c @@ -692,8 +692,9 @@ static irqreturn_t opt3001_irq(int irq, void *_iio) struct opt3001 *opt = iio_priv(iio); int ret; bool wake_result_ready_queue = false; + bool ok_to_ignore_lock = opt->ok_to_ignore_lock; - if (!opt->ok_to_ignore_lock) + if (!ok_to_ignore_lock) mutex_lock(&opt->lock); ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_CONFIGURATION); @@ -730,7 +731,7 @@ static irqreturn_t opt3001_irq(int irq, void *_iio) } out: - if (!opt->ok_to_ignore_lock) + if (!ok_to_ignore_lock) mutex_unlock(&opt->lock); if (wake_result_ready_queue) diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c index 84f6b333c9195..8d9e8a1c94c45 100644 --- a/drivers/iio/pressure/bmp280-core.c +++ b/drivers/iio/pressure/bmp280-core.c @@ -2152,11 +2152,12 @@ int bmp280_common_probe(struct device *dev, /* Bring chip out of reset if there is an assigned GPIO line */ gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(gpiod)) + return dev_err_probe(dev, PTR_ERR(gpiod), "failed to get reset GPIO\n"); + /* Deassert the signal */ - if (gpiod) { - dev_info(dev, "release reset\n"); - gpiod_set_value(gpiod, 0); - } + dev_info(dev, "release reset\n"); + gpiod_set_value(gpiod, 0); data->regmap = regmap; diff --git a/drivers/iio/pressure/mprls0025pa.c b/drivers/iio/pressure/mprls0025pa.c index e3f0de020a40c..829c472812e49 100644 --- a/drivers/iio/pressure/mprls0025pa.c +++ b/drivers/iio/pressure/mprls0025pa.c @@ -87,11 +87,6 @@ static const struct mpr_func_spec mpr_func_spec[] = { [MPR_FUNCTION_C] = {.output_min = 3355443, .output_max = 13421773}, }; -struct mpr_chan { - s32 pres; /* pressure value */ - s64 ts; /* timestamp */ -}; - struct mpr_data { struct i2c_client *client; struct mutex lock; /* @@ -120,7 +115,10 @@ struct mpr_data { * loop until data is ready */ struct completion completion; /* handshake from irq to read */ - struct mpr_chan chan; /* + struct { + s32 pres; /* pressure value */ + aligned_s64 ts; /* timestamp */ + } chan; /* * channel values for buffered * mode */ diff --git a/drivers/iio/pressure/zpa2326.c b/drivers/iio/pressure/zpa2326.c index ef1d0349f4247..a1c694199c989 100644 --- a/drivers/iio/pressure/zpa2326.c +++ b/drivers/iio/pressure/zpa2326.c @@ -582,7 +582,7 @@ static int zpa2326_fill_sample_buffer(struct iio_dev *indio_dev, struct { u32 pressure; u16 temperature; - u64 timestamp; + aligned_s64 timestamp; } sample; int err; diff --git a/drivers/iio/proximity/isl29501.c b/drivers/iio/proximity/isl29501.c index bcebacaf3dab0..9fd3d2e8cea6c 100644 --- a/drivers/iio/proximity/isl29501.c +++ b/drivers/iio/proximity/isl29501.c @@ -938,12 +938,18 @@ static irqreturn_t isl29501_trigger_handler(int irq, void *p) struct iio_dev *indio_dev = pf->indio_dev; struct isl29501_private *isl29501 = iio_priv(indio_dev); const unsigned long *active_mask = indio_dev->active_scan_mask; - u32 buffer[4] __aligned(8) = {}; /* 1x16-bit + naturally aligned ts */ - - if (test_bit(ISL29501_DISTANCE_SCAN_INDEX, active_mask)) - isl29501_register_read(isl29501, REG_DISTANCE, buffer); + u32 value; + struct { + u16 data; + aligned_s64 ts; + } scan = { }; + + if (test_bit(ISL29501_DISTANCE_SCAN_INDEX, active_mask)) { + isl29501_register_read(isl29501, REG_DISTANCE, &value); + scan.data = value; + } - iio_push_to_buffers_with_timestamp(indio_dev, buffer, pf->timestamp); + iio_push_to_buffers_with_timestamp(indio_dev, &scan, pf->timestamp); iio_trigger_notify_done(indio_dev->trig); return IRQ_HANDLED; diff --git a/drivers/iio/temperature/maxim_thermocouple.c b/drivers/iio/temperature/maxim_thermocouple.c index c28a7a6dea5f1..44fba61ccfe27 100644 --- a/drivers/iio/temperature/maxim_thermocouple.c +++ b/drivers/iio/temperature/maxim_thermocouple.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -121,9 +122,16 @@ static const struct maxim_thermocouple_chip maxim_thermocouple_chips[] = { struct maxim_thermocouple_data { struct spi_device *spi; const struct maxim_thermocouple_chip *chip; - - u8 buffer[16] __aligned(IIO_DMA_MINALIGN); char tc_type; + /* Buffer for reading up to 2 hardware channels. */ + struct { + union { + __be16 raw16; + __be32 raw32; + __be16 raw[2]; + }; + aligned_s64 timestamp; + } buffer __aligned(IIO_DMA_MINALIGN); }; static int maxim_thermocouple_read(struct maxim_thermocouple_data *data, @@ -131,18 +139,16 @@ static int maxim_thermocouple_read(struct maxim_thermocouple_data *data, { unsigned int storage_bytes = data->chip->read_size; unsigned int shift = chan->scan_type.shift + (chan->address * 8); - __be16 buf16; - __be32 buf32; int ret; switch (storage_bytes) { case 2: - ret = spi_read(data->spi, (void *)&buf16, storage_bytes); - *val = be16_to_cpu(buf16); + ret = spi_read(data->spi, &data->buffer.raw16, storage_bytes); + *val = be16_to_cpu(data->buffer.raw16); break; case 4: - ret = spi_read(data->spi, (void *)&buf32, storage_bytes); - *val = be32_to_cpu(buf32); + ret = spi_read(data->spi, &data->buffer.raw32, storage_bytes); + *val = be32_to_cpu(data->buffer.raw32); break; default: ret = -EINVAL; @@ -167,9 +173,9 @@ static irqreturn_t maxim_thermocouple_trigger_handler(int irq, void *private) struct maxim_thermocouple_data *data = iio_priv(indio_dev); int ret; - ret = spi_read(data->spi, data->buffer, data->chip->read_size); + ret = spi_read(data->spi, data->buffer.raw, data->chip->read_size); if (!ret) { - iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, + iio_push_to_buffers_with_timestamp(indio_dev, &data->buffer, iio_get_time_ns(indio_dev)); } diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index f20dfe70fa0e4..8bf705921618c 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -460,14 +460,10 @@ static int addr_resolve_neigh(const struct dst_entry *dst, { int ret = 0; - if (ndev_flags & IFF_LOOPBACK) { + if (ndev_flags & IFF_LOOPBACK) memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN); - } else { - if (!(ndev_flags & IFF_NOARP)) { - /* If the device doesn't do ARP internally */ - ret = fetch_ha(dst, addr, dst_in, seq); - } - } + else + ret = fetch_ha(dst, addr, dst_in, seq); return ret; } diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 0b88203720b05..77c0b89259911 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -582,8 +582,8 @@ static int __ib_cache_gid_add(struct ib_device *ib_dev, u32 port, out_unlock: mutex_unlock(&table->lock); if (ret) - pr_warn("%s: unable to add gid %pI6 error=%d\n", - __func__, gid->raw, ret); + pr_warn_ratelimited("%s: unable to add gid %pI6 error=%d\n", + __func__, gid->raw, ret); return ret; } diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 07fb8d3c037f0..50bb3c43f40bf 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -166,7 +166,7 @@ struct cm_port { struct cm_device { struct kref kref; struct list_head list; - spinlock_t mad_agent_lock; + rwlock_t mad_agent_lock; struct ib_device *ib_device; u8 ack_delay; int going_down; @@ -284,7 +284,7 @@ static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv) if (!cm_id_priv->av.port) return ERR_PTR(-EINVAL); - spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); + read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); mad_agent = cm_id_priv->av.port->mad_agent; if (!mad_agent) { m = ERR_PTR(-EINVAL); @@ -315,7 +315,7 @@ static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv) m->context[0] = cm_id_priv; out: - spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); + read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); return m; } @@ -1032,8 +1032,8 @@ static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id, struct cm_id_private *cm_id_priv; cm_id_priv = container_of(cm_id, struct cm_id_private, id); - pr_err("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__, - cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount)); + pr_err_ratelimited("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__, + cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount)); } static void cm_destroy_id(struct ib_cm_id *cm_id, int err) @@ -1294,10 +1294,10 @@ static __be64 cm_form_tid(struct cm_id_private *cm_id_priv) if (!cm_id_priv->av.port) return cpu_to_be64(low_tid); - spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); + read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); if (cm_id_priv->av.port->mad_agent) hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32; - spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); + read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock); return cpu_to_be64(hi_tid | low_tid); } @@ -4374,7 +4374,7 @@ static int cm_add_one(struct ib_device *ib_device) return -ENOMEM; kref_init(&cm_dev->kref); - spin_lock_init(&cm_dev->mad_agent_lock); + rwlock_init(&cm_dev->mad_agent_lock); cm_dev->ib_device = ib_device; cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay; cm_dev->going_down = 0; @@ -4490,9 +4490,9 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data) * The above ensures no call paths from the work are running, * the remaining paths all take the mad_agent_lock. */ - spin_lock(&cm_dev->mad_agent_lock); + write_lock(&cm_dev->mad_agent_lock); port->mad_agent = NULL; - spin_unlock(&cm_dev->mad_agent_lock); + write_unlock(&cm_dev->mad_agent_lock); ib_unregister_mad_agent(mad_agent); ib_port_unregister_client_groups(ib_device, i, cm_counter_groups); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 348527cf1e7bf..a5ceae2e075ad 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -5215,7 +5215,8 @@ static int cma_netevent_callback(struct notifier_block *self, neigh->ha, ETH_ALEN)) continue; cma_id_get(current_id); - queue_work(cma_wq, ¤t_id->id.net_work); + if (!queue_work(cma_wq, ¤t_id->id.net_work)) + cma_id_put(current_id); } out: spin_unlock_irqrestore(&id_table_lock, flags); diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 3e4941754b48d..ce41f235af253 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -367,12 +367,9 @@ EXPORT_SYMBOL(iw_cm_disconnect); /* * CM_ID <-- DESTROYING * - * Clean up all resources associated with the connection and release - * the initial reference taken by iw_create_cm_id. - * - * Returns true if and only if the last cm_id_priv reference has been dropped. + * Clean up all resources associated with the connection. */ -static bool destroy_cm_id(struct iw_cm_id *cm_id) +static void destroy_cm_id(struct iw_cm_id *cm_id) { struct iwcm_id_private *cm_id_priv; struct ib_qp *qp; @@ -441,20 +438,22 @@ static bool destroy_cm_id(struct iw_cm_id *cm_id) iwpm_remove_mapinfo(&cm_id->local_addr, &cm_id->m_local_addr); iwpm_remove_mapping(&cm_id->local_addr, RDMA_NL_IWCM); } - - return iwcm_deref_id(cm_id_priv); } /* - * This function is only called by the application thread and cannot - * be called by the event thread. The function will wait for all - * references to be released on the cm_id and then kfree the cm_id - * object. + * Destroy cm_id. If the cm_id still has other references, wait for all + * references to be released on the cm_id and then release the initial + * reference taken by iw_create_cm_id. */ void iw_destroy_cm_id(struct iw_cm_id *cm_id) { - if (!destroy_cm_id(cm_id)) + struct iwcm_id_private *cm_id_priv; + + cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); + destroy_cm_id(cm_id); + if (refcount_read(&cm_id_priv->refcount) > 1) flush_workqueue(iwcm_wq); + iwcm_deref_id(cm_id_priv); } EXPORT_SYMBOL(iw_destroy_cm_id); @@ -1037,8 +1036,10 @@ static void cm_work_handler(struct work_struct *_work) if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { ret = process_event(cm_id_priv, &levent); - if (ret) - WARN_ON_ONCE(destroy_cm_id(&cm_id_priv->id)); + if (ret) { + destroy_cm_id(&cm_id_priv->id); + WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); + } } else pr_debug("dropping event %d\n", levent.event); if (iwcm_deref_id(cm_id_priv)) diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 6d1dbc9787590..a94723a12bb48 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -1412,10 +1412,11 @@ static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = { }; -static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack, - enum rdma_restrack_type res_type, - res_fill_func_t fill_func) +static noinline_for_stack int +res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh, + struct netlink_ext_ack *extack, + enum rdma_restrack_type res_type, + res_fill_func_t fill_func) { const struct nldev_fill_res_entry *fe = &fill_entries[res_type]; struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; @@ -2153,10 +2154,10 @@ static int nldev_stat_del_doit(struct sk_buff *skb, struct nlmsghdr *nlh, return ret; } -static int stat_get_doit_default_counter(struct sk_buff *skb, - struct nlmsghdr *nlh, - struct netlink_ext_ack *extack, - struct nlattr *tb[]) +static noinline_for_stack int +stat_get_doit_default_counter(struct sk_buff *skb, struct nlmsghdr *nlh, + struct netlink_ext_ack *extack, + struct nlattr *tb[]) { struct rdma_hw_stats *stats; struct nlattr *table_attr; @@ -2246,8 +2247,9 @@ static int stat_get_doit_default_counter(struct sk_buff *skb, return ret; } -static int stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack, struct nlattr *tb[]) +static noinline_for_stack int +stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh, + struct netlink_ext_ack *extack, struct nlattr *tb[]) { static enum rdma_nl_counter_mode mode; diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 8175dde60b0a8..9902bb96c7409 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -1013,6 +1013,8 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb, if (timeout > IB_SA_LOCAL_SVC_TIMEOUT_MAX) timeout = IB_SA_LOCAL_SVC_TIMEOUT_MAX; + spin_lock_irqsave(&ib_nl_request_lock, flags); + delta = timeout - sa_local_svc_timeout_ms; if (delta < 0) abs_delta = -delta; @@ -1020,7 +1022,6 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb, abs_delta = delta; if (delta != 0) { - spin_lock_irqsave(&ib_nl_request_lock, flags); sa_local_svc_timeout_ms = timeout; list_for_each_entry(query, &ib_nl_request_list, list) { if (delta < 0 && abs_delta > query->timeout) @@ -1038,9 +1039,10 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb, if (delay) mod_delayed_work(ib_nl_wq, &ib_nl_timed_work, (unsigned long)delay); - spin_unlock_irqrestore(&ib_nl_request_lock, flags); } + spin_unlock_irqrestore(&ib_nl_request_lock, flags); + settimeout_out: return 0; } diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 07c571c7b6999..c5b6863947605 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -80,9 +80,12 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, unsigned long virt) { - struct scatterlist *sg; + unsigned long curr_len = 0; + dma_addr_t curr_base = ~0; unsigned long va, pgoff; + struct scatterlist *sg; dma_addr_t mask; + dma_addr_t end; int i; umem->iova = va = virt; @@ -107,17 +110,30 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, pgoff = umem->address & ~PAGE_MASK; for_each_sgtable_dma_sg(&umem->sgt_append.sgt, sg, i) { - /* Walk SGL and reduce max page size if VA/PA bits differ - * for any address. + /* If the current entry is physically contiguous with the previous + * one, no need to take its start addresses into consideration. */ - mask |= (sg_dma_address(sg) + pgoff) ^ va; + if (check_add_overflow(curr_base, curr_len, &end) || + end != sg_dma_address(sg)) { + + curr_base = sg_dma_address(sg); + curr_len = 0; + + /* Reduce max page size if VA/PA bits differ */ + mask |= (curr_base + pgoff) ^ va; + + /* The alignment of any VA matching a discontinuity point + * in the physical memory sets the maximum possible page + * size as this must be a starting point of a new page that + * needs to be aligned. + */ + if (i != 0) + mask |= va; + } + + curr_len += sg_dma_len(sg); va += sg_dma_len(sg) - pgoff; - /* Except for the last entry, the ending iova alignment sets - * the maximum possible page size as the low bits of the iova - * must be zero when starting the next chunk. - */ - if (i != (umem->sgt_append.sgt.nents - 1)) - mask |= va; + pgoff = 0; } diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index c6053e82ecf6f..33e2fe0facd52 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -718,8 +718,8 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) goto err_free; pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); - if (!pd) { - ret = -EINVAL; + if (IS_ERR(pd)) { + ret = PTR_ERR(pd); goto err_free; } @@ -809,8 +809,8 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) if (cmd.flags & IB_MR_REREG_PD) { new_pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); - if (!new_pd) { - ret = -EINVAL; + if (IS_ERR(new_pd)) { + ret = PTR_ERR(new_pd); goto put_uobjs; } } else { @@ -919,8 +919,8 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) return PTR_ERR(uobj); pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); - if (!pd) { - ret = -EINVAL; + if (IS_ERR(pd)) { + ret = PTR_ERR(pd); goto err_free; } @@ -1127,8 +1127,8 @@ static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) return ret; cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); - if (!cq) - return -EINVAL; + if (IS_ERR(cq)) + return PTR_ERR(cq); ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata); if (ret) @@ -1189,8 +1189,8 @@ static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs) return ret; cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); - if (!cq) - return -EINVAL; + if (IS_ERR(cq)) + return PTR_ERR(cq); /* we copy a struct ib_uverbs_poll_cq_resp to user space */ header_ptr = attrs->ucore.outbuf; @@ -1238,8 +1238,8 @@ static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs) return ret; cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); - if (!cq) - return -EINVAL; + if (IS_ERR(cq)) + return PTR_ERR(cq); ib_req_notify_cq(cq, cmd.solicited_only ? IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); @@ -1321,8 +1321,8 @@ static int create_qp(struct uverbs_attr_bundle *attrs, ind_tbl = uobj_get_obj_read(rwq_ind_table, UVERBS_OBJECT_RWQ_IND_TBL, cmd->rwq_ind_tbl_handle, attrs); - if (!ind_tbl) { - ret = -EINVAL; + if (IS_ERR(ind_tbl)) { + ret = PTR_ERR(ind_tbl); goto err_put; } @@ -1360,8 +1360,10 @@ static int create_qp(struct uverbs_attr_bundle *attrs, if (cmd->is_srq) { srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd->srq_handle, attrs); - if (!srq || srq->srq_type == IB_SRQT_XRC) { - ret = -EINVAL; + if (IS_ERR(srq) || + srq->srq_type == IB_SRQT_XRC) { + ret = IS_ERR(srq) ? PTR_ERR(srq) : + -EINVAL; goto err_put; } } @@ -1371,23 +1373,29 @@ static int create_qp(struct uverbs_attr_bundle *attrs, rcq = uobj_get_obj_read( cq, UVERBS_OBJECT_CQ, cmd->recv_cq_handle, attrs); - if (!rcq) { - ret = -EINVAL; + if (IS_ERR(rcq)) { + ret = PTR_ERR(rcq); goto err_put; } } } } - if (has_sq) + if (has_sq) { scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd->send_cq_handle, attrs); + if (IS_ERR(scq)) { + ret = PTR_ERR(scq); + goto err_put; + } + } + if (!ind_tbl && cmd->qp_type != IB_QPT_XRC_INI) rcq = rcq ?: scq; pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs); - if (!pd || (!scq && has_sq)) { - ret = -EINVAL; + if (IS_ERR(pd)) { + ret = PTR_ERR(pd); goto err_put; } @@ -1482,18 +1490,18 @@ static int create_qp(struct uverbs_attr_bundle *attrs, err_put: if (!IS_ERR(xrcd_uobj)) uobj_put_read(xrcd_uobj); - if (pd) + if (!IS_ERR_OR_NULL(pd)) uobj_put_obj_read(pd); - if (scq) + if (!IS_ERR_OR_NULL(scq)) rdma_lookup_put_uobject(&scq->uobject->uevent.uobject, UVERBS_LOOKUP_READ); - if (rcq && rcq != scq) + if (!IS_ERR_OR_NULL(rcq) && rcq != scq) rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject, UVERBS_LOOKUP_READ); - if (srq) + if (!IS_ERR_OR_NULL(srq)) rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, UVERBS_LOOKUP_READ); - if (ind_tbl) + if (!IS_ERR_OR_NULL(ind_tbl)) uobj_put_obj_read(ind_tbl); uobj_alloc_abort(&obj->uevent.uobject, attrs); @@ -1655,8 +1663,8 @@ static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs) } qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); - if (!qp) { - ret = -EINVAL; + if (IS_ERR(qp)) { + ret = PTR_ERR(qp); goto out; } @@ -1761,8 +1769,8 @@ static int modify_qp(struct uverbs_attr_bundle *attrs, qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, attrs); - if (!qp) { - ret = -EINVAL; + if (IS_ERR(qp)) { + ret = PTR_ERR(qp); goto out; } @@ -2027,8 +2035,8 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) return -ENOMEM; qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); - if (!qp) { - ret = -EINVAL; + if (IS_ERR(qp)) { + ret = PTR_ERR(qp); goto out; } @@ -2065,9 +2073,9 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH, user_wr->wr.ud.ah, attrs); - if (!ud->ah) { + if (IS_ERR(ud->ah)) { + ret = PTR_ERR(ud->ah); kfree(ud); - ret = -EINVAL; goto out_put; } ud->remote_qpn = user_wr->wr.ud.remote_qpn; @@ -2304,8 +2312,8 @@ static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs) return PTR_ERR(wr); qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); - if (!qp) { - ret = -EINVAL; + if (IS_ERR(qp)) { + ret = PTR_ERR(qp); goto out; } @@ -2355,8 +2363,8 @@ static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs) return PTR_ERR(wr); srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); - if (!srq) { - ret = -EINVAL; + if (IS_ERR(srq)) { + ret = PTR_ERR(srq); goto out; } @@ -2412,8 +2420,8 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs) } pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); - if (!pd) { - ret = -EINVAL; + if (IS_ERR(pd)) { + ret = PTR_ERR(pd); goto err; } @@ -2482,8 +2490,8 @@ static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs) return ret; qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); - if (!qp) - return -EINVAL; + if (IS_ERR(qp)) + return PTR_ERR(qp); obj = qp->uobject; @@ -2532,8 +2540,8 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) return ret; qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); - if (!qp) - return -EINVAL; + if (IS_ERR(qp)) + return PTR_ERR(qp); obj = qp->uobject; mutex_lock(&obj->mcast_lock); @@ -2667,8 +2675,8 @@ static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs, UVERBS_OBJECT_FLOW_ACTION, kern_spec->action.handle, attrs); - if (!ib_spec->action.act) - return -EINVAL; + if (IS_ERR(ib_spec->action.act)) + return PTR_ERR(ib_spec->action.act); ib_spec->action.size = sizeof(struct ib_flow_spec_action_handle); flow_resources_add(uflow_res, @@ -2685,8 +2693,8 @@ static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs, UVERBS_OBJECT_COUNTERS, kern_spec->flow_count.handle, attrs); - if (!ib_spec->flow_count.counters) - return -EINVAL; + if (IS_ERR(ib_spec->flow_count.counters)) + return PTR_ERR(ib_spec->flow_count.counters); ib_spec->flow_count.size = sizeof(struct ib_flow_spec_action_count); flow_resources_add(uflow_res, @@ -2904,14 +2912,14 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) return PTR_ERR(obj); pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); - if (!pd) { - err = -EINVAL; + if (IS_ERR(pd)) { + err = PTR_ERR(pd); goto err_uobj; } cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); - if (!cq) { - err = -EINVAL; + if (IS_ERR(cq)) { + err = PTR_ERR(cq); goto err_put_pd; } @@ -3012,8 +3020,8 @@ static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs) return -EINVAL; wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs); - if (!wq) - return -EINVAL; + if (IS_ERR(wq)) + return PTR_ERR(wq); if (cmd.attr_mask & IB_WQ_FLAGS) { wq_attr.flags = cmd.flags; @@ -3096,8 +3104,8 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) num_read_wqs++) { wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, wqs_handles[num_read_wqs], attrs); - if (!wq) { - err = -EINVAL; + if (IS_ERR(wq)) { + err = PTR_ERR(wq); goto put_wqs; } @@ -3252,8 +3260,8 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) } qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); - if (!qp) { - err = -EINVAL; + if (IS_ERR(qp)) { + err = PTR_ERR(qp); goto err_uobj; } @@ -3399,15 +3407,15 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, if (ib_srq_has_cq(cmd->srq_type)) { attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd->cq_handle, attrs); - if (!attr.ext.cq) { - ret = -EINVAL; + if (IS_ERR(attr.ext.cq)) { + ret = PTR_ERR(attr.ext.cq); goto err_put_xrcd; } } pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs); - if (!pd) { - ret = -EINVAL; + if (IS_ERR(pd)) { + ret = PTR_ERR(pd); goto err_put_cq; } @@ -3514,8 +3522,8 @@ static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs) return ret; srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); - if (!srq) - return -EINVAL; + if (IS_ERR(srq)) + return PTR_ERR(srq); attr.max_wr = cmd.max_wr; attr.srq_limit = cmd.srq_limit; @@ -3542,8 +3550,8 @@ static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs) return ret; srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); - if (!srq) - return -EINVAL; + if (IS_ERR(srq)) + return PTR_ERR(srq); ret = ib_query_srq(srq, &attr); @@ -3668,8 +3676,8 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs) return -EOPNOTSUPP; cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); - if (!cq) - return -EINVAL; + if (IS_ERR(cq)) + return PTR_ERR(cq); ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period); diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index ba05de0380e96..6567d43751280 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -3029,22 +3029,23 @@ EXPORT_SYMBOL(__rdma_block_iter_start); bool __rdma_block_iter_next(struct ib_block_iter *biter) { unsigned int block_offset; - unsigned int sg_delta; + unsigned int delta; if (!biter->__sg_nents || !biter->__sg) return false; biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance; block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1); - sg_delta = BIT_ULL(biter->__pg_bit) - block_offset; + delta = BIT_ULL(biter->__pg_bit) - block_offset; - if (sg_dma_len(biter->__sg) - biter->__sg_advance > sg_delta) { - biter->__sg_advance += sg_delta; - } else { + while (biter->__sg_nents && biter->__sg && + sg_dma_len(biter->__sg) - biter->__sg_advance <= delta) { + delta -= sg_dma_len(biter->__sg) - biter->__sg_advance; biter->__sg_advance = 0; biter->__sg = sg_next(biter->__sg); biter->__sg_nents--; } + biter->__sg_advance += delta; return true; } diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index f7345e4890a14..31fff5885f1a8 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1823,7 +1823,6 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); struct bnxt_re_dev *rdev = srq->rdev; - int rc; switch (srq_attr_mask) { case IB_SRQ_MAX_WR: @@ -1835,11 +1834,8 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, return -EINVAL; srq->qplib_srq.threshold = srq_attr->srq_limit; - rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); - if (rc) { - ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!"); - return rc; - } + bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold); + /* On success, update the shadow */ srq->srq_limit = srq_attr->srq_limit; /* No need to Build and send response back to udata */ diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 68ea4ed0b171b..c19dd732c2354 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -685,9 +685,7 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, srq->dbinfo.db = srq->dpi->dbr; srq->dbinfo.max_slot = 1; srq->dbinfo.priv_db = res->dpi_tbl.priv_db; - if (srq->threshold) - bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); - srq->arm_req = false; + bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); return 0; fail: @@ -697,24 +695,6 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, return rc; } -int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, - struct bnxt_qplib_srq *srq) -{ - struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; - u32 count; - - count = __bnxt_qplib_get_avail(srq_hwq); - if (count > srq->threshold) { - srq->arm_req = false; - bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold); - } else { - /* Deferred arming */ - srq->arm_req = true; - } - - return 0; -} - int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq) { @@ -756,7 +736,6 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; struct rq_wqe *srqe; struct sq_sge *hw_sge; - u32 count = 0; int i, next; spin_lock(&srq_hwq->lock); @@ -788,15 +767,8 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot); - spin_lock(&srq_hwq->lock); - count = __bnxt_qplib_get_avail(srq_hwq); - spin_unlock(&srq_hwq->lock); /* Ring DB */ bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ); - if (srq->arm_req == true && count > srq->threshold) { - srq->arm_req = false; - bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold); - } return 0; } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h index 55fd840359ef2..288196facfd7c 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h @@ -519,8 +519,6 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, srqn_handler_t srq_handler); int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq); -int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, - struct bnxt_qplib_srq *srq); int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq); void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 96ceec1e8199a..77da7cf344274 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -121,6 +121,7 @@ static int __alloc_pbl(struct bnxt_qplib_res *res, pbl->pg_arr = vmalloc_array(pages, sizeof(void *)); if (!pbl->pg_arr) return -ENOMEM; + memset(pbl->pg_arr, 0, pages * sizeof(void *)); pbl->pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t)); if (!pbl->pg_map_arr) { @@ -128,6 +129,7 @@ static int __alloc_pbl(struct bnxt_qplib_res *res, pbl->pg_arr = NULL; return -ENOMEM; } + memset(pbl->pg_map_arr, 0, pages * sizeof(dma_addr_t)); pbl->pg_count = 0; pbl->pg_size = sginfo->pgsize; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index b010c4209ea38..e990690d8b3cd 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -585,7 +585,8 @@ static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev, static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt) { - dma_unmap_sg(&dev->pdev->dev, mtt->sglist, mtt->nsg, DMA_TO_DEVICE); + dma_unmap_sg(&dev->pdev->dev, mtt->sglist, + DIV_ROUND_UP(mtt->size, PAGE_SIZE), DMA_TO_DEVICE); vfree(mtt->sglist); } @@ -978,7 +979,9 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, if (ret) goto err_out_cmd; } else { - init_kernel_qp(dev, qp, attrs); + ret = init_kernel_qp(dev, qp, attrs); + if (ret) + goto err_out_xa; } qp->attrs.max_send_sge = attrs->cap.max_send_sge; diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c index bbc957c578e1e..e5db39f4720de 100644 --- a/drivers/infiniband/hw/hfi1/affinity.c +++ b/drivers/infiniband/hw/hfi1/affinity.c @@ -964,31 +964,35 @@ static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask, struct hfi1_affinity_node_list *affinity) { int possible, curr_cpu, i; - uint num_cores_per_socket = node_affinity.num_online_cpus / + uint num_cores_per_socket; + + cpumask_copy(hw_thread_mask, &affinity->proc.mask); + + if (affinity->num_core_siblings == 0) + return; + + num_cores_per_socket = node_affinity.num_online_cpus / affinity->num_core_siblings / node_affinity.num_online_nodes; - cpumask_copy(hw_thread_mask, &affinity->proc.mask); - if (affinity->num_core_siblings > 0) { - /* Removing other siblings not needed for now */ - possible = cpumask_weight(hw_thread_mask); - curr_cpu = cpumask_first(hw_thread_mask); - for (i = 0; - i < num_cores_per_socket * node_affinity.num_online_nodes; - i++) - curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); - - for (; i < possible; i++) { - cpumask_clear_cpu(curr_cpu, hw_thread_mask); - curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); - } + /* Removing other siblings not needed for now */ + possible = cpumask_weight(hw_thread_mask); + curr_cpu = cpumask_first(hw_thread_mask); + for (i = 0; + i < num_cores_per_socket * node_affinity.num_online_nodes; + i++) + curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); - /* Identifying correct HW threads within physical cores */ - cpumask_shift_left(hw_thread_mask, hw_thread_mask, - num_cores_per_socket * - node_affinity.num_online_nodes * - hw_thread_no); + for (; i < possible; i++) { + cpumask_clear_cpu(curr_cpu, hw_thread_mask); + curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); } + + /* Identifying correct HW threads within physical cores */ + cpumask_shift_left(hw_thread_mask, hw_thread_mask, + num_cores_per_socket * + node_affinity.num_online_nodes * + hw_thread_no); } int hfi1_get_proc_affinity(int node) diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 3df032ddda189..e99890e0c8c37 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -33,7 +33,6 @@ #include #include #include -#include "hnae3.h" #include "hns_roce_device.h" #include "hns_roce_hw_v2.h" diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index aded0a7f42838..f1d4494c7d008 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -42,7 +42,6 @@ #include #include -#include "hnae3.h" #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_cmd.h" @@ -162,6 +161,8 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ, to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); hr_reg_clear(fseg, FRMR_BLK_MODE); + hr_reg_clear(fseg, FRMR_BLOCK_SIZE); + hr_reg_clear(fseg, FRMR_ZBVA); } static void set_atomic_seg(const struct ib_send_wr *wr, @@ -336,9 +337,6 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, int j = 0; int i; - hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX, - (*sge_ind) & (qp->sge.sge_cnt - 1)); - hr_reg_write(rc_sq_wqe, RC_SEND_WQE_INLINE, !!(wr->send_flags & IB_SEND_INLINE)); if (wr->send_flags & IB_SEND_INLINE) @@ -587,6 +585,9 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, hr_reg_write(rc_sq_wqe, RC_SEND_WQE_CQE, (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); + hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX, + curr_idx & (qp->sge.sge_cnt - 1)); + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { if (msg_len != ATOMIC_WR_LEN) @@ -735,6 +736,9 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, owner_bit = ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); + /* RC and UD share the same DirectWQE field layout */ + ((struct hns_roce_v2_rc_send_wqe *)wqe)->byte_4 = 0; + /* Corresponding to the QP type, wqe process separately */ if (ibqp->qp_type == IB_QPT_RC) ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit); @@ -5193,11 +5197,10 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct hns_roce_v2_qp_context ctx[2]; - struct hns_roce_v2_qp_context *context = ctx; - struct hns_roce_v2_qp_context *qpc_mask = ctx + 1; + struct hns_roce_v2_qp_context *context; + struct hns_roce_v2_qp_context *qpc_mask; struct ib_device *ibdev = &hr_dev->ib_dev; - int ret; + int ret = -ENOMEM; if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) return -EOPNOTSUPP; @@ -5208,7 +5211,11 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, * we should set all bits of the relevant fields in context mask to * 0 at the same time, else set them to 0x1. */ - memset(context, 0, hr_dev->caps.qpc_sz); + context = kvzalloc(sizeof(*context), GFP_KERNEL); + qpc_mask = kvzalloc(sizeof(*qpc_mask), GFP_KERNEL); + if (!context || !qpc_mask) + goto out; + memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz); ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state, @@ -5250,6 +5257,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, clear_qp(hr_qp); out: + kvfree(qpc_mask); + kvfree(context); return ret; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index b8e17721f6fde..7875283eb9d63 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -34,6 +34,7 @@ #define _HNS_ROCE_HW_V2_H #include +#include "hnae3.h" #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 #define HNS_ROCE_V2_MTT_ENTRY_SZ 64 diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index a7e4c951f8fe4..5f39a25064d10 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -37,7 +37,6 @@ #include #include #include -#include "hnae3.h" #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_hem.h" diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 0cad6fc7bf32c..26784b296ffa6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -654,7 +654,6 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev, hr_qp->sq.wqe_shift = ucmd->log_sq_stride; hr_qp->sq.wqe_cnt = cnt; - cap->max_send_sge = hr_qp->sq.max_gs; return 0; } @@ -736,7 +735,6 @@ static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, /* sync the parameters of kernel QP to user's configuration */ cap->max_send_wr = cnt; - cap->max_send_sge = hr_qp->sq.max_gs; return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c index 081a01de30553..1fb5e24683647 100644 --- a/drivers/infiniband/hw/hns/hns_roce_restrack.c +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c @@ -4,7 +4,6 @@ #include #include #include -#include "hnae3.h" #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_hw_v2.h" diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c index c0bef11436b94..fa096557adc83 100644 --- a/drivers/infiniband/hw/irdma/pble.c +++ b/drivers/infiniband/hw/irdma/pble.c @@ -71,7 +71,7 @@ int irdma_hmc_init_pble(struct irdma_sc_dev *dev, static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, struct sd_pd_idx *idx) { - idx->sd_idx = (u32)pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE; + idx->sd_idx = pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE; idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE); idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD); } diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 38cecb28d322e..29540b2b2373c 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -2077,6 +2077,7 @@ static int irdma_create_cq(struct ib_cq *ibcq, spin_lock_init(&iwcq->lock); INIT_LIST_HEAD(&iwcq->resize_list); INIT_LIST_HEAD(&iwcq->cmpl_generated); + iwcq->cq_num = cq_num; info.dev = dev; ukinfo->cq_size = max(entries, 4); ukinfo->cq_id = cq_num; @@ -2115,8 +2116,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, goto cq_free_rsrc; } - iwcq->iwpbl = iwpbl; - iwcq->cq_mem_size = 0; cqmr = &iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & @@ -2131,7 +2130,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, err_code = -EPROTO; goto cq_free_rsrc; } - iwcq->iwpbl_shadow = iwpbl_shadow; cqmr_shadow = &iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; cqmr->split = true; diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index 20297a14c9a61..bb9ab945938e0 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -113,21 +113,15 @@ struct irdma_mr { struct irdma_cq { struct ib_cq ibcq; struct irdma_sc_cq sc_cq; - u16 cq_head; - u16 cq_size; - u16 cq_num; + u32 cq_num; bool user_mode; atomic_t armed; enum irdma_cmpl_notify last_notify; - u32 polled_cmpls; - u32 cq_mem_size; struct irdma_dma_mem kmem; struct irdma_dma_mem kmem_shadow; struct completion free_cq; refcount_t refcnt; spinlock_t lock; /* for poll cq */ - struct irdma_pbl *iwpbl; - struct irdma_pbl *iwpbl_shadow; struct list_head resize_list; struct irdma_cq_poll_info cur_cqe; struct list_head cmpl_generated; diff --git a/drivers/infiniband/hw/mlx5/counters.c b/drivers/infiniband/hw/mlx5/counters.c index b049bba215790..d06128501ce4e 100644 --- a/drivers/infiniband/hw/mlx5/counters.c +++ b/drivers/infiniband/hw/mlx5/counters.c @@ -387,7 +387,7 @@ static int do_get_hw_stats(struct ib_device *ibdev, return ret; /* We don't expose device counters over Vports */ - if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0) + if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0) goto done; if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { @@ -407,7 +407,7 @@ static int do_get_hw_stats(struct ib_device *ibdev, */ goto done; } - ret = mlx5_lag_query_cong_counters(dev->mdev, + ret = mlx5_lag_query_cong_counters(mdev, stats->value + cnts->num_q_counters, cnts->num_cong_counters, diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 6e19974ecf6e7..388c95562a927 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -191,6 +191,7 @@ static u16 get_legacy_obj_type(u16 opcode) { switch (opcode) { case MLX5_CMD_OP_CREATE_RQ: + case MLX5_CMD_OP_CREATE_RMP: return MLX5_EVENT_QUEUE_TYPE_RQ; case MLX5_CMD_OP_CREATE_QP: return MLX5_EVENT_QUEUE_TYPE_QP; @@ -1914,6 +1915,7 @@ subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table, /* Level1 is valid for future use, no need to free */ return -ENOMEM; + INIT_LIST_HEAD(&obj_event->obj_sub_list); err = xa_insert(&event->object_ids, key_level2, obj_event, @@ -1922,7 +1924,6 @@ subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table, kfree(obj_event); return err; } - INIT_LIST_HEAD(&obj_event->obj_sub_list); } return 0; diff --git a/drivers/infiniband/hw/mlx5/dm.c b/drivers/infiniband/hw/mlx5/dm.c index 3669c90b2dadc..672e5cfd2fca5 100644 --- a/drivers/infiniband/hw/mlx5/dm.c +++ b/drivers/infiniband/hw/mlx5/dm.c @@ -282,7 +282,7 @@ static struct ib_dm *handle_alloc_dm_memic(struct ib_ucontext *ctx, int err; u64 address; - if (!MLX5_CAP_DEV_MEM(dm_db->dev, memic)) + if (!dm_db || !MLX5_CAP_DEV_MEM(dm_db->dev, memic)) return ERR_PTR(-EOPNOTSUPP); dm = kzalloc(sizeof(*dm), GFP_KERNEL); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index ada7dbf8eb1cf..0adb98b8dacc8 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1690,10 +1690,45 @@ static void deallocate_uars(struct mlx5_ib_dev *dev, context->devx_uid); } +static int mlx5_ib_enable_lb_mp(struct mlx5_core_dev *master, + struct mlx5_core_dev *slave, + struct mlx5_ib_lb_state *lb_state) +{ + int err; + + err = mlx5_nic_vport_update_local_lb(master, true); + if (err) + return err; + + err = mlx5_nic_vport_update_local_lb(slave, true); + if (err) + goto out; + + lb_state->force_enable = true; + return 0; + +out: + mlx5_nic_vport_update_local_lb(master, false); + return err; +} + +static void mlx5_ib_disable_lb_mp(struct mlx5_core_dev *master, + struct mlx5_core_dev *slave, + struct mlx5_ib_lb_state *lb_state) +{ + mlx5_nic_vport_update_local_lb(slave, false); + mlx5_nic_vport_update_local_lb(master, false); + + lb_state->force_enable = false; +} + int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) { int err = 0; + if (dev->lb.force_enable) + return 0; + mutex_lock(&dev->lb.mutex); if (td) dev->lb.user_td++; @@ -1715,6 +1750,9 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) { + if (dev->lb.force_enable) + return; + mutex_lock(&dev->lb.mutex); if (td) dev->lb.user_td--; @@ -3224,6 +3262,8 @@ static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev, lockdep_assert_held(&mlx5_ib_multiport_mutex); + mlx5_ib_disable_lb_mp(ibdev->mdev, mpi->mdev, &ibdev->lb); + mlx5_core_mp_event_replay(ibdev->mdev, MLX5_DRIVER_EVENT_AFFILIATION_REMOVED, NULL); @@ -3319,6 +3359,10 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev, MLX5_DRIVER_EVENT_AFFILIATION_DONE, &key); + err = mlx5_ib_enable_lb_mp(ibdev->mdev, mpi->mdev, &ibdev->lb); + if (err) + goto unbind; + return true; unbind: diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 94678e5c59dd5..9b780f48a382b 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1065,6 +1065,7 @@ struct mlx5_ib_lb_state { u32 user_td; int qps; bool enabled; + bool force_enable; }; struct mlx5_ib_pf_eq { diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c index d9cf6982d645e..9de9bea1025e2 100644 --- a/drivers/infiniband/hw/mlx5/qpc.c +++ b/drivers/infiniband/hw/mlx5/qpc.c @@ -21,8 +21,10 @@ mlx5_get_rsc(struct mlx5_qp_table *table, u32 rsn) spin_lock_irqsave(&table->lock, flags); common = radix_tree_lookup(&table->tree, rsn); - if (common) + if (common && !common->invalid) refcount_inc(&common->refcount); + else + common = NULL; spin_unlock_irqrestore(&table->lock, flags); @@ -178,6 +180,18 @@ static int create_resource_common(struct mlx5_ib_dev *dev, return 0; } +static void modify_resource_common_state(struct mlx5_ib_dev *dev, + struct mlx5_core_qp *qp, + bool invalid) +{ + struct mlx5_qp_table *table = &dev->qp_table; + unsigned long flags; + + spin_lock_irqsave(&table->lock, flags); + qp->common.invalid = invalid; + spin_unlock_irqrestore(&table->lock, flags); +} + static void destroy_resource_common(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp) { @@ -604,8 +618,20 @@ int mlx5_core_create_rq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen, int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev, struct mlx5_core_qp *rq) { + int ret; + + /* The rq destruction can be called again in case it fails, hence we + * mark the common resource as invalid and only once FW destruction + * is completed successfully we actually destroy the resources. + */ + modify_resource_common_state(dev, rq, true); + ret = destroy_rq_tracked(dev, rq->qpn, rq->uid); + if (ret) { + modify_resource_common_state(dev, rq, false); + return ret; + } destroy_resource_common(dev, rq); - return destroy_rq_tracked(dev, rq->qpn, rq->uid); + return 0; } static void destroy_sq_tracked(struct mlx5_ib_dev *dev, u32 sqn, u16 uid) diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c index fec87c9030abd..fffd144d509eb 100644 --- a/drivers/infiniband/sw/rxe/rxe_cq.c +++ b/drivers/infiniband/sw/rxe/rxe_cq.c @@ -56,11 +56,8 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, err = do_mmap_info(rxe, uresp ? &uresp->mi : NULL, udata, cq->queue->buf, cq->queue->buf_size, &cq->queue->ip); - if (err) { - vfree(cq->queue->buf); - kfree(cq->queue); + if (err) return err; - } cq->is_user = uresp; diff --git a/drivers/infiniband/sw/rxe/rxe_task.c b/drivers/infiniband/sw/rxe/rxe_task.c index 80332638d9e3a..be6cd8ce4d97e 100644 --- a/drivers/infiniband/sw/rxe/rxe_task.c +++ b/drivers/infiniband/sw/rxe/rxe_task.c @@ -132,8 +132,12 @@ static void do_task(struct rxe_task *task) * yield the cpu and reschedule the task */ if (!ret) { - task->state = TASK_STATE_IDLE; - resched = 1; + if (task->state != TASK_STATE_DRAINING) { + task->state = TASK_STATE_IDLE; + resched = 1; + } else { + cont = 1; + } goto exit; } diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c index feae920784be8..03e3f6668840f 100644 --- a/drivers/infiniband/sw/siw/siw_qp_tx.c +++ b/drivers/infiniband/sw/siw/siw_qp_tx.c @@ -340,18 +340,17 @@ static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset, if (!sendpage_ok(page[i])) msg.msg_flags &= ~MSG_SPLICE_PAGES; bvec_set_page(&bvec, page[i], bytes, offset); - iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size); + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, bytes); try_page_again: lock_sock(sk); - rv = tcp_sendmsg_locked(sk, &msg, size); + rv = tcp_sendmsg_locked(sk, &msg, bytes); release_sock(sk); if (rv > 0) { size -= rv; sent += rv; if (rv != bytes) { - offset += rv; bytes -= rv; goto try_page_again; } diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index fdbef3254e308..1768b8695c45a 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -761,7 +761,7 @@ int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr, struct siw_wqe *wqe = tx_wqe(qp); unsigned long flags; - int rv = 0; + int rv = 0, imm_err = 0; if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) { siw_dbg_qp(qp, "wr must be empty for user mapped sq\n"); @@ -947,9 +947,17 @@ int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr, * Send directly if SQ processing is not in progress. * Eventual immediate errors (rv < 0) do not affect the involved * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ - * processing, if new work is already pending. But rv must be passed - * to caller. + * processing, if new work is already pending. But rv and pointer + * to failed work request must be passed to caller. */ + if (unlikely(rv < 0)) { + /* + * Immediate error + */ + siw_dbg_qp(qp, "Immediate error %d\n", rv); + imm_err = rv; + *bad_wr = wr; + } if (wqe->wr_status != SIW_WR_IDLE) { spin_unlock_irqrestore(&qp->sq_lock, flags); goto skip_direct_sending; @@ -974,15 +982,10 @@ int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr, up_read(&qp->state_lock); - if (rv >= 0) - return 0; - /* - * Immediate error - */ - siw_dbg_qp(qp, "error %d\n", rv); + if (unlikely(imm_err)) + return imm_err; - *bad_wr = wr; - return rv; + return (rv >= 0) ? 0 : rv; } /* diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index b91467c8e6c40..09fcc14051f28 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -77,12 +77,13 @@ * xbox d-pads should map to buttons, as is required for DDR pads * but we map them to axes when possible to simplify things */ -#define MAP_DPAD_TO_BUTTONS (1 << 0) -#define MAP_TRIGGERS_TO_BUTTONS (1 << 1) -#define MAP_STICKS_TO_NULL (1 << 2) -#define MAP_SELECT_BUTTON (1 << 3) -#define MAP_PADDLES (1 << 4) -#define MAP_PROFILE_BUTTON (1 << 5) +#define MAP_DPAD_TO_BUTTONS BIT(0) +#define MAP_TRIGGERS_TO_BUTTONS BIT(1) +#define MAP_STICKS_TO_NULL BIT(2) +#define MAP_SHARE_BUTTON BIT(3) +#define MAP_PADDLES BIT(4) +#define MAP_PROFILE_BUTTON BIT(5) +#define MAP_SHARE_OFFSET BIT(6) #define DANCEPAD_MAP_CONFIG (MAP_DPAD_TO_BUTTONS | \ MAP_TRIGGERS_TO_BUTTONS | MAP_STICKS_TO_NULL) @@ -135,7 +136,7 @@ static const struct xpad_device { { 0x03f0, 0x048D, "HyperX Clutch", 0, XTYPE_XBOX360 }, /* wireless */ { 0x03f0, 0x0495, "HyperX Clutch Gladiate", 0, XTYPE_XBOXONE }, { 0x03f0, 0x07A0, "HyperX Clutch Gladiate RGB", 0, XTYPE_XBOXONE }, - { 0x03f0, 0x08B6, "HyperX Clutch Gladiate", 0, XTYPE_XBOXONE }, /* v2 */ + { 0x03f0, 0x08B6, "HyperX Clutch Gladiate", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, /* v2 */ { 0x03f0, 0x09B4, "HyperX Clutch Tanto", 0, XTYPE_XBOXONE }, { 0x044f, 0x0f00, "Thrustmaster Wheel", 0, XTYPE_XBOX }, { 0x044f, 0x0f03, "Thrustmaster Wheel", 0, XTYPE_XBOX }, @@ -159,7 +160,7 @@ static const struct xpad_device { { 0x045e, 0x0719, "Xbox 360 Wireless Receiver", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W }, { 0x045e, 0x0b00, "Microsoft X-Box One Elite 2 pad", MAP_PADDLES, XTYPE_XBOXONE }, { 0x045e, 0x0b0a, "Microsoft X-Box Adaptive Controller", MAP_PROFILE_BUTTON, XTYPE_XBOXONE }, - { 0x045e, 0x0b12, "Microsoft Xbox Series S|X Controller", MAP_SELECT_BUTTON, XTYPE_XBOXONE }, + { 0x045e, 0x0b12, "Microsoft Xbox Series S|X Controller", MAP_SHARE_BUTTON | MAP_SHARE_OFFSET, XTYPE_XBOXONE }, { 0x046d, 0xc21d, "Logitech Gamepad F310", 0, XTYPE_XBOX360 }, { 0x046d, 0xc21e, "Logitech Gamepad F510", 0, XTYPE_XBOX360 }, { 0x046d, 0xc21f, "Logitech Gamepad F710", 0, XTYPE_XBOX360 }, @@ -168,6 +169,7 @@ static const struct xpad_device { { 0x046d, 0xca88, "Logitech Compact Controller for Xbox", 0, XTYPE_XBOX }, { 0x046d, 0xca8a, "Logitech Precision Vibration Feedback Wheel", 0, XTYPE_XBOX }, { 0x046d, 0xcaa3, "Logitech DriveFx Racing Wheel", 0, XTYPE_XBOX360 }, + { 0x0502, 0x1305, "Acer NGR200", 0, XTYPE_XBOX360 }, { 0x056e, 0x2004, "Elecom JC-U3613M", 0, XTYPE_XBOX360 }, { 0x05fd, 0x1007, "Mad Catz Controller (unverified)", 0, XTYPE_XBOX }, { 0x05fd, 0x107a, "InterAct 'PowerPad Pro' X-Box pad (Germany)", 0, XTYPE_XBOX }, @@ -205,13 +207,13 @@ static const struct xpad_device { { 0x0738, 0x9871, "Mad Catz Portable Drum", 0, XTYPE_XBOX360 }, { 0x0738, 0xb726, "Mad Catz Xbox controller - MW2", 0, XTYPE_XBOX360 }, { 0x0738, 0xb738, "Mad Catz MVC2TE Stick 2", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, - { 0x0738, 0xbeef, "Mad Catz JOYTECH NEO SE Advanced GamePad", XTYPE_XBOX360 }, + { 0x0738, 0xbeef, "Mad Catz JOYTECH NEO SE Advanced GamePad", 0, XTYPE_XBOX360 }, { 0x0738, 0xcb02, "Saitek Cyborg Rumble Pad - PC/Xbox 360", 0, XTYPE_XBOX360 }, { 0x0738, 0xcb03, "Saitek P3200 Rumble Pad - PC/Xbox 360", 0, XTYPE_XBOX360 }, { 0x0738, 0xcb29, "Saitek Aviator Stick AV8R02", 0, XTYPE_XBOX360 }, { 0x0738, 0xf738, "Super SFIV FightStick TE S", 0, XTYPE_XBOX360 }, { 0x07ff, 0xffff, "Mad Catz GamePad", 0, XTYPE_XBOX360 }, - { 0x0b05, 0x1a38, "ASUS ROG RAIKIRI", 0, XTYPE_XBOXONE }, + { 0x0b05, 0x1a38, "ASUS ROG RAIKIRI", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, { 0x0b05, 0x1abb, "ASUS ROG RAIKIRI PRO", 0, XTYPE_XBOXONE }, { 0x0c12, 0x0005, "Intec wireless", 0, XTYPE_XBOX }, { 0x0c12, 0x8801, "Nyko Xbox Controller", 0, XTYPE_XBOX }, @@ -240,7 +242,7 @@ static const struct xpad_device { { 0x0e6f, 0x0146, "Rock Candy Wired Controller for Xbox One", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x0147, "PDP Marvel Xbox One Controller", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x015c, "PDP Xbox One Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, - { 0x0e6f, 0x015d, "PDP Mirror's Edge Official Wired Controller for Xbox One", XTYPE_XBOXONE }, + { 0x0e6f, 0x015d, "PDP Mirror's Edge Official Wired Controller for Xbox One", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x0161, "PDP Xbox One Controller", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x0162, "PDP Xbox One Controller", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x0163, "PDP Xbox One Controller", 0, XTYPE_XBOXONE }, @@ -288,6 +290,8 @@ static const struct xpad_device { { 0x1038, 0x1430, "SteelSeries Stratus Duo", 0, XTYPE_XBOX360 }, { 0x1038, 0x1431, "SteelSeries Stratus Duo", 0, XTYPE_XBOX360 }, { 0x10f5, 0x7005, "Turtle Beach Recon Controller", 0, XTYPE_XBOXONE }, + { 0x10f5, 0x7008, "Turtle Beach Recon Controller", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, + { 0x10f5, 0x7073, "Turtle Beach Stealth Ultra Controller", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, { 0x11c9, 0x55f0, "Nacon GC-100XF", 0, XTYPE_XBOX360 }, { 0x11ff, 0x0511, "PXN V900", 0, XTYPE_XBOX360 }, { 0x1209, 0x2882, "Ardwiino Controller", 0, XTYPE_XBOX360 }, @@ -352,6 +356,7 @@ static const struct xpad_device { { 0x1ee9, 0x1590, "ZOTAC Gaming Zone", 0, XTYPE_XBOX360 }, { 0x20d6, 0x2001, "BDA Xbox Series X Wired Controller", 0, XTYPE_XBOXONE }, { 0x20d6, 0x2009, "PowerA Enhanced Wired Controller for Xbox Series X|S", 0, XTYPE_XBOXONE }, + { 0x20d6, 0x2064, "PowerA Wired Controller for Xbox", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, { 0x20d6, 0x281f, "PowerA Wired Controller For Xbox 360", 0, XTYPE_XBOX360 }, { 0x24c6, 0x5000, "Razer Atrox Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, { 0x24c6, 0x5300, "PowerA MINI PROEX Controller", 0, XTYPE_XBOX360 }, @@ -386,10 +391,11 @@ static const struct xpad_device { { 0x2dc8, 0x3106, "8BitDo Ultimate Wireless / Pro 2 Wired Controller", 0, XTYPE_XBOX360 }, { 0x2dc8, 0x3109, "8BitDo Ultimate Wireless Bluetooth", 0, XTYPE_XBOX360 }, { 0x2dc8, 0x310a, "8BitDo Ultimate 2C Wireless Controller", 0, XTYPE_XBOX360 }, + { 0x2dc8, 0x310b, "8BitDo Ultimate 2 Wireless Controller", 0, XTYPE_XBOX360 }, { 0x2dc8, 0x6001, "8BitDo SN30 Pro", 0, XTYPE_XBOX360 }, { 0x2e24, 0x0652, "Hyperkin Duke X-Box One pad", 0, XTYPE_XBOXONE }, { 0x2e24, 0x1688, "Hyperkin X91 X-Box One pad", 0, XTYPE_XBOXONE }, - { 0x2e95, 0x0504, "SCUF Gaming Controller", MAP_SELECT_BUTTON, XTYPE_XBOXONE }, + { 0x2e95, 0x0504, "SCUF Gaming Controller", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, { 0x31e3, 0x1100, "Wooting One", 0, XTYPE_XBOX360 }, { 0x31e3, 0x1200, "Wooting Two", 0, XTYPE_XBOX360 }, { 0x31e3, 0x1210, "Wooting Lekker", 0, XTYPE_XBOX360 }, @@ -509,6 +515,7 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOX360_VENDOR(0x045e), /* Microsoft Xbox 360 controllers */ XPAD_XBOXONE_VENDOR(0x045e), /* Microsoft Xbox One controllers */ XPAD_XBOX360_VENDOR(0x046d), /* Logitech Xbox 360-style controllers */ + XPAD_XBOX360_VENDOR(0x0502), /* Acer Inc. Xbox 360 style controllers */ XPAD_XBOX360_VENDOR(0x056e), /* Elecom JC-U3613M */ XPAD_XBOX360_VENDOR(0x06a3), /* Saitek P3600 */ XPAD_XBOX360_VENDOR(0x0738), /* Mad Catz Xbox 360 controllers */ @@ -1025,7 +1032,7 @@ static void xpad360w_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned cha * The report format was gleaned from * https://github.com/kylelemons/xbox/blob/master/xbox.go */ -static void xpadone_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *data) +static void xpadone_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *data, u32 len) { struct input_dev *dev = xpad->dev; bool do_sync = false; @@ -1066,8 +1073,12 @@ static void xpadone_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char /* menu/view buttons */ input_report_key(dev, BTN_START, data[4] & BIT(2)); input_report_key(dev, BTN_SELECT, data[4] & BIT(3)); - if (xpad->mapping & MAP_SELECT_BUTTON) - input_report_key(dev, KEY_RECORD, data[22] & BIT(0)); + if (xpad->mapping & MAP_SHARE_BUTTON) { + if (xpad->mapping & MAP_SHARE_OFFSET) + input_report_key(dev, KEY_RECORD, data[len - 26] & BIT(0)); + else + input_report_key(dev, KEY_RECORD, data[len - 18] & BIT(0)); + } /* buttons A,B,X,Y */ input_report_key(dev, BTN_A, data[4] & BIT(4)); @@ -1215,7 +1226,7 @@ static void xpad_irq_in(struct urb *urb) xpad360w_process_packet(xpad, 0, xpad->idata); break; case XTYPE_XBOXONE: - xpadone_process_packet(xpad, 0, xpad->idata); + xpadone_process_packet(xpad, 0, xpad->idata, urb->actual_length); break; default: xpad_process_packet(xpad, 0, xpad->idata); @@ -1972,7 +1983,7 @@ static int xpad_init_input(struct usb_xpad *xpad) xpad->xtype == XTYPE_XBOXONE) { for (i = 0; xpad360_btn[i] >= 0; i++) input_set_capability(input_dev, EV_KEY, xpad360_btn[i]); - if (xpad->mapping & MAP_SELECT_BUTTON) + if (xpad->mapping & MAP_SHARE_BUTTON) input_set_capability(input_dev, EV_KEY, KEY_RECORD); } else { for (i = 0; xpad_btn[i] >= 0; i++) diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c index aad2d75c03678..14ead0fbb80fd 100644 --- a/drivers/input/keyboard/atkbd.c +++ b/drivers/input/keyboard/atkbd.c @@ -826,7 +826,7 @@ static int atkbd_probe(struct atkbd *atkbd) if (atkbd_skip_getid(atkbd)) { atkbd->id = 0xab83; - return 0; + goto deactivate_kbd; } /* @@ -863,6 +863,7 @@ static int atkbd_probe(struct atkbd *atkbd) return -1; } +deactivate_kbd: /* * Make sure nothing is coming from the keyboard and disturbs our * internal state. diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c index e7ecfca838df4..76fcf90ea7c76 100644 --- a/drivers/input/keyboard/cros_ec_keyb.c +++ b/drivers/input/keyboard/cros_ec_keyb.c @@ -263,6 +263,12 @@ static int cros_ec_keyb_work(struct notifier_block *nb, case EC_MKBP_EVENT_KEY_MATRIX: pm_wakeup_event(ckdev->dev, 0); + if (!ckdev->idev) { + dev_warn_once(ckdev->dev, + "Unexpected key matrix event\n"); + return NOTIFY_OK; + } + if (ckdev->ec->event_size != ckdev->cols) { dev_err(ckdev->dev, "Discarded incomplete key matrix event.\n"); diff --git a/drivers/input/keyboard/gpio_keys.c b/drivers/input/keyboard/gpio_keys.c index 2e7c2c046e675..a34b9533d8f18 100644 --- a/drivers/input/keyboard/gpio_keys.c +++ b/drivers/input/keyboard/gpio_keys.c @@ -455,6 +455,8 @@ static enum hrtimer_restart gpio_keys_irq_timer(struct hrtimer *t) release_timer); struct input_dev *input = bdata->input; + guard(spinlock_irqsave)(&bdata->lock); + if (bdata->key_pressed) { input_report_key(input, *bdata->code, 0); input_sync(input); @@ -493,7 +495,7 @@ static irqreturn_t gpio_keys_irq_isr(int irq, void *dev_id) if (bdata->release_delay) hrtimer_start(&bdata->release_timer, ms_to_ktime(bdata->release_delay), - HRTIMER_MODE_REL_HARD); + HRTIMER_MODE_REL); out: spin_unlock_irqrestore(&bdata->lock, flags); return IRQ_HANDLED; @@ -630,7 +632,7 @@ static int gpio_keys_setup_key(struct platform_device *pdev, bdata->release_delay = button->debounce_interval; hrtimer_init(&bdata->release_timer, - CLOCK_REALTIME, HRTIMER_MODE_REL_HARD); + CLOCK_REALTIME, HRTIMER_MODE_REL); bdata->release_timer.function = gpio_keys_irq_timer; isr = gpio_keys_irq_isr; diff --git a/drivers/input/keyboard/imx_sc_key.c b/drivers/input/keyboard/imx_sc_key.c index d18839f1f4f60..b620cd310cdb7 100644 --- a/drivers/input/keyboard/imx_sc_key.c +++ b/drivers/input/keyboard/imx_sc_key.c @@ -158,7 +158,7 @@ static int imx_sc_key_probe(struct platform_device *pdev) return error; } - error = devm_add_action_or_reset(&pdev->dev, imx_sc_key_action, &priv); + error = devm_add_action_or_reset(&pdev->dev, imx_sc_key_action, priv); if (error) return error; diff --git a/drivers/input/keyboard/mtk-pmic-keys.c b/drivers/input/keyboard/mtk-pmic-keys.c index 4364c3401ff1c..486ca8ff86f83 100644 --- a/drivers/input/keyboard/mtk-pmic-keys.c +++ b/drivers/input/keyboard/mtk-pmic-keys.c @@ -147,8 +147,8 @@ static void mtk_pmic_keys_lp_reset_setup(struct mtk_pmic_keys *keys, u32 value, mask; int error; - kregs_home = keys->keys[MTK_PMIC_HOMEKEY_INDEX].regs; - kregs_pwr = keys->keys[MTK_PMIC_PWRKEY_INDEX].regs; + kregs_home = ®s->keys_regs[MTK_PMIC_HOMEKEY_INDEX]; + kregs_pwr = ®s->keys_regs[MTK_PMIC_PWRKEY_INDEX]; error = of_property_read_u32(keys->dev->of_node, "power-off-time-sec", &long_press_debounce); diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c index 180d90e46061e..5cfe9b56978da 100644 --- a/drivers/input/misc/ims-pcu.c +++ b/drivers/input/misc/ims-pcu.c @@ -845,6 +845,12 @@ static int ims_pcu_flash_firmware(struct ims_pcu *pcu, addr = be32_to_cpu(rec->addr) / 2; len = be16_to_cpu(rec->len); + if (len > sizeof(pcu->cmd_buf) - 1 - sizeof(*fragment)) { + dev_err(pcu->dev, + "Invalid record length in firmware: %d\n", len); + return -EINVAL; + } + fragment = (void *)&pcu->cmd_buf[1]; put_unaligned_le32(addr, &fragment->addr); fragment->len = len; diff --git a/drivers/input/misc/iqs7222.c b/drivers/input/misc/iqs7222.c index b98529568eeb8..eb0e0b37eb41b 100644 --- a/drivers/input/misc/iqs7222.c +++ b/drivers/input/misc/iqs7222.c @@ -301,6 +301,7 @@ struct iqs7222_dev_desc { int allow_offset; int event_offset; int comms_offset; + int ext_chan; bool legacy_gesture; struct iqs7222_reg_grp_desc reg_grps[IQS7222_NUM_REG_GRPS]; }; @@ -315,6 +316,7 @@ static const struct iqs7222_dev_desc iqs7222_devs[] = { .allow_offset = 9, .event_offset = 10, .comms_offset = 12, + .ext_chan = 10, .reg_grps = { [IQS7222_REG_GRP_STAT] = { .base = IQS7222_SYS_STATUS, @@ -373,6 +375,7 @@ static const struct iqs7222_dev_desc iqs7222_devs[] = { .allow_offset = 9, .event_offset = 10, .comms_offset = 12, + .ext_chan = 10, .legacy_gesture = true, .reg_grps = { [IQS7222_REG_GRP_STAT] = { @@ -2244,7 +2247,7 @@ static int iqs7222_parse_chan(struct iqs7222_private *iqs7222, const struct iqs7222_dev_desc *dev_desc = iqs7222->dev_desc; struct i2c_client *client = iqs7222->client; int num_chan = dev_desc->reg_grps[IQS7222_REG_GRP_CHAN].num_row; - int ext_chan = rounddown(num_chan, 10); + int ext_chan = dev_desc->ext_chan ? : num_chan; int error, i; u16 *chan_setup = iqs7222->chan_setup[chan_index]; u16 *sys_setup = iqs7222->sys_setup; @@ -2427,6 +2430,9 @@ static int iqs7222_parse_chan(struct iqs7222_private *iqs7222, if (error) return error; + if (!iqs7222->kp_type[chan_index][i]) + continue; + if (!dev_desc->event_offset) continue; @@ -2448,7 +2454,7 @@ static int iqs7222_parse_sldr(struct iqs7222_private *iqs7222, const struct iqs7222_dev_desc *dev_desc = iqs7222->dev_desc; struct i2c_client *client = iqs7222->client; int num_chan = dev_desc->reg_grps[IQS7222_REG_GRP_CHAN].num_row; - int ext_chan = rounddown(num_chan, 10); + int ext_chan = dev_desc->ext_chan ? : num_chan; int count, error, reg_offset, i; u16 *event_mask = &iqs7222->sys_setup[dev_desc->event_offset]; u16 *sldr_setup = iqs7222->sldr_setup[sldr_index]; diff --git a/drivers/input/misc/sparcspkr.c b/drivers/input/misc/sparcspkr.c index e5dd84725c6e7..d715fd736426d 100644 --- a/drivers/input/misc/sparcspkr.c +++ b/drivers/input/misc/sparcspkr.c @@ -75,9 +75,14 @@ static int bbc_spkr_event(struct input_dev *dev, unsigned int type, unsigned int return -1; switch (code) { - case SND_BELL: if (value) value = 1000; - case SND_TONE: break; - default: return -1; + case SND_BELL: + if (value) + value = 1000; + break; + case SND_TONE: + break; + default: + return -1; } if (value > 20 && value < 32767) @@ -113,9 +118,14 @@ static int grover_spkr_event(struct input_dev *dev, unsigned int type, unsigned return -1; switch (code) { - case SND_BELL: if (value) value = 1000; - case SND_TONE: break; - default: return -1; + case SND_BELL: + if (value) + value = 1000; + break; + case SND_TONE: + break; + default: + return -1; } if (value > 20 && value < 32767) diff --git a/drivers/input/misc/uinput.c b/drivers/input/misc/uinput.c index 2c973f15cab7d..edb62a284548b 100644 --- a/drivers/input/misc/uinput.c +++ b/drivers/input/misc/uinput.c @@ -775,6 +775,7 @@ static int uinput_ff_upload_to_user(char __user *buffer, if (in_compat_syscall()) { struct uinput_ff_upload_compat ff_up_compat; + memset(&ff_up_compat, 0, sizeof(ff_up_compat)); ff_up_compat.request_id = ff_up->request_id; ff_up_compat.retval = ff_up->retval; /* diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index 26677432ac836..3ca6642601c7d 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -163,6 +163,7 @@ static const char * const topbuttonpad_pnp_ids[] = { static const char * const smbus_pnp_ids[] = { /* all of the topbuttonpad_pnp_ids are valid, we just add some extras */ + "DLL060d", /* Dell Precision M3800 */ "LEN0048", /* X1 Carbon 3 */ "LEN0046", /* X250 */ "LEN0049", /* Yoga 11e */ @@ -189,11 +190,15 @@ static const char * const smbus_pnp_ids[] = { "LEN2054", /* E480 */ "LEN2055", /* E580 */ "LEN2068", /* T14 Gen 1 */ + "SYN1221", /* TUXEDO InfinityBook Pro 14 v5 */ + "SYN3003", /* HP EliteBook 850 G1 */ "SYN3015", /* HP EliteBook 840 G2 */ "SYN3052", /* HP EliteBook 840 G4 */ "SYN3221", /* HP 15-ay000 */ "SYN323d", /* HP Spectre X360 13-w013dx */ "SYN3257", /* HP Envy 13-ad105ng */ + "TOS01f6", /* Dynabook Portege X30L-G */ + "TOS0213", /* Dynabook Portege X30-D */ NULL }; diff --git a/drivers/input/rmi4/rmi_f34.c b/drivers/input/rmi4/rmi_f34.c index 0d9a5756e3f59..cae1e41664921 100644 --- a/drivers/input/rmi4/rmi_f34.c +++ b/drivers/input/rmi4/rmi_f34.c @@ -4,6 +4,7 @@ * Copyright (C) 2016 Zodiac Inflight Innovations */ +#include "linux/device.h" #include #include #include @@ -298,39 +299,30 @@ static int rmi_f34_update_firmware(struct f34_data *f34, return ret; } -static int rmi_f34_status(struct rmi_function *fn) -{ - struct f34_data *f34 = dev_get_drvdata(&fn->dev); - - /* - * The status is the percentage complete, or once complete, - * zero for success or a negative return code. - */ - return f34->update_status; -} - static ssize_t rmi_driver_bootloader_id_show(struct device *dev, struct device_attribute *dattr, char *buf) { struct rmi_driver_data *data = dev_get_drvdata(dev); - struct rmi_function *fn = data->f34_container; + struct rmi_function *fn; struct f34_data *f34; - if (fn) { - f34 = dev_get_drvdata(&fn->dev); - - if (f34->bl_version == 5) - return sysfs_emit(buf, "%c%c\n", - f34->bootloader_id[0], - f34->bootloader_id[1]); - else - return sysfs_emit(buf, "V%d.%d\n", - f34->bootloader_id[1], - f34->bootloader_id[0]); - } + fn = data->f34_container; + if (!fn) + return -ENODEV; - return 0; + f34 = dev_get_drvdata(&fn->dev); + if (!f34) + return -ENODEV; + + if (f34->bl_version == 5) + return sysfs_emit(buf, "%c%c\n", + f34->bootloader_id[0], + f34->bootloader_id[1]); + else + return sysfs_emit(buf, "V%d.%d\n", + f34->bootloader_id[1], + f34->bootloader_id[0]); } static DEVICE_ATTR(bootloader_id, 0444, rmi_driver_bootloader_id_show, NULL); @@ -343,13 +335,16 @@ static ssize_t rmi_driver_configuration_id_show(struct device *dev, struct rmi_function *fn = data->f34_container; struct f34_data *f34; - if (fn) { - f34 = dev_get_drvdata(&fn->dev); + fn = data->f34_container; + if (!fn) + return -ENODEV; - return sysfs_emit(buf, "%s\n", f34->configuration_id); - } + f34 = dev_get_drvdata(&fn->dev); + if (!f34) + return -ENODEV; - return 0; + + return sysfs_emit(buf, "%s\n", f34->configuration_id); } static DEVICE_ATTR(configuration_id, 0444, @@ -365,10 +360,14 @@ static int rmi_firmware_update(struct rmi_driver_data *data, if (!data->f34_container) { dev_warn(dev, "%s: No F34 present!\n", __func__); - return -EINVAL; + return -ENODEV; } f34 = dev_get_drvdata(&data->f34_container->dev); + if (!f34) { + dev_warn(dev, "%s: No valid F34 present!\n", __func__); + return -ENODEV; + } if (f34->bl_version >= 7) { if (data->pdt_props & HAS_BSR) { @@ -494,10 +493,18 @@ static ssize_t rmi_driver_update_fw_status_show(struct device *dev, char *buf) { struct rmi_driver_data *data = dev_get_drvdata(dev); - int update_status = 0; + struct f34_data *f34; + int update_status = -ENODEV; - if (data->f34_container) - update_status = rmi_f34_status(data->f34_container); + /* + * The status is the percentage complete, or once complete, + * zero for success or a negative return code. + */ + if (data->f34_container) { + f34 = dev_get_drvdata(&data->f34_container->dev); + if (f34) + update_status = f34->update_status; + } return sysfs_emit(buf, "%d\n", update_status); } @@ -517,33 +524,21 @@ static const struct attribute_group rmi_firmware_attr_group = { .attrs = rmi_firmware_attrs, }; -static int rmi_f34_probe(struct rmi_function *fn) +static int rmi_f34v5_probe(struct f34_data *f34) { - struct f34_data *f34; - unsigned char f34_queries[9]; + struct rmi_function *fn = f34->fn; + u8 f34_queries[9]; bool has_config_id; - u8 version = fn->fd.function_version; - int ret; - - f34 = devm_kzalloc(&fn->dev, sizeof(struct f34_data), GFP_KERNEL); - if (!f34) - return -ENOMEM; - - f34->fn = fn; - dev_set_drvdata(&fn->dev, f34); - - /* v5 code only supported version 0, try V7 probe */ - if (version > 0) - return rmi_f34v7_probe(f34); + int error; f34->bl_version = 5; - ret = rmi_read_block(fn->rmi_dev, fn->fd.query_base_addr, - f34_queries, sizeof(f34_queries)); - if (ret) { + error = rmi_read_block(fn->rmi_dev, fn->fd.query_base_addr, + f34_queries, sizeof(f34_queries)); + if (error) { dev_err(&fn->dev, "%s: Failed to query properties\n", __func__); - return ret; + return error; } snprintf(f34->bootloader_id, sizeof(f34->bootloader_id), @@ -569,11 +564,11 @@ static int rmi_f34_probe(struct rmi_function *fn) f34->v5.config_blocks); if (has_config_id) { - ret = rmi_read_block(fn->rmi_dev, fn->fd.control_base_addr, - f34_queries, sizeof(f34_queries)); - if (ret) { + error = rmi_read_block(fn->rmi_dev, fn->fd.control_base_addr, + f34_queries, sizeof(f34_queries)); + if (error) { dev_err(&fn->dev, "Failed to read F34 config ID\n"); - return ret; + return error; } snprintf(f34->configuration_id, sizeof(f34->configuration_id), @@ -582,12 +577,34 @@ static int rmi_f34_probe(struct rmi_function *fn) f34_queries[2], f34_queries[3]); rmi_dbg(RMI_DEBUG_FN, &fn->dev, "Configuration ID: %s\n", - f34->configuration_id); + f34->configuration_id); } return 0; } +static int rmi_f34_probe(struct rmi_function *fn) +{ + struct f34_data *f34; + u8 version = fn->fd.function_version; + int error; + + f34 = devm_kzalloc(&fn->dev, sizeof(struct f34_data), GFP_KERNEL); + if (!f34) + return -ENOMEM; + + f34->fn = fn; + + /* v5 code only supported version 0 */ + error = version == 0 ? rmi_f34v5_probe(f34) : rmi_f34v7_probe(f34); + if (error) + return error; + + dev_set_drvdata(&fn->dev, f34); + + return 0; +} + int rmi_f34_create_sysfs(struct rmi_device *rmi_dev) { return sysfs_create_group(&rmi_dev->dev.kobj, &rmi_firmware_attr_group); diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h index 8813db7eec397..630cdd5a13283 100644 --- a/drivers/input/serio/i8042-acpipnpio.h +++ b/drivers/input/serio/i8042-acpipnpio.h @@ -1155,6 +1155,20 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = { .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "XxHP4NAx"), + }, + .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | + SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "XxKK4NAx_XxSP4NAx"), + }, + .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS | + SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP) + }, /* * A lot of modern Clevo barebones have touchpad and/or keyboard issues * after suspend fixable with the forcenorestore quirk. diff --git a/drivers/input/tablet/pegasus_notetaker.c b/drivers/input/tablet/pegasus_notetaker.c index a68da2988f9cd..26ab9924a7ae5 100644 --- a/drivers/input/tablet/pegasus_notetaker.c +++ b/drivers/input/tablet/pegasus_notetaker.c @@ -63,6 +63,9 @@ #define BUTTON_PRESSED 0xb5 #define COMMAND_VERSION 0xa9 +/* 1 Status + 1 Color + 2 X + 2 Y = 6 bytes */ +#define NOTETAKER_PACKET_SIZE 6 + /* in xy data packet */ #define BATTERY_NO_REPORT 0x40 #define BATTERY_LOW 0x41 @@ -303,6 +306,12 @@ static int pegasus_probe(struct usb_interface *intf, } pegasus->data_len = usb_maxpacket(dev, pipe); + if (pegasus->data_len < NOTETAKER_PACKET_SIZE) { + dev_err(&intf->dev, "packet size is too small (%d)\n", + pegasus->data_len); + error = -EINVAL; + goto err_free_mem; + } pegasus->data = usb_alloc_coherent(dev, pegasus->data_len, GFP_KERNEL, &pegasus->data_dma); diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c index 20094b9899f09..176b1ab516853 100644 --- a/drivers/input/touchscreen/atmel_mxt_ts.c +++ b/drivers/input/touchscreen/atmel_mxt_ts.c @@ -3324,7 +3324,7 @@ static int mxt_probe(struct i2c_client *client) if (data->reset_gpio) { /* Wait a while and then de-assert the RESET GPIO line */ msleep(MXT_RESET_GPIO_TIME); - gpiod_set_value(data->reset_gpio, 0); + gpiod_set_value_cansleep(data->reset_gpio, 0); msleep(MXT_RESET_INVALID_CHG); } diff --git a/drivers/input/touchscreen/cyttsp4_core.c b/drivers/input/touchscreen/cyttsp4_core.c index 7cb26929dc732..9dc25eb2be445 100644 --- a/drivers/input/touchscreen/cyttsp4_core.c +++ b/drivers/input/touchscreen/cyttsp4_core.c @@ -871,7 +871,7 @@ static void cyttsp4_get_mt_touches(struct cyttsp4_mt_data *md, int num_cur_tch) struct cyttsp4_touch tch; int sig; int i, j, t = 0; - int ids[max(CY_TMA1036_MAX_TCH, CY_TMA4XX_MAX_TCH)]; + int ids[MAX(CY_TMA1036_MAX_TCH, CY_TMA4XX_MAX_TCH)]; memset(ids, 0, si->si_ofs.tch_abs[CY_TCH_T].max * sizeof(int)); for (i = 0; i < num_cur_tch; i++) { diff --git a/drivers/input/touchscreen/cyttsp5.c b/drivers/input/touchscreen/cyttsp5.c index db5a885ecd728..a74b34d8df2a2 100644 --- a/drivers/input/touchscreen/cyttsp5.c +++ b/drivers/input/touchscreen/cyttsp5.c @@ -580,7 +580,7 @@ static int cyttsp5_power_control(struct cyttsp5 *ts, bool on) int rc; SET_CMD_REPORT_TYPE(cmd[0], 0); - SET_CMD_REPORT_ID(cmd[0], HID_POWER_SLEEP); + SET_CMD_REPORT_ID(cmd[0], state); SET_CMD_OPCODE(cmd[1], HID_CMD_SET_POWER); rc = cyttsp5_write(ts, HID_COMMAND_REG, cmd, sizeof(cmd)); @@ -865,13 +865,16 @@ static int cyttsp5_probe(struct device *dev, struct regmap *regmap, int irq, ts->input->phys = ts->phys; input_set_drvdata(ts->input, ts); - /* Reset the gpio to be in a reset state */ + /* Assert gpio to be in a reset state */ ts->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(ts->reset_gpio)) { error = PTR_ERR(ts->reset_gpio); dev_err(dev, "Failed to request reset gpio, error %d\n", error); return error; } + + fsleep(10); /* Ensure long-enough reset pulse (minimum 10us). */ + gpiod_set_value_cansleep(ts->reset_gpio, 0); /* Need a delay to have device up */ diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index b068ff8afbc9a..294bd942ee379 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -1519,6 +1519,7 @@ MODULE_DEVICE_TABLE(i2c, goodix_ts_id); static const struct acpi_device_id goodix_acpi_match[] = { { "GDIX1001", 0 }, { "GDIX1002", 0 }, + { "GDIX1003", 0 }, { "GDX9110", 0 }, { } }; diff --git a/drivers/interconnect/qcom/sc7280.c b/drivers/interconnect/qcom/sc7280.c index a626dbc719995..728589ec80264 100644 --- a/drivers/interconnect/qcom/sc7280.c +++ b/drivers/interconnect/qcom/sc7280.c @@ -165,6 +165,7 @@ static struct qcom_icc_node xm_pcie3_1 = { .id = SC7280_MASTER_PCIE_1, .channels = 1, .buswidth = 8, + .num_links = 1, .links = { SC7280_SLAVE_ANOC_PCIE_GEM_NOC }, }; diff --git a/drivers/interconnect/qcom/sc8180x.c b/drivers/interconnect/qcom/sc8180x.c index a741badaa966e..4dd1d2f2e8216 100644 --- a/drivers/interconnect/qcom/sc8180x.c +++ b/drivers/interconnect/qcom/sc8180x.c @@ -1492,34 +1492,40 @@ static struct qcom_icc_bcm bcm_sh3 = { static struct qcom_icc_bcm bcm_sn0 = { .name = "SN0", + .num_nodes = 1, .nodes = { &slv_qns_gemnoc_sf } }; static struct qcom_icc_bcm bcm_sn1 = { .name = "SN1", + .num_nodes = 1, .nodes = { &slv_qxs_imem } }; static struct qcom_icc_bcm bcm_sn2 = { .name = "SN2", .keepalive = true, + .num_nodes = 1, .nodes = { &slv_qns_gemnoc_gc } }; static struct qcom_icc_bcm bcm_co2 = { .name = "CO2", + .num_nodes = 1, .nodes = { &mas_qnm_npu } }; static struct qcom_icc_bcm bcm_sn3 = { .name = "SN3", .keepalive = true, + .num_nodes = 2, .nodes = { &slv_srvc_aggre1_noc, &slv_qns_cnoc } }; static struct qcom_icc_bcm bcm_sn4 = { .name = "SN4", + .num_nodes = 1, .nodes = { &slv_qxs_pimem } }; diff --git a/drivers/interconnect/qcom/sc8280xp.c b/drivers/interconnect/qcom/sc8280xp.c index 0270f6c64481a..c646cdf8a19bf 100644 --- a/drivers/interconnect/qcom/sc8280xp.c +++ b/drivers/interconnect/qcom/sc8280xp.c @@ -48,6 +48,7 @@ static struct qcom_icc_node qnm_a1noc_cfg = { .id = SC8280XP_MASTER_A1NOC_CFG, .channels = 1, .buswidth = 4, + .num_links = 1, .links = { SC8280XP_SLAVE_SERVICE_A1NOC }, }; diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 3985e7562dc91..43fa49805aea3 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -191,7 +191,6 @@ source "drivers/iommu/iommufd/Kconfig" config IRQ_REMAP bool "Support for Interrupt Remapping" depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI - select DMAR_TABLE if INTEL_IOMMU help Supports Interrupt remapping for IO-APIC and MSI devices. To use x2apic mode in the CPU's which support x2APIC enhancements or diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 7dc30c2b56b30..d872054b874fa 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -540,6 +540,7 @@ struct amd_irte_ops; container_of((x), struct amd_io_pgtable, pgtbl_cfg) struct amd_io_pgtable { + seqcount_t seqcount; /* Protects root/mode update */ struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable iop; int mode; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index ef3fae113dd64..1897619209f14 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -840,11 +840,16 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu) BUG_ON(iommu->cmd_buf == NULL); - entry = iommu_virt_to_phys(iommu->cmd_buf); - entry |= MMIO_CMD_SIZE_512; - - memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, - &entry, sizeof(entry)); + if (!is_kdump_kernel()) { + /* + * Command buffer is re-used for kdump kernel and setting + * of MMIO register is not required. + */ + entry = iommu_virt_to_phys(iommu->cmd_buf); + entry |= MMIO_CMD_SIZE_512; + memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, + &entry, sizeof(entry)); + } amd_iommu_reset_cmd_buffer(iommu); } @@ -893,10 +898,15 @@ static void iommu_enable_event_buffer(struct amd_iommu *iommu) BUG_ON(iommu->evt_buf == NULL); - entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; - - memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, - &entry, sizeof(entry)); + if (!is_kdump_kernel()) { + /* + * Event buffer is re-used for kdump kernel and setting + * of MMIO register is not required. + */ + entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; + memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, + &entry, sizeof(entry)); + } /* set head and tail to zero manually */ writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); @@ -3625,7 +3635,7 @@ static int __init parse_ivrs_acpihid(char *str) { u32 seg = 0, bus, dev, fn; char *hid, *uid, *p, *addr; - char acpiid[ACPIID_LEN] = {0}; + char acpiid[ACPIID_LEN + 1] = { }; /* size with NULL terminator */ int i; addr = strchr(str, '@'); @@ -3651,7 +3661,7 @@ static int __init parse_ivrs_acpihid(char *str) /* We have the '@', make it the terminator to get just the acpiid */ *addr++ = 0; - if (strlen(str) > ACPIID_LEN + 1) + if (strlen(str) > ACPIID_LEN) goto not_found; if (sscanf(str, "=%s", acpiid) != 1) @@ -3682,6 +3692,14 @@ static int __init parse_ivrs_acpihid(char *str) while (*uid == '0' && *(uid + 1)) uid++; + if (strlen(hid) >= ACPIHID_HID_LEN) { + pr_err("Invalid command line: hid is too long\n"); + return 1; + } else if (strlen(uid) >= ACPIHID_UID_LEN) { + pr_err("Invalid command line: uid is too long\n"); + return 1; + } + i = early_acpihid_map_size++; memcpy(early_acpihid_map[i].hid, hid, strlen(hid)); memcpy(early_acpihid_map[i].uid, uid, strlen(uid)); diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index 2892aa1b4dc1d..b785d82399983 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -171,8 +172,11 @@ static bool increase_address_space(struct protection_domain *domain, *pte = PM_LEVEL_PDE(domain->iop.mode, iommu_virt_to_phys(domain->iop.root)); + write_seqcount_begin(&domain->iop.seqcount); domain->iop.root = pte; domain->iop.mode += 1; + write_seqcount_end(&domain->iop.seqcount); + amd_iommu_update_and_flush_device_table(domain); amd_iommu_domain_flush_complete(domain); @@ -199,6 +203,7 @@ static u64 *alloc_pte(struct protection_domain *domain, gfp_t gfp, bool *updated) { + unsigned int seqcount; int level, end_lvl; u64 *pte, *page; @@ -214,8 +219,14 @@ static u64 *alloc_pte(struct protection_domain *domain, } - level = domain->iop.mode - 1; - pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)]; + do { + seqcount = read_seqcount_begin(&domain->iop.seqcount); + + level = domain->iop.mode - 1; + pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)]; + } while (read_seqcount_retry(&domain->iop.seqcount, seqcount)); + + address = PAGE_SIZE_ALIGN(address, page_size); end_lvl = PAGE_SIZE_LEVEL(page_size); @@ -292,6 +303,7 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable, unsigned long *page_size) { int level; + unsigned int seqcount; u64 *pte; *page_size = 0; @@ -299,8 +311,12 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable, if (address > PM_LEVEL_SIZE(pgtable->mode)) return NULL; - level = pgtable->mode - 1; - pte = &pgtable->root[PM_LEVEL_INDEX(level, address)]; + do { + seqcount = read_seqcount_begin(&pgtable->seqcount); + level = pgtable->mode - 1; + pte = &pgtable->root[PM_LEVEL_INDEX(level, address)]; + } while (read_seqcount_retry(&pgtable->seqcount, seqcount)); + *page_size = PTE_LEVEL_PAGE_SIZE(level); while (level > 0) { @@ -524,6 +540,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, cfg->tlb = &v1_flush_ops; + seqcount_init(&pgtable->seqcount); + pgtable->iop.ops.map_pages = iommu_v1_map_pages; pgtable->iop.ops.unmap_pages = iommu_v1_unmap_pages; pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys; diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c index cbf0c46015125..6c0777a3c57b7 100644 --- a/drivers/iommu/amd/io_pgtable_v2.c +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -259,7 +259,7 @@ static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova, pte = v2_alloc_pte(pdom->nid, pdom->iop.pgd, iova, map_size, gfp, &updated); if (!pte) { - ret = -EINVAL; + ret = -ENOMEM; goto out; } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 83c5d786686d0..23cfb98fe90a7 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -780,6 +780,14 @@ int amd_iommu_register_ga_log_notifier(int (*notifier)(u32)) { iommu_ga_log_notifier = notifier; + /* + * Ensure all in-flight IRQ handlers run to completion before returning + * to the caller, e.g. to ensure module code isn't unloaded while it's + * being executed in the IRQ handler. + */ + if (!notifier) + synchronize_rcu(); + return 0; } EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier); @@ -2151,8 +2159,21 @@ static inline u64 dma_max_address(void) if (amd_iommu_pgtable == AMD_IOMMU_V1) return ~0ULL; - /* V2 with 4/5 level page table */ - return ((1ULL << PM_LEVEL_SHIFT(amd_iommu_gpt_level)) - 1); + /* + * V2 with 4/5 level page table. Note that "2.2.6.5 AMD64 4-Kbyte Page + * Translation" shows that the V2 table sign extends the top of the + * address space creating a reserved region in the middle of the + * translation, just like the CPU does. Further Vasant says the docs are + * incomplete and this only applies to non-zero PASIDs. If the AMDv2 + * page table is assigned to the 0 PASID then there is no sign extension + * check. + * + * Since the IOMMU must have a fixed geometry, and the core code does + * not understand sign extended addressing, we have to chop off the high + * bit to get consistent behavior with attachments of the domain to any + * PASID. + */ + return ((1ULL << (PM_LEVEL_SHIFT(amd_iommu_gpt_level) - 1)) - 1); } static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 0b89275084274..0ea450cbb7786 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -121,6 +121,8 @@ #define DART_T8110_ERROR_ADDR_LO 0x170 #define DART_T8110_ERROR_ADDR_HI 0x174 +#define DART_T8110_ERROR_STREAMS 0x1c0 + #define DART_T8110_PROTECT 0x200 #define DART_T8110_UNPROTECT 0x204 #define DART_T8110_PROTECT_LOCK 0x208 @@ -1041,6 +1043,9 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) error, stream_idx, error_code, fault_name, addr); writel(error, dart->regs + DART_T8110_ERROR); + for (int i = 0; i < BITS_TO_U32(dart->num_streams); i++) + writel(U32_MAX, dart->regs + DART_T8110_ERROR_STREAMS + 4 * i); + return IRQ_HANDLED; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c index 4d32b9d5320ea..03f5fd148713d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c @@ -74,7 +74,7 @@ extern struct kvm_iommu_ops kvm_nvhe_sym(smmu_ops); static int atomic_pages; module_param(atomic_pages, int, 0); -phys_addr_t __topup_virt_to_phys(void *virt) +static phys_addr_t __topup_virt_to_phys(void *virt) { return __pa(virt); } @@ -284,15 +284,13 @@ static int kvm_arm_smmu_domain_finalize(struct kvm_arm_smmu_domain *kvm_smmu_dom return 0; } - kvm_smmu_domain->smmu = smmu; - if (kvm_smmu_domain->domain.type == IOMMU_DOMAIN_IDENTITY) { kvm_smmu_domain->id = KVM_IOMMU_DOMAIN_IDMAP_ID; /* * Identity domains doesn't use the DMA API, so no need to * set the domain aperture. */ - return 0; + goto out; } /* Default to stage-1. */ @@ -325,7 +323,13 @@ static int kvm_arm_smmu_domain_finalize(struct kvm_arm_smmu_domain *kvm_smmu_dom ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain, kvm_smmu_domain->id, kvm_smmu_domain->type); + if (ret) { + ida_free(&kvm_arm_smmu_domain_ida, kvm_smmu_domain->id); + return ret; + } +out: + kvm_smmu_domain->smmu = smmu; return ret; } @@ -986,7 +990,7 @@ static int kvm_arm_smmu_remove(struct platform_device *pdev) return 0; } -int kvm_arm_smmu_suspend(struct device *dev) +static int kvm_arm_smmu_suspend(struct device *dev) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); struct host_arm_smmu_device *host_smmu = smmu_to_host(smmu); @@ -996,7 +1000,7 @@ int kvm_arm_smmu_suspend(struct device *dev) return 0; } -int kvm_arm_smmu_resume(struct device *dev) +static int kvm_arm_smmu_resume(struct device *dev) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); struct host_arm_smmu_device *host_smmu = smmu_to_host(smmu); @@ -1046,13 +1050,13 @@ static int kvm_arm_smmu_array_alloc(void) return 0; } -int smmu_put_device(struct device *dev, void *data) +static int smmu_put_device(struct device *dev, void *data) { pm_runtime_put_noidle(dev); return 0; } -int smmu_unregister_smmu(struct device *dev, void *data) +static int smmu_unregister_smmu(struct device *dev, void *data) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); @@ -1181,7 +1185,7 @@ static void kvm_arm_smmu_v3_remove(void) platform_driver_unregister(&kvm_arm_smmu_driver); } -pkvm_handle_t kvm_arm_smmu_v3_id(struct device *dev) +static pkvm_handle_t kvm_arm_smmu_v3_id(struct device *dev) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); struct host_arm_smmu_device *host_smmu = smmu_to_host(smmu); @@ -1189,7 +1193,7 @@ pkvm_handle_t kvm_arm_smmu_v3_id(struct device *dev) return host_smmu->id; } -struct kvm_iommu_driver kvm_smmu_v3_ops = { +static struct kvm_iommu_driver kvm_smmu_v3_ops = { .init_driver = kvm_arm_smmu_v3_init, .remove_driver = kvm_arm_smmu_v3_remove, .get_iommu_id = kvm_arm_smmu_v3_id, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index b964f45c7cb80..31f039801d175 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1231,26 +1231,37 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) return 0; } +static int arm_smmu_streams_cmp_key(const void *lhs, const struct rb_node *rhs) +{ + struct arm_smmu_stream *stream_rhs = + rb_entry(rhs, struct arm_smmu_stream, node); + const u32 *sid_lhs = lhs; + + if (*sid_lhs < stream_rhs->id) + return -1; + if (*sid_lhs > stream_rhs->id) + return 1; + return 0; +} + +static int arm_smmu_streams_cmp_node(struct rb_node *lhs, + const struct rb_node *rhs) +{ + return arm_smmu_streams_cmp_key( + &rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs); +} + static struct arm_smmu_master * arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) { struct rb_node *node; - struct arm_smmu_stream *stream; lockdep_assert_held(&smmu->streams_mutex); - node = smmu->streams.rb_node; - while (node) { - stream = rb_entry(node, struct arm_smmu_stream, node); - if (stream->id < sid) - node = node->rb_right; - else if (stream->id > sid) - node = node->rb_left; - else - return stream->master; - } - - return NULL; + node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key); + if (!node) + return NULL; + return rb_entry(node, struct arm_smmu_stream, node)->master; } /* IRQ and event handlers */ @@ -2345,8 +2356,6 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, { int i; int ret = 0; - struct arm_smmu_stream *new_stream, *cur_stream; - struct rb_node **new_node, *parent_node = NULL; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams), @@ -2357,9 +2366,10 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, mutex_lock(&smmu->streams_mutex); for (i = 0; i < fwspec->num_ids; i++) { + struct arm_smmu_stream *new_stream = &master->streams[i]; + struct rb_node *existing; u32 sid = fwspec->ids[i]; - new_stream = &master->streams[i]; new_stream->id = sid; new_stream->master = master; @@ -2368,28 +2378,23 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, break; /* Insert into SID tree */ - new_node = &(smmu->streams.rb_node); - while (*new_node) { - cur_stream = rb_entry(*new_node, struct arm_smmu_stream, - node); - parent_node = *new_node; - if (cur_stream->id > new_stream->id) { - new_node = &((*new_node)->rb_left); - } else if (cur_stream->id < new_stream->id) { - new_node = &((*new_node)->rb_right); - } else { - dev_warn(master->dev, - "stream %u already in tree\n", - cur_stream->id); - ret = -EINVAL; - break; - } - } - if (ret) - break; + existing = rb_find_add(&new_stream->node, &smmu->streams, + arm_smmu_streams_cmp_node); + if (existing) { + struct arm_smmu_master *existing_master = + rb_entry(existing, struct arm_smmu_stream, node) + ->master; + + /* Bridged PCI devices may end up with duplicated IDs */ + if (existing_master == master) + continue; - rb_link_node(&new_stream->node, parent_node, new_node); - rb_insert_color(&new_stream->node, &smmu->streams); + dev_warn(master->dev, + "stream %u already in tree from dev %s\n", sid, + dev_name(existing_master->dev)); + ret = -EINVAL; + break; + } } if (ret) { diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c index 2e5d4f273c0b0..bbd484dae98ae 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c @@ -61,7 +61,7 @@ struct hyp_arm_smmu_v3_domain { hyp_rwlock_t lock; /* Protects iommu_list. */ hyp_spinlock_t pgt_lock; /* protects page table. */ struct io_pgtable *pgtable; - void *cdptr; + phys_addr_t cdptr_phys; }; struct kvm_iommu_walk_data { @@ -155,7 +155,7 @@ static int smmu_add_cmd(struct hyp_arm_smmu_v3_device *smmu, int ret; u64 cmd[CMDQ_ENT_DWORDS] = {}; int idx = Q_IDX(smmu, smmu->cmdq_prod); - u64 *slot = smmu->cmdq_base + idx * CMDQ_ENT_DWORDS; + __le64 *slot = smmu->cmdq_base + idx * CMDQ_ENT_DWORDS; if (smmu->iommu.power_is_off) return -EPIPE; @@ -245,7 +245,7 @@ static int smmu_send_cmd(struct hyp_arm_smmu_v3_device *smmu, return smmu_sync_cmd(smmu); } -static int smmu_sync_ste(struct hyp_arm_smmu_v3_device *smmu, u64 *step, u32 sid) +static int smmu_sync_ste(struct hyp_arm_smmu_v3_device *smmu, __le64 *step, u32 sid) { struct arm_smmu_cmdq_ent cmd = { .opcode = CMDQ_OP_CFGI_STE, @@ -262,7 +262,7 @@ static int smmu_sync_ste(struct hyp_arm_smmu_v3_device *smmu, u64 *step, u32 sid return smmu_send_cmd(smmu, &cmd); } -static int smmu_sync_cd(struct hyp_arm_smmu_v3_device *smmu, u64 *cd, u32 sid, u32 ssid) +static int smmu_sync_cd(struct hyp_arm_smmu_v3_device *smmu, __le64 *cd, u32 sid, u32 ssid) { struct arm_smmu_cmdq_ent cmd = { .opcode = CMDQ_OP_CFGI_CD, @@ -304,7 +304,7 @@ static int smmu_alloc_l2_strtab(struct hyp_arm_smmu_v3_device *smmu, u32 idx) /* Ensure the empty stream table is visible before the descriptor write */ wmb(); - WRITE_ONCE(smmu->strtab_base[idx], l2ptr | span); + WRITE_ONCE(smmu->strtab_base[idx], cpu_to_le64(l2ptr | span)); if (!(smmu->features & ARM_SMMU_FEAT_COHERENCY)) kvm_flush_dcache_to_poc(&smmu->strtab_base[idx], STRTAB_L1_DESC_DWORDS << 3); @@ -312,11 +312,12 @@ static int smmu_alloc_l2_strtab(struct hyp_arm_smmu_v3_device *smmu, u32 idx) return 0; } -static u64 *smmu_get_ste_ptr(struct hyp_arm_smmu_v3_device *smmu, u32 sid) +static __le64 *smmu_get_ste_ptr(struct hyp_arm_smmu_v3_device *smmu, u32 sid) { u32 idx; int ret; - u64 l1std, span, *base; + u64 l1std, span; + __le64 *base; if (sid >= smmu->strtab_num_entries) return NULL; @@ -326,12 +327,12 @@ static u64 *smmu_get_ste_ptr(struct hyp_arm_smmu_v3_device *smmu, u32 sid) return smmu->strtab_base + sid * STRTAB_STE_DWORDS; idx = sid >> smmu->strtab_split; - l1std = smmu->strtab_base[idx]; + l1std = le64_to_cpu(smmu->strtab_base[idx]); if (!l1std) { ret = smmu_alloc_l2_strtab(smmu, idx); if (ret) return NULL; - l1std = smmu->strtab_base[idx]; + l1std = le64_to_cpu(smmu->strtab_base[idx]); if (WARN_ON(!l1std)) return NULL; } @@ -345,13 +346,13 @@ static u64 *smmu_get_ste_ptr(struct hyp_arm_smmu_v3_device *smmu, u32 sid) return base + idx * STRTAB_STE_DWORDS; } -static u64 *smmu_get_cd_ptr(u64 *cdtab, u32 ssid) +static __le64 *smmu_get_cd_ptr(__le64 *cdtab, u32 ssid) { /* Assume linear for now. */ return cdtab + ssid * CTXDESC_CD_DWORDS; } -static u64 *smmu_alloc_cd(u32 pasid_bits) +static phys_addr_t smmu_alloc_cd(u32 pasid_bits) { u64 *cd_table; u32 requested_order = get_order((1 << pasid_bits) * (CTXDESC_CD_DWORDS << 3)); @@ -361,11 +362,11 @@ static u64 *smmu_alloc_cd(u32 pasid_bits) cd_table = kvm_iommu_donate_pages(requested_order, true); if (!cd_table) - return NULL; - return (u64 *)hyp_virt_to_phys(cd_table); + return 0; + return hyp_virt_to_phys(cd_table); } -static void smmu_free_cd(u64 *cd_table, u32 pasid_bits) +static void smmu_free_cd(__le64 *cd_table, u32 pasid_bits) { u32 order = get_order((1 << pasid_bits) * (CTXDESC_CD_DWORDS << 3)); @@ -772,7 +773,7 @@ static int smmu_init_device(struct hyp_arm_smmu_v3_device *smmu) if (ret) return ret; - smmu->base = hyp_phys_to_virt(smmu->mmio_addr); + smmu->base = (void __iomem *)hyp_phys_to_virt(smmu->mmio_addr); smmu->pgtable_cfg_s1.tlb = &smmu_tlb_ops; smmu->pgtable_cfg_s2.tlb = &smmu_tlb_ops; @@ -827,7 +828,7 @@ static struct kvm_hyp_iommu *smmu_id_to_iommu(pkvm_handle_t smmu_id) return &kvm_hyp_arm_smmu_v3_smmus[smmu_id].iommu; } -int smmu_domain_config_s2(struct kvm_hyp_iommu_domain *domain, u64 *ent) +static int smmu_domain_config_s2(struct kvm_hyp_iommu_domain *domain, u64 *ent) { struct io_pgtable_cfg *cfg; u64 ts, sl, ic, oc, sh, tg, ps; @@ -860,15 +861,14 @@ int smmu_domain_config_s2(struct kvm_hyp_iommu_domain *domain, u64 *ent) return 0; } -int smmu_domain_config_s1(struct hyp_arm_smmu_v3_device *smmu, - struct kvm_hyp_iommu_domain *domain, - u32 sid, u32 pasid, u32 pasid_bits, - u64 *ent, bool *update_ste) +static int smmu_domain_config_s1(struct hyp_arm_smmu_v3_device *smmu, + struct kvm_hyp_iommu_domain *domain, u32 sid, u32 pasid, + u32 pasid_bits, u64 *ent, bool *update_ste) { - u64 *cd_table; - u64 *ste; + phys_addr_t cd_table_phys; + __le64 *ste; u64 val; - u64 *cd_entry; + __le64 *cd_entry; struct io_pgtable_cfg *cfg; struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; @@ -890,27 +890,27 @@ int smmu_domain_config_s1(struct hyp_arm_smmu_v3_device *smmu, */ if (!pasid_bits) { /* The domain already have a CD table. */ - if (smmu_domain->cdptr) { - cd_table = smmu_domain->cdptr; + if (smmu_domain->cdptr_phys) { + cd_table_phys = smmu_domain->cdptr_phys; } else { - cd_table = smmu_alloc_cd(pasid_bits); - if (!cd_table) + cd_table_phys = smmu_alloc_cd(pasid_bits); + if (!cd_table_phys) return -ENOMEM; - smmu_domain->cdptr = cd_table; + smmu_domain->cdptr_phys = cd_table_phys; } } else { if (pasid == 0) { - cd_table = smmu_alloc_cd(pasid_bits); - if (!cd_table) + cd_table_phys = smmu_alloc_cd(pasid_bits); + if (!cd_table_phys) return -ENOMEM; } else { u32 nr_entries = 1 << FIELD_GET(STRTAB_STE_0_S1CDMAX, val); if (pasid >= nr_entries) return -E2BIG; - cd_table = (u64 *)(val & STRTAB_STE_0_S1CTXPTR_MASK); + cd_table_phys = val & STRTAB_STE_0_S1CTXPTR_MASK; /* Device (pasid = 0) must be attached first. */ - if (!cd_table) + if (!cd_table_phys) return -ENODEV; } } @@ -921,7 +921,7 @@ int smmu_domain_config_s1(struct hyp_arm_smmu_v3_device *smmu, FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH); - ent[0] = ((u64)cd_table & STRTAB_STE_0_S1CTXPTR_MASK) | + ent[0] = (cd_table_phys & STRTAB_STE_0_S1CTXPTR_MASK) | FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | FIELD_PREP(STRTAB_STE_0_S1CDMAX, pasid_bits) | FIELD_PREP(STRTAB_STE_0_S1FMT, STRTAB_STE_0_S1FMT_LINEAR) | @@ -929,10 +929,11 @@ int smmu_domain_config_s1(struct hyp_arm_smmu_v3_device *smmu, *update_ste = true; } - cd_entry = smmu_get_cd_ptr(hyp_phys_to_virt((u64)cd_table), pasid); + cd_entry = smmu_get_cd_ptr(hyp_phys_to_virt(cd_table_phys), pasid); if (cd_entry[0]) { /* CD already attached to a different domain. */ - if (cfg->arm_lpae_s1_cfg.ttbr != (cd_entry[1] & CTXDESC_CD_1_TTB0_MASK)) + if (cfg->arm_lpae_s1_cfg.ttbr != (le64_to_cpu(cd_entry[1]) & + CTXDESC_CD_1_TTB0_MASK)) return -EBUSY; /* * This can only happen for PASID = 0 && PASID_BITS =0 with shared cd table, @@ -967,8 +968,8 @@ int smmu_domain_config_s1(struct hyp_arm_smmu_v3_device *smmu, return 0; } -int smmu_domain_finalise(struct hyp_arm_smmu_v3_device *smmu, - struct kvm_hyp_iommu_domain *domain) +static int smmu_domain_finalise(struct hyp_arm_smmu_v3_device *smmu, + struct kvm_hyp_iommu_domain *domain) { int ret; struct io_pgtable_cfg *cfg; @@ -1081,7 +1082,7 @@ static int smmu_attach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom { int i; int ret = -EINVAL; - u64 *dst; + __le64 *dst; u64 ent[STRTAB_STE_DWORDS] = {}; struct hyp_arm_smmu_v3_device *smmu = to_smmu(iommu); struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; @@ -1138,14 +1139,14 @@ static int smmu_attach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom if (smmu_domain->type == KVM_ARM_SMMU_DOMAIN_S2) { /* Device already attached or pasid for s2. */ - if ((dst[0] & ~STRTAB_STE_0_S1CTXPTR_MASK) || pasid) { + if ((le64_to_cpu(dst[0]) & ~STRTAB_STE_0_S1CTXPTR_MASK) || pasid) { ret = -EBUSY; goto out_unlock; } ret = smmu_domain_config_s2(domain, ent); /* Don't lost the CD as we never free it. */ - ent[0] |= dst[0]; + ent[0] |= le64_to_cpu(dst[0]); } else { /* * One drawback to this is that the first attach to this sid dictates @@ -1191,12 +1192,14 @@ static int smmu_attach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom static int smmu_detach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_domain *domain, u32 sid, u32 pasid) { - u64 *dst; + __le64 *dst, *cd_table, *cd; int i, ret = -ENODEV; struct hyp_arm_smmu_v3_device *smmu = to_smmu(iommu); struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; u32 pasid_bits = 0; - u64 *cd_table, *cd; + u64 ste0; + phys_addr_t cd_table_phys; + u32 domain_id, ste_cfg; hyp_write_lock(&smmu_domain->lock); kvm_iommu_lock(iommu); @@ -1204,6 +1207,8 @@ static int smmu_detach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom if (!dst) goto out_unlock; + ste0 = le64_to_cpu(dst[0]); + ste_cfg = FIELD_GET(STRTAB_STE_0_CFG, ste0); /* * Look at smmu_domain_config_s1 for CD allocation and life time * For detach stage-1 domains: @@ -1213,14 +1218,18 @@ static int smmu_detach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom * - PASID_BITS = 0: invalidate the STE, the cdptr per domain would be free at free_domain() */ if (smmu_domain->type == KVM_ARM_SMMU_DOMAIN_S1) { - pasid_bits = FIELD_GET(STRTAB_STE_0_S1CDMAX, dst[0]); + if (ste_cfg != STRTAB_STE_0_CFG_S1_TRANS) { + ret = -EACCES; + goto out_unlock; + } + pasid_bits = FIELD_GET(STRTAB_STE_0_S1CDMAX, ste0); if (pasid >= (1 << pasid_bits)) { ret = -E2BIG; goto out_unlock; } - cd_table = (u64 *)(FIELD_GET(STRTAB_STE_0_S1CTXPTR_MASK, dst[0]) << 6); - BUG_ON(!cd_table); - cd_table = hyp_phys_to_virt((phys_addr_t)cd_table); + cd_table_phys = FIELD_GET(STRTAB_STE_0_S1CTXPTR_MASK, ste0) << 6; + BUG_ON(!cd_table_phys); + cd_table = hyp_phys_to_virt(cd_table_phys); if (pasid_bits) { if (pasid == 0) { int j; @@ -1228,19 +1237,31 @@ static int smmu_detach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom /* PASID 0 must be detach last! */ for (j = 1 ; j < (1 << pasid_bits) ; ++j) { cd = smmu_get_cd_ptr(cd_table, j); - if (cd[0] & CTXDESC_CD_0_V) { + if (le64_to_cpu(cd[0]) & CTXDESC_CD_0_V) { ret = -EINVAL; goto out_unlock; } } + cd = smmu_get_cd_ptr(cd_table, 0); + domain_id = FIELD_GET(CTXDESC_CD_0_ASID, le64_to_cpu(cd[0])); + if (domain->domain_id != domain_id) { + ret = -EACCES; + goto out_unlock; + } } else { cd = smmu_get_cd_ptr(cd_table, pasid); - if (!(cd[0] & CTXDESC_CD_0_V)) { + if (!(le64_to_cpu(cd[0]) & CTXDESC_CD_0_V)) { /* The device is not actually attached! */ ret = -ENOENT; goto out_unlock; } + domain_id = FIELD_GET(CTXDESC_CD_0_ASID, le64_to_cpu(cd[0])); + if (domain->domain_id != domain_id) { + ret = -EACCES; + goto out_unlock; + } + cd[0] = 0; smmu_sync_cd(smmu, cd, sid, pasid); cd[1] = 0; @@ -1250,9 +1271,16 @@ static int smmu_detach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom goto out_skip_ste; } } + } else { + domain_id = FIELD_GET(STRTAB_STE_2_S2VMID, le64_to_cpu(dst[2])); + if ((ste_cfg != STRTAB_STE_0_CFG_S2_TRANS) || + (domain->domain_id != domain_id)) { + ret = -EACCES; + goto out_unlock; + } } /* For stage-2 and pasid = 0 */ - if (!(dst[0] & STRTAB_STE_0_V)) { + if (!(le64_to_cpu(dst[0]) & STRTAB_STE_0_V)) { /* The device is not actually attached! */ ret = -ENOENT; goto out_unlock; @@ -1283,7 +1311,7 @@ static int smmu_detach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom return ret; } -int smmu_alloc_domain(struct kvm_hyp_iommu_domain *domain, u32 type) +static int smmu_alloc_domain(struct kvm_hyp_iommu_domain *domain, u32 type) { struct hyp_arm_smmu_v3_domain *smmu_domain; @@ -1302,7 +1330,7 @@ int smmu_alloc_domain(struct kvm_hyp_iommu_domain *domain, u32 type) return 0; } -void smmu_free_domain(struct kvm_hyp_iommu_domain *domain) +static void smmu_free_domain(struct kvm_hyp_iommu_domain *domain) { struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; /* @@ -1312,13 +1340,13 @@ void smmu_free_domain(struct kvm_hyp_iommu_domain *domain) if (smmu_domain->pgtable) kvm_arm_io_pgtable_free(smmu_domain->pgtable); - if (smmu_domain->cdptr) - smmu_free_cd((u64 *)hyp_phys_to_virt((phys_addr_t)smmu_domain->cdptr), 0); + if (smmu_domain->cdptr_phys) + smmu_free_cd(hyp_phys_to_virt(smmu_domain->cdptr_phys), 0); hyp_free(smmu_domain); } -bool smmu_dabt_device(struct hyp_arm_smmu_v3_device *smmu, +static bool smmu_dabt_device(struct hyp_arm_smmu_v3_device *smmu, struct kvm_cpu_context *host_ctxt, u64 esr, u32 off) { bool is_write = esr & ESR_ELx_WNR; @@ -1361,7 +1389,7 @@ bool smmu_dabt_device(struct hyp_arm_smmu_v3_device *smmu, return true; } -bool smmu_dabt_handler(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr) +static bool smmu_dabt_handler(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr) { struct hyp_arm_smmu_v3_device *smmu; @@ -1373,7 +1401,7 @@ bool smmu_dabt_handler(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr) return false; } -int smmu_suspend(struct kvm_hyp_iommu *iommu) +static int smmu_suspend(struct kvm_hyp_iommu *iommu) { struct hyp_arm_smmu_v3_device *smmu = to_smmu(iommu); @@ -1386,7 +1414,7 @@ int smmu_suspend(struct kvm_hyp_iommu *iommu) return 0; } -int smmu_resume(struct kvm_hyp_iommu *iommu) +static int smmu_resume(struct kvm_hyp_iommu *iommu) { struct hyp_arm_smmu_v3_device *smmu = to_smmu(iommu); @@ -1398,7 +1426,7 @@ int smmu_resume(struct kvm_hyp_iommu *iommu) return 0; } -int smmu_map_pages(struct kvm_hyp_iommu_domain *domain, unsigned long iova, +static int smmu_map_pages(struct kvm_hyp_iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, size_t *total_mapped) { diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h index 143737be23c4b..f4bb06c8961a5 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h @@ -27,9 +27,9 @@ struct hyp_arm_smmu_v3_device { void __iomem *base; u32 cmdq_prod; - u64 *cmdq_base; + __le64 *cmdq_base; size_t cmdq_log2size; - u64 *strtab_base; + __le64 *strtab_base; size_t strtab_num_entries; size_t strtab_num_l1_entries; u8 strtab_split; @@ -49,6 +49,8 @@ enum kvm_arm_smmu_domain_stage { KVM_ARM_SMMU_DOMAIN_S1, KVM_ARM_SMMU_DOMAIN_S2, }; + +extern struct kvm_iommu_ops smmu_ops; #endif /* CONFIG_ARM_SMMU_V3_PKVM */ #endif /* __KVM_ARM_SMMU_V3_H */ diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/io-pgtable-arm.c b/drivers/iommu/arm/arm-smmu-v3/pkvm/io-pgtable-arm.c index 778e96fd36059..a93ee797bef3d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/io-pgtable-arm.c +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/io-pgtable-arm.c @@ -68,8 +68,7 @@ void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, kvm_flush_dcache_to_poc(ptep, sizeof(*ptep) * num_entries); } -int kvm_arm_io_pgtable_init(struct io_pgtable_cfg *cfg, - struct arm_lpae_io_pgtable *data) +static int kvm_arm_io_pgtable_init(struct io_pgtable_cfg *cfg, struct arm_lpae_io_pgtable *data) { int ret = -EINVAL; diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index e6b4bab0dde2e..3d03136697963 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -255,6 +255,7 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { { .compatible = "qcom,sdm670-mdss" }, { .compatible = "qcom,sdm845-mdss" }, { .compatible = "qcom,sdm845-mss-pil" }, + { .compatible = "qcom,sm6115-mdss" }, { .compatible = "qcom,sm6350-mdss" }, { .compatible = "qcom,sm6375-mdss" }, { .compatible = "qcom,sm8150-mdss" }, diff --git a/drivers/iommu/intel/debugfs.c b/drivers/iommu/intel/debugfs.c index 1f925285104ee..aa9cfcb6039e2 100644 --- a/drivers/iommu/intel/debugfs.c +++ b/drivers/iommu/intel/debugfs.c @@ -562,17 +562,11 @@ DEFINE_SHOW_ATTRIBUTE(ir_translation_struct); static void latency_show_one(struct seq_file *m, struct intel_iommu *iommu, struct dmar_drhd_unit *drhd) { - int ret; - seq_printf(m, "IOMMU: %s Register Base Address: %llx\n", iommu->name, drhd->reg_base_addr); - ret = dmar_latency_snapshot(iommu, debug_buf, DEBUG_BUFFER_SIZE); - if (ret < 0) - seq_puts(m, "Failed to get latency snapshot"); - else - seq_puts(m, debug_buf); - seq_puts(m, "\n"); + dmar_latency_snapshot(iommu, debug_buf, DEBUG_BUFFER_SIZE); + seq_printf(m, "%s\n", debug_buf); } static int latency_show(struct seq_file *m, void *v) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index d6381c00bb8dd..ac7b8fb36db8f 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2168,6 +2168,10 @@ static void switch_to_super_page(struct dmar_domain *domain, struct dma_pte *pte = NULL; unsigned long i; + if (WARN_ON(!IS_ALIGNED(start_pfn, lvl_pages) || + !IS_ALIGNED(end_pfn + 1, lvl_pages))) + return; + while (start_pfn <= end_pfn) { if (!pte) pte = pfn_to_dma_pte(domain, start_pfn, &level, @@ -2241,7 +2245,8 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, unsigned long pages_to_remove; pteval |= DMA_PTE_LARGE_PAGE; - pages_to_remove = min_t(unsigned long, nr_pages, + pages_to_remove = min_t(unsigned long, + round_down(nr_pages, lvl_pages), nr_pte_to_next_page(pte) * lvl_pages); end_pfn = iov_pfn + pages_to_remove - 1; switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl); @@ -4410,7 +4415,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) } if (info->ats_supported && ecap_prs(iommu->ecap) && - pci_pri_supported(pdev)) + ecap_pds(iommu->ecap) && pci_pri_supported(pdev)) info->pri_supported = 1; } } @@ -4855,6 +4860,9 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx); +/* QM57/QS57 integrated gfx malfunctions with dmar */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_iommu_igfx); + /* Broadwell igfx malfunctions with dmar */ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx); @@ -4932,7 +4940,6 @@ static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev) } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt); diff --git a/drivers/iommu/intel/perf.c b/drivers/iommu/intel/perf.c index 94ee70ac38e30..ae64e1123f257 100644 --- a/drivers/iommu/intel/perf.c +++ b/drivers/iommu/intel/perf.c @@ -113,7 +113,7 @@ static char *latency_type_names[] = { " svm_prq" }; -int dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size) +void dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size) { struct latency_statistic *lstat = iommu->perf_statistic; unsigned long flags; @@ -122,7 +122,7 @@ int dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size) memset(str, 0, size); for (i = 0; i < COUNTS_NUM; i++) - bytes += snprintf(str + bytes, size - bytes, + bytes += scnprintf(str + bytes, size - bytes, "%s", latency_counter_names[i]); spin_lock_irqsave(&latency_lock, flags); @@ -130,7 +130,7 @@ int dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size) if (!dmar_latency_enabled(iommu, i)) continue; - bytes += snprintf(str + bytes, size - bytes, + bytes += scnprintf(str + bytes, size - bytes, "\n%s", latency_type_names[i]); for (j = 0; j < COUNTS_NUM; j++) { @@ -156,11 +156,9 @@ int dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size) break; } - bytes += snprintf(str + bytes, size - bytes, + bytes += scnprintf(str + bytes, size - bytes, "%12lld", val); } } spin_unlock_irqrestore(&latency_lock, flags); - - return bytes; } diff --git a/drivers/iommu/intel/perf.h b/drivers/iommu/intel/perf.h index fd6db8049d1a7..1e481e9e4ad04 100644 --- a/drivers/iommu/intel/perf.h +++ b/drivers/iommu/intel/perf.h @@ -41,7 +41,7 @@ void dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type); bool dmar_latency_enabled(struct intel_iommu *iommu, enum latency_type type); void dmar_latency_update(struct intel_iommu *iommu, enum latency_type type, u64 latency); -int dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size); +void dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size); #else static inline int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type) @@ -65,9 +65,8 @@ dmar_latency_update(struct intel_iommu *iommu, enum latency_type type, u64 laten { } -static inline int +static inline void dmar_latency_snapshot(struct intel_iommu *iommu, char *str, size_t size) { - return 0; } #endif /* CONFIG_DMAR_PERF */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3f1029c0825e9..3fa5699b9ff19 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -566,6 +566,18 @@ int iommu_probe_device(struct device *dev) mutex_lock(&iommu_probe_device_lock); ret = __iommu_probe_device(dev, NULL); mutex_unlock(&iommu_probe_device_lock); + + /* + * The dma_configure replay paths need bus_iommu_probe() to + * finish before they can call arch_setup_dma_ops() + */ + if (IS_ENABLED(CONFIG_IOMMU_DMA) && !ret && dev->iommu_group) { + mutex_lock(&dev->iommu_group->mutex); + if (!dev->iommu_group->default_domain && + !dev_iommu_ops(dev)->set_platform_dma_ops) + ret = -EPROBE_DEFER; + mutex_unlock(&dev->iommu_group->mutex); + } if (ret) return ret; @@ -2370,6 +2382,7 @@ static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova, unsigned int pgsize_idx, pgsize_idx_next; unsigned long pgsizes; size_t offset, pgsize, pgsize_next; + size_t offset_end; unsigned long addr_merge = paddr | iova; /* Page sizes supported by the hardware and small enough for @size */ @@ -2410,7 +2423,8 @@ static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova, * If size is big enough to accommodate the larger page, reduce * the number of smaller pages. */ - if (offset + pgsize_next <= size) + if (!check_add_overflow(offset, pgsize_next, &offset_end) && + offset_end <= size) size = offset; out_set_count: @@ -3149,6 +3163,12 @@ int iommu_device_use_default_domain(struct device *dev) return 0; mutex_lock(&group->mutex); + /* We may race against bus_iommu_probe() finalising groups here */ + if (IS_ENABLED(CONFIG_IOMMU_DMA) && !group->default_domain && + !dev_iommu_ops(dev)->set_platform_dma_ops) { + ret = -EPROBE_DEFER; + goto unlock_out; + } if (group->owner_cnt) { if (group->owner || !iommu_is_default_domain(group) || !xa_empty(&group->pasid_array)) { diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index e76b229399948..6bd37343061e0 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -69,36 +69,45 @@ struct iopt_area *iopt_area_contig_next(struct iopt_area_contig_iter *iter) return iter->area; } -static bool __alloc_iova_check_hole(struct interval_tree_double_span_iter *span, - unsigned long length, - unsigned long iova_alignment, - unsigned long page_offset) +static bool __alloc_iova_check_range(unsigned long *start, unsigned long last, + unsigned long length, + unsigned long iova_alignment, + unsigned long page_offset) { - if (span->is_used || span->last_hole - span->start_hole < length - 1) + unsigned long aligned_start; + + /* ALIGN_UP() */ + if (check_add_overflow(*start, iova_alignment - 1, &aligned_start)) return false; + aligned_start &= ~(iova_alignment - 1); + aligned_start |= page_offset; - span->start_hole = ALIGN(span->start_hole, iova_alignment) | - page_offset; - if (span->start_hole > span->last_hole || - span->last_hole - span->start_hole < length - 1) + if (aligned_start >= last || last - aligned_start < length - 1) return false; + *start = aligned_start; return true; } -static bool __alloc_iova_check_used(struct interval_tree_span_iter *span, +static bool __alloc_iova_check_hole(struct interval_tree_double_span_iter *span, unsigned long length, unsigned long iova_alignment, unsigned long page_offset) { - if (span->is_hole || span->last_used - span->start_used < length - 1) + if (span->is_used) return false; + return __alloc_iova_check_range(&span->start_hole, span->last_hole, + length, iova_alignment, page_offset); +} - span->start_used = ALIGN(span->start_used, iova_alignment) | - page_offset; - if (span->start_used > span->last_used || - span->last_used - span->start_used < length - 1) +static bool __alloc_iova_check_used(struct interval_tree_span_iter *span, + unsigned long length, + unsigned long iova_alignment, + unsigned long page_offset) +{ + if (span->is_hole) return false; - return true; + return __alloc_iova_check_range(&span->start_used, span->last_used, + length, iova_alignment, page_offset); } /* @@ -479,7 +488,8 @@ static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start, struct iopt_area *area; unsigned long unmapped_bytes = 0; unsigned int tries = 0; - int rc = -ENOENT; + /* If there are no mapped entries then success */ + int rc = 0; /* * The domains_rwsem must be held in read mode any time any area->pages @@ -524,8 +534,10 @@ static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start, iommufd_access_notify_unmap(iopt, area_first, length); /* Something is not responding to unmap requests. */ tries++; - if (WARN_ON(tries > 100)) - return -EDEADLOCK; + if (WARN_ON(tries > 100)) { + rc = -EDEADLOCK; + goto out_unmapped; + } goto again; } @@ -541,12 +553,11 @@ static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start, down_write(&iopt->iova_rwsem); } - if (unmapped_bytes) - rc = 0; out_unlock_iova: up_write(&iopt->iova_rwsem); up_read(&iopt->domains_rwsem); +out_unmapped: if (unmapped) *unmapped = unmapped_bytes; return rc; @@ -578,13 +589,8 @@ int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped) { - int rc; - - rc = iopt_unmap_iova_range(iopt, 0, ULONG_MAX, unmapped); /* If the IOVAs are empty then unmap all succeeds */ - if (rc == -ENOENT) - return 0; - return rc; + return iopt_unmap_iova_range(iopt, 0, ULONG_MAX, unmapped); } /* The caller must always free all the nodes in the allowed_iova rb_root. */ diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c index 0407e2b758ef4..18bbbeef5cccd 100644 --- a/drivers/iommu/iommufd/ioas.c +++ b/drivers/iommu/iommufd/ioas.c @@ -317,6 +317,10 @@ int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd) &unmapped); if (rc) goto out_put; + if (!unmapped) { + rc = -ENOENT; + goto out_put; + } } cmd->length = unmapped; diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 42cffb0ee5e28..2bd899061c453 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -107,6 +107,14 @@ static int of_iommu_configure_device(struct device_node *master_np, of_iommu_configure_dev(master_np, dev); } +static void of_pci_check_device_ats(struct device *dev, struct device_node *np) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + + if (fwspec && of_property_read_bool(np, "ats-supported")) + fwspec->flags |= IOMMU_FWSPEC_PCI_RC_ATS; +} + const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np, const u32 *id) @@ -144,6 +152,7 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, pci_request_acs(); err = pci_for_each_dma_alias(to_pci_dev(dev), of_pci_iommu_init, &info); + of_pci_check_device_ats(dev, master_np); } else { err = of_iommu_configure_device(master_np, dev, id); } diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 8ff69fbf9f65d..36fec26d2a04a 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1177,7 +1177,6 @@ static int rk_iommu_of_xlate(struct device *dev, iommu_dev = of_find_device_by_node(args->np); data->iommu = platform_get_drvdata(iommu_dev); - data->iommu->domain = &rk_identity_domain; dev_iommu_priv_set(dev, data); platform_device_put(iommu_dev); @@ -1217,6 +1216,8 @@ static int rk_iommu_probe(struct platform_device *pdev) if (!iommu) return -ENOMEM; + iommu->domain = &rk_identity_domain; + platform_set_drvdata(pdev, iommu); iommu->dev = dev; iommu->num_mmu = 0; diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index e7b736800dd02..4ff91df769478 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -483,6 +483,7 @@ config IMX_MU_MSI tristate "i.MX MU used as MSI controller" depends on OF && HAS_IOMEM depends on ARCH_MXC || COMPILE_TEST + depends on ARM || ARM64 default m if ARCH_MXC select IRQ_DOMAIN select IRQ_DOMAIN_HIERARCHY diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index 683e8721e3b49..4bce475824171 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -179,14 +179,19 @@ static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, { msi_alloc_info_t *info = args; struct v2m_data *v2m = NULL, *tmp; - int hwirq, offset, i, err = 0; + int hwirq, i, err = 0; + unsigned long offset; + unsigned long align_mask = nr_irqs - 1; spin_lock(&v2m_lock); list_for_each_entry(tmp, &v2m_nodes, entry) { - offset = bitmap_find_free_region(tmp->bm, tmp->nr_spis, - get_count_order(nr_irqs)); - if (offset >= 0) { + unsigned long align_off = tmp->spi_start - (tmp->spi_start & ~align_mask); + + offset = bitmap_find_next_zero_area_off(tmp->bm, tmp->nr_spis, 0, + nr_irqs, align_mask, align_off); + if (offset < tmp->nr_spis) { v2m = tmp; + bitmap_set(v2m->bm, offset, nr_irqs); break; } } diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index ebf94f04c06e4..aa082055e502e 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -965,15 +965,11 @@ static void __init gic_dist_init(void) * enabled. */ affinity = gic_cpu_to_affinity(smp_processor_id()); - for (i = 32; i < GIC_LINE_NR; i++) { - trace_android_vh_gic_v3_affinity_init(i, GICD_IROUTER, &affinity); + for (i = 32; i < GIC_LINE_NR; i++) gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); - } - for (i = 0; i < GIC_ESPI_NR; i++) { - trace_android_vh_gic_v3_affinity_init(i, GICD_IROUTERnE, &affinity); + for (i = 0; i < GIC_ESPI_NR; i++) gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); - } } static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) @@ -1437,9 +1433,6 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, reg = gic_dist_base(d) + offset + (index * 8); val = gic_cpu_to_affinity(cpu); - trace_android_rvh_gic_v3_set_affinity(d, mask_val, &val, force, gic_dist_base(d), - gic_data.redist_regions[0].redist_base, - gic_data.redist_stride); gic_write_irouter(val, reg); /* diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 95dcb83d53356..2c6c50348afd1 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include @@ -818,8 +817,6 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, writeb_relaxed(gic_cpu_map[cpu], reg); irq_data_update_effective_affinity(d, cpumask_of(cpu)); - trace_android_vh_gic_set_affinity(d, mask_val, force, gic_cpu_map, reg); - return IRQ_SET_MASK_OK_DONE; } diff --git a/drivers/irqchip/irq-loongson-pch-lpc.c b/drivers/irqchip/irq-loongson-pch-lpc.c index 9b35492fb6be9..1e9fcea1826f8 100644 --- a/drivers/irqchip/irq-loongson-pch-lpc.c +++ b/drivers/irqchip/irq-loongson-pch-lpc.c @@ -198,8 +198,13 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent, goto iounmap_base; } - priv->lpc_domain = irq_domain_create_linear(irq_handle, LPC_COUNT, - &pch_lpc_domain_ops, priv); + /* + * The LPC interrupt controller is a legacy i8259-compatible device, + * which requires a static 1:1 mapping for IRQs 0-15. + * Use irq_domain_create_legacy to establish this static mapping early. + */ + priv->lpc_domain = irq_domain_create_legacy(irq_handle, LPC_COUNT, 0, 0, + &pch_lpc_domain_ops, priv); if (!priv->lpc_domain) { pr_err("Failed to create IRQ domain\n"); goto free_irq_handle; diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c index 7124565234a58..0807e4aca933f 100644 --- a/drivers/irqchip/irq-qcom-mpm.c +++ b/drivers/irqchip/irq-qcom-mpm.c @@ -226,6 +226,9 @@ static int qcom_mpm_alloc(struct irq_domain *domain, unsigned int virq, if (ret) return ret; + if (pin == GPIO_NO_WAKE_IRQ) + return irq_domain_disconnect_hierarchy(domain, virq); + ret = irq_domain_set_hwirq_and_chip(domain, virq, pin, &qcom_mpm_chip, priv); if (ret) diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c index 627beae9649a2..84cb9cda365ad 100644 --- a/drivers/irqchip/irq-riscv-intc.c +++ b/drivers/irqchip/irq-riscv-intc.c @@ -149,7 +149,8 @@ static int riscv_intc_domain_alloc(struct irq_domain *domain, static const struct irq_domain_ops riscv_intc_domain_ops = { .map = riscv_intc_domain_map, .xlate = irq_domain_xlate_onecell, - .alloc = riscv_intc_domain_alloc + .alloc = riscv_intc_domain_alloc, + .free = irq_domain_free_irqs_top, }; static struct fwnode_handle *riscv_intc_hwnode(void) diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index 5728996691549..a8f5cfad16f7d 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -176,12 +176,14 @@ static int plic_set_affinity(struct irq_data *d, if (cpu >= nr_cpu_ids) return -EINVAL; - plic_irq_disable(d); + /* Invalidate the original routing entry */ + plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 0); irq_data_update_effective_affinity(d, cpumask_of(cpu)); + /* Setting the new routing entry if irq is enabled */ if (!irqd_irq_disabled(d)) - plic_irq_enable(d); + plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 1); return IRQ_SET_MASK_OK_DONE; } @@ -248,11 +250,11 @@ static int plic_irq_suspend(void) priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; - for (i = 0; i < priv->nr_irqs; i++) - if (readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID)) - __set_bit(i, priv->prio_save); - else - __clear_bit(i, priv->prio_save); + /* irq ID 0 is reserved */ + for (i = 1; i < priv->nr_irqs; i++) { + __assign_bit(i, priv->prio_save, + readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID)); + } for_each_cpu(cpu, cpu_present_mask) { struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu); @@ -279,7 +281,8 @@ static void plic_irq_resume(void) priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; - for (i = 0; i < priv->nr_irqs; i++) { + /* irq ID 0 is reserved */ + for (i = 1; i < priv->nr_irqs; i++) { index = BIT_WORD(i); writel((priv->prio_save[index] & BIT_MASK(i)) ? 1 : 0, priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID); diff --git a/drivers/irqchip/irq-sun6i-r.c b/drivers/irqchip/irq-sun6i-r.c index a01e440494154..99958d470d626 100644 --- a/drivers/irqchip/irq-sun6i-r.c +++ b/drivers/irqchip/irq-sun6i-r.c @@ -270,7 +270,7 @@ static const struct irq_domain_ops sun6i_r_intc_domain_ops = { static int sun6i_r_intc_suspend(void) { - u32 buf[BITS_TO_U32(max(SUN6I_NR_TOP_LEVEL_IRQS, SUN6I_NR_MUX_BITS))]; + u32 buf[BITS_TO_U32(MAX(SUN6I_NR_TOP_LEVEL_IRQS, SUN6I_NR_MUX_BITS))]; int i; /* Wake IRQs are enabled during system sleep and shutdown. */ diff --git a/drivers/isdn/hardware/mISDN/hfcsusb.c b/drivers/isdn/hardware/mISDN/hfcsusb.c index b82b89888a5e0..c55438e1f6780 100644 --- a/drivers/isdn/hardware/mISDN/hfcsusb.c +++ b/drivers/isdn/hardware/mISDN/hfcsusb.c @@ -1903,13 +1903,13 @@ setup_instance(struct hfcsusb *hw, struct device *parent) mISDN_freebchannel(&hw->bch[1]); mISDN_freebchannel(&hw->bch[0]); mISDN_freedchannel(&hw->dch); - kfree(hw); return err; } static int hfcsusb_probe(struct usb_interface *intf, const struct usb_device_id *id) { + int err; struct hfcsusb *hw; struct usb_device *dev = interface_to_usbdev(intf); struct usb_host_interface *iface = intf->cur_altsetting; @@ -2100,20 +2100,28 @@ hfcsusb_probe(struct usb_interface *intf, const struct usb_device_id *id) if (!hw->ctrl_urb) { pr_warn("%s: No memory for control urb\n", driver_info->vend_name); - kfree(hw); - return -ENOMEM; + err = -ENOMEM; + goto err_free_hw; } pr_info("%s: %s: detected \"%s\" (%s, if=%d alt=%d)\n", hw->name, __func__, driver_info->vend_name, conf_str[small_match], ifnum, alt_used); - if (setup_instance(hw, dev->dev.parent)) - return -EIO; + if (setup_instance(hw, dev->dev.parent)) { + err = -EIO; + goto err_free_urb; + } hw->intf = intf; usb_set_intfdata(hw->intf, hw); return 0; + +err_free_urb: + usb_free_urb(hw->ctrl_urb); +err_free_hw: + kfree(hw); + return err; } /* function called when an active device is removed */ diff --git a/drivers/isdn/mISDN/dsp_hwec.c b/drivers/isdn/mISDN/dsp_hwec.c index 0b3f29195330a..0cd216e28f009 100644 --- a/drivers/isdn/mISDN/dsp_hwec.c +++ b/drivers/isdn/mISDN/dsp_hwec.c @@ -51,14 +51,14 @@ void dsp_hwec_enable(struct dsp *dsp, const char *arg) goto _do; { - char *dup, *tok, *name, *val; + char *dup, *next, *tok, *name, *val; int tmp; - dup = kstrdup(arg, GFP_ATOMIC); + dup = next = kstrdup(arg, GFP_ATOMIC); if (!dup) return; - while ((tok = strsep(&dup, ","))) { + while ((tok = strsep(&next, ","))) { if (!strlen(tok)) continue; name = strsep(&tok, "="); diff --git a/drivers/leds/flash/leds-qcom-flash.c b/drivers/leds/flash/leds-qcom-flash.c index 17391aefeb941..07ca3e32d4bec 100644 --- a/drivers/leds/flash/leds-qcom-flash.c +++ b/drivers/leds/flash/leds-qcom-flash.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -14,6 +14,9 @@ #include /* registers definitions */ +#define FLASH_REVISION_REG 0x00 +#define FLASH_4CH_REVISION_V0P1 0x01 + #define FLASH_TYPE_REG 0x04 #define FLASH_TYPE_VAL 0x18 @@ -73,6 +76,16 @@ #define UA_PER_MA 1000 +/* thermal threshold constants */ +#define OTST_3CH_MIN_VAL 3 +#define OTST1_4CH_MIN_VAL 0 +#define OTST1_4CH_V0P1_MIN_VAL 3 +#define OTST2_4CH_MIN_VAL 0 + +#define OTST1_MAX_CURRENT_MA 1000 +#define OTST2_MAX_CURRENT_MA 500 +#define OTST3_MAX_CURRENT_MA 200 + enum hw_type { QCOM_MVFLASH_3CH, QCOM_MVFLASH_4CH, @@ -98,31 +111,42 @@ enum { REG_IRESOLUTION, REG_CHAN_STROBE, REG_CHAN_EN, + REG_THERM_THRSH1, + REG_THERM_THRSH2, + REG_THERM_THRSH3, + REG_TORCH_CLAMP, REG_MAX_COUNT, }; -static struct reg_field mvflash_3ch_regs[REG_MAX_COUNT] = { - REG_FIELD(0x08, 0, 7), /* status1 */ - REG_FIELD(0x09, 0, 7), /* status2 */ - REG_FIELD(0x0a, 0, 7), /* status3 */ - REG_FIELD_ID(0x40, 0, 7, 3, 1), /* chan_timer */ - REG_FIELD_ID(0x43, 0, 6, 3, 1), /* itarget */ - REG_FIELD(0x46, 7, 7), /* module_en */ - REG_FIELD(0x47, 0, 5), /* iresolution */ - REG_FIELD_ID(0x49, 0, 2, 3, 1), /* chan_strobe */ - REG_FIELD(0x4c, 0, 2), /* chan_en */ +static const struct reg_field mvflash_3ch_regs[REG_MAX_COUNT] = { + [REG_STATUS1] = REG_FIELD(0x08, 0, 7), + [REG_STATUS2] = REG_FIELD(0x09, 0, 7), + [REG_STATUS3] = REG_FIELD(0x0a, 0, 7), + [REG_CHAN_TIMER] = REG_FIELD_ID(0x40, 0, 7, 3, 1), + [REG_ITARGET] = REG_FIELD_ID(0x43, 0, 6, 3, 1), + [REG_MODULE_EN] = REG_FIELD(0x46, 7, 7), + [REG_IRESOLUTION] = REG_FIELD(0x47, 0, 5), + [REG_CHAN_STROBE] = REG_FIELD_ID(0x49, 0, 2, 3, 1), + [REG_CHAN_EN] = REG_FIELD(0x4c, 0, 2), + [REG_THERM_THRSH1] = REG_FIELD(0x56, 0, 2), + [REG_THERM_THRSH2] = REG_FIELD(0x57, 0, 2), + [REG_THERM_THRSH3] = REG_FIELD(0x58, 0, 2), + [REG_TORCH_CLAMP] = REG_FIELD(0xec, 0, 6), }; -static struct reg_field mvflash_4ch_regs[REG_MAX_COUNT] = { - REG_FIELD(0x06, 0, 7), /* status1 */ - REG_FIELD(0x07, 0, 6), /* status2 */ - REG_FIELD(0x09, 0, 7), /* status3 */ - REG_FIELD_ID(0x3e, 0, 7, 4, 1), /* chan_timer */ - REG_FIELD_ID(0x42, 0, 6, 4, 1), /* itarget */ - REG_FIELD(0x46, 7, 7), /* module_en */ - REG_FIELD(0x49, 0, 3), /* iresolution */ - REG_FIELD_ID(0x4a, 0, 6, 4, 1), /* chan_strobe */ - REG_FIELD(0x4e, 0, 3), /* chan_en */ +static const struct reg_field mvflash_4ch_regs[REG_MAX_COUNT] = { + [REG_STATUS1] = REG_FIELD(0x06, 0, 7), + [REG_STATUS2] = REG_FIELD(0x07, 0, 6), + [REG_STATUS3] = REG_FIELD(0x09, 0, 7), + [REG_CHAN_TIMER] = REG_FIELD_ID(0x3e, 0, 7, 4, 1), + [REG_ITARGET] = REG_FIELD_ID(0x42, 0, 6, 4, 1), + [REG_MODULE_EN] = REG_FIELD(0x46, 7, 7), + [REG_IRESOLUTION] = REG_FIELD(0x49, 0, 3), + [REG_CHAN_STROBE] = REG_FIELD_ID(0x4a, 0, 6, 4, 1), + [REG_CHAN_EN] = REG_FIELD(0x4e, 0, 3), + [REG_THERM_THRSH1] = REG_FIELD(0x7a, 0, 2), + [REG_THERM_THRSH2] = REG_FIELD(0x78, 0, 2), + [REG_TORCH_CLAMP] = REG_FIELD(0xed, 0, 6), }; struct qcom_flash_data { @@ -130,9 +154,12 @@ struct qcom_flash_data { struct regmap_field *r_fields[REG_MAX_COUNT]; struct mutex lock; enum hw_type hw_type; + u32 total_ma; u8 leds_count; u8 max_channels; u8 chan_en_bits; + u8 revision; + u8 torch_clamp; }; struct qcom_flash_led { @@ -143,6 +170,7 @@ struct qcom_flash_led { u32 max_timeout_ms; u32 flash_current_ma; u32 flash_timeout_ms; + u32 current_in_use_ma; u8 *chan_id; u8 chan_count; bool enabled; @@ -172,6 +200,127 @@ static int set_flash_module_en(struct qcom_flash_led *led, bool en) return rc; } +static int update_allowed_flash_current(struct qcom_flash_led *led, u32 *current_ma, bool strobe) +{ + struct qcom_flash_data *flash_data = led->flash_data; + u32 therm_ma, avail_ma, thrsh[3], min_thrsh, sts; + int rc = 0; + + mutex_lock(&flash_data->lock); + /* + * Put previously allocated current into allowed budget in either of these two cases: + * 1) LED is disabled; + * 2) LED is enabled repeatedly + */ + if (!strobe || led->current_in_use_ma != 0) { + if (flash_data->total_ma >= led->current_in_use_ma) + flash_data->total_ma -= led->current_in_use_ma; + else + flash_data->total_ma = 0; + + led->current_in_use_ma = 0; + if (!strobe) + goto unlock; + } + + /* + * Cache the default thermal threshold settings, and set them to the lowest levels before + * reading over-temp real time status. If over-temp has been triggered at the lowest + * threshold, it's very likely that it would be triggered at a higher (default) threshold + * when more flash current is requested. Prevent device from triggering over-temp condition + * by limiting the flash current for the new request. + */ + rc = regmap_field_read(flash_data->r_fields[REG_THERM_THRSH1], &thrsh[0]); + if (rc < 0) + goto unlock; + + rc = regmap_field_read(flash_data->r_fields[REG_THERM_THRSH2], &thrsh[1]); + if (rc < 0) + goto unlock; + + if (flash_data->hw_type == QCOM_MVFLASH_3CH) { + rc = regmap_field_read(flash_data->r_fields[REG_THERM_THRSH3], &thrsh[2]); + if (rc < 0) + goto unlock; + } + + min_thrsh = OTST_3CH_MIN_VAL; + if (flash_data->hw_type == QCOM_MVFLASH_4CH) + min_thrsh = (flash_data->revision == FLASH_4CH_REVISION_V0P1) ? + OTST1_4CH_V0P1_MIN_VAL : OTST1_4CH_MIN_VAL; + + rc = regmap_field_write(flash_data->r_fields[REG_THERM_THRSH1], min_thrsh); + if (rc < 0) + goto unlock; + + if (flash_data->hw_type == QCOM_MVFLASH_4CH) + min_thrsh = OTST2_4CH_MIN_VAL; + + /* + * The default thermal threshold settings have been updated hence + * restore them if any fault happens starting from here. + */ + rc = regmap_field_write(flash_data->r_fields[REG_THERM_THRSH2], min_thrsh); + if (rc < 0) + goto restore; + + if (flash_data->hw_type == QCOM_MVFLASH_3CH) { + rc = regmap_field_write(flash_data->r_fields[REG_THERM_THRSH3], min_thrsh); + if (rc < 0) + goto restore; + } + + /* Read thermal level status to get corresponding derating flash current */ + rc = regmap_field_read(flash_data->r_fields[REG_STATUS2], &sts); + if (rc) + goto restore; + + therm_ma = FLASH_TOTAL_CURRENT_MAX_UA / 1000; + if (flash_data->hw_type == QCOM_MVFLASH_3CH) { + if (sts & FLASH_STS_3CH_OTST3) + therm_ma = OTST3_MAX_CURRENT_MA; + else if (sts & FLASH_STS_3CH_OTST2) + therm_ma = OTST2_MAX_CURRENT_MA; + else if (sts & FLASH_STS_3CH_OTST1) + therm_ma = OTST1_MAX_CURRENT_MA; + } else { + if (sts & FLASH_STS_4CH_OTST2) + therm_ma = OTST2_MAX_CURRENT_MA; + else if (sts & FLASH_STS_4CH_OTST1) + therm_ma = OTST1_MAX_CURRENT_MA; + } + + /* Calculate the allowed flash current for the request */ + if (therm_ma <= flash_data->total_ma) + avail_ma = 0; + else + avail_ma = therm_ma - flash_data->total_ma; + + *current_ma = min_t(u32, *current_ma, avail_ma); + led->current_in_use_ma = *current_ma; + flash_data->total_ma += led->current_in_use_ma; + + dev_dbg(led->flash.led_cdev.dev, "allowed flash current: %dmA, total current: %dmA\n", + led->current_in_use_ma, flash_data->total_ma); + +restore: + /* Restore to default thermal threshold settings */ + rc = regmap_field_write(flash_data->r_fields[REG_THERM_THRSH1], thrsh[0]); + if (rc < 0) + goto unlock; + + rc = regmap_field_write(flash_data->r_fields[REG_THERM_THRSH2], thrsh[1]); + if (rc < 0) + goto unlock; + + if (flash_data->hw_type == QCOM_MVFLASH_3CH) + rc = regmap_field_write(flash_data->r_fields[REG_THERM_THRSH3], thrsh[2]); + +unlock: + mutex_unlock(&flash_data->lock); + return rc; +} + static int set_flash_current(struct qcom_flash_led *led, u32 current_ma, enum led_mode mode) { struct qcom_flash_data *flash_data = led->flash_data; @@ -313,6 +462,10 @@ static int qcom_flash_strobe_set(struct led_classdev_flash *fled_cdev, bool stat if (rc) return rc; + rc = update_allowed_flash_current(led, &led->flash_current_ma, state); + if (rc < 0) + return rc; + rc = set_flash_current(led, led->flash_current_ma, FLASH_MODE); if (rc) return rc; @@ -429,6 +582,10 @@ static int qcom_flash_led_brightness_set(struct led_classdev *led_cdev, if (rc) return rc; + rc = update_allowed_flash_current(led, ¤t_ma, enable); + if (rc < 0) + return rc; + rc = set_flash_current(led, current_ma, TORCH_MODE); if (rc) return rc; @@ -549,6 +706,7 @@ static int qcom_flash_register_led_device(struct device *dev, u32 current_ua, timeout_us; u32 channels[4]; int i, rc, count; + u8 torch_clamp; count = fwnode_property_count_u32(node, "led-sources"); if (count <= 0) { @@ -598,6 +756,12 @@ static int qcom_flash_register_led_device(struct device *dev, current_ua = min_t(u32, current_ua, TORCH_CURRENT_MAX_UA * led->chan_count); led->max_torch_current_ma = current_ua / UA_PER_MA; + torch_clamp = (current_ua / led->chan_count) / TORCH_IRES_UA; + if (torch_clamp != 0) + torch_clamp--; + + flash_data->torch_clamp = max_t(u8, flash_data->torch_clamp, torch_clamp); + if (fwnode_property_present(node, "flash-max-microamp")) { flash->led_cdev.flags |= LED_DEV_CAP_FLASH; @@ -702,11 +866,25 @@ static int qcom_flash_led_probe(struct platform_device *pdev) if (val == FLASH_SUBTYPE_3CH_PM8150_VAL || val == FLASH_SUBTYPE_3CH_PMI8998_VAL) { flash_data->hw_type = QCOM_MVFLASH_3CH; flash_data->max_channels = 3; - regs = mvflash_3ch_regs; + regs = devm_kmemdup(dev, mvflash_3ch_regs, sizeof(mvflash_3ch_regs), + GFP_KERNEL); + if (!regs) + return -ENOMEM; } else if (val == FLASH_SUBTYPE_4CH_VAL) { flash_data->hw_type = QCOM_MVFLASH_4CH; flash_data->max_channels = 4; - regs = mvflash_4ch_regs; + regs = devm_kmemdup(dev, mvflash_4ch_regs, sizeof(mvflash_4ch_regs), + GFP_KERNEL); + if (!regs) + return -ENOMEM; + + rc = regmap_read(regmap, reg_base + FLASH_REVISION_REG, &val); + if (rc < 0) { + dev_err(dev, "Failed to read flash LED module revision, rc=%d\n", rc); + return rc; + } + + flash_data->revision = val; } else { dev_err(dev, "flash LED subtype %#x is not yet supported\n", val); return -ENODEV; @@ -720,6 +898,7 @@ static int qcom_flash_led_probe(struct platform_device *pdev) dev_err(dev, "Failed to allocate regmap field, rc=%d\n", rc); return rc; } + devm_kfree(dev, regs); /* devm_regmap_field_bulk_alloc() makes copies */ platform_set_drvdata(pdev, flash_data); mutex_init(&flash_data->lock); @@ -750,8 +929,7 @@ static int qcom_flash_led_probe(struct platform_device *pdev) flash_data->leds_count++; } - return 0; - + return regmap_field_write(flash_data->r_fields[REG_TORCH_CLAMP], flash_data->torch_clamp); release: fwnode_handle_put(child); while (flash_data->v4l2_flash[flash_data->leds_count] && flash_data->leds_count) diff --git a/drivers/leds/led-class-multicolor.c b/drivers/leds/led-class-multicolor.c index ec62a48116135..e0785935f4ba6 100644 --- a/drivers/leds/led-class-multicolor.c +++ b/drivers/leds/led-class-multicolor.c @@ -61,7 +61,8 @@ static ssize_t multi_intensity_store(struct device *dev, for (i = 0; i < mcled_cdev->num_colors; i++) mcled_cdev->subled_info[i].intensity = intensity_value[i]; - led_set_brightness(led_cdev, led_cdev->brightness); + if (!test_bit(LED_BLINK_SW, &led_cdev->work_flags)) + led_set_brightness(led_cdev, led_cdev->brightness); ret = size; err_out: mutex_unlock(&led_cdev->led_access); diff --git a/drivers/leds/leds-lp50xx.c b/drivers/leds/leds-lp50xx.c index 68c4d9967d683..182a590b02677 100644 --- a/drivers/leds/leds-lp50xx.c +++ b/drivers/leds/leds-lp50xx.c @@ -486,6 +486,7 @@ static int lp50xx_probe_dt(struct lp50xx *priv) } fwnode_for_each_child_node(child, led_node) { + int multi_index; ret = fwnode_property_read_u32(led_node, "color", &color_id); if (ret) { @@ -493,8 +494,16 @@ static int lp50xx_probe_dt(struct lp50xx *priv) dev_err(priv->dev, "Cannot read color\n"); goto child_out; } + ret = fwnode_property_read_u32(led_node, "reg", &multi_index); + if (ret != 0) { + dev_err(priv->dev, "reg must be set\n"); + return -EINVAL; + } else if (multi_index >= LP50XX_LEDS_PER_MODULE) { + dev_err(priv->dev, "reg %i out of range\n", multi_index); + return -EINVAL; + } - mc_led_info[num_colors].color_index = color_id; + mc_led_info[multi_index].color_index = color_id; num_colors++; } diff --git a/drivers/leds/leds-spi-byte.c b/drivers/leds/leds-spi-byte.c index afe9bff7c7c16..b04cf502e6035 100644 --- a/drivers/leds/leds-spi-byte.c +++ b/drivers/leds/leds-spi-byte.c @@ -83,7 +83,7 @@ static int spi_byte_probe(struct spi_device *spi) struct device_node *child; struct device *dev = &spi->dev; struct spi_byte_led *led; - const char *name = "leds-spi-byte::"; + struct led_init_data init_data = {}; const char *state; int ret; @@ -96,12 +96,9 @@ static int spi_byte_probe(struct spi_device *spi) if (!led) return -ENOMEM; - of_property_read_string(child, "label", &name); - strscpy(led->name, name, sizeof(led->name)); led->spi = spi; mutex_init(&led->mutex); led->cdef = device_get_match_data(dev); - led->ldev.name = led->name; led->ldev.brightness = LED_OFF; led->ldev.max_brightness = led->cdef->max_value - led->cdef->off_value; led->ldev.brightness_set_blocking = spi_byte_brightness_set_blocking; @@ -121,7 +118,11 @@ static int spi_byte_probe(struct spi_device *spi) spi_byte_brightness_set_blocking(&led->ldev, led->ldev.brightness); - ret = devm_led_classdev_register(&spi->dev, &led->ldev); + init_data.fwnode = of_fwnode_handle(child); + init_data.devicename = "leds-spi-byte"; + init_data.default_label = ":"; + + ret = devm_led_classdev_register_ext(&spi->dev, &led->ldev, &init_data); if (ret) { of_node_put(child); mutex_destroy(&led->mutex); diff --git a/drivers/leds/rgb/leds-pwm-multicolor.c b/drivers/leds/rgb/leds-pwm-multicolor.c index e1a81e0109e8a..c0aa34b1d0e2d 100644 --- a/drivers/leds/rgb/leds-pwm-multicolor.c +++ b/drivers/leds/rgb/leds-pwm-multicolor.c @@ -135,8 +135,11 @@ static int led_pwm_mc_probe(struct platform_device *pdev) /* init the multicolor's LED class device */ cdev = &priv->mc_cdev.led_cdev; - fwnode_property_read_u32(mcnode, "max-brightness", + ret = fwnode_property_read_u32(mcnode, "max-brightness", &cdev->max_brightness); + if (ret) + goto release_mcnode; + cdev->flags = LED_CORE_SUSPENDRESUME; cdev->brightness_set_blocking = led_pwm_mc_set; diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index ebff3baf30451..cb59b4dbad626 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -415,11 +415,12 @@ struct mbox_chan *mbox_request_channel(struct mbox_client *cl, int index) mutex_lock(&con_mutex); - if (of_parse_phandle_with_args(dev->of_node, "mboxes", - "#mbox-cells", index, &spec)) { + ret = of_parse_phandle_with_args(dev->of_node, "mboxes", "#mbox-cells", + index, &spec); + if (ret) { dev_dbg(dev, "%s: can't parse \"mboxes\" property\n", __func__); mutex_unlock(&con_mutex); - return ERR_PTR(-ENODEV); + return ERR_PTR(ret); } chan = ERR_PTR(-EPROBE_DEFER); @@ -499,8 +500,8 @@ void mbox_free_channel(struct mbox_chan *chan) if (chan->txdone_method == TXDONE_BY_ACK) chan->txdone_method = TXDONE_BY_POLL; - module_put(chan->mbox->dev->driver->owner); spin_unlock_irqrestore(&chan->lock, flags); + module_put(chan->mbox->dev->driver->owner); } EXPORT_SYMBOL_GPL(mbox_free_channel); diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index f8215a8f656a4..49254d99a8ad6 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -419,8 +419,12 @@ int pcc_mbox_ioremap(struct mbox_chan *chan) return -1; pchan_info = chan->con_priv; pcc_mbox_chan = &pchan_info->chan; - pcc_mbox_chan->shmem = ioremap(pcc_mbox_chan->shmem_base_addr, - pcc_mbox_chan->shmem_size); + + pcc_mbox_chan->shmem = acpi_os_ioremap(pcc_mbox_chan->shmem_base_addr, + pcc_mbox_chan->shmem_size); + if (!pcc_mbox_chan->shmem) + return -ENXIO; + return 0; } EXPORT_SYMBOL_GPL(pcc_mbox_ioremap); diff --git a/drivers/mailbox/zynqmp-ipi-mailbox.c b/drivers/mailbox/zynqmp-ipi-mailbox.c index e4fcac97dbfaa..90f248ef2a1fc 100644 --- a/drivers/mailbox/zynqmp-ipi-mailbox.c +++ b/drivers/mailbox/zynqmp-ipi-mailbox.c @@ -616,11 +616,8 @@ static void zynqmp_ipi_free_mboxes(struct zynqmp_ipi_pdata *pdata) i = pdata->num_mboxes; for (; i >= 0; i--) { ipi_mbox = &pdata->ipi_mboxes[i]; - if (ipi_mbox->dev.parent) { - mbox_controller_unregister(&ipi_mbox->mbox); - if (device_is_registered(&ipi_mbox->dev)) - device_unregister(&ipi_mbox->dev); - } + if (device_is_registered(&ipi_mbox->dev)) + device_unregister(&ipi_mbox->dev); } } diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 8440b56e385d5..b9dfebaa9eae5 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1739,7 +1739,12 @@ static void cache_set_flush(struct closure *cl) mutex_unlock(&b->write_lock); } - if (ca->alloc_thread) + /* + * If the register_cache_set() call to bch_cache_set_alloc() failed, + * ca has not been assigned a value and return error. + * So we need check ca is not NULL during bch_cache_set_unregister(). + */ + if (ca && ca->alloc_thread) kthread_stop(ca->alloc_thread); if (c->journal.cur) { diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 7bd7ad1e5333c..c148dac4c41de 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -2746,7 +2746,11 @@ static unsigned long __evict_many(struct dm_bufio_client *c, __make_buffer_clean(b); __free_buffer_wake(b); - cond_resched(); + if (need_resched()) { + dm_bufio_unlock(c); + cond_resched(); + dm_bufio_lock(c); + } } return count; diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index c5851c9f7ec04..0d002d50329da 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -2903,6 +2903,27 @@ static dm_cblock_t get_cache_dev_size(struct cache *cache) return to_cblock(size); } +static bool can_resume(struct cache *cache) +{ + /* + * Disallow retrying the resume operation for devices that failed the + * first resume attempt, as the failure leaves the policy object partially + * initialized. Retrying could trigger BUG_ON when loading cache mappings + * into the incomplete policy object. + */ + if (cache->sized && !cache->loaded_mappings) { + if (get_cache_mode(cache) != CM_WRITE) + DMERR("%s: unable to resume a failed-loaded cache, please check metadata.", + cache_device_name(cache)); + else + DMERR("%s: unable to resume cache due to missing proper cache table reload", + cache_device_name(cache)); + return false; + } + + return true; +} + static bool can_resize(struct cache *cache, dm_cblock_t new_size) { if (from_cblock(new_size) > from_cblock(cache->cache_size)) { @@ -2951,6 +2972,9 @@ static int cache_preresume(struct dm_target *ti) struct cache *cache = ti->private; dm_cblock_t csize = get_cache_dev_size(cache); + if (!can_resume(cache)) + return -EINVAL; + /* * Check to see if the cache has resized. */ diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h index c2b68eda39947..2190f9d1b5124 100644 --- a/drivers/md/dm-core.h +++ b/drivers/md/dm-core.h @@ -161,6 +161,7 @@ struct mapped_device { #define DMF_SUSPENDED_INTERNALLY 7 #define DMF_POST_SUSPENDING 8 #define DMF_EMULATE_ZONE_APPEND 9 +#define DMF_QUEUE_STOPPED 10 void disable_discard(struct mapped_device *md); void disable_write_zeroes(struct mapped_device *md); diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index dc491dc771d71..ada679f4fca67 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c @@ -47,14 +47,15 @@ enum feature_flag_bits { }; struct per_bio_data { - bool bio_submitted; + bool bio_can_corrupt; + struct bvec_iter saved_iter; }; static int parse_features(struct dm_arg_set *as, struct flakey_c *fc, struct dm_target *ti) { - int r; - unsigned int argc; + int r = 0; + unsigned int argc = 0; const char *arg_name; static const struct dm_arg _args[] = { @@ -65,14 +66,13 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc, {0, PROBABILITY_BASE, "Invalid random corrupt argument"}, }; - /* No feature arguments supplied. */ - if (!as->argc) - return 0; - - r = dm_read_arg_group(_args, as, &argc, &ti->error); - if (r) + if (as->argc && (r = dm_read_arg_group(_args, as, &argc, &ti->error))) return r; + /* No feature arguments supplied. */ + if (!argc) + goto error_all_io; + while (argc) { arg_name = dm_shift_arg(as); argc--; @@ -217,6 +217,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc, if (!fc->corrupt_bio_byte && !test_bit(ERROR_READS, &fc->flags) && !test_bit(DROP_WRITES, &fc->flags) && !test_bit(ERROR_WRITES, &fc->flags) && !fc->random_read_corrupt && !fc->random_write_corrupt) { +error_all_io: set_bit(ERROR_WRITES, &fc->flags); set_bit(ERROR_READS, &fc->flags); } @@ -339,7 +340,8 @@ static void flakey_map_bio(struct dm_target *ti, struct bio *bio) } static void corrupt_bio_common(struct bio *bio, unsigned int corrupt_bio_byte, - unsigned char corrupt_bio_value) + unsigned char corrupt_bio_value, + struct bvec_iter start) { struct bvec_iter iter; struct bio_vec bvec; @@ -348,7 +350,7 @@ static void corrupt_bio_common(struct bio *bio, unsigned int corrupt_bio_byte, * Overwrite the Nth byte of the bio's data, on whichever page * it falls. */ - bio_for_each_segment(bvec, bio, iter) { + __bio_for_each_segment(bvec, bio, iter, start) { if (bio_iter_len(bio, iter) > corrupt_bio_byte) { unsigned char *segment = bvec_kmap_local(&bvec); segment[corrupt_bio_byte] = corrupt_bio_value; @@ -357,36 +359,31 @@ static void corrupt_bio_common(struct bio *bio, unsigned int corrupt_bio_byte, "(rw=%c bi_opf=%u bi_sector=%llu size=%u)\n", bio, corrupt_bio_value, corrupt_bio_byte, (bio_data_dir(bio) == WRITE) ? 'w' : 'r', bio->bi_opf, - (unsigned long long)bio->bi_iter.bi_sector, - bio->bi_iter.bi_size); + (unsigned long long)start.bi_sector, + start.bi_size); break; } corrupt_bio_byte -= bio_iter_len(bio, iter); } } -static void corrupt_bio_data(struct bio *bio, struct flakey_c *fc) +static void corrupt_bio_data(struct bio *bio, struct flakey_c *fc, + struct bvec_iter start) { unsigned int corrupt_bio_byte = fc->corrupt_bio_byte - 1; - if (!bio_has_data(bio)) - return; - - corrupt_bio_common(bio, corrupt_bio_byte, fc->corrupt_bio_value); + corrupt_bio_common(bio, corrupt_bio_byte, fc->corrupt_bio_value, start); } -static void corrupt_bio_random(struct bio *bio) +static void corrupt_bio_random(struct bio *bio, struct bvec_iter start) { unsigned int corrupt_byte; unsigned char corrupt_value; - if (!bio_has_data(bio)) - return; - - corrupt_byte = get_random_u32() % bio->bi_iter.bi_size; + corrupt_byte = get_random_u32() % start.bi_size; corrupt_value = get_random_u8(); - corrupt_bio_common(bio, corrupt_byte, corrupt_value); + corrupt_bio_common(bio, corrupt_byte, corrupt_value, start); } static void clone_free(struct bio *clone) @@ -481,7 +478,7 @@ static int flakey_map(struct dm_target *ti, struct bio *bio) unsigned int elapsed; struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data)); - pb->bio_submitted = false; + pb->bio_can_corrupt = false; if (op_is_zone_mgmt(bio_op(bio))) goto map_bio; @@ -490,10 +487,11 @@ static int flakey_map(struct dm_target *ti, struct bio *bio) elapsed = (jiffies - fc->start_time) / HZ; if (elapsed % (fc->up_interval + fc->down_interval) >= fc->up_interval) { bool corrupt_fixed, corrupt_random; - /* - * Flag this bio as submitted while down. - */ - pb->bio_submitted = true; + + if (bio_has_data(bio)) { + pb->bio_can_corrupt = true; + pb->saved_iter = bio->bi_iter; + } /* * Error reads if neither corrupt_bio_byte or drop_writes or error_writes are set. @@ -516,6 +514,8 @@ static int flakey_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_SUBMITTED; } + if (!pb->bio_can_corrupt) + goto map_bio; /* * Corrupt matching writes. */ @@ -535,9 +535,11 @@ static int flakey_map(struct dm_target *ti, struct bio *bio) struct bio *clone = clone_bio(ti, fc, bio); if (clone) { if (corrupt_fixed) - corrupt_bio_data(clone, fc); + corrupt_bio_data(clone, fc, + clone->bi_iter); if (corrupt_random) - corrupt_bio_random(clone); + corrupt_bio_random(clone, + clone->bi_iter); submit_bio(clone); return DM_MAPIO_SUBMITTED; } @@ -559,21 +561,21 @@ static int flakey_end_io(struct dm_target *ti, struct bio *bio, if (op_is_zone_mgmt(bio_op(bio))) return DM_ENDIO_DONE; - if (!*error && pb->bio_submitted && (bio_data_dir(bio) == READ)) { + if (!*error && pb->bio_can_corrupt && (bio_data_dir(bio) == READ)) { if (fc->corrupt_bio_byte) { if ((fc->corrupt_bio_rw == READ) && all_corrupt_bio_flags_match(bio, fc)) { /* * Corrupt successful matching READs while in down state. */ - corrupt_bio_data(bio, fc); + corrupt_bio_data(bio, fc, pb->saved_iter); } } if (fc->random_read_corrupt) { u64 rnd = get_random_u64(); u32 rem = do_div(rnd, PROBABILITY_BASE); if (rem < fc->random_read_corrupt) - corrupt_bio_random(bio); + corrupt_bio_random(bio, pb->saved_iter); } if (test_bit(ERROR_READS, &fc->flags)) { /* diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index eb2b44f4a61f0..6442d41622eff 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -129,7 +129,7 @@ struct journal_sector { commit_id_t commit_id; }; -#define MAX_TAG_SIZE (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK])) +#define MAX_TAG_SIZE 255 #define METADATA_PADDING_SECTORS 8 @@ -1794,7 +1794,7 @@ static void integrity_metadata(struct work_struct *w) struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); char *checksums; unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; - char checksums_onstack[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; + char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; sector_t sector; unsigned int sectors_to_process; @@ -2073,7 +2073,7 @@ static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, } while (++s < ic->sectors_per_block); #ifdef INTERNAL_VERIFY if (ic->internal_hash) { - char checksums_onstack[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; + char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack); if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { @@ -2638,7 +2638,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start unlikely(from_replay) && #endif ic->internal_hash) { - char test_tag[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; + char test_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), (char *)access_journal_data(ic, i, l), test_tag); @@ -4687,7 +4687,7 @@ static void dm_integrity_dtr(struct dm_target *ti) BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); BUG_ON(!list_empty(&ic->wait_list)); - if (ic->mode == 'B') + if (ic->mode == 'B' && ic->bitmap_flush_work.work.func) cancel_delayed_work_sync(&ic->bitmap_flush_work); if (ic->metadata_wq) destroy_workqueue(ic->metadata_wq); diff --git a/drivers/md/dm-ps-historical-service-time.c b/drivers/md/dm-ps-historical-service-time.c index b49e10d76d030..2c8626a83de43 100644 --- a/drivers/md/dm-ps-historical-service-time.c +++ b/drivers/md/dm-ps-historical-service-time.c @@ -541,8 +541,10 @@ static int __init dm_hst_init(void) { int r = dm_register_path_selector(&hst_ps); - if (r < 0) + if (r < 0) { DMERR("register failed %d", r); + return r; + } DMINFO("version " HST_VERSION " loaded"); diff --git a/drivers/md/dm-ps-queue-length.c b/drivers/md/dm-ps-queue-length.c index e305f05ad1e5e..eb543e6431e03 100644 --- a/drivers/md/dm-ps-queue-length.c +++ b/drivers/md/dm-ps-queue-length.c @@ -260,8 +260,10 @@ static int __init dm_ql_init(void) { int r = dm_register_path_selector(&ql_ps); - if (r < 0) + if (r < 0) { DMERR("register failed %d", r); + return r; + } DMINFO("version " QL_VERSION " loaded"); diff --git a/drivers/md/dm-ps-round-robin.c b/drivers/md/dm-ps-round-robin.c index 0f04b673597ae..62ac820125cbd 100644 --- a/drivers/md/dm-ps-round-robin.c +++ b/drivers/md/dm-ps-round-robin.c @@ -220,8 +220,10 @@ static int __init dm_rr_init(void) { int r = dm_register_path_selector(&rr_ps); - if (r < 0) + if (r < 0) { DMERR("register failed %d", r); + return r; + } DMINFO("version " RR_VERSION " loaded"); diff --git a/drivers/md/dm-ps-service-time.c b/drivers/md/dm-ps-service-time.c index 969d31c40272e..f8c43aecdb27a 100644 --- a/drivers/md/dm-ps-service-time.c +++ b/drivers/md/dm-ps-service-time.c @@ -341,8 +341,10 @@ static int __init dm_st_init(void) { int r = dm_register_path_selector(&st_ps); - if (r < 0) + if (r < 0) { DMERR("register failed %d", r); + return r; + } DMINFO("version " ST_VERSION " loaded"); diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 385e24f55ec00..f23edd79df45e 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -2378,7 +2378,7 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) */ sb_retrieve_failed_devices(sb, failed_devices); rdev_for_each(r, mddev) { - if (test_bit(Journal, &rdev->flags) || + if (test_bit(Journal, &r->flags) || !r->sb_page) continue; sb2 = page_address(r->sb_page); diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index 9511dae5b556a..94b6c43dfa5cb 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -133,10 +133,9 @@ static void queue_bio(struct mirror_set *ms, struct bio *bio, int rw) spin_lock_irqsave(&ms->lock, flags); should_wake = !(bl->head); bio_list_add(bl, bio); - spin_unlock_irqrestore(&ms->lock, flags); - if (should_wake) wakeup_mirrord(ms); + spin_unlock_irqrestore(&ms->lock, flags); } static void dispatch_bios(void *context, struct bio_list *bio_list) @@ -646,9 +645,9 @@ static void write_callback(unsigned long error, void *context) if (!ms->failures.head) should_wake = 1; bio_list_add(&ms->failures, bio); - spin_unlock_irqrestore(&ms->lock, flags); if (should_wake) wakeup_mirrord(ms); + spin_unlock_irqrestore(&ms->lock, flags); } static void do_write(struct mirror_set *ms, struct bio *bio) diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index bb09e2815891d..5861389968e9a 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -501,8 +501,9 @@ static char **realloc_argv(unsigned int *size, char **old_argv) gfp = GFP_NOIO; } argv = kmalloc_array(new_size, sizeof(*argv), gfp); - if (argv && old_argv) { - memcpy(argv, old_argv, *size * sizeof(*argv)); + if (argv) { + if (old_argv) + memcpy(argv, old_argv, *size * sizeof(*argv)); *size = new_size; } @@ -669,6 +670,10 @@ int dm_table_add_target(struct dm_table *t, const char *type, DMERR("%s: zero-length target", dm_device_name(t->md)); return -EINVAL; } + if (start + len < start || start + len > LLONG_MAX >> SECTOR_SHIFT) { + DMERR("%s: too large device", dm_device_name(t->md)); + return -EINVAL; + } ti->type = dm_get_target_type(type); if (!ti->type) { @@ -861,17 +866,17 @@ static bool dm_table_supports_dax(struct dm_table *t, return true; } -static int device_is_rq_stackable(struct dm_target *ti, struct dm_dev *dev, - sector_t start, sector_t len, void *data) +static int device_is_not_rq_stackable(struct dm_target *ti, struct dm_dev *dev, + sector_t start, sector_t len, void *data) { struct block_device *bdev = dev->bdev; struct request_queue *q = bdev_get_queue(bdev); /* request-based cannot stack on partitions! */ if (bdev_is_partition(bdev)) - return false; + return true; - return queue_is_mq(q); + return !queue_is_mq(q); } static int dm_table_determine_type(struct dm_table *t) @@ -967,7 +972,7 @@ static int dm_table_determine_type(struct dm_table *t) /* Non-request-stackable devices can't be used for request-based dm */ if (!ti->type->iterate_devices || - !ti->type->iterate_devices(ti, device_is_rq_stackable, NULL)) { + ti->type->iterate_devices(ti, device_is_not_rq_stackable, NULL)) { DMERR("table load rejected: including non-request-stackable devices"); return -EINVAL; } @@ -1245,7 +1250,7 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile, t = dm_get_live_table(md, &srcu_idx); if (!t) - return 0; + goto put_live_table; for (unsigned int i = 0; i < t->num_targets; i++) { struct dm_target *ti = dm_table_get_target(t, i); @@ -1256,6 +1261,7 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile, (void *)key); } +put_live_table: dm_put_live_table(md, srcu_idx); return 0; } diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 1e3c926bdf665..ba49d0cd46a6e 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -601,6 +601,10 @@ int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, (*argc)--; if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { + if (v->fec->dev) { + ti->error = "FEC device already specified"; + return -EINVAL; + } r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev); if (r) { ti->error = "FEC device lookup failed"; diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index b650aadbe4946..5e99099aea0f1 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -1079,6 +1079,9 @@ static int verity_alloc_most_once(struct dm_verity *v) { struct dm_target *ti = v->ti; + if (v->validated_blocks) + return 0; + /* the bitset can only handle INT_MAX blocks */ if (v->data_blocks > INT_MAX) { ti->error = "device too large to use check_at_most_once"; @@ -1102,6 +1105,9 @@ static int verity_alloc_zero_digest(struct dm_verity *v) struct dm_verity_io *io; u8 *zero_data; + if (v->zero_digest) + return 0; + v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); if (!v->zero_digest) @@ -1512,7 +1518,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) goto bad; } - /* Root hash signature is a optional parameter*/ + /* Root hash signature is an optional parameter */ r = verity_verify_root_hash(root_hash_digest_to_validate, strlen(root_hash_digest_to_validate), verify_args.sig, diff --git a/drivers/md/dm-verity-verify-sig.c b/drivers/md/dm-verity-verify-sig.c index 4836508ea50c0..1ced2ef255f32 100644 --- a/drivers/md/dm-verity-verify-sig.c +++ b/drivers/md/dm-verity-verify-sig.c @@ -71,9 +71,14 @@ int verity_verify_sig_parse_opt_args(struct dm_arg_set *as, const char *arg_name) { struct dm_target *ti = v->ti; - int ret = 0; + int ret; const char *sig_key = NULL; + if (v->signature_key_desc) { + ti->error = DM_VERITY_VERIFY_ERR("root_hash_sig_key_desc already specified"); + return -EINVAL; + } + if (!*argc) { ti->error = DM_VERITY_VERIFY_ERR("Signature key not specified"); return -EINVAL; @@ -83,14 +88,18 @@ int verity_verify_sig_parse_opt_args(struct dm_arg_set *as, (*argc)--; ret = verity_verify_get_sig_from_key(sig_key, sig_opts); - if (ret < 0) + if (ret < 0) { ti->error = DM_VERITY_VERIFY_ERR("Invalid key specified"); + return ret; + } v->signature_key_desc = kstrdup(sig_key, GFP_KERNEL); - if (!v->signature_key_desc) + if (!v->signature_key_desc) { + ti->error = DM_VERITY_VERIFY_ERR("Could not allocate memory for signature key"); return -ENOMEM; + } - return ret; + return 0; } /* diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c index 1b2a448a058ac..dc086ea24ec31 100644 --- a/drivers/md/dm-zoned-target.c +++ b/drivers/md/dm-zoned-target.c @@ -1070,7 +1070,7 @@ static int dmz_iterate_devices(struct dm_target *ti, struct dmz_target *dmz = ti->private; unsigned int zone_nr_sectors = dmz_zone_nr_sectors(dmz->metadata); sector_t capacity; - int i, r; + int i, r = 0; for (i = 0; i < dmz->nr_ddevs; i++) { capacity = dmz->dev[i].capacity & ~(zone_nr_sectors - 1); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 3610080ecff64..a5ac25ef5e55e 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1536,14 +1536,18 @@ static void __send_empty_flush(struct clone_info *ci) { struct dm_table *t = ci->map; struct bio flush_bio; + blk_opf_t opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; + + if ((ci->io->orig_bio->bi_opf & (REQ_IDLE | REQ_SYNC)) == + (REQ_IDLE | REQ_SYNC)) + opf |= REQ_IDLE; /* * Use an on-stack bio for this, it's safe since we don't * need to reference it after submit. It's just used as * the basis for the clone(s). */ - bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, - REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC); + bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, opf); ci->bio = &flush_bio; ci->sector_count = 0; @@ -2217,21 +2221,29 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, struct queue_limits *limits) { struct dm_table *old_map; - sector_t size; + sector_t size, old_size; int ret; lockdep_assert_held(&md->suspend_lock); size = dm_table_get_size(t); + old_size = dm_get_size(md); + set_capacity(md->disk, size); + + ret = dm_table_set_restrictions(t, md->queue, limits); + if (ret) { + set_capacity(md->disk, old_size); + old_map = ERR_PTR(ret); + goto out; + } + /* * Wipe any geometry if the size of the table changed. */ - if (size != dm_get_size(md)) + if (size != old_size) memset(&md->geometry, 0, sizeof(md->geometry)); - set_capacity(md->disk, size); - dm_table_event_callback(t, event_callback, md); if (dm_table_request_based(t)) { @@ -2249,10 +2261,10 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, * requests in the queue may refer to bio from the old bioset, * so you must walk through the queue to unprep. */ - if (!md->mempools) { + if (!md->mempools) md->mempools = t->mempools; - t->mempools = NULL; - } + else + dm_free_md_mempools(t->mempools); } else { /* * The md may already have mempools that need changing. @@ -2261,14 +2273,8 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, */ dm_free_md_mempools(md->mempools); md->mempools = t->mempools; - t->mempools = NULL; - } - - ret = dm_table_set_restrictions(t, md->queue, limits); - if (ret) { - old_map = ERR_PTR(ret); - goto out; } + t->mempools = NULL; old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); rcu_assign_pointer(md->map, (void *)t); @@ -2708,7 +2714,7 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map, { bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; - int r; + int r = 0; lockdep_assert_held(&md->suspend_lock); @@ -2760,8 +2766,10 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map, * Stop md->queue before flushing md->wq in case request-based * dm defers requests to md->wq from md->queue. */ - if (dm_request_based(md)) + if (map && dm_request_based(md)) { dm_stop_queue(md->queue); + set_bit(DMF_QUEUE_STOPPED, &md->flags); + } flush_workqueue(md->wq); @@ -2770,7 +2778,8 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map, * We call dm_wait_for_completion to wait for all existing requests * to finish. */ - r = dm_wait_for_completion(md, task_state); + if (map) + r = dm_wait_for_completion(md, task_state); if (!r) set_bit(dmf_suspended_flag, &md->flags); @@ -2783,7 +2792,7 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map, if (r < 0) { dm_queue_flush(md); - if (dm_request_based(md)) + if (test_and_clear_bit(DMF_QUEUE_STOPPED, &md->flags)) dm_start_queue(md->queue); unlock_fs(md); @@ -2867,7 +2876,7 @@ static int __dm_resume(struct mapped_device *md, struct dm_table *map) * so that mapping of targets can work correctly. * Request-based dm is queueing the deferred I/Os in its request_queue. */ - if (dm_request_based(md)) + if (test_and_clear_bit(DMF_QUEUE_STOPPED, &md->flags)) dm_start_queue(md->queue); unlock_fs(md); diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 8317e07b326d0..1f1991634d0a2 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -589,7 +589,7 @@ static int md_bitmap_new_disk_sb(struct bitmap *bitmap) * is a good choice? We choose COUNTER_MAX / 2 arbitrarily. */ write_behind = bitmap->mddev->bitmap_info.max_write_behind; - if (write_behind > COUNTER_MAX) + if (write_behind > COUNTER_MAX / 2) write_behind = COUNTER_MAX / 2; sb->write_behind = cpu_to_le32(write_behind); bitmap->mddev->bitmap_info.max_write_behind = write_behind; @@ -2119,8 +2119,7 @@ int md_bitmap_get_stats(struct bitmap *bitmap, struct md_bitmap_stats *stats) if (!bitmap) return -ENOENT; - if (!bitmap->mddev->bitmap_info.external && - !bitmap->storage.sb_page) + if (!bitmap->storage.sb_page) return -EINVAL; sb = kmap_local_page(bitmap->storage.sb_page); stats->sync_size = le64_to_cpu(sb->sync_size); diff --git a/drivers/md/md.c b/drivers/md/md.c index ca7ae3aad2655..b086cbf240869 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9465,8 +9465,8 @@ void md_check_recovery(struct mddev *mddev) * remove disk. */ rdev_for_each_safe(rdev, tmp, mddev) { - if (test_and_clear_bit(ClusterRemove, &rdev->flags) && - rdev->raid_disk < 0) + if (rdev->raid_disk < 0 && + test_and_clear_bit(ClusterRemove, &rdev->flags)) md_kick_rdev_from_array(rdev); } } @@ -9813,8 +9813,11 @@ static void check_sb_changes(struct mddev *mddev, struct md_rdev *rdev) /* Check for change of roles in the active devices */ rdev_for_each_safe(rdev2, tmp, mddev) { - if (test_bit(Faulty, &rdev2->flags)) + if (test_bit(Faulty, &rdev2->flags)) { + if (test_bit(ClusterRemove, &rdev2->flags)) + set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); continue; + } /* Check if the roles changed */ role = le16_to_cpu(sb->dev_roles[rdev2->desc_nr]); diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 8b25287c89ed6..4c1f86ca55208 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3297,6 +3297,7 @@ static int raid1_reshape(struct mddev *mddev) /* ok, everything is stopped */ oldpool = conf->r1bio_pool; conf->r1bio_pool = newpool; + init_waitqueue_head(&conf->r1bio_pool.wait); for (d = d2 = 0; d < conf->raid_disks; d++) { struct md_rdev *rdev = conf->mirrors[d].rdev; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 36b6bf3f8b29f..a75d090a7fa15 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1205,8 +1205,11 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, rcu_read_unlock(); } - if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) + if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) { + raid_end_bio_io(r10_bio); return; + } + rdev = read_balance(conf, r10_bio, &max_sectors); if (!rdev) { if (err_rdev) { @@ -1428,8 +1431,11 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, } sectors = r10_bio->sectors; - if (!regular_request_wait(mddev, conf, bio, sectors)) + if (!regular_request_wait(mddev, conf, bio, sectors)) { + raid_end_bio_io(r10_bio); return; + } + if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && (mddev->reshape_backwards ? (bio->bi_iter.bi_sector < conf->reshape_safe && diff --git a/drivers/media/cec/usb/rainshadow/rainshadow-cec.c b/drivers/media/cec/usb/rainshadow/rainshadow-cec.c index ee870ea1a8860..6f8d6797c6145 100644 --- a/drivers/media/cec/usb/rainshadow/rainshadow-cec.c +++ b/drivers/media/cec/usb/rainshadow/rainshadow-cec.c @@ -171,11 +171,12 @@ static irqreturn_t rain_interrupt(struct serio *serio, unsigned char data, { struct rain *rain = serio_get_drvdata(serio); + spin_lock(&rain->buf_lock); if (rain->buf_len == DATA_SIZE) { + spin_unlock(&rain->buf_lock); dev_warn_once(rain->dev, "buffer overflow\n"); return IRQ_HANDLED; } - spin_lock(&rain->buf_lock); rain->buf_len++; rain->buf[rain->buf_wr_idx] = data; rain->buf_wr_idx = (rain->buf_wr_idx + 1) & 0xff; diff --git a/drivers/media/common/videobuf2/videobuf2-dma-sg.c b/drivers/media/common/videobuf2/videobuf2-dma-sg.c index 6975a71d740f6..a5aa6a2a028cb 100644 --- a/drivers/media/common/videobuf2/videobuf2-dma-sg.c +++ b/drivers/media/common/videobuf2/videobuf2-dma-sg.c @@ -469,7 +469,7 @@ vb2_dma_sg_dmabuf_ops_begin_cpu_access(struct dma_buf *dbuf, struct vb2_dma_sg_buf *buf = dbuf->priv; struct sg_table *sgt = buf->dma_sgt; - dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir); + dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir); return 0; } @@ -480,7 +480,7 @@ vb2_dma_sg_dmabuf_ops_end_cpu_access(struct dma_buf *dbuf, struct vb2_dma_sg_buf *buf = dbuf->priv; struct sg_table *sgt = buf->dma_sgt; - dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir); + dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir); return 0; } diff --git a/drivers/media/dvb-frontends/dib7000p.c b/drivers/media/dvb-frontends/dib7000p.c index 444fe1c4bf2d7..e561c7cc35bfa 100644 --- a/drivers/media/dvb-frontends/dib7000p.c +++ b/drivers/media/dvb-frontends/dib7000p.c @@ -2198,6 +2198,8 @@ static int w7090p_tuner_write_serpar(struct i2c_adapter *i2c_adap, struct i2c_ms struct dib7000p_state *state = i2c_get_adapdata(i2c_adap); u8 n_overflow = 1; u16 i = 1000; + if (msg[0].len < 3) + return -EOPNOTSUPP; u16 serpar_num = msg[0].buf[0]; while (n_overflow == 1 && i) { @@ -2217,6 +2219,8 @@ static int w7090p_tuner_read_serpar(struct i2c_adapter *i2c_adap, struct i2c_msg struct dib7000p_state *state = i2c_get_adapdata(i2c_adap); u8 n_overflow = 1, n_empty = 1; u16 i = 1000; + if (msg[0].len < 1 || msg[1].len < 2) + return -EOPNOTSUPP; u16 serpar_num = msg[0].buf[0]; u16 read_word; @@ -2261,8 +2265,12 @@ static int dib7090p_rw_on_apb(struct i2c_adapter *i2c_adap, u16 word; if (num == 1) { /* write */ + if (msg[0].len < 3) + return -EOPNOTSUPP; dib7000p_write_word(state, apb_address, ((msg[0].buf[1] << 8) | (msg[0].buf[2]))); } else { + if (msg[1].len < 2) + return -EOPNOTSUPP; word = dib7000p_read_word(state, apb_address); msg[1].buf[0] = (word >> 8) & 0xff; msg[1].buf[1] = (word) & 0xff; diff --git a/drivers/media/dvb-frontends/stv0367_priv.h b/drivers/media/dvb-frontends/stv0367_priv.h index 617f605947b2c..7f056d1cce822 100644 --- a/drivers/media/dvb-frontends/stv0367_priv.h +++ b/drivers/media/dvb-frontends/stv0367_priv.h @@ -25,8 +25,11 @@ #endif /* MACRO definitions */ +#ifndef MIN #define MAX(X, Y) ((X) >= (Y) ? (X) : (Y)) #define MIN(X, Y) ((X) <= (Y) ? (X) : (Y)) +#endif + #define INRANGE(X, Y, Z) \ ((((X) <= (Y)) && ((Y) <= (Z))) || \ (((Z) <= (Y)) && ((Y) <= (X))) ? 1 : 0) diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig index 53b443be5a59e..d4082a86fcedb 100644 --- a/drivers/media/i2c/Kconfig +++ b/drivers/media/i2c/Kconfig @@ -27,7 +27,7 @@ config VIDEO_IR_I2C menuconfig VIDEO_CAMERA_SENSOR bool "Camera sensor devices" - depends on MEDIA_CAMERA_SUPPORT && I2C + depends on MEDIA_CAMERA_SUPPORT && I2C && HAVE_CLK select MEDIA_CONTROLLER select V4L2_FWNODE select VIDEO_V4L2_SUBDEV_API diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c index 99ba925e8ec8e..ecb0e7b1f2a5f 100644 --- a/drivers/media/i2c/adv7180.c +++ b/drivers/media/i2c/adv7180.c @@ -194,6 +194,7 @@ struct adv7180_state; #define ADV7180_FLAG_V2 BIT(1) #define ADV7180_FLAG_MIPI_CSI2 BIT(2) #define ADV7180_FLAG_I2P BIT(3) +#define ADV7180_FLAG_TEST_PATTERN BIT(4) struct adv7180_chip_info { unsigned int flags; @@ -355,32 +356,27 @@ static inline struct adv7180_state *to_state(struct v4l2_subdev *sd) static int adv7180_querystd(struct v4l2_subdev *sd, v4l2_std_id *std) { struct adv7180_state *state = to_state(sd); - int err = mutex_lock_interruptible(&state->mutex); - if (err) - return err; - - if (state->streaming) { - err = -EBUSY; - goto unlock; - } + int ret; - err = adv7180_set_video_standard(state, - ADV7180_STD_AD_PAL_BG_NTSC_J_SECAM); - if (err) - goto unlock; + guard(mutex)(&state->mutex); - msleep(100); - __adv7180_status(state, NULL, std); + /* + * We can't sample the standard if the device is streaming as that would + * interfere with the capture session as the VID_SEL reg is touched. + */ + if (state->streaming) + return -EBUSY; - err = v4l2_std_to_adv7180(state->curr_norm); - if (err < 0) - goto unlock; + /* Set the standard to autodetect PAL B/G/H/I/D, NTSC J or SECAM */ + ret = adv7180_set_video_standard(state, + ADV7180_STD_AD_PAL_BG_NTSC_J_SECAM); + if (ret) + return ret; - err = adv7180_set_video_standard(state, err); + /* Allow some time for the autodetection to run. */ + msleep(100); -unlock: - mutex_unlock(&state->mutex); - return err; + return __adv7180_status(state, NULL, std); } static int adv7180_s_routing(struct v4l2_subdev *sd, u32 input, @@ -673,11 +669,15 @@ static int adv7180_init_controls(struct adv7180_state *state) ADV7180_HUE_MAX, 1, ADV7180_HUE_DEF); v4l2_ctrl_new_custom(&state->ctrl_hdl, &adv7180_ctrl_fast_switch, NULL); - v4l2_ctrl_new_std_menu_items(&state->ctrl_hdl, &adv7180_ctrl_ops, - V4L2_CID_TEST_PATTERN, - ARRAY_SIZE(test_pattern_menu) - 1, - 0, ARRAY_SIZE(test_pattern_menu) - 1, - test_pattern_menu); + if (state->chip_info->flags & ADV7180_FLAG_TEST_PATTERN) { + v4l2_ctrl_new_std_menu_items(&state->ctrl_hdl, + &adv7180_ctrl_ops, + V4L2_CID_TEST_PATTERN, + ARRAY_SIZE(test_pattern_menu) - 1, + 0, + ARRAY_SIZE(test_pattern_menu) - 1, + test_pattern_menu); + } state->sd.ctrl_handler = &state->ctrl_hdl; if (state->ctrl_hdl.error) { @@ -798,12 +798,7 @@ static int adv7180_set_pad_format(struct v4l2_subdev *sd, ret = adv7180_mbus_fmt(sd, &format->format); if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) { - if (state->field != format->format.field) { - state->field = format->format.field; - adv7180_set_power(state, false); - adv7180_set_field_mode(state); - adv7180_set_power(state, true); - } + state->field = format->format.field; } else { framefmt = v4l2_subdev_get_try_format(sd, sd_state, 0); *framefmt = format->format; @@ -1209,7 +1204,7 @@ static const struct adv7180_chip_info adv7182_info = { }; static const struct adv7180_chip_info adv7280_info = { - .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_I2P, + .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_I2P | ADV7180_FLAG_TEST_PATTERN, .valid_input_mask = BIT(ADV7182_INPUT_CVBS_AIN1) | BIT(ADV7182_INPUT_CVBS_AIN2) | BIT(ADV7182_INPUT_CVBS_AIN3) | @@ -1223,7 +1218,8 @@ static const struct adv7180_chip_info adv7280_info = { }; static const struct adv7180_chip_info adv7280_m_info = { - .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2 | ADV7180_FLAG_I2P, + .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2 | ADV7180_FLAG_I2P | + ADV7180_FLAG_TEST_PATTERN, .valid_input_mask = BIT(ADV7182_INPUT_CVBS_AIN1) | BIT(ADV7182_INPUT_CVBS_AIN2) | BIT(ADV7182_INPUT_CVBS_AIN3) | @@ -1244,7 +1240,8 @@ static const struct adv7180_chip_info adv7280_m_info = { }; static const struct adv7180_chip_info adv7281_info = { - .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2, + .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2 | + ADV7180_FLAG_TEST_PATTERN, .valid_input_mask = BIT(ADV7182_INPUT_CVBS_AIN1) | BIT(ADV7182_INPUT_CVBS_AIN2) | BIT(ADV7182_INPUT_CVBS_AIN7) | @@ -1259,7 +1256,8 @@ static const struct adv7180_chip_info adv7281_info = { }; static const struct adv7180_chip_info adv7281_m_info = { - .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2, + .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2 | + ADV7180_FLAG_TEST_PATTERN, .valid_input_mask = BIT(ADV7182_INPUT_CVBS_AIN1) | BIT(ADV7182_INPUT_CVBS_AIN2) | BIT(ADV7182_INPUT_CVBS_AIN3) | @@ -1279,7 +1277,8 @@ static const struct adv7180_chip_info adv7281_m_info = { }; static const struct adv7180_chip_info adv7281_ma_info = { - .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2, + .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2 | + ADV7180_FLAG_TEST_PATTERN, .valid_input_mask = BIT(ADV7182_INPUT_CVBS_AIN1) | BIT(ADV7182_INPUT_CVBS_AIN2) | BIT(ADV7182_INPUT_CVBS_AIN3) | @@ -1304,7 +1303,7 @@ static const struct adv7180_chip_info adv7281_ma_info = { }; static const struct adv7180_chip_info adv7282_info = { - .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_I2P, + .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_I2P | ADV7180_FLAG_TEST_PATTERN, .valid_input_mask = BIT(ADV7182_INPUT_CVBS_AIN1) | BIT(ADV7182_INPUT_CVBS_AIN2) | BIT(ADV7182_INPUT_CVBS_AIN7) | @@ -1319,7 +1318,8 @@ static const struct adv7180_chip_info adv7282_info = { }; static const struct adv7180_chip_info adv7282_m_info = { - .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2 | ADV7180_FLAG_I2P, + .flags = ADV7180_FLAG_V2 | ADV7180_FLAG_MIPI_CSI2 | ADV7180_FLAG_I2P | + ADV7180_FLAG_TEST_PATTERN, .valid_input_mask = BIT(ADV7182_INPUT_CVBS_AIN1) | BIT(ADV7182_INPUT_CVBS_AIN2) | BIT(ADV7182_INPUT_CVBS_AIN3) | @@ -1558,6 +1558,8 @@ static int adv7180_suspend(struct device *dev) struct v4l2_subdev *sd = dev_get_drvdata(dev); struct adv7180_state *state = to_state(sd); + guard(mutex)(&state->mutex); + return adv7180_set_power(state, false); } @@ -1571,6 +1573,8 @@ static int adv7180_resume(struct device *dev) if (ret < 0) return ret; + guard(mutex)(&state->mutex); + ret = adv7180_set_power(state, state->powered); if (ret) return ret; diff --git a/drivers/media/i2c/ccs-pll.c b/drivers/media/i2c/ccs-pll.c index cf8858cb13d4c..611c9823be857 100644 --- a/drivers/media/i2c/ccs-pll.c +++ b/drivers/media/i2c/ccs-pll.c @@ -312,6 +312,11 @@ __ccs_pll_calculate_vt_tree(struct device *dev, dev_dbg(dev, "more_mul2: %u\n", more_mul); pll_fr->pll_multiplier = mul * more_mul; + if (pll_fr->pll_multiplier > lim_fr->max_pll_multiplier) { + dev_dbg(dev, "pll multiplier %u too high\n", + pll_fr->pll_multiplier); + return -EINVAL; + } if (pll_fr->pll_multiplier * pll_fr->pll_ip_clk_freq_hz > lim_fr->max_pll_op_clk_freq_hz) @@ -397,6 +402,8 @@ static int ccs_pll_calculate_vt_tree(struct device *dev, min_pre_pll_clk_div = max_t(u16, min_pre_pll_clk_div, pll->ext_clk_freq_hz / lim_fr->max_pll_ip_clk_freq_hz); + if (!(pll->flags & CCS_PLL_FLAG_EXT_IP_PLL_DIVIDER)) + min_pre_pll_clk_div = clk_div_even(min_pre_pll_clk_div); dev_dbg(dev, "vt min/max_pre_pll_clk_div: %u,%u\n", min_pre_pll_clk_div, max_pre_pll_clk_div); @@ -792,7 +799,7 @@ int ccs_pll_calculate(struct device *dev, const struct ccs_pll_limits *lim, op_lim_fr->min_pre_pll_clk_div, op_lim_fr->max_pre_pll_clk_div); max_op_pre_pll_clk_div = min_t(u16, op_lim_fr->max_pre_pll_clk_div, - clk_div_even(pll->ext_clk_freq_hz / + DIV_ROUND_UP(pll->ext_clk_freq_hz, op_lim_fr->min_pll_ip_clk_freq_hz)); min_op_pre_pll_clk_div = max_t(u16, op_lim_fr->min_pre_pll_clk_div, @@ -815,6 +822,8 @@ int ccs_pll_calculate(struct device *dev, const struct ccs_pll_limits *lim, one_or_more( DIV_ROUND_UP(op_lim_fr->max_pll_op_clk_freq_hz, pll->ext_clk_freq_hz)))); + if (!(pll->flags & CCS_PLL_FLAG_EXT_IP_PLL_DIVIDER)) + min_op_pre_pll_clk_div = clk_div_even(min_op_pre_pll_clk_div); dev_dbg(dev, "pll_op check: min / max op_pre_pll_clk_div: %u / %u\n", min_op_pre_pll_clk_div, max_op_pre_pll_clk_div); diff --git a/drivers/media/i2c/ds90ub913.c b/drivers/media/i2c/ds90ub913.c index ae33d1ecf835d..6ec960d7914e9 100644 --- a/drivers/media/i2c/ds90ub913.c +++ b/drivers/media/i2c/ds90ub913.c @@ -453,10 +453,10 @@ static int ub913_set_fmt(struct v4l2_subdev *sd, if (!fmt) return -EINVAL; - format->format.code = finfo->outcode; - *fmt = format->format; + fmt->code = finfo->outcode; + return 0; } diff --git a/drivers/media/i2c/hi556.c b/drivers/media/i2c/hi556.c index fd56ba1387391..d6ef4a249e94f 100644 --- a/drivers/media/i2c/hi556.c +++ b/drivers/media/i2c/hi556.c @@ -689,21 +689,23 @@ static int hi556_test_pattern(struct hi556 *hi556, u32 pattern) int ret; u32 val; - if (pattern) { - ret = hi556_read_reg(hi556, HI556_REG_ISP, - HI556_REG_VALUE_08BIT, &val); - if (ret) - return ret; + ret = hi556_read_reg(hi556, HI556_REG_ISP, + HI556_REG_VALUE_08BIT, &val); + if (ret) + return ret; - ret = hi556_write_reg(hi556, HI556_REG_ISP, - HI556_REG_VALUE_08BIT, - val | HI556_REG_ISP_TPG_EN); - if (ret) - return ret; - } + val = pattern ? (val | HI556_REG_ISP_TPG_EN) : + (val & ~HI556_REG_ISP_TPG_EN); + + ret = hi556_write_reg(hi556, HI556_REG_ISP, + HI556_REG_VALUE_08BIT, val); + if (ret) + return ret; + + val = pattern ? BIT(pattern - 1) : 0; return hi556_write_reg(hi556, HI556_REG_TEST_PATTERN, - HI556_REG_VALUE_08BIT, pattern); + HI556_REG_VALUE_08BIT, val); } static int hi556_set_ctrl(struct v4l2_ctrl *ctrl) diff --git a/drivers/media/i2c/imx214.c b/drivers/media/i2c/imx214.c index 2f9c8582f9401..db40008f31cf1 100644 --- a/drivers/media/i2c/imx214.c +++ b/drivers/media/i2c/imx214.c @@ -20,7 +20,9 @@ #include #define IMX214_DEFAULT_CLK_FREQ 24000000 -#define IMX214_DEFAULT_LINK_FREQ 480000000 +#define IMX214_DEFAULT_LINK_FREQ 600000000 +/* Keep wrong link frequency for backward compatibility */ +#define IMX214_DEFAULT_LINK_FREQ_LEGACY 480000000 #define IMX214_DEFAULT_PIXEL_RATE ((IMX214_DEFAULT_LINK_FREQ * 8LL) / 10) #define IMX214_FPS 30 #define IMX214_MBUS_CODE MEDIA_BUS_FMT_SRGGB10_1X10 @@ -892,17 +894,26 @@ static int imx214_parse_fwnode(struct device *dev) goto done; } - for (i = 0; i < bus_cfg.nr_of_link_frequencies; i++) + if (bus_cfg.nr_of_link_frequencies != 1) + dev_warn(dev, "Only one link-frequency supported, please review your DT. Continuing anyway\n"); + + for (i = 0; i < bus_cfg.nr_of_link_frequencies; i++) { if (bus_cfg.link_frequencies[i] == IMX214_DEFAULT_LINK_FREQ) break; - - if (i == bus_cfg.nr_of_link_frequencies) { - dev_err(dev, "link-frequencies %d not supported, Please review your DT\n", - IMX214_DEFAULT_LINK_FREQ); - ret = -EINVAL; - goto done; + if (bus_cfg.link_frequencies[i] == + IMX214_DEFAULT_LINK_FREQ_LEGACY) { + dev_warn(dev, + "link-frequencies %d not supported, please review your DT. Continuing anyway\n", + IMX214_DEFAULT_LINK_FREQ); + break; + } } + if (i == bus_cfg.nr_of_link_frequencies) + ret = dev_err_probe(dev, -EINVAL, + "link-frequencies %d not supported, please review your DT\n", + IMX214_DEFAULT_LINK_FREQ); + done: v4l2_fwnode_endpoint_free(&bus_cfg); fwnode_handle_put(endpoint); diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c index a14e571dc62bc..a3d5a8a7c660b 100644 --- a/drivers/media/i2c/imx219.c +++ b/drivers/media/i2c/imx219.c @@ -77,7 +77,7 @@ #define IMX219_VTS_30FPS_640x480 0x06e3 #define IMX219_VTS_MAX 0xffff -#define IMX219_VBLANK_MIN 4 +#define IMX219_VBLANK_MIN 32 /*Frame Length Line*/ #define IMX219_FLL_MIN 0x08a6 diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c index b37a2aaf8ac04..a8026f0f980f9 100644 --- a/drivers/media/i2c/ir-kbd-i2c.c +++ b/drivers/media/i2c/ir-kbd-i2c.c @@ -321,9 +321,9 @@ static int get_key_avermedia_cardbus(struct IR_i2c *ir, enum rc_proto *protocol, static int ir_key_poll(struct IR_i2c *ir) { - enum rc_proto protocol; - u32 scancode; - u8 toggle; + enum rc_proto protocol = 0; + u32 scancode = 0; + u8 toggle = 0; int rc; dev_dbg(&ir->rc->dev, "%s\n", __func__); diff --git a/drivers/media/i2c/mt9v111.c b/drivers/media/i2c/mt9v111.c index 1f7edc0f5b1ab..a184fd9fc6f3b 100644 --- a/drivers/media/i2c/mt9v111.c +++ b/drivers/media/i2c/mt9v111.c @@ -534,8 +534,8 @@ static int mt9v111_calc_frame_rate(struct mt9v111_dev *mt9v111, static int mt9v111_hw_config(struct mt9v111_dev *mt9v111) { struct i2c_client *c = mt9v111->client; - unsigned int ret; u16 outfmtctrl2; + int ret; /* Force device reset. */ ret = __mt9v111_hw_reset(mt9v111); diff --git a/drivers/media/i2c/og01a1b.c b/drivers/media/i2c/og01a1b.c index 365ce56845836..9bd204911651e 100644 --- a/drivers/media/i2c/og01a1b.c +++ b/drivers/media/i2c/og01a1b.c @@ -676,7 +676,7 @@ static void og01a1b_update_pad_format(const struct og01a1b_mode *mode, { fmt->width = mode->width; fmt->height = mode->height; - fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10; + fmt->code = MEDIA_BUS_FMT_Y10_1X10; fmt->field = V4L2_FIELD_NONE; } @@ -867,7 +867,7 @@ static int og01a1b_enum_mbus_code(struct v4l2_subdev *sd, if (code->index > 0) return -EINVAL; - code->code = MEDIA_BUS_FMT_SGRBG10_1X10; + code->code = MEDIA_BUS_FMT_Y10_1X10; return 0; } @@ -879,7 +879,7 @@ static int og01a1b_enum_frame_size(struct v4l2_subdev *sd, if (fse->index >= ARRAY_SIZE(supported_modes)) return -EINVAL; - if (fse->code != MEDIA_BUS_FMT_SGRBG10_1X10) + if (fse->code != MEDIA_BUS_FMT_Y10_1X10) return -EINVAL; fse->min_width = supported_modes[fse->index].width; diff --git a/drivers/media/i2c/ov08x40.c b/drivers/media/i2c/ov08x40.c index 637da4df69011..4d15fa6ac311e 100644 --- a/drivers/media/i2c/ov08x40.c +++ b/drivers/media/i2c/ov08x40.c @@ -2643,7 +2643,7 @@ static int ov08x40_set_ctrl_hflip(struct ov08x40 *ov08x, u32 ctrl_val) return ov08x40_write_reg(ov08x, OV08X40_REG_MIRROR, OV08X40_REG_VALUE_08BIT, - ctrl_val ? val | BIT(2) : val & ~BIT(2)); + ctrl_val ? val & ~BIT(2) : val | BIT(2)); } static int ov08x40_set_ctrl_vflip(struct ov08x40 *ov08x, u32 ctrl_val) diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c index 5429bd2eb0531..5206784726dbd 100644 --- a/drivers/media/i2c/ov2659.c +++ b/drivers/media/i2c/ov2659.c @@ -1479,14 +1479,15 @@ static int ov2659_probe(struct i2c_client *client) V4L2_CID_TEST_PATTERN, ARRAY_SIZE(ov2659_test_pattern_menu) - 1, 0, 0, ov2659_test_pattern_menu); - ov2659->sd.ctrl_handler = &ov2659->ctrls; if (ov2659->ctrls.error) { dev_err(&client->dev, "%s: control initialization error %d\n", __func__, ov2659->ctrls.error); + v4l2_ctrl_handler_free(&ov2659->ctrls); return ov2659->ctrls.error; } + ov2659->sd.ctrl_handler = &ov2659->ctrls; sd = &ov2659->sd; client->flags |= I2C_CLIENT_SCCB; #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API diff --git a/drivers/media/i2c/ov5675.c b/drivers/media/i2c/ov5675.c index c499e7e93c54b..92c3e7951ee72 100644 --- a/drivers/media/i2c/ov5675.c +++ b/drivers/media/i2c/ov5675.c @@ -1339,11 +1339,8 @@ static int ov5675_probe(struct i2c_client *client) return -ENOMEM; ret = ov5675_get_hwcfg(ov5675, &client->dev); - if (ret) { - dev_err(&client->dev, "failed to get HW configuration: %d", - ret); + if (ret) return ret; - } v4l2_i2c_subdev_init(&ov5675->sd, client, &ov5675_subdev_ops); diff --git a/drivers/media/i2c/ov8856.c b/drivers/media/i2c/ov8856.c index f053c3a7676a0..8c93fe6285f80 100644 --- a/drivers/media/i2c/ov8856.c +++ b/drivers/media/i2c/ov8856.c @@ -2323,8 +2323,8 @@ static int ov8856_get_hwcfg(struct ov8856 *ov8856, struct device *dev) if (!is_acpi_node(fwnode)) { ov8856->xvclk = devm_clk_get(dev, "xvclk"); if (IS_ERR(ov8856->xvclk)) { - dev_err(dev, "could not get xvclk clock (%pe)\n", - ov8856->xvclk); + dev_err_probe(dev, PTR_ERR(ov8856->xvclk), + "could not get xvclk clock\n"); return PTR_ERR(ov8856->xvclk); } @@ -2429,11 +2429,8 @@ static int ov8856_probe(struct i2c_client *client) return -ENOMEM; ret = ov8856_get_hwcfg(ov8856, &client->dev); - if (ret) { - dev_err(&client->dev, "failed to get HW configuration: %d", - ret); + if (ret) return ret; - } v4l2_i2c_subdev_init(&ov8856->sd, client, &ov8856_subdev_ops); diff --git a/drivers/media/i2c/rj54n1cb0c.c b/drivers/media/i2c/rj54n1cb0c.c index b430046f9e2a9..09293e90025d5 100644 --- a/drivers/media/i2c/rj54n1cb0c.c +++ b/drivers/media/i2c/rj54n1cb0c.c @@ -1331,10 +1331,13 @@ static int rj54n1_probe(struct i2c_client *client) V4L2_CID_GAIN, 0, 127, 1, 66); v4l2_ctrl_new_std(&rj54n1->hdl, &rj54n1_ctrl_ops, V4L2_CID_AUTO_WHITE_BALANCE, 0, 1, 1, 1); - rj54n1->subdev.ctrl_handler = &rj54n1->hdl; - if (rj54n1->hdl.error) - return rj54n1->hdl.error; + if (rj54n1->hdl.error) { + ret = rj54n1->hdl.error; + goto err_free_ctrl; + } + + rj54n1->subdev.ctrl_handler = &rj54n1->hdl; rj54n1->clk_div = clk_div; rj54n1->rect.left = RJ54N1_COLUMN_SKIP; rj54n1->rect.top = RJ54N1_ROW_SKIP; diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c index c81dd41834043..4b6ad85cf261d 100644 --- a/drivers/media/i2c/tc358743.c +++ b/drivers/media/i2c/tc358743.c @@ -110,7 +110,7 @@ static inline struct tc358743_state *to_state(struct v4l2_subdev *sd) /* --------------- I2C --------------- */ -static void i2c_rd(struct v4l2_subdev *sd, u16 reg, u8 *values, u32 n) +static int i2c_rd(struct v4l2_subdev *sd, u16 reg, u8 *values, u32 n) { struct tc358743_state *state = to_state(sd); struct i2c_client *client = state->i2c_client; @@ -136,6 +136,7 @@ static void i2c_rd(struct v4l2_subdev *sd, u16 reg, u8 *values, u32 n) v4l2_err(sd, "%s: reading register 0x%x from 0x%x failed: %d\n", __func__, reg, client->addr, err); } + return err != ARRAY_SIZE(msgs); } static void i2c_wr(struct v4l2_subdev *sd, u16 reg, u8 *values, u32 n) @@ -192,15 +193,24 @@ static void i2c_wr(struct v4l2_subdev *sd, u16 reg, u8 *values, u32 n) } } -static noinline u32 i2c_rdreg(struct v4l2_subdev *sd, u16 reg, u32 n) +static noinline u32 i2c_rdreg_err(struct v4l2_subdev *sd, u16 reg, u32 n, + int *err) { + int error; __le32 val = 0; - i2c_rd(sd, reg, (u8 __force *)&val, n); + error = i2c_rd(sd, reg, (u8 __force *)&val, n); + if (err) + *err = error; return le32_to_cpu(val); } +static inline u32 i2c_rdreg(struct v4l2_subdev *sd, u16 reg, u32 n) +{ + return i2c_rdreg_err(sd, reg, n, NULL); +} + static noinline void i2c_wrreg(struct v4l2_subdev *sd, u16 reg, u32 val, u32 n) { __le32 raw = cpu_to_le32(val); @@ -229,6 +239,13 @@ static u16 i2c_rd16(struct v4l2_subdev *sd, u16 reg) return i2c_rdreg(sd, reg, 2); } +static int i2c_rd16_err(struct v4l2_subdev *sd, u16 reg, u16 *value) +{ + int err; + *value = i2c_rdreg_err(sd, reg, 2, &err); + return err; +} + static void i2c_wr16(struct v4l2_subdev *sd, u16 reg, u16 val) { i2c_wrreg(sd, reg, val, 2); @@ -1651,12 +1668,23 @@ static int tc358743_enum_mbus_code(struct v4l2_subdev *sd, return 0; } +static u32 tc358743_g_colorspace(u32 code) +{ + switch (code) { + case MEDIA_BUS_FMT_RGB888_1X24: + return V4L2_COLORSPACE_SRGB; + case MEDIA_BUS_FMT_UYVY8_1X16: + return V4L2_COLORSPACE_SMPTE170M; + default: + return 0; + } +} + static int tc358743_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct tc358743_state *state = to_state(sd); - u8 vi_rep = i2c_rd8(sd, VI_REP); if (format->pad != 0) return -EINVAL; @@ -1666,23 +1694,7 @@ static int tc358743_get_fmt(struct v4l2_subdev *sd, format->format.height = state->timings.bt.height; format->format.field = V4L2_FIELD_NONE; - switch (vi_rep & MASK_VOUT_COLOR_SEL) { - case MASK_VOUT_COLOR_RGB_FULL: - case MASK_VOUT_COLOR_RGB_LIMITED: - format->format.colorspace = V4L2_COLORSPACE_SRGB; - break; - case MASK_VOUT_COLOR_601_YCBCR_LIMITED: - case MASK_VOUT_COLOR_601_YCBCR_FULL: - format->format.colorspace = V4L2_COLORSPACE_SMPTE170M; - break; - case MASK_VOUT_COLOR_709_YCBCR_FULL: - case MASK_VOUT_COLOR_709_YCBCR_LIMITED: - format->format.colorspace = V4L2_COLORSPACE_REC709; - break; - default: - format->format.colorspace = 0; - break; - } + format->format.colorspace = tc358743_g_colorspace(format->format.code); return 0; } @@ -1696,19 +1708,14 @@ static int tc358743_set_fmt(struct v4l2_subdev *sd, u32 code = format->format.code; /* is overwritten by get_fmt */ int ret = tc358743_get_fmt(sd, sd_state, format); - format->format.code = code; + if (code == MEDIA_BUS_FMT_RGB888_1X24 || + code == MEDIA_BUS_FMT_UYVY8_1X16) + format->format.code = code; + format->format.colorspace = tc358743_g_colorspace(format->format.code); if (ret) return ret; - switch (code) { - case MEDIA_BUS_FMT_RGB888_1X24: - case MEDIA_BUS_FMT_UYVY8_1X16: - break; - default: - return -EINVAL; - } - if (format->which == V4L2_SUBDEV_FORMAT_TRY) return 0; @@ -1932,8 +1939,19 @@ static int tc358743_probe_of(struct tc358743_state *state) state->pdata.refclk_hz = clk_get_rate(refclk); state->pdata.ddc5v_delay = DDC5V_DELAY_100_MS; state->pdata.enable_hdcp = false; - /* A FIFO level of 16 should be enough for 2-lane 720p60 at 594 MHz. */ - state->pdata.fifo_level = 16; + /* + * Ideally the FIFO trigger level should be set based on the input and + * output data rates, but the calculations required are buried in + * Toshiba's register settings spreadsheet. + * A value of 16 works with a 594Mbps data rate for 720p60 (using 2 + * lanes) and 1080p60 (using 4 lanes), but fails when the data rate + * is increased, or a lower pixel clock is used that result in CSI + * reading out faster than the data is arriving. + * + * A value of 374 works with both those modes at 594Mbps, and with most + * modes on 972Mbps. + */ + state->pdata.fifo_level = 374; /* * The PLL input clock is obtained by dividing refclk by pll_prd. * It must be between 6 MHz and 40 MHz, lower frequency is better. @@ -2021,6 +2039,7 @@ static int tc358743_probe(struct i2c_client *client) struct tc358743_platform_data *pdata = client->dev.platform_data; struct v4l2_subdev *sd; u16 irq_mask = MASK_HDMI_MSK | MASK_CSI_MSK; + u16 chipid; int err; if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA)) @@ -2052,7 +2071,8 @@ static int tc358743_probe(struct i2c_client *client) sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS; /* i2c access */ - if ((i2c_rd16(sd, CHIPID) & MASK_CHIPID) != 0) { + if (i2c_rd16_err(sd, CHIPID, &chipid) || + (chipid & MASK_CHIPID) != 0) { v4l2_info(sd, "not a TC358743 on address 0x%x\n", client->addr << 1); return -ENODEV; @@ -2160,10 +2180,10 @@ static int tc358743_probe(struct i2c_client *client) err_work_queues: cec_unregister_adapter(state->cec_adap); if (!state->i2c_client->irq) { - del_timer(&state->timer); + timer_delete_sync(&state->timer); flush_work(&state->work_i2c_poll); } - cancel_delayed_work(&state->delayed_work_enable_hotplug); + cancel_delayed_work_sync(&state->delayed_work_enable_hotplug); mutex_destroy(&state->confctl_mutex); err_hdl: media_entity_cleanup(&sd->entity); diff --git a/drivers/media/i2c/tc358746.c b/drivers/media/i2c/tc358746.c index 566f5eaddd572..b12a6bd42102e 100644 --- a/drivers/media/i2c/tc358746.c +++ b/drivers/media/i2c/tc358746.c @@ -460,24 +460,20 @@ static int tc358746_apply_misc_config(struct tc358746 *tc358746) return err; } -/* Use MHz as base so the div needs no u64 */ -static u32 tc358746_cfg_to_cnt(unsigned int cfg_val, - unsigned int clk_mhz, - unsigned int time_base) +static u32 tc358746_cfg_to_cnt(unsigned long cfg_val, unsigned long clk_hz, + unsigned long long time_base) { - return DIV_ROUND_UP(cfg_val * clk_mhz, time_base); + return div64_u64((u64)cfg_val * clk_hz + time_base - 1, time_base); } -static u32 tc358746_ps_to_cnt(unsigned int cfg_val, - unsigned int clk_mhz) +static u32 tc358746_ps_to_cnt(unsigned long cfg_val, unsigned long clk_hz) { - return tc358746_cfg_to_cnt(cfg_val, clk_mhz, USEC_PER_SEC); + return tc358746_cfg_to_cnt(cfg_val, clk_hz, PSEC_PER_SEC); } -static u32 tc358746_us_to_cnt(unsigned int cfg_val, - unsigned int clk_mhz) +static u32 tc358746_us_to_cnt(unsigned long cfg_val, unsigned long clk_hz) { - return tc358746_cfg_to_cnt(cfg_val, clk_mhz, 1); + return tc358746_cfg_to_cnt(cfg_val, clk_hz, USEC_PER_SEC); } static int tc358746_apply_dphy_config(struct tc358746 *tc358746) @@ -492,7 +488,6 @@ static int tc358746_apply_dphy_config(struct tc358746 *tc358746) /* The hs_byte_clk is also called SYSCLK in the excel sheet */ hs_byte_clk = cfg->hs_clk_rate / 8; - hs_byte_clk /= HZ_PER_MHZ; hf_clk = hs_byte_clk / 2; val = tc358746_us_to_cnt(cfg->init, hf_clk) - 1; diff --git a/drivers/media/mc/mc-devnode.c b/drivers/media/mc/mc-devnode.c index 94abd042045da..6fd8885f5928e 100644 --- a/drivers/media/mc/mc-devnode.c +++ b/drivers/media/mc/mc-devnode.c @@ -50,11 +50,6 @@ static void media_devnode_release(struct device *cd) { struct media_devnode *devnode = to_media_devnode(cd); - mutex_lock(&media_devnode_lock); - /* Mark device node number as free */ - clear_bit(devnode->minor, media_devnode_nums); - mutex_unlock(&media_devnode_lock); - /* Release media_devnode and perform other cleanups as needed. */ if (devnode->release) devnode->release(devnode); @@ -283,6 +278,7 @@ void media_devnode_unregister(struct media_devnode *devnode) /* Delete the cdev on this minor as well */ cdev_device_del(&devnode->cdev, &devnode->dev); devnode->media_dev = NULL; + clear_bit(devnode->minor, media_devnode_nums); mutex_unlock(&media_devnode_lock); put_device(&devnode->dev); diff --git a/drivers/media/mc/mc-entity.c b/drivers/media/mc/mc-entity.c index 951b79ca125cd..f885e718524c2 100644 --- a/drivers/media/mc/mc-entity.c +++ b/drivers/media/mc/mc-entity.c @@ -678,7 +678,7 @@ static int media_pipeline_explore_next_link(struct media_pipeline *pipe, * (already discovered through iterating over links) and pads * not internally connected. */ - if (origin == local || !local->num_links || + if (origin == local || local->num_links || !media_entity_has_pad_interdep(origin->entity, origin->index, local->index)) continue; diff --git a/drivers/media/pci/b2c2/flexcop-pci.c b/drivers/media/pci/b2c2/flexcop-pci.c index 486c8ec0fa60d..ab53c5b02c48d 100644 --- a/drivers/media/pci/b2c2/flexcop-pci.c +++ b/drivers/media/pci/b2c2/flexcop-pci.c @@ -411,7 +411,7 @@ static void flexcop_pci_remove(struct pci_dev *pdev) struct flexcop_pci *fc_pci = pci_get_drvdata(pdev); if (irq_chk_intv > 0) - cancel_delayed_work(&fc_pci->irq_check_work); + cancel_delayed_work_sync(&fc_pci->irq_check_work); flexcop_pci_dma_exit(fc_pci); flexcop_device_exit(fc_pci->fc_dev); diff --git a/drivers/media/pci/cx18/cx18-queue.c b/drivers/media/pci/cx18/cx18-queue.c index 013694bfcb1c1..7cbb2d5869320 100644 --- a/drivers/media/pci/cx18/cx18-queue.c +++ b/drivers/media/pci/cx18/cx18-queue.c @@ -379,15 +379,22 @@ int cx18_stream_alloc(struct cx18_stream *s) break; } + buf->dma_handle = dma_map_single(&s->cx->pci_dev->dev, + buf->buf, s->buf_size, + s->dma); + if (dma_mapping_error(&s->cx->pci_dev->dev, buf->dma_handle)) { + kfree(buf->buf); + kfree(mdl); + kfree(buf); + break; + } + INIT_LIST_HEAD(&mdl->list); INIT_LIST_HEAD(&mdl->buf_list); mdl->id = s->mdl_base_idx; /* a somewhat safe value */ cx18_enqueue(s, mdl, &s->q_idle); INIT_LIST_HEAD(&buf->list); - buf->dma_handle = dma_map_single(&s->cx->pci_dev->dev, - buf->buf, s->buf_size, - s->dma); cx18_buf_sync_for_cpu(s, buf); list_add_tail(&buf->list, &s->buf_pool); } diff --git a/drivers/media/pci/intel/ivsc/mei_ace.c b/drivers/media/pci/intel/ivsc/mei_ace.c index a0491f3078311..5696a951c0fbc 100644 --- a/drivers/media/pci/intel/ivsc/mei_ace.c +++ b/drivers/media/pci/intel/ivsc/mei_ace.c @@ -528,6 +528,8 @@ static void mei_ace_remove(struct mei_cl_device *cldev) ace_set_camera_owner(ace, ACE_CAMERA_IVSC); + mei_cldev_disable(cldev); + mutex_destroy(&ace->lock); } diff --git a/drivers/media/pci/intel/ivsc/mei_csi.c b/drivers/media/pci/intel/ivsc/mei_csi.c index 685b2ec96071a..6c16721a6226e 100644 --- a/drivers/media/pci/intel/ivsc/mei_csi.c +++ b/drivers/media/pci/intel/ivsc/mei_csi.c @@ -807,6 +807,8 @@ static void mei_csi_remove(struct mei_cl_device *cldev) pm_runtime_disable(&cldev->dev); + mei_cldev_disable(cldev); + mutex_destroy(&csi->lock); } diff --git a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c index 8f346d7da9c8d..269a799ec046c 100644 --- a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c +++ b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c @@ -148,14 +148,12 @@ static int snd_ivtv_pcm_capture_open(struct snd_pcm_substream *substream) s = &itv->streams[IVTV_ENC_STREAM_TYPE_PCM]; - v4l2_fh_init(&item.fh, &s->vdev); item.itv = itv; item.type = s->type; /* See if the stream is available */ if (ivtv_claim_stream(&item, item.type)) { /* No, it's already in use */ - v4l2_fh_exit(&item.fh); snd_ivtv_unlock(itvsc); return -EBUSY; } diff --git a/drivers/media/pci/ivtv/ivtv-driver.h b/drivers/media/pci/ivtv/ivtv-driver.h index ce3a7ca51736e..df2dcef1af3f0 100644 --- a/drivers/media/pci/ivtv/ivtv-driver.h +++ b/drivers/media/pci/ivtv/ivtv-driver.h @@ -322,6 +322,7 @@ struct ivtv_queue { }; struct ivtv; /* forward reference */ +struct ivtv_open_id; struct ivtv_stream { /* These first four fields are always set, even if the stream @@ -331,7 +332,7 @@ struct ivtv_stream { const char *name; /* name of the stream */ int type; /* stream type */ - struct v4l2_fh *fh; /* pointer to the streaming filehandle */ + struct ivtv_open_id *id; /* pointer to the streaming ivtv_open_id */ spinlock_t qlock; /* locks access to the queues */ unsigned long s_flags; /* status flags, see above */ int dma; /* can be PCI_DMA_TODEVICE, PCI_DMA_FROMDEVICE or PCI_DMA_NONE */ diff --git a/drivers/media/pci/ivtv/ivtv-fileops.c b/drivers/media/pci/ivtv/ivtv-fileops.c index 4202c3a47d33e..7ed0d2d85253e 100644 --- a/drivers/media/pci/ivtv/ivtv-fileops.c +++ b/drivers/media/pci/ivtv/ivtv-fileops.c @@ -38,16 +38,16 @@ int ivtv_claim_stream(struct ivtv_open_id *id, int type) if (test_and_set_bit(IVTV_F_S_CLAIMED, &s->s_flags)) { /* someone already claimed this stream */ - if (s->fh == &id->fh) { + if (s->id == id) { /* yes, this file descriptor did. So that's OK. */ return 0; } - if (s->fh == NULL && (type == IVTV_DEC_STREAM_TYPE_VBI || + if (s->id == NULL && (type == IVTV_DEC_STREAM_TYPE_VBI || type == IVTV_ENC_STREAM_TYPE_VBI)) { /* VBI is handled already internally, now also assign the file descriptor to this stream for external reading of the stream. */ - s->fh = &id->fh; + s->id = id; IVTV_DEBUG_INFO("Start Read VBI\n"); return 0; } @@ -55,7 +55,7 @@ int ivtv_claim_stream(struct ivtv_open_id *id, int type) IVTV_DEBUG_INFO("Stream %d is busy\n", type); return -EBUSY; } - s->fh = &id->fh; + s->id = id; if (type == IVTV_DEC_STREAM_TYPE_VBI) { /* Enable reinsertion interrupt */ ivtv_clear_irq_mask(itv, IVTV_IRQ_DEC_VBI_RE_INSERT); @@ -93,7 +93,7 @@ void ivtv_release_stream(struct ivtv_stream *s) struct ivtv *itv = s->itv; struct ivtv_stream *s_vbi; - s->fh = NULL; + s->id = NULL; if ((s->type == IVTV_DEC_STREAM_TYPE_VBI || s->type == IVTV_ENC_STREAM_TYPE_VBI) && test_bit(IVTV_F_S_INTERNAL_USE, &s->s_flags)) { /* this stream is still in use internally */ @@ -125,7 +125,7 @@ void ivtv_release_stream(struct ivtv_stream *s) /* was already cleared */ return; } - if (s_vbi->fh) { + if (s_vbi->id) { /* VBI stream still claimed by a file descriptor */ return; } @@ -349,7 +349,7 @@ static ssize_t ivtv_read(struct ivtv_stream *s, char __user *ubuf, size_t tot_co size_t tot_written = 0; int single_frame = 0; - if (atomic_read(&itv->capturing) == 0 && s->fh == NULL) { + if (atomic_read(&itv->capturing) == 0 && s->id == NULL) { /* shouldn't happen */ IVTV_DEBUG_WARN("Stream %s not initialized before read\n", s->name); return -EIO; @@ -819,7 +819,7 @@ void ivtv_stop_capture(struct ivtv_open_id *id, int gop_end) id->type == IVTV_ENC_STREAM_TYPE_VBI) && test_bit(IVTV_F_S_INTERNAL_USE, &s->s_flags)) { /* Also used internally, don't stop capturing */ - s->fh = NULL; + s->id = NULL; } else { ivtv_stop_v4l2_encode_stream(s, gop_end); @@ -903,7 +903,7 @@ int ivtv_v4l2_close(struct file *filp) v4l2_fh_exit(fh); /* Easy case first: this stream was never claimed by us */ - if (s->fh != &id->fh) + if (s->id != id) goto close_done; /* 'Unclaim' this stream */ diff --git a/drivers/media/pci/ivtv/ivtv-irq.c b/drivers/media/pci/ivtv/ivtv-irq.c index b7aaa8b4a7841..404335e5aff4e 100644 --- a/drivers/media/pci/ivtv/ivtv-irq.c +++ b/drivers/media/pci/ivtv/ivtv-irq.c @@ -305,7 +305,7 @@ static void dma_post(struct ivtv_stream *s) ivtv_process_vbi_data(itv, buf, 0, s->type); s->q_dma.bytesused += buf->bytesused; } - if (s->fh == NULL) { + if (s->id == NULL) { ivtv_queue_move(s, &s->q_dma, NULL, &s->q_free, 0); return; } @@ -330,7 +330,7 @@ static void dma_post(struct ivtv_stream *s) set_bit(IVTV_F_I_HAVE_WORK, &itv->i_flags); } - if (s->fh) + if (s->id) wake_up(&s->waitq); } @@ -351,7 +351,7 @@ void ivtv_dma_stream_dec_prepare(struct ivtv_stream *s, u32 offset, int lock) /* Insert buffer block for YUV if needed */ if (s->type == IVTV_DEC_STREAM_TYPE_YUV && f->offset_y) { - if (yi->blanking_dmaptr) { + if (yi->blanking_ptr) { s->sg_pending[idx].src = yi->blanking_dmaptr; s->sg_pending[idx].dst = offset; s->sg_pending[idx].size = 720 * 16; diff --git a/drivers/media/pci/ivtv/ivtv-yuv.c b/drivers/media/pci/ivtv/ivtv-yuv.c index 2d9274537725a..71f0401066471 100644 --- a/drivers/media/pci/ivtv/ivtv-yuv.c +++ b/drivers/media/pci/ivtv/ivtv-yuv.c @@ -125,7 +125,7 @@ static int ivtv_yuv_prep_user_dma(struct ivtv *itv, struct ivtv_user_dma *dma, ivtv_udma_fill_sg_array(dma, y_buffer_offset, uv_buffer_offset, y_size); /* If we've offset the y plane, ensure top area is blanked */ - if (f->offset_y && yi->blanking_dmaptr) { + if (f->offset_y && yi->blanking_ptr) { dma->SGarray[dma->SG_length].size = cpu_to_le32(720*16); dma->SGarray[dma->SG_length].src = cpu_to_le32(yi->blanking_dmaptr); dma->SGarray[dma->SG_length].dst = cpu_to_le32(IVTV_DECODER_OFFSET + yuv_offset[frame]); @@ -929,6 +929,12 @@ static void ivtv_yuv_init(struct ivtv *itv) yi->blanking_dmaptr = dma_map_single(&itv->pdev->dev, yi->blanking_ptr, 720 * 16, DMA_TO_DEVICE); + if (dma_mapping_error(&itv->pdev->dev, yi->blanking_dmaptr)) { + kfree(yi->blanking_ptr); + yi->blanking_ptr = NULL; + yi->blanking_dmaptr = 0; + IVTV_DEBUG_WARN("Failed to dma_map yuv blanking buffer\n"); + } } else { yi->blanking_dmaptr = 0; IVTV_DEBUG_WARN("Failed to allocate yuv blanking buffer\n"); diff --git a/drivers/media/pci/zoran/zoran.h b/drivers/media/pci/zoran/zoran.h index 56340553b2823..63592c4f9ed8c 100644 --- a/drivers/media/pci/zoran/zoran.h +++ b/drivers/media/pci/zoran/zoran.h @@ -154,12 +154,6 @@ struct zoran_jpg_settings { struct zoran; -/* zoran_fh contains per-open() settings */ -struct zoran_fh { - struct v4l2_fh fh; - struct zoran *zr; -}; - struct card_info { enum card_type type; char name[32]; diff --git a/drivers/media/pci/zoran/zoran_driver.c b/drivers/media/pci/zoran/zoran_driver.c index fa672cc8bc678..feb39854a989e 100644 --- a/drivers/media/pci/zoran/zoran_driver.c +++ b/drivers/media/pci/zoran/zoran_driver.c @@ -511,12 +511,11 @@ static int zoran_s_fmt_vid_cap(struct file *file, void *__fh, struct v4l2_format *fmt) { struct zoran *zr = video_drvdata(file); - struct zoran_fh *fh = __fh; int i; int res = 0; if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG) - return zoran_s_fmt_vid_out(file, fh, fmt); + return zoran_s_fmt_vid_out(file, __fh, fmt); for (i = 0; i < NUM_FORMATS; i++) if (fmt->fmt.pix.pixelformat == zoran_formats[i].fourcc) diff --git a/drivers/media/platform/amphion/vpu_v4l2.c b/drivers/media/platform/amphion/vpu_v4l2.c index 370e7028c7d2e..9dfc9dcb09664 100644 --- a/drivers/media/platform/amphion/vpu_v4l2.c +++ b/drivers/media/platform/amphion/vpu_v4l2.c @@ -709,8 +709,6 @@ static int vpu_v4l2_release(struct vpu_inst *inst) v4l2_ctrl_handler_free(&inst->ctrl_handler); mutex_destroy(&inst->lock); - v4l2_fh_del(&inst->fh); - v4l2_fh_exit(&inst->fh); call_void_vop(inst, cleanup); @@ -779,6 +777,8 @@ int vpu_v4l2_open(struct file *file, struct vpu_inst *inst) return 0; error: + v4l2_fh_del(&inst->fh); + v4l2_fh_exit(&inst->fh); vpu_inst_put(inst); return ret; } @@ -798,6 +798,9 @@ int vpu_v4l2_close(struct file *file) call_void_vop(inst, release); vpu_inst_unlock(inst); + v4l2_fh_del(&inst->fh); + v4l2_fh_exit(&inst->fh); + vpu_inst_unregister(inst); vpu_inst_put(inst); diff --git a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_hevc_req_multi_if.c b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_hevc_req_multi_if.c index 21836dd6ef85a..bbf56016f70a2 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_hevc_req_multi_if.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_hevc_req_multi_if.c @@ -821,7 +821,7 @@ static int vdec_hevc_slice_setup_core_buffer(struct vdec_hevc_slice_inst *inst, inst->vsi_core->fb.y.dma_addr = y_fb_dma; inst->vsi_core->fb.y.size = ctx->picinfo.fb_sz[0]; inst->vsi_core->fb.c.dma_addr = c_fb_dma; - inst->vsi_core->fb.y.size = ctx->picinfo.fb_sz[1]; + inst->vsi_core->fb.c.size = ctx->picinfo.fb_sz[1]; inst->vsi_core->dec.vdec_fb_va = (unsigned long)fb; diff --git a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c index 092d83b7e7954..dc3e0d2928104 100644 --- a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c +++ b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c @@ -752,6 +752,32 @@ static int mxc_get_free_slot(struct mxc_jpeg_slot_data *slot_data) return -1; } +static void mxc_jpeg_free_slot_data(struct mxc_jpeg_dev *jpeg) +{ + /* free descriptor for decoding/encoding phase */ + dma_free_coherent(jpeg->dev, sizeof(struct mxc_jpeg_desc), + jpeg->slot_data.desc, + jpeg->slot_data.desc_handle); + jpeg->slot_data.desc = NULL; + jpeg->slot_data.desc_handle = 0; + + /* free descriptor for encoder configuration phase / decoder DHT */ + dma_free_coherent(jpeg->dev, sizeof(struct mxc_jpeg_desc), + jpeg->slot_data.cfg_desc, + jpeg->slot_data.cfg_desc_handle); + jpeg->slot_data.cfg_desc_handle = 0; + jpeg->slot_data.cfg_desc = NULL; + + /* free configuration stream */ + dma_free_coherent(jpeg->dev, MXC_JPEG_MAX_CFG_STREAM, + jpeg->slot_data.cfg_stream_vaddr, + jpeg->slot_data.cfg_stream_handle); + jpeg->slot_data.cfg_stream_vaddr = NULL; + jpeg->slot_data.cfg_stream_handle = 0; + + jpeg->slot_data.used = false; +} + static bool mxc_jpeg_alloc_slot_data(struct mxc_jpeg_dev *jpeg) { struct mxc_jpeg_desc *desc; @@ -794,30 +820,11 @@ static bool mxc_jpeg_alloc_slot_data(struct mxc_jpeg_dev *jpeg) return true; err: dev_err(jpeg->dev, "Could not allocate descriptors for slot %d", jpeg->slot_data.slot); + mxc_jpeg_free_slot_data(jpeg); return false; } -static void mxc_jpeg_free_slot_data(struct mxc_jpeg_dev *jpeg) -{ - /* free descriptor for decoding/encoding phase */ - dma_free_coherent(jpeg->dev, sizeof(struct mxc_jpeg_desc), - jpeg->slot_data.desc, - jpeg->slot_data.desc_handle); - - /* free descriptor for encoder configuration phase / decoder DHT */ - dma_free_coherent(jpeg->dev, sizeof(struct mxc_jpeg_desc), - jpeg->slot_data.cfg_desc, - jpeg->slot_data.cfg_desc_handle); - - /* free configuration stream */ - dma_free_coherent(jpeg->dev, MXC_JPEG_MAX_CFG_STREAM, - jpeg->slot_data.cfg_stream_vaddr, - jpeg->slot_data.cfg_stream_handle); - - jpeg->slot_data.used = false; -} - static void mxc_jpeg_check_and_set_last_buffer(struct mxc_jpeg_ctx *ctx, struct vb2_v4l2_buffer *src_buf, struct vb2_v4l2_buffer *dst_buf) @@ -1913,9 +1920,19 @@ static void mxc_jpeg_buf_queue(struct vb2_buffer *vb) jpeg_src_buf = vb2_to_mxc_buf(vb); jpeg_src_buf->jpeg_parse_error = false; ret = mxc_jpeg_parse(ctx, vb); - if (ret) + if (ret) { jpeg_src_buf->jpeg_parse_error = true; + /* + * if the capture queue is not setup, the device_run() won't be scheduled, + * need to drop the error buffer, so that the decoding can continue + */ + if (!vb2_is_streaming(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx))) { + v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR); + return; + } + } + end: v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf); } diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h index 2810ebe9b5f75..5a4676d520793 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h @@ -361,7 +361,7 @@ void mxc_isi_channel_get(struct mxc_isi_pipe *pipe); void mxc_isi_channel_put(struct mxc_isi_pipe *pipe); void mxc_isi_channel_enable(struct mxc_isi_pipe *pipe); void mxc_isi_channel_disable(struct mxc_isi_pipe *pipe); -int mxc_isi_channel_chain(struct mxc_isi_pipe *pipe, bool bypass); +int mxc_isi_channel_chain(struct mxc_isi_pipe *pipe); void mxc_isi_channel_unchain(struct mxc_isi_pipe *pipe); void mxc_isi_channel_config(struct mxc_isi_pipe *pipe, diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c index 19e80b95ffeaa..ece352171b936 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c @@ -589,7 +589,7 @@ void mxc_isi_channel_release(struct mxc_isi_pipe *pipe) * * TODO: Support secondary line buffer for downscaling YUV420 images. */ -int mxc_isi_channel_chain(struct mxc_isi_pipe *pipe, bool bypass) +int mxc_isi_channel_chain(struct mxc_isi_pipe *pipe) { /* Channel chaining requires both line and output buffer. */ const u8 resources = MXC_ISI_CHANNEL_RES_OUTPUT_BUF diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c index 9745d6219a166..81223d28ee56e 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c @@ -235,6 +235,65 @@ static void mxc_isi_m2m_vb2_buffer_queue(struct vb2_buffer *vb2) v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf); } +static int mxc_isi_m2m_vb2_prepare_streaming(struct vb2_queue *q) +{ + struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(q); + const struct v4l2_pix_format_mplane *out_pix = &ctx->queues.out.format; + const struct v4l2_pix_format_mplane *cap_pix = &ctx->queues.cap.format; + const struct mxc_isi_format_info *cap_info = ctx->queues.cap.info; + const struct mxc_isi_format_info *out_info = ctx->queues.out.info; + struct mxc_isi_m2m *m2m = ctx->m2m; + int ret; + + guard(mutex)(&m2m->lock); + + if (m2m->usage_count == INT_MAX) + return -EOVERFLOW; + + /* + * Acquire the pipe and initialize the channel with the first user of + * the M2M device. + */ + if (m2m->usage_count == 0) { + bool bypass = cap_pix->width == out_pix->width && + cap_pix->height == out_pix->height && + cap_info->encoding == out_info->encoding; + + ret = mxc_isi_channel_acquire(m2m->pipe, + &mxc_isi_m2m_frame_write_done, + bypass); + if (ret) + return ret; + + mxc_isi_channel_get(m2m->pipe); + } + + m2m->usage_count++; + + /* + * Allocate resources for the channel, counting how many users require + * buffer chaining. + */ + if (!ctx->chained && out_pix->width > MXC_ISI_MAX_WIDTH_UNCHAINED) { + ret = mxc_isi_channel_chain(m2m->pipe); + if (ret) + goto err_deinit; + + m2m->chained_count++; + ctx->chained = true; + } + + return 0; + +err_deinit: + if (--m2m->usage_count == 0) { + mxc_isi_channel_put(m2m->pipe); + mxc_isi_channel_release(m2m->pipe); + } + + return ret; +} + static int mxc_isi_m2m_vb2_start_streaming(struct vb2_queue *q, unsigned int count) { @@ -264,6 +323,35 @@ static void mxc_isi_m2m_vb2_stop_streaming(struct vb2_queue *q) } } +static void mxc_isi_m2m_vb2_unprepare_streaming(struct vb2_queue *q) +{ + struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(q); + struct mxc_isi_m2m *m2m = ctx->m2m; + + guard(mutex)(&m2m->lock); + + /* + * If the last context is this one, reset it to make sure the device + * will be reconfigured when streaming is restarted. + */ + if (m2m->last_ctx == ctx) + m2m->last_ctx = NULL; + + /* Free the channel resources if this is the last chained context. */ + if (ctx->chained && --m2m->chained_count == 0) + mxc_isi_channel_unchain(m2m->pipe); + ctx->chained = false; + + /* Turn off the light with the last user. */ + if (--m2m->usage_count == 0) { + mxc_isi_channel_disable(m2m->pipe); + mxc_isi_channel_put(m2m->pipe); + mxc_isi_channel_release(m2m->pipe); + } + + WARN_ON(m2m->usage_count < 0); +} + static const struct vb2_ops mxc_isi_m2m_vb2_qops = { .queue_setup = mxc_isi_m2m_vb2_queue_setup, .buf_init = mxc_isi_m2m_vb2_buffer_init, @@ -271,8 +359,10 @@ static const struct vb2_ops mxc_isi_m2m_vb2_qops = { .buf_queue = mxc_isi_m2m_vb2_buffer_queue, .wait_prepare = vb2_ops_wait_prepare, .wait_finish = vb2_ops_wait_finish, + .prepare_streaming = mxc_isi_m2m_vb2_prepare_streaming, .start_streaming = mxc_isi_m2m_vb2_start_streaming, .stop_streaming = mxc_isi_m2m_vb2_stop_streaming, + .unprepare_streaming = mxc_isi_m2m_vb2_unprepare_streaming, }; static int mxc_isi_m2m_queue_init(void *priv, struct vb2_queue *src_vq, @@ -482,125 +572,6 @@ static int mxc_isi_m2m_s_fmt_vid(struct file *file, void *fh, return 0; } -static int mxc_isi_m2m_streamon(struct file *file, void *fh, - enum v4l2_buf_type type) -{ - struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(fh); - const struct v4l2_pix_format_mplane *out_pix = &ctx->queues.out.format; - const struct v4l2_pix_format_mplane *cap_pix = &ctx->queues.cap.format; - const struct mxc_isi_format_info *cap_info = ctx->queues.cap.info; - const struct mxc_isi_format_info *out_info = ctx->queues.out.info; - struct mxc_isi_m2m *m2m = ctx->m2m; - bool bypass; - - int ret; - - mutex_lock(&m2m->lock); - - if (m2m->usage_count == INT_MAX) { - ret = -EOVERFLOW; - goto unlock; - } - - bypass = cap_pix->width == out_pix->width && - cap_pix->height == out_pix->height && - cap_info->encoding == out_info->encoding; - - /* - * Acquire the pipe and initialize the channel with the first user of - * the M2M device. - */ - if (m2m->usage_count == 0) { - ret = mxc_isi_channel_acquire(m2m->pipe, - &mxc_isi_m2m_frame_write_done, - bypass); - if (ret) - goto unlock; - - mxc_isi_channel_get(m2m->pipe); - } - - m2m->usage_count++; - - /* - * Allocate resources for the channel, counting how many users require - * buffer chaining. - */ - if (!ctx->chained && out_pix->width > MXC_ISI_MAX_WIDTH_UNCHAINED) { - ret = mxc_isi_channel_chain(m2m->pipe, bypass); - if (ret) - goto deinit; - - m2m->chained_count++; - ctx->chained = true; - } - - /* - * Drop the lock to start the stream, as the .device_run() operation - * needs to acquire it. - */ - mutex_unlock(&m2m->lock); - ret = v4l2_m2m_ioctl_streamon(file, fh, type); - if (ret) { - /* Reacquire the lock for the cleanup path. */ - mutex_lock(&m2m->lock); - goto unchain; - } - - return 0; - -unchain: - if (ctx->chained && --m2m->chained_count == 0) - mxc_isi_channel_unchain(m2m->pipe); - ctx->chained = false; - -deinit: - if (--m2m->usage_count == 0) { - mxc_isi_channel_put(m2m->pipe); - mxc_isi_channel_release(m2m->pipe); - } - -unlock: - mutex_unlock(&m2m->lock); - return ret; -} - -static int mxc_isi_m2m_streamoff(struct file *file, void *fh, - enum v4l2_buf_type type) -{ - struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(fh); - struct mxc_isi_m2m *m2m = ctx->m2m; - - v4l2_m2m_ioctl_streamoff(file, fh, type); - - mutex_lock(&m2m->lock); - - /* - * If the last context is this one, reset it to make sure the device - * will be reconfigured when streaming is restarted. - */ - if (m2m->last_ctx == ctx) - m2m->last_ctx = NULL; - - /* Free the channel resources if this is the last chained context. */ - if (ctx->chained && --m2m->chained_count == 0) - mxc_isi_channel_unchain(m2m->pipe); - ctx->chained = false; - - /* Turn off the light with the last user. */ - if (--m2m->usage_count == 0) { - mxc_isi_channel_disable(m2m->pipe); - mxc_isi_channel_put(m2m->pipe); - mxc_isi_channel_release(m2m->pipe); - } - - WARN_ON(m2m->usage_count < 0); - - mutex_unlock(&m2m->lock); - - return 0; -} - static const struct v4l2_ioctl_ops mxc_isi_m2m_ioctl_ops = { .vidioc_querycap = mxc_isi_m2m_querycap, @@ -621,8 +592,8 @@ static const struct v4l2_ioctl_ops mxc_isi_m2m_ioctl_ops = { .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf, .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, - .vidioc_streamon = mxc_isi_m2m_streamon, - .vidioc_streamoff = mxc_isi_m2m_streamoff, + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, .vidioc_unsubscribe_event = v4l2_event_unsubscribe, diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c index 65d20e9bae69d..483523327c025 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c @@ -851,7 +851,7 @@ int mxc_isi_pipe_acquire(struct mxc_isi_pipe *pipe, /* Chain the channel if needed for wide resolutions. */ if (sink_fmt->width > MXC_ISI_MAX_WIDTH_UNCHAINED) { - ret = mxc_isi_channel_chain(pipe, bypass); + ret = mxc_isi_channel_chain(pipe); if (ret) mxc_isi_channel_release(pipe); } diff --git a/drivers/media/platform/qcom/camss/camss-csid.c b/drivers/media/platform/qcom/camss/camss-csid.c index 6360314f04a63..b90e2e690f3aa 100644 --- a/drivers/media/platform/qcom/camss/camss-csid.c +++ b/drivers/media/platform/qcom/camss/camss-csid.c @@ -239,11 +239,13 @@ static int csid_set_stream(struct v4l2_subdev *sd, int enable) int ret; if (enable) { - ret = v4l2_ctrl_handler_setup(&csid->ctrls); - if (ret < 0) { - dev_err(csid->camss->dev, - "could not sync v4l2 controls: %d\n", ret); - return ret; + if (csid->testgen.nmodes != CSID_PAYLOAD_MODE_DISABLED) { + ret = v4l2_ctrl_handler_setup(&csid->ctrls); + if (ret < 0) { + dev_err(csid->camss->dev, + "could not sync v4l2 controls: %d\n", ret); + return ret; + } } if (!csid->testgen.enabled && @@ -318,7 +320,8 @@ static void csid_try_format(struct csid_device *csid, break; case MSM_CSID_PAD_SRC: - if (csid->testgen_mode->cur.val == 0) { + if (csid->testgen.nmodes == CSID_PAYLOAD_MODE_DISABLED || + csid->testgen_mode->cur.val == 0) { /* Test generator is disabled, */ /* keep pad formats in sync */ u32 code = fmt->code; @@ -368,7 +371,8 @@ static int csid_enum_mbus_code(struct v4l2_subdev *sd, code->code = csid->formats[code->index].code; } else { - if (csid->testgen_mode->cur.val == 0) { + if (csid->testgen.nmodes == CSID_PAYLOAD_MODE_DISABLED || + csid->testgen_mode->cur.val == 0) { struct v4l2_mbus_framefmt *sink_fmt; sink_fmt = __csid_get_format(csid, sd_state, @@ -750,7 +754,8 @@ static int csid_link_setup(struct media_entity *entity, /* If test generator is enabled */ /* do not allow a link from CSIPHY to CSID */ - if (csid->testgen_mode->cur.val != 0) + if (csid->testgen.nmodes != CSID_PAYLOAD_MODE_DISABLED && + csid->testgen_mode->cur.val != 0) return -EBUSY; sd = media_entity_to_v4l2_subdev(remote->entity); @@ -843,24 +848,27 @@ int msm_csid_register_entity(struct csid_device *csid, MSM_CSID_NAME, csid->id); v4l2_set_subdevdata(sd, csid); - ret = v4l2_ctrl_handler_init(&csid->ctrls, 1); - if (ret < 0) { - dev_err(dev, "Failed to init ctrl handler: %d\n", ret); - return ret; - } + if (csid->testgen.nmodes != CSID_PAYLOAD_MODE_DISABLED) { + ret = v4l2_ctrl_handler_init(&csid->ctrls, 1); + if (ret < 0) { + dev_err(dev, "Failed to init ctrl handler: %d\n", ret); + return ret; + } - csid->testgen_mode = v4l2_ctrl_new_std_menu_items(&csid->ctrls, - &csid_ctrl_ops, V4L2_CID_TEST_PATTERN, - csid->testgen.nmodes, 0, 0, - csid->testgen.modes); + csid->testgen_mode = + v4l2_ctrl_new_std_menu_items(&csid->ctrls, + &csid_ctrl_ops, V4L2_CID_TEST_PATTERN, + csid->testgen.nmodes, 0, 0, + csid->testgen.modes); - if (csid->ctrls.error) { - dev_err(dev, "Failed to init ctrl: %d\n", csid->ctrls.error); - ret = csid->ctrls.error; - goto free_ctrl; - } + if (csid->ctrls.error) { + dev_err(dev, "Failed to init ctrl: %d\n", csid->ctrls.error); + ret = csid->ctrls.error; + goto free_ctrl; + } - csid->subdev.ctrl_handler = &csid->ctrls; + csid->subdev.ctrl_handler = &csid->ctrls; + } ret = csid_init_formats(sd, NULL); if (ret < 0) { @@ -891,7 +899,8 @@ int msm_csid_register_entity(struct csid_device *csid, media_cleanup: media_entity_cleanup(&sd->entity); free_ctrl: - v4l2_ctrl_handler_free(&csid->ctrls); + if (csid->testgen.nmodes != CSID_PAYLOAD_MODE_DISABLED) + v4l2_ctrl_handler_free(&csid->ctrls); return ret; } @@ -904,5 +913,6 @@ void msm_csid_unregister_entity(struct csid_device *csid) { v4l2_device_unregister_subdev(&csid->subdev); media_entity_cleanup(&csid->subdev.entity); - v4l2_ctrl_handler_free(&csid->ctrls); + if (csid->testgen.nmodes != CSID_PAYLOAD_MODE_DISABLED) + v4l2_ctrl_handler_free(&csid->ctrls); } diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c index 0754645d26acb..e62245c5c6fc4 100644 --- a/drivers/media/platform/qcom/camss/camss.c +++ b/drivers/media/platform/qcom/camss/camss.c @@ -1660,7 +1660,7 @@ static int camss_probe(struct platform_device *pdev) ret = v4l2_device_register(camss->dev, &camss->v4l2_dev); if (ret < 0) { dev_err(dev, "Failed to register V4L2 device: %d\n", ret); - goto err_genpd_cleanup; + goto err_media_device_cleanup; } v4l2_async_nf_init(&camss->notifier, &camss->v4l2_dev); @@ -1711,6 +1711,8 @@ static int camss_probe(struct platform_device *pdev) v4l2_device_unregister(&camss->v4l2_dev); v4l2_async_nf_cleanup(&camss->notifier); pm_runtime_disable(dev); +err_media_device_cleanup: + media_device_cleanup(&camss->media_dev); err_genpd_cleanup: camss_genpd_cleanup(camss); diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index b570eb8c37568..64a858783c41e 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -334,13 +334,13 @@ static int venus_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&core->work, venus_sys_error_handler); init_waitqueue_head(&core->sys_err_done); - ret = devm_request_threaded_irq(dev, core->irq, hfi_isr, venus_isr_thread, - IRQF_TRIGGER_HIGH | IRQF_ONESHOT, - "venus", core); + ret = hfi_create(core, &venus_core_ops); if (ret) goto err_core_put; - ret = hfi_create(core, &venus_core_ops); + ret = devm_request_threaded_irq(dev, core->irq, hfi_isr, venus_isr_thread, + IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + "venus", core); if (ret) goto err_core_put; @@ -348,7 +348,7 @@ static int venus_probe(struct platform_device *pdev) ret = v4l2_device_register(dev, &core->v4l2_dev); if (ret) - goto err_core_deinit; + goto err_hfi_destroy; platform_set_drvdata(pdev, core); @@ -380,24 +380,24 @@ static int venus_probe(struct platform_device *pdev) ret = venus_enumerate_codecs(core, VIDC_SESSION_TYPE_DEC); if (ret) - goto err_venus_shutdown; + goto err_core_deinit; ret = venus_enumerate_codecs(core, VIDC_SESSION_TYPE_ENC); if (ret) - goto err_venus_shutdown; + goto err_core_deinit; ret = pm_runtime_put_sync(dev); if (ret) { pm_runtime_get_noresume(dev); - goto err_dev_unregister; + goto err_core_deinit; } venus_dbgfs_init(core); return 0; -err_dev_unregister: - v4l2_device_unregister(&core->v4l2_dev); +err_core_deinit: + hfi_core_deinit(core, false); err_venus_shutdown: venus_shutdown(core); err_firmware_deinit: @@ -408,9 +408,9 @@ static int venus_probe(struct platform_device *pdev) pm_runtime_put_noidle(dev); pm_runtime_disable(dev); pm_runtime_set_suspended(dev); + v4l2_device_unregister(&core->v4l2_dev); +err_hfi_destroy: hfi_destroy(core); -err_core_deinit: - hfi_core_deinit(core, false); err_core_put: if (core->pm_ops->core_put) core->pm_ops->core_put(core); diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h index 4a633261ece47..ba8afda6667d4 100644 --- a/drivers/media/platform/qcom/venus/core.h +++ b/drivers/media/platform/qcom/venus/core.h @@ -28,6 +28,8 @@ #define VIDC_PMDOMAINS_NUM_MAX 3 #define VIDC_RESETS_NUM_MAX 2 +#define VENUS_MAX_FPS 240 + extern int venus_fw_debug; struct freq_tbl { diff --git a/drivers/media/platform/qcom/venus/firmware.c b/drivers/media/platform/qcom/venus/firmware.c index fe7da2b304829..01461b2fed51d 100644 --- a/drivers/media/platform/qcom/venus/firmware.c +++ b/drivers/media/platform/qcom/venus/firmware.c @@ -30,7 +30,7 @@ static void venus_reset_cpu(struct venus_core *core) u32 fw_size = core->fw.mapped_mem_size; void __iomem *wrapper_base; - if (IS_IRIS2_1(core)) + if (IS_IRIS2(core) || IS_IRIS2_1(core)) wrapper_base = core->wrapper_tz_base; else wrapper_base = core->wrapper_base; @@ -42,7 +42,7 @@ static void venus_reset_cpu(struct venus_core *core) writel(fw_size, wrapper_base + WRAPPER_NONPIX_START_ADDR); writel(fw_size, wrapper_base + WRAPPER_NONPIX_END_ADDR); - if (IS_IRIS2_1(core)) { + if (IS_IRIS2(core) || IS_IRIS2_1(core)) { /* Bring XTSS out of reset */ writel(0, wrapper_base + WRAPPER_TZ_XTSS_SW_RESET); } else { @@ -68,7 +68,7 @@ int venus_set_hw_state(struct venus_core *core, bool resume) if (resume) { venus_reset_cpu(core); } else { - if (IS_IRIS2_1(core)) + if (IS_IRIS2(core) || IS_IRIS2_1(core)) writel(WRAPPER_XTSS_SW_RESET_BIT, core->wrapper_tz_base + WRAPPER_TZ_XTSS_SW_RESET); else @@ -181,7 +181,7 @@ static int venus_shutdown_no_tz(struct venus_core *core) void __iomem *wrapper_base = core->wrapper_base; void __iomem *wrapper_tz_base = core->wrapper_tz_base; - if (IS_IRIS2_1(core)) { + if (IS_IRIS2(core) || IS_IRIS2_1(core)) { /* Assert the reset to XTSS */ reg = readl(wrapper_tz_base + WRAPPER_TZ_XTSS_SW_RESET); reg |= WRAPPER_XTSS_SW_RESET_BIT; diff --git a/drivers/media/platform/qcom/venus/hfi_msgs.c b/drivers/media/platform/qcom/venus/hfi_msgs.c index 0a041b4db9efc..cf0d97cbc4631 100644 --- a/drivers/media/platform/qcom/venus/hfi_msgs.c +++ b/drivers/media/platform/qcom/venus/hfi_msgs.c @@ -33,8 +33,9 @@ static void event_seq_changed(struct venus_core *core, struct venus_inst *inst, struct hfi_buffer_requirements *bufreq; struct hfi_extradata_input_crop *crop; struct hfi_dpb_counts *dpb_count; + u32 ptype, rem_bytes; + u32 size_read = 0; u8 *data_ptr; - u32 ptype; inst->error = HFI_ERR_NONE; @@ -44,86 +45,118 @@ static void event_seq_changed(struct venus_core *core, struct venus_inst *inst, break; default: inst->error = HFI_ERR_SESSION_INVALID_PARAMETER; - goto done; + inst->ops->event_notify(inst, EVT_SYS_EVENT_CHANGE, &event); + return; } event.event_type = pkt->event_data1; num_properties_changed = pkt->event_data2; - if (!num_properties_changed) { - inst->error = HFI_ERR_SESSION_INSUFFICIENT_RESOURCES; - goto done; - } + if (!num_properties_changed) + goto error; data_ptr = (u8 *)&pkt->ext_event_data[0]; + rem_bytes = pkt->shdr.hdr.size - sizeof(*pkt); + do { + if (rem_bytes < sizeof(u32)) + goto error; ptype = *((u32 *)data_ptr); + + data_ptr += sizeof(u32); + rem_bytes -= sizeof(u32); + switch (ptype) { case HFI_PROPERTY_PARAM_FRAME_SIZE: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_framesize)) + goto error; + frame_sz = (struct hfi_framesize *)data_ptr; event.width = frame_sz->width; event.height = frame_sz->height; - data_ptr += sizeof(*frame_sz); + size_read = sizeof(struct hfi_framesize); break; case HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_profile_level)) + goto error; + profile_level = (struct hfi_profile_level *)data_ptr; event.profile = profile_level->profile; event.level = profile_level->level; - data_ptr += sizeof(*profile_level); + size_read = sizeof(struct hfi_profile_level); break; case HFI_PROPERTY_PARAM_VDEC_PIXEL_BITDEPTH: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_bit_depth)) + goto error; + pixel_depth = (struct hfi_bit_depth *)data_ptr; event.bit_depth = pixel_depth->bit_depth; - data_ptr += sizeof(*pixel_depth); + size_read = sizeof(struct hfi_bit_depth); break; case HFI_PROPERTY_PARAM_VDEC_PIC_STRUCT: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_pic_struct)) + goto error; + pic_struct = (struct hfi_pic_struct *)data_ptr; event.pic_struct = pic_struct->progressive_only; - data_ptr += sizeof(*pic_struct); + size_read = sizeof(struct hfi_pic_struct); break; case HFI_PROPERTY_PARAM_VDEC_COLOUR_SPACE: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_colour_space)) + goto error; + colour_info = (struct hfi_colour_space *)data_ptr; event.colour_space = colour_info->colour_space; - data_ptr += sizeof(*colour_info); + size_read = sizeof(struct hfi_colour_space); break; case HFI_PROPERTY_CONFIG_VDEC_ENTROPY: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(u32)) + goto error; + event.entropy_mode = *(u32 *)data_ptr; - data_ptr += sizeof(u32); + size_read = sizeof(u32); break; case HFI_PROPERTY_CONFIG_BUFFER_REQUIREMENTS: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_buffer_requirements)) + goto error; + bufreq = (struct hfi_buffer_requirements *)data_ptr; event.buf_count = hfi_bufreq_get_count_min(bufreq, ver); - data_ptr += sizeof(*bufreq); + size_read = sizeof(struct hfi_buffer_requirements); break; case HFI_INDEX_EXTRADATA_INPUT_CROP: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_extradata_input_crop)) + goto error; + crop = (struct hfi_extradata_input_crop *)data_ptr; event.input_crop.left = crop->left; event.input_crop.top = crop->top; event.input_crop.width = crop->width; event.input_crop.height = crop->height; - data_ptr += sizeof(*crop); + size_read = sizeof(struct hfi_extradata_input_crop); break; case HFI_PROPERTY_PARAM_VDEC_DPB_COUNTS: - data_ptr += sizeof(u32); + if (rem_bytes < sizeof(struct hfi_dpb_counts)) + goto error; + dpb_count = (struct hfi_dpb_counts *)data_ptr; event.buf_count = dpb_count->fw_min_cnt; - data_ptr += sizeof(*dpb_count); + size_read = sizeof(struct hfi_dpb_counts); break; default: + size_read = 0; break; } + data_ptr += size_read; + rem_bytes -= size_read; num_properties_changed--; } while (num_properties_changed > 0); -done: + inst->ops->event_notify(inst, EVT_SYS_EVENT_CHANGE, &event); + return; + +error: + inst->error = HFI_ERR_SESSION_INSUFFICIENT_RESOURCES; inst->ops->event_notify(inst, EVT_SYS_EVENT_CHANGE, &event); } diff --git a/drivers/media/platform/qcom/venus/hfi_venus.c b/drivers/media/platform/qcom/venus/hfi_venus.c index ab93757fff4b3..8e21152796011 100644 --- a/drivers/media/platform/qcom/venus/hfi_venus.c +++ b/drivers/media/platform/qcom/venus/hfi_venus.c @@ -239,6 +239,7 @@ static int venus_write_queue(struct venus_hfi_device *hdev, static int venus_read_queue(struct venus_hfi_device *hdev, struct iface_queue *queue, void *pkt, u32 *tx_req) { + struct hfi_pkt_hdr *pkt_hdr = NULL; struct hfi_queue_header *qhdr; u32 dwords, new_rd_idx; u32 rd_idx, wr_idx, type, qsize; @@ -304,6 +305,9 @@ static int venus_read_queue(struct venus_hfi_device *hdev, memcpy(pkt, rd_ptr, len); memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2); } + pkt_hdr = (struct hfi_pkt_hdr *)(pkt); + if ((pkt_hdr->size >> 2) != dwords) + return -EINVAL; } else { /* bad packet received, dropping */ new_rd_idx = qhdr->write_idx; @@ -1689,6 +1693,7 @@ void venus_hfi_destroy(struct venus_core *core) venus_interface_queues_release(hdev); mutex_destroy(&hdev->lock); kfree(hdev); + disable_irq(core->irq); core->ops = NULL; } diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c index 884ee6e9d4bd1..8be056210f1d3 100644 --- a/drivers/media/platform/qcom/venus/vdec.c +++ b/drivers/media/platform/qcom/venus/vdec.c @@ -481,11 +481,10 @@ static int vdec_s_parm(struct file *file, void *fh, struct v4l2_streamparm *a) us_per_frame = timeperframe->numerator * (u64)USEC_PER_SEC; do_div(us_per_frame, timeperframe->denominator); - if (!us_per_frame) - return -EINVAL; - + us_per_frame = clamp(us_per_frame, 1, USEC_PER_SEC); fps = (u64)USEC_PER_SEC; do_div(fps, us_per_frame); + fps = min(VENUS_MAX_FPS, fps); inst->fps = fps; inst->timeperframe = *timeperframe; diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c index 44b13696cf82a..dd3840f7bb7bd 100644 --- a/drivers/media/platform/qcom/venus/venc.c +++ b/drivers/media/platform/qcom/venus/venc.c @@ -411,11 +411,10 @@ static int venc_s_parm(struct file *file, void *fh, struct v4l2_streamparm *a) us_per_frame = timeperframe->numerator * (u64)USEC_PER_SEC; do_div(us_per_frame, timeperframe->denominator); - if (!us_per_frame) - return -EINVAL; - + us_per_frame = clamp(us_per_frame, 1, USEC_PER_SEC); fps = (u64)USEC_PER_SEC; do_div(fps, us_per_frame); + fps = min(VENUS_MAX_FPS, fps); inst->timeperframe = *timeperframe; inst->fps = fps; diff --git a/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c b/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c index 5dc1f908b49bd..9aa484126a0dd 100644 --- a/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c +++ b/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c @@ -806,13 +806,12 @@ static int c8sectpfe_probe(struct platform_device *pdev) } tsin->i2c_adapter = of_find_i2c_adapter_by_node(i2c_bus); + of_node_put(i2c_bus); if (!tsin->i2c_adapter) { dev_err(&pdev->dev, "No i2c adapter found\n"); - of_node_put(i2c_bus); ret = -ENODEV; goto err_node_put; } - of_node_put(i2c_bus); /* Acquire reset GPIO and activate it */ tsin->rst_gpio = devm_fwnode_gpiod_get(dev, diff --git a/drivers/media/platform/st/sti/delta/delta-mjpeg-dec.c b/drivers/media/platform/st/sti/delta/delta-mjpeg-dec.c index 0533d4a083d24..a078f1107300e 100644 --- a/drivers/media/platform/st/sti/delta/delta-mjpeg-dec.c +++ b/drivers/media/platform/st/sti/delta/delta-mjpeg-dec.c @@ -239,7 +239,7 @@ static int delta_mjpeg_ipc_open(struct delta_ctx *pctx) return 0; } -static int delta_mjpeg_ipc_decode(struct delta_ctx *pctx, struct delta_au *au) +static int delta_mjpeg_ipc_decode(struct delta_ctx *pctx, dma_addr_t pstart, dma_addr_t pend) { struct delta_dev *delta = pctx->dev; struct delta_mjpeg_ctx *ctx = to_ctx(pctx); @@ -256,8 +256,8 @@ static int delta_mjpeg_ipc_decode(struct delta_ctx *pctx, struct delta_au *au) memset(params, 0, sizeof(*params)); - params->picture_start_addr_p = (u32)(au->paddr); - params->picture_end_addr_p = (u32)(au->paddr + au->size - 1); + params->picture_start_addr_p = pstart; + params->picture_end_addr_p = pend; /* * !WARNING! @@ -374,12 +374,14 @@ static int delta_mjpeg_decode(struct delta_ctx *pctx, struct delta_au *pau) struct delta_dev *delta = pctx->dev; struct delta_mjpeg_ctx *ctx = to_ctx(pctx); int ret; - struct delta_au au = *pau; + void *au_vaddr = pau->vaddr; + dma_addr_t au_dma = pau->paddr; + size_t au_size = pau->size; unsigned int data_offset = 0; struct mjpeg_header *header = &ctx->header_struct; if (!ctx->header) { - ret = delta_mjpeg_read_header(pctx, au.vaddr, au.size, + ret = delta_mjpeg_read_header(pctx, au_vaddr, au_size, header, &data_offset); if (ret) { pctx->stream_errors++; @@ -405,17 +407,17 @@ static int delta_mjpeg_decode(struct delta_ctx *pctx, struct delta_au *pau) goto err; } - ret = delta_mjpeg_read_header(pctx, au.vaddr, au.size, + ret = delta_mjpeg_read_header(pctx, au_vaddr, au_size, ctx->header, &data_offset); if (ret) { pctx->stream_errors++; goto err; } - au.paddr += data_offset; - au.vaddr += data_offset; + au_dma += data_offset; + au_vaddr += data_offset; - ret = delta_mjpeg_ipc_decode(pctx, &au); + ret = delta_mjpeg_ipc_decode(pctx, au_dma, au_dma + au_size - 1); if (ret) goto err; diff --git a/drivers/media/platform/ti/davinci/vpif.c b/drivers/media/platform/ti/davinci/vpif.c index 63cdfed37bc9b..fa9679abf7a48 100644 --- a/drivers/media/platform/ti/davinci/vpif.c +++ b/drivers/media/platform/ti/davinci/vpif.c @@ -505,7 +505,7 @@ static int vpif_probe(struct platform_device *pdev) pdev_display = kzalloc(sizeof(*pdev_display), GFP_KERNEL); if (!pdev_display) { ret = -ENOMEM; - goto err_put_pdev_capture; + goto err_del_pdev_capture; } pdev_display->name = "vpif_display"; @@ -528,6 +528,8 @@ static int vpif_probe(struct platform_device *pdev) err_put_pdev_display: platform_device_put(pdev_display); +err_del_pdev_capture: + platform_device_del(pdev_capture); err_put_pdev_capture: platform_device_put(pdev_capture); err_put_rpm: diff --git a/drivers/media/platform/ti/omap3isp/ispccdc.c b/drivers/media/platform/ti/omap3isp/ispccdc.c index 2fe42aa918004..6eb387ea1dae1 100644 --- a/drivers/media/platform/ti/omap3isp/ispccdc.c +++ b/drivers/media/platform/ti/omap3isp/ispccdc.c @@ -446,8 +446,8 @@ static int ccdc_lsc_config(struct isp_ccdc_device *ccdc, if (ret < 0) goto done; - dma_sync_sg_for_cpu(isp->dev, req->table.sgt.sgl, - req->table.sgt.nents, DMA_TO_DEVICE); + dma_sync_sgtable_for_cpu(isp->dev, &req->table.sgt, + DMA_TO_DEVICE); if (copy_from_user(req->table.addr, config->lsc, req->config.size)) { @@ -455,8 +455,8 @@ static int ccdc_lsc_config(struct isp_ccdc_device *ccdc, goto done; } - dma_sync_sg_for_device(isp->dev, req->table.sgt.sgl, - req->table.sgt.nents, DMA_TO_DEVICE); + dma_sync_sgtable_for_device(isp->dev, &req->table.sgt, + DMA_TO_DEVICE); } spin_lock_irqsave(&ccdc->lsc.req_lock, flags); diff --git a/drivers/media/platform/ti/omap3isp/ispstat.c b/drivers/media/platform/ti/omap3isp/ispstat.c index 68cf68dbcace2..dc496ca9748a6 100644 --- a/drivers/media/platform/ti/omap3isp/ispstat.c +++ b/drivers/media/platform/ti/omap3isp/ispstat.c @@ -161,8 +161,7 @@ static void isp_stat_buf_sync_for_device(struct ispstat *stat, if (ISP_STAT_USES_DMAENGINE(stat)) return; - dma_sync_sg_for_device(stat->isp->dev, buf->sgt.sgl, - buf->sgt.nents, DMA_FROM_DEVICE); + dma_sync_sgtable_for_device(stat->isp->dev, &buf->sgt, DMA_FROM_DEVICE); } static void isp_stat_buf_sync_for_cpu(struct ispstat *stat, @@ -171,8 +170,7 @@ static void isp_stat_buf_sync_for_cpu(struct ispstat *stat, if (ISP_STAT_USES_DMAENGINE(stat)) return; - dma_sync_sg_for_cpu(stat->isp->dev, buf->sgt.sgl, - buf->sgt.nents, DMA_FROM_DEVICE); + dma_sync_sgtable_for_cpu(stat->isp->dev, &buf->sgt, DMA_FROM_DEVICE); } static void isp_stat_buf_clear(struct ispstat *stat) diff --git a/drivers/media/platform/verisilicon/hantro_drv.c b/drivers/media/platform/verisilicon/hantro_drv.c index 1874c976081f8..35833ee8beb51 100644 --- a/drivers/media/platform/verisilicon/hantro_drv.c +++ b/drivers/media/platform/verisilicon/hantro_drv.c @@ -910,6 +910,8 @@ static int hantro_add_func(struct hantro_dev *vpu, unsigned int funcid) vpu->decoder = func; v4l2_disable_ioctl(vfd, VIDIOC_TRY_ENCODER_CMD); v4l2_disable_ioctl(vfd, VIDIOC_ENCODER_CMD); + v4l2_disable_ioctl(vfd, VIDIOC_G_SELECTION); + v4l2_disable_ioctl(vfd, VIDIOC_S_SELECTION); } video_set_drvdata(vfd, vpu); diff --git a/drivers/media/platform/verisilicon/hantro_v4l2.c b/drivers/media/platform/verisilicon/hantro_v4l2.c index db145519fc5d3..52009c5b0db4b 100644 --- a/drivers/media/platform/verisilicon/hantro_v4l2.c +++ b/drivers/media/platform/verisilicon/hantro_v4l2.c @@ -655,8 +655,7 @@ static int vidioc_g_selection(struct file *file, void *priv, struct hantro_ctx *ctx = fh_to_ctx(priv); /* Crop only supported on source. */ - if (!ctx->is_encoder || - sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) return -EINVAL; switch (sel->target) { @@ -688,8 +687,7 @@ static int vidioc_s_selection(struct file *file, void *priv, struct vb2_queue *vq; /* Crop only supported on source. */ - if (!ctx->is_encoder || - sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) return -EINVAL; /* Change not allowed if the queue is streaming. */ diff --git a/drivers/media/platform/verisilicon/rockchip_vpu_hw.c b/drivers/media/platform/verisilicon/rockchip_vpu_hw.c index f975276707835..df5b7dadb1b4e 100644 --- a/drivers/media/platform/verisilicon/rockchip_vpu_hw.c +++ b/drivers/media/platform/verisilicon/rockchip_vpu_hw.c @@ -17,7 +17,6 @@ #define RK3066_ACLK_MAX_FREQ (300 * 1000 * 1000) #define RK3288_ACLK_MAX_FREQ (400 * 1000 * 1000) -#define RK3588_ACLK_MAX_FREQ (300 * 1000 * 1000) #define ROCKCHIP_VPU981_MIN_SIZE 64 @@ -441,13 +440,6 @@ static int rk3066_vpu_hw_init(struct hantro_dev *vpu) return 0; } -static int rk3588_vpu981_hw_init(struct hantro_dev *vpu) -{ - /* Bump ACLKs to max. possible freq. to improve performance. */ - clk_set_rate(vpu->clocks[0].clk, RK3588_ACLK_MAX_FREQ); - return 0; -} - static int rockchip_vpu_hw_init(struct hantro_dev *vpu) { /* Bump ACLK to max. possible freq. to improve performance. */ @@ -808,7 +800,6 @@ const struct hantro_variant rk3588_vpu981_variant = { .codec_ops = rk3588_vpu981_codec_ops, .irqs = rk3588_vpu981_irqs, .num_irqs = ARRAY_SIZE(rk3588_vpu981_irqs), - .init = rk3588_vpu981_hw_init, .clk_names = rk3588_vpu981_vpu_clk_names, .num_clocks = ARRAY_SIZE(rk3588_vpu981_vpu_clk_names) }; diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c index e5590a708f1c5..bb4aabb08c06e 100644 --- a/drivers/media/rc/imon.c +++ b/drivers/media/rc/imon.c @@ -536,7 +536,9 @@ static int display_open(struct inode *inode, struct file *file) mutex_lock(&ictx->lock); - if (!ictx->display_supported) { + if (ictx->disconnected) { + retval = -ENODEV; + } else if (!ictx->display_supported) { pr_err("display not supported by device\n"); retval = -ENODEV; } else if (ictx->display_isopen) { @@ -598,6 +600,9 @@ static int send_packet(struct imon_context *ictx) int retval = 0; struct usb_ctrlrequest *control_req = NULL; + if (ictx->disconnected) + return -ENODEV; + /* Check if we need to use control or interrupt urb */ if (!ictx->tx_control) { pipe = usb_sndintpipe(ictx->usbdev_intf0, @@ -645,12 +650,15 @@ static int send_packet(struct imon_context *ictx) smp_rmb(); /* ensure later readers know we're not busy */ pr_err_ratelimited("error submitting urb(%d)\n", retval); } else { - /* Wait for transmission to complete (or abort) */ - retval = wait_for_completion_interruptible( - &ictx->tx.finished); - if (retval) { + /* Wait for transmission to complete (or abort or timeout) */ + retval = wait_for_completion_interruptible_timeout(&ictx->tx.finished, 10 * HZ); + if (retval <= 0) { usb_kill_urb(ictx->tx_urb); pr_err_ratelimited("task interrupted\n"); + if (retval < 0) + ictx->tx.status = retval; + else + ictx->tx.status = -ETIMEDOUT; } ictx->tx.busy = false; @@ -949,12 +957,14 @@ static ssize_t vfd_write(struct file *file, const char __user *buf, static const unsigned char vfd_packet6[] = { 0x01, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF }; - if (ictx->disconnected) - return -ENODEV; - if (mutex_lock_interruptible(&ictx->lock)) return -ERESTARTSYS; + if (ictx->disconnected) { + retval = -ENODEV; + goto exit; + } + if (!ictx->dev_present_intf0) { pr_err_ratelimited("no iMON device present\n"); retval = -ENODEV; @@ -1029,11 +1039,13 @@ static ssize_t lcd_write(struct file *file, const char __user *buf, int retval = 0; struct imon_context *ictx = file->private_data; - if (ictx->disconnected) - return -ENODEV; - mutex_lock(&ictx->lock); + if (ictx->disconnected) { + retval = -ENODEV; + goto exit; + } + if (!ictx->display_supported) { pr_err_ratelimited("no iMON display present\n"); retval = -ENODEV; @@ -1745,14 +1757,6 @@ static void usb_rx_callback_intf0(struct urb *urb) if (!ictx) return; - /* - * if we get a callback before we're done configuring the hardware, we - * can't yet process the data, as there's nowhere to send it, but we - * still need to submit a new rx URB to avoid wedging the hardware - */ - if (!ictx->dev_present_intf0) - goto out; - switch (urb->status) { case -ENOENT: /* usbcore unlink successful! */ return; @@ -1761,16 +1765,29 @@ static void usb_rx_callback_intf0(struct urb *urb) break; case 0: - imon_incoming_packet(ictx, urb, intfnum); + /* + * if we get a callback before we're done configuring the hardware, we + * can't yet process the data, as there's nowhere to send it, but we + * still need to submit a new rx URB to avoid wedging the hardware + */ + if (ictx->dev_present_intf0) + imon_incoming_packet(ictx, urb, intfnum); break; + case -ECONNRESET: + case -EILSEQ: + case -EPROTO: + case -EPIPE: + dev_warn(ictx->dev, "imon %s: status(%d)\n", + __func__, urb->status); + return; + default: dev_warn(ictx->dev, "imon %s: status(%d): ignored\n", __func__, urb->status); break; } -out: usb_submit_urb(ictx->rx_urb_intf0, GFP_ATOMIC); } @@ -1786,14 +1803,6 @@ static void usb_rx_callback_intf1(struct urb *urb) if (!ictx) return; - /* - * if we get a callback before we're done configuring the hardware, we - * can't yet process the data, as there's nowhere to send it, but we - * still need to submit a new rx URB to avoid wedging the hardware - */ - if (!ictx->dev_present_intf1) - goto out; - switch (urb->status) { case -ENOENT: /* usbcore unlink successful! */ return; @@ -1802,16 +1811,29 @@ static void usb_rx_callback_intf1(struct urb *urb) break; case 0: - imon_incoming_packet(ictx, urb, intfnum); + /* + * if we get a callback before we're done configuring the hardware, we + * can't yet process the data, as there's nowhere to send it, but we + * still need to submit a new rx URB to avoid wedging the hardware + */ + if (ictx->dev_present_intf1) + imon_incoming_packet(ictx, urb, intfnum); break; + case -ECONNRESET: + case -EILSEQ: + case -EPROTO: + case -EPIPE: + dev_warn(ictx->dev, "imon %s: status(%d)\n", + __func__, urb->status); + return; + default: dev_warn(ictx->dev, "imon %s: status(%d): ignored\n", __func__, urb->status); break; } -out: usb_submit_urb(ictx->rx_urb_intf1, GFP_ATOMIC); } @@ -2499,7 +2521,11 @@ static void imon_disconnect(struct usb_interface *interface) int ifnum; ictx = usb_get_intfdata(interface); + + mutex_lock(&ictx->lock); ictx->disconnected = true; + mutex_unlock(&ictx->lock); + dev = ictx->dev; ifnum = interface->cur_altsetting->desc.bInterfaceNumber; diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c index f8901d6fbe9bf..ff3088691d5e1 100644 --- a/drivers/media/rc/lirc_dev.c +++ b/drivers/media/rc/lirc_dev.c @@ -735,11 +735,11 @@ int lirc_register(struct rc_dev *dev) cdev_init(&dev->lirc_cdev, &lirc_fops); + get_device(&dev->dev); + err = cdev_device_add(&dev->lirc_cdev, &dev->lirc_dev); if (err) - goto out_ida; - - get_device(&dev->dev); + goto out_put_device; switch (dev->driver_type) { case RC_DRIVER_SCANCODE: @@ -763,7 +763,8 @@ int lirc_register(struct rc_dev *dev) return 0; -out_ida: +out_put_device: + put_device(&dev->lirc_dev); ida_free(&lirc_ida, minor); return err; } diff --git a/drivers/media/rc/redrat3.c b/drivers/media/rc/redrat3.c index 9f2947af33aa7..880981e1c507e 100644 --- a/drivers/media/rc/redrat3.c +++ b/drivers/media/rc/redrat3.c @@ -422,7 +422,7 @@ static int redrat3_send_cmd(int cmd, struct redrat3_dev *rr3) static int redrat3_enable_detector(struct redrat3_dev *rr3) { struct device *dev = rr3->dev; - u8 ret; + int ret; ret = redrat3_send_cmd(RR3_RC_DET_ENABLE, rr3); if (ret != 0) diff --git a/drivers/media/test-drivers/vidtv/vidtv_channel.c b/drivers/media/test-drivers/vidtv/vidtv_channel.c index 7838e62727128..f3023e91b3ebc 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_channel.c +++ b/drivers/media/test-drivers/vidtv/vidtv_channel.c @@ -497,7 +497,7 @@ int vidtv_channel_si_init(struct vidtv_mux *m) vidtv_psi_sdt_table_destroy(m->si.sdt); free_pat: vidtv_psi_pat_table_destroy(m->si.pat); - return 0; + return -EINVAL; } void vidtv_channel_si_destroy(struct vidtv_mux *m) diff --git a/drivers/media/test-drivers/vivid/vivid-ctrls.c b/drivers/media/test-drivers/vivid/vivid-ctrls.c index f2b20e25a7a49..5ca385a6a1365 100644 --- a/drivers/media/test-drivers/vivid/vivid-ctrls.c +++ b/drivers/media/test-drivers/vivid/vivid-ctrls.c @@ -240,7 +240,8 @@ static const struct v4l2_ctrl_config vivid_ctrl_u8_pixel_array = { .min = 0x00, .max = 0xff, .step = 1, - .dims = { 640 / PIXEL_ARRAY_DIV, 360 / PIXEL_ARRAY_DIV }, + .dims = { DIV_ROUND_UP(360, PIXEL_ARRAY_DIV), + DIV_ROUND_UP(640, PIXEL_ARRAY_DIV) }, }; static const struct v4l2_ctrl_config vivid_ctrl_s32_array = { diff --git a/drivers/media/test-drivers/vivid/vivid-kthread-cap.c b/drivers/media/test-drivers/vivid/vivid-kthread-cap.c index 42048727d7ff3..b8cdffc9a1e9e 100644 --- a/drivers/media/test-drivers/vivid/vivid-kthread-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-kthread-cap.c @@ -765,9 +765,14 @@ static int vivid_thread_vid_cap(void *data) next_jiffies_since_start = jiffies_since_start; wait_jiffies = next_jiffies_since_start - jiffies_since_start; - while (time_is_after_jiffies(cur_jiffies + wait_jiffies) && - !kthread_should_stop()) - schedule(); + if (!time_is_after_jiffies(cur_jiffies + wait_jiffies)) + continue; + + wait_queue_head_t wait; + + init_waitqueue_head(&wait); + wait_event_interruptible_timeout(wait, kthread_should_stop(), + cur_jiffies + wait_jiffies - jiffies); } dprintk(dev, 1, "Video Capture Thread End\n"); return 0; diff --git a/drivers/media/test-drivers/vivid/vivid-kthread-out.c b/drivers/media/test-drivers/vivid/vivid-kthread-out.c index fac6208b51da8..015a7b166a1e6 100644 --- a/drivers/media/test-drivers/vivid/vivid-kthread-out.c +++ b/drivers/media/test-drivers/vivid/vivid-kthread-out.c @@ -235,9 +235,14 @@ static int vivid_thread_vid_out(void *data) next_jiffies_since_start = jiffies_since_start; wait_jiffies = next_jiffies_since_start - jiffies_since_start; - while (time_is_after_jiffies(cur_jiffies + wait_jiffies) && - !kthread_should_stop()) - schedule(); + if (!time_is_after_jiffies(cur_jiffies + wait_jiffies)) + continue; + + wait_queue_head_t wait; + + init_waitqueue_head(&wait); + wait_event_interruptible_timeout(wait, kthread_should_stop(), + cur_jiffies + wait_jiffies - jiffies); } dprintk(dev, 1, "Video Output Thread End\n"); return 0; diff --git a/drivers/media/test-drivers/vivid/vivid-kthread-touch.c b/drivers/media/test-drivers/vivid/vivid-kthread-touch.c index fa711ee36a3fb..c862689786b69 100644 --- a/drivers/media/test-drivers/vivid/vivid-kthread-touch.c +++ b/drivers/media/test-drivers/vivid/vivid-kthread-touch.c @@ -135,9 +135,14 @@ static int vivid_thread_touch_cap(void *data) next_jiffies_since_start = jiffies_since_start; wait_jiffies = next_jiffies_since_start - jiffies_since_start; - while (time_is_after_jiffies(cur_jiffies + wait_jiffies) && - !kthread_should_stop()) - schedule(); + if (!time_is_after_jiffies(cur_jiffies + wait_jiffies)) + continue; + + wait_queue_head_t wait; + + init_waitqueue_head(&wait); + wait_event_interruptible_timeout(wait, kthread_should_stop(), + cur_jiffies + wait_jiffies - jiffies); } dprintk(dev, 1, "Touch Capture Thread End\n"); return 0; diff --git a/drivers/media/test-drivers/vivid/vivid-sdr-cap.c b/drivers/media/test-drivers/vivid/vivid-sdr-cap.c index a81f26b769883..1dd59c710dae7 100644 --- a/drivers/media/test-drivers/vivid/vivid-sdr-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-sdr-cap.c @@ -206,9 +206,14 @@ static int vivid_thread_sdr_cap(void *data) next_jiffies_since_start = jiffies_since_start; wait_jiffies = next_jiffies_since_start - jiffies_since_start; - while (time_is_after_jiffies(cur_jiffies + wait_jiffies) && - !kthread_should_stop()) - schedule(); + if (!time_is_after_jiffies(cur_jiffies + wait_jiffies)) + continue; + + wait_queue_head_t wait; + + init_waitqueue_head(&wait); + wait_event_interruptible_timeout(wait, kthread_should_stop(), + cur_jiffies + wait_jiffies - jiffies); } dprintk(dev, 1, "SDR Capture Thread End\n"); return 0; diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c index 9443dbb04699a..5d1f78c7604de 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c @@ -460,8 +460,8 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls) if (keep_controls) return; - dims[0] = roundup(dev->src_rect.width, PIXEL_ARRAY_DIV); - dims[1] = roundup(dev->src_rect.height, PIXEL_ARRAY_DIV); + dims[0] = DIV_ROUND_UP(dev->src_rect.height, PIXEL_ARRAY_DIV); + dims[1] = DIV_ROUND_UP(dev->src_rect.width, PIXEL_ARRAY_DIV); v4l2_ctrl_modify_dimensions(dev->pixel_array, dims); } @@ -954,8 +954,8 @@ int vivid_vid_cap_s_selection(struct file *file, void *fh, struct v4l2_selection if (dev->has_compose_cap) { v4l2_rect_set_min_size(compose, &min_rect); v4l2_rect_set_max_size(compose, &max_rect); - v4l2_rect_map_inside(compose, &fmt); } + v4l2_rect_map_inside(compose, &fmt); dev->fmt_cap_rect = fmt; tpg_s_buf_height(&dev->tpg, fmt.height); } else if (dev->has_compose_cap) { diff --git a/drivers/media/tuners/xc4000.c b/drivers/media/tuners/xc4000.c index 29bc63021c5aa..6fb3550811a28 100644 --- a/drivers/media/tuners/xc4000.c +++ b/drivers/media/tuners/xc4000.c @@ -1087,12 +1087,12 @@ static int check_firmware(struct dvb_frontend *fe, unsigned int type, static void xc_debug_dump(struct xc4000_priv *priv) { - u16 adc_envelope; + u16 adc_envelope = 0; u32 freq_error_hz = 0; - u16 lock_status; + u16 lock_status = 0; u32 hsync_freq_hz = 0; - u16 frame_lines; - u16 quality; + u16 frame_lines = 0; + u16 quality = 0; u16 signal = 0; u16 noise = 0; u8 hw_majorversion = 0, hw_minorversion = 0; diff --git a/drivers/media/tuners/xc5000.c b/drivers/media/tuners/xc5000.c index 2182e5b7b6064..a28481edd22ed 100644 --- a/drivers/media/tuners/xc5000.c +++ b/drivers/media/tuners/xc5000.c @@ -58,7 +58,7 @@ struct xc5000_priv { struct dvb_frontend *fe; struct delayed_work timer_sleep; - const struct firmware *firmware; + bool inited; }; /* Misc Defines */ @@ -622,14 +622,14 @@ static int xc5000_fwupload(struct dvb_frontend *fe, static void xc_debug_dump(struct xc5000_priv *priv) { - u16 adc_envelope; + u16 adc_envelope = 0; u32 freq_error_hz = 0; - u16 lock_status; + u16 lock_status = 0; u32 hsync_freq_hz = 0; - u16 frame_lines; - u16 quality; - u16 snr; - u16 totalgain; + u16 frame_lines = 0; + u16 quality = 0; + u16 snr = 0; + u16 totalgain = 0; u8 hw_majorversion = 0, hw_minorversion = 0; u8 fw_majorversion = 0, fw_minorversion = 0; u16 fw_buildversion = 0; @@ -1110,23 +1110,19 @@ static int xc_load_fw_and_init_tuner(struct dvb_frontend *fe, int force) if (!force && xc5000_is_firmware_loaded(fe) == 0) return 0; - if (!priv->firmware) { - ret = request_firmware(&fw, desired_fw->name, - priv->i2c_props.adap->dev.parent); - if (ret) { - pr_err("xc5000: Upload failed. rc %d\n", ret); - return ret; - } - dprintk(1, "firmware read %zu bytes.\n", fw->size); + ret = request_firmware(&fw, desired_fw->name, + priv->i2c_props.adap->dev.parent); + if (ret) { + pr_err("xc5000: Upload failed. rc %d\n", ret); + return ret; + } + dprintk(1, "firmware read %zu bytes.\n", fw->size); - if (fw->size != desired_fw->size) { - pr_err("xc5000: Firmware file with incorrect size\n"); - release_firmware(fw); - return -EINVAL; - } - priv->firmware = fw; - } else - fw = priv->firmware; + if (fw->size != desired_fw->size) { + pr_err("xc5000: Firmware file with incorrect size\n"); + release_firmware(fw); + return -EINVAL; + } /* Try up to 5 times to load firmware */ for (i = 0; i < 5; i++) { @@ -1204,6 +1200,7 @@ static int xc_load_fw_and_init_tuner(struct dvb_frontend *fe, int force) } err: + release_firmware(fw); if (!ret) printk(KERN_INFO "xc5000: Firmware %s loaded and running.\n", desired_fw->name); @@ -1274,7 +1271,7 @@ static int xc5000_resume(struct dvb_frontend *fe) /* suspended before firmware is loaded. Avoid firmware load in resume path. */ - if (!priv->firmware) + if (!priv->inited) return 0; return xc5000_set_params(fe); @@ -1293,6 +1290,8 @@ static int xc5000_init(struct dvb_frontend *fe) if (debug) xc_debug_dump(priv); + priv->inited = true; + return 0; } @@ -1305,11 +1304,7 @@ static void xc5000_release(struct dvb_frontend *fe) mutex_lock(&xc5000_list_mutex); if (priv) { - cancel_delayed_work(&priv->timer_sleep); - if (priv->firmware) { - release_firmware(priv->firmware); - priv->firmware = NULL; - } + cancel_delayed_work_sync(&priv->timer_sleep); hybrid_tuner_release_state(priv); } diff --git a/drivers/media/usb/cx231xx/cx231xx-417.c b/drivers/media/usb/cx231xx/cx231xx-417.c index c5e21785fafe2..02343e88cc618 100644 --- a/drivers/media/usb/cx231xx/cx231xx-417.c +++ b/drivers/media/usb/cx231xx/cx231xx-417.c @@ -1722,6 +1722,8 @@ static void cx231xx_video_dev_init( vfd->lock = &dev->lock; vfd->release = video_device_release_empty; vfd->ctrl_handler = &dev->mpeg_ctrl_handler.hdl; + vfd->device_caps = V4L2_CAP_READWRITE | V4L2_CAP_STREAMING | + V4L2_CAP_VIDEO_CAPTURE; video_set_drvdata(vfd, dev); if (dev->tuner_type == TUNER_ABSENT) { v4l2_disable_ioctl(vfd, VIDIOC_G_FREQUENCY); diff --git a/drivers/media/usb/dvb-usb/cxusb.c b/drivers/media/usb/dvb-usb/cxusb.c index 1d98d3465e28d..ce52c936cb931 100644 --- a/drivers/media/usb/dvb-usb/cxusb.c +++ b/drivers/media/usb/dvb-usb/cxusb.c @@ -119,9 +119,8 @@ static void cxusb_gpio_tuner(struct dvb_usb_device *d, int onoff) o[0] = GPIO_TUNER; o[1] = onoff; - cxusb_ctrl_msg(d, CMD_GPIO_WRITE, o, 2, &i, 1); - if (i != 0x01) + if (!cxusb_ctrl_msg(d, CMD_GPIO_WRITE, o, 2, &i, 1) && i != 0x01) dev_info(&d->udev->dev, "gpio_write failed.\n"); st->gpio_write_state[GPIO_TUNER] = onoff; diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c b/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c index 5a47dcbf1c8e5..303b055fefea9 100644 --- a/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c +++ b/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c @@ -520,12 +520,13 @@ static int hdcs_init(struct sd *sd) static int hdcs_dump(struct sd *sd) { u16 reg, val; + int err = 0; pr_info("Dumping sensor registers:\n"); - for (reg = HDCS_IDENT; reg <= HDCS_ROWEXPH; reg++) { - stv06xx_read_sensor(sd, reg, &val); + for (reg = HDCS_IDENT; reg <= HDCS_ROWEXPH && !err; reg++) { + err = stv06xx_read_sensor(sd, reg, &val); pr_info("reg 0x%02x = 0x%02x\n", reg, val); } - return 0; + return (err < 0) ? err : 0; } diff --git a/drivers/media/usb/gspca/vicam.c b/drivers/media/usb/gspca/vicam.c index d98343fd33fe3..91e177aa8136f 100644 --- a/drivers/media/usb/gspca/vicam.c +++ b/drivers/media/usb/gspca/vicam.c @@ -227,6 +227,7 @@ static int sd_init(struct gspca_dev *gspca_dev) const struct ihex_binrec *rec; const struct firmware *fw; u8 *firmware_buf; + int len; ret = request_ihex_firmware(&fw, VICAM_FIRMWARE, &gspca_dev->dev->dev); @@ -241,9 +242,14 @@ static int sd_init(struct gspca_dev *gspca_dev) goto exit; } for (rec = (void *)fw->data; rec; rec = ihex_next_binrec(rec)) { - memcpy(firmware_buf, rec->data, be16_to_cpu(rec->len)); + len = be16_to_cpu(rec->len); + if (len > PAGE_SIZE) { + ret = -EINVAL; + break; + } + memcpy(firmware_buf, rec->data, len); ret = vicam_control_msg(gspca_dev, 0xff, 0, 0, firmware_buf, - be16_to_cpu(rec->len)); + len); if (ret < 0) break; } diff --git a/drivers/media/usb/hdpvr/hdpvr-i2c.c b/drivers/media/usb/hdpvr/hdpvr-i2c.c index 070559b01b01b..54956a8ff15e8 100644 --- a/drivers/media/usb/hdpvr/hdpvr-i2c.c +++ b/drivers/media/usb/hdpvr/hdpvr-i2c.c @@ -165,10 +165,16 @@ static const struct i2c_algorithm hdpvr_algo = { .functionality = hdpvr_functionality, }; +/* prevent invalid 0-length usb_control_msg */ +static const struct i2c_adapter_quirks hdpvr_quirks = { + .flags = I2C_AQ_NO_ZERO_LEN_READ, +}; + static const struct i2c_adapter hdpvr_i2c_adapter_template = { .name = "Hauppauge HD PVR I2C", .owner = THIS_MODULE, .algo = &hdpvr_algo, + .quirks = &hdpvr_quirks, }; static int hdpvr_activate_ir(struct hdpvr_device *dev) diff --git a/drivers/media/usb/usbtv/usbtv-video.c b/drivers/media/usb/usbtv/usbtv-video.c index 7495df6b51912..f3633448e8b96 100644 --- a/drivers/media/usb/usbtv/usbtv-video.c +++ b/drivers/media/usb/usbtv/usbtv-video.c @@ -73,6 +73,10 @@ static int usbtv_configure_for_norm(struct usbtv *usbtv, v4l2_std_id norm) } if (params) { + if (vb2_is_busy(&usbtv->vb2q) && + (usbtv->width != params->cap_width || + usbtv->height != params->cap_height)) + return -EBUSY; usbtv->width = params->cap_width; usbtv->height = params->cap_height; usbtv->n_chunks = usbtv->width * usbtv->height diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c index 028c4a5049af9..bd90d8bacd5ef 100644 --- a/drivers/media/usb/uvc/uvc_ctrl.c +++ b/drivers/media/usb/uvc/uvc_ctrl.c @@ -815,6 +815,25 @@ static inline void uvc_clear_bit(u8 *data, int bit) data[bit >> 3] &= ~(1 << (bit & 7)); } +static s32 uvc_menu_to_v4l2_menu(struct uvc_control_mapping *mapping, s32 val) +{ + unsigned int i; + + for (i = 0; BIT(i) <= mapping->menu_mask; ++i) { + u32 menu_value; + + if (!test_bit(i, &mapping->menu_mask)) + continue; + + menu_value = uvc_mapping_get_menu_value(mapping, i); + + if (menu_value == val) + return i; + } + + return val; +} + /* * Extract the bit string specified by mapping->offset and mapping->size * from the little-endian data stored at 'data' and return the result as @@ -849,6 +868,16 @@ static s32 uvc_get_le_value(struct uvc_control_mapping *mapping, if (mapping->data_type == UVC_CTRL_DATA_TYPE_SIGNED) value |= -(value & (1 << (mapping->size - 1))); + /* If it is a menu, convert from uvc to v4l2. */ + if (mapping->v4l2_type != V4L2_CTRL_TYPE_MENU) + return value; + + switch (query) { + case UVC_GET_CUR: + case UVC_GET_DEF: + return uvc_menu_to_v4l2_menu(mapping, value); + } + return value; } @@ -1013,32 +1042,6 @@ static int uvc_ctrl_populate_cache(struct uvc_video_chain *chain, return 0; } -static s32 __uvc_ctrl_get_value(struct uvc_control_mapping *mapping, - const u8 *data) -{ - s32 value = mapping->get(mapping, UVC_GET_CUR, data); - - if (mapping->v4l2_type == V4L2_CTRL_TYPE_MENU) { - unsigned int i; - - for (i = 0; BIT(i) <= mapping->menu_mask; ++i) { - u32 menu_value; - - if (!test_bit(i, &mapping->menu_mask)) - continue; - - menu_value = uvc_mapping_get_menu_value(mapping, i); - - if (menu_value == value) { - value = i; - break; - } - } - } - - return value; -} - static int __uvc_ctrl_load_cur(struct uvc_video_chain *chain, struct uvc_control *ctrl) { @@ -1089,8 +1092,8 @@ static int __uvc_ctrl_get(struct uvc_video_chain *chain, if (ret < 0) return ret; - *value = __uvc_ctrl_get_value(mapping, - uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT)); + *value = mapping->get(mapping, UVC_GET_CUR, + uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT)); return 0; } @@ -1240,7 +1243,6 @@ static int __uvc_query_v4l2_ctrl(struct uvc_video_chain *chain, { struct uvc_control_mapping *master_map = NULL; struct uvc_control *master_ctrl = NULL; - unsigned int i; memset(v4l2_ctrl, 0, sizeof(*v4l2_ctrl)); v4l2_ctrl->id = mapping->id; @@ -1283,21 +1285,6 @@ static int __uvc_query_v4l2_ctrl(struct uvc_video_chain *chain, v4l2_ctrl->minimum = ffs(mapping->menu_mask) - 1; v4l2_ctrl->maximum = fls(mapping->menu_mask) - 1; v4l2_ctrl->step = 1; - - for (i = 0; BIT(i) <= mapping->menu_mask; ++i) { - u32 menu_value; - - if (!test_bit(i, &mapping->menu_mask)) - continue; - - menu_value = uvc_mapping_get_menu_value(mapping, i); - - if (menu_value == v4l2_ctrl->default_value) { - v4l2_ctrl->default_value = i; - break; - } - } - return 0; case V4L2_CTRL_TYPE_BOOLEAN: @@ -1580,7 +1567,7 @@ void uvc_ctrl_status_event(struct uvc_video_chain *chain, uvc_ctrl_set_handle(handle, ctrl, NULL); list_for_each_entry(mapping, &ctrl->info.mappings, list) { - s32 value = __uvc_ctrl_get_value(mapping, data); + s32 value = mapping->get(mapping, UVC_GET_CUR, data); /* * handle may be NULL here if the device sends auto-update @@ -1655,7 +1642,9 @@ static bool uvc_ctrl_xctrls_has_control(const struct v4l2_ext_control *xctrls, } static void uvc_ctrl_send_events(struct uvc_fh *handle, - const struct v4l2_ext_control *xctrls, unsigned int xctrls_count) + struct uvc_entity *entity, + const struct v4l2_ext_control *xctrls, + unsigned int xctrls_count) { struct uvc_control_mapping *mapping; struct uvc_control *ctrl; @@ -1666,6 +1655,9 @@ static void uvc_ctrl_send_events(struct uvc_fh *handle, u32 changes = V4L2_EVENT_CTRL_CH_VALUE; ctrl = uvc_find_control(handle->chain, xctrls[i].id, &mapping); + if (ctrl->entity != entity) + continue; + if (ctrl->info.flags & UVC_CTRL_FLAG_ASYNCHRONOUS) /* Notification will be sent from an Interrupt event. */ continue; @@ -1796,15 +1788,20 @@ int uvc_ctrl_begin(struct uvc_video_chain *chain) return mutex_lock_interruptible(&chain->ctrl_mutex) ? -ERESTARTSYS : 0; } +/* + * Returns the number of uvc controls that have been correctly set, or a + * negative number if there has been an error. + */ static int uvc_ctrl_commit_entity(struct uvc_device *dev, struct uvc_fh *handle, struct uvc_entity *entity, int rollback, struct uvc_control **err_ctrl) { + unsigned int processed_ctrls = 0; struct uvc_control *ctrl; unsigned int i; - int ret; + int ret = 0; if (entity == NULL) return 0; @@ -1833,8 +1830,9 @@ static int uvc_ctrl_commit_entity(struct uvc_device *dev, dev->intfnum, ctrl->info.selector, uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT), ctrl->info.size); - else - ret = 0; + + if (!ret) + processed_ctrls++; if (rollback || ret < 0) memcpy(uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT), @@ -1843,18 +1841,26 @@ static int uvc_ctrl_commit_entity(struct uvc_device *dev, ctrl->dirty = 0; - if (ret < 0) { + if (!rollback && handle && + ctrl->info.flags & UVC_CTRL_FLAG_ASYNCHRONOUS) + uvc_ctrl_set_handle(handle, ctrl, handle); + + if (ret < 0 && !rollback) { if (err_ctrl) *err_ctrl = ctrl; - return ret; + /* + * If we fail to set a control, we need to rollback + * the next ones. + */ + rollback = 1; } - if (!rollback && handle && - ctrl->info.flags & UVC_CTRL_FLAG_ASYNCHRONOUS) - uvc_ctrl_set_handle(handle, ctrl, handle); } - return 0; + if (ret) + return ret; + + return processed_ctrls; } static int uvc_ctrl_find_ctrl_idx(struct uvc_entity *entity, @@ -1884,7 +1890,8 @@ int __uvc_ctrl_commit(struct uvc_fh *handle, int rollback, struct uvc_video_chain *chain = handle->chain; struct uvc_control *err_ctrl; struct uvc_entity *entity; - int ret = 0; + int ret_out = 0; + int ret; /* Find the control. */ list_for_each_entry(entity, &chain->entities, chain) { @@ -1895,15 +1902,23 @@ int __uvc_ctrl_commit(struct uvc_fh *handle, int rollback, ctrls->error_idx = uvc_ctrl_find_ctrl_idx(entity, ctrls, err_ctrl); - goto done; + /* + * When we fail to commit an entity, we need to + * restore the UVC_CTRL_DATA_BACKUP for all the + * controls in the other entities, otherwise our cache + * and the hardware will be out of sync. + */ + rollback = 1; + + ret_out = ret; + } else if (ret > 0 && !rollback) { + uvc_ctrl_send_events(handle, entity, + ctrls->controls, ctrls->count); } } - if (!rollback) - uvc_ctrl_send_events(handle, ctrls->controls, ctrls->count); -done: mutex_unlock(&chain->ctrl_mutex); - return ret; + return ret_out; } int uvc_ctrl_get(struct uvc_video_chain *chain, diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c index 02cfa12b9cb90..a4b9a7ca4e50f 100644 --- a/drivers/media/usb/uvc/uvc_driver.c +++ b/drivers/media/usb/uvc/uvc_driver.c @@ -135,6 +135,9 @@ struct uvc_entity *uvc_entity_by_id(struct uvc_device *dev, int id) { struct uvc_entity *entity; + if (id == UVC_INVALID_ENTITY_ID) + return NULL; + list_for_each_entry(entity, &dev->entities, list) { if (entity->id == id) return entity; @@ -162,13 +165,26 @@ static struct uvc_entity *uvc_entity_by_reference(struct uvc_device *dev, static struct uvc_streaming *uvc_stream_by_id(struct uvc_device *dev, int id) { - struct uvc_streaming *stream; + struct uvc_streaming *stream, *last_stream; + unsigned int count = 0; list_for_each_entry(stream, &dev->streams, list) { + count += 1; + last_stream = stream; if (stream->header.bTerminalLink == id) return stream; } + /* + * If the streaming entity is referenced by an invalid ID, notify the + * user and use heuristics to guess the correct entity. + */ + if (count == 1 && id == UVC_INVALID_ENTITY_ID) { + dev_warn(&dev->intf->dev, + "UVC non compliance: Invalid USB header. The streaming entity has an invalid ID, guessing the correct one."); + return last_stream; + } + return NULL; } @@ -235,6 +251,9 @@ static int uvc_parse_format(struct uvc_device *dev, unsigned int i, n; u8 ftype; + if (buflen < 4) + return -EINVAL; + format->type = buffer[2]; format->index = buffer[3]; format->frames = frames; @@ -775,14 +794,27 @@ static const u8 uvc_media_transport_input_guid[16] = UVC_GUID_UVC_MEDIA_TRANSPORT_INPUT; static const u8 uvc_processing_guid[16] = UVC_GUID_UVC_PROCESSING; -static struct uvc_entity *uvc_alloc_entity(u16 type, u16 id, - unsigned int num_pads, unsigned int extra_size) +static struct uvc_entity *uvc_alloc_new_entity(struct uvc_device *dev, u16 type, + u16 id, unsigned int num_pads, + unsigned int extra_size) { struct uvc_entity *entity; unsigned int num_inputs; unsigned int size; unsigned int i; + /* Per UVC 1.1+ spec 3.7.2, the ID should be non-zero. */ + if (id == 0) { + dev_err(&dev->intf->dev, "Found Unit with invalid ID 0\n"); + id = UVC_INVALID_ENTITY_ID; + } + + /* Per UVC 1.1+ spec 3.7.2, the ID is unique. */ + if (uvc_entity_by_id(dev, id)) { + dev_err(&dev->intf->dev, "Found multiple Units with ID %u\n", id); + id = UVC_INVALID_ENTITY_ID; + } + extra_size = roundup(extra_size, sizeof(*entity->pads)); if (num_pads) num_inputs = type & UVC_TERM_OUTPUT ? num_pads : num_pads - 1; @@ -792,7 +824,7 @@ static struct uvc_entity *uvc_alloc_entity(u16 type, u16 id, + num_inputs; entity = kzalloc(size, GFP_KERNEL); if (entity == NULL) - return NULL; + return ERR_PTR(-ENOMEM); entity->id = id; entity->type = type; @@ -904,10 +936,10 @@ static int uvc_parse_vendor_control(struct uvc_device *dev, break; } - unit = uvc_alloc_entity(UVC_VC_EXTENSION_UNIT, buffer[3], - p + 1, 2*n); - if (unit == NULL) - return -ENOMEM; + unit = uvc_alloc_new_entity(dev, UVC_VC_EXTENSION_UNIT, + buffer[3], p + 1, 2 * n); + if (IS_ERR(unit)) + return PTR_ERR(unit); memcpy(unit->guid, &buffer[4], 16); unit->extension.bNumControls = buffer[20]; @@ -1016,10 +1048,10 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return -EINVAL; } - term = uvc_alloc_entity(type | UVC_TERM_INPUT, buffer[3], - 1, n + p); - if (term == NULL) - return -ENOMEM; + term = uvc_alloc_new_entity(dev, type | UVC_TERM_INPUT, + buffer[3], 1, n + p); + if (IS_ERR(term)) + return PTR_ERR(term); if (UVC_ENTITY_TYPE(term) == UVC_ITT_CAMERA) { term->camera.bControlSize = n; @@ -1075,10 +1107,10 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return 0; } - term = uvc_alloc_entity(type | UVC_TERM_OUTPUT, buffer[3], - 1, 0); - if (term == NULL) - return -ENOMEM; + term = uvc_alloc_new_entity(dev, type | UVC_TERM_OUTPUT, + buffer[3], 1, 0); + if (IS_ERR(term)) + return PTR_ERR(term); memcpy(term->baSourceID, &buffer[7], 1); @@ -1097,9 +1129,10 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return -EINVAL; } - unit = uvc_alloc_entity(buffer[2], buffer[3], p + 1, 0); - if (unit == NULL) - return -ENOMEM; + unit = uvc_alloc_new_entity(dev, buffer[2], buffer[3], + p + 1, 0); + if (IS_ERR(unit)) + return PTR_ERR(unit); memcpy(unit->baSourceID, &buffer[5], p); @@ -1119,9 +1152,9 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return -EINVAL; } - unit = uvc_alloc_entity(buffer[2], buffer[3], 2, n); - if (unit == NULL) - return -ENOMEM; + unit = uvc_alloc_new_entity(dev, buffer[2], buffer[3], 2, n); + if (IS_ERR(unit)) + return PTR_ERR(unit); memcpy(unit->baSourceID, &buffer[4], 1); unit->processing.wMaxMultiplier = @@ -1148,9 +1181,10 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return -EINVAL; } - unit = uvc_alloc_entity(buffer[2], buffer[3], p + 1, n); - if (unit == NULL) - return -ENOMEM; + unit = uvc_alloc_new_entity(dev, buffer[2], buffer[3], + p + 1, n); + if (IS_ERR(unit)) + return PTR_ERR(unit); memcpy(unit->guid, &buffer[4], 16); unit->extension.bNumControls = buffer[20]; @@ -1290,9 +1324,10 @@ static int uvc_gpio_parse(struct uvc_device *dev) return dev_err_probe(&dev->intf->dev, irq, "No IRQ for privacy GPIO\n"); - unit = uvc_alloc_entity(UVC_EXT_GPIO_UNIT, UVC_EXT_GPIO_UNIT_ID, 0, 1); - if (!unit) - return -ENOMEM; + unit = uvc_alloc_new_entity(dev, UVC_EXT_GPIO_UNIT, + UVC_EXT_GPIO_UNIT_ID, 0, 1); + if (IS_ERR(unit)) + return PTR_ERR(unit); unit->gpio.gpio_privacy = gpio_privacy; unit->gpio.irq = irq; @@ -2217,13 +2252,16 @@ static int uvc_probe(struct usb_interface *intf, #endif /* Parse the Video Class control descriptor. */ - if (uvc_parse_control(dev) < 0) { + ret = uvc_parse_control(dev); + if (ret < 0) { + ret = -ENODEV; uvc_dbg(dev, PROBE, "Unable to parse UVC descriptors\n"); goto error; } /* Parse the associated GPIOs. */ - if (uvc_gpio_parse(dev) < 0) { + ret = uvc_gpio_parse(dev); + if (ret < 0) { uvc_dbg(dev, PROBE, "Unable to parse UVC GPIOs\n"); goto error; } @@ -2249,24 +2287,32 @@ static int uvc_probe(struct usb_interface *intf, } /* Register the V4L2 device. */ - if (v4l2_device_register(&intf->dev, &dev->vdev) < 0) + ret = v4l2_device_register(&intf->dev, &dev->vdev); + if (ret < 0) goto error; /* Scan the device for video chains. */ - if (uvc_scan_device(dev) < 0) + if (uvc_scan_device(dev) < 0) { + ret = -ENODEV; goto error; + } /* Initialize controls. */ - if (uvc_ctrl_init_device(dev) < 0) + if (uvc_ctrl_init_device(dev) < 0) { + ret = -ENODEV; goto error; + } /* Register video device nodes. */ - if (uvc_register_chains(dev) < 0) + if (uvc_register_chains(dev) < 0) { + ret = -ENODEV; goto error; + } #ifdef CONFIG_MEDIA_CONTROLLER /* Register the media device node */ - if (media_device_register(&dev->mdev) < 0) + ret = media_device_register(&dev->mdev); + if (ret < 0) goto error; #endif /* Save our data pointer in the interface data. */ @@ -2300,7 +2346,7 @@ static int uvc_probe(struct usb_interface *intf, error: uvc_unregister_video(dev); kref_put(&dev->ref, uvc_delete); - return -ENODEV; + return ret; } static void uvc_disconnect(struct usb_interface *intf) diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c index 7bcd706281daf..cb7d9fb589fca 100644 --- a/drivers/media/usb/uvc/uvc_v4l2.c +++ b/drivers/media/usb/uvc/uvc_v4l2.c @@ -106,6 +106,12 @@ static int uvc_ioctl_xu_ctrl_map(struct uvc_video_chain *chain, struct uvc_control_mapping *map; int ret; + if (xmap->data_type > UVC_CTRL_DATA_TYPE_BITMASK) { + uvc_dbg(chain->dev, CONTROL, + "Unsupported UVC data type %u\n", xmap->data_type); + return -EINVAL; + } + map = kzalloc(sizeof(*map), GFP_KERNEL); if (map == NULL) return -ENOMEM; diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 9572fdfe74f24..a9f880eb518ad 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -258,6 +258,15 @@ static void uvc_fixup_video_ctrl(struct uvc_streaming *stream, ctrl->dwMaxPayloadTransferSize = bandwidth; } + + if (stream->intf->num_altsetting > 1 && + ctrl->dwMaxPayloadTransferSize > stream->maxpsize) { + dev_warn_ratelimited(&stream->intf->dev, + "UVC non compliance: the max payload transmission size (%u) exceeds the size of the ep max packet (%u). Using the max size.\n", + ctrl->dwMaxPayloadTransferSize, + stream->maxpsize); + ctrl->dwMaxPayloadTransferSize = stream->maxpsize; + } } static size_t uvc_video_ctrl_size(struct uvc_streaming *stream) @@ -1405,12 +1414,6 @@ static void uvc_video_decode_meta(struct uvc_streaming *stream, if (!meta_buf || length == 2) return; - if (meta_buf->length - meta_buf->bytesused < - length + sizeof(meta->ns) + sizeof(meta->sof)) { - meta_buf->error = 1; - return; - } - has_pts = mem[1] & UVC_STREAM_PTS; has_scr = mem[1] & UVC_STREAM_SCR; @@ -1431,6 +1434,12 @@ static void uvc_video_decode_meta(struct uvc_streaming *stream, !memcmp(scr, stream->clock.last_scr, 6))) return; + if (meta_buf->length - meta_buf->bytesused < + length + sizeof(meta->ns) + sizeof(meta->sof)) { + meta_buf->error = 1; + return; + } + meta = (struct uvc_meta_buf *)((u8 *)meta_buf->mem + meta_buf->bytesused); local_irq_save(flags); time = uvc_video_get_time(); diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h index e99bfaa622669..cbb5ce963cd09 100644 --- a/drivers/media/usb/uvc/uvcvideo.h +++ b/drivers/media/usb/uvc/uvcvideo.h @@ -41,6 +41,8 @@ #define UVC_EXT_GPIO_UNIT 0x7ffe #define UVC_EXT_GPIO_UNIT_ID 0x100 +#define UVC_INVALID_ENTITY_ID 0xffff + /* ------------------------------------------------------------------------ * Driver specific constants. */ diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c index 3a4b15a98e021..b37507e093399 100644 --- a/drivers/media/v4l2-core/v4l2-common.c +++ b/drivers/media/v4l2-core/v4l2-common.c @@ -487,10 +487,10 @@ s64 v4l2_get_link_freq(struct v4l2_ctrl_handler *handler, unsigned int mul, freq = div_u64(v4l2_ctrl_g_ctrl_int64(ctrl) * mul, div); - pr_warn("%s: Link frequency estimated using pixel rate: result might be inaccurate\n", - __func__); - pr_warn("%s: Consider implementing support for V4L2_CID_LINK_FREQ in the transmitter driver\n", - __func__); + pr_warn_once("%s: Link frequency estimated using pixel rate: result might be inaccurate\n", + __func__); + pr_warn_once("%s: Consider implementing support for V4L2_CID_LINK_FREQ in the transmitter driver\n", + __func__); } return freq > 0 ? freq : -EINVAL; diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c index a662fb60f73f4..a1d3e93a40956 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls-core.c +++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c @@ -894,12 +894,12 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, p_h264_sps->flags &= ~V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; - - if (p_h264_sps->chroma_format_idc < 3) - p_h264_sps->flags &= - ~V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; } + if (p_h264_sps->chroma_format_idc < 3) + p_h264_sps->flags &= + ~V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; + if (p_h264_sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY) p_h264_sps->flags &= ~V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; @@ -1578,7 +1578,6 @@ void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl) kvfree(hdl->buckets); hdl->buckets = NULL; hdl->cached = NULL; - hdl->error = 0; mutex_unlock(hdl->lock); mutex_destroy(&hdl->_lock); } diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c index 77bbf276ae89d..c4d3d110c5007 100644 --- a/drivers/media/v4l2-core/v4l2-dev.c +++ b/drivers/media/v4l2-core/v4l2-dev.c @@ -1033,25 +1033,25 @@ int __video_register_device(struct video_device *vdev, vdev->dev.class = &video_class; vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor); vdev->dev.parent = vdev->dev_parent; + vdev->dev.release = v4l2_device_release; dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num); + + /* Increase v4l2_device refcount */ + v4l2_device_get(vdev->v4l2_dev); + mutex_lock(&videodev_lock); ret = device_register(&vdev->dev); if (ret < 0) { mutex_unlock(&videodev_lock); pr_err("%s: device_register failed\n", __func__); - goto cleanup; + put_device(&vdev->dev); + return ret; } - /* Register the release callback that will be called when the last - reference to the device goes away. */ - vdev->dev.release = v4l2_device_release; if (nr != -1 && nr != vdev->num && warn_if_nr_in_use) pr_warn("%s: requested %s%d, got %s\n", __func__, name_base, nr, video_device_node_name(vdev)); - /* Increase v4l2_device refcount */ - v4l2_device_get(vdev->v4l2_dev); - /* Part 5: Register the entity. */ ret = video_register_media_controller(vdev); diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c index a32ef739eb449..097413bc6c8ea 100644 --- a/drivers/media/v4l2-core/v4l2-subdev.c +++ b/drivers/media/v4l2-core/v4l2-subdev.c @@ -351,6 +351,8 @@ static int call_enum_dv_timings(struct v4l2_subdev *sd, static int call_get_mbus_config(struct v4l2_subdev *sd, unsigned int pad, struct v4l2_mbus_config *config) { + memset(config, 0, sizeof(*config)); + return check_pad(sd, pad) ? : sd->ops->pad->get_mbus_config(sd, pad, config); } diff --git a/drivers/memory/samsung/exynos-srom.c b/drivers/memory/samsung/exynos-srom.c index e73dd330af477..d913fb901973f 100644 --- a/drivers/memory/samsung/exynos-srom.c +++ b/drivers/memory/samsung/exynos-srom.c @@ -121,20 +121,18 @@ static int exynos_srom_probe(struct platform_device *pdev) return -ENOMEM; srom->dev = dev; - srom->reg_base = of_iomap(np, 0); - if (!srom->reg_base) { + srom->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(srom->reg_base)) { dev_err(&pdev->dev, "iomap of exynos srom controller failed\n"); - return -ENOMEM; + return PTR_ERR(srom->reg_base); } platform_set_drvdata(pdev, srom); srom->reg_offset = exynos_srom_alloc_reg_dump(exynos_srom_offsets, ARRAY_SIZE(exynos_srom_offsets)); - if (!srom->reg_offset) { - iounmap(srom->reg_base); + if (!srom->reg_offset) return -ENOMEM; - } for_each_child_of_node(np, child) { if (exynos_srom_configure_bank(srom, child)) { diff --git a/drivers/memstick/core/memstick.c b/drivers/memstick/core/memstick.c index bbfaf6536903d..65af43201b693 100644 --- a/drivers/memstick/core/memstick.c +++ b/drivers/memstick/core/memstick.c @@ -323,7 +323,7 @@ EXPORT_SYMBOL(memstick_init_req); static int h_memstick_read_dev_id(struct memstick_dev *card, struct memstick_request **mrq) { - struct ms_id_register id_reg; + struct ms_id_register id_reg = {}; if (!(*mrq)) { memstick_init_req(&card->current_mrq, MS_TPC_READ_REG, &id_reg, @@ -367,7 +367,9 @@ int memstick_set_rw_addr(struct memstick_dev *card) { card->next_request = h_memstick_set_rw_addr; memstick_new_req(card->host); - wait_for_completion(&card->mrq_complete); + if (!wait_for_completion_timeout(&card->mrq_complete, + msecs_to_jiffies(500))) + card->current_mrq.error = -ETIMEDOUT; return card->current_mrq.error; } @@ -401,7 +403,9 @@ static struct memstick_dev *memstick_alloc_card(struct memstick_host *host) card->next_request = h_memstick_read_dev_id; memstick_new_req(host); - wait_for_completion(&card->mrq_complete); + if (!wait_for_completion_timeout(&card->mrq_complete, + msecs_to_jiffies(500))) + card->current_mrq.error = -ETIMEDOUT; if (card->current_mrq.error) goto err_out; @@ -548,7 +552,6 @@ EXPORT_SYMBOL(memstick_add_host); */ void memstick_remove_host(struct memstick_host *host) { - host->removing = 1; flush_workqueue(workqueue); mutex_lock(&host->lock); if (host->card) diff --git a/drivers/memstick/host/rtsx_usb_ms.c b/drivers/memstick/host/rtsx_usb_ms.c index dec279845a752..43ec4948daa20 100644 --- a/drivers/memstick/host/rtsx_usb_ms.c +++ b/drivers/memstick/host/rtsx_usb_ms.c @@ -812,6 +812,7 @@ static int rtsx_usb_ms_drv_remove(struct platform_device *pdev) int err; host->eject = true; + msh->removing = true; cancel_work_sync(&host->handle_req); cancel_delayed_work_sync(&host->poll_card); diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c index 87603eeaa2770..2b85da0fcf27e 100644 --- a/drivers/mfd/axp20x.c +++ b/drivers/mfd/axp20x.c @@ -936,7 +936,8 @@ static const struct mfd_cell axp152_cells[] = { }; static struct mfd_cell axp313a_cells[] = { - MFD_CELL_NAME("axp20x-regulator"), + /* AXP323 is sometimes paired with AXP717 as sub-PMIC */ + MFD_CELL_BASIC("axp20x-regulator", NULL, NULL, 0, 1), MFD_CELL_RES("axp313a-pek", axp313a_pek_resources), }; diff --git a/drivers/mfd/da9063-i2c.c b/drivers/mfd/da9063-i2c.c index d715cf9a9e688..7b829bfe04bc7 100644 --- a/drivers/mfd/da9063-i2c.c +++ b/drivers/mfd/da9063-i2c.c @@ -37,9 +37,13 @@ enum da9063_page_sel_buf_fmt { DA9063_PAGE_SEL_BUF_SIZE, }; +enum da9063_page_sel_msgs { + DA9063_PAGE_SEL_MSG = 0, + DA9063_PAGE_SEL_CNT, +}; + enum da9063_paged_read_msgs { - DA9063_PAGED_READ_MSG_PAGE_SEL = 0, - DA9063_PAGED_READ_MSG_REG_SEL, + DA9063_PAGED_READ_MSG_REG_SEL = 0, DA9063_PAGED_READ_MSG_DATA, DA9063_PAGED_READ_MSG_CNT, }; @@ -65,10 +69,21 @@ static int da9063_i2c_blockreg_read(struct i2c_client *client, u16 addr, (page_num << DA9063_I2C_PAGE_SEL_SHIFT) & DA9063_REG_PAGE_MASK; /* Write reg address, page selection */ - xfer[DA9063_PAGED_READ_MSG_PAGE_SEL].addr = client->addr; - xfer[DA9063_PAGED_READ_MSG_PAGE_SEL].flags = 0; - xfer[DA9063_PAGED_READ_MSG_PAGE_SEL].len = DA9063_PAGE_SEL_BUF_SIZE; - xfer[DA9063_PAGED_READ_MSG_PAGE_SEL].buf = page_sel_buf; + xfer[DA9063_PAGE_SEL_MSG].addr = client->addr; + xfer[DA9063_PAGE_SEL_MSG].flags = 0; + xfer[DA9063_PAGE_SEL_MSG].len = DA9063_PAGE_SEL_BUF_SIZE; + xfer[DA9063_PAGE_SEL_MSG].buf = page_sel_buf; + + ret = i2c_transfer(client->adapter, xfer, DA9063_PAGE_SEL_CNT); + if (ret < 0) { + dev_err(&client->dev, "Page switch failed: %d\n", ret); + return ret; + } + + if (ret != DA9063_PAGE_SEL_CNT) { + dev_err(&client->dev, "Page switch failed to complete\n"); + return -EIO; + } /* Select register address */ xfer[DA9063_PAGED_READ_MSG_REG_SEL].addr = client->addr; diff --git a/drivers/mfd/exynos-lpass.c b/drivers/mfd/exynos-lpass.c index 1506d8d352b19..5e39d91b728fc 100644 --- a/drivers/mfd/exynos-lpass.c +++ b/drivers/mfd/exynos-lpass.c @@ -141,7 +141,6 @@ static int exynos_lpass_remove(struct platform_device *pdev) { struct exynos_lpass *lpass = platform_get_drvdata(pdev); - exynos_lpass_disable(lpass); pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) exynos_lpass_disable(lpass); diff --git a/drivers/mfd/intel_soc_pmic_chtdc_ti.c b/drivers/mfd/intel_soc_pmic_chtdc_ti.c index 992855bfda3e4..6daf33e07ea0a 100644 --- a/drivers/mfd/intel_soc_pmic_chtdc_ti.c +++ b/drivers/mfd/intel_soc_pmic_chtdc_ti.c @@ -81,8 +81,9 @@ static struct mfd_cell chtdc_ti_dev[] = { static const struct regmap_config chtdc_ti_regmap_config = { .reg_bits = 8, .val_bits = 8, - .max_register = 128, - .cache_type = REGCACHE_NONE, + .max_register = 0xff, + /* The hardware does not support reading multiple registers at once */ + .use_single_read = true, }; static const struct regmap_irq chtdc_ti_irqs[] = { diff --git a/drivers/mfd/madera-core.c b/drivers/mfd/madera-core.c index bdbd5bfc97145..2f74a8c644a32 100644 --- a/drivers/mfd/madera-core.c +++ b/drivers/mfd/madera-core.c @@ -456,7 +456,7 @@ int madera_dev_init(struct madera *madera) struct device *dev = madera->dev; unsigned int hwid; int (*patch_fn)(struct madera *) = NULL; - const struct mfd_cell *mfd_devs; + const struct mfd_cell *mfd_devs = NULL; int n_devs = 0; int i, ret; @@ -670,7 +670,7 @@ int madera_dev_init(struct madera *madera) goto err_reset; } - if (!n_devs) { + if (!n_devs || !mfd_devs) { dev_err(madera->dev, "Device ID 0x%x not a %s\n", hwid, madera->type_name); ret = -ENODEV; diff --git a/drivers/mfd/max14577.c b/drivers/mfd/max14577.c index 1f4f5002595c0..17672eeb1732a 100644 --- a/drivers/mfd/max14577.c +++ b/drivers/mfd/max14577.c @@ -463,6 +463,7 @@ static void max14577_i2c_remove(struct i2c_client *i2c) { struct max14577 *max14577 = i2c_get_clientdata(i2c); + device_init_wakeup(max14577->dev, false); mfd_remove_devices(max14577->dev); regmap_del_irq_chip(max14577->irq, max14577->irq_data); if (max14577->dev_type == MAXIM_DEVICE_TYPE_MAX77836) diff --git a/drivers/mfd/rz-mtu3.c b/drivers/mfd/rz-mtu3.c index f3dac4a29a832..9cdfef610398f 100644 --- a/drivers/mfd/rz-mtu3.c +++ b/drivers/mfd/rz-mtu3.c @@ -32,7 +32,7 @@ static const unsigned long rz_mtu3_8bit_ch_reg_offs[][13] = { [RZ_MTU3_CHAN_2] = MTU_8BIT_CH_1_2(0x204, 0x092, 0x205, 0x200, 0x20c, 0x201, 0x202), [RZ_MTU3_CHAN_3] = MTU_8BIT_CH_3_4_6_7(0x008, 0x093, 0x02c, 0x000, 0x04c, 0x002, 0x004, 0x005, 0x038), [RZ_MTU3_CHAN_4] = MTU_8BIT_CH_3_4_6_7(0x009, 0x094, 0x02d, 0x001, 0x04d, 0x003, 0x006, 0x007, 0x039), - [RZ_MTU3_CHAN_5] = MTU_8BIT_CH_5(0xab2, 0x1eb, 0xab4, 0xab6, 0xa84, 0xa85, 0xa86, 0xa94, 0xa95, 0xa96, 0xaa4, 0xaa5, 0xaa6), + [RZ_MTU3_CHAN_5] = MTU_8BIT_CH_5(0xab2, 0x895, 0xab4, 0xab6, 0xa84, 0xa85, 0xa86, 0xa94, 0xa95, 0xa96, 0xaa4, 0xaa5, 0xaa6), [RZ_MTU3_CHAN_6] = MTU_8BIT_CH_3_4_6_7(0x808, 0x893, 0x82c, 0x800, 0x84c, 0x802, 0x804, 0x805, 0x838), [RZ_MTU3_CHAN_7] = MTU_8BIT_CH_3_4_6_7(0x809, 0x894, 0x82d, 0x801, 0x84d, 0x803, 0x806, 0x807, 0x839), [RZ_MTU3_CHAN_8] = MTU_8BIT_CH_8(0x404, 0x098, 0x400, 0x406, 0x401, 0x402, 0x403) diff --git a/drivers/mfd/stmpe-i2c.c b/drivers/mfd/stmpe-i2c.c index fe018bedab983..7e2ca39758825 100644 --- a/drivers/mfd/stmpe-i2c.c +++ b/drivers/mfd/stmpe-i2c.c @@ -137,3 +137,4 @@ module_exit(stmpe_exit); MODULE_DESCRIPTION("STMPE MFD I2C Interface Driver"); MODULE_AUTHOR("Rabin Vincent "); +MODULE_LICENSE("GPL"); diff --git a/drivers/mfd/stmpe-spi.c b/drivers/mfd/stmpe-spi.c index 792236f56399a..b9cc85ea2c401 100644 --- a/drivers/mfd/stmpe-spi.c +++ b/drivers/mfd/stmpe-spi.c @@ -129,7 +129,7 @@ static const struct spi_device_id stmpe_spi_id[] = { { "stmpe2403", STMPE2403 }, { } }; -MODULE_DEVICE_TABLE(spi, stmpe_id); +MODULE_DEVICE_TABLE(spi, stmpe_spi_id); static struct spi_driver stmpe_spi_driver = { .driver = { diff --git a/drivers/mfd/stmpe.c b/drivers/mfd/stmpe.c index 9c3cf58457a7d..be6a84a3062cc 100644 --- a/drivers/mfd/stmpe.c +++ b/drivers/mfd/stmpe.c @@ -1485,6 +1485,9 @@ int stmpe_probe(struct stmpe_client_info *ci, enum stmpe_partnum partnum) void stmpe_remove(struct stmpe *stmpe) { + if (stmpe->domain) + irq_domain_remove(stmpe->domain); + if (!IS_ERR(stmpe->vio) && regulator_is_enabled(stmpe->vio)) regulator_disable(stmpe->vio); if (!IS_ERR(stmpe->vcc) && regulator_is_enabled(stmpe->vcc)) diff --git a/drivers/mfd/tps65219.c b/drivers/mfd/tps65219.c index 0e0c42e4fdfc7..72a5f51fe32a5 100644 --- a/drivers/mfd/tps65219.c +++ b/drivers/mfd/tps65219.c @@ -228,7 +228,6 @@ static struct regmap_irq_chip tps65219_irq_chip = { static int tps65219_probe(struct i2c_client *client) { struct tps65219 *tps; - unsigned int chipid; bool pwr_button; int ret; @@ -253,12 +252,6 @@ static int tps65219_probe(struct i2c_client *client) if (ret) return ret; - ret = regmap_read(tps->regmap, TPS65219_REG_TI_DEV_ID, &chipid); - if (ret) { - dev_err(tps->dev, "Failed to read device ID: %d\n", ret); - return ret; - } - ret = devm_mfd_add_devices(tps->dev, PLATFORM_DEVID_AUTO, tps65219_cells, ARRAY_SIZE(tps65219_cells), NULL, 0, regmap_irq_get_domain(tps->irq_data)); diff --git a/drivers/mfd/vexpress-sysreg.c b/drivers/mfd/vexpress-sysreg.c index eab82619ec314..8e41942081138 100644 --- a/drivers/mfd/vexpress-sysreg.c +++ b/drivers/mfd/vexpress-sysreg.c @@ -90,6 +90,7 @@ static int vexpress_sysreg_probe(struct platform_device *pdev) struct resource *mem; void __iomem *base; struct gpio_chip *mmc_gpio_chip; + int ret; mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!mem) @@ -110,7 +111,10 @@ static int vexpress_sysreg_probe(struct platform_device *pdev) bgpio_init(mmc_gpio_chip, &pdev->dev, 0x4, base + SYS_MCI, NULL, NULL, NULL, NULL, 0); mmc_gpio_chip->ngpio = 2; - devm_gpiochip_add_data(&pdev->dev, mmc_gpio_chip, NULL); + + ret = devm_gpiochip_add_data(&pdev->dev, mmc_gpio_chip, NULL); + if (ret) + return ret; return devm_mfd_add_devices(&pdev->dev, PLATFORM_DEVID_AUTO, vexpress_sysreg_cells, diff --git a/drivers/misc/cardreader/rtsx_usb.c b/drivers/misc/cardreader/rtsx_usb.c index f150d8769f198..f546b050cb495 100644 --- a/drivers/misc/cardreader/rtsx_usb.c +++ b/drivers/misc/cardreader/rtsx_usb.c @@ -698,6 +698,12 @@ static void rtsx_usb_disconnect(struct usb_interface *intf) } #ifdef CONFIG_PM +static int rtsx_usb_resume_child(struct device *dev, void *data) +{ + pm_request_resume(dev); + return 0; +} + static int rtsx_usb_suspend(struct usb_interface *intf, pm_message_t message) { struct rtsx_ucr *ucr = @@ -713,8 +719,10 @@ static int rtsx_usb_suspend(struct usb_interface *intf, pm_message_t message) mutex_unlock(&ucr->dev_mutex); /* Defer the autosuspend if card exists */ - if (val & (SD_CD | MS_CD)) + if (val & (SD_CD | MS_CD)) { + device_for_each_child(&intf->dev, NULL, rtsx_usb_resume_child); return -EAGAIN; + } } else { /* There is an ongoing operation*/ return -EAGAIN; @@ -724,12 +732,6 @@ static int rtsx_usb_suspend(struct usb_interface *intf, pm_message_t message) return 0; } -static int rtsx_usb_resume_child(struct device *dev, void *data) -{ - pm_request_resume(dev); - return 0; -} - static int rtsx_usb_resume(struct usb_interface *intf) { device_for_each_child(&intf->dev, NULL, rtsx_usb_resume_child); diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index fc021e265edc0..c110fb606dfb8 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -322,11 +322,11 @@ static void fastrpc_free_map(struct kref *ref) perm.vmid = QCOM_SCM_VMID_HLOS; perm.perm = QCOM_SCM_PERM_RWX; - err = qcom_scm_assign_mem(map->phys, map->size, + err = qcom_scm_assign_mem(map->phys, map->len, &src_perms, &perm, 1); if (err) { - dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d", - map->phys, map->size, err); + dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n", + map->phys, map->len, err); return; } } @@ -362,32 +362,29 @@ static int fastrpc_map_get(struct fastrpc_map *map) static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, - struct fastrpc_map **ppmap, bool take_ref) + struct fastrpc_map **ppmap) { - struct fastrpc_session_ctx *sess = fl->sctx; struct fastrpc_map *map = NULL; + struct dma_buf *buf; int ret = -ENOENT; + buf = dma_buf_get(fd); + if (IS_ERR(buf)) + return PTR_ERR(buf); + spin_lock(&fl->lock); list_for_each_entry(map, &fl->maps, node) { - if (map->fd != fd) + if (map->fd != fd || map->buf != buf) continue; - if (take_ref) { - ret = fastrpc_map_get(map); - if (ret) { - dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n", - __func__, fd, ret); - break; - } - } - *ppmap = map; ret = 0; break; } spin_unlock(&fl->lock); + dma_buf_put(buf); + return ret; } @@ -751,16 +748,14 @@ static const struct dma_buf_ops fastrpc_dma_buf_ops = { .release = fastrpc_release, }; -static int fastrpc_map_create(struct fastrpc_user *fl, int fd, +static int fastrpc_map_attach(struct fastrpc_user *fl, int fd, u64 len, u32 attr, struct fastrpc_map **ppmap) { struct fastrpc_session_ctx *sess = fl->sctx; struct fastrpc_map *map = NULL; struct sg_table *table; - int err = 0; - - if (!fastrpc_map_lookup(fl, fd, ppmap, true)) - return 0; + struct scatterlist *sgl = NULL; + int err = 0, sgl_index = 0; map = kzalloc(sizeof(*map), GFP_KERNEL); if (!map) @@ -797,7 +792,15 @@ static int fastrpc_map_create(struct fastrpc_user *fl, int fd, map->phys = sg_dma_address(map->table->sgl); map->phys += ((u64)fl->sctx->sid << 32); } - map->size = len; + for_each_sg(map->table->sgl, sgl, map->table->nents, + sgl_index) + map->size += sg_dma_len(sgl); + if (len > map->size) { + dev_dbg(sess->dev, "Bad size passed len 0x%llx map size 0x%llx\n", + len, map->size); + err = -EINVAL; + goto map_err; + } map->va = sg_virt(map->table->sgl); map->len = len; @@ -814,10 +817,10 @@ static int fastrpc_map_create(struct fastrpc_user *fl, int fd, dst_perms[1].vmid = fl->cctx->vmperms[0].vmid; dst_perms[1].perm = QCOM_SCM_PERM_RWX; map->attr = attr; - err = qcom_scm_assign_mem(map->phys, (u64)map->size, &src_perms, dst_perms, 2); + err = qcom_scm_assign_mem(map->phys, (u64)map->len, &src_perms, dst_perms, 2); if (err) { - dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d", - map->phys, map->size, err); + dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n", + map->phys, map->len, err); goto map_err; } } @@ -838,6 +841,24 @@ static int fastrpc_map_create(struct fastrpc_user *fl, int fd, return err; } +static int fastrpc_map_create(struct fastrpc_user *fl, int fd, + u64 len, u32 attr, struct fastrpc_map **ppmap) +{ + struct fastrpc_session_ctx *sess = fl->sctx; + int err = 0; + + if (!fastrpc_map_lookup(fl, fd, ppmap)) { + if (!fastrpc_map_get(*ppmap)) + return 0; + dev_dbg(sess->dev, "%s: Failed to get map fd=%d\n", + __func__, fd); + } + + err = fastrpc_map_attach(fl, fd, len, attr, ppmap); + + return err; +} + /* * Fastrpc payload buffer with metadata looks like: * @@ -910,8 +931,12 @@ static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx) ctx->args[i].length == 0) continue; - err = fastrpc_map_create(ctx->fl, ctx->args[i].fd, - ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]); + if (i < ctx->nbufs) + err = fastrpc_map_create(ctx->fl, ctx->args[i].fd, + ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]); + else + err = fastrpc_map_attach(ctx->fl, ctx->args[i].fd, + ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]); if (err) { dev_err(dev, "Error Creating map %d\n", err); return -EINVAL; @@ -1067,6 +1092,7 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, struct fastrpc_phy_page *pages; u64 *fdlist; int i, inbufs, outbufs, handles; + int ret = 0; inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc); @@ -1082,23 +1108,26 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, u64 len = rpra[i].buf.len; if (!kernel) { - if (copy_to_user((void __user *)dst, src, len)) - return -EFAULT; + if (copy_to_user((void __user *)dst, src, len)) { + ret = -EFAULT; + goto cleanup_fdlist; + } } else { memcpy(dst, src, len); } } } +cleanup_fdlist: /* Clean up fdlist which is updated by DSP */ for (i = 0; i < FASTRPC_MAX_FDLIST; i++) { if (!fdlist[i]) break; - if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false)) + if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap)) fastrpc_map_put(mmap); } - return 0; + return ret; } static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, @@ -1222,7 +1251,7 @@ static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_reques * that does not support unsigned PD offload */ if (!fl->cctx->unsigned_support || !unsigned_pd_request) { - dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD"); + dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n"); return true; } } @@ -1286,7 +1315,7 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, &src_perms, fl->cctx->vmperms, fl->cctx->vmcount); if (err) { - dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d", + dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n", fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); goto err_map; } @@ -1340,7 +1369,7 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, (u64)fl->cctx->remote_heap->size, &src_perms, &dst_perms, 1); if (err) - dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d", + dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n", fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); } err_map: @@ -2045,7 +2074,7 @@ static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp) args[0].length = sizeof(req_msg); pages.addr = map->phys; - pages.size = map->size; + pages.size = map->len; args[1].ptr = (u64) (uintptr_t) &pages; args[1].length = sizeof(pages); @@ -2060,7 +2089,7 @@ static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp) err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]); if (err) { dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n", - req.fd, req.vaddrin, map->size); + req.fd, req.vaddrin, map->len); goto err_invoke; } @@ -2073,7 +2102,7 @@ static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp) if (copy_to_user((void __user *)argp, &req, sizeof(req))) { /* unmap the memory and release the buffer */ req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr; - req_unmap.length = map->size; + req_unmap.length = map->len; fastrpc_req_mem_unmap_impl(fl, &req_unmap); return -EFAULT; } diff --git a/drivers/misc/genwqe/card_ddcb.c b/drivers/misc/genwqe/card_ddcb.c index 500b1feaf1f6f..fd7d5cd50d396 100644 --- a/drivers/misc/genwqe/card_ddcb.c +++ b/drivers/misc/genwqe/card_ddcb.c @@ -923,7 +923,7 @@ int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, } if (cmd->asv_length > DDCB_ASV_LENGTH) { dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n", - __func__, cmd->asiv_length); + __func__, cmd->asv_length); return -EINVAL; } rc = __genwqe_enqueue_ddcb(cd, req, f_flags); diff --git a/drivers/misc/lkdtm/fortify.c b/drivers/misc/lkdtm/fortify.c index 0159276656780..00ed2147113e6 100644 --- a/drivers/misc/lkdtm/fortify.c +++ b/drivers/misc/lkdtm/fortify.c @@ -44,6 +44,9 @@ static void lkdtm_FORTIFY_STR_MEMBER(void) char *src; src = kmalloc(size, GFP_KERNEL); + if (!src) + return; + strscpy(src, "over ten bytes", size); size = strlen(src) + 1; @@ -109,6 +112,9 @@ static void lkdtm_FORTIFY_MEM_MEMBER(void) char *src; src = kmalloc(size, GFP_KERNEL); + if (!src) + return; + strscpy(src, "over ten bytes", size); size = strlen(src) + 1; diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 2e65ce6bdec7d..b94cf7393fad6 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -1269,10 +1269,16 @@ static void mei_dev_bus_put(struct mei_device *bus) static void mei_cl_bus_dev_release(struct device *dev) { struct mei_cl_device *cldev = to_mei_cl_device(dev); + struct mei_device *mdev = cldev->cl->dev; + struct mei_cl *cl; mei_cl_flush_queues(cldev->cl, NULL); mei_me_cl_put(cldev->me_cl); mei_dev_bus_put(cldev->bus); + + list_for_each_entry(cl, &mdev->file_list, link) + WARN_ON(cl == cldev->cl); + kfree(cldev->cl); kfree(cldev); } diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index 4adfa5af162f1..fcfc6c7e6dc8a 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -120,6 +120,8 @@ #define MEI_DEV_ID_PTL_H 0xE370 /* Panther Lake H */ #define MEI_DEV_ID_PTL_P 0xE470 /* Panther Lake P */ +#define MEI_DEV_ID_WCL_P 0x4D70 /* Wildcat Lake P */ + /* * MEI HW Section */ diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 93b98a7f4c7fd..1a1df0390a40d 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -127,6 +127,8 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_PTL_H, MEI_ME_PCH15_CFG)}, {MEI_PCI_DEVICE(MEI_DEV_ID_PTL_P, MEI_ME_PCH15_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_WCL_P, MEI_ME_PCH15_CFG)}, + /* required last entry */ {0, } }; diff --git a/drivers/misc/tps6594-pfsm.c b/drivers/misc/tps6594-pfsm.c index 88dcac8148922..71fbe31542e56 100644 --- a/drivers/misc/tps6594-pfsm.c +++ b/drivers/misc/tps6594-pfsm.c @@ -260,6 +260,9 @@ static int tps6594_pfsm_probe(struct platform_device *pdev) pfsm->miscdev.minor = MISC_DYNAMIC_MINOR; pfsm->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "pfsm-%ld-0x%02x", tps->chip_id, tps->reg); + if (!pfsm->miscdev.name) + return -ENOMEM; + pfsm->miscdev.fops = &tps6594_pfsm_fops; pfsm->miscdev.parent = dev->parent; diff --git a/drivers/misc/vmw_vmci/vmci_host.c b/drivers/misc/vmw_vmci/vmci_host.c index abe79f6fd2a79..b64944367ac53 100644 --- a/drivers/misc/vmw_vmci/vmci_host.c +++ b/drivers/misc/vmw_vmci/vmci_host.c @@ -227,6 +227,7 @@ static int drv_cp_harray_to_user(void __user *user_buf_uva, static int vmci_host_setup_notify(struct vmci_ctx *context, unsigned long uva) { + struct page *page; int retval; if (context->notify_page) { @@ -243,13 +244,11 @@ static int vmci_host_setup_notify(struct vmci_ctx *context, /* * Lock physical page backing a given user VA. */ - retval = get_user_pages_fast(uva, 1, FOLL_WRITE, &context->notify_page); - if (retval != 1) { - context->notify_page = NULL; + retval = get_user_pages_fast(uva, 1, FOLL_WRITE, &page); + if (retval != 1) return VMCI_ERROR_GENERIC; - } - if (context->notify_page == NULL) - return VMCI_ERROR_UNAVAILABLE; + + context->notify_page = page; /* * Map the locked page and set up notify pointer. diff --git a/drivers/mmc/core/card.h b/drivers/mmc/core/card.h index 8476754b1b170..fe0b2fa3bb89d 100644 --- a/drivers/mmc/core/card.h +++ b/drivers/mmc/core/card.h @@ -86,6 +86,7 @@ struct mmc_fixup { #define CID_MANFID_MICRON 0x13 #define CID_MANFID_SAMSUNG 0x15 #define CID_MANFID_APACER 0x27 +#define CID_MANFID_SWISSBIT 0x5D #define CID_MANFID_KINGSTON 0x70 #define CID_MANFID_HYNIX 0x90 #define CID_MANFID_KINGSTON_SD 0x9F @@ -291,4 +292,9 @@ static inline int mmc_card_broken_sd_poweroff_notify(const struct mmc_card *c) return c->quirks & MMC_QUIRK_BROKEN_SD_POWEROFF_NOTIFY; } +static inline int mmc_card_no_uhs_ddr50_tuning(const struct mmc_card *c) +{ + return c->quirks & MMC_QUIRK_NO_UHS_DDR50_TUNING; +} + #endif diff --git a/drivers/mmc/core/quirks.h b/drivers/mmc/core/quirks.h index 89b512905be14..c417ed34c0576 100644 --- a/drivers/mmc/core/quirks.h +++ b/drivers/mmc/core/quirks.h @@ -34,6 +34,22 @@ static const struct mmc_fixup __maybe_unused mmc_sd_fixups[] = { MMC_QUIRK_BROKEN_SD_CACHE | MMC_QUIRK_BROKEN_SD_POWEROFF_NOTIFY, EXT_CSD_REV_ANY), + /* + * Swissbit series S46-u cards throw I/O errors during tuning requests + * after the initial tuning request expectedly times out. This has + * only been observed on cards manufactured on 01/2019 that are using + * Bay Trail host controllers. + */ + _FIXUP_EXT("0016G", CID_MANFID_SWISSBIT, 0x5342, 2019, 1, + 0, -1ull, SDIO_ANY_ID, SDIO_ANY_ID, add_quirk_sd, + MMC_QUIRK_NO_UHS_DDR50_TUNING, EXT_CSD_REV_ANY), + + /* + * Some SD cards reports discard support while they don't + */ + MMC_FIXUP(CID_NAME_ANY, CID_MANFID_SANDISK_SD, 0x5344, add_quirk_sd, + MMC_QUIRK_BROKEN_SD_DISCARD), + END_FIXUP }; @@ -137,12 +153,6 @@ static const struct mmc_fixup __maybe_unused mmc_blk_fixups[] = { MMC_FIXUP("M62704", CID_MANFID_KINGSTON, 0x0100, add_quirk_mmc, MMC_QUIRK_TRIM_BROKEN), - /* - * Some SD cards reports discard support while they don't - */ - MMC_FIXUP(CID_NAME_ANY, CID_MANFID_SANDISK_SD, 0x5344, add_quirk_sd, - MMC_QUIRK_BROKEN_SD_DISCARD), - END_FIXUP }; diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index f02c3e5eb5c85..a06f3011e2b58 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -618,6 +618,29 @@ static int sd_set_current_limit(struct mmc_card *card, u8 *status) return 0; } +/* + * Determine if the card should tune or not. + */ +static bool mmc_sd_use_tuning(struct mmc_card *card) +{ + /* + * SPI mode doesn't define CMD19 and tuning is only valid for SDR50 and + * SDR104 mode SD-cards. Note that tuning is mandatory for SDR104. + */ + if (mmc_host_is_spi(card->host)) + return false; + + switch (card->host->ios.timing) { + case MMC_TIMING_UHS_SDR50: + case MMC_TIMING_UHS_SDR104: + return true; + case MMC_TIMING_UHS_DDR50: + return !mmc_card_no_uhs_ddr50_tuning(card); + } + + return false; +} + /* * UHS-I specific initialization procedure */ @@ -661,14 +684,7 @@ static int mmc_sd_init_uhs_card(struct mmc_card *card) if (err) goto out; - /* - * SPI mode doesn't define CMD19 and tuning is only valid for SDR50 and - * SDR104 mode SD-cards. Note that tuning is mandatory for SDR104. - */ - if (!mmc_host_is_spi(card->host) && - (card->host->ios.timing == MMC_TIMING_UHS_SDR50 || - card->host->ios.timing == MMC_TIMING_UHS_DDR50 || - card->host->ios.timing == MMC_TIMING_UHS_SDR104)) { + if (mmc_sd_use_tuning(card)) { err = mmc_execute_tuning(card); /* diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c index 86b4c68a02323..73db83c697027 100644 --- a/drivers/mmc/core/sdio.c +++ b/drivers/mmc/core/sdio.c @@ -947,7 +947,11 @@ static void mmc_sdio_remove(struct mmc_host *host) */ static int mmc_sdio_alive(struct mmc_host *host) { - return mmc_select_card(host->card); + if (!mmc_host_is_spi(host)) + return mmc_select_card(host->card); + else + return mmc_io_rw_direct(host->card, 0, 0, SDIO_CCCR_CCCR, 0, + NULL); } /* diff --git a/drivers/mmc/host/bcm2835.c b/drivers/mmc/host/bcm2835.c index 35d8fdea668b9..f923447ed2ce2 100644 --- a/drivers/mmc/host/bcm2835.c +++ b/drivers/mmc/host/bcm2835.c @@ -502,7 +502,8 @@ void bcm2835_prepare_dma(struct bcm2835_host *host, struct mmc_data *data) DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!desc) { - dma_unmap_sg(dma_chan->device->dev, data->sg, sg_len, dir_data); + dma_unmap_sg(dma_chan->device->dev, data->sg, data->sg_len, + dir_data); return; } diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c index 698408e8bad03..381a61adf06ee 100644 --- a/drivers/mmc/host/dw_mmc-exynos.c +++ b/drivers/mmc/host/dw_mmc-exynos.c @@ -28,6 +28,8 @@ enum dw_mci_exynos_type { DW_MCI_TYPE_EXYNOS5420_SMU, DW_MCI_TYPE_EXYNOS7, DW_MCI_TYPE_EXYNOS7_SMU, + DW_MCI_TYPE_EXYNOS7870, + DW_MCI_TYPE_EXYNOS7870_SMU, DW_MCI_TYPE_ARTPEC8, }; @@ -70,6 +72,12 @@ static struct dw_mci_exynos_compatible { }, { .compatible = "samsung,exynos7-dw-mshc-smu", .ctrl_type = DW_MCI_TYPE_EXYNOS7_SMU, + }, { + .compatible = "samsung,exynos7870-dw-mshc", + .ctrl_type = DW_MCI_TYPE_EXYNOS7870, + }, { + .compatible = "samsung,exynos7870-dw-mshc-smu", + .ctrl_type = DW_MCI_TYPE_EXYNOS7870_SMU, }, { .compatible = "axis,artpec8-dw-mshc", .ctrl_type = DW_MCI_TYPE_ARTPEC8, @@ -86,6 +94,8 @@ static inline u8 dw_mci_exynos_get_ciu_div(struct dw_mci *host) return EXYNOS4210_FIXED_CIU_CLK_DIV; else if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) return SDMMC_CLKSEL_GET_DIV(mci_readl(host, CLKSEL64)) + 1; else @@ -101,7 +111,8 @@ static void dw_mci_exynos_config_smu(struct dw_mci *host) * set for non-ecryption mode at this time. */ if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS5420_SMU || - priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU) { + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU) { mci_writel(host, MPSBEGIN0, 0); mci_writel(host, MPSEND0, SDMMC_ENDING_SEC_NR_MAX); mci_writel(host, MPSCTRL0, SDMMC_MPSCTRL_SECURE_WRITE_BIT | @@ -127,6 +138,12 @@ static int dw_mci_exynos_priv_init(struct dw_mci *host) DQS_CTRL_GET_RD_DELAY(priv->saved_strobe_ctrl); } + if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU) { + /* Quirk needed for certain Exynos SoCs */ + host->quirks |= DW_MMC_QUIRK_FIFO64_32; + } + if (priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) { /* Quirk needed for the ARTPEC-8 SoC */ host->quirks |= DW_MMC_QUIRK_EXTENDED_TMOUT; @@ -144,6 +161,8 @@ static void dw_mci_exynos_set_clksel_timing(struct dw_mci *host, u32 timing) if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) clksel = mci_readl(host, CLKSEL64); else @@ -153,6 +172,8 @@ static void dw_mci_exynos_set_clksel_timing(struct dw_mci *host, u32 timing) if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) mci_writel(host, CLKSEL64, clksel); else @@ -223,6 +244,8 @@ static int dw_mci_exynos_resume_noirq(struct device *dev) if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) clksel = mci_readl(host, CLKSEL64); else @@ -231,6 +254,8 @@ static int dw_mci_exynos_resume_noirq(struct device *dev) if (clksel & SDMMC_CLKSEL_WAKEUP_INT) { if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) mci_writel(host, CLKSEL64, clksel); else @@ -410,6 +435,8 @@ static inline u8 dw_mci_exynos_get_clksmpl(struct dw_mci *host) if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) return SDMMC_CLKSEL_CCLK_SAMPLE(mci_readl(host, CLKSEL64)); else @@ -423,6 +450,8 @@ static inline void dw_mci_exynos_set_clksmpl(struct dw_mci *host, u8 sample) if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) clksel = mci_readl(host, CLKSEL64); else @@ -430,6 +459,8 @@ static inline void dw_mci_exynos_set_clksmpl(struct dw_mci *host, u8 sample) clksel = SDMMC_CLKSEL_UP_SAMPLE(clksel, sample); if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) mci_writel(host, CLKSEL64, clksel); else @@ -444,6 +475,8 @@ static inline u8 dw_mci_exynos_move_next_clksmpl(struct dw_mci *host) if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) clksel = mci_readl(host, CLKSEL64); else @@ -454,6 +487,8 @@ static inline u8 dw_mci_exynos_move_next_clksmpl(struct dw_mci *host) if (priv->ctrl_type == DW_MCI_TYPE_EXYNOS7 || priv->ctrl_type == DW_MCI_TYPE_EXYNOS7_SMU || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870 || + priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU || priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) mci_writel(host, CLKSEL64, clksel); else @@ -633,6 +668,10 @@ static const struct of_device_id dw_mci_exynos_match[] = { .data = &exynos_drv_data, }, { .compatible = "samsung,exynos7-dw-mshc-smu", .data = &exynos_drv_data, }, + { .compatible = "samsung,exynos7870-dw-mshc", + .data = &exynos_drv_data, }, + { .compatible = "samsung,exynos7870-dw-mshc-smu", + .data = &exynos_drv_data, }, { .compatible = "axis,artpec8-dw-mshc", .data = &artpec_drv_data, }, {}, diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c index 02f3748e46c14..cf685c0a17edc 100644 --- a/drivers/mmc/host/mtk-sd.c +++ b/drivers/mmc/host/mtk-sd.c @@ -770,12 +770,18 @@ static inline void msdc_dma_setup(struct msdc_host *host, struct msdc_dma *dma, static void msdc_prepare_data(struct msdc_host *host, struct mmc_data *data) { if (!(data->host_cookie & MSDC_PREPARE_FLAG)) { - data->host_cookie |= MSDC_PREPARE_FLAG; data->sg_count = dma_map_sg(host->dev, data->sg, data->sg_len, mmc_get_dma_dir(data)); + if (data->sg_count) + data->host_cookie |= MSDC_PREPARE_FLAG; } } +static bool msdc_data_prepared(struct mmc_data *data) +{ + return data->host_cookie & MSDC_PREPARE_FLAG; +} + static void msdc_unprepare_data(struct msdc_host *host, struct mmc_data *data) { if (data->host_cookie & MSDC_ASYNC_FLAG) @@ -1338,8 +1344,19 @@ static void msdc_ops_request(struct mmc_host *mmc, struct mmc_request *mrq) WARN_ON(host->mrq); host->mrq = mrq; - if (mrq->data) + if (mrq->data) { msdc_prepare_data(host, mrq->data); + if (!msdc_data_prepared(mrq->data)) { + host->mrq = NULL; + /* + * Failed to prepare DMA area, fail fast before + * starting any commands. + */ + mrq->cmd->error = -ENOSPC; + mmc_request_done(mmc_from_priv(host), mrq); + return; + } + } /* if SBC is required, we have HW option and SW option. * if HW option is enabled, and SBC does not have "special" flags, diff --git a/drivers/mmc/host/mvsdio.c b/drivers/mmc/host/mvsdio.c index ca01b7d204ba6..955df7f370a78 100644 --- a/drivers/mmc/host/mvsdio.c +++ b/drivers/mmc/host/mvsdio.c @@ -292,7 +292,7 @@ static u32 mvsd_finish_data(struct mvsd_host *host, struct mmc_data *data, host->pio_ptr = NULL; host->pio_size = 0; } else { - dma_unmap_sg(mmc_dev(host->mmc), data->sg, host->sg_frags, + dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len, mmc_get_dma_dir(data)); } diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c index c675dec587efb..cffacf4434b55 100644 --- a/drivers/mmc/host/renesas_sdhi_core.c +++ b/drivers/mmc/host/renesas_sdhi_core.c @@ -220,7 +220,11 @@ static void renesas_sdhi_set_clock(struct tmio_mmc_host *host, clk &= ~0xff; } - sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, clk & CLK_CTL_DIV_MASK); + clock = clk & CLK_CTL_DIV_MASK; + if (clock != 0xff) + host->mmc->actual_clock /= (1 << (ffs(clock) + 1)); + + sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, clock); if (!(host->pdata->flags & TMIO_MMC_MIN_RCAR2)) usleep_range(10000, 11000); @@ -1107,26 +1111,26 @@ int renesas_sdhi_probe(struct platform_device *pdev, num_irqs = platform_irq_count(pdev); if (num_irqs < 0) { ret = num_irqs; - goto eirq; + goto edisclk; } /* There must be at least one IRQ source */ if (!num_irqs) { ret = -ENXIO; - goto eirq; + goto edisclk; } for (i = 0; i < num_irqs; i++) { irq = platform_get_irq(pdev, i); if (irq < 0) { ret = irq; - goto eirq; + goto edisclk; } ret = devm_request_irq(&pdev->dev, irq, tmio_mmc_irq, 0, dev_name(&pdev->dev), host); if (ret) - goto eirq; + goto edisclk; } ret = tmio_mmc_host_probe(host); @@ -1138,8 +1142,6 @@ int renesas_sdhi_probe(struct platform_device *pdev, return ret; -eirq: - tmio_mmc_host_remove(host); edisclk: renesas_sdhi_clk_disable(host); efree: diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c index ded9b6849e35e..90ea92bbdb2cf 100644 --- a/drivers/mmc/host/rtsx_usb_sdmmc.c +++ b/drivers/mmc/host/rtsx_usb_sdmmc.c @@ -1032,9 +1032,7 @@ static int sd_set_power_mode(struct rtsx_usb_sdmmc *host, err = sd_power_on(host); } - if (!err) - host->power_mode = power_mode; - + host->power_mode = power_mode; return err; } diff --git a/drivers/mmc/host/sdhci-cadence.c b/drivers/mmc/host/sdhci-cadence.c index be1505e8c536e..7759531ccca70 100644 --- a/drivers/mmc/host/sdhci-cadence.c +++ b/drivers/mmc/host/sdhci-cadence.c @@ -433,6 +433,13 @@ static const struct sdhci_cdns_drv_data sdhci_elba_drv_data = { }, }; +static const struct sdhci_cdns_drv_data sdhci_eyeq_drv_data = { + .pltfm_data = { + .ops = &sdhci_cdns_ops, + .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, + }, +}; + static const struct sdhci_cdns_drv_data sdhci_cdns_drv_data = { .pltfm_data = { .ops = &sdhci_cdns_ops, @@ -595,6 +602,10 @@ static const struct of_device_id sdhci_cdns_match[] = { .compatible = "amd,pensando-elba-sd4hc", .data = &sdhci_elba_drv_data, }, + { + .compatible = "mobileye,eyeq-sd4hc", + .data = &sdhci_eyeq_drv_data, + }, { .compatible = "cdns,sd4hc" }, { /* sentinel */ } }; diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c index 5133f60558555..a6215b08ecdee 100644 --- a/drivers/mmc/host/sdhci-msm.c +++ b/drivers/mmc/host/sdhci-msm.c @@ -81,6 +81,7 @@ #define CORE_IO_PAD_PWR_SWITCH_EN BIT(15) #define CORE_IO_PAD_PWR_SWITCH BIT(16) #define CORE_HC_SELECT_IN_EN BIT(18) +#define CORE_HC_SELECT_IN_SDR50 (4 << 19) #define CORE_HC_SELECT_IN_HS400 (6 << 19) #define CORE_HC_SELECT_IN_MASK (7 << 19) @@ -1133,6 +1134,10 @@ static bool sdhci_msm_is_tuning_needed(struct sdhci_host *host) { struct mmc_ios *ios = &host->mmc->ios; + if (ios->timing == MMC_TIMING_UHS_SDR50 && + host->flags & SDHCI_SDR50_NEEDS_TUNING) + return true; + /* * Tuning is required for SDR104, HS200 and HS400 cards and * if clock frequency is greater than 100MHz in these modes. @@ -1201,6 +1206,8 @@ static int sdhci_msm_execute_tuning(struct mmc_host *mmc, u32 opcode) struct mmc_ios ios = host->mmc->ios; struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); + const struct sdhci_msm_offset *msm_offset = msm_host->offset; + u32 config; if (!sdhci_msm_is_tuning_needed(host)) { msm_host->use_cdr = false; @@ -1217,6 +1224,14 @@ static int sdhci_msm_execute_tuning(struct mmc_host *mmc, u32 opcode) */ msm_host->tuning_done = 0; + if (ios.timing == MMC_TIMING_UHS_SDR50 && + host->flags & SDHCI_SDR50_NEEDS_TUNING) { + config = readl_relaxed(host->ioaddr + msm_offset->core_vendor_spec); + config &= ~CORE_HC_SELECT_IN_MASK; + config |= CORE_HC_SELECT_IN_EN | CORE_HC_SELECT_IN_SDR50; + writel_relaxed(config, host->ioaddr + msm_offset->core_vendor_spec); + } + /* * For HS400 tuning in HS200 timing requires: * - select MCLK/2 in VENDOR_SPEC @@ -1564,6 +1579,7 @@ static void sdhci_msm_check_power_status(struct sdhci_host *host, u32 req_type) { struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); + struct mmc_host *mmc = host->mmc; bool done = false; u32 val = SWITCHABLE_SIGNALING_VOLTAGE; const struct sdhci_msm_offset *msm_offset = @@ -1621,6 +1637,12 @@ static void sdhci_msm_check_power_status(struct sdhci_host *host, u32 req_type) "%s: pwr_irq for req: (%d) timed out\n", mmc_hostname(host->mmc), req_type); } + + if ((req_type & REQ_BUS_ON) && mmc->card && !mmc->ops->get_cd(mmc)) { + sdhci_writeb(host, 0, SDHCI_POWER_CONTROL); + host->pwr = 0; + } + pr_debug("%s: %s: request %d done\n", mmc_hostname(host->mmc), __func__, req_type); } @@ -1679,6 +1701,13 @@ static void sdhci_msm_handle_pwr_irq(struct sdhci_host *host, int irq) udelay(10); } + if ((irq_status & CORE_PWRCTL_BUS_ON) && mmc->card && + !mmc->ops->get_cd(mmc)) { + msm_host_writel(msm_host, CORE_PWRCTL_BUS_FAIL, host, + msm_offset->core_pwrctl_ctl); + return; + } + /* Handle BUS ON/OFF*/ if (irq_status & CORE_PWRCTL_BUS_ON) { pwr_state = REQ_BUS_ON; diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index a0524127ca073..4c6fa92a95a6e 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -54,7 +54,7 @@ #define DLL_TXCLK_TAPNUM_DEFAULT 0x10 #define DLL_TXCLK_TAPNUM_90_DEGREES 0xA #define DLL_TXCLK_TAPNUM_FROM_SW BIT(24) -#define DLL_STRBIN_TAPNUM_DEFAULT 0x8 +#define DLL_STRBIN_TAPNUM_DEFAULT 0x4 #define DLL_STRBIN_TAPNUM_FROM_SW BIT(24) #define DLL_STRBIN_DELAY_NUM_SEL BIT(26) #define DLL_STRBIN_DELAY_NUM_OFFSET 16 diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 6b351810a301c..2ea5357e3bf0e 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -608,8 +608,12 @@ static void sdhci_intel_set_power(struct sdhci_host *host, unsigned char mode, sdhci_set_power(host, mode, vdd); - if (mode == MMC_POWER_OFF) + if (mode == MMC_POWER_OFF) { + if (slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_APL_SD || + slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_BYT_SD) + usleep_range(15000, 17500); return; + } /* * Bus power might not enable after D3 -> D0 transition due to the @@ -907,7 +911,8 @@ static bool glk_broken_cqhci(struct sdhci_pci_slot *slot) { return slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_GLK_EMMC && (dmi_match(DMI_BIOS_VENDOR, "LENOVO") || - dmi_match(DMI_SYS_VENDOR, "IRBIS")); + dmi_match(DMI_SYS_VENDOR, "IRBIS") || + dmi_match(DMI_SYS_VENDOR, "Positivo Tecnologia SA")); } static bool jsl_broken_hs400es(struct sdhci_pci_slot *slot) diff --git a/drivers/mmc/host/sdhci-pci-gli.c b/drivers/mmc/host/sdhci-pci-gli.c index 11c404374d79d..1dc1b9274b68c 100644 --- a/drivers/mmc/host/sdhci-pci-gli.c +++ b/drivers/mmc/host/sdhci-pci-gli.c @@ -28,9 +28,6 @@ #define PCI_GLI_9750_PM_CTRL 0xFC #define PCI_GLI_9750_PM_STATE GENMASK(1, 0) -#define PCI_GLI_9750_CORRERR_MASK 0x214 -#define PCI_GLI_9750_CORRERR_MASK_REPLAY_TIMER_TIMEOUT BIT(12) - #define SDHCI_GLI_9750_CFG2 0x848 #define SDHCI_GLI_9750_CFG2_L1DLY GENMASK(28, 24) #define GLI_9750_CFG2_L1DLY_VALUE 0x1F @@ -155,9 +152,6 @@ #define PCI_GLI_9755_PM_CTRL 0xFC #define PCI_GLI_9755_PM_STATE GENMASK(1, 0) -#define PCI_GLI_9755_CORRERR_MASK 0x214 -#define PCI_GLI_9755_CORRERR_MASK_REPLAY_TIMER_TIMEOUT BIT(12) - #define SDHCI_GLI_9767_GM_BURST_SIZE 0x510 #define SDHCI_GLI_9767_GM_BURST_SIZE_AXI_ALWAYS_SET BIT(8) @@ -227,6 +221,20 @@ #define GLI_MAX_TUNING_LOOP 40 /* Genesys Logic chipset */ +static void sdhci_gli_mask_replay_timer_timeout(struct pci_dev *pdev) +{ + int aer; + u32 value; + + /* mask the replay timer timeout of AER */ + aer = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); + if (aer) { + pci_read_config_dword(pdev, aer + PCI_ERR_COR_MASK, &value); + value |= PCI_ERR_COR_REP_TIMER; + pci_write_config_dword(pdev, aer + PCI_ERR_COR_MASK, value); + } +} + static inline void gl9750_wt_on(struct sdhci_host *host) { u32 wt_value; @@ -568,9 +576,7 @@ static void gl9750_hw_setting(struct sdhci_host *host) pci_write_config_dword(pdev, PCI_GLI_9750_PM_CTRL, value); /* mask the replay timer timeout of AER */ - pci_read_config_dword(pdev, PCI_GLI_9750_CORRERR_MASK, &value); - value |= PCI_GLI_9750_CORRERR_MASK_REPLAY_TIMER_TIMEOUT; - pci_write_config_dword(pdev, PCI_GLI_9750_CORRERR_MASK, value); + sdhci_gli_mask_replay_timer_timeout(pdev); gl9750_wt_off(host); } @@ -782,9 +788,7 @@ static void gl9755_hw_setting(struct sdhci_pci_slot *slot) pci_write_config_dword(pdev, PCI_GLI_9755_PM_CTRL, value); /* mask the replay timer timeout of AER */ - pci_read_config_dword(pdev, PCI_GLI_9755_CORRERR_MASK, &value); - value |= PCI_GLI_9755_CORRERR_MASK_REPLAY_TIMER_TIMEOUT; - pci_write_config_dword(pdev, PCI_GLI_9755_CORRERR_MASK, value); + sdhci_gli_mask_replay_timer_timeout(pdev); gl9755_wt_off(pdev); } @@ -1343,7 +1347,7 @@ static int gl9763e_add_host(struct sdhci_pci_slot *slot) return ret; } -static void gli_set_gl9763e(struct sdhci_pci_slot *slot) +static void gl9763e_hw_setting(struct sdhci_pci_slot *slot) { struct pci_dev *pdev = slot->chip->pdev; u32 value; @@ -1372,6 +1376,9 @@ static void gli_set_gl9763e(struct sdhci_pci_slot *slot) value |= FIELD_PREP(GLI_9763E_HS400_RXDLY, GLI_9763E_HS400_RXDLY_5); pci_write_config_dword(pdev, PCIE_GLI_9763E_CLKRXDLY, value); + /* mask the replay timer timeout of AER */ + sdhci_gli_mask_replay_timer_timeout(pdev); + pci_read_config_dword(pdev, PCIE_GLI_9763E_VHS, &value); value &= ~GLI_9763E_VHS_REV; value |= FIELD_PREP(GLI_9763E_VHS_REV, GLI_9763E_VHS_REV_R); @@ -1515,7 +1522,7 @@ static int gli_probe_slot_gl9763e(struct sdhci_pci_slot *slot) gli_pcie_enable_msi(slot); host->mmc_host_ops.hs400_enhanced_strobe = gl9763e_hs400_enhanced_strobe; - gli_set_gl9763e(slot); + gl9763e_hw_setting(slot); sdhci_enable_v4_mode(host); return 0; diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h index a5758cdd03a5d..83e0677f242dd 100644 --- a/drivers/mmc/host/sdhci.h +++ b/drivers/mmc/host/sdhci.h @@ -830,4 +830,20 @@ void sdhci_switch_external_dma(struct sdhci_host *host, bool en); void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable); void __sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd); +#if defined(CONFIG_DYNAMIC_DEBUG) || \ + (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) +#define SDHCI_DBG_ANYWAY 0 +#elif defined(DEBUG) +#define SDHCI_DBG_ANYWAY 1 +#else +#define SDHCI_DBG_ANYWAY 0 +#endif + +#define sdhci_dbg_dumpregs(host, fmt) \ +do { \ + DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt); \ + if (DYNAMIC_DEBUG_BRANCH(descriptor) || SDHCI_DBG_ANYWAY) \ + sdhci_dumpregs(host); \ +} while (0) + #endif /* __SDHCI_HW_H */ diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c index 562034af653eb..fb89b6062351a 100644 --- a/drivers/mmc/host/sdhci_am654.c +++ b/drivers/mmc/host/sdhci_am654.c @@ -559,7 +559,8 @@ static struct sdhci_ops sdhci_am654_ops = { static const struct sdhci_pltfm_data sdhci_am654_pdata = { .ops = &sdhci_am654_ops, .quirks = SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12, - .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, + .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN | + SDHCI_QUIRK2_DISABLE_HW_TIMEOUT, }; static const struct sdhci_am654_driver_data sdhci_am654_sr1_drvdata = { @@ -589,7 +590,8 @@ static struct sdhci_ops sdhci_j721e_8bit_ops = { static const struct sdhci_pltfm_data sdhci_j721e_8bit_pdata = { .ops = &sdhci_j721e_8bit_ops, .quirks = SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12, - .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, + .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN | + SDHCI_QUIRK2_DISABLE_HW_TIMEOUT, }; static const struct sdhci_am654_driver_data sdhci_j721e_8bit_drvdata = { @@ -613,7 +615,8 @@ static struct sdhci_ops sdhci_j721e_4bit_ops = { static const struct sdhci_pltfm_data sdhci_j721e_4bit_pdata = { .ops = &sdhci_j721e_4bit_ops, .quirks = SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12, - .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, + .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN | + SDHCI_QUIRK2_DISABLE_HW_TIMEOUT, }; static const struct sdhci_am654_driver_data sdhci_j721e_4bit_drvdata = { diff --git a/drivers/most/core.c b/drivers/most/core.c index e4412c7d25b0e..5d073b3d27967 100644 --- a/drivers/most/core.c +++ b/drivers/most/core.c @@ -538,8 +538,8 @@ static struct most_channel *get_channel(char *mdev, char *mdev_ch) dev = bus_find_device_by_name(&mostbus, NULL, mdev); if (!dev) return NULL; - put_device(dev); iface = dev_get_drvdata(dev); + put_device(dev); list_for_each_entry_safe(c, tmp, &iface->p->channel_list, list) { if (!strcmp(dev_name(&c->dev), mdev_ch)) return c; diff --git a/drivers/most/most_usb.c b/drivers/most/most_usb.c index 485d5ca399513..988e33f279704 100644 --- a/drivers/most/most_usb.c +++ b/drivers/most/most_usb.c @@ -929,6 +929,10 @@ static void release_mdev(struct device *dev) { struct most_dev *mdev = to_mdev_from_dev(dev); + kfree(mdev->busy_urbs); + kfree(mdev->cap); + kfree(mdev->conf); + kfree(mdev->ep_address); kfree(mdev); } /** @@ -1093,7 +1097,7 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id) err_free_conf: kfree(mdev->conf); err_free_mdev: - put_device(&mdev->dev); + kfree(mdev); return ret; } @@ -1121,13 +1125,6 @@ static void hdm_disconnect(struct usb_interface *interface) if (mdev->dci) device_unregister(&mdev->dci->dev); most_deregister_interface(&mdev->iface); - - kfree(mdev->busy_urbs); - kfree(mdev->cap); - kfree(mdev->conf); - kfree(mdev->ep_address); - put_device(&mdev->dci->dev); - put_device(&mdev->dev); } static int hdm_suspend(struct usb_interface *interface, pm_message_t message) diff --git a/drivers/mtd/ftl.c b/drivers/mtd/ftl.c index 8c22064ead387..f2bd1984609cc 100644 --- a/drivers/mtd/ftl.c +++ b/drivers/mtd/ftl.c @@ -344,7 +344,7 @@ static int erase_xfer(partition_t *part, return -ENOMEM; erase->addr = xfer->Offset; - erase->len = 1 << part->header.EraseUnitSize; + erase->len = 1ULL << part->header.EraseUnitSize; ret = mtd_erase(part->mbd.mtd, erase); if (!ret) { diff --git a/drivers/mtd/mtdchar.c b/drivers/mtd/mtdchar.c index 8dc4f5c493fcb..335c702633ffe 100644 --- a/drivers/mtd/mtdchar.c +++ b/drivers/mtd/mtdchar.c @@ -599,6 +599,7 @@ mtdchar_write_ioctl(struct mtd_info *mtd, struct mtd_write_req __user *argp) uint8_t *datbuf = NULL, *oobbuf = NULL; size_t datbuf_len, oobbuf_len; int ret = 0; + u64 end; if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT; @@ -618,7 +619,7 @@ mtdchar_write_ioctl(struct mtd_info *mtd, struct mtd_write_req __user *argp) req.len &= 0xffffffff; req.ooblen &= 0xffffffff; - if (req.start + req.len > mtd->size) + if (check_add_overflow(req.start, req.len, &end) || end > mtd->size) return -EINVAL; datbuf_len = min_t(size_t, req.len, mtd->erasesize); @@ -698,6 +699,7 @@ mtdchar_read_ioctl(struct mtd_info *mtd, struct mtd_read_req __user *argp) size_t datbuf_len, oobbuf_len; size_t orig_len, orig_ooblen; int ret = 0; + u64 end; if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT; @@ -724,7 +726,7 @@ mtdchar_read_ioctl(struct mtd_info *mtd, struct mtd_read_req __user *argp) req.len &= 0xffffffff; req.ooblen &= 0xffffffff; - if (req.start + req.len > mtd->size) { + if (check_add_overflow(req.start, req.len, &end) || end > mtd->size) { ret = -EINVAL; goto out; } diff --git a/drivers/mtd/nand/ecc-mxic.c b/drivers/mtd/nand/ecc-mxic.c index 47e10945b8d27..63cb206269dd9 100644 --- a/drivers/mtd/nand/ecc-mxic.c +++ b/drivers/mtd/nand/ecc-mxic.c @@ -614,7 +614,7 @@ static int mxic_ecc_finish_io_req_external(struct nand_device *nand, { struct mxic_ecc_engine *mxic = nand_to_mxic(nand); struct mxic_ecc_ctx *ctx = nand_to_ecc_ctx(nand); - int nents, step, ret; + int nents, step, ret = 0; if (req->mode == MTD_OPS_RAW) return 0; diff --git a/drivers/mtd/nand/onenand/onenand_samsung.c b/drivers/mtd/nand/onenand/onenand_samsung.c index fd6890a03d557..0e21d443078e4 100644 --- a/drivers/mtd/nand/onenand/onenand_samsung.c +++ b/drivers/mtd/nand/onenand/onenand_samsung.c @@ -906,7 +906,7 @@ static int s3c_onenand_probe(struct platform_device *pdev) err = devm_request_irq(&pdev->dev, r->start, s5pc110_onenand_irq, IRQF_SHARED, "onenand", - &onenand); + onenand); if (err) { dev_err(&pdev->dev, "failed to get irq\n"); return err; diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c index 3f494f7c7ecbd..14a09285f1551 100644 --- a/drivers/mtd/nand/raw/atmel/nand-controller.c +++ b/drivers/mtd/nand/raw/atmel/nand-controller.c @@ -373,7 +373,7 @@ static int atmel_nand_dma_transfer(struct atmel_nand_controller *nc, dma_cookie_t cookie; buf_dma = dma_map_single(nc->dev, buf, len, dir); - if (dma_mapping_error(nc->dev, dev_dma)) { + if (dma_mapping_error(nc->dev, buf_dma)) { dev_err(nc->dev, "Failed to prepare a buffer for DMA access\n"); goto err; @@ -1378,13 +1378,23 @@ static int atmel_smc_nand_prepare_smcconf(struct atmel_nand *nand, return ret; /* - * The write cycle timing is directly matching tWC, but is also + * Read setup timing depends on the operation done on the NAND: + * + * NRD_SETUP = max(tAR, tCLR) + */ + timeps = max(conf->timings.sdr.tAR_min, conf->timings.sdr.tCLR_min); + ncycles = DIV_ROUND_UP(timeps, mckperiodps); + totalcycles += ncycles; + ret = atmel_smc_cs_conf_set_setup(smcconf, ATMEL_SMC_NRD_SHIFT, ncycles); + if (ret) + return ret; + + /* + * The read cycle timing is directly matching tRC, but is also * dependent on the setup and hold timings we calculated earlier, * which gives: * - * NRD_CYCLE = max(tRC, NRD_PULSE + NRD_HOLD) - * - * NRD_SETUP is always 0. + * NRD_CYCLE = max(tRC, NRD_SETUP + NRD_PULSE + NRD_HOLD) */ ncycles = DIV_ROUND_UP(conf->timings.sdr.tRC_min, mckperiodps); ncycles = max(totalcycles, ncycles); @@ -1848,7 +1858,7 @@ atmel_nand_controller_legacy_add_nands(struct atmel_nand_controller *nc) static int atmel_nand_controller_add_nands(struct atmel_nand_controller *nc) { - struct device_node *np, *nand_np; + struct device_node *np; struct device *dev = nc->dev; int ret, reg_cells; u32 val; @@ -1875,7 +1885,7 @@ static int atmel_nand_controller_add_nands(struct atmel_nand_controller *nc) reg_cells += val; - for_each_child_of_node(np, nand_np) { + for_each_child_of_node_scoped(np, nand_np) { struct atmel_nand *nand; nand = atmel_nand_create(nc, nand_np, reg_cells); diff --git a/drivers/mtd/nand/raw/atmel/pmecc.c b/drivers/mtd/nand/raw/atmel/pmecc.c index 3c7dee1be21df..0b402823b619c 100644 --- a/drivers/mtd/nand/raw/atmel/pmecc.c +++ b/drivers/mtd/nand/raw/atmel/pmecc.c @@ -143,6 +143,7 @@ struct atmel_pmecc_caps { int nstrengths; int el_offset; bool correct_erased_chunks; + bool clk_ctrl; }; struct atmel_pmecc { @@ -843,6 +844,10 @@ static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev, if (IS_ERR(pmecc->regs.errloc)) return ERR_CAST(pmecc->regs.errloc); + /* pmecc data setup time */ + if (caps->clk_ctrl) + writel(PMECC_CLK_133MHZ, pmecc->regs.base + ATMEL_PMECC_CLK); + /* Disable all interrupts before registering the PMECC handler. */ writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR); atmel_pmecc_reset(pmecc); @@ -896,6 +901,7 @@ static struct atmel_pmecc_caps at91sam9g45_caps = { .strengths = atmel_pmecc_strengths, .nstrengths = 5, .el_offset = 0x8c, + .clk_ctrl = true, }; static struct atmel_pmecc_caps sama5d4_caps = { diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index 4f37ca894d18a..202b4fc064fa3 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -2876,7 +2876,7 @@ cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl) static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl) { dma_cap_mask_t mask; - struct dma_device *dma_dev = cdns_ctrl->dmac->device; + struct dma_device *dma_dev; int ret; cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev, @@ -2920,6 +2920,7 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl) } } + dma_dev = cdns_ctrl->dmac->device; cdns_ctrl->io.iova_dma = dma_map_resource(dma_dev->dev, cdns_ctrl->io.dma, cdns_ctrl->io.size, DMA_BIDIRECTIONAL, 0); diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c index 811982da35574..b0a70badf3eb7 100644 --- a/drivers/mtd/nand/raw/fsmc_nand.c +++ b/drivers/mtd/nand/raw/fsmc_nand.c @@ -503,6 +503,8 @@ static int dma_xfer(struct fsmc_nand_data *host, void *buffer, int len, dma_dev = chan->device; dma_addr = dma_map_single(dma_dev->dev, buffer, len, direction); + if (dma_mapping_error(dma_dev->dev, dma_addr)) + return -EINVAL; if (direction == DMA_TO_DEVICE) { dma_src = dma_addr; @@ -874,10 +876,14 @@ static int fsmc_nand_probe_config_dt(struct platform_device *pdev, if (!of_property_read_u32(np, "bank-width", &val)) { if (val == 2) { nand->options |= NAND_BUSWIDTH_16; - } else if (val != 1) { + } else if (val == 1) { + nand->options |= NAND_BUSWIDTH_AUTO; + } else { dev_err(&pdev->dev, "invalid bank-width %u\n", val); return -EINVAL; } + } else { + nand->options |= NAND_BUSWIDTH_AUTO; } if (of_property_read_bool(np, "nand-skip-bbtscan")) diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c index b8cff9240b286..275d34119acdc 100644 --- a/drivers/mtd/nand/raw/qcom_nandc.c +++ b/drivers/mtd/nand/raw/qcom_nandc.c @@ -2858,7 +2858,12 @@ static int qcom_param_page_type_exec(struct nand_chip *chip, const struct nand_ const struct nand_op_instr *instr = NULL; unsigned int op_id = 0; unsigned int len = 0; - int ret; + int ret, reg_base; + + reg_base = NAND_READ_LOCATION_0; + + if (nandc->props->qpic_v2) + reg_base = NAND_READ_LOCATION_LAST_CW_0; ret = qcom_parse_instructions(chip, subop, &q_op); if (ret) @@ -2910,14 +2915,17 @@ static int qcom_param_page_type_exec(struct nand_chip *chip, const struct nand_ op_id = q_op.data_instr_idx; len = nand_subop_get_data_len(subop, op_id); - nandc_set_read_loc(chip, 0, 0, 0, len, 1); + if (nandc->props->qpic_v2) + nandc_set_read_loc_last(chip, reg_base, 0, len, 1); + else + nandc_set_read_loc_first(chip, reg_base, 0, len, 1); if (!nandc->props->qpic_v2) { write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0); write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL); } - nandc->buf_count = len; + nandc->buf_count = 512; memset(nandc->data_buffer, 0xff, nandc->buf_count); config_nand_single_cw_page_read(chip, false, 0); diff --git a/drivers/mtd/nand/raw/renesas-nand-controller.c b/drivers/mtd/nand/raw/renesas-nand-controller.c index 589021ea9eb2a..a9e79f0acbe2a 100644 --- a/drivers/mtd/nand/raw/renesas-nand-controller.c +++ b/drivers/mtd/nand/raw/renesas-nand-controller.c @@ -426,6 +426,9 @@ static int rnandc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf, /* Configure DMA */ dma_addr = dma_map_single(rnandc->dev, rnandc->buf, mtd->writesize, DMA_FROM_DEVICE); + if (dma_mapping_error(rnandc->dev, dma_addr)) + return -ENOMEM; + writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG); writel(mtd->writesize, rnandc->regs + DMA_CNT_REG); writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG); @@ -606,6 +609,9 @@ static int rnandc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf, /* Configure DMA */ dma_addr = dma_map_single(rnandc->dev, (void *)rnandc->buf, mtd->writesize, DMA_TO_DEVICE); + if (dma_mapping_error(rnandc->dev, dma_addr)) + return -ENOMEM; + writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG); writel(mtd->writesize, rnandc->regs + DMA_CNT_REG); writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG); diff --git a/drivers/mtd/nand/raw/rockchip-nand-controller.c b/drivers/mtd/nand/raw/rockchip-nand-controller.c index 2a95dd63b8c20..f68600ce5bfad 100644 --- a/drivers/mtd/nand/raw/rockchip-nand-controller.c +++ b/drivers/mtd/nand/raw/rockchip-nand-controller.c @@ -656,9 +656,16 @@ static int rk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf, dma_data = dma_map_single(nfc->dev, (void *)nfc->page_buf, mtd->writesize, DMA_TO_DEVICE); + if (dma_mapping_error(nfc->dev, dma_data)) + return -ENOMEM; + dma_oob = dma_map_single(nfc->dev, nfc->oob_buf, ecc->steps * oob_step, DMA_TO_DEVICE); + if (dma_mapping_error(nfc->dev, dma_oob)) { + dma_unmap_single(nfc->dev, dma_data, mtd->writesize, DMA_TO_DEVICE); + return -ENOMEM; + } reinit_completion(&nfc->done); writel(INT_DMA, nfc->regs + nfc->cfg->int_en_off); @@ -772,9 +779,17 @@ static int rk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *buf, int oob_on, dma_data = dma_map_single(nfc->dev, nfc->page_buf, mtd->writesize, DMA_FROM_DEVICE); + if (dma_mapping_error(nfc->dev, dma_data)) + return -ENOMEM; + dma_oob = dma_map_single(nfc->dev, nfc->oob_buf, ecc->steps * oob_step, DMA_FROM_DEVICE); + if (dma_mapping_error(nfc->dev, dma_oob)) { + dma_unmap_single(nfc->dev, dma_data, mtd->writesize, + DMA_FROM_DEVICE); + return -ENOMEM; + } /* * The first blocks (4, 8 or 16 depending on the device) diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c index 88811139aaf5b..c795629839717 100644 --- a/drivers/mtd/nand/raw/stm32_fmc2_nand.c +++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c @@ -263,6 +263,7 @@ struct stm32_fmc2_nfc { struct sg_table dma_data_sg; struct sg_table dma_ecc_sg; u8 *ecc_buf; + dma_addr_t dma_ecc_addr; int dma_ecc_len; struct completion complete; @@ -885,17 +886,10 @@ static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf, if (!write_data && !raw) { /* Configure DMA ECC status */ - p = nfc->ecc_buf; for_each_sg(nfc->dma_ecc_sg.sgl, sg, eccsteps, s) { - sg_set_buf(sg, p, nfc->dma_ecc_len); - p += nfc->dma_ecc_len; - } - - ret = dma_map_sg(nfc->dev, nfc->dma_ecc_sg.sgl, - eccsteps, dma_data_dir); - if (!ret) { - ret = -EIO; - goto err_unmap_data; + sg_dma_address(sg) = nfc->dma_ecc_addr + + s * nfc->dma_ecc_len; + sg_dma_len(sg) = nfc->dma_ecc_len; } desc_ecc = dmaengine_prep_slave_sg(nfc->dma_ecc_ch, @@ -904,7 +898,7 @@ static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf, DMA_PREP_INTERRUPT); if (!desc_ecc) { ret = -ENOMEM; - goto err_unmap_ecc; + goto err_unmap_data; } reinit_completion(&nfc->dma_ecc_complete); @@ -912,7 +906,7 @@ static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf, desc_ecc->callback_param = &nfc->dma_ecc_complete; ret = dma_submit_error(dmaengine_submit(desc_ecc)); if (ret) - goto err_unmap_ecc; + goto err_unmap_data; dma_async_issue_pending(nfc->dma_ecc_ch); } @@ -932,7 +926,7 @@ static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf, if (!write_data && !raw) dmaengine_terminate_all(nfc->dma_ecc_ch); ret = -ETIMEDOUT; - goto err_unmap_ecc; + goto err_unmap_data; } /* Wait DMA data transfer completion */ @@ -952,11 +946,6 @@ static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf, } } -err_unmap_ecc: - if (!write_data && !raw) - dma_unmap_sg(nfc->dev, nfc->dma_ecc_sg.sgl, - eccsteps, dma_data_dir); - err_unmap_data: dma_unmap_sg(nfc->dev, nfc->dma_data_sg.sgl, eccsteps, dma_data_dir); @@ -979,9 +968,21 @@ static int stm32_fmc2_nfc_seq_write(struct nand_chip *chip, const u8 *buf, /* Write oob */ if (oob_required) { - ret = nand_change_write_column_op(chip, mtd->writesize, - chip->oob_poi, mtd->oobsize, - false); + unsigned int offset_in_page = mtd->writesize; + const void *buf = chip->oob_poi; + unsigned int len = mtd->oobsize; + + if (!raw) { + struct mtd_oob_region oob_free; + + mtd_ooblayout_free(mtd, 0, &oob_free); + offset_in_page += oob_free.offset; + buf += oob_free.offset; + len = oob_free.length; + } + + ret = nand_change_write_column_op(chip, offset_in_page, + buf, len, false); if (ret) return ret; } @@ -1582,7 +1583,8 @@ static int stm32_fmc2_nfc_dma_setup(struct stm32_fmc2_nfc *nfc) return ret; /* Allocate a buffer to store ECC status registers */ - nfc->ecc_buf = devm_kzalloc(nfc->dev, FMC2_MAX_ECC_BUF_LEN, GFP_KERNEL); + nfc->ecc_buf = dmam_alloc_coherent(nfc->dev, FMC2_MAX_ECC_BUF_LEN, + &nfc->dma_ecc_addr, GFP_KERNEL); if (!nfc->ecc_buf) return -ENOMEM; diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index 9abf38049d35f..4469618ee6033 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -817,6 +817,7 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand, if (ret) return ret; + sunxi_nfc_randomizer_config(nand, page, false); sunxi_nfc_randomizer_enable(nand); writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, nfc->regs + NFC_REG_CMD); @@ -1049,6 +1050,7 @@ static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand, if (ret) return ret; + sunxi_nfc_randomizer_config(nand, page, false); sunxi_nfc_randomizer_enable(nand); sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page); diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 393ff37f0d23c..ee61b2d882320 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -624,7 +624,10 @@ static int spinand_write_page(struct spinand_device *spinand, SPINAND_WRITE_INITIAL_DELAY_US, SPINAND_WRITE_POLL_DELAY_US, &status); - if (!ret && (status & STATUS_PROG_FAILED)) + if (ret) + return ret; + + if (status & STATUS_PROG_FAILED) return -EIO; return nand_ecc_finish_io_req(nand, (struct nand_page_io_req *)req); @@ -1316,6 +1319,7 @@ static void spinand_cleanup(struct spinand_device *spinand) { struct nand_device *nand = spinand_to_nand(spinand); + nanddev_ecc_engine_cleanup(nand); nanddev_cleanup(nand); spinand_manufacturer_cleanup(spinand); kfree(spinand->databuf); diff --git a/drivers/mtd/spi-nor/swp.c b/drivers/mtd/spi-nor/swp.c index 5ab9d53248608..a186d1fde8694 100644 --- a/drivers/mtd/spi-nor/swp.c +++ b/drivers/mtd/spi-nor/swp.c @@ -50,7 +50,6 @@ static u64 spi_nor_get_min_prot_length_sr(struct spi_nor *nor) static void spi_nor_get_locked_range_sr(struct spi_nor *nor, u8 sr, loff_t *ofs, uint64_t *len) { - struct mtd_info *mtd = &nor->mtd; u64 min_prot_len; u8 mask = spi_nor_get_sr_bp_mask(nor); u8 tb_mask = spi_nor_get_sr_tb_mask(nor); @@ -71,13 +70,13 @@ static void spi_nor_get_locked_range_sr(struct spi_nor *nor, u8 sr, loff_t *ofs, min_prot_len = spi_nor_get_min_prot_length_sr(nor); *len = min_prot_len << (bp - 1); - if (*len > mtd->size) - *len = mtd->size; + if (*len > nor->params->size) + *len = nor->params->size; if (nor->flags & SNOR_F_HAS_SR_TB && sr & tb_mask) *ofs = 0; else - *ofs = mtd->size - *len; + *ofs = nor->params->size - *len; } /* @@ -153,7 +152,6 @@ static bool spi_nor_is_unlocked_sr(struct spi_nor *nor, loff_t ofs, */ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, uint64_t len) { - struct mtd_info *mtd = &nor->mtd; u64 min_prot_len; int ret, status_old, status_new; u8 mask = spi_nor_get_sr_bp_mask(nor); @@ -178,7 +176,7 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, uint64_t len) can_be_bottom = false; /* If anything above us is unlocked, we can't use 'top' protection */ - if (!spi_nor_is_locked_sr(nor, ofs + len, mtd->size - (ofs + len), + if (!spi_nor_is_locked_sr(nor, ofs + len, nor->params->size - (ofs + len), status_old)) can_be_top = false; @@ -190,11 +188,11 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, uint64_t len) /* lock_len: length of region that should end up locked */ if (use_top) - lock_len = mtd->size - ofs; + lock_len = nor->params->size - ofs; else lock_len = ofs + len; - if (lock_len == mtd->size) { + if (lock_len == nor->params->size) { val = mask; } else { min_prot_len = spi_nor_get_min_prot_length_sr(nor); @@ -243,7 +241,6 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, uint64_t len) */ static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len) { - struct mtd_info *mtd = &nor->mtd; u64 min_prot_len; int ret, status_old, status_new; u8 mask = spi_nor_get_sr_bp_mask(nor); @@ -268,7 +265,7 @@ static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len) can_be_top = false; /* If anything above us is locked, we can't use 'bottom' protection */ - if (!spi_nor_is_unlocked_sr(nor, ofs + len, mtd->size - (ofs + len), + if (!spi_nor_is_unlocked_sr(nor, ofs + len, nor->params->size - (ofs + len), status_old)) can_be_bottom = false; @@ -280,7 +277,7 @@ static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len) /* lock_len: length of region that should remain locked */ if (use_top) - lock_len = mtd->size - (ofs + len); + lock_len = nor->params->size - (ofs + len); else lock_len = ofs; diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c index c99ffe6c683a3..d02a91cefec89 100644 --- a/drivers/net/bonding/bond_3ad.c +++ b/drivers/net/bonding/bond_3ad.c @@ -99,13 +99,16 @@ static int ad_marker_send(struct port *port, struct bond_marker *marker); static void ad_mux_machine(struct port *port, bool *update_slave_arr); static void ad_rx_machine(struct lacpdu *lacpdu, struct port *port); static void ad_tx_machine(struct port *port); -static void ad_periodic_machine(struct port *port, struct bond_params *bond_params); +static void ad_periodic_machine(struct port *port); static void ad_port_selection_logic(struct port *port, bool *update_slave_arr); static void ad_agg_selection_logic(struct aggregator *aggregator, bool *update_slave_arr); static void ad_clear_agg(struct aggregator *aggregator); static void ad_initialize_agg(struct aggregator *aggregator); -static void ad_initialize_port(struct port *port, int lacp_fast); +static void ad_initialize_port(struct port *port, const struct bond_params *bond_params); +static void ad_enable_collecting(struct port *port); +static void ad_disable_distributing(struct port *port, + bool *update_slave_arr); static void ad_enable_collecting_distributing(struct port *port, bool *update_slave_arr); static void ad_disable_collecting_distributing(struct port *port, @@ -171,9 +174,38 @@ static inline int __agg_has_partner(struct aggregator *agg) return !is_zero_ether_addr(agg->partner_system.mac_addr_value); } +/** + * __disable_distributing_port - disable the port's slave for distributing. + * Port will still be able to collect. + * @port: the port we're looking at + * + * This will disable only distributing on the port's slave. + */ +static void __disable_distributing_port(struct port *port) +{ + bond_set_slave_tx_disabled_flags(port->slave, BOND_SLAVE_NOTIFY_LATER); +} + +/** + * __enable_collecting_port - enable the port's slave for collecting, + * if it's up + * @port: the port we're looking at + * + * This will enable only collecting on the port's slave. + */ +static void __enable_collecting_port(struct port *port) +{ + struct slave *slave = port->slave; + + if (slave->link == BOND_LINK_UP && bond_slave_is_up(slave)) + bond_set_slave_rx_enabled_flags(slave, BOND_SLAVE_NOTIFY_LATER); +} + /** * __disable_port - disable the port's slave * @port: the port we're looking at + * + * This will disable both collecting and distributing on the port's slave. */ static inline void __disable_port(struct port *port) { @@ -183,6 +215,8 @@ static inline void __disable_port(struct port *port) /** * __enable_port - enable the port's slave, if it's up * @port: the port we're looking at + * + * This will enable both collecting and distributing on the port's slave. */ static inline void __enable_port(struct port *port) { @@ -193,10 +227,27 @@ static inline void __enable_port(struct port *port) } /** - * __port_is_enabled - check if the port's slave is in active state + * __port_move_to_attached_state - check if port should transition back to attached + * state. * @port: the port we're looking at */ -static inline int __port_is_enabled(struct port *port) +static bool __port_move_to_attached_state(struct port *port) +{ + if (!(port->sm_vars & AD_PORT_SELECTED) || + (port->sm_vars & AD_PORT_STANDBY) || + !(port->partner_oper.port_state & LACP_STATE_SYNCHRONIZATION) || + !(port->actor_oper_port_state & LACP_STATE_SYNCHRONIZATION)) + port->sm_mux_state = AD_MUX_ATTACHED; + + return port->sm_mux_state == AD_MUX_ATTACHED; +} + +/** + * __port_is_collecting_distributing - check if the port's slave is in the + * combined collecting/distributing state + * @port: the port we're looking at + */ +static int __port_is_collecting_distributing(struct port *port) { return bond_is_active_slave(port->slave); } @@ -942,6 +993,7 @@ static int ad_marker_send(struct port *port, struct bond_marker *marker) */ static void ad_mux_machine(struct port *port, bool *update_slave_arr) { + struct bonding *bond = __get_bond_by_port(port); mux_states_t last_state; /* keep current State Machine state to compare later if it was @@ -999,9 +1051,13 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) if ((port->sm_vars & AD_PORT_SELECTED) && (port->partner_oper.port_state & LACP_STATE_SYNCHRONIZATION) && !__check_agg_selection_timer(port)) { - if (port->aggregator->is_active) - port->sm_mux_state = - AD_MUX_COLLECTING_DISTRIBUTING; + if (port->aggregator->is_active) { + int state = AD_MUX_COLLECTING_DISTRIBUTING; + + if (!bond->params.coupled_control) + state = AD_MUX_COLLECTING; + port->sm_mux_state = state; + } } else if (!(port->sm_vars & AD_PORT_SELECTED) || (port->sm_vars & AD_PORT_STANDBY)) { /* if UNSELECTED or STANDBY */ @@ -1019,11 +1075,45 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) } break; case AD_MUX_COLLECTING_DISTRIBUTING: + if (!__port_move_to_attached_state(port)) { + /* if port state hasn't changed make + * sure that a collecting distributing + * port in an active aggregator is enabled + */ + if (port->aggregator->is_active && + !__port_is_collecting_distributing(port)) { + __enable_port(port); + *update_slave_arr = true; + } + } + break; + case AD_MUX_COLLECTING: + if (!__port_move_to_attached_state(port)) { + if ((port->sm_vars & AD_PORT_SELECTED) && + (port->partner_oper.port_state & LACP_STATE_SYNCHRONIZATION) && + (port->partner_oper.port_state & LACP_STATE_COLLECTING)) { + port->sm_mux_state = AD_MUX_DISTRIBUTING; + } else { + /* If port state hasn't changed, make sure that a collecting + * port is enabled for an active aggregator. + */ + struct slave *slave = port->slave; + + if (port->aggregator->is_active && + bond_is_slave_rx_disabled(slave)) { + ad_enable_collecting(port); + *update_slave_arr = true; + } + } + } + break; + case AD_MUX_DISTRIBUTING: if (!(port->sm_vars & AD_PORT_SELECTED) || (port->sm_vars & AD_PORT_STANDBY) || + !(port->partner_oper.port_state & LACP_STATE_COLLECTING) || !(port->partner_oper.port_state & LACP_STATE_SYNCHRONIZATION) || !(port->actor_oper_port_state & LACP_STATE_SYNCHRONIZATION)) { - port->sm_mux_state = AD_MUX_ATTACHED; + port->sm_mux_state = AD_MUX_COLLECTING; } else { /* if port state hasn't changed make * sure that a collecting distributing @@ -1031,7 +1121,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) */ if (port->aggregator && port->aggregator->is_active && - !__port_is_enabled(port)) { + !__port_is_collecting_distributing(port)) { __enable_port(port); *update_slave_arr = true; } @@ -1082,6 +1172,20 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) update_slave_arr); port->ntt = true; break; + case AD_MUX_COLLECTING: + port->actor_oper_port_state |= LACP_STATE_COLLECTING; + port->actor_oper_port_state &= ~LACP_STATE_DISTRIBUTING; + port->actor_oper_port_state |= LACP_STATE_SYNCHRONIZATION; + ad_enable_collecting(port); + ad_disable_distributing(port, update_slave_arr); + port->ntt = true; + break; + case AD_MUX_DISTRIBUTING: + port->actor_oper_port_state |= LACP_STATE_DISTRIBUTING; + port->actor_oper_port_state |= LACP_STATE_SYNCHRONIZATION; + ad_enable_collecting_distributing(port, + update_slave_arr); + break; default: break; } @@ -1196,10 +1300,16 @@ static void ad_rx_machine(struct lacpdu *lacpdu, struct port *port) * case of EXPIRED even if LINK_DOWN didn't arrive for * the port. */ - port->partner_oper.port_state &= ~LACP_STATE_SYNCHRONIZATION; port->sm_vars &= ~AD_PORT_MATCHED; + /* Based on IEEE 8021AX-2014, Figure 6-18 - Receive + * machine state diagram, the statue should be + * Partner_Oper_Port_State.Synchronization = FALSE; + * Partner_Oper_Port_State.LACP_Timeout = Short Timeout; + * start current_while_timer(Short Timeout); + * Actor_Oper_Port_State.Expired = TRUE; + */ + port->partner_oper.port_state &= ~LACP_STATE_SYNCHRONIZATION; port->partner_oper.port_state |= LACP_STATE_LACP_TIMEOUT; - port->partner_oper.port_state |= LACP_STATE_LACP_ACTIVITY; port->sm_rx_timer_counter = __ad_timer_to_ticks(AD_CURRENT_WHILE_TIMER, (u16)(AD_SHORT_TIMEOUT)); port->actor_oper_port_state |= LACP_STATE_EXPIRED; port->sm_vars |= AD_PORT_CHURNED; @@ -1305,11 +1415,10 @@ static void ad_tx_machine(struct port *port) /** * ad_periodic_machine - handle a port's periodic state machine * @port: the port we're looking at - * @bond_params: bond parameters we will use * * Turn ntt flag on priodically to perform periodic transmission of lacpdu's. */ -static void ad_periodic_machine(struct port *port, struct bond_params *bond_params) +static void ad_periodic_machine(struct port *port) { periodic_states_t last_state; @@ -1318,8 +1427,7 @@ static void ad_periodic_machine(struct port *port, struct bond_params *bond_para /* check if port was reinitialized */ if (((port->sm_vars & AD_PORT_BEGIN) || !(port->sm_vars & AD_PORT_LACP_ENABLED) || !port->is_enabled) || - (!(port->actor_oper_port_state & LACP_STATE_LACP_ACTIVITY) && !(port->partner_oper.port_state & LACP_STATE_LACP_ACTIVITY)) || - !bond_params->lacp_active) { + (!(port->actor_oper_port_state & LACP_STATE_LACP_ACTIVITY) && !(port->partner_oper.port_state & LACP_STATE_LACP_ACTIVITY))) { port->sm_periodic_state = AD_NO_PERIODIC; } /* check if state machine should change state */ @@ -1843,16 +1951,16 @@ static void ad_initialize_agg(struct aggregator *aggregator) /** * ad_initialize_port - initialize a given port's parameters * @port: the port we're looking at - * @lacp_fast: boolean. whether fast periodic should be used + * @bond_params: bond parameters we will use */ -static void ad_initialize_port(struct port *port, int lacp_fast) +static void ad_initialize_port(struct port *port, const struct bond_params *bond_params) { static const struct port_params tmpl = { .system_priority = 0xffff, .key = 1, .port_number = 1, .port_priority = 0xff, - .port_state = 1, + .port_state = 0, }; static const struct lacpdu lacpdu = { .subtype = 0x01, @@ -1870,12 +1978,14 @@ static void ad_initialize_port(struct port *port, int lacp_fast) port->actor_port_priority = 0xff; port->actor_port_aggregator_identifier = 0; port->ntt = false; - port->actor_admin_port_state = LACP_STATE_AGGREGATION | - LACP_STATE_LACP_ACTIVITY; - port->actor_oper_port_state = LACP_STATE_AGGREGATION | - LACP_STATE_LACP_ACTIVITY; + port->actor_admin_port_state = LACP_STATE_AGGREGATION; + port->actor_oper_port_state = LACP_STATE_AGGREGATION; + if (bond_params->lacp_active) { + port->actor_admin_port_state |= LACP_STATE_LACP_ACTIVITY; + port->actor_oper_port_state |= LACP_STATE_LACP_ACTIVITY; + } - if (lacp_fast) + if (bond_params->lacp_fast) port->actor_oper_port_state |= LACP_STATE_LACP_TIMEOUT; memcpy(&port->partner_admin, &tmpl, sizeof(tmpl)); @@ -1906,6 +2016,45 @@ static void ad_initialize_port(struct port *port, int lacp_fast) } } +/** + * ad_enable_collecting - enable a port's receive + * @port: the port we're looking at + * + * Enable @port if it's in an active aggregator + */ +static void ad_enable_collecting(struct port *port) +{ + if (port->aggregator->is_active) { + struct slave *slave = port->slave; + + slave_dbg(slave->bond->dev, slave->dev, + "Enabling collecting on port %d (LAG %d)\n", + port->actor_port_number, + port->aggregator->aggregator_identifier); + __enable_collecting_port(port); + } +} + +/** + * ad_disable_distributing - disable a port's transmit + * @port: the port we're looking at + * @update_slave_arr: Does slave array need update? + */ +static void ad_disable_distributing(struct port *port, bool *update_slave_arr) +{ + if (port->aggregator && + !MAC_ADDRESS_EQUAL(&port->aggregator->partner_system, + &(null_mac_addr))) { + slave_dbg(port->slave->bond->dev, port->slave->dev, + "Disabling distributing on port %d (LAG %d)\n", + port->actor_port_number, + port->aggregator->aggregator_identifier); + __disable_distributing_port(port); + /* Slave array needs an update */ + *update_slave_arr = true; + } +} + /** * ad_enable_collecting_distributing - enable a port's transmit/receive * @port: the port we're looking at @@ -2052,7 +2201,7 @@ void bond_3ad_bind_slave(struct slave *slave) /* port initialization */ port = &(SLAVE_AD_INFO(slave)->port); - ad_initialize_port(port, bond->params.lacp_fast); + ad_initialize_port(port, &bond->params); port->slave = slave; port->actor_port_number = SLAVE_AD_INFO(slave)->id; @@ -2364,7 +2513,7 @@ void bond_3ad_state_machine_handler(struct work_struct *work) } ad_rx_machine(NULL, port); - ad_periodic_machine(port, &bond->params); + ad_periodic_machine(port); ad_port_selection_logic(port, &update_slave_arr); ad_mux_machine(port, &update_slave_arr); ad_tx_machine(port); @@ -2734,6 +2883,31 @@ void bond_3ad_update_lacp_rate(struct bonding *bond) spin_unlock_bh(&bond->mode_lock); } +/** + * bond_3ad_update_lacp_active - change the lacp active + * @bond: bonding struct + * + * Update actor_oper_port_state when lacp_active is modified. + */ +void bond_3ad_update_lacp_active(struct bonding *bond) +{ + struct port *port = NULL; + struct list_head *iter; + struct slave *slave; + int lacp_active; + + lacp_active = bond->params.lacp_active; + spin_lock_bh(&bond->mode_lock); + bond_for_each_slave(bond, slave, iter) { + port = &(SLAVE_AD_INFO(slave)->port); + if (lacp_active) + port->actor_oper_port_state |= LACP_STATE_LACP_ACTIVITY; + else + port->actor_oper_port_state &= ~LACP_STATE_LACP_ACTIVITY; + } + spin_unlock_bh(&bond->mode_lock); +} + size_t bond_3ad_stats_size(void) { return nla_total_size_64bit(sizeof(u64)) + /* BOND_3AD_STAT_LACPDU_RX */ diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 7eb62fe55947f..9aa328b958be4 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -2040,15 +2040,27 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, * set the master's mac address to that of the first slave */ memcpy(ss.__data, bond_dev->dev_addr, bond_dev->addr_len); - ss.ss_family = slave_dev->type; - res = dev_set_mac_address(slave_dev, (struct sockaddr *)&ss, - extack); - if (res) { - slave_err(bond_dev, slave_dev, "Error %d calling set_mac_address\n", res); - goto err_restore_mtu; - } + } else if (bond->params.fail_over_mac == BOND_FOM_FOLLOW && + BOND_MODE(bond) == BOND_MODE_ACTIVEBACKUP && + bond_has_slaves(bond) && + memcmp(slave_dev->dev_addr, bond_dev->dev_addr, bond_dev->addr_len) == 0) { + /* Set slave to random address to avoid duplicate mac + * address in later fail over. + */ + eth_random_addr(ss.__data); + } else { + goto skip_mac_set; + } + + ss.ss_family = slave_dev->type; + res = dev_set_mac_address(slave_dev, (struct sockaddr *)&ss, extack); + if (res) { + slave_err(bond_dev, slave_dev, "Error %d calling set_mac_address\n", res); + goto err_restore_mtu; } +skip_mac_set: + /* set no_addrconf flag before open to prevent IPv6 addrconf */ slave_dev->priv_flags |= IFF_NO_ADDRCONF; @@ -2469,7 +2481,7 @@ static int __bond_release_one(struct net_device *bond_dev, RCU_INIT_POINTER(bond->current_arp_slave, NULL); - if (!all && (!bond->params.fail_over_mac || + if (!all && (bond->params.fail_over_mac != BOND_FOM_ACTIVE || BOND_MODE(bond) != BOND_MODE_ACTIVEBACKUP)) { if (ether_addr_equal_64bits(bond_dev->dev_addr, slave->perm_hwaddr) && bond_has_slaves(bond)) @@ -2862,7 +2874,7 @@ static void bond_mii_monitor(struct work_struct *work) { struct bonding *bond = container_of(work, struct bonding, mii_work.work); - bool should_notify_peers = false; + bool should_notify_peers; bool commit; unsigned long delay; struct slave *slave; @@ -2874,30 +2886,33 @@ static void bond_mii_monitor(struct work_struct *work) goto re_arm; rcu_read_lock(); + should_notify_peers = bond_should_notify_peers(bond); commit = !!bond_miimon_inspect(bond); - if (bond->send_peer_notif) { - rcu_read_unlock(); - if (rtnl_trylock()) { - bond->send_peer_notif--; - rtnl_unlock(); - } - } else { - rcu_read_unlock(); - } - if (commit) { + rcu_read_unlock(); + + if (commit || bond->send_peer_notif) { /* Race avoidance with bond_close cancel of workqueue */ if (!rtnl_trylock()) { delay = 1; - should_notify_peers = false; goto re_arm; } - bond_for_each_slave(bond, slave, iter) { - bond_commit_link_state(slave, BOND_SLAVE_NOTIFY_LATER); + if (commit) { + bond_for_each_slave(bond, slave, iter) { + bond_commit_link_state(slave, + BOND_SLAVE_NOTIFY_LATER); + } + bond_miimon_commit(bond); + } + + if (bond->send_peer_notif) { + bond->send_peer_notif--; + if (should_notify_peers) + call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, + bond->dev); } - bond_miimon_commit(bond); rtnl_unlock(); /* might sleep, hold no other locks */ } @@ -2905,13 +2920,6 @@ static void bond_mii_monitor(struct work_struct *work) re_arm: if (bond->params.miimon) queue_delayed_work(bond->wq, &bond->mii_work, delay); - - if (should_notify_peers) { - if (!rtnl_trylock()) - return; - call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, bond->dev); - rtnl_unlock(); - } } static int bond_upper_dev_walk(struct net_device *upper, @@ -3249,7 +3257,6 @@ static void bond_ns_send_all(struct bonding *bond, struct slave *slave) /* Find out through which dev should the packet go */ memset(&fl6, 0, sizeof(struct flowi6)); fl6.daddr = targets[i]; - fl6.flowi6_oif = bond->dev->ifindex; dst = ip6_route_output(dev_net(bond->dev), NULL, &fl6); if (dst->error) { @@ -6388,6 +6395,7 @@ static int __init bond_check_params(struct bond_params *params) params->ad_actor_sys_prio = ad_actor_sys_prio; eth_zero_addr(params->ad_actor_system); params->ad_user_port_key = ad_user_port_key; + params->coupled_control = 1; if (packets_per_slave > 0) { params->reciprocal_packets_per_slave = reciprocal_value(packets_per_slave); diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 27cbe148f0db5..aebc814ad495d 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -122,6 +122,7 @@ static const struct nla_policy bond_policy[IFLA_BOND_MAX + 1] = { [IFLA_BOND_PEER_NOTIF_DELAY] = NLA_POLICY_FULL_RANGE(NLA_U32, &delay_range), [IFLA_BOND_MISSED_MAX] = { .type = NLA_U8 }, [IFLA_BOND_NS_IP6_TARGET] = { .type = NLA_NESTED }, + [IFLA_BOND_COUPLED_CONTROL] = { .type = NLA_U8 }, }; static const struct nla_policy bond_slave_policy[IFLA_BOND_SLAVE_MAX + 1] = { @@ -549,6 +550,16 @@ static int bond_changelink(struct net_device *bond_dev, struct nlattr *tb[], return err; } + if (data[IFLA_BOND_COUPLED_CONTROL]) { + int coupled_control = nla_get_u8(data[IFLA_BOND_COUPLED_CONTROL]); + + bond_opt_initval(&newval, coupled_control); + err = __bond_opt_set(bond, BOND_OPT_COUPLED_CONTROL, &newval, + data[IFLA_BOND_COUPLED_CONTROL], extack); + if (err) + return err; + } + return 0; } @@ -615,6 +626,7 @@ static size_t bond_get_size(const struct net_device *bond_dev) /* IFLA_BOND_NS_IP6_TARGET */ nla_total_size(sizeof(struct nlattr)) + nla_total_size(sizeof(struct in6_addr)) * BOND_MAX_NS_TARGETS + + nla_total_size(sizeof(u8)) + /* IFLA_BOND_COUPLED_CONTROL */ 0; } @@ -774,6 +786,10 @@ static int bond_fill_info(struct sk_buff *skb, bond->params.missed_max)) goto nla_put_failure; + if (nla_put_u8(skb, IFLA_BOND_COUPLED_CONTROL, + bond->params.coupled_control)) + goto nla_put_failure; + if (BOND_MODE(bond) == BOND_MODE_8023AD) { struct ad_info info; diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index 6d003c0ef6698..8291803e4f00a 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -85,7 +85,8 @@ static int bond_option_ad_user_port_key_set(struct bonding *bond, const struct bond_opt_value *newval); static int bond_option_missed_max_set(struct bonding *bond, const struct bond_opt_value *newval); - +static int bond_option_coupled_control_set(struct bonding *bond, + const struct bond_opt_value *newval); static const struct bond_opt_value bond_mode_tbl[] = { { "balance-rr", BOND_MODE_ROUNDROBIN, BOND_VALFLAG_DEFAULT}, @@ -233,6 +234,12 @@ static const struct bond_opt_value bond_missed_max_tbl[] = { { NULL, -1, 0}, }; +static const struct bond_opt_value bond_coupled_control_tbl[] = { + { "on", 1, BOND_VALFLAG_DEFAULT}, + { "off", 0, 0}, + { NULL, -1, 0}, +}; + static const struct bond_option bond_opts[BOND_OPT_LAST] = { [BOND_OPT_MODE] = { .id = BOND_OPT_MODE, @@ -497,6 +504,15 @@ static const struct bond_option bond_opts[BOND_OPT_LAST] = { .desc = "Delay between each peer notification on failover event, in milliseconds", .values = bond_peer_notif_delay_tbl, .set = bond_option_peer_notif_delay_set + }, + [BOND_OPT_COUPLED_CONTROL] = { + .id = BOND_OPT_COUPLED_CONTROL, + .name = "coupled_control", + .desc = "Opt into using coupled control MUX for LACP states", + .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)), + .flags = BOND_OPTFLAG_IFDOWN, + .values = bond_coupled_control_tbl, + .set = bond_option_coupled_control_set, } }; @@ -1618,6 +1634,7 @@ static int bond_option_lacp_active_set(struct bonding *bond, netdev_dbg(bond->dev, "Setting LACP active to %s (%llu)\n", newval->string, newval->value); bond->params.lacp_active = newval->value; + bond_3ad_update_lacp_active(bond); return 0; } @@ -1811,3 +1828,13 @@ static int bond_option_ad_user_port_key_set(struct bonding *bond, bond->params.ad_user_port_key = newval->value; return 0; } + +static int bond_option_coupled_control_set(struct bonding *bond, + const struct bond_opt_value *newval) +{ + netdev_info(bond->dev, "Setting coupled_control to %s (%llu)\n", + newval->string, newval->value); + + bond->params.coupled_control = newval->value; + return 0; +} diff --git a/drivers/net/can/bxcan.c b/drivers/net/can/bxcan.c index 49cf9682b9254..247d02447fc3f 100644 --- a/drivers/net/can/bxcan.c +++ b/drivers/net/can/bxcan.c @@ -842,7 +842,7 @@ static netdev_tx_t bxcan_start_xmit(struct sk_buff *skb, u32 id; int i, j; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; if (bxcan_tx_busy(priv)) diff --git a/drivers/net/can/c_can/c_can_platform.c b/drivers/net/can/c_can/c_can_platform.c index 7f405bcf11c23..603680792f1ff 100644 --- a/drivers/net/can/c_can/c_can_platform.c +++ b/drivers/net/can/c_can/c_can_platform.c @@ -333,7 +333,7 @@ static int c_can_plat_probe(struct platform_device *pdev) /* Check if we need custom RAMINIT via syscon. Mostly for TI * platforms. Only supported with DT boot. */ - if (np && of_property_read_bool(np, "syscon-raminit")) { + if (np && of_property_present(np, "syscon-raminit")) { u32 id; struct c_can_raminit *raminit = &priv->raminit_sys; diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index 7d9a7c92d4cf6..6018444a76b9b 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -125,13 +125,16 @@ void can_change_state(struct net_device *dev, struct can_frame *cf, EXPORT_SYMBOL_GPL(can_change_state); /* CAN device restart for bus-off recovery */ -static void can_restart(struct net_device *dev) +static int can_restart(struct net_device *dev) { struct can_priv *priv = netdev_priv(dev); struct sk_buff *skb; struct can_frame *cf; int err; + if (!priv->do_set_mode) + return -EOPNOTSUPP; + if (netif_carrier_ok(dev)) netdev_err(dev, "Attempt to restart for bus-off recovery, but carrier is OK?\n"); @@ -142,24 +145,25 @@ static void can_restart(struct net_device *dev) /* send restart message upstream */ skb = alloc_can_err_skb(dev, &cf); - if (!skb) - goto restart; - - cf->can_id |= CAN_ERR_RESTARTED; - - netif_rx(skb); - -restart: - netdev_dbg(dev, "restarted\n"); - priv->can_stats.restarts++; + if (skb) { + cf->can_id |= CAN_ERR_RESTARTED; + netif_rx(skb); + } /* Now restart the device */ netif_carrier_on(dev); err = priv->do_set_mode(dev, CAN_MODE_START); if (err) { - netdev_err(dev, "Error %d during restart", err); + netdev_err(dev, "Restart failed, error %pe\n", ERR_PTR(err)); netif_carrier_off(dev); + + return err; + } else { + netdev_dbg(dev, "Restarted\n"); + priv->can_stats.restarts++; } + + return 0; } static void can_restart_work(struct work_struct *work) @@ -184,9 +188,8 @@ int can_restart_now(struct net_device *dev) return -EBUSY; cancel_delayed_work_sync(&priv->restart_work); - can_restart(dev); - return 0; + return can_restart(dev); } /* CAN bus-off diff --git a/drivers/net/can/dev/netlink.c b/drivers/net/can/dev/netlink.c index 01aacdcda2606..77d165ed0d534 100644 --- a/drivers/net/can/dev/netlink.c +++ b/drivers/net/can/dev/netlink.c @@ -285,13 +285,27 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[], } if (data[IFLA_CAN_RESTART_MS]) { + unsigned int restart_ms = nla_get_u32(data[IFLA_CAN_RESTART_MS]); + + if (restart_ms != 0 && !priv->do_set_mode) { + NL_SET_ERR_MSG(extack, + "Device doesn't support restart from Bus Off"); + return -EOPNOTSUPP; + } + /* Do not allow changing restart delay while running */ if (dev->flags & IFF_UP) return -EBUSY; - priv->restart_ms = nla_get_u32(data[IFLA_CAN_RESTART_MS]); + priv->restart_ms = restart_ms; } if (data[IFLA_CAN_RESTART]) { + if (!priv->do_set_mode) { + NL_SET_ERR_MSG(extack, + "Device doesn't support restart from Bus Off"); + return -EOPNOTSUPP; + } + /* Do not allow a restart while not running */ if (!(dev->flags & IFF_UP)) return -EINVAL; diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c index c490b4ba065ba..0b74b79b08f0a 100644 --- a/drivers/net/can/kvaser_pciefd.c +++ b/drivers/net/can/kvaser_pciefd.c @@ -927,6 +927,7 @@ static int kvaser_pciefd_setup_can_ctrls(struct kvaser_pciefd *pcie) can->err_rep_cnt = 0; can->bec.txerr = 0; can->bec.rxerr = 0; + can->can.dev->dev_port = i; init_completion(&can->start_comp); init_completion(&can->flush_comp); @@ -1137,7 +1138,7 @@ static int kvaser_pciefd_handle_data_packet(struct kvaser_pciefd *pcie, skb = alloc_canfd_skb(priv->dev, &cf); if (!skb) { priv->dev->stats.rx_dropped++; - return -ENOMEM; + return 0; } cf->len = can_fd_dlc2len(dlc); @@ -1149,7 +1150,7 @@ static int kvaser_pciefd_handle_data_packet(struct kvaser_pciefd *pcie, skb = alloc_can_skb(priv->dev, (struct can_frame **)&cf); if (!skb) { priv->dev->stats.rx_dropped++; - return -ENOMEM; + return 0; } can_frame_set_cc_len((struct can_frame *)cf, dlc, priv->ctrlmode); } @@ -1167,7 +1168,9 @@ static int kvaser_pciefd_handle_data_packet(struct kvaser_pciefd *pcie, priv->dev->stats.rx_packets++; kvaser_pciefd_set_skb_timestamp(pcie, skb, p->timestamp); - return netif_rx(skb); + netif_rx(skb); + + return 0; } static void kvaser_pciefd_change_state(struct kvaser_pciefd_can *can, @@ -1580,24 +1583,28 @@ static int kvaser_pciefd_read_buffer(struct kvaser_pciefd *pcie, int dma_buf) return res; } -static u32 kvaser_pciefd_receive_irq(struct kvaser_pciefd *pcie) +static void kvaser_pciefd_receive_irq(struct kvaser_pciefd *pcie) { + void __iomem *srb_cmd_reg = KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG; u32 irq = ioread32(KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IRQ_REG); - if (irq & KVASER_PCIEFD_SRB_IRQ_DPD0) + iowrite32(irq, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IRQ_REG); + + if (irq & KVASER_PCIEFD_SRB_IRQ_DPD0) { kvaser_pciefd_read_buffer(pcie, 0); + iowrite32(KVASER_PCIEFD_SRB_CMD_RDB0, srb_cmd_reg); /* Rearm buffer */ + } - if (irq & KVASER_PCIEFD_SRB_IRQ_DPD1) + if (irq & KVASER_PCIEFD_SRB_IRQ_DPD1) { kvaser_pciefd_read_buffer(pcie, 1); + iowrite32(KVASER_PCIEFD_SRB_CMD_RDB1, srb_cmd_reg); /* Rearm buffer */ + } if (irq & KVASER_PCIEFD_SRB_IRQ_DOF0 || irq & KVASER_PCIEFD_SRB_IRQ_DOF1 || irq & KVASER_PCIEFD_SRB_IRQ_DUF0 || irq & KVASER_PCIEFD_SRB_IRQ_DUF1) dev_err(&pcie->pci->dev, "DMA IRQ error 0x%08X\n", irq); - - iowrite32(irq, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IRQ_REG); - return irq; } static void kvaser_pciefd_transmit_irq(struct kvaser_pciefd_can *can) @@ -1625,29 +1632,22 @@ static irqreturn_t kvaser_pciefd_irq_handler(int irq, void *dev) struct kvaser_pciefd *pcie = (struct kvaser_pciefd *)dev; const struct kvaser_pciefd_irq_mask *irq_mask = pcie->driver_data->irq_mask; u32 pci_irq = ioread32(KVASER_PCIEFD_PCI_IRQ_ADDR(pcie)); - u32 srb_irq = 0; - u32 srb_release = 0; int i; if (!(pci_irq & irq_mask->all)) return IRQ_NONE; + iowrite32(0, KVASER_PCIEFD_PCI_IEN_ADDR(pcie)); + if (pci_irq & irq_mask->kcan_rx0) - srb_irq = kvaser_pciefd_receive_irq(pcie); + kvaser_pciefd_receive_irq(pcie); for (i = 0; i < pcie->nr_channels; i++) { if (pci_irq & irq_mask->kcan_tx[i]) kvaser_pciefd_transmit_irq(pcie->can[i]); } - if (srb_irq & KVASER_PCIEFD_SRB_IRQ_DPD0) - srb_release |= KVASER_PCIEFD_SRB_CMD_RDB0; - - if (srb_irq & KVASER_PCIEFD_SRB_IRQ_DPD1) - srb_release |= KVASER_PCIEFD_SRB_CMD_RDB1; - - if (srb_release) - iowrite32(srb_release, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG); + iowrite32(irq_mask->all, KVASER_PCIEFD_PCI_IEN_ADDR(pcie)); return IRQ_HANDLED; } @@ -1667,13 +1667,22 @@ static void kvaser_pciefd_teardown_can_ctrls(struct kvaser_pciefd *pcie) } } +static void kvaser_pciefd_disable_irq_srcs(struct kvaser_pciefd *pcie) +{ + unsigned int i; + + /* Masking PCI_IRQ is insufficient as running ISR will unmask it */ + iowrite32(0, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IEN_REG); + for (i = 0; i < pcie->nr_channels; ++i) + iowrite32(0, pcie->can[i]->reg_base + KVASER_PCIEFD_KCAN_IEN_REG); +} + static int kvaser_pciefd_probe(struct pci_dev *pdev, const struct pci_device_id *id) { int err; struct kvaser_pciefd *pcie; const struct kvaser_pciefd_irq_mask *irq_mask; - void __iomem *irq_en_base; pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL); if (!pcie) @@ -1726,8 +1735,7 @@ static int kvaser_pciefd_probe(struct pci_dev *pdev, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_IEN_REG); /* Enable PCI interrupts */ - irq_en_base = KVASER_PCIEFD_PCI_IEN_ADDR(pcie); - iowrite32(irq_mask->all, irq_en_base); + iowrite32(irq_mask->all, KVASER_PCIEFD_PCI_IEN_ADDR(pcie)); /* Ready the DMA buffers */ iowrite32(KVASER_PCIEFD_SRB_CMD_RDB0, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CMD_REG); @@ -1741,8 +1749,7 @@ static int kvaser_pciefd_probe(struct pci_dev *pdev, return 0; err_free_irq: - /* Disable PCI interrupts */ - iowrite32(0, irq_en_base); + kvaser_pciefd_disable_irq_srcs(pcie); free_irq(pcie->pci->irq, pcie); err_teardown_can_ctrls: @@ -1762,35 +1769,25 @@ static int kvaser_pciefd_probe(struct pci_dev *pdev, return err; } -static void kvaser_pciefd_remove_all_ctrls(struct kvaser_pciefd *pcie) -{ - int i; - - for (i = 0; i < pcie->nr_channels; i++) { - struct kvaser_pciefd_can *can = pcie->can[i]; - - if (can) { - iowrite32(0, can->reg_base + KVASER_PCIEFD_KCAN_IEN_REG); - unregister_candev(can->can.dev); - del_timer(&can->bec_poll_timer); - kvaser_pciefd_pwm_stop(can); - free_candev(can->can.dev); - } - } -} - static void kvaser_pciefd_remove(struct pci_dev *pdev) { struct kvaser_pciefd *pcie = pci_get_drvdata(pdev); + unsigned int i; - kvaser_pciefd_remove_all_ctrls(pcie); + for (i = 0; i < pcie->nr_channels; ++i) { + struct kvaser_pciefd_can *can = pcie->can[i]; - /* Disable interrupts */ - iowrite32(0, KVASER_PCIEFD_SRB_ADDR(pcie) + KVASER_PCIEFD_SRB_CTRL_REG); - iowrite32(0, KVASER_PCIEFD_PCI_IEN_ADDR(pcie)); + unregister_candev(can->can.dev); + del_timer(&can->bec_poll_timer); + kvaser_pciefd_pwm_stop(can); + } + kvaser_pciefd_disable_irq_srcs(pcie); free_irq(pcie->pci->irq, pcie); + for (i = 0; i < pcie->nr_channels; ++i) + free_candev(pcie->can[i]->can.dev); + pci_iounmap(pdev, pcie->reg_base); pci_release_regions(pdev); pci_disable_device(pdev); diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index 2a258986eed02..e6a74d66f0d8c 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -606,7 +606,7 @@ static int m_can_handle_lost_msg(struct net_device *dev) struct can_frame *frame; u32 timestamp = 0; - netdev_err(dev, "msg lost in rxf0\n"); + netdev_dbg(dev, "msg lost in rxf0\n"); stats->rx_errors++; stats->rx_over_errors++; @@ -2125,9 +2125,9 @@ EXPORT_SYMBOL_GPL(m_can_class_register); void m_can_class_unregister(struct m_can_classdev *cdev) { + unregister_candev(cdev->net); if (cdev->is_peripheral) can_rx_offload_del(&cdev->offload); - unregister_candev(cdev->net); } EXPORT_SYMBOL_GPL(m_can_class_unregister); diff --git a/drivers/net/can/m_can/m_can_platform.c b/drivers/net/can/m_can/m_can_platform.c index cdb28d6a092c6..e5477775992e6 100644 --- a/drivers/net/can/m_can/m_can_platform.c +++ b/drivers/net/can/m_can/m_can_platform.c @@ -183,7 +183,7 @@ static void m_can_plat_remove(struct platform_device *pdev) struct m_can_classdev *mcan_class = &priv->cdev; m_can_class_unregister(mcan_class); - + pm_runtime_disable(mcan_class->dev); m_can_class_free_dev(mcan_class->net); } diff --git a/drivers/net/can/m_can/tcan4x5x-core.c b/drivers/net/can/m_can/tcan4x5x-core.c index ae8c42f5debd4..4d440ecfc211c 100644 --- a/drivers/net/can/m_can/tcan4x5x-core.c +++ b/drivers/net/can/m_can/tcan4x5x-core.c @@ -385,10 +385,11 @@ static int tcan4x5x_can_probe(struct spi_device *spi) priv = cdev_to_priv(mcan_class); priv->power = devm_regulator_get_optional(&spi->dev, "vsup"); - if (PTR_ERR(priv->power) == -EPROBE_DEFER) { - ret = -EPROBE_DEFER; - goto out_m_can_class_free_dev; - } else { + if (IS_ERR(priv->power)) { + if (PTR_ERR(priv->power) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto out_m_can_class_free_dev; + } priv->power = NULL; } diff --git a/drivers/net/can/rcar/rcar_can.c b/drivers/net/can/rcar/rcar_can.c index f5aa5dbacaf21..1f26aba620b98 100644 --- a/drivers/net/can/rcar/rcar_can.c +++ b/drivers/net/can/rcar/rcar_can.c @@ -861,7 +861,6 @@ static int __maybe_unused rcar_can_resume(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct rcar_can_priv *priv = netdev_priv(ndev); - u16 ctlr; int err; if (!netif_running(ndev)) @@ -873,12 +872,7 @@ static int __maybe_unused rcar_can_resume(struct device *dev) return err; } - ctlr = readw(&priv->regs->ctlr); - ctlr &= ~RCAR_CAN_CTLR_SLPM; - writew(ctlr, &priv->regs->ctlr); - ctlr &= ~RCAR_CAN_CTLR_CANM; - writew(ctlr, &priv->regs->ctlr); - priv->can.state = CAN_STATE_ERROR_ACTIVE; + rcar_can_start(ndev); netif_device_attach(ndev); netif_start_queue(ndev); diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index e501b55678d1d..ae4ebcee60779 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -738,9 +738,6 @@ static int rcar_canfd_reset_controller(struct rcar_canfd_global *gpriv) /* Reset Global error flags */ rcar_canfd_write(gpriv->base, RCANFD_GERFL, 0x0); - /* Set the controller into appropriate mode */ - rcar_canfd_set_mode(gpriv); - /* Transition all Channels to reset mode */ for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { rcar_canfd_clear_bit(gpriv->base, @@ -760,6 +757,10 @@ static int rcar_canfd_reset_controller(struct rcar_canfd_global *gpriv) return err; } } + + /* Set the controller into appropriate mode */ + rcar_canfd_set_mode(gpriv); + return 0; } diff --git a/drivers/net/can/slcan/slcan-core.c b/drivers/net/can/slcan/slcan-core.c index 24c6622d36bd8..58ff2ec1d9757 100644 --- a/drivers/net/can/slcan/slcan-core.c +++ b/drivers/net/can/slcan/slcan-core.c @@ -71,12 +71,21 @@ MODULE_AUTHOR("Dario Binacchi "); #define SLCAN_CMD_LEN 1 #define SLCAN_SFF_ID_LEN 3 #define SLCAN_EFF_ID_LEN 8 +#define SLCAN_DATA_LENGTH_LEN 1 +#define SLCAN_ERROR_LEN 1 #define SLCAN_STATE_LEN 1 #define SLCAN_STATE_BE_RXCNT_LEN 3 #define SLCAN_STATE_BE_TXCNT_LEN 3 -#define SLCAN_STATE_FRAME_LEN (1 + SLCAN_CMD_LEN + \ - SLCAN_STATE_BE_RXCNT_LEN + \ - SLCAN_STATE_BE_TXCNT_LEN) +#define SLCAN_STATE_MSG_LEN (SLCAN_CMD_LEN + \ + SLCAN_STATE_LEN + \ + SLCAN_STATE_BE_RXCNT_LEN + \ + SLCAN_STATE_BE_TXCNT_LEN) +#define SLCAN_ERROR_MSG_LEN_MIN (SLCAN_CMD_LEN + \ + SLCAN_ERROR_LEN + \ + SLCAN_DATA_LENGTH_LEN) +#define SLCAN_FRAME_MSG_LEN_MIN (SLCAN_CMD_LEN + \ + SLCAN_SFF_ID_LEN + \ + SLCAN_DATA_LENGTH_LEN) struct slcan { struct can_priv can; @@ -176,6 +185,9 @@ static void slcan_bump_frame(struct slcan *sl) u32 tmpid; char *cmd = sl->rbuff; + if (sl->rcount < SLCAN_FRAME_MSG_LEN_MIN) + return; + skb = alloc_can_skb(sl->dev, &cf); if (unlikely(!skb)) { sl->dev->stats.rx_dropped++; @@ -281,7 +293,7 @@ static void slcan_bump_state(struct slcan *sl) return; } - if (state == sl->can.state || sl->rcount < SLCAN_STATE_FRAME_LEN) + if (state == sl->can.state || sl->rcount != SLCAN_STATE_MSG_LEN) return; cmd += SLCAN_STATE_BE_RXCNT_LEN + SLCAN_CMD_LEN + 1; @@ -328,6 +340,9 @@ static void slcan_bump_err(struct slcan *sl) bool rx_errors = false, tx_errors = false, rx_over_errors = false; int i, len; + if (sl->rcount < SLCAN_ERROR_MSG_LEN_MIN) + return; + /* get len from sanitized ASCII value */ len = cmd[1]; if (len >= '0' && len < '9') @@ -456,8 +471,7 @@ static void slcan_bump(struct slcan *sl) static void slcan_unesc(struct slcan *sl, unsigned char s) { if ((s == '\r') || (s == '\a')) { /* CR or BEL ends the pdu */ - if (!test_and_clear_bit(SLF_ERROR, &sl->flags) && - sl->rcount > 4) + if (!test_and_clear_bit(SLF_ERROR, &sl->flags)) slcan_bump(sl); sl->rcount = 0; diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index b757555ed4c4f..1acd4fc7adc8b 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -545,8 +545,6 @@ static int hi3110_stop(struct net_device *net) priv->force_quit = 1; free_irq(spi->irq, priv); - destroy_workqueue(priv->wq); - priv->wq = NULL; mutex_lock(&priv->hi3110_lock); @@ -771,34 +769,23 @@ static int hi3110_open(struct net_device *net) goto out_close; } - priv->wq = alloc_workqueue("hi3110_wq", WQ_FREEZABLE | WQ_MEM_RECLAIM, - 0); - if (!priv->wq) { - ret = -ENOMEM; - goto out_free_irq; - } - INIT_WORK(&priv->tx_work, hi3110_tx_work_handler); - INIT_WORK(&priv->restart_work, hi3110_restart_work_handler); - ret = hi3110_hw_reset(spi); if (ret) - goto out_free_wq; + goto out_free_irq; ret = hi3110_setup(net); if (ret) - goto out_free_wq; + goto out_free_irq; ret = hi3110_set_normal_mode(spi); if (ret) - goto out_free_wq; + goto out_free_irq; netif_wake_queue(net); mutex_unlock(&priv->hi3110_lock); return 0; - out_free_wq: - destroy_workqueue(priv->wq); out_free_irq: free_irq(spi->irq, priv); hi3110_hw_sleep(spi); @@ -813,6 +800,7 @@ static const struct net_device_ops hi3110_netdev_ops = { .ndo_open = hi3110_open, .ndo_stop = hi3110_stop, .ndo_start_xmit = hi3110_hard_start_xmit, + .ndo_change_mtu = can_change_mtu, }; static const struct ethtool_ops hi3110_ethtool_ops = { @@ -914,6 +902,15 @@ static int hi3110_can_probe(struct spi_device *spi) if (ret) goto out_clk; + priv->wq = alloc_workqueue("hi3110_wq", WQ_FREEZABLE | WQ_MEM_RECLAIM, + 0); + if (!priv->wq) { + ret = -ENOMEM; + goto out_clk; + } + INIT_WORK(&priv->tx_work, hi3110_tx_work_handler); + INIT_WORK(&priv->restart_work, hi3110_restart_work_handler); + priv->spi = spi; mutex_init(&priv->hi3110_lock); @@ -949,6 +946,8 @@ static int hi3110_can_probe(struct spi_device *spi) return 0; error_probe: + destroy_workqueue(priv->wq); + priv->wq = NULL; hi3110_power_enable(priv->power, 0); out_clk: @@ -969,6 +968,9 @@ static void hi3110_can_remove(struct spi_device *spi) hi3110_power_enable(priv->power, 0); + destroy_workqueue(priv->wq); + priv->wq = NULL; + clk_disable_unprepare(priv->clk); free_candev(net); diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c index 6fecfe4cd0804..21ae3a89924e9 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c @@ -75,6 +75,24 @@ static const struct can_bittiming_const mcp251xfd_data_bittiming_const = { .brp_inc = 1, }; +/* The datasheet of the mcp2518fd (DS20006027B) specifies a range of + * [-64,63] for TDCO, indicating a relative TDCO. + * + * Manual tests have shown, that using a relative TDCO configuration + * results in bus off, while an absolute configuration works. + * + * For TDCO use the max value (63) from the data sheet, but 0 as the + * minimum. + */ +static const struct can_tdc_const mcp251xfd_tdc_const = { + .tdcv_min = 0, + .tdcv_max = 63, + .tdco_min = 0, + .tdco_max = 63, + .tdcf_min = 0, + .tdcf_max = 0, +}; + static const char *__mcp251xfd_get_model_str(enum mcp251xfd_model model) { switch (model) { @@ -510,8 +528,7 @@ static int mcp251xfd_set_bittiming(const struct mcp251xfd_priv *priv) { const struct can_bittiming *bt = &priv->can.bittiming; const struct can_bittiming *dbt = &priv->can.data_bittiming; - u32 val = 0; - s8 tdco; + u32 tdcmod, val = 0; int err; /* CAN Control Register @@ -575,11 +592,16 @@ static int mcp251xfd_set_bittiming(const struct mcp251xfd_priv *priv) return err; /* Transmitter Delay Compensation */ - tdco = clamp_t(int, dbt->brp * (dbt->prop_seg + dbt->phase_seg1), - -64, 63); - val = FIELD_PREP(MCP251XFD_REG_TDC_TDCMOD_MASK, - MCP251XFD_REG_TDC_TDCMOD_AUTO) | - FIELD_PREP(MCP251XFD_REG_TDC_TDCO_MASK, tdco); + if (priv->can.ctrlmode & CAN_CTRLMODE_TDC_AUTO) + tdcmod = MCP251XFD_REG_TDC_TDCMOD_AUTO; + else if (priv->can.ctrlmode & CAN_CTRLMODE_TDC_MANUAL) + tdcmod = MCP251XFD_REG_TDC_TDCMOD_MANUAL; + else + tdcmod = MCP251XFD_REG_TDC_TDCMOD_DISABLED; + + val = FIELD_PREP(MCP251XFD_REG_TDC_TDCMOD_MASK, tdcmod) | + FIELD_PREP(MCP251XFD_REG_TDC_TDCV_MASK, priv->can.tdc.tdcv) | + FIELD_PREP(MCP251XFD_REG_TDC_TDCO_MASK, priv->can.tdc.tdco); return regmap_write(priv->map_reg, MCP251XFD_REG_TDC, val); } @@ -2083,10 +2105,12 @@ static int mcp251xfd_probe(struct spi_device *spi) priv->can.do_get_berr_counter = mcp251xfd_get_berr_counter; priv->can.bittiming_const = &mcp251xfd_bittiming_const; priv->can.data_bittiming_const = &mcp251xfd_data_bittiming_const; + priv->can.tdc_const = &mcp251xfd_tdc_const; priv->can.ctrlmode_supported = CAN_CTRLMODE_LOOPBACK | CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_BERR_REPORTING | CAN_CTRLMODE_FD | CAN_CTRLMODE_FD_NON_ISO | - CAN_CTRLMODE_CC_LEN8_DLC; + CAN_CTRLMODE_CC_LEN8_DLC | CAN_CTRLMODE_TDC_AUTO | + CAN_CTRLMODE_TDC_MANUAL; set_bit(MCP251XFD_FLAGS_DOWN, priv->flags); priv->ndev = ndev; priv->spi = spi; @@ -2179,8 +2203,8 @@ static void mcp251xfd_remove(struct spi_device *spi) struct mcp251xfd_priv *priv = spi_get_drvdata(spi); struct net_device *ndev = priv->ndev; - can_rx_offload_del(&priv->offload); mcp251xfd_unregister(priv); + can_rx_offload_del(&priv->offload); spi->max_speed_hz = priv->spi_max_speed_hz_orig; free_candev(ndev); } diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index 283fbf59e66d5..5ab1f9c7288e5 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -768,6 +768,7 @@ static const struct net_device_ops sun4ican_netdev_ops = { .ndo_open = sun4ican_open, .ndo_stop = sun4ican_close, .ndo_start_xmit = sun4ican_start_xmit, + .ndo_change_mtu = can_change_mtu, }; static const struct ethtool_ops sun4ican_ethtool_ops = { diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c index 5e3a72b7c4691..bb49a2c0a9a5c 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.c +++ b/drivers/net/can/usb/etas_es58x/es58x_core.c @@ -7,7 +7,7 @@ * * Copyright (c) 2019 Robert Bosch Engineering and Business Solutions. All rights reserved. * Copyright (c) 2020 ETAS K.K.. All rights reserved. - * Copyright (c) 2020-2022 Vincent Mailhol + * Copyright (c) 2020-2025 Vincent Mailhol */ #include @@ -1977,6 +1977,7 @@ static const struct net_device_ops es58x_netdev_ops = { .ndo_stop = es58x_stop, .ndo_start_xmit = es58x_start_xmit, .ndo_eth_ioctl = can_eth_ioctl_hwts, + .ndo_change_mtu = can_change_mtu, }; static const struct ethtool_ops es58x_ethtool_ops = { diff --git a/drivers/net/can/usb/etas_es58x/es58x_devlink.c b/drivers/net/can/usb/etas_es58x/es58x_devlink.c index e763a9904bedd..0d155eb1b9e99 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_devlink.c +++ b/drivers/net/can/usb/etas_es58x/es58x_devlink.c @@ -215,7 +215,7 @@ static int es58x_devlink_info_get(struct devlink *devlink, struct es58x_sw_version *fw_ver = &es58x_dev->firmware_version; struct es58x_sw_version *bl_ver = &es58x_dev->bootloader_version; struct es58x_hw_revision *hw_rev = &es58x_dev->hardware_revision; - char buf[max(sizeof("xx.xx.xx"), sizeof("axxx/xxx"))]; + char buf[MAX(sizeof("xx.xx.xx"), sizeof("axxx/xxx"))]; int ret = 0; if (es58x_sw_version_is_valid(fw_ver)) { diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index de616d6589c0b..9bd61fd8e5013 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -286,11 +286,6 @@ struct gs_host_frame { #define GS_MAX_RX_URBS 30 #define GS_NAPI_WEIGHT 32 -/* Maximum number of interfaces the driver supports per device. - * Current hardware only supports 3 interfaces. The future may vary. - */ -#define GS_MAX_INTF 3 - struct gs_tx_context { struct gs_can *dev; unsigned int echo_id; @@ -321,7 +316,6 @@ struct gs_can { /* usb interface struct */ struct gs_usb { - struct gs_can *canch[GS_MAX_INTF]; struct usb_anchor rx_submitted; struct usb_device *udev; @@ -333,9 +327,11 @@ struct gs_usb { unsigned int hf_size_rx; u8 active_channels; + u8 channel_cnt; unsigned int pipe_in; unsigned int pipe_out; + struct gs_can *canch[] __counted_by(channel_cnt); }; /* 'allocate' a tx context. @@ -596,7 +592,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) } /* device reports out of range channel id */ - if (hf->channel >= GS_MAX_INTF) + if (hf->channel >= parent->channel_cnt) goto device_detach; dev = parent->canch[hf->channel]; @@ -696,7 +692,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) /* USB failure take down all interfaces */ if (rc == -ENODEV) { device_detach: - for (rc = 0; rc < GS_MAX_INTF; rc++) { + for (rc = 0; rc < parent->channel_cnt; rc++) { if (parent->canch[rc]) netif_device_detach(parent->canch[rc]->netdev); } @@ -1246,6 +1242,7 @@ static struct gs_can *gs_make_candev(unsigned int channel, netdev->flags |= IFF_ECHO; /* we support full roundtrip echo */ netdev->dev_id = channel; + netdev->dev_port = channel; /* dev setup */ strcpy(dev->bt_const.name, KBUILD_MODNAME); @@ -1457,17 +1454,19 @@ static int gs_usb_probe(struct usb_interface *intf, icount = dconf.icount + 1; dev_info(&intf->dev, "Configuring for %u interfaces\n", icount); - if (icount > GS_MAX_INTF) { + if (icount > type_max(parent->channel_cnt)) { dev_err(&intf->dev, "Driver cannot handle more that %u CAN interfaces\n", - GS_MAX_INTF); + type_max(parent->channel_cnt)); return -EINVAL; } - parent = kzalloc(sizeof(*parent), GFP_KERNEL); + parent = kzalloc(struct_size(parent, canch, icount), GFP_KERNEL); if (!parent) return -ENOMEM; + parent->channel_cnt = icount; + init_usb_anchor(&parent->rx_submitted); usb_set_intfdata(intf, parent); @@ -1528,7 +1527,7 @@ static void gs_usb_disconnect(struct usb_interface *intf) return; } - for (i = 0; i < GS_MAX_INTF; i++) + for (i = 0; i < parent->channel_cnt; i++) if (parent->canch[i]) gs_destroy_candev(parent->canch[i]); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c index 15f28b6fe758e..022b5b79247c5 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c @@ -856,6 +856,7 @@ static int kvaser_usb_init_one(struct kvaser_usb *dev, int channel) } SET_NETDEV_DEV(netdev, &dev->intf->dev); netdev->dev_id = channel; + netdev->dev_port = channel; dev->nets[channel] = priv; diff --git a/drivers/net/can/usb/mcba_usb.c b/drivers/net/can/usb/mcba_usb.c index 47619e9cb0055..ecc489afb8416 100644 --- a/drivers/net/can/usb/mcba_usb.c +++ b/drivers/net/can/usb/mcba_usb.c @@ -761,6 +761,7 @@ static const struct net_device_ops mcba_netdev_ops = { .ndo_open = mcba_usb_open, .ndo_stop = mcba_usb_close, .ndo_start_xmit = mcba_usb_start_xmit, + .ndo_change_mtu = can_change_mtu, }; static const struct ethtool_ops mcba_ethtool_ops = { diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index 24ad9f593a773..9d162d1004c26 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -111,7 +111,7 @@ void peak_usb_update_ts_now(struct peak_time_ref *time_ref, u32 ts_now) u32 delta_ts = time_ref->ts_dev_2 - time_ref->ts_dev_1; if (time_ref->ts_dev_2 < time_ref->ts_dev_1) - delta_ts &= (1 << time_ref->adapter->ts_used_bits) - 1; + delta_ts &= (1ULL << time_ref->adapter->ts_used_bits) - 1; time_ref->ts_total += delta_ts; } diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c index 4d85b29a17b78..ebefc274b50a5 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c @@ -49,7 +49,7 @@ struct __packed pcan_ufd_fw_info { __le32 ser_no; /* S/N */ __le32 flags; /* special functions */ - /* extended data when type == PCAN_USBFD_TYPE_EXT */ + /* extended data when type >= PCAN_USBFD_TYPE_EXT */ u8 cmd_out_ep; /* ep for cmd */ u8 cmd_in_ep; /* ep for replies */ u8 data_out_ep[2]; /* ep for CANx TX */ @@ -982,10 +982,11 @@ static int pcan_usb_fd_init(struct peak_usb_device *dev) dev->can.ctrlmode |= CAN_CTRLMODE_FD_NON_ISO; } - /* if vendor rsp is of type 2, then it contains EP numbers to - * use for cmds pipes. If not, then default EP should be used. + /* if vendor rsp type is greater than or equal to 2, then it + * contains EP numbers to use for cmds pipes. If not, then + * default EP should be used. */ - if (fw_info->type != cpu_to_le16(PCAN_USBFD_TYPE_EXT)) { + if (le16_to_cpu(fw_info->type) < PCAN_USBFD_TYPE_EXT) { fw_info->cmd_out_ep = PCAN_USBPRO_EP_CMDOUT; fw_info->cmd_in_ep = PCAN_USBPRO_EP_CMDIN; } @@ -1018,11 +1019,11 @@ static int pcan_usb_fd_init(struct peak_usb_device *dev) dev->can_channel_id = le32_to_cpu(pdev->usb_if->fw_info.dev_id[dev->ctrl_idx]); - /* if vendor rsp is of type 2, then it contains EP numbers to - * use for data pipes. If not, then statically defined EP are used - * (see peak_usb_create_dev()). + /* if vendor rsp type is greater than or equal to 2, then it contains EP + * numbers to use for data pipes. If not, then statically defined EP are + * used (see peak_usb_create_dev()). */ - if (fw_info->type == cpu_to_le16(PCAN_USBFD_TYPE_EXT)) { + if (le16_to_cpu(fw_info->type) >= PCAN_USBFD_TYPE_EXT) { dev->ep_msg_in = fw_info->data_in_ep; dev->ep_msg_out = fw_info->data_out_ep[dev->ctrl_idx]; } diff --git a/drivers/net/can/xilinx_can.c b/drivers/net/can/xilinx_can.c index abe58f1030433..57d1209134f11 100644 --- a/drivers/net/can/xilinx_can.c +++ b/drivers/net/can/xilinx_can.c @@ -628,14 +628,6 @@ static void xcan_write_frame(struct net_device *ndev, struct sk_buff *skb, dlc |= XCAN_DLCR_EDL_MASK; } - if (!(priv->devtype.flags & XCAN_FLAG_TX_MAILBOXES) && - (priv->devtype.flags & XCAN_FLAG_TXFEMP)) - can_put_echo_skb(skb, ndev, priv->tx_head % priv->tx_max, 0); - else - can_put_echo_skb(skb, ndev, 0, 0); - - priv->tx_head++; - priv->write_reg(priv, XCAN_FRAME_ID_OFFSET(frame_offset), id); /* If the CAN frame is RTR frame this write triggers transmission * (not on CAN FD) @@ -668,6 +660,14 @@ static void xcan_write_frame(struct net_device *ndev, struct sk_buff *skb, data[1]); } } + + if (!(priv->devtype.flags & XCAN_FLAG_TX_MAILBOXES) && + (priv->devtype.flags & XCAN_FLAG_TXFEMP)) + can_put_echo_skb(skb, ndev, priv->tx_head % priv->tx_max, 0); + else + can_put_echo_skb(skb, ndev, 0, 0); + + priv->tx_head++; } /** diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index cfcda893f1a16..ffe8db7c2f1f4 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -339,18 +339,23 @@ static void b53_set_forwarding(struct b53_device *dev, int enable) b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, mgmt); - /* Include IMP port in dumb forwarding mode - */ - b53_read8(dev, B53_CTRL_PAGE, B53_SWITCH_CTRL, &mgmt); - mgmt |= B53_MII_DUMB_FWDG_EN; - b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_CTRL, mgmt); - - /* Look at B53_UC_FWD_EN and B53_MC_FWD_EN to decide whether - * frames should be flooded or not. - */ - b53_read8(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, &mgmt); - mgmt |= B53_UC_FWD_EN | B53_MC_FWD_EN | B53_IPMC_FWD_EN; - b53_write8(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, mgmt); + if (!is5325(dev)) { + /* Include IMP port in dumb forwarding mode */ + b53_read8(dev, B53_CTRL_PAGE, B53_SWITCH_CTRL, &mgmt); + mgmt |= B53_MII_DUMB_FWDG_EN; + b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_CTRL, mgmt); + + /* Look at B53_UC_FWD_EN and B53_MC_FWD_EN to decide whether + * frames should be flooded or not. + */ + b53_read8(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, &mgmt); + mgmt |= B53_UC_FWD_EN | B53_MC_FWD_EN | B53_IP_MC; + b53_write8(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, mgmt); + } else { + b53_read8(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, &mgmt); + mgmt |= B53_IP_MC; + b53_write8(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, mgmt); + } } static void b53_enable_vlan(struct b53_device *dev, int port, bool enable, @@ -373,15 +378,17 @@ static void b53_enable_vlan(struct b53_device *dev, int port, bool enable, b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5, &vc5); } + vc1 &= ~VC1_RX_MCST_FWD_EN; + if (enable) { vc0 |= VC0_VLAN_EN | VC0_VID_CHK_EN | VC0_VID_HASH_VID; - vc1 |= VC1_RX_MCST_UNTAG_EN | VC1_RX_MCST_FWD_EN; + vc1 |= VC1_RX_MCST_UNTAG_EN; vc4 &= ~VC4_ING_VID_CHECK_MASK; if (enable_filtering) { vc4 |= VC4_ING_VID_VIO_DROP << VC4_ING_VID_CHECK_S; vc5 |= VC5_DROP_VTABLE_MISS; } else { - vc4 |= VC4_ING_VID_VIO_FWD << VC4_ING_VID_CHECK_S; + vc4 |= VC4_NO_ING_VID_CHK << VC4_ING_VID_CHECK_S; vc5 &= ~VC5_DROP_VTABLE_MISS; } @@ -393,7 +400,7 @@ static void b53_enable_vlan(struct b53_device *dev, int port, bool enable, } else { vc0 &= ~(VC0_VLAN_EN | VC0_VID_CHK_EN | VC0_VID_HASH_VID); - vc1 &= ~(VC1_RX_MCST_UNTAG_EN | VC1_RX_MCST_FWD_EN); + vc1 &= ~VC1_RX_MCST_UNTAG_EN; vc4 &= ~VC4_ING_VID_CHECK_MASK; vc5 &= ~VC5_DROP_VTABLE_MISS; @@ -505,6 +512,10 @@ void b53_imp_vlan_setup(struct dsa_switch *ds, int cpu_port) unsigned int i; u16 pvlan; + /* BCM5325 CPU port is at 8 */ + if ((is5325(dev) || is5365(dev)) && cpu_port == B53_CPU_PORT_25) + cpu_port = B53_CPU_PORT; + /* Enable the IMP port to be in the same VLAN as the other ports * on a per-port basis such that we only have Port i and IMP in * the same VLAN. @@ -555,6 +566,9 @@ static void b53_port_set_learning(struct b53_device *dev, int port, { u16 reg; + if (is5325(dev)) + return; + b53_read16(dev, B53_CTRL_PAGE, B53_DIS_LEARNING, ®); if (learning) reg &= ~BIT(port); @@ -1161,6 +1175,8 @@ static void b53_force_link(struct b53_device *dev, int port, int link) if (port == dev->imp_port) { off = B53_PORT_OVERRIDE_CTRL; val = PORT_OVERRIDE_EN; + } else if (is5325(dev)) { + return; } else { off = B53_GMII_PORT_OVERRIDE_CTRL(port); val = GMII_PO_EN; @@ -1185,6 +1201,8 @@ static void b53_force_port_config(struct b53_device *dev, int port, if (port == dev->imp_port) { off = B53_PORT_OVERRIDE_CTRL; val = PORT_OVERRIDE_EN; + } else if (is5325(dev)) { + return; } else { off = B53_GMII_PORT_OVERRIDE_CTRL(port); val = GMII_PO_EN; @@ -1197,6 +1215,10 @@ static void b53_force_port_config(struct b53_device *dev, int port, else reg &= ~PORT_OVERRIDE_FULL_DUPLEX; + reg &= ~(0x3 << GMII_PO_SPEED_S); + if (is5301x(dev) || is58xx(dev)) + reg &= ~PORT_OVERRIDE_SPEED_2000M; + switch (speed) { case 2000: reg |= PORT_OVERRIDE_SPEED_2000M; @@ -1215,10 +1237,24 @@ static void b53_force_port_config(struct b53_device *dev, int port, return; } - if (rx_pause) - reg |= PORT_OVERRIDE_RX_FLOW; - if (tx_pause) - reg |= PORT_OVERRIDE_TX_FLOW; + if (is5325(dev)) + reg &= ~PORT_OVERRIDE_LP_FLOW_25; + else + reg &= ~(PORT_OVERRIDE_RX_FLOW | PORT_OVERRIDE_TX_FLOW); + + if (rx_pause) { + if (is5325(dev)) + reg |= PORT_OVERRIDE_LP_FLOW_25; + else + reg |= PORT_OVERRIDE_RX_FLOW; + } + + if (tx_pause) { + if (is5325(dev)) + reg |= PORT_OVERRIDE_LP_FLOW_25; + else + reg |= PORT_OVERRIDE_TX_FLOW; + } b53_write8(dev, B53_CTRL_PAGE, off, reg); } @@ -1235,24 +1271,7 @@ static void b53_adjust_63xx_rgmii(struct dsa_switch *ds, int port, off = B53_RGMII_CTRL_P(port); b53_read8(dev, B53_CTRL_PAGE, off, &rgmii_ctrl); - - switch (interface) { - case PHY_INTERFACE_MODE_RGMII_ID: - rgmii_ctrl |= (RGMII_CTRL_DLL_RXC | RGMII_CTRL_DLL_TXC); - break; - case PHY_INTERFACE_MODE_RGMII_RXID: - rgmii_ctrl &= ~(RGMII_CTRL_DLL_TXC); - rgmii_ctrl |= RGMII_CTRL_DLL_RXC; - break; - case PHY_INTERFACE_MODE_RGMII_TXID: - rgmii_ctrl &= ~(RGMII_CTRL_DLL_RXC); - rgmii_ctrl |= RGMII_CTRL_DLL_TXC; - break; - case PHY_INTERFACE_MODE_RGMII: - default: - rgmii_ctrl &= ~(RGMII_CTRL_DLL_RXC | RGMII_CTRL_DLL_TXC); - break; - } + rgmii_ctrl &= ~(RGMII_CTRL_DLL_RXC | RGMII_CTRL_DLL_TXC); if (port != dev->imp_port) { if (is63268(dev)) @@ -1392,6 +1411,10 @@ static void b53_phylink_get_caps(struct dsa_switch *ds, int port, __set_bit(PHY_INTERFACE_MODE_MII, config->supported_interfaces); __set_bit(PHY_INTERFACE_MODE_REVMII, config->supported_interfaces); + /* BCM63xx RGMII ports support RGMII */ + if (is63xx(dev) && in_range(port, B53_63XX_RGMII0, 4)) + phy_interface_set_rgmii(config->supported_interfaces); + config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100; @@ -1519,12 +1542,21 @@ int b53_vlan_add(struct dsa_switch *ds, int port, bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED; bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID; struct b53_vlan *vl; + u16 old_pvid, new_pvid; int err; err = b53_vlan_prepare(ds, port, vlan); if (err) return err; + b53_read16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port), &old_pvid); + if (pvid) + new_pvid = vlan->vid; + else if (!pvid && vlan->vid == old_pvid) + new_pvid = b53_default_pvid(dev); + else + new_pvid = old_pvid; + vl = &dev->vlans[vlan->vid]; b53_get_vlan_entry(dev, vlan->vid, vl); @@ -1541,10 +1573,10 @@ int b53_vlan_add(struct dsa_switch *ds, int port, b53_set_vlan_entry(dev, vlan->vid, vl); b53_fast_age_vlan(dev, vlan->vid); - if (pvid && !dsa_is_cpu_port(ds, port)) { + if (!dsa_is_cpu_port(ds, port) && new_pvid != old_pvid) { b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port), - vlan->vid); - b53_fast_age_vlan(dev, vlan->vid); + new_pvid); + b53_fast_age_vlan(dev, old_pvid); } return 0; @@ -1784,7 +1816,7 @@ static int b53_arl_search_wait(struct b53_device *dev) do { b53_read8(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_CTL, ®); if (!(reg & ARL_SRCH_STDN)) - return 0; + return -ENOENT; if (reg & ARL_SRCH_VLID) return 0; @@ -1956,7 +1988,7 @@ EXPORT_SYMBOL(b53_br_join); void b53_br_leave(struct dsa_switch *ds, int port, struct dsa_bridge bridge) { struct b53_device *dev = ds->priv; - struct b53_vlan *vl = &dev->vlans[0]; + struct b53_vlan *vl; s8 cpu_port = dsa_to_port(ds, port)->cpu_dp->index; unsigned int i; u16 pvlan, reg, pvid; @@ -1982,6 +2014,7 @@ void b53_br_leave(struct dsa_switch *ds, int port, struct dsa_bridge bridge) dev->ports[port].vlan_ctl_mask = pvlan; pvid = b53_default_pvid(dev); + vl = &dev->vlans[pvid]; /* Make this port join all VLANs without VLAN entries */ if (is58xx(dev)) { @@ -1990,12 +2023,12 @@ void b53_br_leave(struct dsa_switch *ds, int port, struct dsa_bridge bridge) if (!(reg & BIT(cpu_port))) reg |= BIT(cpu_port); b53_write16(dev, B53_VLAN_PAGE, B53_JOIN_ALL_VLAN_EN, reg); - } else { - b53_get_vlan_entry(dev, pvid, vl); - vl->members |= BIT(port) | BIT(cpu_port); - vl->untag |= BIT(port) | BIT(cpu_port); - b53_set_vlan_entry(dev, pvid, vl); } + + b53_get_vlan_entry(dev, pvid, vl); + vl->members |= BIT(port) | BIT(cpu_port); + vl->untag |= BIT(port) | BIT(cpu_port); + b53_set_vlan_entry(dev, pvid, vl); } EXPORT_SYMBOL(b53_br_leave); @@ -2046,7 +2079,13 @@ int b53_br_flags_pre(struct dsa_switch *ds, int port, struct switchdev_brport_flags flags, struct netlink_ext_ack *extack) { - if (flags.mask & ~(BR_FLOOD | BR_MCAST_FLOOD | BR_LEARNING)) + struct b53_device *dev = ds->priv; + unsigned long mask = (BR_FLOOD | BR_MCAST_FLOOD); + + if (!is5325(dev)) + mask |= BR_LEARNING; + + if (flags.mask & ~mask) return -EINVAL; return 0; diff --git a/drivers/net/dsa/b53/b53_regs.h b/drivers/net/dsa/b53/b53_regs.h index bfbcb66bef662..38e2d60dab7d5 100644 --- a/drivers/net/dsa/b53/b53_regs.h +++ b/drivers/net/dsa/b53/b53_regs.h @@ -92,6 +92,7 @@ #define PORT_OVERRIDE_SPEED_10M (0 << PORT_OVERRIDE_SPEED_S) #define PORT_OVERRIDE_SPEED_100M (1 << PORT_OVERRIDE_SPEED_S) #define PORT_OVERRIDE_SPEED_1000M (2 << PORT_OVERRIDE_SPEED_S) +#define PORT_OVERRIDE_LP_FLOW_25 BIT(3) /* BCM5325 only */ #define PORT_OVERRIDE_RV_MII_25 BIT(4) /* BCM5325 only */ #define PORT_OVERRIDE_RX_FLOW BIT(4) #define PORT_OVERRIDE_TX_FLOW BIT(5) @@ -103,7 +104,7 @@ /* IP Multicast control (8 bit) */ #define B53_IP_MULTICAST_CTRL 0x21 -#define B53_IPMC_FWD_EN BIT(1) +#define B53_IP_MC BIT(0) #define B53_UC_FWD_EN BIT(6) #define B53_MC_FWD_EN BIT(7) diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c index 5b139f2206b6e..48cf9d300bbf5 100644 --- a/drivers/net/dsa/dsa_loop.c +++ b/drivers/net/dsa/dsa_loop.c @@ -378,13 +378,10 @@ static struct mdio_driver dsa_loop_drv = { static void dsa_loop_phydevs_unregister(void) { - unsigned int i; - - for (i = 0; i < NUM_FIXED_PHYS; i++) - if (!IS_ERR(phydevs[i])) { + for (int i = 0; i < NUM_FIXED_PHYS; i++) { + if (!IS_ERR(phydevs[i])) fixed_phy_unregister(phydevs[i]); - phy_device_free(phydevs[i]); - } + } } static int __init dsa_loop_init(void) diff --git a/drivers/net/dsa/hirschmann/hellcreek_ptp.c b/drivers/net/dsa/hirschmann/hellcreek_ptp.c index 5249a1c2a80b8..1bb994f785963 100644 --- a/drivers/net/dsa/hirschmann/hellcreek_ptp.c +++ b/drivers/net/dsa/hirschmann/hellcreek_ptp.c @@ -371,8 +371,18 @@ static int hellcreek_led_setup(struct hellcreek *hellcreek) hellcreek_set_brightness(hellcreek, STATUS_OUT_IS_GM, 1); /* Register both leds */ - led_classdev_register(hellcreek->dev, &hellcreek->led_sync_good); - led_classdev_register(hellcreek->dev, &hellcreek->led_is_gm); + ret = led_classdev_register(hellcreek->dev, &hellcreek->led_sync_good); + if (ret) { + dev_err(hellcreek->dev, "Failed to register sync_good LED\n"); + goto out; + } + + ret = led_classdev_register(hellcreek->dev, &hellcreek->led_is_gm); + if (ret) { + dev_err(hellcreek->dev, "Failed to register is_gm LED\n"); + led_classdev_unregister(&hellcreek->led_sync_good); + goto out; + } ret = 0; diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c index 3c76a1a14aee3..c1a9ab9259768 100644 --- a/drivers/net/dsa/lantiq_gswip.c +++ b/drivers/net/dsa/lantiq_gswip.c @@ -682,30 +682,24 @@ static int gswip_add_single_port_br(struct gswip_priv *priv, int port, bool add) return 0; } -static int gswip_port_enable(struct dsa_switch *ds, int port, - struct phy_device *phydev) +static int gswip_port_setup(struct dsa_switch *ds, int port) { struct gswip_priv *priv = ds->priv; int err; - if (!dsa_is_user_port(ds, port)) - return 0; - if (!dsa_is_cpu_port(ds, port)) { err = gswip_add_single_port_br(priv, port, true); if (err) return err; } - /* RMON Counter Enable for port */ - gswip_switch_w(priv, GSWIP_BM_PCFG_CNTEN, GSWIP_BM_PCFGp(port)); + return 0; +} - /* enable port fetch/store dma & VLAN Modification */ - gswip_switch_mask(priv, 0, GSWIP_FDMA_PCTRL_EN | - GSWIP_FDMA_PCTRL_VLANMOD_BOTH, - GSWIP_FDMA_PCTRLp(port)); - gswip_switch_mask(priv, 0, GSWIP_SDMA_PCTRL_EN, - GSWIP_SDMA_PCTRLp(port)); +static int gswip_port_enable(struct dsa_switch *ds, int port, + struct phy_device *phydev) +{ + struct gswip_priv *priv = ds->priv; if (!dsa_is_cpu_port(ds, port)) { u32 mdio_phy = 0; @@ -717,6 +711,16 @@ static int gswip_port_enable(struct dsa_switch *ds, int port, GSWIP_MDIO_PHYp(port)); } + /* RMON Counter Enable for port */ + gswip_switch_w(priv, GSWIP_BM_PCFG_CNTEN, GSWIP_BM_PCFGp(port)); + + /* enable port fetch/store dma & VLAN Modification */ + gswip_switch_mask(priv, 0, GSWIP_FDMA_PCTRL_EN | + GSWIP_FDMA_PCTRL_VLANMOD_BOTH, + GSWIP_FDMA_PCTRLp(port)); + gswip_switch_mask(priv, 0, GSWIP_SDMA_PCTRL_EN, + GSWIP_SDMA_PCTRLp(port)); + return 0; } @@ -724,9 +728,6 @@ static void gswip_port_disable(struct dsa_switch *ds, int port) { struct gswip_priv *priv = ds->priv; - if (!dsa_is_user_port(ds, port)) - return; - gswip_switch_mask(priv, GSWIP_FDMA_PCTRL_EN, 0, GSWIP_FDMA_PCTRLp(port)); gswip_switch_mask(priv, GSWIP_SDMA_PCTRL_EN, 0, @@ -1365,8 +1366,9 @@ static int gswip_port_fdb(struct dsa_switch *ds, int port, int i; int err; + /* Operation not supported on the CPU port, don't throw errors */ if (!bridge) - return -EINVAL; + return 0; for (i = max_ports; i < ARRAY_SIZE(priv->vlans); i++) { if (priv->vlans[i].bridge == bridge) { @@ -1821,6 +1823,7 @@ static int gswip_get_sset_count(struct dsa_switch *ds, int port, int sset) static const struct dsa_switch_ops gswip_xrx200_switch_ops = { .get_tag_protocol = gswip_get_tag_protocol, .setup = gswip_setup, + .port_setup = gswip_port_setup, .port_enable = gswip_port_enable, .port_disable = gswip_port_disable, .port_bridge_join = gswip_port_bridge_join, diff --git a/drivers/net/dsa/microchip/ksz9477.c b/drivers/net/dsa/microchip/ksz9477.c index 59134d117846d..a11f2c1aabaca 100644 --- a/drivers/net/dsa/microchip/ksz9477.c +++ b/drivers/net/dsa/microchip/ksz9477.c @@ -1087,9 +1087,15 @@ void ksz9477_config_cpu_port(struct dsa_switch *ds) } } +#define RESV_MCAST_CNT 8 + +static u8 reserved_mcast_map[RESV_MCAST_CNT] = { 0, 1, 3, 16, 32, 33, 2, 17 }; + int ksz9477_enable_stp_addr(struct ksz_device *dev) { + u8 i, ports, update; const u32 *masks; + bool override; u32 data; int ret; @@ -1098,23 +1104,87 @@ int ksz9477_enable_stp_addr(struct ksz_device *dev) /* Enable Reserved multicast table */ ksz_cfg(dev, REG_SW_LUE_CTRL_0, SW_RESV_MCAST_ENABLE, true); - /* Set the Override bit for forwarding BPDU packet to CPU */ - ret = ksz_write32(dev, REG_SW_ALU_VAL_B, - ALU_V_OVERRIDE | BIT(dev->cpu_port)); - if (ret < 0) - return ret; + /* The reserved multicast address table has 8 entries. Each entry has + * a default value of which port to forward. It is assumed the host + * port is the last port in most of the switches, but that is not the + * case for KSZ9477 or maybe KSZ9897. For LAN937X family the default + * port is port 5, the first RGMII port. It is okay for LAN9370, a + * 5-port switch, but may not be correct for the other 8-port + * versions. It is necessary to update the whole table to forward to + * the right ports. + * Furthermore PTP messages can use a reserved multicast address and + * the host will not receive them if this table is not correct. + */ + for (i = 0; i < RESV_MCAST_CNT; i++) { + data = reserved_mcast_map[i] << + dev->info->shifts[ALU_STAT_INDEX]; + data |= ALU_STAT_START | + masks[ALU_STAT_DIRECT] | + masks[ALU_RESV_MCAST_ADDR] | + masks[ALU_STAT_READ]; + ret = ksz_write32(dev, REG_SW_ALU_STAT_CTRL__4, data); + if (ret < 0) + return ret; - data = ALU_STAT_START | ALU_RESV_MCAST_ADDR | masks[ALU_STAT_WRITE]; + /* wait to be finished */ + ret = ksz9477_wait_alu_sta_ready(dev); + if (ret < 0) + return ret; - ret = ksz_write32(dev, REG_SW_ALU_STAT_CTRL__4, data); - if (ret < 0) - return ret; + ret = ksz_read32(dev, REG_SW_ALU_VAL_B, &data); + if (ret < 0) + return ret; - /* wait to be finished */ - ret = ksz9477_wait_alu_sta_ready(dev); - if (ret < 0) { - dev_err(dev->dev, "Failed to update Reserved Multicast table\n"); - return ret; + override = false; + ports = data & dev->port_mask; + switch (i) { + case 0: + case 6: + /* Change the host port. */ + update = BIT(dev->cpu_port); + override = true; + break; + case 2: + /* Change the host port. */ + update = BIT(dev->cpu_port); + break; + case 4: + case 5: + case 7: + /* Skip the host port. */ + update = dev->port_mask & ~BIT(dev->cpu_port); + break; + default: + update = ports; + break; + } + if (update != ports || override) { + data &= ~dev->port_mask; + data |= update; + /* Set Override bit to receive frame even when port is + * closed. + */ + if (override) + data |= ALU_V_OVERRIDE; + ret = ksz_write32(dev, REG_SW_ALU_VAL_B, data); + if (ret < 0) + return ret; + + data = reserved_mcast_map[i] << + dev->info->shifts[ALU_STAT_INDEX]; + data |= ALU_STAT_START | + masks[ALU_STAT_DIRECT] | + masks[ALU_RESV_MCAST_ADDR] | + masks[ALU_STAT_WRITE]; + ret = ksz_write32(dev, REG_SW_ALU_STAT_CTRL__4, data); + if (ret < 0) + return ret; + + /* wait to be finished */ + ret = ksz9477_wait_alu_sta_ready(dev); + if (ret < 0) + return ret; + } } return 0; diff --git a/drivers/net/dsa/microchip/ksz9477_reg.h b/drivers/net/dsa/microchip/ksz9477_reg.h index d0886ed984c57..c3ad8ce707f85 100644 --- a/drivers/net/dsa/microchip/ksz9477_reg.h +++ b/drivers/net/dsa/microchip/ksz9477_reg.h @@ -2,7 +2,7 @@ /* * Microchip KSZ9477 register definitions * - * Copyright (C) 2017-2024 Microchip Technology Inc. + * Copyright (C) 2017-2025 Microchip Technology Inc. */ #ifndef __KSZ9477_REGS_H @@ -422,7 +422,6 @@ #define ALU_RESV_MCAST_INDEX_M (BIT(6) - 1) #define ALU_STAT_START BIT(7) -#define ALU_RESV_MCAST_ADDR BIT(1) #define REG_SW_ALU_VAL_A 0x0420 diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index 997c225dfba4d..cff83a8fb7d28 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -437,6 +437,8 @@ static const u16 ksz9477_regs[] = { static const u32 ksz9477_masks[] = { [ALU_STAT_WRITE] = 0, [ALU_STAT_READ] = 1, + [ALU_STAT_DIRECT] = 0, + [ALU_RESV_MCAST_ADDR] = BIT(1), [P_MII_TX_FLOW_CTRL] = BIT(5), [P_MII_RX_FLOW_CTRL] = BIT(3), }; @@ -464,6 +466,8 @@ static const u8 ksz9477_xmii_ctrl1[] = { static const u32 lan937x_masks[] = { [ALU_STAT_WRITE] = 1, [ALU_STAT_READ] = 2, + [ALU_STAT_DIRECT] = BIT(3), + [ALU_RESV_MCAST_ADDR] = BIT(2), [P_MII_TX_FLOW_CTRL] = BIT(5), [P_MII_RX_FLOW_CTRL] = BIT(3), }; diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index a4de58847deab..0e51f2277381d 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -255,6 +255,8 @@ enum ksz_masks { DYNAMIC_MAC_TABLE_TIMESTAMP, ALU_STAT_WRITE, ALU_STAT_READ, + ALU_STAT_DIRECT, + ALU_RESV_MCAST_ADDR, P_MII_TX_FLOW_CTRL, P_MII_RX_FLOW_CTRL, }; diff --git a/drivers/net/dsa/microchip/lan937x_main.c b/drivers/net/dsa/microchip/lan937x_main.c index dde37e61faa35..d2e98b5f0aca4 100644 --- a/drivers/net/dsa/microchip/lan937x_main.c +++ b/drivers/net/dsa/microchip/lan937x_main.c @@ -336,6 +336,7 @@ static void lan937x_set_tune_adj(struct ksz_device *dev, int port, ksz_pread16(dev, port, reg, &data16); /* Update tune Adjust */ + data16 &= ~PORT_TUNE_ADJ; data16 |= FIELD_PREP(PORT_TUNE_ADJ, val); ksz_pwrite16(dev, port, reg, data16); diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 8d27933c3733b..f91f25578f075 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -1543,7 +1543,7 @@ static void vsc9959_tas_clock_adjust(struct ocelot *ocelot) struct tc_taprio_qopt_offload *taprio; struct ocelot_port *ocelot_port; struct timespec64 base_ts; - int port; + int i, port; u32 val; mutex_lock(&ocelot->fwd_domain_lock); @@ -1575,6 +1575,9 @@ static void vsc9959_tas_clock_adjust(struct ocelot *ocelot) QSYS_PARAM_CFG_REG_3_BASE_TIME_SEC_MSB_M, QSYS_PARAM_CFG_REG_3); + for (i = 0; i < taprio->num_entries; i++) + vsc9959_tas_gcl_set(ocelot, i, &taprio->entries[i]); + ocelot_rmw(ocelot, QSYS_TAS_PARAM_CFG_CTRL_CONFIG_CHANGE, QSYS_TAS_PARAM_CFG_CTRL_CONFIG_CHANGE, QSYS_TAS_PARAM_CFG_CTRL); diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 1a367e64bc3b1..843e50b5a0ec5 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -2076,6 +2076,7 @@ static void sja1105_bridge_stp_state_set(struct dsa_switch *ds, int port, switch (state) { case BR_STATE_DISABLED: case BR_STATE_BLOCKING: + case BR_STATE_LISTENING: /* From UM10944 description of DRPDTAG (why put this there?): * "Management traffic flows to the port regardless of the state * of the INGRESS flag". So BPDUs are still be allowed to pass. @@ -2085,11 +2086,6 @@ static void sja1105_bridge_stp_state_set(struct dsa_switch *ds, int port, mac[port].egress = false; mac[port].dyn_learn = false; break; - case BR_STATE_LISTENING: - mac[port].ingress = true; - mac[port].egress = false; - mac[port].dyn_learn = false; - break; case BR_STATE_LEARNING: mac[port].ingress = true; mac[port].egress = false; diff --git a/drivers/net/ethernet/agere/et131x.c b/drivers/net/ethernet/agere/et131x.c index 3d9220f9c9fe7..294dbe2c3797a 100644 --- a/drivers/net/ethernet/agere/et131x.c +++ b/drivers/net/ethernet/agere/et131x.c @@ -2459,6 +2459,10 @@ static int nic_send_packet(struct et131x_adapter *adapter, struct tcb *tcb) skb->data, skb_headlen(skb), DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, + dma_addr)) + return -ENOMEM; + desc[frag].addr_lo = lower_32_bits(dma_addr); desc[frag].addr_hi = upper_32_bits(dma_addr); frag++; @@ -2468,6 +2472,10 @@ static int nic_send_packet(struct et131x_adapter *adapter, struct tcb *tcb) skb->data, skb_headlen(skb) / 2, DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, + dma_addr)) + return -ENOMEM; + desc[frag].addr_lo = lower_32_bits(dma_addr); desc[frag].addr_hi = upper_32_bits(dma_addr); frag++; @@ -2478,6 +2486,10 @@ static int nic_send_packet(struct et131x_adapter *adapter, struct tcb *tcb) skb_headlen(skb) / 2, skb_headlen(skb) / 2, DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, + dma_addr)) + goto unmap_first_out; + desc[frag].addr_lo = lower_32_bits(dma_addr); desc[frag].addr_hi = upper_32_bits(dma_addr); frag++; @@ -2489,6 +2501,9 @@ static int nic_send_packet(struct et131x_adapter *adapter, struct tcb *tcb) 0, desc[frag].len_vlan, DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, dma_addr)) + goto unmap_out; + desc[frag].addr_lo = lower_32_bits(dma_addr); desc[frag].addr_hi = upper_32_bits(dma_addr); frag++; @@ -2578,6 +2593,27 @@ static int nic_send_packet(struct et131x_adapter *adapter, struct tcb *tcb) &adapter->regs->global.watchdog_timer); } return 0; + +unmap_out: + // Unmap the body of the packet with map_page + while (--i) { + frag--; + dma_addr = desc[frag].addr_lo; + dma_addr |= (u64)desc[frag].addr_hi << 32; + dma_unmap_page(&adapter->pdev->dev, dma_addr, + desc[frag].len_vlan, DMA_TO_DEVICE); + } + +unmap_first_out: + // Unmap the header with map_single + while (frag--) { + dma_addr = desc[frag].addr_lo; + dma_addr |= (u64)desc[frag].addr_hi << 32; + dma_unmap_single(&adapter->pdev->dev, dma_addr, + desc[frag].len_vlan, DMA_TO_DEVICE); + } + + return -ENOMEM; } static int send_packet(struct sk_buff *skb, struct et131x_adapter *adapter) diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index d901877544445..82a7c52fdb721 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -752,7 +752,10 @@ static u32 ena_get_rxfh_indir_size(struct net_device *netdev) static u32 ena_get_rxfh_key_size(struct net_device *netdev) { - return ENA_HASH_KEY_SIZE; + struct ena_adapter *adapter = netdev_priv(netdev); + struct ena_rss *rss = &adapter->ena_dev->rss; + + return rss->hash_key ? ENA_HASH_KEY_SIZE : 0; } static int ena_indirection_table_set(struct ena_adapter *adapter, diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 0d201a57d7e29..dd9c50d3ec0f0 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -1310,7 +1310,7 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, } if (xdp_flags & ENA_XDP_REDIRECT) - xdp_do_flush_map(); + xdp_do_flush(); return work_done; diff --git a/drivers/net/ethernet/amd/pds_core/auxbus.c b/drivers/net/ethernet/amd/pds_core/auxbus.c index fb7a5403e630d..889a18962270a 100644 --- a/drivers/net/ethernet/amd/pds_core/auxbus.c +++ b/drivers/net/ethernet/amd/pds_core/auxbus.c @@ -172,48 +172,57 @@ static struct pds_auxiliary_dev *pdsc_auxbus_dev_register(struct pdsc *cf, return padev; } -int pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf) +void pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf, + struct pds_auxiliary_dev **pd_ptr) { struct pds_auxiliary_dev *padev; - int err = 0; + + if (!*pd_ptr) + return; mutex_lock(&pf->config_lock); - padev = pf->vfs[cf->vf_id].padev; - if (padev) { - pds_client_unregister(pf, padev->client_id); - auxiliary_device_delete(&padev->aux_dev); - auxiliary_device_uninit(&padev->aux_dev); - padev->client_id = 0; - } - pf->vfs[cf->vf_id].padev = NULL; + padev = *pd_ptr; + pds_client_unregister(pf, padev->client_id); + auxiliary_device_delete(&padev->aux_dev); + auxiliary_device_uninit(&padev->aux_dev); + *pd_ptr = NULL; mutex_unlock(&pf->config_lock); - return err; } -int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf) +int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf, + enum pds_core_vif_types vt, + struct pds_auxiliary_dev **pd_ptr) { struct pds_auxiliary_dev *padev; - enum pds_core_vif_types vt; char devname[PDS_DEVNAME_LEN]; + unsigned long mask; u16 vt_support; int client_id; int err = 0; + if (!cf) + return -ENODEV; + + if (vt >= PDS_DEV_TYPE_MAX) + return -EINVAL; + mutex_lock(&pf->config_lock); - /* We only support vDPA so far, so it is the only one to - * be verified that it is available in the Core device and - * enabled in the devlink param. In the future this might - * become a loop for several VIF types. - */ + mask = BIT_ULL(PDSC_S_FW_DEAD) | + BIT_ULL(PDSC_S_STOPPING_DRIVER); + if (cf->state & mask) { + dev_err(pf->dev, "%s: can't add dev, VF client in bad state %#lx\n", + __func__, cf->state); + err = -ENXIO; + goto out_unlock; + } /* Verify that the type is supported and enabled. It is not * an error if there is no auxbus device support for this * VF, it just means something else needs to happen with it. */ - vt = PDS_DEV_TYPE_VDPA; vt_support = !!le16_to_cpu(pf->dev_ident.vif_types[vt]); if (!(vt_support && pf->viftype_status[vt].supported && @@ -239,7 +248,7 @@ int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf) err = PTR_ERR(padev); goto out_unlock; } - pf->vfs[cf->vf_id].padev = padev; + *pd_ptr = padev; out_unlock: mutex_unlock(&pf->config_lock); diff --git a/drivers/net/ethernet/amd/pds_core/core.c b/drivers/net/ethernet/amd/pds_core/core.c index b3fa867c8ccd9..c2ef55cff6b3e 100644 --- a/drivers/net/ethernet/amd/pds_core/core.c +++ b/drivers/net/ethernet/amd/pds_core/core.c @@ -413,10 +413,7 @@ int pdsc_setup(struct pdsc *pdsc, bool init) if (err) return err; - /* Scale the descriptor ring length based on number of CPUs and VFs */ - numdescs = max_t(int, PDSC_ADMINQ_MIN_LENGTH, num_online_cpus()); - numdescs += 2 * pci_sriov_get_totalvfs(pdsc->pdev); - numdescs = roundup_pow_of_two(numdescs); + numdescs = PDSC_ADMINQ_MAX_LENGTH; err = pdsc_qcq_alloc(pdsc, PDS_CORE_QTYPE_ADMINQ, 0, "adminq", PDS_CORE_QCQ_F_CORE | PDS_CORE_QCQ_F_INTR, numdescs, diff --git a/drivers/net/ethernet/amd/pds_core/core.h b/drivers/net/ethernet/amd/pds_core/core.h index 858bebf797762..4213714085031 100644 --- a/drivers/net/ethernet/amd/pds_core/core.h +++ b/drivers/net/ethernet/amd/pds_core/core.h @@ -16,7 +16,7 @@ #define PDSC_WATCHDOG_SECS 5 #define PDSC_QUEUE_NAME_MAX_SZ 16 -#define PDSC_ADMINQ_MIN_LENGTH 16 /* must be a power of two */ +#define PDSC_ADMINQ_MAX_LENGTH 16 /* must be a power of two */ #define PDSC_NOTIFYQ_LENGTH 64 /* must be a power of two */ #define PDSC_TEARDOWN_RECOVERY false #define PDSC_TEARDOWN_REMOVING true @@ -300,8 +300,11 @@ void pdsc_health_thread(struct work_struct *work); int pdsc_register_notify(struct notifier_block *nb); void pdsc_unregister_notify(struct notifier_block *nb); void pdsc_notify(unsigned long event, void *data); -int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf); -int pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf); +int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf, + enum pds_core_vif_types vt, + struct pds_auxiliary_dev **pd_ptr); +void pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf, + struct pds_auxiliary_dev **pd_ptr); void pdsc_process_adminq(struct pdsc_qcq *qcq); void pdsc_work_thread(struct work_struct *work); diff --git a/drivers/net/ethernet/amd/pds_core/dev.c b/drivers/net/ethernet/amd/pds_core/dev.c index f0e39ab400450..e65a1632df505 100644 --- a/drivers/net/ethernet/amd/pds_core/dev.c +++ b/drivers/net/ethernet/amd/pds_core/dev.c @@ -42,6 +42,8 @@ int pdsc_err_to_errno(enum pds_core_status_code code) return -ERANGE; case PDS_RC_BAD_ADDR: return -EFAULT; + case PDS_RC_BAD_PCI: + return -ENXIO; case PDS_RC_EOPCODE: case PDS_RC_EINTR: case PDS_RC_DEV_CMD: @@ -65,7 +67,7 @@ bool pdsc_is_fw_running(struct pdsc *pdsc) /* Firmware is useful only if the running bit is set and * fw_status != 0xff (bad PCI read) */ - return (pdsc->fw_status != 0xff) && + return (pdsc->fw_status != PDS_RC_BAD_PCI) && (pdsc->fw_status & PDS_CORE_FW_STS_F_RUNNING); } @@ -131,6 +133,7 @@ static int pdsc_devcmd_wait(struct pdsc *pdsc, u8 opcode, int max_seconds) unsigned long max_wait; unsigned long duration; int timeout = 0; + bool running; int done = 0; int err = 0; int status; @@ -139,6 +142,10 @@ static int pdsc_devcmd_wait(struct pdsc *pdsc, u8 opcode, int max_seconds) max_wait = start_time + (max_seconds * HZ); while (!done && !timeout) { + running = pdsc_is_fw_running(pdsc); + if (!running) + break; + done = pdsc_devcmd_done(pdsc); if (done) break; @@ -155,7 +162,7 @@ static int pdsc_devcmd_wait(struct pdsc *pdsc, u8 opcode, int max_seconds) dev_dbg(dev, "DEVCMD %d %s after %ld secs\n", opcode, pdsc_devcmd_str(opcode), duration / HZ); - if (!done || timeout) { + if ((!done || timeout) && running) { dev_err(dev, "DEVCMD %d %s timeout, done %d timeout %d max_seconds=%d\n", opcode, pdsc_devcmd_str(opcode), done, timeout, max_seconds); diff --git a/drivers/net/ethernet/amd/pds_core/devlink.c b/drivers/net/ethernet/amd/pds_core/devlink.c index 0032e8e351811..bee70e46e34c6 100644 --- a/drivers/net/ethernet/amd/pds_core/devlink.c +++ b/drivers/net/ethernet/amd/pds_core/devlink.c @@ -55,8 +55,11 @@ int pdsc_dl_enable_set(struct devlink *dl, u32 id, for (vf_id = 0; vf_id < pdsc->num_vfs; vf_id++) { struct pdsc *vf = pdsc->vfs[vf_id].vf; - err = ctx->val.vbool ? pdsc_auxbus_dev_add(vf, pdsc) : - pdsc_auxbus_dev_del(vf, pdsc); + if (ctx->val.vbool) + err = pdsc_auxbus_dev_add(vf, pdsc, vt_entry->vif_id, + &pdsc->vfs[vf_id].padev); + else + pdsc_auxbus_dev_del(vf, pdsc, &pdsc->vfs[vf_id].padev); } return err; diff --git a/drivers/net/ethernet/amd/pds_core/main.c b/drivers/net/ethernet/amd/pds_core/main.c index eddbf0acdde77..76652e0e5b6d9 100644 --- a/drivers/net/ethernet/amd/pds_core/main.c +++ b/drivers/net/ethernet/amd/pds_core/main.c @@ -189,7 +189,8 @@ static int pdsc_init_vf(struct pdsc *vf) devl_unlock(dl); pf->vfs[vf->vf_id].vf = vf; - err = pdsc_auxbus_dev_add(vf, pf); + err = pdsc_auxbus_dev_add(vf, pf, PDS_DEV_TYPE_VDPA, + &pf->vfs[vf->vf_id].padev); if (err) { devl_lock(dl); devl_unregister(dl); @@ -415,7 +416,7 @@ static void pdsc_remove(struct pci_dev *pdev) pf = pdsc_get_pf_struct(pdsc->pdev); if (!IS_ERR(pf)) { - pdsc_auxbus_dev_del(pdsc, pf); + pdsc_auxbus_dev_del(pdsc, pf, &pf->vfs[pdsc->vf_id].padev); pf->vfs[pdsc->vf_id].vf = NULL; } } else { @@ -475,6 +476,15 @@ static void pdsc_reset_prepare(struct pci_dev *pdev) pdsc_stop_health_thread(pdsc); pdsc_fw_down(pdsc); + if (pdev->is_virtfn) { + struct pdsc *pf; + + pf = pdsc_get_pf_struct(pdsc->pdev); + if (!IS_ERR(pf)) + pdsc_auxbus_dev_del(pdsc, pf, + &pf->vfs[pdsc->vf_id].padev); + } + pdsc_unmap_bars(pdsc); pci_release_regions(pdev); pci_disable_device(pdev); @@ -510,6 +520,15 @@ static void pdsc_reset_done(struct pci_dev *pdev) pdsc_fw_up(pdsc); pdsc_restart_health_thread(pdsc); + + if (pdev->is_virtfn) { + struct pdsc *pf; + + pf = pdsc_get_pf_struct(pdsc->pdev); + if (!IS_ERR(pf)) + pdsc_auxbus_dev_add(pdsc, pf, PDS_DEV_TYPE_VDPA, + &pf->vfs[pdsc->vf_id].padev); + } } static const struct pci_error_handlers pdsc_err_handler = { diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h index 3b70f67376331..aa25a8a0a106f 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h @@ -1373,6 +1373,8 @@ #define MDIO_VEND2_CTRL1_SS13 BIT(13) #endif +#define XGBE_VEND2_MAC_AUTO_SW BIT(9) + /* MDIO mask values */ #define XGBE_AN_CL73_INT_CMPLT BIT(0) #define XGBE_AN_CL73_INC_LINK BIT(1) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c index 230726d7b74f6..d41b58fad37bb 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c @@ -373,8 +373,13 @@ static int xgbe_map_rx_buffer(struct xgbe_prv_data *pdata, } /* Set up the header page info */ - xgbe_set_buffer_data(&rdata->rx.hdr, &ring->rx_hdr_pa, - XGBE_SKB_ALLOC_SIZE); + if (pdata->netdev->features & NETIF_F_RXCSUM) { + xgbe_set_buffer_data(&rdata->rx.hdr, &ring->rx_hdr_pa, + XGBE_SKB_ALLOC_SIZE); + } else { + xgbe_set_buffer_data(&rdata->rx.hdr, &ring->rx_hdr_pa, + pdata->rx_buf_size); + } /* Set up the buffer page info */ xgbe_set_buffer_data(&rdata->rx.buf, &ring->rx_buf_pa, diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index f393228d41c7b..f1b0fb02b3cd1 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -320,6 +320,18 @@ static void xgbe_config_sph_mode(struct xgbe_prv_data *pdata) XGMAC_IOWRITE_BITS(pdata, MAC_RCR, HDSMS, XGBE_SPH_HDSMS_SIZE); } +static void xgbe_disable_sph_mode(struct xgbe_prv_data *pdata) +{ + unsigned int i; + + for (i = 0; i < pdata->channel_count; i++) { + if (!pdata->channel[i]->rx_ring) + break; + + XGMAC_DMA_IOWRITE_BITS(pdata->channel[i], DMA_CH_CR, SPH, 0); + } +} + static int xgbe_write_rss_reg(struct xgbe_prv_data *pdata, unsigned int type, unsigned int index, unsigned int val) { @@ -3545,8 +3557,12 @@ static int xgbe_init(struct xgbe_prv_data *pdata) xgbe_config_tx_coalesce(pdata); xgbe_config_rx_buffer_size(pdata); xgbe_config_tso_mode(pdata); - xgbe_config_sph_mode(pdata); - xgbe_config_rss(pdata); + + if (pdata->netdev->features & NETIF_F_RXCSUM) { + xgbe_config_sph_mode(pdata); + xgbe_config_rss(pdata); + } + desc_if->wrapper_tx_desc_init(pdata); desc_if->wrapper_rx_desc_init(pdata); xgbe_enable_dma_interrupts(pdata); @@ -3702,5 +3718,9 @@ void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *hw_if) hw_if->disable_vxlan = xgbe_disable_vxlan; hw_if->set_vxlan_id = xgbe_set_vxlan_id; + /* For Split Header*/ + hw_if->enable_sph = xgbe_config_sph_mode; + hw_if->disable_sph = xgbe_disable_sph_mode; + DBGPR("<--xgbe_init_function_ptrs\n"); } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 6b73648b37793..b4d57da71de2a 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -1172,7 +1172,6 @@ static void xgbe_free_rx_data(struct xgbe_prv_data *pdata) static int xgbe_phy_reset(struct xgbe_prv_data *pdata) { - pdata->phy_link = -1; pdata->phy_speed = SPEED_UNKNOWN; return pdata->phy_if.phy_reset(pdata); @@ -2257,10 +2256,17 @@ static int xgbe_set_features(struct net_device *netdev, if (ret) return ret; - if ((features & NETIF_F_RXCSUM) && !rxcsum) + if ((features & NETIF_F_RXCSUM) && !rxcsum) { + hw_if->enable_sph(pdata); + hw_if->enable_vxlan(pdata); hw_if->enable_rx_csum(pdata); - else if (!(features & NETIF_F_RXCSUM) && rxcsum) + schedule_work(&pdata->restart_work); + } else if (!(features & NETIF_F_RXCSUM) && rxcsum) { + hw_if->disable_sph(pdata); + hw_if->disable_vxlan(pdata); hw_if->disable_rx_csum(pdata); + schedule_work(&pdata->restart_work); + } if ((features & NETIF_F_HW_VLAN_CTAG_RX) && !rxvlan) hw_if->enable_rx_vlan_stripping(pdata); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c index 4a2dc705b5280..63012119f2c8e 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c @@ -375,6 +375,10 @@ static void xgbe_an37_set(struct xgbe_prv_data *pdata, bool enable, reg |= MDIO_VEND2_CTRL1_AN_RESTART; XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_CTRL1, reg); + + reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL); + reg |= XGBE_VEND2_MAC_AUTO_SW; + XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL, reg); } static void xgbe_an37_restart(struct xgbe_prv_data *pdata) @@ -1003,6 +1007,11 @@ static void xgbe_an37_init(struct xgbe_prv_data *pdata) netif_dbg(pdata, link, pdata->netdev, "CL37 AN (%s) initialized\n", (pdata->an_mode == XGBE_AN_MODE_CL37) ? "BaseX" : "SGMII"); + + reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_CTRL1); + reg &= ~MDIO_AN_CTRL1_ENABLE; + XMDIO_WRITE(pdata, MDIO_MMD_AN, MDIO_CTRL1, reg); + } static void xgbe_an73_init(struct xgbe_prv_data *pdata) @@ -1404,6 +1413,10 @@ static void xgbe_phy_status(struct xgbe_prv_data *pdata) pdata->phy.link = pdata->phy_if.phy_impl.link_status(pdata, &an_restart); + /* bail out if the link status register read fails */ + if (pdata->phy.link < 0) + return; + if (an_restart) { xgbe_phy_config_aneg(pdata); goto adjust_link; @@ -1651,6 +1664,7 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata) pdata->phy.duplex = DUPLEX_FULL; } + pdata->phy_link = 0; pdata->phy.link = 0; pdata->phy.pause_autoneg = pdata->pause_autoneg; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index 268399dfcf22f..32e633d113484 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -2855,8 +2855,7 @@ static bool xgbe_phy_valid_speed(struct xgbe_prv_data *pdata, int speed) static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) { struct xgbe_phy_data *phy_data = pdata->phy_data; - unsigned int reg; - int ret; + int reg, ret; *an_restart = 0; @@ -2890,11 +2889,20 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) return 0; } - /* Link status is latched low, so read once to clear - * and then read again to get current state - */ - reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); + if (reg < 0) + return reg; + + /* Link status is latched low so that momentary link drops + * can be detected. If link was already down read again + * to get the latest state. + */ + + if (!pdata->phy.link && !(reg & MDIO_STAT1_LSTATUS)) { + reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); + if (reg < 0) + return reg; + } if (pdata->en_rx_adap) { /* if the link is available and adaptation is done, @@ -2913,9 +2921,7 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) xgbe_phy_set_mode(pdata, phy_data->cur_mode); } - /* check again for the link and adaptation status */ - reg = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_STAT1); - if ((reg & MDIO_STAT1_LSTATUS) && pdata->rx_adapt_done) + if (pdata->rx_adapt_done) return 1; } else if (reg & MDIO_STAT1_LSTATUS) return 1; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index ad136ed493ed1..a596cd08124fa 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -292,12 +292,12 @@ #define XGBE_LINK_TIMEOUT 5 #define XGBE_KR_TRAINING_WAIT_ITER 50 -#define XGBE_SGMII_AN_LINK_STATUS BIT(1) +#define XGBE_SGMII_AN_LINK_DUPLEX BIT(1) #define XGBE_SGMII_AN_LINK_SPEED (BIT(2) | BIT(3)) #define XGBE_SGMII_AN_LINK_SPEED_10 0x00 #define XGBE_SGMII_AN_LINK_SPEED_100 0x04 #define XGBE_SGMII_AN_LINK_SPEED_1000 0x08 -#define XGBE_SGMII_AN_LINK_DUPLEX BIT(4) +#define XGBE_SGMII_AN_LINK_STATUS BIT(4) /* ECC correctable error notification window (seconds) */ #define XGBE_ECC_LIMIT 60 @@ -865,6 +865,10 @@ struct xgbe_hw_if { void (*enable_vxlan)(struct xgbe_prv_data *); void (*disable_vxlan)(struct xgbe_prv_data *); void (*set_vxlan_id)(struct xgbe_prv_data *); + + /* For Split Header */ + void (*enable_sph)(struct xgbe_prv_data *pdata); + void (*disable_sph)(struct xgbe_prv_data *pdata); }; /* This structure represents implementation specific routines for an diff --git a/drivers/net/ethernet/apm/xgene-v2/main.c b/drivers/net/ethernet/apm/xgene-v2/main.c index 379d19d18dbed..5808e3c73a8f4 100644 --- a/drivers/net/ethernet/apm/xgene-v2/main.c +++ b/drivers/net/ethernet/apm/xgene-v2/main.c @@ -9,8 +9,6 @@ #include "main.h" -static const struct acpi_device_id xge_acpi_match[]; - static int xge_get_resources(struct xge_pdata *pdata) { struct platform_device *pdev; @@ -733,7 +731,7 @@ MODULE_DEVICE_TABLE(acpi, xge_acpi_match); static struct platform_driver xge_driver = { .driver = { .name = "xgene-enet-v2", - .acpi_match_table = ACPI_PTR(xge_acpi_match), + .acpi_match_table = xge_acpi_match, }, .probe = xge_probe, .remove = xge_remove, diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h index dbd2846601358..7f616abd3db2c 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h +++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h @@ -113,6 +113,8 @@ struct aq_stats_s { #define AQ_HW_POWER_STATE_D0 0U #define AQ_HW_POWER_STATE_D3 3U +#define AQ_FW_WAKE_ON_LINK_RTPM BIT(10) + #define AQ_HW_FLAG_STARTED 0x00000004U #define AQ_HW_FLAG_STOPPING 0x00000008U #define AQ_HW_FLAG_RESETTING 0x00000010U diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_main.c b/drivers/net/ethernet/aquantia/atlantic/aq_main.c index 0b2a52199914b..75d436c906968 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_main.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_main.c @@ -123,7 +123,6 @@ static netdev_tx_t aq_ndev_start_xmit(struct sk_buff *skb, struct net_device *nd } #endif - skb_tx_timestamp(skb); return aq_nic_xmit(aq_nic, skb); } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c index c9b0d57696a48..07392174f6437 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c @@ -898,6 +898,8 @@ int aq_nic_xmit(struct aq_nic_s *self, struct sk_buff *skb) frags = aq_nic_map_skb(self, skb, ring); + skb_tx_timestamp(skb); + if (likely(frags)) { err = self->aq_hw_ops->hw_ring_tx_xmit(self->aq_hw, ring, frags); diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c index 52e2070a4a2f0..7370e3f76b620 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c @@ -462,6 +462,44 @@ static int aq_a2_fw_get_mac_temp(struct aq_hw_s *self, int *temp) return aq_a2_fw_get_phy_temp(self, temp); } +static int aq_a2_fw_set_wol_params(struct aq_hw_s *self, const u8 *mac, u32 wol) +{ + struct mac_address_aligned_s mac_address; + struct link_control_s link_control; + struct wake_on_lan_s wake_on_lan; + + memcpy(mac_address.aligned.mac_address, mac, ETH_ALEN); + hw_atl2_shared_buffer_write(self, mac_address, mac_address); + + memset(&wake_on_lan, 0, sizeof(wake_on_lan)); + + if (wol & WAKE_MAGIC) + wake_on_lan.wake_on_magic_packet = 1U; + + if (wol & (WAKE_PHY | AQ_FW_WAKE_ON_LINK_RTPM)) + wake_on_lan.wake_on_link_up = 1U; + + hw_atl2_shared_buffer_write(self, sleep_proxy, wake_on_lan); + + hw_atl2_shared_buffer_get(self, link_control, link_control); + link_control.mode = AQ_HOST_MODE_SLEEP_PROXY; + hw_atl2_shared_buffer_write(self, link_control, link_control); + + return hw_atl2_shared_buffer_finish_ack(self); +} + +static int aq_a2_fw_set_power(struct aq_hw_s *self, unsigned int power_state, + const u8 *mac) +{ + u32 wol = self->aq_nic_cfg->wol; + int err = 0; + + if (wol) + err = aq_a2_fw_set_wol_params(self, mac, wol); + + return err; +} + static int aq_a2_fw_set_eee_rate(struct aq_hw_s *self, u32 speed) { struct link_options_s link_options; @@ -605,6 +643,7 @@ const struct aq_fw_ops aq_a2_fw_ops = { .set_state = aq_a2_fw_set_state, .update_link_status = aq_a2_fw_update_link_status, .update_stats = aq_a2_fw_update_stats, + .set_power = aq_a2_fw_set_power, .get_mac_temp = aq_a2_fw_get_mac_temp, .get_phy_temp = aq_a2_fw_get_phy_temp, .set_eee_rate = aq_a2_fw_set_eee_rate, diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c index 009e0b3066fa3..baf12ae0b8c4c 100644 --- a/drivers/net/ethernet/atheros/ag71xx.c +++ b/drivers/net/ethernet/atheros/ag71xx.c @@ -1234,6 +1234,11 @@ static bool ag71xx_fill_rx_buf(struct ag71xx *ag, struct ag71xx_buf *buf, buf->rx.rx_buf = data; buf->rx.dma_addr = dma_map_single(&ag->pdev->dev, data, ag->rx_buf_size, DMA_FROM_DEVICE); + if (dma_mapping_error(&ag->pdev->dev, buf->rx.dma_addr)) { + skb_free_frag(data); + buf->rx.rx_buf = NULL; + return false; + } desc->data = (u32)buf->rx.dma_addr + offset; return true; } @@ -1532,6 +1537,10 @@ static netdev_tx_t ag71xx_hard_start_xmit(struct sk_buff *skb, dma_addr = dma_map_single(&ag->pdev->dev, skb->data, skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(&ag->pdev->dev, dma_addr)) { + netif_dbg(ag, tx_err, ndev, "DMA mapping error\n"); + goto err_drop; + } i = ring->curr & ring_mask; desc = ag71xx_ring_desc(ring, i); diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index 02aa6fd8ebc2d..4ed165702d58e 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -1861,14 +1861,21 @@ static u16 atl1_alloc_rx_buffers(struct atl1_adapter *adapter) break; } - buffer_info->alloced = 1; - buffer_info->skb = skb; - buffer_info->length = (u16) adapter->rx_buffer_len; page = virt_to_page(skb->data); offset = offset_in_page(skb->data); buffer_info->dma = dma_map_page(&pdev->dev, page, offset, adapter->rx_buffer_len, DMA_FROM_DEVICE); + if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { + kfree_skb(skb); + adapter->soft_stats.rx_dropped++; + break; + } + + buffer_info->alloced = 1; + buffer_info->skb = skb; + buffer_info->length = (u16)adapter->rx_buffer_len; + rfd_desc->buffer_addr = cpu_to_le64(buffer_info->dma); rfd_desc->buf_len = cpu_to_le16(adapter->rx_buffer_len); rfd_desc->coalese = 0; @@ -2183,8 +2190,8 @@ static int atl1_tx_csum(struct atl1_adapter *adapter, struct sk_buff *skb, return 0; } -static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, - struct tx_packet_desc *ptpd) +static bool atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, + struct tx_packet_desc *ptpd) { struct atl1_tpd_ring *tpd_ring = &adapter->tpd_ring; struct atl1_buffer *buffer_info; @@ -2194,6 +2201,7 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, unsigned int nr_frags; unsigned int f; int retval; + u16 first_mapped; u16 next_to_use; u16 data_len; u8 hdr_len; @@ -2201,6 +2209,7 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, buf_len -= skb->data_len; nr_frags = skb_shinfo(skb)->nr_frags; next_to_use = atomic_read(&tpd_ring->next_to_use); + first_mapped = next_to_use; buffer_info = &tpd_ring->buffer_info[next_to_use]; BUG_ON(buffer_info->skb); /* put skb in last TPD */ @@ -2216,6 +2225,8 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, buffer_info->dma = dma_map_page(&adapter->pdev->dev, page, offset, hdr_len, DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) + goto dma_err; if (++next_to_use == tpd_ring->count) next_to_use = 0; @@ -2242,6 +2253,9 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, page, offset, buffer_info->length, DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, + buffer_info->dma)) + goto dma_err; if (++next_to_use == tpd_ring->count) next_to_use = 0; } @@ -2254,6 +2268,8 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, buffer_info->dma = dma_map_page(&adapter->pdev->dev, page, offset, buf_len, DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) + goto dma_err; if (++next_to_use == tpd_ring->count) next_to_use = 0; } @@ -2277,6 +2293,9 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, buffer_info->dma = skb_frag_dma_map(&adapter->pdev->dev, frag, i * ATL1_MAX_TX_BUF_LEN, buffer_info->length, DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->pdev->dev, + buffer_info->dma)) + goto dma_err; if (++next_to_use == tpd_ring->count) next_to_use = 0; @@ -2285,6 +2304,22 @@ static void atl1_tx_map(struct atl1_adapter *adapter, struct sk_buff *skb, /* last tpd's buffer-info */ buffer_info->skb = skb; + + return true; + + dma_err: + while (first_mapped != next_to_use) { + buffer_info = &tpd_ring->buffer_info[first_mapped]; + dma_unmap_page(&adapter->pdev->dev, + buffer_info->dma, + buffer_info->length, + DMA_TO_DEVICE); + buffer_info->dma = 0; + + if (++first_mapped == tpd_ring->count) + first_mapped = 0; + } + return false; } static void atl1_tx_queue(struct atl1_adapter *adapter, u16 count, @@ -2355,10 +2390,8 @@ static netdev_tx_t atl1_xmit_frame(struct sk_buff *skb, len = skb_headlen(skb); - if (unlikely(skb->len <= 0)) { - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } + if (unlikely(skb->len <= 0)) + goto drop_packet; nr_frags = skb_shinfo(skb)->nr_frags; for (f = 0; f < nr_frags; f++) { @@ -2371,10 +2404,9 @@ static netdev_tx_t atl1_xmit_frame(struct sk_buff *skb, if (mss) { if (skb->protocol == htons(ETH_P_IP)) { proto_hdr_len = skb_tcp_all_headers(skb); - if (unlikely(proto_hdr_len > len)) { - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } + if (unlikely(proto_hdr_len > len)) + goto drop_packet; + /* need additional TPD ? */ if (proto_hdr_len != len) count += (len - proto_hdr_len + @@ -2406,23 +2438,26 @@ static netdev_tx_t atl1_xmit_frame(struct sk_buff *skb, } tso = atl1_tso(adapter, skb, ptpd); - if (tso < 0) { - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } + if (tso < 0) + goto drop_packet; if (!tso) { ret_val = atl1_tx_csum(adapter, skb, ptpd); - if (ret_val < 0) { - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } + if (ret_val < 0) + goto drop_packet; } - atl1_tx_map(adapter, skb, ptpd); + if (!atl1_tx_map(adapter, skb, ptpd)) + goto drop_packet; + atl1_tx_queue(adapter, count, ptpd); atl1_update_mailbox(adapter); return NETDEV_TX_OK; + +drop_packet: + adapter->soft_stats.tx_errors++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; } static int atl1_rings_clean(struct napi_struct *napi, int budget) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 6bf4a21853858..8e4e8291d8c66 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -2491,6 +2491,7 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, { struct bnxt_napi *bnapi = cpr->bnapi; u32 raw_cons = cpr->cp_raw_cons; + bool flush_xdp = false; u32 cons; int tx_pkts = 0; int rx_pkts = 0; @@ -2528,6 +2529,8 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, else rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, &event); + if (event & BNXT_REDIRECT_EVENT) + flush_xdp = true; if (likely(rc >= 0)) rx_pkts += rc; /* Increment rx_pkts when rc is -ENOMEM to count towards @@ -2555,8 +2558,10 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } } - if (event & BNXT_REDIRECT_EVENT) + if (flush_xdp) { xdp_do_flush(); + event &= ~BNXT_REDIRECT_EVENT; + } if (event & BNXT_TX_EVENT) { struct bnxt_tx_ring_info *txr = bnapi->tx_ring; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.c index c067898820360..32813cdd5aa5c 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.c @@ -66,20 +66,30 @@ static int bnxt_hwrm_dbg_dma_data(struct bnxt *bp, void *msg, } } + if (cmn_req->req_type == + cpu_to_le16(HWRM_DBG_COREDUMP_RETRIEVE)) + info->dest_buf_size += len; + if (info->dest_buf) { if ((info->seg_start + off + len) <= BNXT_COREDUMP_BUF_LEN(info->buf_len)) { - memcpy(info->dest_buf + off, dma_buf, len); + u16 copylen = min_t(u16, len, + info->dest_buf_size - off); + + memcpy(info->dest_buf + off, dma_buf, copylen); + if (copylen < len) + break; } else { rc = -ENOBUFS; + if (cmn_req->req_type == + cpu_to_le16(HWRM_DBG_COREDUMP_LIST)) { + kfree(info->dest_buf); + info->dest_buf = NULL; + } break; } } - if (cmn_req->req_type == - cpu_to_le16(HWRM_DBG_COREDUMP_RETRIEVE)) - info->dest_buf_size += len; - if (!(cmn_resp->flags & HWRM_DBG_CMN_FLAGS_MORE)) break; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c index 63e0670383852..1727e9bb1479d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c @@ -487,7 +487,9 @@ static int bnxt_ets_validate(struct bnxt *bp, struct ieee_ets *ets, u8 *tc) if ((ets->tc_tx_bw[i] || ets->tc_tsa[i]) && i > bp->max_tc) return -EINVAL; + } + for (i = 0; i < max_tc; i++) { switch (ets->tc_tsa[i]) { case IEEE_8021QAZ_TSA_STRICT: break; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 2e7ddbca9d53b..dcedafa4d2e14 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -1393,6 +1393,17 @@ static int bnxt_get_regs_len(struct net_device *dev) return reg_len; } +#define BNXT_PCIE_32B_ENTRY(start, end) \ + { offsetof(struct pcie_ctx_hw_stats, start), \ + offsetof(struct pcie_ctx_hw_stats, end) } + +static const struct { + u16 start; + u16 end; +} bnxt_pcie_32b_entries[] = { + BNXT_PCIE_32B_ENTRY(pcie_ltssm_histogram[0], pcie_ltssm_histogram[3]), +}; + static void bnxt_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *_p) { @@ -1424,12 +1435,27 @@ static void bnxt_get_regs(struct net_device *dev, struct ethtool_regs *regs, req->pcie_stat_host_addr = cpu_to_le64(hw_pcie_stats_addr); rc = hwrm_req_send(bp, req); if (!rc) { - __le64 *src = (__le64 *)hw_pcie_stats; - u64 *dst = (u64 *)(_p + BNXT_PXP_REG_LEN); - int i; - - for (i = 0; i < sizeof(*hw_pcie_stats) / sizeof(__le64); i++) - dst[i] = le64_to_cpu(src[i]); + u8 *dst = (u8 *)(_p + BNXT_PXP_REG_LEN); + u8 *src = (u8 *)hw_pcie_stats; + int i, j; + + for (i = 0, j = 0; i < sizeof(*hw_pcie_stats); ) { + if (i >= bnxt_pcie_32b_entries[j].start && + i <= bnxt_pcie_32b_entries[j].end) { + u32 *dst32 = (u32 *)(dst + i); + + *dst32 = le32_to_cpu(*(__le32 *)(src + i)); + i += 4; + if (i > bnxt_pcie_32b_entries[j].end && + j < ARRAY_SIZE(bnxt_pcie_32b_entries) - 1) + j++; + } else { + u64 *dst64 = (u64 *)(dst + i); + + *dst64 = le64_to_cpu(*(__le64 *)(src + i)); + i += 8; + } + } } hwrm_req_drop(bp, req); } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c index bbe8657f6545b..404b433f1bc08 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c @@ -917,9 +917,9 @@ static void bnxt_ptp_free(struct bnxt *bp) if (ptp->ptp_clock) { ptp_clock_unregister(ptp->ptp_clock); ptp->ptp_clock = NULL; - kfree(ptp->ptp_info.pin_config); - ptp->ptp_info.pin_config = NULL; } + kfree(ptp->ptp_info.pin_config); + ptp->ptp_info.pin_config = NULL; } int bnxt_ptp_init(struct bnxt *bp) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c index 273c9ba48f09a..b896f8d20ff33 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c @@ -244,7 +244,7 @@ bnxt_tc_parse_pedit(struct bnxt *bp, struct bnxt_tc_actions *actions, offset < offset_of_ip6_daddr + 16) { actions->nat.src_xlate = false; idx = (offset - offset_of_ip6_daddr) / 4; - actions->nat.l3.ipv6.saddr.s6_addr32[idx] = htonl(val); + actions->nat.l3.ipv6.daddr.s6_addr32[idx] = htonl(val); } else { netdev_err(bp->dev, "%s: IPv6_hdr: Invalid pedit field\n", diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 758f51366ef03..07a458ecb7cc9 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -115,7 +115,7 @@ static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, tx_buf->action = XDP_REDIRECT; tx_buf->xdpf = xdpf; dma_unmap_addr_set(tx_buf, mapping, mapping); - dma_unmap_len_set(tx_buf, len, 0); + dma_unmap_len_set(tx_buf, len, len); } void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) diff --git a/drivers/net/ethernet/broadcom/cnic.c b/drivers/net/ethernet/broadcom/cnic.c index 7926aaef8f0c5..ad2745c07c1ae 100644 --- a/drivers/net/ethernet/broadcom/cnic.c +++ b/drivers/net/ethernet/broadcom/cnic.c @@ -4220,8 +4220,7 @@ static void cnic_cm_stop_bnx2x_hw(struct cnic_dev *dev) cnic_bnx2x_delete_wait(dev, 0); - cancel_delayed_work(&cp->delete_task); - flush_workqueue(cnic_wq); + cancel_delayed_work_sync(&cp->delete_task); if (atomic_read(&cp->iscsi_conn) != 0) netdev_warn(dev->netdev, "%d iSCSI connections not destroyed\n", diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index b3878975bd9c0..ea4973096aa28 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -5814,7 +5814,7 @@ static int tg3_setup_fiber_mii_phy(struct tg3 *tp, bool force_reset) u32 current_speed = SPEED_UNKNOWN; u8 current_duplex = DUPLEX_UNKNOWN; bool current_link_up = false; - u32 local_adv, remote_adv, sgsr; + u32 local_adv = 0, remote_adv = 0, sgsr; if ((tg3_asic_rev(tp) == ASIC_REV_5719 || tg3_asic_rev(tp) == ASIC_REV_5720) && @@ -5955,9 +5955,6 @@ static int tg3_setup_fiber_mii_phy(struct tg3 *tp, bool force_reset) else current_duplex = DUPLEX_HALF; - local_adv = 0; - remote_adv = 0; - if (bmcr & BMCR_ANENABLE) { u32 common; diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 4325d0ace1f26..7593255e6e53d 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -281,9 +281,9 @@ static void macb_set_hwaddr(struct macb *bp) u32 bottom; u16 top; - bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); + bottom = get_unaligned_le32(bp->dev->dev_addr); macb_or_gem_writel(bp, SA1B, bottom); - top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); + top = get_unaligned_le16(bp->dev->dev_addr + 4); macb_or_gem_writel(bp, SA1T, top); if (gem_has_ptp(bp)) { @@ -1016,22 +1016,15 @@ static void macb_update_stats(struct macb *bp) static int macb_halt_tx(struct macb *bp) { - unsigned long halt_time, timeout; - u32 status; + u32 status; macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); - timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); - do { - halt_time = jiffies; - status = macb_readl(bp, TSR); - if (!(status & MACB_BIT(TGO))) - return 0; - - udelay(250); - } while (time_before(halt_time, timeout)); - - return -ETIMEDOUT; + /* Poll TSR until TGO is cleared or timeout. */ + return read_poll_timeout_atomic(macb_readl, status, + !(status & MACB_BIT(TGO)), + 250, MACB_HALT_TIMEOUT, false, + bp, TSR); } static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) @@ -1241,11 +1234,12 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) { struct macb *bp = queue->bp; u16 queue_index = queue - bp->queues; + unsigned long flags; unsigned int tail; unsigned int head; int packets = 0; - spin_lock(&queue->tx_ptr_lock); + spin_lock_irqsave(&queue->tx_ptr_lock, flags); head = queue->tx_head; for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { struct macb_tx_skb *tx_skb; @@ -1304,7 +1298,7 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) CIRC_CNT(queue->tx_head, queue->tx_tail, bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) netif_wake_subqueue(bp->dev, queue_index); - spin_unlock(&queue->tx_ptr_lock); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); return packets; } @@ -1720,8 +1714,9 @@ static void macb_tx_restart(struct macb_queue *queue) { struct macb *bp = queue->bp; unsigned int head_idx, tbqp; + unsigned long flags; - spin_lock(&queue->tx_ptr_lock); + spin_lock_irqsave(&queue->tx_ptr_lock, flags); if (queue->tx_head == queue->tx_tail) goto out_tx_ptr_unlock; @@ -1733,19 +1728,20 @@ static void macb_tx_restart(struct macb_queue *queue) if (tbqp == head_idx) goto out_tx_ptr_unlock; - spin_lock_irq(&bp->lock); + spin_lock(&bp->lock); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); - spin_unlock_irq(&bp->lock); + spin_unlock(&bp->lock); out_tx_ptr_unlock: - spin_unlock(&queue->tx_ptr_lock); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); } static bool macb_tx_complete_pending(struct macb_queue *queue) { bool retval = false; + unsigned long flags; - spin_lock(&queue->tx_ptr_lock); + spin_lock_irqsave(&queue->tx_ptr_lock, flags); if (queue->tx_head != queue->tx_tail) { /* Make hw descriptor updates visible to CPU */ rmb(); @@ -1753,7 +1749,7 @@ static bool macb_tx_complete_pending(struct macb_queue *queue) if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) retval = true; } - spin_unlock(&queue->tx_ptr_lock); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); return retval; } @@ -2321,6 +2317,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) struct macb_queue *queue = &bp->queues[queue_index]; unsigned int desc_cnt, nr_frags, frag_size, f; unsigned int hdrlen; + unsigned long flags; bool is_lso; netdev_tx_t ret = NETDEV_TX_OK; @@ -2381,7 +2378,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); } - spin_lock_bh(&queue->tx_ptr_lock); + spin_lock_irqsave(&queue->tx_ptr_lock, flags); /* This is a hard error, log it. */ if (CIRC_SPACE(queue->tx_head, queue->tx_tail, @@ -2403,15 +2400,15 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) wmb(); skb_tx_timestamp(skb); - spin_lock_irq(&bp->lock); + spin_lock(&bp->lock); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); - spin_unlock_irq(&bp->lock); + spin_unlock(&bp->lock); if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) netif_stop_subqueue(dev, queue_index); unlock: - spin_unlock_bh(&queue->tx_ptr_lock); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); return ret; } @@ -5077,7 +5074,11 @@ static int macb_probe(struct platform_device *pdev) #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (err) { + dev_err(&pdev->dev, "failed to set DMA mask\n"); + goto err_out_free_netdev; + } bp->hw_dma_cap |= HW_DMA_CAP_64B; } #endif diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c index de8a6ce86ad7e..12105ffb5dac6 100644 --- a/drivers/net/ethernet/cavium/liquidio/request_manager.c +++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c @@ -126,7 +126,7 @@ int octeon_init_instr_queue(struct octeon_device *oct, oct->io_qmask.iq |= BIT_ULL(iq_no); /* Set the 32B/64B mode for each input queue */ - oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no); + oct->io_qmask.iq64B |= ((u64)(conf->instr_type == 64) << iq_no); iq->iqcmd_64B = (conf->instr_type == 64); oct->fn_list.setup_iq_regs(oct, iq_no); diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c index a317feb8decb6..a423a93882115 100644 --- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c +++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c @@ -1427,9 +1427,9 @@ static acpi_status bgx_acpi_match_id(acpi_handle handle, u32 lvl, { struct acpi_buffer string = { ACPI_ALLOCATE_BUFFER, NULL }; struct bgx *bgx = context; - char bgx_sel[5]; + char bgx_sel[7]; - snprintf(bgx_sel, 5, "BGX%d", bgx->bgx_id); + snprintf(bgx_sel, sizeof(bgx_sel), "BGX%d", bgx->bgx_id); if (ACPI_FAILURE(acpi_get_name(handle, ACPI_SINGLE_NAME, &string))) { pr_warn("Invalid link device\n"); return AE_OK; @@ -1491,13 +1491,17 @@ static int bgx_init_of_phy(struct bgx *bgx) * this cortina phy, for which there is no driver * support, ignore it. */ - if (phy_np && - !of_device_is_compatible(phy_np, "cortina,cs4223-slice")) { - /* Wait until the phy drivers are available */ - pd = of_phy_find_device(phy_np); - if (!pd) - goto defer; - bgx->lmac[lmac].phydev = pd; + if (phy_np) { + if (!of_device_is_compatible(phy_np, "cortina,cs4223-slice")) { + /* Wait until the phy drivers are available */ + pd = of_phy_find_device(phy_np); + if (!pd) { + of_node_put(phy_np); + goto defer; + } + bgx->lmac[lmac].phydev = pd; + } + of_node_put(phy_np); } lmac++; @@ -1513,11 +1517,11 @@ static int bgx_init_of_phy(struct bgx *bgx) * for phy devices we may have already found. */ while (lmac) { + lmac--; if (bgx->lmac[lmac].phydev) { put_device(&bgx->lmac[lmac].phydev->mdio.dev); bgx->lmac[lmac].phydev = NULL; } - lmac--; } of_node_put(node); return -EPROBE_DEFER; diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index cccf0db2fb4e5..48701032c20c5 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -2057,10 +2057,10 @@ static int enic_change_mtu(struct net_device *netdev, int new_mtu) if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic)) return -EOPNOTSUPP; - if (netdev->mtu > enic->port_mtu) + if (new_mtu > enic->port_mtu) netdev_warn(netdev, "interface MTU (%d) set higher than port MTU (%d)\n", - netdev->mtu, enic->port_mtu); + new_mtu, enic->port_mtu); return _enic_change_mtu(netdev, new_mtu); } diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 5af98fba74803..fce2ff1e1d834 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -1148,6 +1148,7 @@ static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb, struct gmac_txdesc *txd; skb_frag_t *skb_frag; dma_addr_t mapping; + bool tcp = false; void *buffer; u16 mss; int ret; @@ -1155,6 +1156,13 @@ static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb, word1 = skb->len; word3 = SOF_BIT; + /* Determine if we are doing TCP */ + if (skb->protocol == htons(ETH_P_IP)) + tcp = (ip_hdr(skb)->protocol == IPPROTO_TCP); + else + /* IPv6 */ + tcp = (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP); + mss = skb_shinfo(skb)->gso_size; if (mss) { /* This means we are dealing with TCP and skb->len is the @@ -1167,8 +1175,26 @@ static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb, mss, skb->len); word1 |= TSS_MTU_ENABLE_BIT; word3 |= mss; + } else if (tcp) { + /* Even if we are not using TSO, use the hardware offloader + * for transferring the TCP frame: this hardware has partial + * TCP awareness (called TOE - TCP Offload Engine) and will + * according to the datasheet put packets belonging to the + * same TCP connection in the same queue for the TOE/TSO + * engine to process. The engine will deal with chopping + * up frames that exceed ETH_DATA_LEN which the + * checksumming engine cannot handle (see below) into + * manageable chunks. It flawlessly deals with quite big + * frames and frames containing custom DSA EtherTypes. + */ + mss = netdev->mtu + skb_tcp_all_headers(skb); + mss = min(mss, skb->len); + netdev_dbg(netdev, "TOE/TSO len %04x mtu %04x mss %04x\n", + skb->len, netdev->mtu, mss); + word1 |= TSS_MTU_ENABLE_BIT; + word3 |= mss; } else if (skb->len >= ETH_FRAME_LEN) { - /* Hardware offloaded checksumming isn't working on frames + /* Hardware offloaded checksumming isn't working on non-TCP frames * bigger than 1514 bytes. A hypothesis about this is that the * checksum buffer is only 1518 bytes, so when the frames get * bigger they get truncated, or the last few bytes get @@ -1185,21 +1211,16 @@ static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb, } if (skb->ip_summed == CHECKSUM_PARTIAL) { - int tcp = 0; - /* We do not switch off the checksumming on non TCP/UDP * frames: as is shown from tests, the checksumming engine * is smart enough to see that a frame is not actually TCP * or UDP and then just pass it through without any changes * to the frame. */ - if (skb->protocol == htons(ETH_P_IP)) { + if (skb->protocol == htons(ETH_P_IP)) word1 |= TSS_IP_CHKSUM_BIT; - tcp = ip_hdr(skb)->protocol == IPPROTO_TCP; - } else { /* IPv6 */ + else word1 |= TSS_IPV6_ENABLE_BIT; - tcp = ipv6_hdr(skb)->nexthdr == IPPROTO_TCP; - } word1 |= tcp ? TSS_TCP_CHKSUM_BIT : TSS_UDP_CHKSUM_BIT; } diff --git a/drivers/net/ethernet/dlink/dl2k.c b/drivers/net/ethernet/dlink/dl2k.c index db6615aa921b1..72c97dcd0fee0 100644 --- a/drivers/net/ethernet/dlink/dl2k.c +++ b/drivers/net/ethernet/dlink/dl2k.c @@ -146,6 +146,8 @@ rio_probe1 (struct pci_dev *pdev, const struct pci_device_id *ent) np->ioaddr = ioaddr; np->chip_id = chip_idx; np->pdev = pdev; + + spin_lock_init(&np->stats_lock); spin_lock_init (&np->tx_lock); spin_lock_init (&np->rx_lock); @@ -352,7 +354,7 @@ parse_eeprom (struct net_device *dev) eth_hw_addr_set(dev, psrom->mac_addr); if (np->chip_id == CHIP_IP1000A) { - np->led_mode = psrom->led_mode; + np->led_mode = le16_to_cpu(psrom->led_mode); return 0; } @@ -496,25 +498,34 @@ static int alloc_list(struct net_device *dev) for (i = 0; i < RX_RING_SIZE; i++) { /* Allocated fixed size of skbuff */ struct sk_buff *skb; + dma_addr_t addr; skb = netdev_alloc_skb_ip_align(dev, np->rx_buf_sz); np->rx_skbuff[i] = skb; - if (!skb) { - free_list(dev); - return -ENOMEM; - } + if (!skb) + goto err_free_list; + + addr = dma_map_single(&np->pdev->dev, skb->data, + np->rx_buf_sz, DMA_FROM_DEVICE); + if (dma_mapping_error(&np->pdev->dev, addr)) + goto err_kfree_skb; np->rx_ring[i].next_desc = cpu_to_le64(np->rx_ring_dma + ((i + 1) % RX_RING_SIZE) * sizeof(struct netdev_desc)); /* Rubicon now supports 40 bits of addressing space. */ - np->rx_ring[i].fraginfo = - cpu_to_le64(dma_map_single(&np->pdev->dev, skb->data, - np->rx_buf_sz, DMA_FROM_DEVICE)); + np->rx_ring[i].fraginfo = cpu_to_le64(addr); np->rx_ring[i].fraginfo |= cpu_to_le64((u64)np->rx_buf_sz << 48); } return 0; + +err_kfree_skb: + dev_kfree_skb(np->rx_skbuff[i]); + np->rx_skbuff[i] = NULL; +err_free_list: + free_list(dev); + return -ENOMEM; } static void rio_hw_init(struct net_device *dev) @@ -866,7 +877,6 @@ tx_error (struct net_device *dev, int tx_status) frame_id = (tx_status & 0xffff0000); printk (KERN_ERR "%s: Transmit error, TxStatus %4.4x, FrameId %d.\n", dev->name, tx_status, frame_id); - dev->stats.tx_errors++; /* Ttransmit Underrun */ if (tx_status & 0x10) { dev->stats.tx_fifo_errors++; @@ -903,9 +913,15 @@ tx_error (struct net_device *dev, int tx_status) rio_set_led_mode(dev); /* Let TxStartThresh stay default value */ } + + spin_lock(&np->stats_lock); /* Maximum Collisions */ if (tx_status & 0x08) dev->stats.collisions++; + + dev->stats.tx_errors++; + spin_unlock(&np->stats_lock); + /* Restart the Tx */ dw32(MACCtrl, dr16(MACCtrl) | TxEnable); } @@ -947,15 +963,18 @@ receive_packet (struct net_device *dev) } else { struct sk_buff *skb; + skb = NULL; /* Small skbuffs for short packets */ - if (pkt_len > copy_thresh) { + if (pkt_len <= copy_thresh) + skb = netdev_alloc_skb_ip_align(dev, pkt_len); + if (!skb) { dma_unmap_single(&np->pdev->dev, desc_to_dma(desc), np->rx_buf_sz, DMA_FROM_DEVICE); skb_put (skb = np->rx_skbuff[entry], pkt_len); np->rx_skbuff[entry] = NULL; - } else if ((skb = netdev_alloc_skb_ip_align(dev, pkt_len))) { + } else { dma_sync_single_for_cpu(&np->pdev->dev, desc_to_dma(desc), np->rx_buf_sz, @@ -1074,7 +1093,9 @@ get_stats (struct net_device *dev) int i; #endif unsigned int stat_reg; + unsigned long flags; + spin_lock_irqsave(&np->stats_lock, flags); /* All statistics registers need to be acknowledged, else statistic overflow could cause problems */ @@ -1083,7 +1104,7 @@ get_stats (struct net_device *dev) dev->stats.rx_bytes += dr32(OctetRcvOk); dev->stats.tx_bytes += dr32(OctetXmtOk); - dev->stats.multicast = dr32(McstFramesRcvdOk); + dev->stats.multicast += dr32(McstFramesRcvdOk); dev->stats.collisions += dr32(SingleColFrames) + dr32(MultiColFrames); @@ -1124,6 +1145,9 @@ get_stats (struct net_device *dev) dr16(TCPCheckSumErrors); dr16(UDPCheckSumErrors); dr16(IPCheckSumErrors); + + spin_unlock_irqrestore(&np->stats_lock, flags); + return &dev->stats; } diff --git a/drivers/net/ethernet/dlink/dl2k.h b/drivers/net/ethernet/dlink/dl2k.h index 195dc6cfd8955..56aff2f0bdbfa 100644 --- a/drivers/net/ethernet/dlink/dl2k.h +++ b/drivers/net/ethernet/dlink/dl2k.h @@ -335,7 +335,7 @@ typedef struct t_SROM { u16 sub_system_id; /* 0x06 */ u16 pci_base_1; /* 0x08 (IP1000A only) */ u16 pci_base_2; /* 0x0a (IP1000A only) */ - u16 led_mode; /* 0x0c (IP1000A only) */ + __le16 led_mode; /* 0x0c (IP1000A only) */ u16 reserved1[9]; /* 0x0e-0x1f */ u8 mac_addr[6]; /* 0x20-0x25 */ u8 reserved2[10]; /* 0x26-0x2f */ @@ -372,6 +372,8 @@ struct netdev_private { struct pci_dev *pdev; void __iomem *ioaddr; void __iomem *eeprom_addr; + // To ensure synchronization when stats are updated. + spinlock_t stats_lock; spinlock_t tx_lock; spinlock_t rx_lock; unsigned int rx_buf_sz; /* Based on MTU+slack. */ diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c index 51b8377edd1d0..779f1324bb5f8 100644 --- a/drivers/net/ethernet/emulex/benet/be_cmds.c +++ b/drivers/net/ethernet/emulex/benet/be_cmds.c @@ -1609,7 +1609,7 @@ int be_cmd_get_stats(struct be_adapter *adapter, struct be_dma_mem *nonemb_cmd) /* version 1 of the cmd is not supported only by BE2 */ if (BE2_chip(adapter)) hdr->version = 0; - if (BE3_chip(adapter) || lancer_chip(adapter)) + else if (BE3_chip(adapter) || lancer_chip(adapter)) hdr->version = 1; else hdr->version = 2; @@ -3852,8 +3852,8 @@ int be_cmd_set_mac_list(struct be_adapter *adapter, u8 *mac_array, status = be_mcc_notify_wait(adapter); err: - dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma); spin_unlock_bh(&adapter->mcc_lock); + dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma); return status; } diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index 6bc0fde95f9dc..011c8cc8429e3 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -1296,7 +1296,8 @@ static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo) (adapter->bmc_filt_mask & BMC_FILT_MULTICAST) static bool be_send_pkt_to_bmc(struct be_adapter *adapter, - struct sk_buff **skb) + struct sk_buff **skb, + struct be_wrb_params *wrb_params) { struct ethhdr *eh = (struct ethhdr *)(*skb)->data; bool os2bmc = false; @@ -1360,7 +1361,7 @@ static bool be_send_pkt_to_bmc(struct be_adapter *adapter, * to BMC, asic expects the vlan to be inline in the packet. */ if (os2bmc) - *skb = be_insert_vlan_in_pkt(adapter, *skb, NULL); + *skb = be_insert_vlan_in_pkt(adapter, *skb, wrb_params); return os2bmc; } @@ -1387,7 +1388,7 @@ static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev) /* if os2bmc is enabled and if the pkt is destined to bmc, * enqueue the pkt a 2nd time with mgmt bit set. */ - if (be_send_pkt_to_bmc(adapter, &skb)) { + if (be_send_pkt_to_bmc(adapter, &skb, &wrb_params)) { BE_WRB_F_SET(wrb_params.features, OS2BMC, 1); wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); if (unlikely(!wrb_cnt)) @@ -1465,10 +1466,10 @@ static void be_tx_timeout(struct net_device *netdev, unsigned int txqueue) ntohs(tcphdr->source)); dev_info(dev, "TCP dest port %d\n", ntohs(tcphdr->dest)); - dev_info(dev, "TCP sequence num %d\n", - ntohs(tcphdr->seq)); - dev_info(dev, "TCP ack_seq %d\n", - ntohs(tcphdr->ack_seq)); + dev_info(dev, "TCP sequence num %u\n", + ntohl(tcphdr->seq)); + dev_info(dev, "TCP ack_seq %u\n", + ntohl(tcphdr->ack_seq)); } else if (ip_hdr(skb)->protocol == IPPROTO_UDP) { udphdr = udp_hdr(skb); diff --git a/drivers/net/ethernet/engleder/tsnep_hw.h b/drivers/net/ethernet/engleder/tsnep_hw.h index 55e1caf193a69..64c97eb66f671 100644 --- a/drivers/net/ethernet/engleder/tsnep_hw.h +++ b/drivers/net/ethernet/engleder/tsnep_hw.h @@ -181,6 +181,8 @@ struct tsnep_gcl_operation { #define TSNEP_DESC_SIZE 256 #define TSNEP_DESC_SIZE_DATA_AFTER 2048 #define TSNEP_DESC_OFFSET 128 +#define TSNEP_DESC_SIZE_DATA_AFTER_INLINE (64 - sizeof(struct tsnep_tx_desc) + \ + sizeof_field(struct tsnep_tx_desc, tx)) #define TSNEP_DESC_OWNER_COUNTER_MASK 0xC0000000 #define TSNEP_DESC_OWNER_COUNTER_SHIFT 30 #define TSNEP_DESC_LENGTH_MASK 0x00003FFF diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index 4f36b29d66c86..215ae6745932a 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -51,12 +51,24 @@ #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) -#define TSNEP_TX_TYPE_SKB BIT(0) -#define TSNEP_TX_TYPE_SKB_FRAG BIT(1) -#define TSNEP_TX_TYPE_XDP_TX BIT(2) -#define TSNEP_TX_TYPE_XDP_NDO BIT(3) -#define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO) -#define TSNEP_TX_TYPE_XSK BIT(4) +/* mapping type */ +#define TSNEP_TX_TYPE_MAP BIT(0) +#define TSNEP_TX_TYPE_MAP_PAGE BIT(1) +#define TSNEP_TX_TYPE_INLINE BIT(2) +/* buffer type */ +#define TSNEP_TX_TYPE_SKB BIT(8) +#define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP) +#define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE) +#define TSNEP_TX_TYPE_SKB_FRAG BIT(9) +#define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE) +#define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE) +#define TSNEP_TX_TYPE_XDP_TX BIT(10) +#define TSNEP_TX_TYPE_XDP_NDO BIT(11) +#define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE) +#define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO) +#define TSNEP_TX_TYPE_XSK BIT(12) +#define TSNEP_TX_TYPE_TSTAMP BIT(13) +#define TSNEP_TX_TYPE_SKB_TSTAMP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_TSTAMP) #define TSNEP_XDP_TX BIT(0) #define TSNEP_XDP_REDIRECT BIT(1) @@ -375,8 +387,7 @@ static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, if (entry->skb) { entry->properties = length & TSNEP_DESC_LENGTH_MASK; entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; - if ((entry->type & TSNEP_TX_TYPE_SKB) && - (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)) + if ((entry->type & TSNEP_TX_TYPE_SKB_TSTAMP) == TSNEP_TX_TYPE_SKB_TSTAMP) entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; /* toggle user flag to prevent false acknowledge @@ -416,6 +427,8 @@ static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; entry->desc->more_properties = __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); + if (entry->type & TSNEP_TX_TYPE_INLINE) + entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG; /* descriptor properties shall be written last, because valid data is * signaled there @@ -433,39 +446,83 @@ static int tsnep_tx_desc_available(struct tsnep_tx *tx) return tx->read - tx->write - 1; } -static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) +static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry, + struct device *dmadev, dma_addr_t *dma) +{ + unsigned int len; + int mapped; + + len = skb_frag_size(frag); + if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { + *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE); + if (dma_mapping_error(dmadev, *dma)) + return -ENOMEM; + entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE; + mapped = 1; + } else { + void *fragdata = skb_frag_address_safe(frag); + + if (likely(fragdata)) { + memcpy(&entry->desc->tx, fragdata, len); + } else { + struct page *page = skb_frag_page(frag); + + fragdata = kmap_local_page(page); + memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag), + len); + kunmap_local(fragdata); + } + entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE; + mapped = 0; + } + + return mapped; +} + +static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count, + bool do_tstamp) { struct device *dmadev = tx->adapter->dmadev; struct tsnep_tx_entry *entry; unsigned int len; - dma_addr_t dma; int map_len = 0; - int i; + dma_addr_t dma; + int i, mapped; for (i = 0; i < count; i++) { entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; if (!i) { len = skb_headlen(skb); - dma = dma_map_single(dmadev, skb->data, len, - DMA_TO_DEVICE); + if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { + dma = dma_map_single(dmadev, skb->data, len, + DMA_TO_DEVICE); + if (dma_mapping_error(dmadev, dma)) + return -ENOMEM; + entry->type = TSNEP_TX_TYPE_SKB_MAP; + mapped = 1; + } else { + memcpy(&entry->desc->tx, skb->data, len); + entry->type = TSNEP_TX_TYPE_SKB_INLINE; + mapped = 0; + } - entry->type = TSNEP_TX_TYPE_SKB; + if (do_tstamp) + entry->type |= TSNEP_TX_TYPE_TSTAMP; } else { - len = skb_frag_size(&skb_shinfo(skb)->frags[i - 1]); - dma = skb_frag_dma_map(dmadev, - &skb_shinfo(skb)->frags[i - 1], - 0, len, DMA_TO_DEVICE); + skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; - entry->type = TSNEP_TX_TYPE_SKB_FRAG; + len = skb_frag_size(frag); + mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma); + if (mapped < 0) + return mapped; } - if (dma_mapping_error(dmadev, dma)) - return -ENOMEM; entry->len = len; - dma_unmap_addr_set(entry, dma, dma); - - entry->desc->tx = __cpu_to_le64(dma); + if (likely(mapped)) { + dma_unmap_addr_set(entry, dma, dma); + entry->desc->tx = __cpu_to_le64(dma); + } map_len += len; } @@ -484,13 +541,12 @@ static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) entry = &tx->entry[(index + i) & TSNEP_RING_MASK]; if (entry->len) { - if (entry->type & TSNEP_TX_TYPE_SKB) + if (entry->type & TSNEP_TX_TYPE_MAP) dma_unmap_single(dmadev, dma_unmap_addr(entry, dma), dma_unmap_len(entry, len), DMA_TO_DEVICE); - else if (entry->type & - (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_XDP_NDO)) + else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE) dma_unmap_page(dmadev, dma_unmap_addr(entry, dma), dma_unmap_len(entry, len), @@ -506,11 +562,12 @@ static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, struct tsnep_tx *tx) { - int count = 1; struct tsnep_tx_entry *entry; + bool do_tstamp = false; + int count = 1; int length; - int i; int retval; + int i; if (skb_shinfo(skb)->nr_frags > 0) count += skb_shinfo(skb)->nr_frags; @@ -527,7 +584,13 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, entry = &tx->entry[tx->write]; entry->skb = skb; - retval = tsnep_tx_map(skb, tx, count); + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + tx->adapter->hwtstamp_config.tx_type == HWTSTAMP_TX_ON) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + do_tstamp = true; + } + + retval = tsnep_tx_map(skb, tx, count, do_tstamp); if (retval < 0) { tsnep_tx_unmap(tx, tx->write, count); dev_kfree_skb_any(entry->skb); @@ -539,9 +602,6 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, } length = retval; - if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) - skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; - for (i = 0; i < count; i++) tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, i == count - 1); @@ -586,7 +646,7 @@ static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx, if (dma_mapping_error(dmadev, dma)) return -ENOMEM; - entry->type = TSNEP_TX_TYPE_XDP_NDO; + entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE; } else { page = unlikely(frag) ? skb_frag_page(frag) : virt_to_page(xdpf->data); @@ -792,8 +852,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) length = tsnep_tx_unmap(tx, tx->read, count); - if ((entry->type & TSNEP_TX_TYPE_SKB) && - (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && + if (((entry->type & TSNEP_TX_TYPE_SKB_TSTAMP) == TSNEP_TX_TYPE_SKB_TSTAMP) && (__le32_to_cpu(entry->desc_wb->properties) & TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { struct skb_shared_hwtstamps hwtstamps; diff --git a/drivers/net/ethernet/faraday/Kconfig b/drivers/net/ethernet/faraday/Kconfig index c699bd6bcbb93..474073c7f94d7 100644 --- a/drivers/net/ethernet/faraday/Kconfig +++ b/drivers/net/ethernet/faraday/Kconfig @@ -31,6 +31,7 @@ config FTGMAC100 depends on ARM || COMPILE_TEST depends on !64BIT || BROKEN select PHYLIB + select FIXED_PHY select MDIO_ASPEED if MACH_ASPEED_G6 select CRC32 help diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index e7bf70ac9a4ca..6b7e1bb5c62d9 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -3141,7 +3140,6 @@ static const struct net_device_ops dpaa_ops = { .ndo_stop = dpaa_eth_stop, .ndo_tx_timeout = dpaa_tx_timeout, .ndo_get_stats64 = dpaa_get_stats64, - .ndo_change_carrier = fixed_phy_change_carrier, .ndo_set_mac_address = dpaa_set_mac_address, .ndo_validate_addr = eth_validate_addr, .ndo_set_rx_mode = dpaa_set_rx_mode, diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c index 3f8cd4a7d8457..20d73a4ed22cd 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c @@ -415,8 +415,10 @@ static int dpaa_get_ts_info(struct net_device *net_dev, of_node_put(ptp_node); } - if (ptp_dev) + if (ptp_dev) { ptp = platform_get_drvdata(ptp_dev); + put_device(&ptp_dev->dev); + } if (ptp) info->phc_index = ptp->phc_index; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index 40e8818295951..61bd2389ef4b5 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -1077,8 +1077,7 @@ static int dpaa2_eth_build_single_fd(struct dpaa2_eth_priv *priv, dma_addr_t addr; buffer_start = skb->data - dpaa2_eth_needed_headroom(skb); - aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, - DPAA2_ETH_TX_BUF_ALIGN); + aligned_start = PTR_ALIGN(buffer_start, DPAA2_ETH_TX_BUF_ALIGN); if (aligned_start >= skb->head) buffer_start = aligned_start; else @@ -3928,6 +3927,7 @@ static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, MEM_TYPE_PAGE_ORDER0, NULL); if (err) { dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); + xdp_rxq_info_unreg(&fq->channel->xdp_rxq); return err; } @@ -4421,17 +4421,25 @@ static int dpaa2_eth_bind_dpni(struct dpaa2_eth_priv *priv) return -EINVAL; } if (err) - return err; + goto out; } err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, DPNI_QUEUE_TX, &priv->tx_qdid); if (err) { dev_err(dev, "dpni_get_qdid() failed\n"); - return err; + goto out; } return 0; + +out: + while (i--) { + if (priv->fq[i].type == DPAA2_RX_FQ && + xdp_rxq_info_is_reg(&priv->fq[i].channel->xdp_rxq)) + xdp_rxq_info_unreg(&priv->fq[i].channel->xdp_rxq); + } + return err; } /* Allocate rings for storing incoming frame descriptors */ @@ -4645,12 +4653,19 @@ static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) return PTR_ERR(dpmac_dev); } - if (IS_ERR(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) + if (IS_ERR(dpmac_dev)) return 0; + if (dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) { + err = 0; + goto out_put_device; + } + mac = kzalloc(sizeof(struct dpaa2_mac), GFP_KERNEL); - if (!mac) - return -ENOMEM; + if (!mac) { + err = -ENOMEM; + goto out_put_device; + } mac->mc_dev = dpmac_dev; mac->mc_io = priv->mc_io; @@ -4684,6 +4699,8 @@ static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) dpaa2_mac_close(mac); err_free_mac: kfree(mac); +out_put_device: + put_device(&dpmac_dev->dev); return err; } @@ -4813,6 +4830,17 @@ static void dpaa2_eth_del_ch_napi(struct dpaa2_eth_priv *priv) } } +static void dpaa2_eth_free_rx_xdp_rxq(struct dpaa2_eth_priv *priv) +{ + int i; + + for (i = 0; i < priv->num_fqs; i++) { + if (priv->fq[i].type == DPAA2_RX_FQ && + xdp_rxq_info_is_reg(&priv->fq[i].channel->xdp_rxq)) + xdp_rxq_info_unreg(&priv->fq[i].channel->xdp_rxq); + } +} + static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) { struct device *dev; @@ -5016,6 +5044,7 @@ static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) free_percpu(priv->percpu_stats); err_alloc_percpu_stats: dpaa2_eth_del_ch_napi(priv); + dpaa2_eth_free_rx_xdp_rxq(priv); err_bind: dpaa2_eth_free_dpbps(priv); err_dpbp_setup: @@ -5068,6 +5097,7 @@ static void dpaa2_eth_remove(struct fsl_mc_device *ls_dev) free_percpu(priv->percpu_extras); dpaa2_eth_del_ch_napi(priv); + dpaa2_eth_free_rx_xdp_rxq(priv); dpaa2_eth_free_dpbps(priv); dpaa2_eth_free_dpio(priv); dpaa2_eth_free_dpni(priv); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index a05a8525caa45..cdab37e9634d4 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -1447,12 +1447,19 @@ static int dpaa2_switch_port_connect_mac(struct ethsw_port_priv *port_priv) if (PTR_ERR(dpmac_dev) == -EPROBE_DEFER) return PTR_ERR(dpmac_dev); - if (IS_ERR(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) + if (IS_ERR(dpmac_dev)) return 0; + if (dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) { + err = 0; + goto out_put_device; + } + mac = kzalloc(sizeof(*mac), GFP_KERNEL); - if (!mac) - return -ENOMEM; + if (!mac) { + err = -ENOMEM; + goto out_put_device; + } mac->mc_dev = dpmac_dev; mac->mc_io = port_priv->ethsw_data->mc_io; @@ -1482,6 +1489,8 @@ static int dpaa2_switch_port_connect_mac(struct ethsw_port_priv *port_priv) dpaa2_mac_close(mac); err_free_mac: kfree(mac); +out_put_device: + put_device(&dpmac_dev->dev); return err; } @@ -2691,7 +2700,7 @@ static int dpaa2_switch_setup_dpbp(struct ethsw_core *ethsw) dev_err(dev, "dpsw_ctrl_if_set_pools() failed\n"); goto err_get_attr; } - ethsw->bpid = dpbp_attrs.id; + ethsw->bpid = dpbp_attrs.bpid; return 0; diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 8feb7d4226bb5..7accf3a3e9f0d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -1246,6 +1246,8 @@ static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, /* next descriptor to process */ i = rx_ring->next_to_clean; + enetc_lock_mdio(); + while (likely(rx_frm_cnt < work_limit)) { union enetc_rx_bd *rxbd; struct sk_buff *skb; @@ -1281,7 +1283,9 @@ static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, rx_byte_cnt += skb->len + ETH_HLEN; rx_frm_cnt++; + enetc_unlock_mdio(); napi_gro_receive(napi, skb); + enetc_lock_mdio(); } rx_ring->next_to_clean = i; @@ -1289,6 +1293,8 @@ static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, rx_ring->stats.packets += rx_frm_cnt; rx_ring->stats.bytes += rx_byte_cnt; + enetc_unlock_mdio(); + return rx_frm_cnt; } @@ -1572,6 +1578,16 @@ static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, } } +static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first, + int rx_ring_last) +{ + while (rx_ring_first != rx_ring_last) { + enetc_flip_rx_buff(rx_ring, + &rx_ring->rx_swbd[rx_ring_first]); + enetc_bdr_idx_inc(rx_ring, &rx_ring_first); + } +} + static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, struct napi_struct *napi, int work_limit, struct bpf_prog *prog) @@ -1588,6 +1604,8 @@ static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, /* next descriptor to process */ i = rx_ring->next_to_clean; + enetc_lock_mdio(); + while (likely(rx_frm_cnt < work_limit)) { union enetc_rx_bd *rxbd, *orig_rxbd; int orig_i, orig_cleaned_cnt; @@ -1647,7 +1665,9 @@ static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, if (unlikely(!skb)) goto out; + enetc_unlock_mdio(); napi_gro_receive(napi, skb); + enetc_lock_mdio(); break; case XDP_TX: tx_ring = priv->xdp_tx_ring[rx_ring->index]; @@ -1682,16 +1702,14 @@ static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, } break; case XDP_REDIRECT: + enetc_unlock_mdio(); err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); + enetc_lock_mdio(); if (unlikely(err)) { enetc_xdp_drop(rx_ring, orig_i, i); rx_ring->stats.xdp_redirect_failures++; } else { - while (orig_i != i) { - enetc_flip_rx_buff(rx_ring, - &rx_ring->rx_swbd[orig_i]); - enetc_bdr_idx_inc(rx_ring, &orig_i); - } + enetc_bulk_flip_buff(rx_ring, orig_i, i); xdp_redirect_frm_cnt++; rx_ring->stats.xdp_redirect++; } @@ -1706,8 +1724,11 @@ static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, rx_ring->stats.packets += rx_frm_cnt; rx_ring->stats.bytes += rx_byte_cnt; - if (xdp_redirect_frm_cnt) - xdp_do_flush_map(); + if (xdp_redirect_frm_cnt) { + enetc_unlock_mdio(); + xdp_do_flush(); + enetc_lock_mdio(); + } if (xdp_tx_frm_cnt) enetc_update_tx_ring_tail(tx_ring); @@ -1716,6 +1737,8 @@ static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - rx_ring->xdp.xdp_tx_in_flight); + enetc_unlock_mdio(); + return rx_frm_cnt; } @@ -1734,6 +1757,7 @@ static int enetc_poll(struct napi_struct *napi, int budget) for (i = 0; i < v->count_tx_rings; i++) if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) complete = false; + enetc_unlock_mdio(); prog = rx_ring->xdp.prog; if (prog) @@ -1745,10 +1769,8 @@ static int enetc_poll(struct napi_struct *napi, int budget) if (work_done) v->rx_napi_work = true; - if (!complete) { - enetc_unlock_mdio(); + if (!complete) return budget; - } napi_complete_done(napi, work_done); @@ -1757,6 +1779,7 @@ static int enetc_poll(struct napi_struct *napi, int budget) v->rx_napi_work = false; + enetc_lock_mdio(); /* enable interrupts */ enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index 860ecee302f1a..dcf3e4b4e3f55 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -41,7 +41,7 @@ struct enetc_tx_swbd { }; #define ENETC_RX_MAXFRM_SIZE ENETC_MAC_MAXFRM_SIZE -#define ENETC_RXB_TRUESIZE 2048 /* PAGE_SIZE >> 1 */ +#define ENETC_RXB_TRUESIZE (PAGE_SIZE >> 1) #define ENETC_RXB_PAD NET_SKB_PAD /* add extra space if needed */ #define ENETC_RXB_DMA_SIZE \ (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - ENETC_RXB_PAD) diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h index 1619943fb2637..4e8881b479e48 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h @@ -485,7 +485,7 @@ static inline u64 _enetc_rd_reg64(void __iomem *reg) tmp = ioread32(reg + 4); } while (high != tmp); - return le64_to_cpu((__le64)high << 32 | low); + return (u64)high << 32 | low; } #endif diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index a856047f1dfd6..0ad3ea380e9b1 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -1179,19 +1179,29 @@ static int enetc_pf_register_with_ierb(struct pci_dev *pdev) { struct platform_device *ierb_pdev; struct device_node *ierb_node; + int ret; ierb_node = of_find_compatible_node(NULL, NULL, "fsl,ls1028a-enetc-ierb"); - if (!ierb_node || !of_device_is_available(ierb_node)) + if (!ierb_node) return -ENODEV; + if (!of_device_is_available(ierb_node)) { + of_node_put(ierb_node); + return -ENODEV; + } + ierb_pdev = of_find_device_by_node(ierb_node); of_node_put(ierb_node); if (!ierb_pdev) return -EPROBE_DEFER; - return enetc_ierb_register_pf(ierb_pdev, pdev); + ret = enetc_ierb_register_pf(ierb_pdev, pdev); + + put_device(&ierb_pdev->dev); + + return ret; } static struct enetc_si *enetc_psi_create(struct pci_dev *pdev) diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 2d6b50903c923..ee0306ab97714 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -695,7 +695,12 @@ static int fec_enet_txq_submit_skb(struct fec_enet_priv_tx_q *txq, txq->bd.cur = bdp; /* Trigger transmission start */ - writel(0, txq->bd.reg_desc_active); + if (!(fep->quirks & FEC_QUIRK_ERR007885) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active)) + writel(0, txq->bd.reg_desc_active); return 0; } @@ -1074,6 +1079,29 @@ static void fec_enet_enable_ring(struct net_device *ndev) } } +/* Whack a reset. We should wait for this. + * For i.MX6SX SOC, enet use AXI bus, we use disable MAC + * instead of reset MAC itself. + */ +static void fec_ctrl_reset(struct fec_enet_private *fep, bool allow_wol) +{ + u32 val; + + if (!allow_wol || !(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || + ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { + writel(0, fep->hwp + FEC_ECNTRL); + } else { + writel(FEC_ECR_RESET, fep->hwp + FEC_ECNTRL); + udelay(10); + } + } else { + val = readl(fep->hwp + FEC_ECNTRL); + val |= (FEC_ECR_MAGICEN | FEC_ECR_SLEEP); + writel(val, fep->hwp + FEC_ECNTRL); + } +} + /* * This function is called to start or restart the FEC during a link * change, transmit timeout, or to reconfigure the FEC. The network @@ -1090,17 +1118,7 @@ fec_restart(struct net_device *ndev) if (fep->bufdesc_ex) fec_ptp_save_state(fep); - /* Whack a reset. We should wait for this. - * For i.MX6SX SOC, enet use AXI bus, we use disable MAC - * instead of reset MAC itself. - */ - if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || - ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { - writel(0, fep->hwp + FEC_ECNTRL); - } else { - writel(1, fep->hwp + FEC_ECNTRL); - udelay(10); - } + fec_ctrl_reset(fep, false); /* * enet-mac reset will reset mac address registers too, @@ -1354,22 +1372,7 @@ fec_stop(struct net_device *ndev) if (fep->bufdesc_ex) fec_ptp_save_state(fep); - /* Whack a reset. We should wait for this. - * For i.MX6SX SOC, enet use AXI bus, we use disable MAC - * instead of reset MAC itself. - */ - if (!(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { - if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { - writel(0, fep->hwp + FEC_ECNTRL); - } else { - writel(FEC_ECR_RESET, fep->hwp + FEC_ECNTRL); - udelay(10); - } - } else { - val = readl(fep->hwp + FEC_ECNTRL); - val |= (FEC_ECR_MAGICEN | FEC_ECR_SLEEP); - writel(val, fep->hwp + FEC_ECNTRL); - } + fec_ctrl_reset(fep, true); writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); @@ -1773,6 +1776,8 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) ndev->stats.rx_packets++; pkt_len = fec16_to_cpu(bdp->cbd_datlen); ndev->stats.rx_bytes += pkt_len; + if (fep->quirks & FEC_QUIRK_HAS_RACC) + ndev->stats.rx_bytes -= 2; index = fec_enet_get_bd_index(bdp, &rxq->bd); page = rxq->rx_skb_info[index].page; @@ -1901,7 +1906,7 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) rxq->bd.cur = bdp; if (xdp_result & FEC_ENET_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); return pkt_received; } @@ -2297,7 +2302,8 @@ static void fec_enet_phy_reset_after_clk_enable(struct net_device *ndev) */ phy_dev = of_phy_find_device(fep->phy_node); phy_reset_after_clk_enable(phy_dev); - put_device(&phy_dev->mdio.dev); + if (phy_dev) + put_device(&phy_dev->mdio.dev); } } @@ -3066,27 +3072,25 @@ static int fec_enet_us_to_itr_clock(struct net_device *ndev, int us) static void fec_enet_itr_coal_set(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); - int rx_itr, tx_itr; + u32 rx_itr = 0, tx_itr = 0; + int rx_ictt, tx_ictt; - /* Must be greater than zero to avoid unpredictable behavior */ - if (!fep->rx_time_itr || !fep->rx_pkts_itr || - !fep->tx_time_itr || !fep->tx_pkts_itr) - return; - - /* Select enet system clock as Interrupt Coalescing - * timer Clock Source - */ - rx_itr = FEC_ITR_CLK_SEL; - tx_itr = FEC_ITR_CLK_SEL; + rx_ictt = fec_enet_us_to_itr_clock(ndev, fep->rx_time_itr); + tx_ictt = fec_enet_us_to_itr_clock(ndev, fep->tx_time_itr); - /* set ICFT and ICTT */ - rx_itr |= FEC_ITR_ICFT(fep->rx_pkts_itr); - rx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->rx_time_itr)); - tx_itr |= FEC_ITR_ICFT(fep->tx_pkts_itr); - tx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->tx_time_itr)); + if (rx_ictt > 0 && fep->rx_pkts_itr > 1) { + /* Enable with enet system clock as Interrupt Coalescing timer Clock Source */ + rx_itr = FEC_ITR_EN | FEC_ITR_CLK_SEL; + rx_itr |= FEC_ITR_ICFT(fep->rx_pkts_itr); + rx_itr |= FEC_ITR_ICTT(rx_ictt); + } - rx_itr |= FEC_ITR_EN; - tx_itr |= FEC_ITR_EN; + if (tx_ictt > 0 && fep->tx_pkts_itr > 1) { + /* Enable with enet system clock as Interrupt Coalescing timer Clock Source */ + tx_itr = FEC_ITR_EN | FEC_ITR_CLK_SEL; + tx_itr |= FEC_ITR_ICFT(fep->tx_pkts_itr); + tx_itr |= FEC_ITR_ICTT(tx_ictt); + } writel(tx_itr, fep->hwp + FEC_TXIC0); writel(rx_itr, fep->hwp + FEC_RXIC0); diff --git a/drivers/net/ethernet/freescale/fsl_pq_mdio.c b/drivers/net/ethernet/freescale/fsl_pq_mdio.c index eee675a25b2c3..ef9adecb639ae 100644 --- a/drivers/net/ethernet/freescale/fsl_pq_mdio.c +++ b/drivers/net/ethernet/freescale/fsl_pq_mdio.c @@ -483,10 +483,12 @@ static int fsl_pq_mdio_probe(struct platform_device *pdev) "missing 'reg' property in node %pOF\n", tbi); err = -EBUSY; + of_node_put(tbi); goto error; } set_tbipa(*prop, pdev, data->get_tbipa, priv->map, &res); + of_node_put(tbi); } } diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c index 7a15b9245698b..30c7bc019d2fe 100644 --- a/drivers/net/ethernet/freescale/gianfar_ethtool.c +++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c @@ -1468,8 +1468,10 @@ static int gfar_get_ts_info(struct net_device *dev, if (ptp_node) { ptp_dev = of_find_device_by_node(ptp_node); of_node_put(ptp_node); - if (ptp_dev) + if (ptp_dev) { ptp = platform_get_drvdata(ptp_dev); + put_device(&ptp_dev->dev); + } } if (ptp) diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c index 79db7a6d42bc2..9c50febb42717 100644 --- a/drivers/net/ethernet/google/gve/gve_adminq.c +++ b/drivers/net/ethernet/google/gve/gve_adminq.c @@ -431,6 +431,7 @@ static int gve_adminq_issue_cmd(struct gve_priv *priv, break; default: dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode); + return -EINVAL; } return 0; diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 8cd098fe88ef2..241a541b8edd2 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -1801,49 +1801,56 @@ static void gve_turnup(struct gve_priv *priv) gve_set_napi_enabled(priv); } -static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) +static struct gve_notify_block *gve_get_tx_notify_block(struct gve_priv *priv, + unsigned int txqueue) { - struct gve_notify_block *block; - struct gve_tx_ring *tx = NULL; - struct gve_priv *priv; - u32 last_nic_done; - u32 current_time; u32 ntfy_idx; - netdev_info(dev, "Timeout on tx queue, %d", txqueue); - priv = netdev_priv(dev); if (txqueue > priv->tx_cfg.num_queues) - goto reset; + return NULL; ntfy_idx = gve_tx_idx_to_ntfy(priv, txqueue); if (ntfy_idx >= priv->num_ntfy_blks) - goto reset; + return NULL; + + return &priv->ntfy_blocks[ntfy_idx]; +} + +static bool gve_tx_timeout_try_q_kick(struct gve_priv *priv, + unsigned int txqueue) +{ + struct gve_notify_block *block; + u32 current_time; + + block = gve_get_tx_notify_block(priv, txqueue); - block = &priv->ntfy_blocks[ntfy_idx]; - tx = block->tx; + if (!block) + return false; current_time = jiffies_to_msecs(jiffies); - if (tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time) - goto reset; + if (block->tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time) + return false; - /* Check to see if there are missed completions, which will allow us to - * kick the queue. - */ - last_nic_done = gve_tx_load_event_counter(priv, tx); - if (last_nic_done - tx->done) { - netdev_info(dev, "Kicking queue %d", txqueue); - iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); - napi_schedule(&block->napi); - tx->last_kick_msec = current_time; - goto out; - } // Else reset. + netdev_info(priv->dev, "Kicking queue %d", txqueue); + napi_schedule(&block->napi); + block->tx->last_kick_msec = current_time; + return true; +} -reset: - gve_schedule_reset(priv); +static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) +{ + struct gve_notify_block *block; + struct gve_priv *priv; -out: - if (tx) - tx->queue_timeout++; + netdev_info(dev, "Timeout on tx queue, %d", txqueue); + priv = netdev_priv(dev); + + if (!gve_tx_timeout_try_q_kick(priv, txqueue)) + gve_schedule_reset(priv); + + block = gve_get_tx_notify_block(priv, txqueue); + if (block) + block->tx->queue_timeout++; priv->tx_timeo_cnt++; } @@ -1992,7 +1999,7 @@ void gve_handle_report_stats(struct gve_priv *priv) }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(RX_BUFFERS_POSTED), - .value = cpu_to_be64(priv->rx[0].fill_cnt), + .value = cpu_to_be64(priv->rx[idx].fill_cnt), .queue_id = cpu_to_be32(idx), }; } @@ -2366,6 +2373,8 @@ static void gve_shutdown(struct pci_dev *pdev) struct gve_priv *priv = netdev_priv(netdev); bool was_up = netif_carrier_ok(priv->dev); + netif_device_detach(netdev); + rtnl_lock(); if (was_up && gve_close(priv->dev)) { /* If the dev was up, attempt to close, if close fails, reset */ diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 89b62b8d16e14..857749fef37cf 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -716,6 +716,9 @@ static int gve_tx_add_skb_dqo(struct gve_tx_ring *tx, s16 completion_tag; pkt = gve_alloc_pending_packet(tx); + if (!pkt) + return -ENOMEM; + pkt->skb = skb; completion_tag = pkt - tx->dqo.pending_packets; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index 4f385a18d288e..36206273453f3 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -60,7 +60,7 @@ static struct hns3_dbg_cmd_info hns3_dbg_cmd[] = { .name = "tm_qset", .cmd = HNAE3_DBG_CMD_TM_QSET, .dentry = HNS3_DBG_DENTRY_TM, - .buf_len = HNS3_DBG_READ_LEN, + .buf_len = HNS3_DBG_READ_LEN_1MB, .init = hns3_dbg_common_file_init, }, { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 801801e8803e9..dbf44a17987eb 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -473,20 +474,14 @@ static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector, writel(mask_en, tqp_vector->mask_addr); } -static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector) +static void hns3_irq_enable(struct hns3_enet_tqp_vector *tqp_vector) { napi_enable(&tqp_vector->napi); enable_irq(tqp_vector->vector_irq); - - /* enable vector */ - hns3_mask_vector_irq(tqp_vector, 1); } -static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector) +static void hns3_irq_disable(struct hns3_enet_tqp_vector *tqp_vector) { - /* disable vector */ - hns3_mask_vector_irq(tqp_vector, 0); - disable_irq(tqp_vector->vector_irq); napi_disable(&tqp_vector->napi); cancel_work_sync(&tqp_vector->rx_group.dim.work); @@ -707,11 +702,42 @@ static int hns3_set_rx_cpu_rmap(struct net_device *netdev) return 0; } +static void hns3_enable_irqs_and_tqps(struct net_device *netdev) +{ + struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hnae3_handle *h = priv->ae_handle; + u16 i; + + for (i = 0; i < priv->vector_num; i++) + hns3_irq_enable(&priv->tqp_vector[i]); + + for (i = 0; i < priv->vector_num; i++) + hns3_mask_vector_irq(&priv->tqp_vector[i], 1); + + for (i = 0; i < h->kinfo.num_tqps; i++) + hns3_tqp_enable(h->kinfo.tqp[i]); +} + +static void hns3_disable_irqs_and_tqps(struct net_device *netdev) +{ + struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hnae3_handle *h = priv->ae_handle; + u16 i; + + for (i = 0; i < h->kinfo.num_tqps; i++) + hns3_tqp_disable(h->kinfo.tqp[i]); + + for (i = 0; i < priv->vector_num; i++) + hns3_mask_vector_irq(&priv->tqp_vector[i], 0); + + for (i = 0; i < priv->vector_num; i++) + hns3_irq_disable(&priv->tqp_vector[i]); +} + static int hns3_nic_net_up(struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); struct hnae3_handle *h = priv->ae_handle; - int i, j; int ret; ret = hns3_nic_reset_all_ring(h); @@ -720,23 +746,13 @@ static int hns3_nic_net_up(struct net_device *netdev) clear_bit(HNS3_NIC_STATE_DOWN, &priv->state); - /* enable the vectors */ - for (i = 0; i < priv->vector_num; i++) - hns3_vector_enable(&priv->tqp_vector[i]); - - /* enable rcb */ - for (j = 0; j < h->kinfo.num_tqps; j++) - hns3_tqp_enable(h->kinfo.tqp[j]); + hns3_enable_irqs_and_tqps(netdev); /* start the ae_dev */ ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0; if (ret) { set_bit(HNS3_NIC_STATE_DOWN, &priv->state); - while (j--) - hns3_tqp_disable(h->kinfo.tqp[j]); - - for (j = i - 1; j >= 0; j--) - hns3_vector_disable(&priv->tqp_vector[j]); + hns3_disable_irqs_and_tqps(netdev); } return ret; @@ -823,17 +839,9 @@ static void hns3_reset_tx_queue(struct hnae3_handle *h) static void hns3_nic_net_down(struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); - struct hnae3_handle *h = hns3_get_handle(netdev); const struct hnae3_ae_ops *ops; - int i; - - /* disable vectors */ - for (i = 0; i < priv->vector_num; i++) - hns3_vector_disable(&priv->tqp_vector[i]); - /* disable rcb */ - for (i = 0; i < h->kinfo.num_tqps; i++) - hns3_tqp_disable(h->kinfo.tqp[i]); + hns3_disable_irqs_and_tqps(netdev); /* stop ae_dev */ ops = priv->ae_handle->ae_algo->ops; @@ -1032,6 +1040,8 @@ static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring, static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) { u32 alloc_size = ring->tqp->handle->kinfo.tx_spare_buf_size; + struct net_device *netdev = ring_to_netdev(ring); + struct hns3_nic_priv *priv = netdev_priv(netdev); struct hns3_tx_spare *tx_spare; struct page *page; dma_addr_t dma; @@ -1073,6 +1083,7 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) tx_spare->buf = page_address(page); tx_spare->len = PAGE_SIZE << order; ring->tx_spare = tx_spare; + ring->tx_copybreak = priv->tx_copybreak; return; dma_mapping_error: @@ -4872,6 +4883,30 @@ static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv) devm_kfree(&pdev->dev, priv->tqp_vector); } +static void hns3_update_tx_spare_buf_config(struct hns3_nic_priv *priv) +{ +#define HNS3_MIN_SPARE_BUF_SIZE (2 * 1024 * 1024) +#define HNS3_MAX_PACKET_SIZE (64 * 1024) + + struct iommu_domain *domain = iommu_get_domain_for_dev(priv->dev); + struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle); + struct hnae3_handle *handle = priv->ae_handle; + + if (ae_dev->dev_version < HNAE3_DEVICE_VERSION_V3) + return; + + if (!(domain && iommu_is_dma_domain(domain))) + return; + + priv->min_tx_copybreak = HNS3_MAX_PACKET_SIZE; + priv->min_tx_spare_buf_size = HNS3_MIN_SPARE_BUF_SIZE; + + if (priv->tx_copybreak < priv->min_tx_copybreak) + priv->tx_copybreak = priv->min_tx_copybreak; + if (handle->kinfo.tx_spare_buf_size < priv->min_tx_spare_buf_size) + handle->kinfo.tx_spare_buf_size = priv->min_tx_spare_buf_size; +} + static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, unsigned int ring_type) { @@ -5106,6 +5141,7 @@ int hns3_init_all_ring(struct hns3_nic_priv *priv) int i, j; int ret; + hns3_update_tx_spare_buf_config(priv); for (i = 0; i < ring_num; i++) { ret = hns3_alloc_ring_memory(&priv->ring[i]); if (ret) { @@ -5310,6 +5346,8 @@ static int hns3_client_init(struct hnae3_handle *handle) priv->ae_handle = handle; priv->tx_timeout_count = 0; priv->max_non_tso_bd_num = ae_dev->dev_specs.max_non_tso_bd_num; + priv->min_tx_copybreak = 0; + priv->min_tx_spare_buf_size = 0; set_bit(HNS3_NIC_STATE_DOWN, &priv->state); handle->msg_enable = netif_msg_init(debug, DEFAULT_MSG_LEVEL); @@ -5870,8 +5908,6 @@ int hns3_set_channels(struct net_device *netdev, void hns3_external_lb_prepare(struct net_device *ndev, bool if_running) { struct hns3_nic_priv *priv = netdev_priv(ndev); - struct hnae3_handle *h = priv->ae_handle; - int i; if (!if_running) return; @@ -5882,11 +5918,7 @@ void hns3_external_lb_prepare(struct net_device *ndev, bool if_running) netif_carrier_off(ndev); netif_tx_disable(ndev); - for (i = 0; i < priv->vector_num; i++) - hns3_vector_disable(&priv->tqp_vector[i]); - - for (i = 0; i < h->kinfo.num_tqps; i++) - hns3_tqp_disable(h->kinfo.tqp[i]); + hns3_disable_irqs_and_tqps(ndev); /* delay ring buffer clearing to hns3_reset_notify_uninit_enet * during reset process, because driver may not be able @@ -5902,7 +5934,6 @@ void hns3_external_lb_restore(struct net_device *ndev, bool if_running) { struct hns3_nic_priv *priv = netdev_priv(ndev); struct hnae3_handle *h = priv->ae_handle; - int i; if (!if_running) return; @@ -5918,11 +5949,7 @@ void hns3_external_lb_restore(struct net_device *ndev, bool if_running) clear_bit(HNS3_NIC_STATE_DOWN, &priv->state); - for (i = 0; i < priv->vector_num; i++) - hns3_vector_enable(&priv->tqp_vector[i]); - - for (i = 0; i < h->kinfo.num_tqps; i++) - hns3_tqp_enable(h->kinfo.tqp[i]); + hns3_enable_irqs_and_tqps(ndev); netif_tx_wake_all_queues(ndev); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index d36c4ed16d8dd..caf7a4df85852 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -596,6 +596,8 @@ struct hns3_nic_priv { struct hns3_enet_coalesce rx_coal; u32 tx_copybreak; u32 rx_copybreak; + u32 min_tx_copybreak; + u32 min_tx_spare_buf_size; }; union l3_hdr_info { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 4d318af748a0b..2fa64099e8be2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -9346,8 +9346,7 @@ static int hclge_mii_ioctl(struct hclge_dev *hdev, struct ifreq *ifr, int cmd) /* this command reads phy id and register at the same time */ fallthrough; case SIOCGMIIREG: - data->val_out = hclge_read_phy_reg(hdev, data->reg_num); - return 0; + return hclge_read_phy_reg(hdev, data->reg_num, &data->val_out); case SIOCSMIIREG: return hclge_write_phy_reg(hdev, data->reg_num, data->val_in); @@ -9493,33 +9492,36 @@ static bool hclge_need_enable_vport_vlan_filter(struct hclge_vport *vport) return false; } -int hclge_enable_vport_vlan_filter(struct hclge_vport *vport, bool request_en) +static int __hclge_enable_vport_vlan_filter(struct hclge_vport *vport, + bool request_en) { - struct hclge_dev *hdev = vport->back; bool need_en; int ret; - mutex_lock(&hdev->vport_lock); - - vport->req_vlan_fltr_en = request_en; - need_en = hclge_need_enable_vport_vlan_filter(vport); - if (need_en == vport->cur_vlan_fltr_en) { - mutex_unlock(&hdev->vport_lock); + if (need_en == vport->cur_vlan_fltr_en) return 0; - } ret = hclge_set_vport_vlan_filter(vport, need_en); - if (ret) { - mutex_unlock(&hdev->vport_lock); + if (ret) return ret; - } vport->cur_vlan_fltr_en = need_en; + return 0; +} + +int hclge_enable_vport_vlan_filter(struct hclge_vport *vport, bool request_en) +{ + struct hclge_dev *hdev = vport->back; + int ret; + + mutex_lock(&hdev->vport_lock); + vport->req_vlan_fltr_en = request_en; + ret = __hclge_enable_vport_vlan_filter(vport, request_en); mutex_unlock(&hdev->vport_lock); - return 0; + return ret; } static int hclge_enable_vlan_filter(struct hnae3_handle *handle, bool enable) @@ -10540,16 +10542,19 @@ static void hclge_sync_vlan_fltr_state(struct hclge_dev *hdev) &vport->state)) continue; - ret = hclge_enable_vport_vlan_filter(vport, - vport->req_vlan_fltr_en); + mutex_lock(&hdev->vport_lock); + ret = __hclge_enable_vport_vlan_filter(vport, + vport->req_vlan_fltr_en); if (ret) { dev_err(&hdev->pdev->dev, "failed to sync vlan filter state for vport%u, ret = %d\n", vport->vport_id, ret); set_bit(HCLGE_VPORT_STATE_VLAN_FLTR_CHANGE, &vport->state); + mutex_unlock(&hdev->vport_lock); return; } + mutex_unlock(&hdev->vport_lock); } } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c index 80079657afebe..b8dbf932caf94 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c @@ -274,7 +274,7 @@ void hclge_mac_stop_phy(struct hclge_dev *hdev) phy_stop(phydev); } -u16 hclge_read_phy_reg(struct hclge_dev *hdev, u16 reg_addr) +int hclge_read_phy_reg(struct hclge_dev *hdev, u16 reg_addr, u16 *val) { struct hclge_phy_reg_cmd *req; struct hclge_desc desc; @@ -286,11 +286,14 @@ u16 hclge_read_phy_reg(struct hclge_dev *hdev, u16 reg_addr) req->reg_addr = cpu_to_le16(reg_addr); ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) + if (ret) { dev_err(&hdev->pdev->dev, "failed to read phy reg, ret = %d.\n", ret); + return ret; + } - return le16_to_cpu(req->reg_val); + *val = le16_to_cpu(req->reg_val); + return 0; } int hclge_write_phy_reg(struct hclge_dev *hdev, u16 reg_addr, u16 val) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h index 4200d0b6d9317..21d434c82475b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h @@ -13,7 +13,7 @@ int hclge_mac_connect_phy(struct hnae3_handle *handle); void hclge_mac_disconnect_phy(struct hnae3_handle *handle); void hclge_mac_start_phy(struct hclge_dev *hdev); void hclge_mac_stop_phy(struct hclge_dev *hdev); -u16 hclge_read_phy_reg(struct hclge_dev *hdev, u16 reg_addr); +int hclge_read_phy_reg(struct hclge_dev *hdev, u16 reg_addr, u16 *val); int hclge_write_phy_reg(struct hclge_dev *hdev, u16 reg_addr, u16 val); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.c index ddc691424c816..c1e88e67ebb65 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.c @@ -440,6 +440,13 @@ static int hclge_ptp_create_clock(struct hclge_dev *hdev) ptp->info.settime64 = hclge_ptp_settime; ptp->info.n_alarm = 0; + + spin_lock_init(&ptp->lock); + ptp->io_base = hdev->hw.hw.io_base + HCLGE_PTP_REG_OFFSET; + ptp->ts_cfg.rx_filter = HWTSTAMP_FILTER_NONE; + ptp->ts_cfg.tx_type = HWTSTAMP_TX_OFF; + hdev->ptp = ptp; + ptp->clock = ptp_clock_register(&ptp->info, &hdev->pdev->dev); if (IS_ERR(ptp->clock)) { dev_err(&hdev->pdev->dev, @@ -451,12 +458,6 @@ static int hclge_ptp_create_clock(struct hclge_dev *hdev) return -ENODEV; } - spin_lock_init(&ptp->lock); - ptp->io_base = hdev->hw.hw.io_base + HCLGE_PTP_REG_OFFSET; - ptp->ts_cfg.rx_filter = HWTSTAMP_FILTER_NONE; - ptp->ts_cfg.tx_type = HWTSTAMP_TX_OFF; - hdev->ptp = ptp; - return 0; } @@ -496,14 +497,14 @@ int hclge_ptp_init(struct hclge_dev *hdev) if (ret) { dev_err(&hdev->pdev->dev, "failed to init freq, ret = %d\n", ret); - goto out; + goto out_clear_int; } ret = hclge_ptp_set_ts_mode(hdev, &hdev->ptp->ts_cfg); if (ret) { dev_err(&hdev->pdev->dev, "failed to init ts mode, ret = %d\n", ret); - goto out; + goto out_clear_int; } ktime_get_real_ts64(&ts); @@ -511,7 +512,7 @@ int hclge_ptp_init(struct hclge_dev *hdev) if (ret) { dev_err(&hdev->pdev->dev, "failed to init ts time, ret = %d\n", ret); - goto out; + goto out_clear_int; } set_bit(HCLGE_STATE_PTP_EN, &hdev->state); @@ -519,6 +520,9 @@ int hclge_ptp_init(struct hclge_dev *hdev) return 0; +out_clear_int: + clear_bit(HCLGE_PTP_FLAG_EN, &hdev->ptp->flags); + hclge_ptp_int_en(hdev, false); out: hclge_ptp_destroy_clock(hdev); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 69bfcfb148def..68a9aeeed3da0 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -1257,9 +1257,8 @@ static void hclgevf_sync_vlan_filter(struct hclgevf_dev *hdev) rtnl_unlock(); } -static int hclgevf_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable) +static int hclgevf_en_hw_strip_rxvtag_cmd(struct hclgevf_dev *hdev, bool enable) { - struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); struct hclge_vf_to_pf_msg send_msg; hclgevf_build_send_msg(&send_msg, HCLGE_MBX_SET_VLAN, @@ -1268,6 +1267,19 @@ static int hclgevf_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable) return hclgevf_send_mbx_msg(hdev, &send_msg, false, NULL, 0); } +static int hclgevf_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable) +{ + struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); + int ret; + + ret = hclgevf_en_hw_strip_rxvtag_cmd(hdev, enable); + if (ret) + return ret; + + hdev->rxvtag_strip_en = enable; + return 0; +} + static int hclgevf_reset_tqp(struct hnae3_handle *handle) { #define HCLGEVF_RESET_ALL_QUEUE_DONE 1U @@ -2143,12 +2155,13 @@ static int hclgevf_rss_init_hw(struct hclgevf_dev *hdev) tc_valid, tc_size); } -static int hclgevf_init_vlan_config(struct hclgevf_dev *hdev) +static int hclgevf_init_vlan_config(struct hclgevf_dev *hdev, + bool rxvtag_strip_en) { struct hnae3_handle *nic = &hdev->nic; int ret; - ret = hclgevf_en_hw_strip_rxvtag(nic, true); + ret = hclgevf_en_hw_strip_rxvtag(nic, rxvtag_strip_en); if (ret) { dev_err(&hdev->pdev->dev, "failed to enable rx vlan offload, ret = %d\n", ret); @@ -2815,7 +2828,7 @@ static int hclgevf_reset_hdev(struct hclgevf_dev *hdev) if (ret) return ret; - ret = hclgevf_init_vlan_config(hdev); + ret = hclgevf_init_vlan_config(hdev, hdev->rxvtag_strip_en); if (ret) { dev_err(&hdev->pdev->dev, "failed(%d) to initialize VLAN config\n", ret); @@ -2928,7 +2941,7 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) goto err_config; } - ret = hclgevf_init_vlan_config(hdev); + ret = hclgevf_init_vlan_config(hdev, true); if (ret) { dev_err(&hdev->pdev->dev, "failed(%d) to initialize VLAN config\n", ret); @@ -3016,11 +3029,7 @@ static void hclgevf_uninit_ae_dev(struct hnae3_ae_dev *ae_dev) static u32 hclgevf_get_max_channels(struct hclgevf_dev *hdev) { - struct hnae3_handle *nic = &hdev->nic; - struct hnae3_knic_private_info *kinfo = &nic->kinfo; - - return min_t(u32, hdev->rss_size_max, - hdev->num_tqps / kinfo->tc_info.num_tc); + return min(hdev->rss_size_max, hdev->num_tqps); } /** diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h index cccef32284616..0208425ab594f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h @@ -253,6 +253,7 @@ struct hclgevf_dev { int *vector_irq; bool gro_en; + bool rxvtag_strip_en; unsigned long vlan_del_fail_bmap[BITS_TO_LONGS(VLAN_N_VID)]; diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index b3fc18db4f4c3..dfbda2e5ec88a 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -211,7 +211,6 @@ struct ibmvnic_statistics { u8 reserved[72]; } __packed __aligned(8); -#define NUM_TX_STATS 3 struct ibmvnic_tx_queue_stats { u64 batched_packets; u64 direct_packets; @@ -219,13 +218,18 @@ struct ibmvnic_tx_queue_stats { u64 dropped_packets; }; -#define NUM_RX_STATS 3 +#define NUM_TX_STATS \ + (sizeof(struct ibmvnic_tx_queue_stats) / sizeof(u64)) + struct ibmvnic_rx_queue_stats { u64 packets; u64 bytes; u64 interrupts; }; +#define NUM_RX_STATS \ + (sizeof(struct ibmvnic_rx_queue_stats) / sizeof(u64)) + struct ibmvnic_acl_buffer { __be32 len; __be32 version; diff --git a/drivers/net/ethernet/intel/e1000e/defines.h b/drivers/net/ethernet/intel/e1000e/defines.h index 0a35d36c2c857..955bb11618572 100644 --- a/drivers/net/ethernet/intel/e1000e/defines.h +++ b/drivers/net/ethernet/intel/e1000e/defines.h @@ -638,6 +638,9 @@ /* For checksumming, the sum of all words in the NVM should equal 0xBABA. */ #define NVM_SUM 0xBABA +/* Uninitialized ("empty") checksum word value */ +#define NVM_CHECKSUM_UNINITIALIZED 0xFFFF + /* PBA (printed board assembly) number words */ #define NVM_PBA_OFFSET_0 8 #define NVM_PBA_OFFSET_1 9 diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index fc0f98ea61332..a1abc51584a18 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -567,12 +567,12 @@ static int e1000_set_eeprom(struct net_device *netdev, { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; + size_t total_len, max_len; u16 *eeprom_buff; - void *ptr; - int max_len; + int ret_val = 0; int first_word; int last_word; - int ret_val = 0; + void *ptr; u16 i; if (eeprom->len == 0) @@ -587,6 +587,10 @@ static int e1000_set_eeprom(struct net_device *netdev, max_len = hw->nvm.word_size * 2; + if (check_add_overflow(eeprom->offset, eeprom->len, &total_len) || + total_len > max_len) + return -EFBIG; + first_word = eeprom->offset >> 1; last_word = (eeprom->offset + eeprom->len - 1) >> 1; eeprom_buff = kmalloc(max_len, GFP_KERNEL); diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index 364378133526a..df4e7d781cb1c 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -4274,6 +4274,8 @@ static s32 e1000_validate_nvm_checksum_ich8lan(struct e1000_hw *hw) ret_val = e1000e_update_nvm_checksum(hw); if (ret_val) return ret_val; + } else if (hw->mac.type == e1000_pch_tgp) { + return 0; } } diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 721c098f2bb1b..7e4fea0e186b6 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -3540,9 +3540,6 @@ s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: - case e1000_pch_mtp: - case e1000_pch_lnp: - case e1000_pch_ptp: case e1000_pch_nvp: if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { /* Stable 24MHz frequency */ @@ -3558,6 +3555,17 @@ s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca) adapter->cc.shift = shift; } break; + case e1000_pch_mtp: + case e1000_pch_lnp: + case e1000_pch_ptp: + /* System firmware can misreport this value, so set it to a + * stable 38400KHz frequency. + */ + incperiod = INCPERIOD_38400KHZ; + incvalue = INCVALUE_38400KHZ; + shift = INCVALUE_SHIFT_38400KHZ; + adapter->cc.shift = shift; + break; case e1000_82574: case e1000_82583: /* Stable 25MHz frequency */ diff --git a/drivers/net/ethernet/intel/e1000e/nvm.c b/drivers/net/ethernet/intel/e1000e/nvm.c index e609f4df86f45..16369e6d245a4 100644 --- a/drivers/net/ethernet/intel/e1000e/nvm.c +++ b/drivers/net/ethernet/intel/e1000e/nvm.c @@ -558,6 +558,12 @@ s32 e1000e_validate_nvm_checksum_generic(struct e1000_hw *hw) checksum += nvm_data; } + if (hw->mac.type == e1000_pch_tgp && + nvm_data == NVM_CHECKSUM_UNINITIALIZED) { + e_dbg("Uninitialized NVM Checksum on TGP platform - ignoring\n"); + return 0; + } + if (checksum != (u16)NVM_SUM) { e_dbg("NVM Checksum Invalid\n"); return -E1000_ERR_NVM; diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c index bbcfd529399b0..d039dea48ca32 100644 --- a/drivers/net/ethernet/intel/e1000e/ptp.c +++ b/drivers/net/ethernet/intel/e1000e/ptp.c @@ -294,15 +294,17 @@ void e1000e_ptp_init(struct e1000_adapter *adapter) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: - case e1000_pch_mtp: - case e1000_pch_lnp: - case e1000_pch_ptp: case e1000_pch_nvp: if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) adapter->ptp_clock_info.max_adj = MAX_PPB_24MHZ; else adapter->ptp_clock_info.max_adj = MAX_PPB_38400KHZ; break; + case e1000_pch_mtp: + case e1000_pch_lnp: + case e1000_pch_ptp: + adapter->ptp_clock_info.max_adj = MAX_PPB_38400KHZ; + break; case e1000_82574: case e1000_82583: adapter->ptp_clock_info.max_adj = MAX_PPB_25MHZ; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k.h b/drivers/net/ethernet/intel/fm10k/fm10k.h index 6119a41088381..65a2816142d96 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k.h +++ b/drivers/net/ethernet/intel/fm10k/fm10k.h @@ -189,13 +189,14 @@ struct fm10k_q_vector { struct fm10k_ring_container rx, tx; struct napi_struct napi; + struct rcu_head rcu; /* to avoid race with update stats on free */ + cpumask_t affinity_mask; char name[IFNAMSIZ + 9]; #ifdef CONFIG_DEBUG_FS struct dentry *dbg_q_vector; #endif /* CONFIG_DEBUG_FS */ - struct rcu_head rcu; /* to avoid race with update stats on free */ /* for dynamic allocation of rings associated with this q_vector */ struct fm10k_ring ring[] ____cacheline_internodealigned_in_smp; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.c b/drivers/net/ethernet/intel/fm10k/fm10k_common.c index f51a63fca513e..1f919a50c7653 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_common.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.c @@ -447,17 +447,16 @@ void fm10k_update_hw_stats_q(struct fm10k_hw *hw, struct fm10k_hw_stats_q *q, /** * fm10k_unbind_hw_stats_q - Unbind the queue counters from their queues * @q: pointer to the ring of hardware statistics queue - * @idx: index pointing to the start of the ring iteration * @count: number of queues to iterate over * * Function invalidates the index values for the queues so any updates that * may have happened are ignored and the base for the queue stats is reset. **/ -void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 idx, u32 count) +void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 count) { u32 i; - for (i = 0; i < count; i++, idx++, q++) { + for (i = 0; i < count; i++, q++) { q->rx_stats_idx = 0; q->tx_stats_idx = 0; } diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.h b/drivers/net/ethernet/intel/fm10k/fm10k_common.h index 4c48fb73b3e78..13fca6a91a01b 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_common.h +++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.h @@ -43,6 +43,6 @@ u32 fm10k_read_hw_stats_32b(struct fm10k_hw *hw, u32 addr, void fm10k_update_hw_stats_q(struct fm10k_hw *hw, struct fm10k_hw_stats_q *q, u32 idx, u32 count); #define fm10k_unbind_hw_stats_32b(s) ((s)->base_h = 0) -void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 idx, u32 count); +void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 count); s32 fm10k_get_host_state_generic(struct fm10k_hw *hw, bool *host_ready); #endif /* _FM10K_COMMON_H_ */ diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c index aed5e0bf6313e..b51b6003ad032 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c @@ -1510,7 +1510,7 @@ static void fm10k_rebind_hw_stats_pf(struct fm10k_hw *hw, fm10k_unbind_hw_stats_32b(&stats->nodesc_drop); /* Unbind Queue Statistics */ - fm10k_unbind_hw_stats_q(stats->q, 0, hw->mac.max_queues); + fm10k_unbind_hw_stats_q(stats->q, hw->mac.max_queues); /* Reinitialize bases for all stats */ fm10k_update_hw_stats_pf(hw, stats); diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c index 7fb1961f29210..6861a0bdc14e1 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c @@ -465,7 +465,7 @@ static void fm10k_rebind_hw_stats_vf(struct fm10k_hw *hw, struct fm10k_hw_stats *stats) { /* Unbind Queue Statistics */ - fm10k_unbind_hw_stats_q(stats->q, 0, hw->mac.max_queues); + fm10k_unbind_hw_stats_q(stats->q, hw->mac.max_queues); /* Reinitialize bases for all stats */ fm10k_update_hw_stats_vf(hw, stats); diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 68f403dd2f526..f095069dc32a9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -23,6 +23,7 @@ #define I40E_MAX_VEB 16 #define I40E_MAX_NUM_DESCRIPTORS 4096 +#define I40E_MAX_NUM_DESCRIPTORS_XL710 8160 #define I40E_MAX_CSR_SPACE (4 * 1024 * 1024 - 64 * 1024) #define I40E_DEFAULT_NUM_DESCRIPTORS 512 #define I40E_REQ_DESCRIPTOR_MULTIPLE 32 @@ -953,6 +954,7 @@ struct i40e_q_vector { u16 reg_idx; /* register index of the interrupt */ struct napi_struct napi; + struct rcu_head rcu; /* to avoid race with update stats on free */ struct i40e_ring_container rx; struct i40e_ring_container tx; @@ -963,7 +965,6 @@ struct i40e_q_vector { cpumask_t affinity_mask; struct irq_affinity_notify affinity_notify; - struct rcu_head rcu; /* to avoid race with update stats on free */ char name[I40E_INT_NAME_STR_LEN]; bool arm_wb_state; bool in_busy_poll; @@ -1228,7 +1229,8 @@ struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr); int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr); bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi); -int i40e_count_filters(struct i40e_vsi *vsi); +int i40e_count_all_filters(struct i40e_vsi *vsi); +int i40e_count_active_filters(struct i40e_vsi *vsi); struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr); void i40e_vlan_stripping_enable(struct i40e_vsi *vsi); static inline bool i40e_is_sw_dcb(struct i40e_pf *pf) diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c index 306758428aefd..a569d2fcc90af 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_client.c +++ b/drivers/net/ethernet/intel/i40e/i40e_client.c @@ -361,8 +361,8 @@ static void i40e_client_add_instance(struct i40e_pf *pf) if (i40e_client_get_params(vsi, &cdev->lan_info.params)) goto free_cdev; - mac = list_first_entry(&cdev->lan_info.netdev->dev_addrs.list, - struct netdev_hw_addr, list); + mac = list_first_entry_or_null(&cdev->lan_info.netdev->dev_addrs.list, + struct netdev_hw_addr, list); if (mac) ether_addr_copy(cdev->lan_info.lanmac, mac->addr); else diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c index 4d7caa1199719..5d46a8e5376da 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_common.c +++ b/drivers/net/ethernet/intel/i40e/i40e_common.c @@ -1067,10 +1067,11 @@ int i40e_pf_reset(struct i40e_hw *hw) void i40e_clear_hw(struct i40e_hw *hw) { u32 num_queues, base_queue; - u32 num_pf_int; - u32 num_vf_int; + s32 num_pf_int; + s32 num_vf_int; u32 num_vfs; - u32 i, j; + s32 i; + u32 j; u32 val; u32 eol = 0x7ff; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 4e90570ba7803..077d5daaedcca 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -246,6 +246,7 @@ static const struct i40e_stats i40e_gstrings_net_stats[] = { I40E_NETDEV_STAT(rx_errors), I40E_NETDEV_STAT(tx_errors), I40E_NETDEV_STAT(rx_dropped), + I40E_NETDEV_STAT(rx_missed_errors), I40E_NETDEV_STAT(tx_dropped), I40E_NETDEV_STAT(collisions), I40E_NETDEV_STAT(rx_length_errors), @@ -322,7 +323,7 @@ static const struct i40e_stats i40e_gstrings_stats[] = { I40E_PF_STAT("port.rx_broadcast", stats.eth.rx_broadcast), I40E_PF_STAT("port.tx_broadcast", stats.eth.tx_broadcast), I40E_PF_STAT("port.tx_errors", stats.eth.tx_errors), - I40E_PF_STAT("port.rx_dropped", stats.eth.rx_discards), + I40E_PF_STAT("port.rx_discards", stats.eth.rx_discards), I40E_PF_STAT("port.tx_dropped_link_down", stats.tx_dropped_link_down), I40E_PF_STAT("port.rx_crc_errors", stats.crc_errors), I40E_PF_STAT("port.illegal_bytes", stats.illegal_bytes), @@ -2014,6 +2015,18 @@ static void i40e_get_drvinfo(struct net_device *netdev, drvinfo->n_priv_flags += I40E_GL_PRIV_FLAGS_STR_LEN; } +static u32 i40e_get_max_num_descriptors(struct i40e_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + + switch (hw->mac.type) { + case I40E_MAC_XL710: + return I40E_MAX_NUM_DESCRIPTORS_XL710; + default: + return I40E_MAX_NUM_DESCRIPTORS; + } +} + static void i40e_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct kernel_ethtool_ringparam *kernel_ring, @@ -2023,8 +2036,8 @@ static void i40e_get_ringparam(struct net_device *netdev, struct i40e_pf *pf = np->vsi->back; struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; - ring->rx_max_pending = I40E_MAX_NUM_DESCRIPTORS; - ring->tx_max_pending = I40E_MAX_NUM_DESCRIPTORS; + ring->rx_max_pending = i40e_get_max_num_descriptors(pf); + ring->tx_max_pending = i40e_get_max_num_descriptors(pf); ring->rx_mini_max_pending = 0; ring->rx_jumbo_max_pending = 0; ring->rx_pending = vsi->rx_rings[0]->count; @@ -2049,12 +2062,12 @@ static int i40e_set_ringparam(struct net_device *netdev, struct kernel_ethtool_ringparam *kernel_ring, struct netlink_ext_ack *extack) { + u32 new_rx_count, new_tx_count, max_num_descriptors; struct i40e_ring *tx_rings = NULL, *rx_rings = NULL; struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_hw *hw = &np->vsi->back->hw; struct i40e_vsi *vsi = np->vsi; struct i40e_pf *pf = vsi->back; - u32 new_rx_count, new_tx_count; u16 tx_alloc_queue_pairs; int timeout = 50; int i, err = 0; @@ -2062,14 +2075,15 @@ static int i40e_set_ringparam(struct net_device *netdev, if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) return -EINVAL; - if (ring->tx_pending > I40E_MAX_NUM_DESCRIPTORS || + max_num_descriptors = i40e_get_max_num_descriptors(pf); + if (ring->tx_pending > max_num_descriptors || ring->tx_pending < I40E_MIN_NUM_DESCRIPTORS || - ring->rx_pending > I40E_MAX_NUM_DESCRIPTORS || + ring->rx_pending > max_num_descriptors || ring->rx_pending < I40E_MIN_NUM_DESCRIPTORS) { netdev_info(netdev, "Descriptors requested (Tx: %d / Rx: %d) out of range [%d-%d]\n", ring->tx_pending, ring->rx_pending, - I40E_MIN_NUM_DESCRIPTORS, I40E_MAX_NUM_DESCRIPTORS); + I40E_MIN_NUM_DESCRIPTORS, max_num_descriptors); return -EINVAL; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 806cec458a072..affdbd3ee76c5 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -509,6 +509,7 @@ static void i40e_get_netdev_stats_struct(struct net_device *netdev, stats->tx_dropped = vsi_stats->tx_dropped; stats->rx_errors = vsi_stats->rx_errors; stats->rx_dropped = vsi_stats->rx_dropped; + stats->rx_missed_errors = vsi_stats->rx_missed_errors; stats->rx_crc_errors = vsi_stats->rx_crc_errors; stats->rx_length_errors = vsi_stats->rx_length_errors; } @@ -700,17 +701,13 @@ i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw, struct i40e_eth_stats *stat_offset, struct i40e_eth_stats *stat) { - u64 rx_rdpc, rx_rxerr; - i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded, - &stat_offset->rx_discards, &rx_rdpc); + &stat_offset->rx_discards, &stat->rx_discards); i40e_stat_update64(hw, I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)), I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)), offset_loaded, &stat_offset->rx_discards_other, - &rx_rxerr); - - stat->rx_discards = rx_rdpc + rx_rxerr; + &stat->rx_discards_other); } /** @@ -732,9 +729,6 @@ void i40e_update_eth_stats(struct i40e_vsi *vsi) i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), vsi->stat_offsets_loaded, &oes->tx_errors, &es->tx_errors); - i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_discards, &es->rx_discards); i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), vsi->stat_offsets_loaded, &oes->rx_unknown_protocol, &es->rx_unknown_protocol); @@ -991,8 +985,10 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi) ns->tx_errors = es->tx_errors; ons->multicast = oes->rx_multicast; ns->multicast = es->rx_multicast; - ons->rx_dropped = oes->rx_discards; - ns->rx_dropped = es->rx_discards; + ons->rx_dropped = oes->rx_discards_other; + ns->rx_dropped = es->rx_discards_other; + ons->rx_missed_errors = oes->rx_discards; + ns->rx_missed_errors = es->rx_discards; ons->tx_dropped = oes->tx_discards; ns->tx_dropped = es->tx_discards; @@ -1251,12 +1247,30 @@ void i40e_update_stats(struct i40e_vsi *vsi) } /** - * i40e_count_filters - counts VSI mac filters + * i40e_count_all_filters - counts VSI MAC filters * @vsi: the VSI to be searched * - * Returns count of mac filters - **/ -int i40e_count_filters(struct i40e_vsi *vsi) + * Return: count of MAC filters in any state. + */ +int i40e_count_all_filters(struct i40e_vsi *vsi) +{ + struct i40e_mac_filter *f; + struct hlist_node *h; + int bkt, cnt = 0; + + hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) + cnt++; + + return cnt; +} + +/** + * i40e_count_active_filters - counts VSI MAC filters + * @vsi: the VSI to be searched + * + * Return: count of active MAC filters. + */ +int i40e_count_active_filters(struct i40e_vsi *vsi) { struct i40e_mac_filter *f; struct hlist_node *h; @@ -4214,7 +4228,7 @@ static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) irq_num = pf->msix_entries[base + vector].vector; irq_set_affinity_notifier(irq_num, NULL); irq_update_affinity_hint(irq_num, NULL); - free_irq(irq_num, &vsi->q_vectors[vector]); + free_irq(irq_num, vsi->q_vectors[vector]); } return err; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index c962987d8b51b..99604379c87b6 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -950,9 +950,6 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, if (!eop_desc) break; - /* prevent any other reads prior to eop_desc */ - smp_rmb(); - i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf); /* we have caught up to head, no work left to do */ if (tx_head == tx_desc) @@ -2401,7 +2398,7 @@ void i40e_update_rx_stats(struct i40e_ring *rx_ring, void i40e_finalize_xdp_rx(struct i40e_ring *rx_ring, unsigned int xdp_res) { if (xdp_res & I40E_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); if (xdp_res & I40E_XDP_TX) { struct i40e_ring *xdp_ring = diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index d5509bc16d0d5..7f5538e2c9de5 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -448,7 +448,7 @@ static void i40e_config_irq_link_list(struct i40e_vf *vf, u16 vsi_id, (qtype << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | (pf_queue_id << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | BIT(I40E_QINT_RQCTL_CAUSE_ENA_SHIFT) | - (itr_idx << I40E_QINT_RQCTL_ITR_INDX_SHIFT); + FIELD_PREP(I40E_QINT_RQCTL_ITR_INDX_MASK, itr_idx); wr32(hw, reg_idx, reg); } @@ -657,6 +657,13 @@ static int i40e_config_vsi_tx_queue(struct i40e_vf *vf, u16 vsi_id, /* only set the required fields */ tx_ctx.base = info->dma_ring_addr / 128; + + /* ring_len has to be multiple of 8 */ + if (!IS_ALIGNED(info->ring_len, 8) || + info->ring_len > I40E_MAX_NUM_DESCRIPTORS_XL710) { + ret = -EINVAL; + goto error_context; + } tx_ctx.qlen = info->ring_len; tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[0]); tx_ctx.rdylist_act = 0; @@ -722,6 +729,13 @@ static int i40e_config_vsi_rx_queue(struct i40e_vf *vf, u16 vsi_id, /* only set the required fields */ rx_ctx.base = info->dma_ring_addr / 128; + + /* ring_len has to be multiple of 32 */ + if (!IS_ALIGNED(info->ring_len, 32) || + info->ring_len > I40E_MAX_NUM_DESCRIPTORS_XL710) { + ret = -EINVAL; + goto error_param; + } rx_ctx.qlen = info->ring_len; if (info->splithdr_enabled) { @@ -1459,6 +1473,7 @@ static void i40e_trigger_vf_reset(struct i40e_vf *vf, bool flr) * functions that may still be running at this point. */ clear_bit(I40E_VF_STATE_INIT, &vf->vf_states); + clear_bit(I40E_VF_STATE_RESOURCES_LOADED, &vf->vf_states); /* In the case of a VFLR, the HW has already reset the VF and we * just need to clean up, so don't hit the VFRTRIG register. @@ -1552,8 +1567,8 @@ static void i40e_cleanup_reset_vf(struct i40e_vf *vf) * @vf: pointer to the VF structure * @flr: VFLR was issued or not * - * Returns true if the VF is in reset, resets successfully, or resets - * are disabled and false otherwise. + * Return: True if reset was performed successfully or if resets are disabled. + * False if reset is already in progress. **/ bool i40e_reset_vf(struct i40e_vf *vf, bool flr) { @@ -1572,7 +1587,7 @@ bool i40e_reset_vf(struct i40e_vf *vf, bool flr) /* If VF is being reset already we don't need to continue. */ if (test_and_set_bit(I40E_VF_STATE_RESETTING, &vf->vf_states)) - return true; + return false; i40e_trigger_vf_reset(vf, flr); @@ -2125,7 +2140,10 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg) size_t len = 0; int ret; - if (!i40e_sync_vf_state(vf, I40E_VF_STATE_INIT)) { + i40e_sync_vf_state(vf, I40E_VF_STATE_INIT); + + if (!test_bit(I40E_VF_STATE_INIT, &vf->vf_states) || + test_bit(I40E_VF_STATE_RESOURCES_LOADED, &vf->vf_states)) { aq_ret = -EINVAL; goto err; } @@ -2228,6 +2246,7 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg) vf->default_lan_addr.addr); } set_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states); + set_bit(I40E_VF_STATE_RESOURCES_LOADED, &vf->vf_states); err: /* send the response back to the VF */ @@ -2390,7 +2409,7 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) } if (vf->adq_enabled) { - if (idx >= ARRAY_SIZE(vf->ch)) { + if (idx >= vf->num_tc) { aq_ret = -ENODEV; goto error_param; } @@ -2411,7 +2430,7 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg) * to its appropriate VSIs based on TC mapping */ if (vf->adq_enabled) { - if (idx >= ARRAY_SIZE(vf->ch)) { + if (idx >= vf->num_tc) { aq_ret = -ENODEV; goto error_param; } @@ -2461,8 +2480,10 @@ static int i40e_validate_queue_map(struct i40e_vf *vf, u16 vsi_id, u16 vsi_queue_id, queue_id; for_each_set_bit(vsi_queue_id, &queuemap, I40E_MAX_VSI_QP) { - if (vf->adq_enabled) { - vsi_id = vf->ch[vsi_queue_id / I40E_MAX_VF_VSI].vsi_id; + u16 idx = vsi_queue_id / I40E_MAX_VF_VSI; + + if (vf->adq_enabled && idx < vf->num_tc) { + vsi_id = vf->ch[idx].vsi_id; queue_id = (vsi_queue_id % I40E_DEFAULT_QUEUES_PER_VF); } else { queue_id = vsi_queue_id; @@ -2850,24 +2871,6 @@ static int i40e_vc_get_stats_msg(struct i40e_vf *vf, u8 *msg) (u8 *)&stats, sizeof(stats)); } -/** - * i40e_can_vf_change_mac - * @vf: pointer to the VF info - * - * Return true if the VF is allowed to change its MAC filters, false otherwise - */ -static bool i40e_can_vf_change_mac(struct i40e_vf *vf) -{ - /* If the VF MAC address has been set administratively (via the - * ndo_set_vf_mac command), then deny permission to the VF to - * add/delete unicast MAC addresses, unless the VF is trusted - */ - if (vf->pf_set_mac && !vf->trusted) - return false; - - return true; -} - #define I40E_MAX_MACVLAN_PER_HW 3072 #define I40E_MAX_MACVLAN_PER_PF(num_ports) (I40E_MAX_MACVLAN_PER_HW / \ (num_ports)) @@ -2906,8 +2909,10 @@ static inline int i40e_check_vf_permission(struct i40e_vf *vf, struct i40e_pf *pf = vf->pf; struct i40e_vsi *vsi = pf->vsi[vf->lan_vsi_idx]; struct i40e_hw *hw = &pf->hw; - int mac2add_cnt = 0; - int i; + int i, mac_add_max, mac_add_cnt = 0; + bool vf_trusted; + + vf_trusted = test_bit(I40E_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps); for (i = 0; i < al->num_elements; i++) { struct i40e_mac_filter *f; @@ -2927,9 +2932,8 @@ static inline int i40e_check_vf_permission(struct i40e_vf *vf, * The VF may request to set the MAC address filter already * assigned to it so do not return an error in that case. */ - if (!i40e_can_vf_change_mac(vf) && - !is_multicast_ether_addr(addr) && - !ether_addr_equal(addr, vf->default_lan_addr.addr)) { + if (!vf_trusted && !is_multicast_ether_addr(addr) && + vf->pf_set_mac && !ether_addr_equal(addr, vf->default_lan_addr.addr)) { dev_err(&pf->pdev->dev, "VF attempting to override administratively set MAC address, bring down and up the VF interface to resume normal operation\n"); return -EPERM; @@ -2938,29 +2942,33 @@ static inline int i40e_check_vf_permission(struct i40e_vf *vf, /*count filters that really will be added*/ f = i40e_find_mac(vsi, addr); if (!f) - ++mac2add_cnt; + ++mac_add_cnt; } /* If this VF is not privileged, then we can't add more than a limited - * number of addresses. Check to make sure that the additions do not - * push us over the limit. - */ - if (!test_bit(I40E_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps)) { - if ((i40e_count_filters(vsi) + mac2add_cnt) > - I40E_VC_MAX_MAC_ADDR_PER_VF) { - dev_err(&pf->pdev->dev, - "Cannot add more MAC addresses, VF is not trusted, switch the VF to trusted to add more functionality\n"); - return -EPERM; - } - /* If this VF is trusted, it can use more resources than untrusted. + * number of addresses. + * + * If this VF is trusted, it can use more resources than untrusted. * However to ensure that every trusted VF has appropriate number of * resources, divide whole pool of resources per port and then across * all VFs. */ - } else { - if ((i40e_count_filters(vsi) + mac2add_cnt) > - I40E_VC_MAX_MACVLAN_PER_TRUSTED_VF(pf->num_alloc_vfs, - hw->num_ports)) { + if (!vf_trusted) + mac_add_max = I40E_VC_MAX_MAC_ADDR_PER_VF; + else + mac_add_max = I40E_VC_MAX_MACVLAN_PER_TRUSTED_VF(pf->num_alloc_vfs, hw->num_ports); + + /* VF can replace all its filters in one step, in this case mac_add_max + * will be added as active and another mac_add_max will be in + * a to-be-removed state. Account for that. + */ + if ((i40e_count_active_filters(vsi) + mac_add_cnt) > mac_add_max || + (i40e_count_all_filters(vsi) + mac_add_cnt) > 2 * mac_add_max) { + if (!vf_trusted) { + dev_err(&pf->pdev->dev, + "Cannot add more MAC addresses, VF is not trusted, switch the VF to trusted to add more functionality\n"); + return -EPERM; + } else { dev_err(&pf->pdev->dev, "Cannot add more MAC addresses, trusted VF exhausted it's resources\n"); return -EPERM; @@ -3143,10 +3151,10 @@ static int i40e_vc_del_mac_addr_msg(struct i40e_vf *vf, u8 *msg) const u8 *addr = al->list[i].addr; /* Allow to delete VF primary MAC only if it was not set - * administratively by PF or if VF is trusted. + * administratively by PF. */ if (ether_addr_equal(addr, vf->default_lan_addr.addr)) { - if (i40e_can_vf_change_mac(vf)) + if (!vf->pf_set_mac) was_unimac_deleted = true; else continue; @@ -3593,7 +3601,7 @@ static int i40e_validate_cloud_filter(struct i40e_vf *vf, /* action_meta is TC number here to which the filter is applied */ if (!tc_filter->action_meta || - tc_filter->action_meta > vf->num_tc) { + tc_filter->action_meta >= vf->num_tc) { dev_info(&pf->pdev->dev, "VF %d: Invalid TC number %u\n", vf->vf_id, tc_filter->action_meta); goto err; @@ -3891,6 +3899,8 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg) aq_ret); } +#define I40E_MAX_VF_CLOUD_FILTER 0xFF00 + /** * i40e_vc_add_cloud_filter * @vf: pointer to the VF info @@ -3930,6 +3940,14 @@ static int i40e_vc_add_cloud_filter(struct i40e_vf *vf, u8 *msg) goto err_out; } + if (vf->num_cloud_filters >= I40E_MAX_VF_CLOUD_FILTER) { + dev_warn(&pf->pdev->dev, + "VF %d: Max number of filters reached, can't apply cloud filter\n", + vf->vf_id); + aq_ret = -ENOSPC; + goto err_out; + } + cfilter = kzalloc(sizeof(*cfilter), GFP_KERNEL); if (!cfilter) { aq_ret = -ENOMEM; @@ -4332,7 +4350,10 @@ int i40e_vc_process_vflr_event(struct i40e_pf *pf) reg = rd32(hw, I40E_GLGEN_VFLRSTAT(reg_idx)); if (reg & BIT(bit_idx)) /* i40e_reset_vf will clear the bit in GLGEN_VFLRSTAT */ - i40e_reset_vf(vf, true); + if (!i40e_reset_vf(vf, true)) { + /* At least one VF did not finish resetting, retry next time */ + set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); + } } return 0; @@ -5007,8 +5028,8 @@ int i40e_get_vf_stats(struct net_device *netdev, int vf_id, vf_stats->tx_bytes = stats->tx_bytes; vf_stats->broadcast = stats->rx_broadcast; vf_stats->multicast = stats->rx_multicast; - vf_stats->rx_dropped = stats->rx_discards; - vf_stats->tx_dropped = stats->tx_discards; + vf_stats->rx_dropped = stats->rx_discards + stats->rx_discards_other; + vf_stats->tx_dropped = stats->tx_errors; return 0; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h index 66f95e2f3146a..e0e797fea138a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h @@ -41,7 +41,8 @@ enum i40e_vf_states { I40E_VF_STATE_MC_PROMISC, I40E_VF_STATE_UC_PROMISC, I40E_VF_STATE_PRE_ENABLE, - I40E_VF_STATE_RESETTING + I40E_VF_STATE_RESETTING, + I40E_VF_STATE_RESOURCES_LOADED, }; /* VF capabilities */ diff --git a/drivers/net/ethernet/intel/ice/ice_arfs.c b/drivers/net/ethernet/intel/ice/ice_arfs.c index d7e0116f67737..7036070bc0208 100644 --- a/drivers/net/ethernet/intel/ice/ice_arfs.c +++ b/drivers/net/ethernet/intel/ice/ice_arfs.c @@ -376,6 +376,50 @@ ice_arfs_is_perfect_flow_set(struct ice_hw *hw, __be16 l3_proto, u8 l4_proto) return false; } +/** + * ice_arfs_cmp - Check if aRFS filter matches this flow. + * @fltr_info: filter info of the saved ARFS entry. + * @fk: flow dissector keys. + * @n_proto: One of htons(ETH_P_IP) or htons(ETH_P_IPV6). + * @ip_proto: One of IPPROTO_TCP or IPPROTO_UDP. + * + * Since this function assumes limited values for n_proto and ip_proto, it + * is meant to be called only from ice_rx_flow_steer(). + * + * Return: + * * true - fltr_info refers to the same flow as fk. + * * false - fltr_info and fk refer to different flows. + */ +static bool +ice_arfs_cmp(const struct ice_fdir_fltr *fltr_info, const struct flow_keys *fk, + __be16 n_proto, u8 ip_proto) +{ + /* Determine if the filter is for IPv4 or IPv6 based on flow_type, + * which is one of ICE_FLTR_PTYPE_NONF_IPV{4,6}_{TCP,UDP}. + */ + bool is_v4 = fltr_info->flow_type == ICE_FLTR_PTYPE_NONF_IPV4_TCP || + fltr_info->flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP; + + /* Following checks are arranged in the quickest and most discriminative + * fields first for early failure. + */ + if (is_v4) + return n_proto == htons(ETH_P_IP) && + fltr_info->ip.v4.src_port == fk->ports.src && + fltr_info->ip.v4.dst_port == fk->ports.dst && + fltr_info->ip.v4.src_ip == fk->addrs.v4addrs.src && + fltr_info->ip.v4.dst_ip == fk->addrs.v4addrs.dst && + fltr_info->ip.v4.proto == ip_proto; + + return fltr_info->ip.v6.src_port == fk->ports.src && + fltr_info->ip.v6.dst_port == fk->ports.dst && + fltr_info->ip.v6.proto == ip_proto && + !memcmp(&fltr_info->ip.v6.src_ip, &fk->addrs.v6addrs.src, + sizeof(struct in6_addr)) && + !memcmp(&fltr_info->ip.v6.dst_ip, &fk->addrs.v6addrs.dst, + sizeof(struct in6_addr)); +} + /** * ice_rx_flow_steer - steer the Rx flow to where application is being run * @netdev: ptr to the netdev being adjusted @@ -447,6 +491,10 @@ ice_rx_flow_steer(struct net_device *netdev, const struct sk_buff *skb, continue; fltr_info = &arfs_entry->fltr_info; + + if (!ice_arfs_cmp(fltr_info, &fk, n_proto, ip_proto)) + continue; + ret = fltr_info->fltr_id; if (fltr_info->q_index == rxq_idx || diff --git a/drivers/net/ethernet/intel/ice/ice_ddp.c b/drivers/net/ethernet/intel/ice/ice_ddp.c index b27ec93638b6b..cd97974b451a0 100644 --- a/drivers/net/ethernet/intel/ice/ice_ddp.c +++ b/drivers/net/ethernet/intel/ice/ice_ddp.c @@ -1884,6 +1884,8 @@ enum ice_ddp_state ice_copy_and_init_pkg(struct ice_hw *hw, const u8 *buf, return ICE_DDP_PKG_ERR; buf_copy = devm_kmemdup(ice_hw_to_dev(hw), buf, len, GFP_KERNEL); + if (!buf_copy) + return ICE_DDP_PKG_ERR; state = ice_init_pkg(hw, buf_copy, len); if (!ice_is_init_pkg_successful(state)) { diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 39b5f24be7e4f..dd58b2372dc0c 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -3329,8 +3329,7 @@ static u32 ice_get_combined_cnt(struct ice_vsi *vsi) ice_for_each_q_vector(vsi, q_idx) { struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; - if (q_vector->rx.rx_ring && q_vector->tx.tx_ring) - combined++; + combined += min(q_vector->num_ring_tx, q_vector->num_ring_rx); } return combined; diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c index ad82ff7d19957..09f9c7ba52795 100644 --- a/drivers/net/ethernet/intel/ice/ice_irq.c +++ b/drivers/net/ethernet/intel/ice/ice_irq.c @@ -45,7 +45,7 @@ static void ice_free_irq_res(struct ice_pf *pf, u16 index) /** * ice_get_irq_res - get an interrupt resource * @pf: board private structure - * @dyn_only: force entry to be dynamically allocated + * @dyn_allowed: allow entry to be dynamically allocated * * Allocate new irq entry in the free slot of the tracker. Since xarray * is used, always allocate new entry at the lowest possible index. Set @@ -53,11 +53,12 @@ static void ice_free_irq_res(struct ice_pf *pf, u16 index) * * Returns allocated irq entry or NULL on failure. */ -static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only) +static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, + bool dyn_allowed) { - struct xa_limit limit = { .max = pf->irq_tracker.num_entries, + struct xa_limit limit = { .max = pf->irq_tracker.num_entries - 1, .min = 0 }; - unsigned int num_static = pf->irq_tracker.num_static; + unsigned int num_static = pf->irq_tracker.num_static - 1; struct ice_irq_entry *entry; unsigned int index; int ret; @@ -66,9 +67,9 @@ static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only) if (!entry) return NULL; - /* skip preallocated entries if the caller says so */ - if (dyn_only) - limit.min = num_static; + /* only already allocated if the caller says so */ + if (!dyn_allowed) + limit.max = num_static; ret = xa_alloc(&pf->irq_tracker.entries, &index, entry, limit, GFP_KERNEL); @@ -78,7 +79,7 @@ static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only) entry = NULL; } else { entry->index = index; - entry->dynamic = index >= num_static; + entry->dynamic = index > num_static; } return entry; @@ -272,7 +273,7 @@ int ice_init_interrupt_scheme(struct ice_pf *pf) /** * ice_alloc_irq - Allocate new interrupt vector * @pf: board private structure - * @dyn_only: force dynamic allocation of the interrupt + * @dyn_allowed: allow dynamic allocation of the interrupt * * Allocate new interrupt vector for a given owner id. * return struct msi_map with interrupt details and track @@ -285,20 +286,20 @@ int ice_init_interrupt_scheme(struct ice_pf *pf) * interrupt will be allocated with pci_msix_alloc_irq_at. * * Some callers may only support dynamically allocated interrupts. - * This is indicated with dyn_only flag. + * This is indicated with dyn_allowed flag. * * On failure, return map with negative .index. The caller * is expected to check returned map index. * */ -struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_only) +struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_allowed) { int sriov_base_vector = pf->sriov_base_vector; struct msi_map map = { .index = -ENOENT }; struct device *dev = ice_pf_to_dev(pf); struct ice_irq_entry *entry; - entry = ice_get_irq_res(pf, dyn_only); + entry = ice_get_irq_res(pf, dyn_allowed); if (!entry) return map; diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c index 4e675c7c199fa..8ed9918ea4e89 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.c +++ b/drivers/net/ethernet/intel/ice/ice_lag.c @@ -1229,12 +1229,18 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) */ if (!primary_lag) { lag->primary = true; + if (!ice_is_switchdev_running(lag->pf)) + return; + /* Configure primary's SWID to be shared */ ice_lag_primary_swid(lag, true); primary_lag = lag; } else { u16 swid; + if (!ice_is_switchdev_running(primary_lag->pf)) + return; + swid = primary_lag->pf->hw.port_info->sw_id; ice_lag_set_swid(swid, lag, true); ice_lag_add_prune_list(primary_lag, lag->pf); @@ -2123,7 +2129,8 @@ bool ice_lag_is_switchdev_running(struct ice_pf *pf) struct ice_lag *lag = pf->lag; struct net_device *tmp_nd; - if (!ice_is_feature_supported(pf, ICE_F_SRIOV_LAG) || !lag) + if (!ice_is_feature_supported(pf, ICE_F_SRIOV_LAG) || + !lag || !lag->upper_netdev) return false; rcu_read_lock(); diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 1fc4805353eb5..a6a290514e548 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -587,6 +587,8 @@ ice_vsi_alloc_def(struct ice_vsi *vsi, struct ice_channel *ch) return -ENOMEM; } + vsi->irq_dyn_alloc = pci_msix_can_alloc_dyn(vsi->back->pdev); + switch (vsi->type) { case ICE_VSI_SWITCHDEV_CTRL: /* Setup eswitch MSIX irq handler for VSI */ diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 0ae7bdfff83fb..e846246261b94 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -2650,6 +2650,27 @@ static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) bpf_prog_put(old_prog); } +/** + * ice_unmap_xdp_rings - Unmap XDP rings from interrupt vectors + * @vsi: the VSI with XDP rings being unmapped + */ +static void ice_unmap_xdp_rings(struct ice_vsi *vsi) +{ + int v_idx; + + ice_for_each_q_vector(vsi, v_idx) { + struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; + struct ice_tx_ring *ring; + + ice_for_each_tx_ring(ring, q_vector->tx) + if (!ring->tx_buf || !ice_ring_is_xdp(ring)) + break; + + /* restore the value of last node prior to XDP setup */ + q_vector->tx.tx_ring = ring; + } +} + /** * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP * @vsi: VSI to bring up Tx rings used by XDP @@ -2749,7 +2770,7 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog, if (status) { dev_err(dev, "Failed VSI LAN queue config for XDP, error: %d\n", status); - goto clear_xdp_rings; + goto unmap_xdp_rings; } /* assign the prog only when it's not already present on VSI; @@ -2765,6 +2786,8 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog, ice_vsi_assign_bpf_prog(vsi, prog); return 0; +unmap_xdp_rings: + ice_unmap_xdp_rings(vsi); clear_xdp_rings: ice_for_each_xdp_txq(vsi, i) if (vsi->xdp_rings[i]) { @@ -2781,6 +2804,8 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog, mutex_unlock(&pf->avail_q_mutex); devm_kfree(dev, vsi->xdp_rings); + vsi->xdp_rings = NULL; + return -ENOMEM; } @@ -2796,7 +2821,7 @@ int ice_destroy_xdp_rings(struct ice_vsi *vsi, enum ice_xdp_cfg cfg_type) { u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; struct ice_pf *pf = vsi->back; - int i, v_idx; + int i; /* q_vectors are freed in reset path so there's no point in detaching * rings @@ -2804,17 +2829,7 @@ int ice_destroy_xdp_rings(struct ice_vsi *vsi, enum ice_xdp_cfg cfg_type) if (cfg_type == ICE_XDP_CFG_PART) goto free_qmap; - ice_for_each_q_vector(vsi, v_idx) { - struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; - struct ice_tx_ring *ring; - - ice_for_each_tx_ring(ring, q_vector->tx) - if (!ring->tx_buf || !ice_ring_is_xdp(ring)) - break; - - /* restore the value of last node prior to XDP setup */ - q_vector->tx.tx_ring = ring; - } + ice_unmap_xdp_rings(vsi); free_qmap: mutex_lock(&pf->avail_q_mutex); @@ -2956,11 +2971,14 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, xdp_ring_err = ice_vsi_determine_xdp_res(vsi); if (xdp_ring_err) { NL_SET_ERR_MSG_MOD(extack, "Not enough Tx resources for XDP"); + goto resume_if; } else { xdp_ring_err = ice_prepare_xdp_rings(vsi, prog, ICE_XDP_CFG_FULL); - if (xdp_ring_err) + if (xdp_ring_err) { NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); + goto resume_if; + } } xdp_features_set_redirect_target(vsi->netdev, true); /* reallocate Rx queues that are used for zero-copy */ @@ -2978,6 +2996,7 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Rx resources failed"); } +resume_if: if (if_running) ret = ice_up(vsi); @@ -8746,7 +8765,7 @@ static int ice_create_q_channels(struct ice_vsi *vsi) list_add_tail(&ch->list, &vsi->ch_list); vsi->tc_map_vsi[i] = ch->ch_vsi; dev_dbg(ice_pf_to_dev(pf), - "successfully created channel: VSI %pK\n", ch->ch_vsi); + "successfully created channel: VSI %p\n", ch->ch_vsi); } return 0; diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index 908bcd0738033..c2de166f05515 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -84,6 +84,27 @@ ice_sched_find_node_by_teid(struct ice_sched_node *start_node, u32 teid) return NULL; } +/** + * ice_sched_find_next_vsi_node - find the next node for a given VSI + * @vsi_node: VSI support node to start search with + * + * Return: Next VSI support node, or NULL. + * + * The function returns a pointer to the next node from the VSI layer + * assigned to the given VSI, or NULL if there is no such a node. + */ +static struct ice_sched_node * +ice_sched_find_next_vsi_node(struct ice_sched_node *vsi_node) +{ + unsigned int vsi_handle = vsi_node->vsi_handle; + + while ((vsi_node = vsi_node->sibling) != NULL) + if (vsi_node->vsi_handle == vsi_handle) + break; + + return vsi_node; +} + /** * ice_aqc_send_sched_elem_cmd - send scheduling elements cmd * @hw: pointer to the HW struct @@ -1096,8 +1117,10 @@ ice_sched_add_nodes_to_layer(struct ice_port_info *pi, if (parent->num_children < max_child_nodes) { new_num_nodes = max_child_nodes - parent->num_children; } else { - /* This parent is full, try the next sibling */ - parent = parent->sibling; + /* This parent is full, + * try the next available sibling. + */ + parent = ice_sched_find_next_vsi_node(parent); /* Don't modify the first node TEID memory if the * first node was added already in the above call. * Instead send some temp memory for all other @@ -1538,12 +1561,23 @@ ice_sched_get_free_qparent(struct ice_port_info *pi, u16 vsi_handle, u8 tc, /* get the first queue group node from VSI sub-tree */ qgrp_node = ice_sched_get_first_node(pi, vsi_node, qgrp_layer); while (qgrp_node) { + struct ice_sched_node *next_vsi_node; + /* make sure the qgroup node is part of the VSI subtree */ if (ice_sched_find_node_in_subtree(pi->hw, vsi_node, qgrp_node)) if (qgrp_node->num_children < max_children && qgrp_node->owner == owner) break; qgrp_node = qgrp_node->sibling; + if (qgrp_node) + continue; + + next_vsi_node = ice_sched_find_next_vsi_node(vsi_node); + if (!next_vsi_node) + break; + + vsi_node = next_vsi_node; + qgrp_node = ice_sched_get_first_node(pi, vsi_node, qgrp_layer); } /* Select the best queue group */ @@ -1614,16 +1648,16 @@ ice_sched_get_agg_node(struct ice_port_info *pi, struct ice_sched_node *tc_node, /** * ice_sched_calc_vsi_child_nodes - calculate number of VSI child nodes * @hw: pointer to the HW struct - * @num_qs: number of queues + * @num_new_qs: number of new queues that will be added to the tree * @num_nodes: num nodes array * * This function calculates the number of VSI child nodes based on the * number of queues. */ static void -ice_sched_calc_vsi_child_nodes(struct ice_hw *hw, u16 num_qs, u16 *num_nodes) +ice_sched_calc_vsi_child_nodes(struct ice_hw *hw, u16 num_new_qs, u16 *num_nodes) { - u16 num = num_qs; + u16 num = num_new_qs; u8 i, qgl, vsil; qgl = ice_sched_get_qgrp_layer(hw); @@ -1789,7 +1823,11 @@ ice_sched_add_vsi_support_nodes(struct ice_port_info *pi, u16 vsi_handle, if (!parent) return -EIO; - if (i == vsil) + /* Do not modify the VSI handle for already existing VSI nodes, + * (if no new VSI node was added to the tree). + * Assign the VSI handle only to newly added VSI nodes. + */ + if (i == vsil && num_added) parent->vsi_handle = vsi_handle; } @@ -1822,6 +1860,41 @@ ice_sched_add_vsi_to_topo(struct ice_port_info *pi, u16 vsi_handle, u8 tc) num_nodes); } +/** + * ice_sched_recalc_vsi_support_nodes - recalculate VSI support nodes count + * @hw: pointer to the HW struct + * @vsi_node: pointer to the leftmost VSI node that needs to be extended + * @new_numqs: new number of queues that has to be handled by the VSI + * @new_num_nodes: pointer to nodes count table to modify the VSI layer entry + * + * This function recalculates the number of supported nodes that need to + * be added after adding more Tx queues for a given VSI. + * The number of new VSI support nodes that shall be added will be saved + * to the @new_num_nodes table for the VSI layer. + */ +static void +ice_sched_recalc_vsi_support_nodes(struct ice_hw *hw, + struct ice_sched_node *vsi_node, + unsigned int new_numqs, u16 *new_num_nodes) +{ + u32 vsi_nodes_cnt = 1; + u32 max_queue_cnt = 1; + u32 qgl, vsil; + + qgl = ice_sched_get_qgrp_layer(hw); + vsil = ice_sched_get_vsi_layer(hw); + + for (u32 i = vsil; i <= qgl; i++) + max_queue_cnt *= hw->max_children[i]; + + while ((vsi_node = ice_sched_find_next_vsi_node(vsi_node)) != NULL) + vsi_nodes_cnt++; + + if (new_numqs > (max_queue_cnt * vsi_nodes_cnt)) + new_num_nodes[vsil] = DIV_ROUND_UP(new_numqs, max_queue_cnt) - + vsi_nodes_cnt; +} + /** * ice_sched_update_vsi_child_nodes - update VSI child nodes * @pi: port information structure @@ -1873,15 +1946,25 @@ ice_sched_update_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle, return status; } - if (new_numqs) - ice_sched_calc_vsi_child_nodes(hw, new_numqs, new_num_nodes); - /* Keep the max number of queue configuration all the time. Update the - * tree only if number of queues > previous number of queues. This may + ice_sched_recalc_vsi_support_nodes(hw, vsi_node, + new_numqs, new_num_nodes); + ice_sched_calc_vsi_child_nodes(hw, new_numqs - prev_numqs, + new_num_nodes); + + /* Never decrease the number of queues in the tree. Update the tree + * only if number of queues > previous number of queues. This may * leave some extra nodes in the tree if number of queues < previous * number but that wouldn't harm anything. Removing those extra nodes * may complicate the code if those nodes are part of SRL or * individually rate limited. + * Also, add the required VSI support nodes if the existing ones cannot + * handle the requested new number of queues. */ + status = ice_sched_add_vsi_support_nodes(pi, vsi_handle, tc_node, + new_num_nodes); + if (status) + return status; + status = ice_sched_add_vsi_child_nodes(pi, vsi_handle, tc_node, new_num_nodes, owner); if (status) @@ -2022,6 +2105,58 @@ static bool ice_sched_is_leaf_node_present(struct ice_sched_node *node) return (node->info.data.elem_type == ICE_AQC_ELEM_TYPE_LEAF); } +/** + * ice_sched_rm_vsi_subtree - remove all nodes assigned to a given VSI + * @pi: port information structure + * @vsi_node: pointer to the leftmost node of the VSI to be removed + * @owner: LAN or RDMA + * @tc: TC number + * + * Return: Zero in case of success, or -EBUSY if the VSI has leaf nodes in TC. + * + * This function removes all the VSI support nodes associated with a given VSI + * and its LAN or RDMA children nodes from the scheduler tree. + */ +static int +ice_sched_rm_vsi_subtree(struct ice_port_info *pi, + struct ice_sched_node *vsi_node, u8 owner, u8 tc) +{ + u16 vsi_handle = vsi_node->vsi_handle; + bool all_vsi_nodes_removed = true; + int j = 0; + + while (vsi_node) { + struct ice_sched_node *next_vsi_node; + + if (ice_sched_is_leaf_node_present(vsi_node)) { + ice_debug(pi->hw, ICE_DBG_SCHED, "VSI has leaf nodes in TC %d\n", tc); + return -EBUSY; + } + while (j < vsi_node->num_children) { + if (vsi_node->children[j]->owner == owner) + ice_free_sched_node(pi, vsi_node->children[j]); + else + j++; + } + + next_vsi_node = ice_sched_find_next_vsi_node(vsi_node); + + /* remove the VSI if it has no children */ + if (!vsi_node->num_children) + ice_free_sched_node(pi, vsi_node); + else + all_vsi_nodes_removed = false; + + vsi_node = next_vsi_node; + } + + /* clean up aggregator related VSI info if any */ + if (all_vsi_nodes_removed) + ice_sched_rm_agg_vsi_info(pi, vsi_handle); + + return 0; +} + /** * ice_sched_rm_vsi_cfg - remove the VSI and its children nodes * @pi: port information structure @@ -2048,7 +2183,6 @@ ice_sched_rm_vsi_cfg(struct ice_port_info *pi, u16 vsi_handle, u8 owner) ice_for_each_traffic_class(i) { struct ice_sched_node *vsi_node, *tc_node; - u8 j = 0; tc_node = ice_sched_get_tc_node(pi, i); if (!tc_node) @@ -2058,31 +2192,12 @@ ice_sched_rm_vsi_cfg(struct ice_port_info *pi, u16 vsi_handle, u8 owner) if (!vsi_node) continue; - if (ice_sched_is_leaf_node_present(vsi_node)) { - ice_debug(pi->hw, ICE_DBG_SCHED, "VSI has leaf nodes in TC %d\n", i); - status = -EBUSY; + status = ice_sched_rm_vsi_subtree(pi, vsi_node, owner, i); + if (status) goto exit_sched_rm_vsi_cfg; - } - while (j < vsi_node->num_children) { - if (vsi_node->children[j]->owner == owner) { - ice_free_sched_node(pi, vsi_node->children[j]); - /* reset the counter again since the num - * children will be updated after node removal - */ - j = 0; - } else { - j++; - } - } - /* remove the VSI if it has no children */ - if (!vsi_node->num_children) { - ice_free_sched_node(pi, vsi_node); - vsi_ctx->sched.vsi_node[i] = NULL; + vsi_ctx->sched.vsi_node[i] = NULL; - /* clean up aggregator related VSI info if any */ - ice_sched_rm_agg_vsi_info(pi, vsi_handle); - } if (owner == ICE_SCHED_NODE_OWNER_LAN) vsi_ctx->sched.max_lanq[i] = 0; else diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index 19f730a68fa21..ac004ef1d724d 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -3024,7 +3024,7 @@ ice_add_update_vsi_list(struct ice_hw *hw, u16 vsi_handle_arr[2]; /* A rule already exists with the new VSI being added */ - if (cur_fltr->fwd_id.hw_vsi_id == new_fltr->fwd_id.hw_vsi_id) + if (cur_fltr->vsi_handle == new_fltr->vsi_handle) return -EEXIST; vsi_handle_arr[0] = cur_fltr->vsi_handle; @@ -5991,7 +5991,7 @@ ice_adv_add_update_vsi_list(struct ice_hw *hw, /* A rule already exists with the new VSI being added */ if (test_bit(vsi_handle, m_entry->vsi_list_info->vsi_map)) - return 0; + return -EEXIST; /* Update the previously created VSI list set with * the new VSI ID passed in diff --git a/drivers/net/ethernet/intel/ice/ice_trace.h b/drivers/net/ethernet/intel/ice/ice_trace.h index b2f5c9fe01492..a41e210a310d1 100644 --- a/drivers/net/ethernet/intel/ice/ice_trace.h +++ b/drivers/net/ethernet/intel/ice/ice_trace.h @@ -130,7 +130,7 @@ DECLARE_EVENT_CLASS(ice_tx_template, __entry->buf = buf; __assign_str(devname, ring->netdev->name);), - TP_printk("netdev: %s ring: %pK desc: %pK buf %pK", __get_str(devname), + TP_printk("netdev: %s ring: %p desc: %p buf %p", __get_str(devname), __entry->ring, __entry->desc, __entry->buf) ); @@ -158,7 +158,7 @@ DECLARE_EVENT_CLASS(ice_rx_template, __entry->desc = desc; __assign_str(devname, ring->netdev->name);), - TP_printk("netdev: %s ring: %pK desc: %pK", __get_str(devname), + TP_printk("netdev: %s ring: %p desc: %p", __get_str(devname), __entry->ring, __entry->desc) ); DEFINE_EVENT(ice_rx_template, ice_clean_rx_irq, @@ -182,7 +182,7 @@ DECLARE_EVENT_CLASS(ice_rx_indicate_template, __entry->skb = skb; __assign_str(devname, ring->netdev->name);), - TP_printk("netdev: %s ring: %pK desc: %pK skb %pK", __get_str(devname), + TP_printk("netdev: %s ring: %p desc: %p skb %p", __get_str(devname), __entry->ring, __entry->desc, __entry->skb) ); @@ -205,7 +205,7 @@ DECLARE_EVENT_CLASS(ice_xmit_template, __entry->skb = skb; __assign_str(devname, ring->netdev->name);), - TP_printk("netdev: %s skb: %pK ring: %pK", __get_str(devname), + TP_printk("netdev: %s skb: %p ring: %p", __get_str(devname), __entry->skb, __entry->ring) ); @@ -228,7 +228,7 @@ DECLARE_EVENT_CLASS(ice_tx_tstamp_template, TP_fast_assign(__entry->skb = skb; __entry->idx = idx;), - TP_printk("skb %pK idx %d", + TP_printk("skb %p idx %d", __entry->skb, __entry->idx) ); #define DEFINE_TX_TSTAMP_OP_EVENT(name) \ diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index f5023ac9ab832..eae4376c68595 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -527,14 +527,14 @@ int ice_setup_rx_ring(struct ice_rx_ring *rx_ring) * @xdp: xdp_buff used as input to the XDP program * @xdp_prog: XDP program to run * @xdp_ring: ring to be used for XDP_TX action - * @rx_buf: Rx buffer to store the XDP action + * @eop_desc: Last descriptor in packet to read metadata from * * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR} */ -static void +static u32 ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring, - struct ice_rx_buf *rx_buf) + union ice_32b_rx_flex_desc *eop_desc) { unsigned int ret = ICE_XDP_PASS; u32 act; @@ -542,6 +542,8 @@ ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, if (!xdp_prog) goto exit; + ice_xdp_meta_set_desc(xdp, eop_desc); + act = bpf_prog_run_xdp(xdp_prog, xdp); switch (act) { case XDP_PASS: @@ -571,7 +573,7 @@ ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, ret = ICE_XDP_CONSUMED; } exit: - ice_set_rx_bufs_act(xdp, rx_ring, ret); + return ret; } /** @@ -857,10 +859,8 @@ ice_add_xdp_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, xdp_buff_set_frags_flag(xdp); } - if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { - ice_set_rx_bufs_act(xdp, rx_ring, ICE_XDP_CONSUMED); + if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) return -ENOMEM; - } __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buf->page, rx_buf->page_offset, size); @@ -921,7 +921,6 @@ ice_get_rx_buf(struct ice_rx_ring *rx_ring, const unsigned int size, struct ice_rx_buf *rx_buf; rx_buf = &rx_ring->rx_buf[ntc]; - rx_buf->pgcnt = page_count(rx_buf->page); prefetchw(rx_buf->page); if (!size) @@ -937,6 +936,31 @@ ice_get_rx_buf(struct ice_rx_ring *rx_ring, const unsigned int size, return rx_buf; } +/** + * ice_get_pgcnts - grab page_count() for gathered fragments + * @rx_ring: Rx descriptor ring to store the page counts on + * + * This function is intended to be called right before running XDP + * program so that the page recycling mechanism will be able to take + * a correct decision regarding underlying pages; this is done in such + * way as XDP program can change the refcount of page + */ +static void ice_get_pgcnts(struct ice_rx_ring *rx_ring) +{ + u32 nr_frags = rx_ring->nr_frags + 1; + u32 idx = rx_ring->first_desc; + struct ice_rx_buf *rx_buf; + u32 cnt = rx_ring->count; + + for (int i = 0; i < nr_frags; i++) { + rx_buf = &rx_ring->rx_buf[idx]; + rx_buf->pgcnt = page_count(rx_buf->page); + + if (++idx == cnt) + idx = 0; + } +} + /** * ice_build_skb - Build skb around an existing buffer * @rx_ring: Rx descriptor ring to transact packets on @@ -1049,12 +1073,12 @@ ice_construct_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) rx_buf->page_offset + headlen, size, xdp->frame_sz); } else { - /* buffer is unused, change the act that should be taken later - * on; data was copied onto skb's linear part so there's no + /* buffer is unused, restore biased page count in Rx buffer; + * data was copied onto skb's linear part so there's no * need for adjusting page offset and we can reuse this buffer * as-is */ - rx_buf->act = ICE_SKB_CONSUMED; + rx_buf->pagecnt_bias++; } if (unlikely(xdp_buff_has_frags(xdp))) { @@ -1107,29 +1131,34 @@ ice_put_rx_buf(struct ice_rx_ring *rx_ring, struct ice_rx_buf *rx_buf) * @xdp: XDP buffer carrying linear + frags part * @xdp_xmit: XDP_TX/XDP_REDIRECT verdict storage * @ntc: a current next_to_clean value to be stored at rx_ring + * @verdict: return code from XDP program execution * * Walk through gathered fragments and satisfy internal page * recycle mechanism; we take here an action related to verdict * returned by XDP program; */ static void ice_put_rx_mbuf(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, - u32 *xdp_xmit, u32 ntc) + u32 *xdp_xmit, u32 ntc, u32 verdict) { u32 nr_frags = rx_ring->nr_frags + 1; u32 idx = rx_ring->first_desc; u32 cnt = rx_ring->count; + u32 post_xdp_frags = 1; struct ice_rx_buf *buf; int i; - for (i = 0; i < nr_frags; i++) { + if (unlikely(xdp_buff_has_frags(xdp))) + post_xdp_frags += xdp_get_shared_info_from_buff(xdp)->nr_frags; + + for (i = 0; i < post_xdp_frags; i++) { buf = &rx_ring->rx_buf[idx]; - if (buf->act & (ICE_XDP_TX | ICE_XDP_REDIR)) { + if (verdict & (ICE_XDP_TX | ICE_XDP_REDIR)) { ice_rx_buf_adjust_pg_offset(buf, xdp->frame_sz); - *xdp_xmit |= buf->act; - } else if (buf->act & ICE_XDP_CONSUMED) { + *xdp_xmit |= verdict; + } else if (verdict & ICE_XDP_CONSUMED) { buf->pagecnt_bias++; - } else if (buf->act == ICE_XDP_PASS) { + } else if (verdict == ICE_XDP_PASS) { ice_rx_buf_adjust_pg_offset(buf, xdp->frame_sz); } @@ -1138,6 +1167,17 @@ static void ice_put_rx_mbuf(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, if (++idx == cnt) idx = 0; } + /* handle buffers that represented frags released by XDP prog; + * for these we keep pagecnt_bias as-is; refcount from struct page + * has been decremented within XDP prog and we do not have to increase + * the biased refcnt + */ + for (; i < nr_frags; i++) { + buf = &rx_ring->rx_buf[idx]; + ice_put_rx_buf(rx_ring, buf); + if (++idx == cnt) + idx = 0; + } xdp->data = NULL; rx_ring->first_desc = ntc; @@ -1164,9 +1204,9 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) struct ice_tx_ring *xdp_ring = NULL; struct bpf_prog *xdp_prog = NULL; u32 ntc = rx_ring->next_to_clean; + u32 cached_ntu, xdp_verdict; u32 cnt = rx_ring->count; u32 xdp_xmit = 0; - u32 cached_ntu; bool failure; xdp_prog = READ_ONCE(rx_ring->xdp_prog); @@ -1230,7 +1270,7 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) xdp_prepare_buff(xdp, hard_start, offset, size, !!offset); xdp_buff_clear_frags_flag(xdp); } else if (ice_add_xdp_frag(rx_ring, xdp, rx_buf, size)) { - ice_put_rx_mbuf(rx_ring, xdp, NULL, ntc); + ice_put_rx_mbuf(rx_ring, xdp, NULL, ntc, ICE_XDP_CONSUMED); break; } if (++ntc == cnt) @@ -1240,13 +1280,14 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) if (ice_is_non_eop(rx_ring, rx_desc)) continue; - ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_buf); - if (rx_buf->act == ICE_XDP_PASS) + ice_get_pgcnts(rx_ring); + xdp_verdict = ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_desc); + if (xdp_verdict == ICE_XDP_PASS) goto construct_skb; total_rx_bytes += xdp_get_buff_len(xdp); total_rx_pkts++; - ice_put_rx_mbuf(rx_ring, xdp, &xdp_xmit, ntc); + ice_put_rx_mbuf(rx_ring, xdp, &xdp_xmit, ntc, xdp_verdict); continue; construct_skb: @@ -1256,13 +1297,10 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) skb = ice_construct_skb(rx_ring, xdp); /* exit if we failed to retrieve a buffer */ if (!skb) { - rx_ring->ring_stats->rx_stats.alloc_page_failed++; - rx_buf->act = ICE_XDP_CONSUMED; - if (unlikely(xdp_buff_has_frags(xdp))) - ice_set_rx_bufs_act(xdp, rx_ring, - ICE_XDP_CONSUMED); + rx_ring->ring_stats->rx_stats.alloc_buf_failed++; + xdp_verdict = ICE_XDP_CONSUMED; } - ice_put_rx_mbuf(rx_ring, xdp, &xdp_xmit, ntc); + ice_put_rx_mbuf(rx_ring, xdp, &xdp_xmit, ntc, xdp_verdict); if (!skb) break; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index 407d4c320097f..53a155dde3e32 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -201,7 +201,6 @@ struct ice_rx_buf { struct page *page; unsigned int page_offset; unsigned int pgcnt; - unsigned int act; unsigned int pagecnt_bias; }; @@ -257,6 +256,14 @@ enum ice_rx_dtype { ICE_RX_DTYPE_SPLIT_ALWAYS = 2, }; +struct ice_xdp_buff { + struct xdp_buff xdp_buff; + const union ice_32b_rx_flex_desc *eop_desc; +}; + +/* Required for compatibility with xdp_buffs from xsk_pool */ +static_assert(offsetof(struct ice_xdp_buff, xdp_buff) == 0); + /* indices into GLINT_ITR registers */ #define ICE_RX_ITR ICE_IDX_ITR0 #define ICE_TX_ITR ICE_IDX_ITR1 @@ -298,7 +305,6 @@ enum ice_dynamic_itr { /* descriptor ring, associated with a VSI */ struct ice_rx_ring { /* CL1 - 1st cacheline starts here */ - struct ice_rx_ring *next; /* pointer to next ring in q_vector */ void *desc; /* Descriptor ring memory */ struct device *dev; /* Used for DMA mapping */ struct net_device *netdev; /* netdev ring maps to */ @@ -310,12 +316,16 @@ struct ice_rx_ring { u16 count; /* Number of descriptors */ u16 reg_idx; /* HW register index of the ring */ u16 next_to_alloc; - /* CL2 - 2nd cacheline starts here */ + union { struct ice_rx_buf *rx_buf; struct xdp_buff **xdp_buf; }; - struct xdp_buff xdp; + /* CL2 - 2nd cacheline starts here */ + union { + struct ice_xdp_buff xdp_ext; + struct xdp_buff xdp; + }; /* CL3 - 3rd cacheline starts here */ struct bpf_prog *xdp_prog; u16 rx_offset; @@ -332,6 +342,7 @@ struct ice_rx_ring { /* CL4 - 4th cacheline starts here */ struct ice_channel *ch; struct ice_tx_ring *xdp_ring; + struct ice_rx_ring *next; /* pointer to next ring in q_vector */ struct xsk_buff_pool *xsk_pool; u32 nr_frags; dma_addr_t dma; /* physical address of ring */ diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c index c8322fb6f2b37..7e06373e14d98 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c @@ -450,7 +450,7 @@ void ice_finalize_xdp_rx(struct ice_tx_ring *xdp_ring, unsigned int xdp_res, struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[first_idx]; if (xdp_res & ICE_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); if (xdp_res & ICE_XDP_TX) { if (static_branch_unlikely(&ice_xdp_locking_key)) diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h index b0e56675f98b2..41efafc5eb386 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h @@ -5,49 +5,6 @@ #define _ICE_TXRX_LIB_H_ #include "ice.h" -/** - * ice_set_rx_bufs_act - propagate Rx buffer action to frags - * @xdp: XDP buffer representing frame (linear and frags part) - * @rx_ring: Rx ring struct - * act: action to store onto Rx buffers related to XDP buffer parts - * - * Set action that should be taken before putting Rx buffer from first frag - * to the last. - */ -static inline void -ice_set_rx_bufs_act(struct xdp_buff *xdp, const struct ice_rx_ring *rx_ring, - const unsigned int act) -{ - u32 sinfo_frags = xdp_get_shared_info_from_buff(xdp)->nr_frags; - u32 nr_frags = rx_ring->nr_frags + 1; - u32 idx = rx_ring->first_desc; - u32 cnt = rx_ring->count; - struct ice_rx_buf *buf; - - for (int i = 0; i < nr_frags; i++) { - buf = &rx_ring->rx_buf[idx]; - buf->act = act; - - if (++idx == cnt) - idx = 0; - } - - /* adjust pagecnt_bias on frags freed by XDP prog */ - if (sinfo_frags < rx_ring->nr_frags && act == ICE_XDP_CONSUMED) { - u32 delta = rx_ring->nr_frags - sinfo_frags; - - while (delta) { - if (idx == 0) - idx = cnt - 1; - else - idx--; - buf = &rx_ring->rx_buf[idx]; - buf->pagecnt_bias--; - delta--; - } - } -} - /** * ice_test_staterr - tests bits in Rx descriptor status and error fields * @status_err_n: Rx descriptor status_error0 or status_error1 bits @@ -164,4 +121,14 @@ ice_process_skb_fields(struct ice_rx_ring *rx_ring, struct sk_buff *skb, u16 ptype); void ice_receive_skb(struct ice_rx_ring *rx_ring, struct sk_buff *skb, u16 vlan_tag); + +static inline void +ice_xdp_meta_set_desc(struct xdp_buff *xdp, + union ice_32b_rx_flex_desc *eop_desc) +{ + struct ice_xdp_buff *xdp_ext = container_of(xdp, struct ice_xdp_buff, + xdp_buff); + + xdp_ext->eop_desc = eop_desc; +} #endif /* !_ICE_TXRX_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index e709b10a29761..1edcf93031831 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -3769,7 +3769,6 @@ static int ice_vc_repr_add_mac(struct ice_vf *vf, u8 *msg) } ice_vfhw_mac_add(vf, &al->list[i]); - vf->num_mac++; break; } diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c index 3ca5f44dea26e..88c1acd5e8f05 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c @@ -1824,6 +1824,11 @@ int ice_vc_add_fdir_fltr(struct ice_vf *vf, u8 *msg) pf = vf->pf; dev = ice_pf_to_dev(pf); vf_vsi = ice_get_vf_vsi(vf); + if (!vf_vsi) { + dev_err(dev, "Can not get FDIR vf_vsi for VF %u\n", vf->vf_id); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err_exit; + } #define ICE_VF_MAX_FDIR_FILTERS 128 if (!ice_fdir_num_avail_fltr(&pf->hw, vf_vsi) || diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 92b2be06a6e93..f11cba65e5d85 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -2081,11 +2081,8 @@ static void igb_diag_test(struct net_device *netdev, } else { dev_info(&adapter->pdev->dev, "online testing starting\n"); - /* PHY is powered down when interface is down */ - if (if_running && igb_link_test(adapter, &data[TEST_LINK])) + if (igb_link_test(adapter, &data[TEST_LINK])) eth_test->flags |= ETH_TEST_FL_FAILED; - else - data[TEST_LINK] = 0; /* Online tests aren't run; pass by default */ data[TEST_REG] = 0; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index e2f5c4384455e..3e1408e1c1fcf 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -6772,6 +6772,17 @@ static int igc_probe(struct pci_dev *pdev, adapter->port_num = hw->bus.func; adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); + /* PCI config space info */ + hw->vendor_id = pdev->vendor; + hw->device_id = pdev->device; + hw->revision_id = pdev->revision; + hw->subsystem_vendor_id = pdev->subsystem_vendor; + hw->subsystem_device_id = pdev->subsystem_device; + + /* Disable ASPM L1.2 on I226 devices to avoid packet loss */ + if (igc_is_device_id_i226(hw)) + pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); + err = pci_save_state(pdev); if (err) goto err_ioremap; @@ -6793,13 +6804,6 @@ static int igc_probe(struct pci_dev *pdev, netdev->mem_start = pci_resource_start(pdev, 0); netdev->mem_end = pci_resource_end(pdev, 0); - /* PCI config space info */ - hw->vendor_id = pdev->vendor; - hw->device_id = pdev->device; - hw->revision_id = pdev->revision; - hw->subsystem_vendor_id = pdev->subsystem_vendor; - hw->subsystem_device_id = pdev->subsystem_device; - /* Copy the default MAC and PHY function pointers */ memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); @@ -7144,6 +7148,9 @@ static int __maybe_unused igc_resume(struct device *dev) pci_enable_wake(pdev, PCI_D3hot, 0); pci_enable_wake(pdev, PCI_D3cold, 0); + if (igc_is_device_id_i226(hw)) + pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); + if (igc_init_interrupt_scheme(adapter, true)) { netdev_err(netdev, "Unable to allocate memory for queues\n"); return -ENOMEM; @@ -7259,6 +7266,9 @@ static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) pci_enable_wake(pdev, PCI_D3hot, 0); pci_enable_wake(pdev, PCI_D3cold, 0); + if (igc_is_device_id_i226(hw)) + pci_disable_link_state_locked(pdev, PCIE_LINK_STATE_L1_2); + /* In case of PCI error, adapter loses its HW address * so we should re-assign it here. */ diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index b6bb01a486d9d..a82af96e6bd12 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -1237,6 +1237,8 @@ void igc_ptp_reset(struct igc_adapter *adapter) /* reset the tstamp_config */ igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config); + mutex_lock(&adapter->ptm_lock); + spin_lock_irqsave(&adapter->tmreg_lock, flags); switch (adapter->hw.mac.type) { @@ -1255,7 +1257,6 @@ void igc_ptp_reset(struct igc_adapter *adapter) if (!igc_is_crosststamp_supported(adapter)) break; - mutex_lock(&adapter->ptm_lock); wr32(IGC_PCIE_DIG_DELAY, IGC_PCIE_DIG_DELAY_DEFAULT); wr32(IGC_PCIE_PHY_DELAY, IGC_PCIE_PHY_DELAY_DEFAULT); @@ -1279,7 +1280,6 @@ void igc_ptp_reset(struct igc_adapter *adapter) netdev_err(adapter->netdev, "Timeout reading IGC_PTM_STAT register\n"); igc_ptm_reset(hw); - mutex_unlock(&adapter->ptm_lock); break; default: /* No work to do. */ @@ -1296,5 +1296,7 @@ void igc_ptp_reset(struct igc_adapter *adapter) out: spin_unlock_irqrestore(&adapter->tmreg_lock, flags); + mutex_unlock(&adapter->ptm_lock); + wrfl(); } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index b6f0376e42f4b..d15182657cead 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -503,9 +503,10 @@ struct ixgbe_q_vector { struct ixgbe_ring_container rx, tx; struct napi_struct napi; + struct rcu_head rcu; /* to avoid race with update stats on free */ + cpumask_t affinity_mask; int numa_node; - struct rcu_head rcu; /* to avoid race with update stats on free */ char name[IFNAMSIZ + 9]; /* for dynamic allocation of rings associated with this q_vector */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index f245f3df40fca..99876b765b08b 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -2421,7 +2421,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } if (xdp_xmit & IXGBE_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); if (xdp_xmit & IXGBE_XDP_TX) { struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 1703c640a434d..9fdd19acf2242 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -351,7 +351,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, } if (xdp_xmit & IXGBE_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); if (xdp_xmit & IXGBE_XDP_TX) { struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter); @@ -403,7 +403,7 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) dma_addr_t dma; u32 cmd_type; - while (budget-- > 0) { + while (likely(budget)) { if (unlikely(!ixgbe_desc_unused(xdp_ring))) { work_done = false; break; @@ -438,6 +438,8 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) xdp_ring->next_to_use++; if (xdp_ring->next_to_use == xdp_ring->count) xdp_ring->next_to_use = 0; + + budget--; } if (tx_desc) { diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index 5f08779c0e4e3..e177d1d58696a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright(c) 1999 - 2018 Intel Corporation. */ +/* Copyright(c) 1999 - 2024 Intel Corporation. */ #ifndef _IXGBEVF_DEFINES_H_ #define _IXGBEVF_DEFINES_H_ @@ -16,6 +16,9 @@ #define IXGBE_DEV_ID_X550_VF_HV 0x1564 #define IXGBE_DEV_ID_X550EM_X_VF_HV 0x15A9 +#define IXGBE_DEV_ID_E610_VF 0x57AD +#define IXGBE_SUBDEV_ID_E610_VF_HV 0x00FF + #define IXGBE_VF_IRQ_CLEAR_MASK 7 #define IXGBE_VF_MAX_TX_QUEUES 8 #define IXGBE_VF_MAX_RX_QUEUES 8 @@ -25,6 +28,7 @@ /* Link speed */ typedef u32 ixgbe_link_speed; +#define IXGBE_LINK_SPEED_UNKNOWN 0 #define IXGBE_LINK_SPEED_1GB_FULL 0x0020 #define IXGBE_LINK_SPEED_10GB_FULL 0x0080 #define IXGBE_LINK_SPEED_100_FULL 0x0008 diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index f804b35d79c72..83b3243f172c7 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -271,6 +271,9 @@ static int ixgbevf_ipsec_add_sa(struct xfrm_state *xs, adapter = netdev_priv(dev); ipsec = adapter->ipsec; + if (!(adapter->pf_features & IXGBEVF_PF_SUP_IPSEC)) + return -EOPNOTSUPP; + if (xs->id.proto != IPPROTO_ESP && xs->id.proto != IPPROTO_AH) { NL_SET_ERR_MSG_MOD(extack, "Unsupported protocol for IPsec offload"); return -EINVAL; @@ -400,6 +403,9 @@ static void ixgbevf_ipsec_del_sa(struct xfrm_state *xs) adapter = netdev_priv(dev); ipsec = adapter->ipsec; + if (!(adapter->pf_features & IXGBEVF_PF_SUP_IPSEC)) + return; + if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { sa_idx = xs->xso.offload_handle - IXGBE_IPSEC_BASE_RX_INDEX; @@ -628,6 +634,10 @@ void ixgbevf_init_ipsec_offload(struct ixgbevf_adapter *adapter) size_t size; switch (adapter->hw.api_version) { + case ixgbe_mbox_api_17: + if (!(adapter->pf_features & IXGBEVF_PF_SUP_IPSEC)) + return; + break; case ixgbe_mbox_api_14: break; default: diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 130cb868774c4..f31068e24e867 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright(c) 1999 - 2018 Intel Corporation. */ +/* Copyright(c) 1999 - 2024 Intel Corporation. */ #ifndef _IXGBEVF_H_ #define _IXGBEVF_H_ @@ -366,6 +366,13 @@ struct ixgbevf_adapter { /* Interrupt Throttle Rate */ u32 eitr_param; + u32 pf_features; +#define IXGBEVF_PF_SUP_IPSEC BIT(0) +#define IXGBEVF_PF_SUP_ESX_MBX BIT(1) + +#define IXGBEVF_SUPPORTED_FEATURES (IXGBEVF_PF_SUP_IPSEC | \ + IXGBEVF_PF_SUP_ESX_MBX) + struct ixgbevf_hw_stats stats; unsigned long state; @@ -418,6 +425,8 @@ enum ixgbevf_boards { board_X550EM_x_vf, board_X550EM_x_vf_hv, board_x550em_a_vf, + board_e610_vf, + board_e610_vf_hv, }; enum ixgbevf_xcast_modes { @@ -434,11 +443,13 @@ extern const struct ixgbevf_info ixgbevf_X550EM_x_vf_info; extern const struct ixgbe_mbx_operations ixgbevf_mbx_ops; extern const struct ixgbe_mbx_operations ixgbevf_mbx_ops_legacy; extern const struct ixgbevf_info ixgbevf_x550em_a_vf_info; +extern const struct ixgbevf_info ixgbevf_e610_vf_info; extern const struct ixgbevf_info ixgbevf_82599_vf_hv_info; extern const struct ixgbevf_info ixgbevf_X540_vf_hv_info; extern const struct ixgbevf_info ixgbevf_X550_vf_hv_info; extern const struct ixgbevf_info ixgbevf_X550EM_x_vf_hv_info; +extern const struct ixgbevf_info ixgbevf_e610_vf_hv_info; extern const struct ixgbe_mbx_operations ixgbevf_hv_mbx_ops; /* needed by ethtool.c */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index a44e4bd561421..72b17a0f052c2 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 1999 - 2018 Intel Corporation. */ +/* Copyright(c) 1999 - 2024 Intel Corporation. */ /****************************************************************************** Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code @@ -39,7 +39,7 @@ static const char ixgbevf_driver_string[] = "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; static char ixgbevf_copyright[] = - "Copyright (c) 2009 - 2018 Intel Corporation."; + "Copyright (c) 2009 - 2024 Intel Corporation."; static const struct ixgbevf_info *ixgbevf_info_tbl[] = { [board_82599_vf] = &ixgbevf_82599_vf_info, @@ -51,6 +51,8 @@ static const struct ixgbevf_info *ixgbevf_info_tbl[] = { [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, [board_X550EM_x_vf_hv] = &ixgbevf_X550EM_x_vf_hv_info, [board_x550em_a_vf] = &ixgbevf_x550em_a_vf_info, + [board_e610_vf] = &ixgbevf_e610_vf_info, + [board_e610_vf_hv] = &ixgbevf_e610_vf_hv_info, }; /* ixgbevf_pci_tbl - PCI Device ID Table @@ -71,6 +73,9 @@ static const struct pci_device_id ixgbevf_pci_tbl[] = { {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv}, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf }, + {PCI_VDEVICE_SUB(INTEL, IXGBE_DEV_ID_E610_VF, PCI_ANY_ID, + IXGBE_SUBDEV_ID_E610_VF_HV), board_e610_vf_hv}, + {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_VF), board_e610_vf}, /* required last entry */ {0, } }; @@ -2270,10 +2275,36 @@ static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; } +/** + * ixgbevf_set_features - Set features supported by PF + * @adapter: pointer to the adapter struct + * + * Negotiate with PF supported features and then set pf_features accordingly. + */ +static void ixgbevf_set_features(struct ixgbevf_adapter *adapter) +{ + u32 *pf_features = &adapter->pf_features; + struct ixgbe_hw *hw = &adapter->hw; + int err; + + err = hw->mac.ops.negotiate_features(hw, pf_features); + if (err && err != -EOPNOTSUPP) + netdev_dbg(adapter->netdev, + "PF feature negotiation failed.\n"); + + /* Address also pre API 1.7 cases */ + if (hw->api_version == ixgbe_mbox_api_14) + *pf_features |= IXGBEVF_PF_SUP_IPSEC; + else if (hw->api_version == ixgbe_mbox_api_15) + *pf_features |= IXGBEVF_PF_SUP_ESX_MBX; +} + static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; static const int api[] = { + ixgbe_mbox_api_17, + ixgbe_mbox_api_16, ixgbe_mbox_api_15, ixgbe_mbox_api_14, ixgbe_mbox_api_13, @@ -2293,7 +2324,9 @@ static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) idx++; } - if (hw->api_version >= ixgbe_mbox_api_15) { + ixgbevf_set_features(adapter); + + if (adapter->pf_features & IXGBEVF_PF_SUP_ESX_MBX) { hw->mbx.ops.init_params(hw); memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, sizeof(struct ixgbe_mbx_operations)); @@ -2650,6 +2683,8 @@ static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) case ixgbe_mbox_api_13: case ixgbe_mbox_api_14: case ixgbe_mbox_api_15: + case ixgbe_mbox_api_16: + case ixgbe_mbox_api_17: if (adapter->xdp_prog && hw->mac.max_tx_queues == rss) rss = rss > 3 ? 2 : 1; @@ -4644,6 +4679,8 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) case ixgbe_mbox_api_13: case ixgbe_mbox_api_14: case ixgbe_mbox_api_15: + case ixgbe_mbox_api_16: + case ixgbe_mbox_api_17: netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); break; @@ -4694,6 +4731,9 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) case ixgbe_mac_X540_vf: dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); break; + case ixgbe_mac_e610_vf: + dev_info(&pdev->dev, "Intel(R) E610 Virtual Function\n"); + break; case ixgbe_mac_82599_vf: default: dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); diff --git a/drivers/net/ethernet/intel/ixgbevf/mbx.h b/drivers/net/ethernet/intel/ixgbevf/mbx.h index 835bbcc5cc8e6..a8ed23ee66aa8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/mbx.h +++ b/drivers/net/ethernet/intel/ixgbevf/mbx.h @@ -66,6 +66,8 @@ enum ixgbe_pfvf_api_rev { ixgbe_mbox_api_13, /* API version 1.3, linux/freebsd VF driver */ ixgbe_mbox_api_14, /* API version 1.4, linux/freebsd VF driver */ ixgbe_mbox_api_15, /* API version 1.5, linux/freebsd VF driver */ + ixgbe_mbox_api_16, /* API version 1.6, linux/freebsd VF driver */ + ixgbe_mbox_api_17, /* API version 1.7, linux/freebsd VF driver */ /* This value should always be last */ ixgbe_mbox_api_unknown, /* indicates that API version is not known */ }; @@ -102,6 +104,12 @@ enum ixgbe_pfvf_api_rev { #define IXGBE_VF_GET_LINK_STATE 0x10 /* get vf link state */ +/* mailbox API, version 1.6 VF requests */ +#define IXGBE_VF_GET_PF_LINK_STATE 0x11 /* request PF to send link info */ + +/* mailbox API, version 1.7 VF requests */ +#define IXGBE_VF_FEATURES_NEGOTIATE 0x12 /* get features supported by PF*/ + /* length of permanent address message returned from PF */ #define IXGBE_VF_PERMADDR_MSG_LEN 4 /* word in permanent address message with the current multicast type */ diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.c b/drivers/net/ethernet/intel/ixgbevf/vf.c index 1641d00d8ed35..65257107dfc8a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/vf.c +++ b/drivers/net/ethernet/intel/ixgbevf/vf.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 1999 - 2018 Intel Corporation. */ +/* Copyright(c) 1999 - 2024 Intel Corporation. */ #include "vf.h" #include "ixgbevf.h" @@ -313,6 +313,8 @@ int ixgbevf_get_reta_locked(struct ixgbe_hw *hw, u32 *reta, int num_rx_queues) * is not supported for this device type. */ switch (hw->api_version) { + case ixgbe_mbox_api_17: + case ixgbe_mbox_api_16: case ixgbe_mbox_api_15: case ixgbe_mbox_api_14: case ixgbe_mbox_api_13: @@ -382,6 +384,8 @@ int ixgbevf_get_rss_key_locked(struct ixgbe_hw *hw, u8 *rss_key) * or if the operation is not supported for this device type. */ switch (hw->api_version) { + case ixgbe_mbox_api_17: + case ixgbe_mbox_api_16: case ixgbe_mbox_api_15: case ixgbe_mbox_api_14: case ixgbe_mbox_api_13: @@ -552,6 +556,8 @@ static s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode) case ixgbe_mbox_api_13: case ixgbe_mbox_api_14: case ixgbe_mbox_api_15: + case ixgbe_mbox_api_16: + case ixgbe_mbox_api_17: break; default: return -EOPNOTSUPP; @@ -624,6 +630,85 @@ static s32 ixgbevf_hv_get_link_state_vf(struct ixgbe_hw *hw, bool *link_state) return -EOPNOTSUPP; } +/** + * ixgbevf_get_pf_link_state - Get PF's link status + * @hw: pointer to the HW structure + * @speed: link speed + * @link_up: indicate if link is up/down + * + * Ask PF to provide link_up state and speed of the link. + * + * Return: IXGBE_ERR_MBX in the case of mailbox error, + * -EOPNOTSUPP if the op is not supported or 0 on success. + */ +static int ixgbevf_get_pf_link_state(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + bool *link_up) +{ + u32 msgbuf[3] = {}; + int err; + + switch (hw->api_version) { + case ixgbe_mbox_api_16: + case ixgbe_mbox_api_17: + break; + default: + return -EOPNOTSUPP; + } + + msgbuf[0] = IXGBE_VF_GET_PF_LINK_STATE; + + err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, + ARRAY_SIZE(msgbuf)); + if (err || (msgbuf[0] & IXGBE_VT_MSGTYPE_FAILURE)) { + err = IXGBE_ERR_MBX; + *speed = IXGBE_LINK_SPEED_UNKNOWN; + /* No need to set @link_up to false as it will be done by + * ixgbe_check_mac_link_vf(). + */ + } else { + *speed = msgbuf[1]; + *link_up = msgbuf[2]; + } + + return err; +} + +/** + * ixgbevf_negotiate_features_vf - negotiate supported features with PF driver + * @hw: pointer to the HW structure + * @pf_features: bitmask of features supported by PF + * + * Return: IXGBE_ERR_MBX in the case of mailbox error, + * -EOPNOTSUPP if the op is not supported or 0 on success. + */ +static int ixgbevf_negotiate_features_vf(struct ixgbe_hw *hw, u32 *pf_features) +{ + u32 msgbuf[2] = {}; + int err; + + switch (hw->api_version) { + case ixgbe_mbox_api_17: + break; + default: + return -EOPNOTSUPP; + } + + msgbuf[0] = IXGBE_VF_FEATURES_NEGOTIATE; + msgbuf[1] = IXGBEVF_SUPPORTED_FEATURES; + + err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, + ARRAY_SIZE(msgbuf)); + + if (err || (msgbuf[0] & IXGBE_VT_MSGTYPE_FAILURE)) { + err = IXGBE_ERR_MBX; + *pf_features = 0x0; + } else { + *pf_features = msgbuf[1]; + } + + return err; +} + /** * ixgbevf_set_vfta_vf - Set/Unset VLAN filter table address * @hw: pointer to the HW structure @@ -658,6 +743,58 @@ static s32 ixgbevf_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, return err; } +/** + * ixgbe_read_vflinks - Read VFLINKS register + * @hw: pointer to the HW structure + * @speed: link speed + * @link_up: indicate if link is up/down + * + * Get linkup status and link speed from the VFLINKS register. + */ +static void ixgbe_read_vflinks(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + bool *link_up) +{ + u32 vflinks = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + + /* if link status is down no point in checking to see if PF is up */ + if (!(vflinks & IXGBE_LINKS_UP)) { + *link_up = false; + return; + } + + /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs + * before the link status is correct + */ + if (hw->mac.type == ixgbe_mac_82599_vf) { + for (int i = 0; i < 5; i++) { + udelay(100); + vflinks = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + + if (!(vflinks & IXGBE_LINKS_UP)) { + *link_up = false; + return; + } + } + } + + /* We reached this point so there's link */ + *link_up = true; + + switch (vflinks & IXGBE_LINKS_SPEED_82599) { + case IXGBE_LINKS_SPEED_10G_82599: + *speed = IXGBE_LINK_SPEED_10GB_FULL; + break; + case IXGBE_LINKS_SPEED_1G_82599: + *speed = IXGBE_LINK_SPEED_1GB_FULL; + break; + case IXGBE_LINKS_SPEED_100_82599: + *speed = IXGBE_LINK_SPEED_100_FULL; + break; + default: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + } +} + /** * ixgbevf_hv_set_vfta_vf - * Hyper-V variant - just a stub. * @hw: unused @@ -702,10 +839,10 @@ static s32 ixgbevf_check_mac_link_vf(struct ixgbe_hw *hw, bool *link_up, bool autoneg_wait_to_complete) { + struct ixgbevf_adapter *adapter = hw->back; struct ixgbe_mbx_info *mbx = &hw->mbx; struct ixgbe_mac_info *mac = &hw->mac; s32 ret_val = 0; - u32 links_reg; u32 in_msg = 0; /* If we were hit with a reset drop the link */ @@ -715,43 +852,21 @@ static s32 ixgbevf_check_mac_link_vf(struct ixgbe_hw *hw, if (!mac->get_link_status) goto out; - /* if link status is down no point in checking to see if pf is up */ - links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); - if (!(links_reg & IXGBE_LINKS_UP)) - goto out; - - /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs - * before the link status is correct - */ - if (mac->type == ixgbe_mac_82599_vf) { - int i; - - for (i = 0; i < 5; i++) { - udelay(100); - links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); - - if (!(links_reg & IXGBE_LINKS_UP)) - goto out; - } - } - - switch (links_reg & IXGBE_LINKS_SPEED_82599) { - case IXGBE_LINKS_SPEED_10G_82599: - *speed = IXGBE_LINK_SPEED_10GB_FULL; - break; - case IXGBE_LINKS_SPEED_1G_82599: - *speed = IXGBE_LINK_SPEED_1GB_FULL; - break; - case IXGBE_LINKS_SPEED_100_82599: - *speed = IXGBE_LINK_SPEED_100_FULL; - break; + if (hw->mac.type == ixgbe_mac_e610_vf) { + ret_val = ixgbevf_get_pf_link_state(hw, speed, link_up); + if (ret_val) + goto out; + } else { + ixgbe_read_vflinks(hw, speed, link_up); + if (*link_up == false) + goto out; } /* if the read failed it could just be a mailbox collision, best wait * until we are called again and don't report an error */ if (mbx->ops.read(hw, &in_msg, 1)) { - if (hw->api_version >= ixgbe_mbox_api_15) + if (adapter->pf_features & IXGBEVF_PF_SUP_ESX_MBX) mac->get_link_status = false; goto out; } @@ -951,6 +1066,8 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, case ixgbe_mbox_api_13: case ixgbe_mbox_api_14: case ixgbe_mbox_api_15: + case ixgbe_mbox_api_16: + case ixgbe_mbox_api_17: break; default: return 0; @@ -1005,6 +1122,7 @@ static const struct ixgbe_mac_operations ixgbevf_mac_ops = { .setup_link = ixgbevf_setup_mac_link_vf, .check_link = ixgbevf_check_mac_link_vf, .negotiate_api_version = ixgbevf_negotiate_api_version_vf, + .negotiate_features = ixgbevf_negotiate_features_vf, .set_rar = ixgbevf_set_rar_vf, .update_mc_addr_list = ixgbevf_update_mc_addr_list_vf, .update_xcast_mode = ixgbevf_update_xcast_mode, @@ -1076,3 +1194,13 @@ const struct ixgbevf_info ixgbevf_x550em_a_vf_info = { .mac = ixgbe_mac_x550em_a_vf, .mac_ops = &ixgbevf_mac_ops, }; + +const struct ixgbevf_info ixgbevf_e610_vf_info = { + .mac = ixgbe_mac_e610_vf, + .mac_ops = &ixgbevf_mac_ops, +}; + +const struct ixgbevf_info ixgbevf_e610_vf_hv_info = { + .mac = ixgbe_mac_e610_vf, + .mac_ops = &ixgbevf_hv_mac_ops, +}; diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.h b/drivers/net/ethernet/intel/ixgbevf/vf.h index b4eef5b6c172b..4f19b8900c29a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/vf.h +++ b/drivers/net/ethernet/intel/ixgbevf/vf.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright(c) 1999 - 2018 Intel Corporation. */ +/* Copyright(c) 1999 - 2024 Intel Corporation. */ #ifndef __IXGBE_VF_H__ #define __IXGBE_VF_H__ @@ -26,6 +26,7 @@ struct ixgbe_mac_operations { s32 (*stop_adapter)(struct ixgbe_hw *); s32 (*get_bus_info)(struct ixgbe_hw *); s32 (*negotiate_api_version)(struct ixgbe_hw *hw, int api); + int (*negotiate_features)(struct ixgbe_hw *hw, u32 *pf_features); /* Link */ s32 (*setup_link)(struct ixgbe_hw *, ixgbe_link_speed, bool, bool); @@ -54,6 +55,8 @@ enum ixgbe_mac_type { ixgbe_mac_X550_vf, ixgbe_mac_X550EM_x_vf, ixgbe_mac_x550em_a_vf, + ixgbe_mac_e610, + ixgbe_mac_e610_vf, ixgbe_num_macs }; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 165f76d1231c1..2941721b65152 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -2520,7 +2520,7 @@ static int mvneta_rx_swbm(struct napi_struct *napi, mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1); if (ps.xdp_redirect) - xdp_do_flush_map(); + xdp_do_flush(); if (ps.rx_packets) mvneta_update_stats(pp, &ps); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index fce57faf345ce..aabc39f7690f8 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -4055,7 +4055,7 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, } if (xdp_ret & MVPP2_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); if (ps.rx_packets) { struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c index 6f1fe7e283d4e..7a30095b3486f 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c @@ -917,7 +917,7 @@ static void octep_hb_timeout_task(struct work_struct *work) miss_cnt); rtnl_lock(); if (netif_running(oct->netdev)) - octep_stop(oct->netdev); + dev_close(oct->netdev); rtnl_unlock(); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index 52792546fe00d..729d1833a829a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -21,8 +21,7 @@ #include "rvu.h" #include "lmac_common.h" -#define DRV_NAME "Marvell-CGX/RPM" -#define DRV_STRING "Marvell CGX/RPM Driver" +#define DRV_NAME "Marvell-CGX-RPM" #define CGX_RX_STAT_GLOBAL_INDEX 9 @@ -66,8 +65,18 @@ static int cgx_fwi_link_change(struct cgx *cgx, int lmac_id, bool en); /* Supported devices */ static const struct pci_device_id cgx_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_CGX) }, - { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RPM) }, - { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10KB_RPM) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RPM, + PCI_ANY_ID, PCI_SUBSYS_DEVID_CN10K_A) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RPM, + PCI_ANY_ID, PCI_SUBSYS_DEVID_CNF10K_A) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RPM, + PCI_ANY_ID, PCI_SUBSYS_DEVID_CNF10K_B) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10KB_RPM, + PCI_ANY_ID, PCI_SUBSYS_DEVID_CN10K_B) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10KB_RPM, + PCI_ANY_ID, PCI_SUBSYS_DEVID_CN20KA) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10KB_RPM, + PCI_ANY_ID, PCI_SUBSYS_DEVID_CNF20KA) }, { 0, } /* end of table */ }; @@ -707,6 +716,11 @@ int cgx_get_rx_stats(void *cgxd, int lmac_id, int idx, u64 *rx_stat) if (!is_lmac_valid(cgx, lmac_id)) return -ENODEV; + + /* pass lmac as 0 for CGX_CMR_RX_STAT9-12 */ + if (idx >= CGX_RX_STAT_GLOBAL_INDEX) + lmac_id = 0; + *rx_stat = cgx_read(cgx, lmac_id, CGXX_CMRX_RX_STAT0 + (idx * 8)); return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index a607c7294b0c5..9fbc071ef29b0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -30,6 +30,8 @@ #define PCI_SUBSYS_DEVID_CNF10K_A 0xBA00 #define PCI_SUBSYS_DEVID_CNF10K_B 0xBC00 #define PCI_SUBSYS_DEVID_CN10K_B 0xBD00 +#define PCI_SUBSYS_DEVID_CN20KA 0xC220 +#define PCI_SUBSYS_DEVID_CNF20KA 0xC320 /* PCI BAR nos */ #define PCI_AF_REG_BAR_NUM 0 diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c index 0e74c5a2231e6..1e4cd4f7d0cfd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c @@ -13,19 +13,26 @@ /* RVU LMTST */ #define LMT_TBL_OP_READ 0 #define LMT_TBL_OP_WRITE 1 -#define LMT_MAP_TABLE_SIZE (128 * 1024) #define LMT_MAPTBL_ENTRY_SIZE 16 +#define LMT_MAX_VFS 256 + +#define LMT_MAP_ENTRY_ENA BIT_ULL(20) +#define LMT_MAP_ENTRY_LINES GENMASK_ULL(18, 16) /* Function to perform operations (read/write) on lmtst map table */ static int lmtst_map_table_ops(struct rvu *rvu, u32 index, u64 *val, int lmt_tbl_op) { void __iomem *lmt_map_base; - u64 tbl_base; + u64 tbl_base, cfg; + int pfs, vfs; tbl_base = rvu_read64(rvu, BLKADDR_APR, APR_AF_LMT_MAP_BASE); + cfg = rvu_read64(rvu, BLKADDR_APR, APR_AF_LMT_CFG); + vfs = 1 << (cfg & 0xF); + pfs = 1 << ((cfg >> 4) & 0x7); - lmt_map_base = ioremap_wc(tbl_base, LMT_MAP_TABLE_SIZE); + lmt_map_base = ioremap_wc(tbl_base, pfs * vfs * LMT_MAPTBL_ENTRY_SIZE); if (!lmt_map_base) { dev_err(rvu->dev, "Failed to setup lmt map table mapping!!\n"); return -ENOMEM; @@ -35,6 +42,13 @@ static int lmtst_map_table_ops(struct rvu *rvu, u32 index, u64 *val, *val = readq(lmt_map_base + index); } else { writeq((*val), (lmt_map_base + index)); + + cfg = FIELD_PREP(LMT_MAP_ENTRY_ENA, 0x1); + /* 2048 LMTLINES */ + cfg |= FIELD_PREP(LMT_MAP_ENTRY_LINES, 0x6); + + writeq(cfg, (lmt_map_base + (index + 8))); + /* Flushing the AP interceptor cache to make APR_LMT_MAP_ENTRY_S * changes effective. Write 1 for flush and read is being used as a * barrier and sets up a data dependency. Write to 0 after a write @@ -52,7 +66,7 @@ static int lmtst_map_table_ops(struct rvu *rvu, u32 index, u64 *val, #define LMT_MAP_TBL_W1_OFF 8 static u32 rvu_get_lmtst_tbl_index(struct rvu *rvu, u16 pcifunc) { - return ((rvu_get_pf(pcifunc) * rvu->hw->total_vfs) + + return ((rvu_get_pf(pcifunc) * LMT_MAX_VFS) + (pcifunc & RVU_PFVF_FUNC_MASK)) * LMT_MAPTBL_ENTRY_SIZE; } @@ -69,7 +83,7 @@ static int rvu_get_lmtaddr(struct rvu *rvu, u16 pcifunc, mutex_lock(&rvu->rsrc_lock); rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_SMMU_ADDR_REQ, iova); - pf = rvu_get_pf(pcifunc) & 0x1F; + pf = rvu_get_pf(pcifunc) & RVU_PFVF_PF_MASK; val = BIT_ULL(63) | BIT_ULL(14) | BIT_ULL(13) | pf << 8 | ((pcifunc & RVU_PFVF_FUNC_MASK) & 0xFF); rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_SMMU_TXN_REQ, val); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index feca86e429df2..56dab11833b53 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -580,6 +580,7 @@ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, u64 lmt_addr, val, tbl_base; int pf, vf, num_vfs, hw_vfs; void __iomem *lmt_map_base; + int apr_pfs, apr_vfs; int buf_size = 10240; size_t off = 0; int index = 0; @@ -595,8 +596,12 @@ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, return -ENOMEM; tbl_base = rvu_read64(rvu, BLKADDR_APR, APR_AF_LMT_MAP_BASE); + val = rvu_read64(rvu, BLKADDR_APR, APR_AF_LMT_CFG); + apr_vfs = 1 << (val & 0xF); + apr_pfs = 1 << ((val >> 4) & 0x7); - lmt_map_base = ioremap_wc(tbl_base, 128 * 1024); + lmt_map_base = ioremap_wc(tbl_base, apr_pfs * apr_vfs * + LMT_MAPTBL_ENTRY_SIZE); if (!lmt_map_base) { dev_err(rvu->dev, "Failed to setup lmt map table mapping!!\n"); kfree(buf); @@ -618,7 +623,7 @@ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, off += scnprintf(&buf[off], buf_size - 1 - off, "PF%d \t\t\t", pf); - index = pf * rvu->hw->total_vfs * LMT_MAPTBL_ENTRY_SIZE; + index = pf * apr_vfs * LMT_MAPTBL_ENTRY_SIZE; off += scnprintf(&buf[off], buf_size - 1 - off, " 0x%llx\t\t", (tbl_base + index)); lmt_addr = readq(lmt_map_base + index); @@ -631,7 +636,7 @@ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, /* Reading num of VFs per PF */ rvu_get_pf_numvfs(rvu, pf, &num_vfs, &hw_vfs); for (vf = 0; vf < num_vfs; vf++) { - index = (pf * rvu->hw->total_vfs * 16) + + index = (pf * apr_vfs * LMT_MAPTBL_ENTRY_SIZE) + ((vf + 1) * LMT_MAPTBL_ENTRY_SIZE); off += scnprintf(&buf[off], buf_size - 1 - off, "PF%d:VF%d \t\t", pf, vf); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c index 237f82082ebe5..0f4e462d39c2e 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c @@ -580,8 +580,8 @@ static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf) if (!npc_check_field(rvu, blkaddr, NPC_LB, intf)) *features &= ~BIT_ULL(NPC_OUTER_VID); - /* Set SPI flag only if AH/ESP and IPSEC_SPI are in the key */ - if (npc_check_field(rvu, blkaddr, NPC_IPSEC_SPI, intf) && + /* Allow extracting SPI field from AH and ESP headers at same offset */ + if (npc_is_field_present(rvu, NPC_IPSEC_SPI, intf) && (*features & (BIT_ULL(NPC_IPPROTO_ESP) | BIT_ULL(NPC_IPPROTO_AH)))) *features |= BIT_ULL(NPC_IPSEC_SPI); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c index 7417087b6db59..a2807a1e4f4a6 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c @@ -352,9 +352,12 @@ int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) mutex_lock(&pfvf->mbox.lock); /* Remove RQ's policer mapping */ - for (qidx = 0; qidx < hw->rx_queues; qidx++) - cn10k_map_unmap_rq_policer(pfvf, qidx, - hw->matchall_ipolicer, false); + for (qidx = 0; qidx < hw->rx_queues; qidx++) { + rc = cn10k_map_unmap_rq_policer(pfvf, qidx, hw->matchall_ipolicer, false); + if (rc) + dev_warn(pfvf->dev, "Failed to unmap RQ %d's policer (error %d).", + qidx, rc); + } rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c index 6cc7a78968fc1..74953f67a2bf9 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c @@ -533,7 +533,8 @@ static int cn10k_mcs_write_tx_secy(struct otx2_nic *pfvf, if (sw_tx_sc->encrypt) sectag_tci |= (MCS_TCI_E | MCS_TCI_C); - policy = FIELD_PREP(MCS_TX_SECY_PLCY_MTU, secy->netdev->mtu); + policy = FIELD_PREP(MCS_TX_SECY_PLCY_MTU, + pfvf->netdev->mtu + OTX2_ETH_HLEN); /* Write SecTag excluding AN bits(1..0) */ policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_TCI, sectag_tci >> 2); policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_OFFSET, tag_offset); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 47adccf7a7776..1999918ca500f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -988,6 +988,7 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) int err, pool_id, non_xdp_queues; struct nix_aq_enq_req *aq; struct otx2_cq_queue *cq; + struct otx2_pool *pool; cq = &qset->cq[qidx]; cq->cq_idx = qidx; @@ -996,8 +997,13 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) cq->cq_type = CQ_RX; cq->cint_idx = qidx; cq->cqe_cnt = qset->rqe_cnt; - if (pfvf->xdp_prog) + if (pfvf->xdp_prog) { + pool = &qset->pool[qidx]; xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0); + xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, + MEM_TYPE_PAGE_POOL, + pool->page_pool); + } } else if (qidx < non_xdp_queues) { cq->cq_type = CQ_TX; cq->cint_idx = qidx - pfvf->hw.rx_queues; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c index 3a72b0793d4a7..82725923555c5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c @@ -476,7 +476,7 @@ void otx2_ptp_destroy(struct otx2_nic *pfvf) if (!ptp) return; - cancel_delayed_work(&pfvf->ptp->synctstamp_work); + cancel_delayed_work_sync(&pfvf->ptp->synctstamp_work); ptp_clock_unregister(ptp->ptp_clock); kfree(ptp); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 46bdbee9d38ad..635cfb9a3e2c5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -1116,7 +1116,6 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, free_leaf: otx2_tc_del_from_flow_list(flow_cfg, new_node); - kfree_rcu(new_node, rcu); if (new_node->is_act_police) { mutex_lock(&nic->mbox.lock); @@ -1136,6 +1135,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, mutex_unlock(&nic->mbox.lock); } + kfree_rcu(new_node, rcu); return rc; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c index 37db19584c143..92861f102590f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c @@ -1560,6 +1560,7 @@ static int otx2_qos_leaf_del_last(struct otx2_nic *pfvf, u16 classid, bool force if (!node->is_static) dwrr_del_node = true; + WRITE_ONCE(node->qid, OTX2_QOS_QID_INNER); /* destroy the leaf node */ otx2_qos_disable_sq(pfvf, qid); otx2_qos_destroy_node(pfvf, node); @@ -1604,9 +1605,6 @@ static int otx2_qos_leaf_del_last(struct otx2_nic *pfvf, u16 classid, bool force } kfree(new_cfg); - /* update tx_real_queues */ - otx2_qos_update_tx_netdev_queues(pfvf); - return 0; } diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index dc89dbc13b251..c843e6531449b 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -1586,6 +1586,13 @@ static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev) bool gso = false; int tx_num; + if (skb_vlan_tag_present(skb) && + !eth_proto_is_802_3(eth_hdr(skb)->h_proto)) { + skb = __vlan_hwaccel_push_inside(skb); + if (!skb) + goto dropped; + } + /* normally we can rely on the stack not calling this more than once, * however we have 2 queues running on the same ring so we need to lock * the ring access @@ -1631,8 +1638,9 @@ static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev) drop: spin_unlock(ð->page_lock); - stats->tx_dropped++; dev_kfree_skb_any(skb); +dropped: + stats->tx_dropped++; return NETDEV_TX_OK; } @@ -2180,14 +2188,18 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, ring->data[idx] = new_data; rxd->rxd1 = (unsigned int)dma_addr; release_desc: + if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA)) { + if (unlikely(dma_addr == DMA_MAPPING_ERROR)) + addr64 = FIELD_GET(RX_DMA_ADDR64_MASK, + rxd->rxd2); + else + addr64 = RX_DMA_PREP_ADDR64(dma_addr); + } + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) rxd->rxd2 = RX_DMA_LSO; else - rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size); - - if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA) && - likely(dma_addr != DMA_MAPPING_ERROR)) - rxd->rxd2 |= RX_DMA_PREP_ADDR64(dma_addr); + rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size) | addr64; ring->calc_idx = idx; done++; @@ -2209,7 +2221,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, net_dim(ð->rx_dim, dim_sample); if (xdp_flush) - xdp_do_flush_map(); + xdp_do_flush(); return done; } @@ -3113,11 +3125,19 @@ static int mtk_dma_init(struct mtk_eth *eth) static void mtk_dma_free(struct mtk_eth *eth) { const struct mtk_soc_data *soc = eth->soc; - int i; + int i, j, txqs = 1; + + if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) + txqs = MTK_QDMA_NUM_QUEUES; + + for (i = 0; i < MTK_MAX_DEVS; i++) { + if (!eth->netdev[i]) + continue; + + for (j = 0; j < txqs; j++) + netdev_tx_reset_subqueue(eth->netdev[i], j); + } - for (i = 0; i < MTK_MAX_DEVS; i++) - if (eth->netdev[i]) - netdev_reset_queue(eth->netdev[i]); if (!MTK_HAS_CAPS(soc->caps, MTK_SRAM) && eth->scratch_ring) { dma_free_coherent(eth->dma_dev, MTK_QDMA_RING_SIZE * soc->txrx.txd_size, @@ -4616,7 +4636,7 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) } if (mtk_is_netsys_v3_or_greater(mac->hw) && - MTK_HAS_CAPS(mac->hw->soc->caps, MTK_ESW_BIT) && + MTK_HAS_CAPS(mac->hw->soc->caps, MTK_ESW) && id == MTK_GMAC1_ID) { mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index a4efbeb162084..11e16c9e4e922 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -34,8 +34,10 @@ struct mtk_flow_data { u16 vlan_in; struct { - u16 id; - __be16 proto; + struct { + u16 id; + __be16 proto; + } vlans[2]; u8 num; } vlan; struct { @@ -99,7 +101,9 @@ mtk_flow_get_wdma_info(struct net_device *dev, const u8 *addr, struct mtk_wdma_i if (!IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED)) return -1; + rcu_read_lock(); err = dev_fill_forward_path(dev, addr, &stack); + rcu_read_unlock(); if (err) return err; @@ -330,18 +334,19 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f, case FLOW_ACTION_CSUM: break; case FLOW_ACTION_VLAN_PUSH: - if (data.vlan.num == 1 || + if (data.vlan.num + data.pppoe.num == 2 || act->vlan.proto != htons(ETH_P_8021Q)) return -EOPNOTSUPP; - data.vlan.id = act->vlan.vid; - data.vlan.proto = act->vlan.proto; + data.vlan.vlans[data.vlan.num].id = act->vlan.vid; + data.vlan.vlans[data.vlan.num].proto = act->vlan.proto; data.vlan.num++; break; case FLOW_ACTION_VLAN_POP: break; case FLOW_ACTION_PPPOE_PUSH: - if (data.pppoe.num == 1) + if (data.pppoe.num == 1 || + data.vlan.num == 2) return -EOPNOTSUPP; data.pppoe.sid = act->pppoe.sid; @@ -431,12 +436,9 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f, if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE) foe.bridge.vlan = data.vlan_in; - if (data.vlan.num == 1) { - if (data.vlan.proto != htons(ETH_P_8021Q)) - return -EOPNOTSUPP; + for (i = 0; i < data.vlan.num; i++) + mtk_foe_entry_set_vlan(eth, &foe, data.vlan.vlans[i].id); - mtk_foe_entry_set_vlan(eth, &foe, data.vlan.id); - } if (data.pppoe.num == 1) mtk_foe_entry_set_pppoe(eth, &foe, data.pppoe.sid); diff --git a/drivers/net/ethernet/mediatek/mtk_star_emac.c b/drivers/net/ethernet/mediatek/mtk_star_emac.c index 25989c79c92e6..5eb7a97e7eb17 100644 --- a/drivers/net/ethernet/mediatek/mtk_star_emac.c +++ b/drivers/net/ethernet/mediatek/mtk_star_emac.c @@ -1163,6 +1163,7 @@ static int mtk_star_tx_poll(struct napi_struct *napi, int budget) struct net_device *ndev = priv->ndev; unsigned int head = ring->head; unsigned int entry = ring->tail; + unsigned long flags; while (entry != head && count < (MTK_STAR_RING_NUM_DESCS - 1)) { ret = mtk_star_tx_complete_one(priv); @@ -1182,9 +1183,9 @@ static int mtk_star_tx_poll(struct napi_struct *napi, int budget) netif_wake_queue(ndev); if (napi_complete(napi)) { - spin_lock(&priv->lock); + spin_lock_irqsave(&priv->lock, flags); mtk_star_enable_dma_irq(priv, false, true); - spin_unlock(&priv->lock); + spin_unlock_irqrestore(&priv->lock, flags); } return 0; @@ -1341,16 +1342,16 @@ static int mtk_star_rx(struct mtk_star_priv *priv, int budget) static int mtk_star_rx_poll(struct napi_struct *napi, int budget) { struct mtk_star_priv *priv; + unsigned long flags; int work_done = 0; priv = container_of(napi, struct mtk_star_priv, rx_napi); work_done = mtk_star_rx(priv, budget); - if (work_done < budget) { - napi_complete_done(napi, work_done); - spin_lock(&priv->lock); + if (work_done < budget && napi_complete_done(napi, work_done)) { + spin_lock_irqsave(&priv->lock, flags); mtk_star_enable_dma_irq(priv, true, false); - spin_unlock(&priv->lock); + spin_unlock_irqrestore(&priv->lock, flags); } return work_done; @@ -1467,6 +1468,8 @@ static __maybe_unused int mtk_star_suspend(struct device *dev) if (netif_running(ndev)) mtk_star_disable(ndev); + netif_device_detach(ndev); + clk_bulk_disable_unprepare(MTK_STAR_NCLKS, priv->clks); return 0; @@ -1491,6 +1494,8 @@ static __maybe_unused int mtk_star_resume(struct device *dev) clk_bulk_disable_unprepare(MTK_STAR_NCLKS, priv->clks); } + netif_device_attach(ndev); + return ret; } diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c index 85a9ad2b86bff..525e2d365cd95 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed.c +++ b/drivers/net/ethernet/mediatek/mtk_wed.c @@ -1886,7 +1886,6 @@ void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, if (!pdev) goto err_of_node_put; - get_device(&pdev->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) goto err_put_device; diff --git a/drivers/net/ethernet/mellanox/mlx4/alloc.c b/drivers/net/ethernet/mellanox/mlx4/alloc.c index b330020dc0d67..f2bded847e61d 100644 --- a/drivers/net/ethernet/mellanox/mlx4/alloc.c +++ b/drivers/net/ethernet/mellanox/mlx4/alloc.c @@ -682,9 +682,9 @@ static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) } static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, - struct mlx4_db *db, int order) + struct mlx4_db *db, unsigned int order) { - int o; + unsigned int o; int i; for (o = order; o <= 1; ++o) { @@ -712,7 +712,7 @@ static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, return 0; } -int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) +int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, unsigned int order) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_db_pgdir *pgdir; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_clock.c b/drivers/net/ethernet/mellanox/mlx4/en_clock.c index 9e3b761820881..2d5b86207e079 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_clock.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_clock.c @@ -249,7 +249,7 @@ static const struct ptp_clock_info mlx4_en_ptp_clock_info = { static u32 freq_to_shift(u16 freq) { u32 freq_khz = freq * 1000; - u64 max_val_cycles = freq_khz * 1000 * MLX4_EN_WRAP_AROUND_SEC; + u64 max_val_cycles = freq_khz * 1000ULL * MLX4_EN_WRAP_AROUND_SEC; u64 max_val_cycles_rounded = 1ULL << fls64(max_val_cycles - 1); /* calculate max possible multiplier in order to fit in 64bit */ u64 max_mul = div64_u64(ULLONG_MAX, max_val_cycles_rounded); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 164a13272faa2..07dced3c2b1c0 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -1916,6 +1916,7 @@ static int mlx4_en_get_ts_info(struct net_device *dev, if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { info->so_timestamping |= SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_TX_SOFTWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 33bbcced81059..275561272721e 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -1177,9 +1177,9 @@ static void mlx4_en_do_uc_filter(struct mlx4_en_priv *priv, mlx4_unregister_mac(mdev->dev, priv->port, mac); hlist_del_rcu(&entry->hlist); - kfree_rcu(entry, rcu); en_dbg(DRV, priv, "Removed MAC %pM on port:%d\n", entry->mac, priv->port); + kfree_rcu(entry, rcu); ++removed; } } diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 65cb63f6c4658..61a0fd8424a2c 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -450,6 +450,8 @@ int mlx4_en_process_tx_cq(struct net_device *dev, if (unlikely(!priv->port_up)) return 0; + if (unlikely(!napi_budget) && cq->type == TX_XDP) + return 0; netdev_txq_bql_complete_prefetchw(ring->tx_queue); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 3e6bd27f6315d..336e0a54b3dc2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -289,6 +289,10 @@ static void poll_timeout(struct mlx5_cmd_work_ent *ent) return; } cond_resched(); + if (mlx5_cmd_is_down(dev)) { + ent->ret = -ENXIO; + return; + } } while (time_before(jiffies, poll_end)); ent->ret = -ETIMEDOUT; @@ -1059,7 +1063,7 @@ static void cmd_work_handler(struct work_struct *work) poll_timeout(ent); /* make sure we read the descriptor after ownership is SW */ rmb(); - mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT)); + mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, !!ent->ret); } } @@ -1916,8 +1920,8 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context, pages_queue, token, force_polling); - if (callback) - return err; + if (callback && !err) + return 0; if (err > 0) /* Failed in FW, command didn't execute */ err = deliv_status_to_err(err); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index f66788a2ed77e..8489b5087d9c6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -107,7 +107,7 @@ static int mlx5_devlink_reload_fw_activate(struct devlink *devlink, struct netli if (err) return err; - mlx5_unload_one_devl_locked(dev, true); + mlx5_sync_reset_unload_flow(dev, true); err = mlx5_health_wait_pci_up(dev); if (err) NL_SET_ERR_MSG_MOD(extack, "FW activate aborted, PCI reads fail after reset"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 20a6bc1a234f4..455d02b6500d0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -93,8 +93,6 @@ struct page_pool; #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \ MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD)) -#define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18 - /* Keep in sync with mlx5e_mpwrq_log_wqe_sz. * These are theoretical maximums, which can be further restricted by * capabilities. These values are used for static resource allocations and @@ -521,6 +519,12 @@ struct mlx5e_xdpsq { struct mlx5e_channel *channel; } ____cacheline_aligned_in_smp; +struct mlx5e_xdp_buff { + struct xdp_buff xdp; + struct mlx5_cqe64 *cqe; + struct mlx5e_rq *rq; +}; + struct mlx5e_ktls_resync_resp; struct mlx5e_icosq { @@ -719,6 +723,7 @@ struct mlx5e_rq { struct mlx5e_xdpsq *xdpsq; DECLARE_BITMAP(flags, 8); struct page_pool *page_pool; + struct mlx5e_xdp_buff mxbuf; /* AF_XDP zero-copy */ struct xsk_buff_pool *xsk_pool; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 775010e94cb7c..9c22d64af6853 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -9,6 +9,9 @@ #include #include +#define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18 +#define MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ 17 + static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev) { u8 min_page_shift = MLX5_CAP_GEN_2(mdev, log_min_mkey_entity_size); @@ -95,25 +98,29 @@ u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode) return sizeof(struct mlx5_ksm) * 4; } WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", mode); - return 0; + return 1; } u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, enum mlx5e_mpwrq_umr_mode umr_mode) { u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); - u8 max_pages_per_wqe, max_log_mpwqe_size; + u8 max_pages_per_wqe, max_log_wqe_size_calc; + u8 max_log_wqe_size_cap; u16 max_wqe_size; /* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */ max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB; max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe), MLX5_UMR_FLEX_ALIGNMENT) / umr_entry_size; - max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift; + max_log_wqe_size_calc = ilog2(max_pages_per_wqe) + page_shift; + + WARN_ON_ONCE(max_log_wqe_size_calc < MLX5E_ORDER2_MAX_PACKET_MTU); - WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU); + max_log_wqe_size_cap = mlx5_core_is_ecpf(mdev) ? + MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ : MLX5_MPWRQ_MAX_LOG_WQE_SZ; - return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ); + return min_t(u8, max_log_wqe_size_calc, max_log_wqe_size_cap); } u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c index 8e25f4ef5ccce..4720523813b97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c @@ -272,8 +272,8 @@ static int port_update_shared_buffer(struct mlx5_core_dev *mdev, /* Total shared buffer size is split in a ratio of 3:1 between * lossy and lossless pools respectively. */ - lossy_epool_size = (shared_buffer_size / 4) * 3; lossless_ipool_size = shared_buffer_size / 4; + lossy_epool_size = shared_buffer_size - lossless_ipool_size; mlx5e_port_set_sbpr(mdev, 0, MLX5_EGRESS_DIR, MLX5_LOSSY_POOL, 0, lossy_epool_size); @@ -288,14 +288,12 @@ static int port_set_buffer(struct mlx5e_priv *priv, u16 port_buff_cell_sz = priv->dcbx.port_buff_cell_sz; struct mlx5_core_dev *mdev = priv->mdev; int sz = MLX5_ST_SZ_BYTES(pbmc_reg); - u32 new_headroom_size = 0; - u32 current_headroom_size; + u32 current_headroom_cells = 0; + u32 new_headroom_cells = 0; void *in; int err; int i; - current_headroom_size = port_buffer->headroom_size; - in = kzalloc(sz, GFP_KERNEL); if (!in) return -ENOMEM; @@ -306,12 +304,14 @@ static int port_set_buffer(struct mlx5e_priv *priv, for (i = 0; i < MLX5E_MAX_NETWORK_BUFFER; i++) { void *buffer = MLX5_ADDR_OF(pbmc_reg, in, buffer[i]); + current_headroom_cells += MLX5_GET(bufferx_reg, buffer, size); + u64 size = port_buffer->buffer[i].size; u64 xoff = port_buffer->buffer[i].xoff; u64 xon = port_buffer->buffer[i].xon; - new_headroom_size += size; do_div(size, port_buff_cell_sz); + new_headroom_cells += size; do_div(xoff, port_buff_cell_sz); do_div(xon, port_buff_cell_sz); MLX5_SET(bufferx_reg, buffer, size, size); @@ -320,10 +320,8 @@ static int port_set_buffer(struct mlx5e_priv *priv, MLX5_SET(bufferx_reg, buffer, xon_threshold, xon); } - new_headroom_size /= port_buff_cell_sz; - current_headroom_size /= port_buff_cell_sz; - err = port_update_shared_buffer(priv->mdev, current_headroom_size, - new_headroom_size); + err = port_update_shared_buffer(priv->mdev, current_headroom_cells, + new_headroom_cells); if (err) goto out; @@ -331,6 +329,9 @@ static int port_set_buffer(struct mlx5e_priv *priv, if (err) goto out; + /* RO bits should be set to 0 on write */ + MLX5_SET(pbmc_reg, in, port_buffer_size, 0); + err = mlx5e_port_set_pbmc(mdev, in); out: kfree(in); @@ -574,7 +575,6 @@ int mlx5e_port_manual_buffer_config(struct mlx5e_priv *priv, if (err) return err; } - priv->dcbx.xoff = xoff; /* Apply the settings */ if (update_buffer) { @@ -583,6 +583,8 @@ int mlx5e_port_manual_buffer_config(struct mlx5e_priv *priv, return err; } + priv->dcbx.xoff = xoff; + if (update_prio2buffer) err = mlx5e_port_set_priority2buffer(priv->mdev, prio2buffer); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index d9acc37afe1c8..74729bf168b13 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -362,7 +362,7 @@ void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_que void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) { struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); - struct Qdisc *qdisc = dev_queue->qdisc_sleeping; + struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); if (!qdisc) return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index b723ff5e5249c..13c7ed1bb37e9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -895,7 +895,7 @@ void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq) mlx5e_xmit_xdp_doorbell(xdpsq); if (test_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags)) { - xdp_do_flush_map(); + xdp_do_flush(); __clear_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index ecfe93a479da8..38e9ff6aa3aee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -44,12 +44,6 @@ (MLX5E_XDP_INLINE_WQE_MAX_DS_CNT * MLX5_SEND_WQE_DS - \ sizeof(struct mlx5_wqe_inline_seg)) -struct mlx5e_xdp_buff { - struct xdp_buff xdp; - struct mlx5_cqe64 *cqe; - struct mlx5e_rq *rq; -}; - /* XDP packets can be transmitted in different ways. On completion, we need to * distinguish between them to clean up things in a proper way. */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 463c23ae0ad1e..5161bf51fa110 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -1093,7 +1093,7 @@ mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, static int mlx5e_xfrm_add_policy(struct xfrm_policy *x, struct netlink_ext_ack *extack) { - struct net_device *netdev = x->xdo.real_dev; + struct net_device *netdev = x->xdo.dev; struct mlx5e_ipsec_pol_entry *pol_entry; struct mlx5e_priv *priv; int err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 51a144246ea62..f96e9bbf8fc69 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -333,6 +333,10 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, if (unlikely(!sa_entry)) { rcu_read_unlock(); atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss); + /* Clear secpath to prevent invalid dereference + * in downstream XFRM policy checks. + */ + secpath_reset(skb); return; } xfrm_state_hold(sa_entry->x); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c index 8705cffc747ff..29e633e6dd3f0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c @@ -587,32 +587,55 @@ static int mlx5e_dcbnl_ieee_setmaxrate(struct net_device *netdev, struct mlx5_core_dev *mdev = priv->mdev; u8 max_bw_value[IEEE_8021QAZ_MAX_TCS]; u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS]; - __u64 upper_limit_mbps = roundup(255 * MLX5E_100MB, MLX5E_1GB); + __u64 upper_limit_mbps; + __u64 upper_limit_gbps; int i; + struct { + int scale; + const char *units_str; + } units[] = { + [MLX5_100_MBPS_UNIT] = { + .scale = 100, + .units_str = "Mbps", + }, + [MLX5_GBPS_UNIT] = { + .scale = 1, + .units_str = "Gbps", + }, + }; memset(max_bw_value, 0, sizeof(max_bw_value)); memset(max_bw_unit, 0, sizeof(max_bw_unit)); + upper_limit_mbps = 255 * MLX5E_100MB; + upper_limit_gbps = 255 * MLX5E_1GB; for (i = 0; i <= mlx5_max_tc(mdev); i++) { if (!maxrate->tc_maxrate[i]) { max_bw_unit[i] = MLX5_BW_NO_LIMIT; continue; } - if (maxrate->tc_maxrate[i] < upper_limit_mbps) { + if (maxrate->tc_maxrate[i] <= upper_limit_mbps) { max_bw_value[i] = div_u64(maxrate->tc_maxrate[i], MLX5E_100MB); max_bw_value[i] = max_bw_value[i] ? max_bw_value[i] : 1; max_bw_unit[i] = MLX5_100_MBPS_UNIT; - } else { + } else if (max_bw_value[i] <= upper_limit_gbps) { max_bw_value[i] = div_u64(maxrate->tc_maxrate[i], MLX5E_1GB); max_bw_unit[i] = MLX5_GBPS_UNIT; + } else { + netdev_err(netdev, + "tc_%d maxrate %llu Kbps exceeds limit %llu\n", + i, maxrate->tc_maxrate[i], + upper_limit_gbps); + return -EINVAL; } } for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { - netdev_dbg(netdev, "%s: tc_%d <=> max_bw %d Gbps\n", - __func__, i, max_bw_value[i]); + netdev_dbg(netdev, "%s: tc_%d <=> max_bw %u %s\n", __func__, i, + max_bw_value[i] * units[max_bw_unit[i]].scale, + units[max_bw_unit[i]].units_str); } return mlx5_modify_port_ets_rate_limit(mdev, max_bw_value, max_bw_unit); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 54379297a7489..ccd2ebfd26737 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1839,12 +1839,12 @@ static int mlx5e_get_module_eeprom_by_page(struct net_device *netdev, if (!size_read) return i; - if (size_read == -EINVAL) - return -EINVAL; if (size_read < 0) { - netdev_err(priv->netdev, "%s: mlx5_query_module_eeprom_by_page failed:0x%x\n", - __func__, size_read); - return i; + NL_SET_ERR_MSG_FMT_MOD( + extack, + "Query module eeprom by page failed, read %u bytes, err %d\n", + i, size_read); + return size_read; } i += size_read; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 8a892614015cd..5c6f01abdcb91 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3627,8 +3627,11 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, /* MQPRIO is another toplevel qdisc that can't be attached * simultaneously with the offloaded HTB. */ - if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq))) - return -EINVAL; + if (mlx5e_selq_is_htb_enabled(&priv->selq)) { + NL_SET_ERR_MSG_MOD(mqprio->extack, + "MQPRIO cannot be configured when HTB offload is enabled."); + return -EOPNOTSUPP; + } switch (mqprio->mode) { case TC_MQPRIO_MODE_DCB: @@ -4136,6 +4139,10 @@ static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n"); + features &= ~NETIF_F_HW_MACSEC; + if (netdev->features & NETIF_F_HW_MACSEC) + netdev_warn(netdev, "Disabling HW MACsec offload, not supported in switchdev mode\n"); + return features; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 751d3ffcd2f6c..656a7b65f4c7b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -63,6 +63,7 @@ #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \ max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE) #define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1 +#define MLX5E_REP_PARAMS_DEF_LOG_RQ_SIZE 0x8 static const char mlx5e_rep_driver_name[] = "mlx5e_rep"; @@ -798,6 +799,8 @@ static void mlx5e_build_rep_params(struct net_device *netdev) /* RQ */ mlx5e_build_rq_params(mdev, params); + if (!mlx5e_is_uplink_rep(priv) && mlx5_core_is_ecpf(mdev)) + params->log_rq_mtu_frames = MLX5E_REP_PARAMS_DEF_LOG_RQ_SIZE; /* If netdev is already registered (e.g. move from nic profile to uplink, * RTNL lock must be held before triggering netdev notifiers. @@ -829,6 +832,8 @@ static void mlx5e_build_rep_netdev(struct net_device *netdev, netdev->ethtool_ops = &mlx5e_rep_ethtool_ops; netdev->watchdog_timeo = 15 * HZ; + if (mlx5_core_is_ecpf(mdev)) + netdev->tx_queue_len = 1 << MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE; #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) netdev->hw_features |= NETIF_F_HW_TC; @@ -1443,12 +1448,21 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = { static int mlx5e_vport_uplink_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) { - struct mlx5e_priv *priv = netdev_priv(mlx5_uplink_netdev_get(dev)); struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); + struct net_device *netdev; + struct mlx5e_priv *priv; + int err; + + netdev = mlx5_uplink_netdev_get(dev); + if (!netdev) + return 0; + priv = netdev_priv(netdev); rpriv->netdev = priv->netdev; - return mlx5e_netdev_change_profile(priv, &mlx5e_uplink_rep_profile, - rpriv); + err = mlx5e_netdev_change_profile(priv, &mlx5e_uplink_rep_profile, + rpriv); + mlx5_uplink_netdev_put(dev, netdev); + return err; } static void @@ -1560,8 +1574,16 @@ mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) { struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); struct net_device *netdev = rpriv->netdev; - struct mlx5e_priv *priv = netdev_priv(netdev); - void *ppriv = priv->ppriv; + struct mlx5e_priv *priv; + void *ppriv; + + if (!netdev) { + ppriv = rpriv; + goto free_ppriv; + } + + priv = netdev_priv(netdev); + ppriv = priv->ppriv; if (rep->vport == MLX5_VPORT_UPLINK) { mlx5e_vport_uplink_rep_unload(rpriv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 57b0e26696e30..1d586451900b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1159,8 +1159,9 @@ static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp) } } -static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe, - u32 cqe_bcnt) +static unsigned int mlx5e_lro_update_hdr(struct sk_buff *skb, + struct mlx5_cqe64 *cqe, + u32 cqe_bcnt) { struct ethhdr *eth = (struct ethhdr *)(skb->data); struct tcphdr *tcp; @@ -1211,6 +1212,8 @@ static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe, tcp->check = csum_ipv6_magic(&ipv6->saddr, &ipv6->daddr, payload_len, IPPROTO_TCP, check); } + + return (unsigned int)((unsigned char *)tcp + tcp->doff * 4 - skb->data); } static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index) @@ -1567,8 +1570,10 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, mlx5e_macsec_offload_handle_rx_skb(netdev, skb, cqe); if (lro_num_seg > 1) { - mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt); - skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg); + unsigned int hdrlen = mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt); + + skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt - hdrlen, lro_num_seg); + skb_shinfo(skb)->gso_segs = lro_num_seg; /* Subtract one since we already counted this as one * "regular" packet in mlx5e_complete_rx_cqe() */ @@ -1692,17 +1697,17 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, prog = rcu_dereference(rq->xdp_prog); if (prog) { - struct mlx5e_xdp_buff mxbuf; + struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; net_prefetchw(va); /* xdp_frame data area */ mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, rq->buff.frame0_sz, - cqe_bcnt, &mxbuf); - if (mlx5e_xdp_handle(rq, prog, &mxbuf)) + cqe_bcnt, mxbuf); + if (mlx5e_xdp_handle(rq, prog, mxbuf)) return NULL; /* page/packet was consumed by XDP */ - rx_headroom = mxbuf.xdp.data - mxbuf.xdp.data_hard_start; - metasize = mxbuf.xdp.data - mxbuf.xdp.data_meta; - cqe_bcnt = mxbuf.xdp.data_end - mxbuf.xdp.data; + rx_headroom = mxbuf->xdp.data - mxbuf->xdp.data_hard_start; + metasize = mxbuf->xdp.data - mxbuf->xdp.data_meta; + cqe_bcnt = mxbuf->xdp.data_end - mxbuf->xdp.data; } frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt, metasize); @@ -1721,11 +1726,11 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi struct mlx5_cqe64 *cqe, u32 cqe_bcnt) { struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; + struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; struct mlx5e_wqe_frag_info *head_wi = wi; u16 rx_headroom = rq->buff.headroom; struct mlx5e_frag_page *frag_page; struct skb_shared_info *sinfo; - struct mlx5e_xdp_buff mxbuf; u32 frag_consumed_bytes; struct bpf_prog *prog; struct sk_buff *skb; @@ -1745,8 +1750,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi net_prefetch(va + rx_headroom); mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, rq->buff.frame0_sz, - frag_consumed_bytes, &mxbuf); - sinfo = xdp_get_shared_info_from_buff(&mxbuf.xdp); + frag_consumed_bytes, mxbuf); + sinfo = xdp_get_shared_info_from_buff(&mxbuf->xdp); truesize = 0; cqe_bcnt -= frag_consumed_bytes; @@ -1758,8 +1763,9 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt); - mlx5e_add_skb_shared_info_frag(rq, sinfo, &mxbuf.xdp, frag_page, - wi->offset, frag_consumed_bytes); + mlx5e_add_skb_shared_info_frag(rq, sinfo, &mxbuf->xdp, + frag_page, wi->offset, + frag_consumed_bytes); truesize += frag_info->frag_stride; cqe_bcnt -= frag_consumed_bytes; @@ -1768,31 +1774,46 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi } prog = rcu_dereference(rq->xdp_prog); - if (prog && mlx5e_xdp_handle(rq, prog, &mxbuf)) { - if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { - struct mlx5e_wqe_frag_info *pwi; + if (prog) { + u8 nr_frags_free, old_nr_frags = sinfo->nr_frags; + + if (mlx5e_xdp_handle(rq, prog, mxbuf)) { + if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, + rq->flags)) { + struct mlx5e_wqe_frag_info *pwi; - for (pwi = head_wi; pwi < wi; pwi++) - pwi->frag_page->frags++; + wi -= old_nr_frags - sinfo->nr_frags; + + for (pwi = head_wi; pwi < wi; pwi++) + pwi->frag_page->frags++; + } + return NULL; /* page/packet was consumed by XDP */ + } + + nr_frags_free = old_nr_frags - sinfo->nr_frags; + if (unlikely(nr_frags_free)) { + wi -= nr_frags_free; + truesize -= nr_frags_free * frag_info->frag_stride; } - return NULL; /* page/packet was consumed by XDP */ } - skb = mlx5e_build_linear_skb(rq, mxbuf.xdp.data_hard_start, rq->buff.frame0_sz, - mxbuf.xdp.data - mxbuf.xdp.data_hard_start, - mxbuf.xdp.data_end - mxbuf.xdp.data, - mxbuf.xdp.data - mxbuf.xdp.data_meta); + skb = mlx5e_build_linear_skb( + rq, mxbuf->xdp.data_hard_start, rq->buff.frame0_sz, + mxbuf->xdp.data - mxbuf->xdp.data_hard_start, + mxbuf->xdp.data_end - mxbuf->xdp.data, + mxbuf->xdp.data - mxbuf->xdp.data_meta); if (unlikely(!skb)) return NULL; skb_mark_for_recycle(skb); head_wi->frag_page->frags++; - if (xdp_buff_has_frags(&mxbuf.xdp)) { + if (xdp_buff_has_frags(&mxbuf->xdp)) { /* sinfo->nr_frags is reset by build_skb, calculate again. */ xdp_update_skb_shared_info(skb, wi - head_wi - 1, sinfo->xdp_frags_size, truesize, - xdp_buff_is_frag_pfmemalloc(&mxbuf.xdp)); + xdp_buff_is_frag_pfmemalloc( + &mxbuf->xdp)); for (struct mlx5e_wqe_frag_info *pwi = head_wi + 1; pwi < wi; pwi++) pwi->frag_page->frags++; @@ -1998,11 +2019,12 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx]; u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); struct mlx5e_frag_page *head_page = frag_page; + struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; u32 frag_offset = head_offset; u32 byte_cnt = cqe_bcnt; struct skb_shared_info *sinfo; - struct mlx5e_xdp_buff mxbuf; unsigned int truesize = 0; + u32 pg_consumed_bytes; struct bpf_prog *prog; struct sk_buff *skb; u32 linear_frame_sz; @@ -2047,20 +2069,23 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w } } - mlx5e_fill_mxbuf(rq, cqe, va, linear_hr, linear_frame_sz, linear_data_len, &mxbuf); + mlx5e_fill_mxbuf(rq, cqe, va, linear_hr, linear_frame_sz, + linear_data_len, mxbuf); - sinfo = xdp_get_shared_info_from_buff(&mxbuf.xdp); + sinfo = xdp_get_shared_info_from_buff(&mxbuf->xdp); while (byte_cnt) { /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ - u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - frag_offset, byte_cnt); + pg_consumed_bytes = + min_t(u32, PAGE_SIZE - frag_offset, byte_cnt); if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) truesize += pg_consumed_bytes; else truesize += ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); - mlx5e_add_skb_shared_info_frag(rq, sinfo, &mxbuf.xdp, frag_page, frag_offset, + mlx5e_add_skb_shared_info_frag(rq, sinfo, &mxbuf->xdp, + frag_page, frag_offset, pg_consumed_bytes); byte_cnt -= pg_consumed_bytes; frag_offset = 0; @@ -2068,10 +2093,15 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w } if (prog) { - if (mlx5e_xdp_handle(rq, prog, &mxbuf)) { + u8 nr_frags_free, old_nr_frags = sinfo->nr_frags; + u32 len; + + if (mlx5e_xdp_handle(rq, prog, mxbuf)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { struct mlx5e_frag_page *pfp; + frag_page -= old_nr_frags - sinfo->nr_frags; + for (pfp = head_page; pfp < frag_page; pfp++) pfp->frags++; @@ -2081,10 +2111,20 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w return NULL; /* page/packet was consumed by XDP */ } - skb = mlx5e_build_linear_skb(rq, mxbuf.xdp.data_hard_start, - linear_frame_sz, - mxbuf.xdp.data - mxbuf.xdp.data_hard_start, 0, - mxbuf.xdp.data - mxbuf.xdp.data_meta); + nr_frags_free = old_nr_frags - sinfo->nr_frags; + if (unlikely(nr_frags_free)) { + frag_page -= nr_frags_free; + truesize -= (nr_frags_free - 1) * PAGE_SIZE + + ALIGN(pg_consumed_bytes, + BIT(rq->mpwqe.log_stride_sz)); + } + + len = mxbuf->xdp.data_end - mxbuf->xdp.data; + + skb = mlx5e_build_linear_skb( + rq, mxbuf->xdp.data_hard_start, linear_frame_sz, + mxbuf->xdp.data - mxbuf->xdp.data_hard_start, len, + mxbuf->xdp.data - mxbuf->xdp.data_meta); if (unlikely(!skb)) { mlx5e_page_release_fragmented(rq, &wi->linear_page); return NULL; @@ -2094,29 +2134,34 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w wi->linear_page.frags++; mlx5e_page_release_fragmented(rq, &wi->linear_page); - if (xdp_buff_has_frags(&mxbuf.xdp)) { + if (xdp_buff_has_frags(&mxbuf->xdp)) { struct mlx5e_frag_page *pagep; /* sinfo->nr_frags is reset by build_skb, calculate again. */ xdp_update_skb_shared_info(skb, frag_page - head_page, sinfo->xdp_frags_size, truesize, - xdp_buff_is_frag_pfmemalloc(&mxbuf.xdp)); + xdp_buff_is_frag_pfmemalloc( + &mxbuf->xdp)); pagep = head_page; do pagep->frags++; while (++pagep < frag_page); + + headlen = min_t(u16, MLX5E_RX_MAX_HEAD - len, + skb->data_len); + __pskb_pull_tail(skb, headlen); } - __pskb_pull_tail(skb, headlen); } else { dma_addr_t addr; - if (xdp_buff_has_frags(&mxbuf.xdp)) { + if (xdp_buff_has_frags(&mxbuf->xdp)) { struct mlx5e_frag_page *pagep; xdp_update_skb_shared_info(skb, sinfo->nr_frags, sinfo->xdp_frags_size, truesize, - xdp_buff_is_frag_pfmemalloc(&mxbuf.xdp)); + xdp_buff_is_frag_pfmemalloc( + &mxbuf->xdp)); pagep = frag_page - sinfo->nr_frags; do @@ -2166,20 +2211,20 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, prog = rcu_dereference(rq->xdp_prog); if (prog) { - struct mlx5e_xdp_buff mxbuf; + struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; net_prefetchw(va); /* xdp_frame data area */ mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, rq->buff.frame0_sz, - cqe_bcnt, &mxbuf); - if (mlx5e_xdp_handle(rq, prog, &mxbuf)) { + cqe_bcnt, mxbuf); + if (mlx5e_xdp_handle(rq, prog, mxbuf)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) frag_page->frags++; return NULL; /* page/packet was consumed by XDP */ } - rx_headroom = mxbuf.xdp.data - mxbuf.xdp.data_hard_start; - metasize = mxbuf.xdp.data - mxbuf.xdp.data_meta; - cqe_bcnt = mxbuf.xdp.data_end - mxbuf.xdp.data; + rx_headroom = mxbuf->xdp.data - mxbuf->xdp.data_hard_start; + metasize = mxbuf->xdp.data - mxbuf->xdp.data_meta; + cqe_bcnt = mxbuf->xdp.data_end - mxbuf->xdp.data; } frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt, metasize); @@ -2276,7 +2321,10 @@ mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt) { int nr_frags = skb_shinfo(skb)->nr_frags; - return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; + if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE) + return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE; + else + return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; } static void diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c index 08a75654f5f18..c170503b3aace 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c @@ -165,6 +165,9 @@ mlx5e_test_loopback_validate(struct sk_buff *skb, struct udphdr *udph; struct iphdr *iph; + if (skb_linearize(skb)) + goto out; + /* We are only going to peek, no need to clone the SKB */ if (MLX5E_TEST_PKT_SIZE - ETH_HLEN > skb_headlen(skb)) goto out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 4b96ad657145b..1c69244e00d75 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1339,16 +1339,13 @@ static void fec_set_rs_stats(struct ethtool_fec_stats *fec_stats, u32 *ppcnt) } static void fec_set_block_stats(struct mlx5e_priv *priv, + int mode, struct ethtool_fec_stats *fec_stats) { struct mlx5_core_dev *mdev = priv->mdev; u32 out[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); - int mode = fec_active_mode(mdev); - - if (mode == MLX5E_FEC_NOFEC) - return; MLX5_SET(ppcnt_reg, in, local_port, 1); MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP); @@ -1389,11 +1386,14 @@ static void fec_set_corrected_bits_total(struct mlx5e_priv *priv, void mlx5e_stats_fec_get(struct mlx5e_priv *priv, struct ethtool_fec_stats *fec_stats) { - if (!MLX5_CAP_PCAM_FEATURE(priv->mdev, ppcnt_statistical_group)) + int mode = fec_active_mode(priv->mdev); + + if (mode == MLX5E_FEC_NOFEC || + !MLX5_CAP_PCAM_FEATURE(priv->mdev, ppcnt_statistical_group)) return; fec_set_corrected_bits_total(priv, fec_stats); - fec_set_block_stats(priv, fec_stats); + fec_set_block_stats(priv, mode, fec_stats); } #define PPORT_ETH_EXT_OFF(c) \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index dc9b157a44993..2be9c69daad5f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -1915,9 +1915,8 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, return err; } -static bool mlx5_flow_has_geneve_opt(struct mlx5e_tc_flow *flow) +static bool mlx5_flow_has_geneve_opt(struct mlx5_flow_spec *spec) { - struct mlx5_flow_spec *spec = &flow->attr->parse_attr->spec; void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_3); @@ -1956,7 +1955,7 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, } complete_all(&flow->del_hw_done); - if (mlx5_flow_has_geneve_opt(flow)) + if (mlx5_flow_has_geneve_opt(&attr->parse_attr->spec)) mlx5_geneve_tlv_option_del(priv->mdev->geneve); if (flow->decap_route) @@ -2456,12 +2455,13 @@ static int parse_tunnel_attr(struct mlx5e_priv *priv, err = mlx5e_tc_tun_parse(filter_dev, priv, tmp_spec, f, match_level); if (err) { - kvfree(tmp_spec); NL_SET_ERR_MSG_MOD(extack, "Failed to parse tunnel attributes"); netdev_warn(priv->netdev, "Failed to parse tunnel attributes"); - return err; + } else { + err = mlx5e_tc_set_attr_rx_tun(flow, tmp_spec); } - err = mlx5e_tc_set_attr_rx_tun(flow, tmp_spec); + if (mlx5_flow_has_geneve_opt(tmp_spec)) + mlx5_geneve_tlv_option_del(priv->mdev->geneve); kvfree(tmp_spec); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index d8e739cbcbced..91319b5acd3de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -47,10 +47,12 @@ static void mlx5_esw_offloads_pf_vf_devlink_port_attrs_set(struct mlx5_eswitch * devlink_port_attrs_pci_vf_set(dl_port, controller_num, pfnum, vport_num - 1, external); } else if (mlx5_core_is_ec_vf_vport(esw->dev, vport_num)) { + u16 base_vport = mlx5_core_ec_vf_vport_base(dev); + memcpy(dl_port->attrs.switch_id.id, ppid.id, ppid.id_len); dl_port->attrs.switch_id.id_len = ppid.id_len; devlink_port_attrs_pci_vf_set(dl_port, 0, pfnum, - vport_num - 1, false); + vport_num - base_vport, false); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c index 8587cd572da53..bdb825aa87268 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c @@ -96,7 +96,7 @@ static int esw_create_legacy_fdb_table(struct mlx5_eswitch *esw) if (!flow_group_in) return -ENOMEM; - ft_attr.max_fte = POOL_NEXT_SIZE; + ft_attr.max_fte = MLX5_FS_MAX_POOL_SIZE; ft_attr.prio = LEGACY_FDB_PRIO; fdb = mlx5_create_flow_table(root_ns, &ft_attr); if (IS_ERR(fdb)) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c index cc0f2be21a265..05fbd2098b268 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c @@ -2,6 +2,7 @@ /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ #include "eswitch.h" +#include "lib/mlx5.h" #include "esw/qos.h" #include "en/port.h" #define CREATE_TRACE_POINTS @@ -712,6 +713,71 @@ int mlx5_esw_qos_set_vport_rate(struct mlx5_eswitch *esw, struct mlx5_vport *vpo return err; } +static u32 mlx5_esw_qos_lag_link_speed_get_locked(struct mlx5_core_dev *mdev) +{ + struct ethtool_link_ksettings lksettings; + struct net_device *slave, *master; + u32 speed = SPEED_UNKNOWN; + + /* Lock ensures a stable reference to master and slave netdevice + * while port speed of master is queried. + */ + ASSERT_RTNL(); + + slave = mlx5_uplink_netdev_get(mdev); + if (!slave) + goto out; + + master = netdev_master_upper_dev_get(slave); + if (master && !__ethtool_get_link_ksettings(master, &lksettings)) + speed = lksettings.base.speed; + +out: + mlx5_uplink_netdev_put(mdev, slave); + return speed; +} + +static int mlx5_esw_qos_max_link_speed_get(struct mlx5_core_dev *mdev, u32 *link_speed_max, + bool hold_rtnl_lock, struct netlink_ext_ack *extack) +{ + int err; + + if (!mlx5_lag_is_active(mdev)) + goto skip_lag; + + if (hold_rtnl_lock) + rtnl_lock(); + + *link_speed_max = mlx5_esw_qos_lag_link_speed_get_locked(mdev); + + if (hold_rtnl_lock) + rtnl_unlock(); + + if (*link_speed_max != (u32)SPEED_UNKNOWN) + return 0; + +skip_lag: + err = mlx5_port_max_linkspeed(mdev, link_speed_max); + if (err) + NL_SET_ERR_MSG_MOD(extack, "Failed to get link maximum speed"); + + return err; +} + +static int mlx5_esw_qos_link_speed_verify(struct mlx5_core_dev *mdev, + const char *name, u32 link_speed_max, + u64 value, struct netlink_ext_ack *extack) +{ + if (value > link_speed_max) { + pr_err("%s rate value %lluMbps exceed link maximum speed %u.\n", + name, value, link_speed_max); + NL_SET_ERR_MSG_MOD(extack, "TX rate value exceed link maximum speed"); + return -EINVAL; + } + + return 0; +} + int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32 rate_mbps) { u32 ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; @@ -755,12 +821,6 @@ static int esw_qos_devlink_rate_to_mbps(struct mlx5_core_dev *mdev, const char * u64 value; int err; - err = mlx5_port_max_linkspeed(mdev, &link_speed_max); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Failed to get link maximum speed"); - return err; - } - value = div_u64_rem(*rate, MLX5_LINKSPEED_UNIT, &remainder); if (remainder) { pr_err("%s rate value %lluBps not in link speed units of 1Mbps.\n", @@ -769,12 +829,13 @@ static int esw_qos_devlink_rate_to_mbps(struct mlx5_core_dev *mdev, const char * return -EINVAL; } - if (value > link_speed_max) { - pr_err("%s rate value %lluMbps exceed link maximum speed %u.\n", - name, value, link_speed_max); - NL_SET_ERR_MSG_MOD(extack, "TX rate value exceed link maximum speed"); - return -EINVAL; - } + err = mlx5_esw_qos_max_link_speed_get(mdev, &link_speed_max, true, extack); + if (err) + return err; + + err = mlx5_esw_qos_link_speed_verify(mdev, name, link_speed_max, value, extack); + if (err) + return err; *rate = value; return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index f6022c135ec02..914b380fd3eeb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1295,12 +1295,15 @@ mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw, ret = mlx5_eswitch_load_pf_vf_vport(esw, MLX5_VPORT_ECPF, enabled_events); if (ret) goto ecpf_err; - if (mlx5_core_ec_sriov_enabled(esw->dev)) { - ret = mlx5_eswitch_load_ec_vf_vports(esw, esw->esw_funcs.num_ec_vfs, - enabled_events); - if (ret) - goto ec_vf_err; - } + } + + /* Enable ECVF vports */ + if (mlx5_core_ec_sriov_enabled(esw->dev)) { + ret = mlx5_eswitch_load_ec_vf_vports(esw, + esw->esw_funcs.num_ec_vfs, + enabled_events); + if (ret) + goto ec_vf_err; } /* Enable VF vports */ @@ -1331,9 +1334,11 @@ void mlx5_eswitch_disable_pf_vf_vports(struct mlx5_eswitch *esw) { mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); + if (mlx5_core_ec_sriov_enabled(esw->dev)) + mlx5_eswitch_unload_ec_vf_vports(esw, + esw->esw_funcs.num_ec_vfs); + if (mlx5_ecpf_vport_exists(esw->dev)) { - if (mlx5_core_ec_sriov_enabled(esw->dev)) - mlx5_eswitch_unload_ec_vf_vports(esw, esw->esw_funcs.num_vfs); mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_ECPF); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 7eba3a5bb97ca..86fb8197594f5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -1170,19 +1170,19 @@ static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, struct mlx5_core_dev *peer_dev) { + struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch; struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {0}; struct mlx5_flow_handle **flows; - /* total vports is the same for both e-switches */ - int nvports = esw->total_vports; struct mlx5_flow_handle *flow; + struct mlx5_vport *peer_vport; struct mlx5_flow_spec *spec; - struct mlx5_vport *vport; int err, pfindex; unsigned long i; void *misc; - if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(esw->dev)) + if (!MLX5_VPORT_MANAGER(peer_dev) && + !mlx5_core_is_ecpf_esw_manager(peer_dev)) return 0; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); @@ -1191,7 +1191,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, peer_miss_rules_setup(esw, peer_dev, spec, &dest); - flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL); + flows = kvcalloc(peer_esw->total_vports, sizeof(*flows), GFP_KERNEL); if (!flows) { err = -ENOMEM; goto alloc_flows_err; @@ -1201,10 +1201,10 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); - if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { - vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); - esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, - spec, MLX5_VPORT_PF); + if (mlx5_core_is_ecpf_esw_manager(peer_dev)) { + peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); + esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, + MLX5_VPORT_PF); flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); @@ -1212,11 +1212,11 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, err = PTR_ERR(flow); goto add_pf_flow_err; } - flows[vport->index] = flow; + flows[peer_vport->index] = flow; } - if (mlx5_ecpf_vport_exists(esw->dev)) { - vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); + if (mlx5_ecpf_vport_exists(peer_dev)) { + peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); @@ -1224,13 +1224,14 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, err = PTR_ERR(flow); goto add_ecpf_flow_err; } - flows[vport->index] = flow; + flows[peer_vport->index] = flow; } - mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { + mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_vfs(peer_dev)) { esw_set_peer_miss_rule_source_port(esw, - peer_dev->priv.eswitch, - spec, vport->vport); + peer_esw, + spec, peer_vport->vport); flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), spec, &flow_act, &dest, 1); @@ -1238,22 +1239,22 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, err = PTR_ERR(flow); goto add_vf_flow_err; } - flows[vport->index] = flow; + flows[peer_vport->index] = flow; } - if (mlx5_core_ec_sriov_enabled(esw->dev)) { - mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { - if (i >= mlx5_core_max_ec_vfs(peer_dev)) - break; - esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, - spec, vport->vport); + if (mlx5_core_ec_sriov_enabled(peer_dev)) { + mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_ec_vfs(peer_dev)) { + esw_set_peer_miss_rule_source_port(esw, peer_esw, + spec, + peer_vport->vport); flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec, &flow_act, &dest, 1); if (IS_ERR(flow)) { err = PTR_ERR(flow); goto add_ec_vf_flow_err; } - flows[vport->index] = flow; + flows[peer_vport->index] = flow; } } @@ -1270,25 +1271,27 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, return 0; add_ec_vf_flow_err: - mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { - if (!flows[vport->index]) + mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_ec_vfs(peer_dev)) { + if (!flows[peer_vport->index]) continue; - mlx5_del_flow_rules(flows[vport->index]); + mlx5_del_flow_rules(flows[peer_vport->index]); } add_vf_flow_err: - mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { - if (!flows[vport->index]) + mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_vfs(peer_dev)) { + if (!flows[peer_vport->index]) continue; - mlx5_del_flow_rules(flows[vport->index]); + mlx5_del_flow_rules(flows[peer_vport->index]); } - if (mlx5_ecpf_vport_exists(esw->dev)) { - vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); - mlx5_del_flow_rules(flows[vport->index]); + if (mlx5_ecpf_vport_exists(peer_dev)) { + peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); + mlx5_del_flow_rules(flows[peer_vport->index]); } add_ecpf_flow_err: - if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { - vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); - mlx5_del_flow_rules(flows[vport->index]); + if (mlx5_core_is_ecpf_esw_manager(peer_dev)) { + peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); + mlx5_del_flow_rules(flows[peer_vport->index]); } add_pf_flow_err: esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); @@ -1301,37 +1304,34 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, struct mlx5_core_dev *peer_dev) { + struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch; u16 peer_index = mlx5_get_dev_index(peer_dev); struct mlx5_flow_handle **flows; - struct mlx5_vport *vport; + struct mlx5_vport *peer_vport; unsigned long i; flows = esw->fdb_table.offloads.peer_miss_rules[peer_index]; if (!flows) return; - if (mlx5_core_ec_sriov_enabled(esw->dev)) { - mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { - /* The flow for a particular vport could be NULL if the other ECPF - * has fewer or no VFs enabled - */ - if (!flows[vport->index]) - continue; - mlx5_del_flow_rules(flows[vport->index]); - } + if (mlx5_core_ec_sriov_enabled(peer_dev)) { + mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_ec_vfs(peer_dev)) + mlx5_del_flow_rules(flows[peer_vport->index]); } - mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) - mlx5_del_flow_rules(flows[vport->index]); + mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, + mlx5_core_max_vfs(peer_dev)) + mlx5_del_flow_rules(flows[peer_vport->index]); - if (mlx5_ecpf_vport_exists(esw->dev)) { - vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); - mlx5_del_flow_rules(flows[vport->index]); + if (mlx5_ecpf_vport_exists(peer_dev)) { + peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); + mlx5_del_flow_rules(flows[peer_vport->index]); } - if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { - vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); - mlx5_del_flow_rules(flows[vport->index]); + if (mlx5_core_is_ecpf_esw_manager(peer_dev)) { + peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); + mlx5_del_flow_rules(flows[peer_vport->index]); } kvfree(flows); @@ -3499,7 +3499,9 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) int err; mutex_init(&esw->offloads.termtbl_mutex); - mlx5_rdma_enable_roce(esw->dev); + err = mlx5_rdma_enable_roce(esw->dev); + if (err) + goto err_roce; err = mlx5_esw_host_number_init(esw); if (err) @@ -3560,6 +3562,7 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) esw_offloads_metadata_uninit(esw); err_metadata: mlx5_rdma_disable_roce(esw->dev); +err_roce: mutex_destroy(&esw->offloads.termtbl_mutex); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index 3ec892d51f57d..e7143d32b2211 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -163,11 +163,16 @@ static int temp_warn(struct notifier_block *nb, unsigned long type, void *data) u64 value_msb; value_lsb = be64_to_cpu(eqe->data.temp_warning.sensor_warning_lsb); + /* bit 1-63 are not supported for NICs, + * hence read only bit 0 (asic) from lsb. + */ + value_lsb &= 0x1; value_msb = be64_to_cpu(eqe->data.temp_warning.sensor_warning_msb); - mlx5_core_warn(events->dev, - "High temperature on sensors with bit set %llx %llx", - value_msb, value_lsb); + if (net_ratelimit()) + mlx5_core_warn(events->dev, + "High temperature on sensors with bit set %llx %llx", + value_msb, value_lsb); return NOTIFY_OK; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index d2dc375f5e49c..5f35a6fc03054 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -1984,6 +1984,7 @@ try_add_to_existing_fg(struct mlx5_flow_table *ft, struct mlx5_flow_handle *rule; struct match_list *iter; bool take_write = false; + bool try_again = false; struct fs_fte *fte; u64 version = 0; int err; @@ -2043,6 +2044,7 @@ try_add_to_existing_fg(struct mlx5_flow_table *ft, nested_down_write_ref_node(&g->node, FS_LOCK_PARENT); if (!g->node.active) { + try_again = true; up_write_ref_node(&g->node, false); continue; } @@ -2064,7 +2066,8 @@ try_add_to_existing_fg(struct mlx5_flow_table *ft, tree_put_node(&fte->node, false); return rule; } - rule = ERR_PTR(-ENOENT); + err = try_again ? -EAGAIN : -ENOENT; + rule = ERR_PTR(err); out: kmem_cache_free(steering->ftes_cache, fte); return rule; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.c index c14590acc7726..f6abfd00d7e68 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.c @@ -50,10 +50,12 @@ mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type tab int i, found_i = -1; for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { - if (dev->priv.ft_pool->ft_left[i] && FT_POOLS[i] >= desired_size && + if (dev->priv.ft_pool->ft_left[i] && + (FT_POOLS[i] >= desired_size || + desired_size == MLX5_FS_MAX_POOL_SIZE) && FT_POOLS[i] <= max_ft_size) { found_i = i; - if (desired_size != POOL_NEXT_SIZE) + if (desired_size != MLX5_FS_MAX_POOL_SIZE) break; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.h index 25f4274b372b5..173e312db7204 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.h @@ -7,8 +7,6 @@ #include #include "fs_core.h" -#define POOL_NEXT_SIZE 0 - int mlx5_ft_pool_init(struct mlx5_core_dev *dev); void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index 6b17346aa4cef..4d64d179b5dd7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -6,13 +6,15 @@ #include "fw_reset.h" #include "diag/fw_tracer.h" #include "lib/tout.h" +#include "sf/sf.h" enum { MLX5_FW_RESET_FLAGS_RESET_REQUESTED, MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, MLX5_FW_RESET_FLAGS_PENDING_COMP, MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, - MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED + MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, + MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, }; struct mlx5_fw_reset { @@ -25,7 +27,9 @@ struct mlx5_fw_reset { struct work_struct reset_reload_work; struct work_struct reset_now_work; struct work_struct reset_abort_work; + struct delayed_work reset_timeout_work; unsigned long reset_flags; + u8 reset_method; struct timer_list timer; struct completion done; int ret; @@ -94,7 +98,7 @@ static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level, } static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level, - u8 *reset_type, u8 *reset_state) + u8 *reset_type, u8 *reset_state, u8 *reset_method) { u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {}; u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {}; @@ -110,13 +114,26 @@ static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level, *reset_type = MLX5_GET(mfrl_reg, out, reset_type); if (reset_state) *reset_state = MLX5_GET(mfrl_reg, out, reset_state); + if (reset_method) + *reset_method = MLX5_GET(mfrl_reg, out, pci_reset_req_method); return 0; } int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type) { - return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL); + return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL, NULL); +} + +static int mlx5_fw_reset_get_reset_method(struct mlx5_core_dev *dev, + u8 *reset_method) +{ + if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method)) { + *reset_method = MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE; + return 0; + } + + return mlx5_reg_mfrl_query(dev, NULL, NULL, NULL, reset_method); } static int mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev *dev, @@ -124,7 +141,7 @@ static int mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev *dev, { u8 reset_state; - if (mlx5_reg_mfrl_query(dev, NULL, NULL, &reset_state)) + if (mlx5_reg_mfrl_query(dev, NULL, NULL, &reset_state, NULL)) goto out; if (!reset_state) @@ -203,7 +220,7 @@ int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev) return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false); } -static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev, bool unloaded) +static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev) { struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; struct devlink *devlink = priv_to_devlink(dev); @@ -212,8 +229,7 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev, bool unload if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) { complete(&fw_reset->done); } else { - if (!unloaded) - mlx5_unload_one(dev, false); + mlx5_sync_reset_unload_flow(dev, false); if (mlx5_health_wait_pci_up(dev)) mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); else @@ -242,6 +258,8 @@ static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool return -EALREADY; } + if (current_work() != &fw_reset->reset_timeout_work.work) + cancel_delayed_work(&fw_reset->reset_timeout_work); mlx5_stop_sync_reset_poll(dev); if (poll_health) mlx5_start_health_poll(dev); @@ -256,7 +274,7 @@ static void mlx5_sync_reset_reload_work(struct work_struct *work) mlx5_sync_reset_clear_reset_requested(dev, false); mlx5_enter_error_state(dev, true); - mlx5_fw_reset_complete_reload(dev, false); + mlx5_fw_reset_complete_reload(dev); } #define MLX5_RESET_POLL_INTERVAL (HZ / 10) @@ -312,6 +330,11 @@ static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev) } mlx5_stop_health_poll(dev, true); mlx5_start_sync_reset_poll(dev); + + if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, + &fw_reset->reset_flags)) + schedule_delayed_work(&fw_reset->reset_timeout_work, + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_SYNC_UPDATE))); return 0; } @@ -383,6 +406,11 @@ static bool mlx5_is_reset_now_capable(struct mlx5_core_dev *dev) return false; } + if (!mlx5_core_is_ecpf(dev) && !mlx5_sf_table_empty(dev)) { + mlx5_core_warn(dev, "SFs should be removed before reset\n"); + return false; + } + #if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE) err = mlx5_check_hotplug_interrupt(dev); if (err) @@ -402,7 +430,11 @@ static void mlx5_sync_reset_request_event(struct work_struct *work) struct mlx5_core_dev *dev = fw_reset->dev; int err; - if (test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags) || + err = mlx5_fw_reset_get_reset_method(dev, &fw_reset->reset_method); + if (err) + mlx5_core_warn(dev, "Failed reading MFRL, err %d\n", err); + + if (err || test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags) || !mlx5_is_reset_now_capable(dev)) { err = mlx5_fw_reset_set_reset_sync_nack(dev); mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s", @@ -419,21 +451,15 @@ static void mlx5_sync_reset_request_event(struct work_struct *work) mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n"); } -static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev) +static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev, u16 dev_id) { struct pci_bus *bridge_bus = dev->pdev->bus; struct pci_dev *bridge = bridge_bus->self; unsigned long timeout; struct pci_dev *sdev; - u16 reg16, dev_id; int cap, err; + u16 reg16; - err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id); - if (err) - return pcibios_err_to_errno(err); - err = mlx5_check_dev_ids(dev, dev_id); - if (err) - return err; cap = pci_find_capability(bridge, PCI_CAP_ID_EXP); if (!cap) return -EOPNOTSUPP; @@ -503,64 +529,60 @@ static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev) return err; } -static void mlx5_sync_reset_now_event(struct work_struct *work) +static int mlx5_pci_reset_bus(struct mlx5_core_dev *dev) { - struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, - reset_now_work); - struct mlx5_core_dev *dev = fw_reset->dev; - int err; + if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method)) + return -EOPNOTSUPP; - if (mlx5_sync_reset_clear_reset_requested(dev, false)) - return; + return pci_reset_bus(dev->pdev); +} - mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n"); +static int mlx5_sync_pci_reset(struct mlx5_core_dev *dev, u8 reset_method) +{ + u16 dev_id; + int err; - err = mlx5_cmd_fast_teardown_hca(dev); - if (err) { - mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err); - goto done; - } + err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id); + if (err) + return pcibios_err_to_errno(err); + err = mlx5_check_dev_ids(dev, dev_id); + if (err) + return err; - err = mlx5_pci_link_toggle(dev); - if (err) { - mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, no reset done, err %d\n", err); - set_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags); + switch (reset_method) { + case MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE: + err = mlx5_pci_link_toggle(dev, dev_id); + if (err) + mlx5_core_warn(dev, "mlx5_pci_link_toggle failed\n"); + break; + case MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET: + err = mlx5_pci_reset_bus(dev); + if (err) + mlx5_core_warn(dev, "mlx5_pci_reset_bus failed\n"); + break; + default: + return -EOPNOTSUPP; } - mlx5_enter_error_state(dev, true); -done: - fw_reset->ret = err; - mlx5_fw_reset_complete_reload(dev, false); + return err; } -static void mlx5_sync_reset_unload_event(struct work_struct *work) +void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked) { - struct mlx5_fw_reset *fw_reset; - struct mlx5_core_dev *dev; + struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; unsigned long timeout; bool reset_action; u8 rst_state; int err; - fw_reset = container_of(work, struct mlx5_fw_reset, reset_unload_work); - dev = fw_reset->dev; - - if (mlx5_sync_reset_clear_reset_requested(dev, false)) - return; - - mlx5_core_warn(dev, "Sync Reset Unload. Function is forced down.\n"); - - err = mlx5_cmd_fast_teardown_hca(dev); - if (err) - mlx5_core_warn(dev, "Fast teardown failed, unloading, err %d\n", err); - else - mlx5_enter_error_state(dev, true); - - if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) + if (locked) mlx5_unload_one_devl_locked(dev, false); else mlx5_unload_one(dev, false); + if (!test_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags)) + return; + mlx5_set_fw_rst_ack(dev); mlx5_core_warn(dev, "Sync Reset Unload done, device reset expected\n"); @@ -583,17 +605,73 @@ static void mlx5_sync_reset_unload_event(struct work_struct *work) goto done; } - mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n", rst_state); + mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n", + rst_state); if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ) { - err = mlx5_pci_link_toggle(dev); + err = mlx5_sync_pci_reset(dev, fw_reset->reset_method); if (err) { - mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, err %d\n", err); + mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, err %d\n", + err); fw_reset->ret = err; } } done: - mlx5_fw_reset_complete_reload(dev, true); + clear_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags); +} + +static void mlx5_sync_reset_now_event(struct work_struct *work) +{ + struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, + reset_now_work); + struct mlx5_core_dev *dev = fw_reset->dev; + int err; + + if (mlx5_sync_reset_clear_reset_requested(dev, false)) + return; + + mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n"); + + err = mlx5_cmd_fast_teardown_hca(dev); + if (err) { + mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err); + goto done; + } + + err = mlx5_sync_pci_reset(dev, fw_reset->reset_method); + if (err) { + mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, no reset done, err %d\n", err); + set_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags); + } + + mlx5_enter_error_state(dev, true); +done: + fw_reset->ret = err; + mlx5_fw_reset_complete_reload(dev); +} + +static void mlx5_sync_reset_unload_event(struct work_struct *work) +{ + struct mlx5_fw_reset *fw_reset; + struct mlx5_core_dev *dev; + int err; + + fw_reset = container_of(work, struct mlx5_fw_reset, reset_unload_work); + dev = fw_reset->dev; + + if (mlx5_sync_reset_clear_reset_requested(dev, false)) + return; + + set_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags); + mlx5_core_warn(dev, "Sync Reset Unload. Function is forced down.\n"); + + err = mlx5_cmd_fast_teardown_hca(dev); + if (err) + mlx5_core_warn(dev, "Fast teardown failed, unloading, err %d\n", err); + else + mlx5_enter_error_state(dev, true); + + mlx5_fw_reset_complete_reload(dev); } static void mlx5_sync_reset_abort_event(struct work_struct *work) @@ -630,6 +708,19 @@ static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct } } +static void mlx5_sync_reset_timeout_work(struct work_struct *work) +{ + struct delayed_work *dwork = container_of(work, struct delayed_work, + work); + struct mlx5_fw_reset *fw_reset = + container_of(dwork, struct mlx5_fw_reset, reset_timeout_work); + struct mlx5_core_dev *dev = fw_reset->dev; + + if (mlx5_sync_reset_clear_reset_requested(dev, true)) + return; + mlx5_core_warn(dev, "PCI Sync FW Update Reset Timeout.\n"); +} + static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data) { struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb); @@ -713,6 +804,7 @@ void mlx5_drain_fw_reset(struct mlx5_core_dev *dev) cancel_work_sync(&fw_reset->reset_reload_work); cancel_work_sync(&fw_reset->reset_now_work); cancel_work_sync(&fw_reset->reset_abort_work); + cancel_delayed_work(&fw_reset->reset_timeout_work); } static const struct devlink_param mlx5_fw_reset_devlink_params[] = { @@ -756,6 +848,8 @@ int mlx5_fw_reset_init(struct mlx5_core_dev *dev) INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work); INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event); INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event); + INIT_DELAYED_WORK(&fw_reset->reset_timeout_work, + mlx5_sync_reset_timeout_work); init_completion(&fw_reset->done); return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h index ea527d06a85f0..d5b28525c960d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h @@ -12,6 +12,7 @@ int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel, int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev); int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev); +void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked); int mlx5_fw_reset_verify_fw_complete(struct mlx5_core_dev *dev, struct netlink_ext_ack *extack); void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index d798834c4e755..3ac8043f76dac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -833,6 +833,7 @@ static void poll_health(struct timer_list *t) health->prev = count; if (health->miss_counter == MAX_MISSES) { mlx5_core_err(dev, "device's health compromised - reached miss count\n"); + health->synd = ioread8(&h->synd); print_health_info(dev); queue_work(health->wq, &health->report_work); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c index 9482e51ac82a5..bdbbfaf504d98 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c @@ -28,7 +28,7 @@ struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) dm = kzalloc(sizeof(*dm), GFP_KERNEL); if (!dm) - return ERR_PTR(-ENOMEM); + return NULL; spin_lock_init(&dm->lock); @@ -80,7 +80,7 @@ struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) err_steering: kfree(dm); - return ERR_PTR(-ENOMEM); + return NULL; } void mlx5_dm_cleanup(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c index 711d14dea2485..d313cb7f0ed88 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c @@ -161,7 +161,8 @@ mlx5_chains_create_table(struct mlx5_fs_chains *chains, ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); - sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE; + sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? + FT_TBL_SZ : MLX5_FS_MAX_POOL_SIZE; ft_attr.max_fte = sz; /* We use chains_default_ft(chains) as the table's next_ft till diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 2b5826a785c4f..adcc2bc9c8c87 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -52,6 +52,19 @@ static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) static inline struct net_device *mlx5_uplink_netdev_get(struct mlx5_core_dev *mdev) { - return mdev->mlx5e_res.uplink_netdev; + struct mlx5e_resources *mlx5e_res = &mdev->mlx5e_res; + struct net_device *netdev; + + mutex_lock(&mlx5e_res->uplink_netdev_lock); + netdev = mlx5e_res->uplink_netdev; + netdev_hold(netdev, &mlx5e_res->tracker, GFP_KERNEL); + mutex_unlock(&mlx5e_res->uplink_netdev_lock); + return netdev; +} + +static inline void mlx5_uplink_netdev_put(struct mlx5_core_dev *mdev, + struct net_device *netdev) +{ + netdev_put(netdev, &mdev->mlx5e_res.tracker); } #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 96136229b1b07..8a11e410f7c13 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -627,6 +627,9 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_with_driver_unload)) MLX5_SET(cmd_hca_cap, set_hca_cap, pci_sync_for_fw_update_with_driver_unload, 1); + if (MLX5_CAP_GEN_MAX(dev, pcie_reset_using_hotreset_method)) + MLX5_SET(cmd_hca_cap, set_hca_cap, + pcie_reset_using_hotreset_method, 1); if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports)) MLX5_SET(cmd_hca_cap, @@ -1055,9 +1058,6 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) } dev->dm = mlx5_dm_create(dev); - if (IS_ERR(dev->dm)) - mlx5_core_warn(dev, "Failed to init device memory %ld\n", PTR_ERR(dev->dm)); - dev->tracer = mlx5_fw_tracer_create(dev); dev->hv_vhca = mlx5_hv_vhca_create(dev); dev->rsc_dump = mlx5_rsc_dump_create(dev); @@ -2206,6 +2206,7 @@ static const struct pci_device_id mlx5_core_pci_table[] = { { PCI_VDEVICE(MELLANOX, 0x1021) }, /* ConnectX-7 */ { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */ { PCI_VDEVICE(MELLANOX, 0x1025) }, /* ConnectX-9 */ + { PCI_VDEVICE(MELLANOX, 0x1027) }, /* ConnectX-10 */ { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */ { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */ { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index dcf58efac159c..5df1472fe2be2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -291,7 +291,7 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 function) static int alloc_system_page(struct mlx5_core_dev *dev, u32 function) { struct device *device = mlx5_core_dma_dev(dev); - int nid = dev_to_node(device); + int nid = dev->priv.numa_node; struct page *page; u64 zero_addr = 1; u64 addr; @@ -489,9 +489,12 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev, u32 func_id; u32 npages; u32 i = 0; + int err; - if (!mlx5_cmd_is_down(dev)) - return mlx5_cmd_do(dev, in, in_size, out, out_size); + err = mlx5_cmd_do(dev, in, in_size, out, out_size); + /* If FW is gone (-ENXIO), proceed to forceful reclaim */ + if (err != -ENXIO) + return err; /* No hard feelings, we want our pages back! */ npages = MLX5_GET(manage_pages_in, in, input_num_entries); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index a8d6fd18c0f55..487d8b413a41d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -323,10 +323,8 @@ struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i, free_irq(irq->map.virq, &irq->nh); err_req_irq: #ifdef CONFIG_RFS_ACCEL - if (i && rmap && *rmap) { - free_irq_cpu_rmap(*rmap); - *rmap = NULL; - } + if (i && rmap && *rmap) + irq_cpu_rmap_remove(*rmap, irq->map.virq); err_irq_rmap: #endif if (i && pci_msix_can_alloc_dyn(dev->pdev)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rdma.c b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c index a42f6cd99b744..5c552b71e371c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/rdma.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c @@ -118,8 +118,8 @@ static void mlx5_rdma_make_default_gid(struct mlx5_core_dev *dev, union ib_gid * static int mlx5_rdma_add_roce_addr(struct mlx5_core_dev *dev) { + u8 mac[ETH_ALEN] = {}; union ib_gid gid; - u8 mac[ETH_ALEN]; mlx5_rdma_make_default_gid(dev, &gid); return mlx5_core_roce_gid_set(dev, 0, @@ -140,17 +140,17 @@ void mlx5_rdma_disable_roce(struct mlx5_core_dev *dev) mlx5_nic_vport_disable_roce(dev); } -void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) +int mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) { int err; if (!MLX5_CAP_GEN(dev, roce)) - return; + return 0; err = mlx5_nic_vport_enable_roce(dev); if (err) { mlx5_core_err(dev, "Failed to enable RoCE: %d\n", err); - return; + return err; } err = mlx5_rdma_add_roce_addr(dev); @@ -165,10 +165,11 @@ void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) goto del_roce_addr; } - return; + return err; del_roce_addr: mlx5_rdma_del_roce_addr(dev); disable_roce: mlx5_nic_vport_disable_roce(dev); + return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rdma.h b/drivers/net/ethernet/mellanox/mlx5/core/rdma.h index 750cff2a71a4b..3d9e76c3d42fb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/rdma.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/rdma.h @@ -8,12 +8,12 @@ #ifdef CONFIG_MLX5_ESWITCH -void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev); +int mlx5_rdma_enable_roce(struct mlx5_core_dev *dev); void mlx5_rdma_disable_roce(struct mlx5_core_dev *dev); #else /* CONFIG_MLX5_ESWITCH */ -static inline void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) {} +static inline int mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) { return 0; } static inline void mlx5_rdma_disable_roce(struct mlx5_core_dev *dev) {} #endif /* CONFIG_MLX5_ESWITCH */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index e34a8f88c518c..d5b2b6cfc8d21 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -20,9 +20,16 @@ struct mlx5_sf { u16 hw_state; }; +static void *mlx5_sf_by_dl_port(struct devlink_port *dl_port) +{ + struct mlx5_devlink_port *mlx5_dl_port = mlx5_devlink_port_get(dl_port); + + return container_of(mlx5_dl_port, struct mlx5_sf, dl_port); +} + struct mlx5_sf_table { struct mlx5_core_dev *dev; /* To refer from notifier context. */ - struct xarray port_indices; /* port index based lookup. */ + struct xarray function_ids; /* function id based lookup. */ refcount_t refcount; struct completion disable_complete; struct mutex sf_state_lock; /* Serializes sf state among user cmds & vhca event handler. */ @@ -30,33 +37,20 @@ struct mlx5_sf_table { struct notifier_block vhca_nb; }; -static struct mlx5_sf * -mlx5_sf_lookup_by_index(struct mlx5_sf_table *table, unsigned int port_index) -{ - return xa_load(&table->port_indices, port_index); -} - static struct mlx5_sf * mlx5_sf_lookup_by_function_id(struct mlx5_sf_table *table, unsigned int fn_id) { - unsigned long index; - struct mlx5_sf *sf; - - xa_for_each(&table->port_indices, index, sf) { - if (sf->hw_fn_id == fn_id) - return sf; - } - return NULL; + return xa_load(&table->function_ids, fn_id); } -static int mlx5_sf_id_insert(struct mlx5_sf_table *table, struct mlx5_sf *sf) +static int mlx5_sf_function_id_insert(struct mlx5_sf_table *table, struct mlx5_sf *sf) { - return xa_insert(&table->port_indices, sf->port_index, sf, GFP_KERNEL); + return xa_insert(&table->function_ids, sf->hw_fn_id, sf, GFP_KERNEL); } -static void mlx5_sf_id_erase(struct mlx5_sf_table *table, struct mlx5_sf *sf) +static void mlx5_sf_function_id_erase(struct mlx5_sf_table *table, struct mlx5_sf *sf) { - xa_erase(&table->port_indices, sf->port_index); + xa_erase(&table->function_ids, sf->hw_fn_id); } static struct mlx5_sf * @@ -93,7 +87,7 @@ mlx5_sf_alloc(struct mlx5_sf_table *table, struct mlx5_eswitch *esw, sf->hw_state = MLX5_VHCA_STATE_ALLOCATED; sf->controller = controller; - err = mlx5_sf_id_insert(table, sf); + err = mlx5_sf_function_id_insert(table, sf); if (err) goto insert_err; @@ -111,7 +105,6 @@ mlx5_sf_alloc(struct mlx5_sf_table *table, struct mlx5_eswitch *esw, static void mlx5_sf_free(struct mlx5_sf_table *table, struct mlx5_sf *sf) { - mlx5_sf_id_erase(table, sf); mlx5_sf_hw_table_sf_free(table->dev, sf->controller, sf->id); trace_mlx5_sf_free(table->dev, sf->port_index, sf->controller, sf->hw_fn_id); kfree(sf); @@ -172,26 +165,19 @@ int mlx5_devlink_sf_port_fn_state_get(struct devlink_port *dl_port, struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(dl_port->devlink); + struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); struct mlx5_sf_table *table; - struct mlx5_sf *sf; - int err = 0; table = mlx5_sf_table_try_get(dev); if (!table) return -EOPNOTSUPP; - sf = mlx5_sf_lookup_by_index(table, dl_port->index); - if (!sf) { - err = -EOPNOTSUPP; - goto sf_err; - } mutex_lock(&table->sf_state_lock); *state = mlx5_sf_to_devlink_state(sf->hw_state); *opstate = mlx5_sf_to_devlink_opstate(sf->hw_state); mutex_unlock(&table->sf_state_lock); -sf_err: mlx5_sf_table_put(table); - return err; + return 0; } static int mlx5_sf_activate(struct mlx5_core_dev *dev, struct mlx5_sf *sf, @@ -257,8 +243,8 @@ int mlx5_devlink_sf_port_fn_state_set(struct devlink_port *dl_port, struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(dl_port->devlink); + struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); struct mlx5_sf_table *table; - struct mlx5_sf *sf; int err; table = mlx5_sf_table_try_get(dev); @@ -267,14 +253,7 @@ int mlx5_devlink_sf_port_fn_state_set(struct devlink_port *dl_port, "Port state set is only supported in eswitch switchdev mode or SF ports are disabled."); return -EOPNOTSUPP; } - sf = mlx5_sf_lookup_by_index(table, dl_port->index); - if (!sf) { - err = -ENODEV; - goto out; - } - err = mlx5_sf_state_set(dev, table, sf, state, extack); -out: mlx5_sf_table_put(table); return err; } @@ -301,6 +280,7 @@ static int mlx5_sf_add(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, return 0; esw_err: + mlx5_sf_function_id_erase(table, sf); mlx5_sf_free(table, sf); return err; } @@ -361,6 +341,8 @@ int mlx5_devlink_sf_port_new(struct devlink *devlink, static void mlx5_sf_dealloc(struct mlx5_sf_table *table, struct mlx5_sf *sf) { + mlx5_sf_function_id_erase(table, sf); + if (sf->hw_state == MLX5_VHCA_STATE_ALLOCATED) { mlx5_sf_free(table, sf); } else if (mlx5_sf_is_active(sf)) { @@ -383,10 +365,9 @@ int mlx5_devlink_sf_port_del(struct devlink *devlink, struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(devlink); + struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); struct mlx5_eswitch *esw = dev->priv.eswitch; struct mlx5_sf_table *table; - struct mlx5_sf *sf; - int err = 0; table = mlx5_sf_table_try_get(dev); if (!table) { @@ -394,21 +375,14 @@ int mlx5_devlink_sf_port_del(struct devlink *devlink, "Port del is only supported in eswitch switchdev mode or SF ports are disabled."); return -EOPNOTSUPP; } - sf = mlx5_sf_lookup_by_index(table, dl_port->index); - if (!sf) { - err = -ENODEV; - goto sf_err; - } mlx5_eswitch_unload_sf_vport(esw, sf->hw_fn_id); - mlx5_sf_id_erase(table, sf); mutex_lock(&table->sf_state_lock); mlx5_sf_dealloc(table, sf); mutex_unlock(&table->sf_state_lock); -sf_err: mlx5_sf_table_put(table); - return err; + return 0; } static bool mlx5_sf_state_update_check(const struct mlx5_sf *sf, u8 new_state) @@ -471,9 +445,8 @@ static void mlx5_sf_deactivate_all(struct mlx5_sf_table *table) /* At this point, no new user commands can start and no vhca event can * arrive. It is safe to destroy all user created SFs. */ - xa_for_each(&table->port_indices, index, sf) { + xa_for_each(&table->function_ids, index, sf) { mlx5_eswitch_unload_sf_vport(esw, sf->hw_fn_id); - mlx5_sf_id_erase(table, sf); mlx5_sf_dealloc(table, sf); } } @@ -531,7 +504,7 @@ int mlx5_sf_table_init(struct mlx5_core_dev *dev) mutex_init(&table->sf_state_lock); table->dev = dev; - xa_init(&table->port_indices); + xa_init(&table->function_ids); dev->priv.sf_table = table; refcount_set(&table->refcount, 0); table->esw_nb.notifier_call = mlx5_sf_esw_event; @@ -566,6 +539,16 @@ void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev) mlx5_esw_event_notifier_unregister(dev->priv.eswitch, &table->esw_nb); WARN_ON(refcount_read(&table->refcount)); mutex_destroy(&table->sf_state_lock); - WARN_ON(!xa_empty(&table->port_indices)); + WARN_ON(!xa_empty(&table->function_ids)); kfree(table); } + +bool mlx5_sf_table_empty(const struct mlx5_core_dev *dev) +{ + struct mlx5_sf_table *table = dev->priv.sf_table; + + if (!table) + return true; + + return xa_empty(&table->function_ids); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h index 860f9ddb7107b..89559a37997ad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h @@ -17,6 +17,7 @@ void mlx5_sf_hw_table_destroy(struct mlx5_core_dev *dev); int mlx5_sf_table_init(struct mlx5_core_dev *dev); void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev); +bool mlx5_sf_table_empty(const struct mlx5_core_dev *dev); int mlx5_devlink_sf_port_new(struct devlink *devlink, const struct devlink_port_new_attrs *add_attr, @@ -61,6 +62,11 @@ static inline void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev) { } +static inline bool mlx5_sf_table_empty(const struct mlx5_core_dev *dev) +{ + return true; +} + #endif #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index 21753f3278685..06b5265b6e6db 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -444,19 +444,22 @@ int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); + int err; out = kvzalloc(outlen, GFP_KERNEL); if (!out) return -ENOMEM; - mlx5_query_nic_vport_context(mdev, 0, out); + err = mlx5_query_nic_vport_context(mdev, 0, out); + if (err) + goto out; *node_guid = MLX5_GET64(query_nic_vport_context_out, out, nic_vport_context.node_guid); - +out: kvfree(out); - return 0; + return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_node_guid); @@ -498,19 +501,22 @@ int mlx5_query_nic_vport_qkey_viol_cntr(struct mlx5_core_dev *mdev, { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); + int err; out = kvzalloc(outlen, GFP_KERNEL); if (!out) return -ENOMEM; - mlx5_query_nic_vport_context(mdev, 0, out); + err = mlx5_query_nic_vport_context(mdev, 0, out); + if (err) + goto out; *qkey_viol_cntr = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.qkey_violation_counter); - +out: kvfree(out); - return 0; + return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_qkey_viol_cntr); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c b/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c index b032d5a4b3b84..10f5bc4892fc7 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c @@ -601,6 +601,8 @@ int mlxsw_linecard_devlink_info_get(struct mlxsw_linecard *linecard, err = devlink_info_version_fixed_put(req, DEVLINK_INFO_VERSION_GENERIC_FW_PSID, info->psid); + if (err) + goto unlock; sprintf(buf, "%u.%u.%u", info->fw_major, info->fw_minor, info->fw_sub_minor); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 9dbd5edff0b02..51f49510826a3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -2528,6 +2528,8 @@ static const struct mlxsw_listener mlxsw_sp_listener[] = { ROUTER_EXP, false), MLXSW_SP_RXL_NO_MARK(DISCARD_ING_ROUTER_DIP_LINK_LOCAL, FORWARD, ROUTER_EXP, false), + MLXSW_SP_RXL_NO_MARK(DISCARD_ING_ROUTER_SIP_LINK_LOCAL, FORWARD, + ROUTER_EXP, false), /* Multicast Router Traps */ MLXSW_SP_RXL_MARK(ACL1, TRAP_TO_CPU, MULTICAST, false), MLXSW_SP_RXL_L3_MARK(ACL2, TRAP_TO_CPU, MULTICAST, false), diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c index 9fd1ca0792584..fba545938fd7d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c @@ -816,8 +816,10 @@ int mlxsw_sp_flower_stats(struct mlxsw_sp *mlxsw_sp, return -EINVAL; rule = mlxsw_sp_acl_rule_lookup(mlxsw_sp, ruleset, f->cookie); - if (!rule) - return -EINVAL; + if (!rule) { + err = -EINVAL; + goto err_rule_get_stats; + } err = mlxsw_sp_acl_rule_get_stats(mlxsw_sp, rule, &packets, &bytes, &drops, &lastuse, &used_hw_stats); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index d15aa6b25a888..0534b10e29c5c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -3013,6 +3013,9 @@ static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp, .rif = rif, }; + if (!mlxsw_sp_dev_lower_is_port(mlxsw_sp_rif_dev(rif))) + return 0; + neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); if (rms.err) goto err_arp; diff --git a/drivers/net/ethernet/mellanox/mlxsw/trap.h b/drivers/net/ethernet/mellanox/mlxsw/trap.h index 83477c8e6971b..5bfc1499347a9 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/trap.h +++ b/drivers/net/ethernet/mellanox/mlxsw/trap.h @@ -95,6 +95,7 @@ enum { MLXSW_TRAP_ID_DISCARD_ING_ROUTER_IPV4_SIP_BC = 0x16A, MLXSW_TRAP_ID_DISCARD_ING_ROUTER_IPV4_DIP_LOCAL_NET = 0x16B, MLXSW_TRAP_ID_DISCARD_ING_ROUTER_DIP_LINK_LOCAL = 0x16C, + MLXSW_TRAP_ID_DISCARD_ING_ROUTER_SIP_LINK_LOCAL = 0x16D, MLXSW_TRAP_ID_DISCARD_ROUTER_IRIF_EN = 0x178, MLXSW_TRAP_ID_DISCARD_ROUTER_ERIF_EN = 0x179, MLXSW_TRAP_ID_DISCARD_ROUTER_LPM4 = 0x17B, diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index 72b3092d35f71..39a58c3578a02 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -18,6 +18,8 @@ #define EEPROM_MAC_OFFSET (0x01) #define MAX_EEPROM_SIZE (512) #define MAX_OTP_SIZE (1024) +#define MAX_HS_OTP_SIZE (8 * 1024) +#define MAX_HS_EEPROM_SIZE (64 * 1024) #define OTP_INDICATOR_1 (0xF3) #define OTP_INDICATOR_2 (0xF7) @@ -272,6 +274,9 @@ static int lan743x_hs_otp_read(struct lan743x_adapter *adapter, u32 offset, int ret; int i; + if (offset + length > MAX_HS_OTP_SIZE) + return -EINVAL; + ret = lan743x_hs_syslock_acquire(adapter, LOCK_TIMEOUT_MAX_CNT); if (ret < 0) return ret; @@ -320,6 +325,9 @@ static int lan743x_hs_otp_write(struct lan743x_adapter *adapter, u32 offset, int ret; int i; + if (offset + length > MAX_HS_OTP_SIZE) + return -EINVAL; + ret = lan743x_hs_syslock_acquire(adapter, LOCK_TIMEOUT_MAX_CNT); if (ret < 0) return ret; @@ -497,6 +505,9 @@ static int lan743x_hs_eeprom_read(struct lan743x_adapter *adapter, u32 val; int i; + if (offset + length > MAX_HS_EEPROM_SIZE) + return -EINVAL; + retval = lan743x_hs_syslock_acquire(adapter, LOCK_TIMEOUT_MAX_CNT); if (retval < 0) return retval; @@ -539,6 +550,9 @@ static int lan743x_hs_eeprom_write(struct lan743x_adapter *adapter, u32 val; int i; + if (offset + length > MAX_HS_EEPROM_SIZE) + return -EINVAL; + retval = lan743x_hs_syslock_acquire(adapter, LOCK_TIMEOUT_MAX_CNT); if (retval < 0) return retval; @@ -604,9 +618,9 @@ static int lan743x_ethtool_get_eeprom_len(struct net_device *netdev) struct lan743x_adapter *adapter = netdev_priv(netdev); if (adapter->flags & LAN743X_ADAPTER_FLAG_OTP) - return MAX_OTP_SIZE; + return adapter->is_pci11x1x ? MAX_HS_OTP_SIZE : MAX_OTP_SIZE; - return MAX_EEPROM_SIZE; + return adapter->is_pci11x1x ? MAX_HS_EEPROM_SIZE : MAX_EEPROM_SIZE; } static int lan743x_ethtool_get_eeprom(struct net_device *netdev, diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index 92010bfe5e413..781440d5756f3 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -1373,7 +1373,7 @@ static int lan743x_mac_set_mtu(struct lan743x_adapter *adapter, int new_mtu) } /* PHY */ -static int lan743x_phy_reset(struct lan743x_adapter *adapter) +static int lan743x_hw_reset_phy(struct lan743x_adapter *adapter) { u32 data; @@ -1407,7 +1407,7 @@ static void lan743x_phy_update_flowcontrol(struct lan743x_adapter *adapter, static int lan743x_phy_init(struct lan743x_adapter *adapter) { - return lan743x_phy_reset(adapter); + return lan743x_hw_reset_phy(adapter); } static void lan743x_phy_link_status_change(struct net_device *netdev) @@ -1949,6 +1949,7 @@ static void lan743x_tx_frame_add_lso(struct lan743x_tx *tx, if (nr_frags <= 0) { tx->frame_data0 |= TX_DESC_DATA0_LS_; tx->frame_data0 |= TX_DESC_DATA0_IOC_; + tx->frame_last = tx->frame_first; } tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail]; tx_descriptor->data0 = cpu_to_le32(tx->frame_data0); @@ -2018,6 +2019,7 @@ static int lan743x_tx_frame_add_fragment(struct lan743x_tx *tx, tx->frame_first = 0; tx->frame_data0 = 0; tx->frame_tail = 0; + tx->frame_last = 0; return -ENOMEM; } @@ -2058,16 +2060,18 @@ static void lan743x_tx_frame_end(struct lan743x_tx *tx, TX_DESC_DATA0_DTYPE_DATA_) { tx->frame_data0 |= TX_DESC_DATA0_LS_; tx->frame_data0 |= TX_DESC_DATA0_IOC_; + tx->frame_last = tx->frame_tail; } - tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail]; - buffer_info = &tx->buffer_info[tx->frame_tail]; + tx_descriptor = &tx->ring_cpu_ptr[tx->frame_last]; + buffer_info = &tx->buffer_info[tx->frame_last]; buffer_info->skb = skb; if (time_stamp) buffer_info->flags |= TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED; if (ignore_sync) buffer_info->flags |= TX_BUFFER_INFO_FLAG_IGNORE_SYNC; + tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail]; tx_descriptor->data0 = cpu_to_le32(tx->frame_data0); tx->frame_tail = lan743x_tx_next_index(tx, tx->frame_tail); tx->last_tail = tx->frame_tail; @@ -3255,6 +3259,7 @@ static int lan743x_hardware_init(struct lan743x_adapter *adapter, struct pci_dev *pdev) { struct lan743x_tx *tx; + u32 sgmii_ctl; int index; int ret; @@ -3267,6 +3272,15 @@ static int lan743x_hardware_init(struct lan743x_adapter *adapter, spin_lock_init(&adapter->eth_syslock_spinlock); mutex_init(&adapter->sgmii_rw_lock); pci11x1x_set_rfe_rd_fifo_threshold(adapter); + sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL); + if (adapter->is_sgmii_en) { + sgmii_ctl |= SGMII_CTL_SGMII_ENABLE_; + sgmii_ctl &= ~SGMII_CTL_SGMII_POWER_DN_; + } else { + sgmii_ctl &= ~SGMII_CTL_SGMII_ENABLE_; + sgmii_ctl |= SGMII_CTL_SGMII_POWER_DN_; + } + lan743x_csr_write(adapter, SGMII_CTL, sgmii_ctl); } else { adapter->max_tx_channels = LAN743X_MAX_TX_CHANNELS; adapter->used_tx_channels = LAN743X_USED_TX_CHANNELS; @@ -3315,7 +3329,6 @@ static int lan743x_hardware_init(struct lan743x_adapter *adapter, static int lan743x_mdiobus_init(struct lan743x_adapter *adapter) { - u32 sgmii_ctl; int ret; adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); @@ -3327,10 +3340,6 @@ static int lan743x_mdiobus_init(struct lan743x_adapter *adapter) adapter->mdiobus->priv = (void *)adapter; if (adapter->is_pci11x1x) { if (adapter->is_sgmii_en) { - sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL); - sgmii_ctl |= SGMII_CTL_SGMII_ENABLE_; - sgmii_ctl &= ~SGMII_CTL_SGMII_POWER_DN_; - lan743x_csr_write(adapter, SGMII_CTL, sgmii_ctl); netif_dbg(adapter, drv, adapter->netdev, "SGMII operation\n"); adapter->mdiobus->read = lan743x_mdiobus_read_c22; @@ -3341,10 +3350,6 @@ static int lan743x_mdiobus_init(struct lan743x_adapter *adapter) netif_dbg(adapter, drv, adapter->netdev, "lan743x-mdiobus-c45\n"); } else { - sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL); - sgmii_ctl &= ~SGMII_CTL_SGMII_ENABLE_; - sgmii_ctl |= SGMII_CTL_SGMII_POWER_DN_; - lan743x_csr_write(adapter, SGMII_CTL, sgmii_ctl); netif_dbg(adapter, drv, adapter->netdev, "RGMII operation\n"); // Only C22 support when RGMII I/F diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h index 3b2c6046eb3ad..b6c83c68241e6 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.h +++ b/drivers/net/ethernet/microchip/lan743x_main.h @@ -974,6 +974,7 @@ struct lan743x_tx { u32 frame_first; u32 frame_data0; u32 frame_tail; + u32 frame_last; struct lan743x_tx_buffer_info *buffer_info; diff --git a/drivers/net/ethernet/microchip/lan743x_ptp.c b/drivers/net/ethernet/microchip/lan743x_ptp.c index 39e1066ecd5ff..47f2531198f62 100644 --- a/drivers/net/ethernet/microchip/lan743x_ptp.c +++ b/drivers/net/ethernet/microchip/lan743x_ptp.c @@ -58,7 +58,7 @@ int lan743x_gpio_init(struct lan743x_adapter *adapter) static void lan743x_ptp_wait_till_cmd_done(struct lan743x_adapter *adapter, u32 bit_mask) { - int timeout = 1000; + int timeout = PTP_CMD_CTL_TIMEOUT_CNT; u32 data = 0; while (timeout && diff --git a/drivers/net/ethernet/microchip/lan743x_ptp.h b/drivers/net/ethernet/microchip/lan743x_ptp.h index e26d4eff71336..225e8232474d7 100644 --- a/drivers/net/ethernet/microchip/lan743x_ptp.h +++ b/drivers/net/ethernet/microchip/lan743x_ptp.h @@ -18,9 +18,10 @@ */ #define LAN743X_PTP_N_EVENT_CHAN 2 #define LAN743X_PTP_N_PEROUT LAN743X_PTP_N_EVENT_CHAN -#define LAN743X_PTP_N_EXTTS 4 -#define LAN743X_PTP_N_PPS 0 #define PCI11X1X_PTP_IO_MAX_CHANNELS 8 +#define LAN743X_PTP_N_EXTTS PCI11X1X_PTP_IO_MAX_CHANNELS +#define LAN743X_PTP_N_PPS 0 +#define PTP_CMD_CTL_TIMEOUT_CNT 50 struct lan743x_adapter; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c index 06811c60d598e..df10a0b68a08e 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c @@ -294,7 +294,7 @@ static void lan966x_stats_update(struct lan966x *lan966x) { int i, j; - mutex_lock(&lan966x->stats_lock); + spin_lock(&lan966x->stats_lock); for (i = 0; i < lan966x->num_phys_ports; i++) { uint idx = i * lan966x->num_stats; @@ -310,7 +310,7 @@ static void lan966x_stats_update(struct lan966x *lan966x) } } - mutex_unlock(&lan966x->stats_lock); + spin_unlock(&lan966x->stats_lock); } static int lan966x_get_sset_count(struct net_device *dev, int sset) @@ -365,7 +365,7 @@ static void lan966x_get_eth_mac_stats(struct net_device *dev, idx = port->chip_port * lan966x->num_stats; - mutex_lock(&lan966x->stats_lock); + spin_lock(&lan966x->stats_lock); mac_stats->FramesTransmittedOK = lan966x->stats[idx + SYS_COUNT_TX_UC] + @@ -424,7 +424,7 @@ static void lan966x_get_eth_mac_stats(struct net_device *dev, lan966x->stats[idx + SYS_COUNT_RX_LONG] + lan966x->stats[idx + SYS_COUNT_RX_PMAC_LONG]; - mutex_unlock(&lan966x->stats_lock); + spin_unlock(&lan966x->stats_lock); } static const struct ethtool_rmon_hist_range lan966x_rmon_ranges[] = { @@ -450,7 +450,7 @@ static void lan966x_get_eth_rmon_stats(struct net_device *dev, idx = port->chip_port * lan966x->num_stats; - mutex_lock(&lan966x->stats_lock); + spin_lock(&lan966x->stats_lock); rmon_stats->undersize_pkts = lan966x->stats[idx + SYS_COUNT_RX_SHORT] + @@ -508,7 +508,7 @@ static void lan966x_get_eth_rmon_stats(struct net_device *dev, lan966x->stats[idx + SYS_COUNT_TX_SZ_1024_1526] + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_1024_1526]; - mutex_unlock(&lan966x->stats_lock); + spin_unlock(&lan966x->stats_lock); *ranges = lan966x_rmon_ranges; } @@ -614,7 +614,7 @@ void lan966x_stats_get(struct net_device *dev, idx = port->chip_port * lan966x->num_stats; - mutex_lock(&lan966x->stats_lock); + spin_lock(&lan966x->stats_lock); stats->rx_bytes = lan966x->stats[idx + SYS_COUNT_RX_OCT] + lan966x->stats[idx + SYS_COUNT_RX_PMAC_OCT]; @@ -696,7 +696,7 @@ void lan966x_stats_get(struct net_device *dev, stats->collisions = lan966x->stats[idx + SYS_COUNT_TX_COL]; - mutex_unlock(&lan966x->stats_lock); + spin_unlock(&lan966x->stats_lock); } int lan966x_stats_init(struct lan966x *lan966x) @@ -712,7 +712,7 @@ int lan966x_stats_init(struct lan966x *lan966x) return -ENOMEM; /* Init stats worker */ - mutex_init(&lan966x->stats_lock); + spin_lock_init(&lan966x->stats_lock); snprintf(queue_name, sizeof(queue_name), "%s-stats", dev_name(lan966x->dev)); lan966x->stats_queue = create_singlethread_workqueue(queue_name); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index c3f6c10bc2393..5466f14e000ce 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -353,6 +353,11 @@ static void lan966x_ifh_set_rew_op(void *ifh, u64 rew_op) lan966x_ifh_set(ifh, rew_op, IFH_POS_REW_CMD, IFH_WID_REW_CMD); } +static void lan966x_ifh_set_oam_type(void *ifh, u64 oam_type) +{ + lan966x_ifh_set(ifh, oam_type, IFH_POS_PDU_TYPE, IFH_WID_PDU_TYPE); +} + static void lan966x_ifh_set_timestamp(void *ifh, u64 timestamp) { lan966x_ifh_set(ifh, timestamp, IFH_POS_TIMESTAMP, IFH_WID_TIMESTAMP); @@ -380,6 +385,7 @@ static netdev_tx_t lan966x_port_xmit(struct sk_buff *skb, return err; lan966x_ifh_set_rew_op(ifh, LAN966X_SKB_CB(skb)->rew_op); + lan966x_ifh_set_oam_type(ifh, LAN966X_SKB_CB(skb)->pdu_type); lan966x_ifh_set_timestamp(ifh, LAN966X_SKB_CB(skb)->ts_id); } @@ -875,6 +881,7 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, lan966x_vlan_port_set_vlan_aware(port, 0); lan966x_vlan_port_set_vid(port, HOST_PVID, false, false); lan966x_vlan_port_apply(port); + lan966x_vlan_port_rew_host(port); return 0; } @@ -1256,7 +1263,6 @@ static int lan966x_probe(struct platform_device *pdev) cancel_delayed_work_sync(&lan966x->stats_work); destroy_workqueue(lan966x->stats_queue); - mutex_destroy(&lan966x->stats_lock); debugfs_remove_recursive(lan966x->debugfs_root); @@ -1274,7 +1280,6 @@ static int lan966x_remove(struct platform_device *pdev) cancel_delayed_work_sync(&lan966x->stats_work); destroy_workqueue(lan966x->stats_queue); - mutex_destroy(&lan966x->stats_lock); lan966x_mac_purge_entries(lan966x); lan966x_mdb_deinit(lan966x); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h index caa9e0533c96b..1f5a18b205bf2 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -74,6 +74,10 @@ #define IFH_REW_OP_ONE_STEP_PTP 0x3 #define IFH_REW_OP_TWO_STEP_PTP 0x4 +#define IFH_PDU_TYPE_NONE 0 +#define IFH_PDU_TYPE_IPV4 7 +#define IFH_PDU_TYPE_IPV6 8 + #define FDMA_RX_DCB_MAX_DBS 1 #define FDMA_TX_DCB_MAX_DBS 1 #define FDMA_DCB_INFO_DATAL(x) ((x) & GENMASK(15, 0)) @@ -306,6 +310,7 @@ struct lan966x_phc { struct lan966x_skb_cb { u8 rew_op; + u8 pdu_type; u16 ts_id; unsigned long jiffies; }; @@ -342,8 +347,8 @@ struct lan966x { const struct lan966x_stat_layout *stats_layout; u32 num_stats; - /* workqueue for reading stats */ - struct mutex stats_lock; + /* lock for reading stats */ + spinlock_t stats_lock; u64 *stats; struct delayed_work stats_work; struct workqueue_struct *stats_queue; @@ -544,6 +549,7 @@ void lan966x_vlan_port_apply(struct lan966x_port *port); bool lan966x_vlan_cpu_member_cpu_vlan_mask(struct lan966x *lan966x, u16 vid); void lan966x_vlan_port_set_vlan_aware(struct lan966x_port *port, bool vlan_aware); +void lan966x_vlan_port_rew_host(struct lan966x_port *port); int lan966x_vlan_port_set_vid(struct lan966x_port *port, u16 vid, bool pvid, diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c index 63905bb5a63a8..87e5e81d40dc6 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c @@ -322,34 +322,55 @@ void lan966x_ptp_hwtstamp_get(struct lan966x_port *port, *cfg = phc->hwtstamp_config; } -static int lan966x_ptp_classify(struct lan966x_port *port, struct sk_buff *skb) +static void lan966x_ptp_classify(struct lan966x_port *port, struct sk_buff *skb, + u8 *rew_op, u8 *pdu_type) { struct ptp_header *header; u8 msgtype; int type; - if (port->ptp_tx_cmd == IFH_REW_OP_NOOP) - return IFH_REW_OP_NOOP; + if (port->ptp_tx_cmd == IFH_REW_OP_NOOP) { + *rew_op = IFH_REW_OP_NOOP; + *pdu_type = IFH_PDU_TYPE_NONE; + return; + } type = ptp_classify_raw(skb); - if (type == PTP_CLASS_NONE) - return IFH_REW_OP_NOOP; + if (type == PTP_CLASS_NONE) { + *rew_op = IFH_REW_OP_NOOP; + *pdu_type = IFH_PDU_TYPE_NONE; + return; + } header = ptp_parse_header(skb, type); - if (!header) - return IFH_REW_OP_NOOP; + if (!header) { + *rew_op = IFH_REW_OP_NOOP; + *pdu_type = IFH_PDU_TYPE_NONE; + return; + } - if (port->ptp_tx_cmd == IFH_REW_OP_TWO_STEP_PTP) - return IFH_REW_OP_TWO_STEP_PTP; + if (type & PTP_CLASS_L2) + *pdu_type = IFH_PDU_TYPE_NONE; + if (type & PTP_CLASS_IPV4) + *pdu_type = IFH_PDU_TYPE_IPV4; + if (type & PTP_CLASS_IPV6) + *pdu_type = IFH_PDU_TYPE_IPV6; + + if (port->ptp_tx_cmd == IFH_REW_OP_TWO_STEP_PTP) { + *rew_op = IFH_REW_OP_TWO_STEP_PTP; + return; + } /* If it is sync and run 1 step then set the correct operation, * otherwise run as 2 step */ msgtype = ptp_get_msgtype(header, type); - if ((msgtype & 0xf) == 0) - return IFH_REW_OP_ONE_STEP_PTP; + if ((msgtype & 0xf) == 0) { + *rew_op = IFH_REW_OP_ONE_STEP_PTP; + return; + } - return IFH_REW_OP_TWO_STEP_PTP; + *rew_op = IFH_REW_OP_TWO_STEP_PTP; } static void lan966x_ptp_txtstamp_old_release(struct lan966x_port *port) @@ -374,10 +395,12 @@ int lan966x_ptp_txtstamp_request(struct lan966x_port *port, { struct lan966x *lan966x = port->lan966x; unsigned long flags; + u8 pdu_type; u8 rew_op; - rew_op = lan966x_ptp_classify(port, skb); + lan966x_ptp_classify(port, skb, &rew_op, &pdu_type); LAN966X_SKB_CB(skb)->rew_op = rew_op; + LAN966X_SKB_CB(skb)->pdu_type = pdu_type; if (rew_op != IFH_REW_OP_TWO_STEP_PTP) return 0; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c b/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c index 1c88120eb291a..bcb4db76b75cd 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c @@ -297,6 +297,7 @@ static void lan966x_port_bridge_leave(struct lan966x_port *port, lan966x_vlan_port_set_vlan_aware(port, false); lan966x_vlan_port_set_vid(port, HOST_PVID, false, false); lan966x_vlan_port_apply(port); + lan966x_vlan_port_rew_host(port); } int lan966x_port_changeupper(struct net_device *dev, diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c index a4414f63c9b1c..c266f903ea8a6 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c @@ -403,11 +403,11 @@ static void lan966x_es0_read_esdx_counter(struct lan966x *lan966x, u32 counter; id = id & 0xff; /* counter limit */ - mutex_lock(&lan966x->stats_lock); + spin_lock(&lan966x->stats_lock); lan_wr(SYS_STAT_CFG_STAT_VIEW_SET(id), lan966x, SYS_STAT_CFG); counter = lan_rd(lan966x, SYS_CNT(LAN966X_STAT_ESDX_GRN_PKTS)) + lan_rd(lan966x, SYS_CNT(LAN966X_STAT_ESDX_YEL_PKTS)); - mutex_unlock(&lan966x->stats_lock); + spin_unlock(&lan966x->stats_lock); if (counter) admin->cache.counter = counter; } @@ -417,14 +417,14 @@ static void lan966x_es0_write_esdx_counter(struct lan966x *lan966x, { id = id & 0xff; /* counter limit */ - mutex_lock(&lan966x->stats_lock); + spin_lock(&lan966x->stats_lock); lan_wr(SYS_STAT_CFG_STAT_VIEW_SET(id), lan966x, SYS_STAT_CFG); lan_wr(0, lan966x, SYS_CNT(LAN966X_STAT_ESDX_GRN_BYTES)); lan_wr(admin->cache.counter, lan966x, SYS_CNT(LAN966X_STAT_ESDX_GRN_PKTS)); lan_wr(0, lan966x, SYS_CNT(LAN966X_STAT_ESDX_YEL_BYTES)); lan_wr(0, lan966x, SYS_CNT(LAN966X_STAT_ESDX_YEL_PKTS)); - mutex_unlock(&lan966x->stats_lock); + spin_unlock(&lan966x->stats_lock); } static void lan966x_vcap_cache_write(struct net_device *dev, diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c index 3c44660128dae..ffb245fb7d678 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c @@ -149,6 +149,27 @@ void lan966x_vlan_port_set_vlan_aware(struct lan966x_port *port, port->vlan_aware = vlan_aware; } +/* When the interface is in host mode, the interface should not be vlan aware + * but it should insert all the tags that it gets from the network stack. + * The tags are not in the data of the frame but actually in the skb and the ifh + * is configured already to get this tag. So what we need to do is to update the + * rewriter to insert the vlan tag for all frames which have a vlan tag + * different than 0. + */ +void lan966x_vlan_port_rew_host(struct lan966x_port *port) +{ + struct lan966x *lan966x = port->lan966x; + u32 val; + + /* Tag all frames except when VID=0*/ + val = REW_TAG_CFG_TAG_CFG_SET(2); + + /* Update only some bits in the register */ + lan_rmw(val, + REW_TAG_CFG_TAG_CFG, + lan966x, REW_TAG_CFG(port->chip_port)); +} + void lan966x_vlan_port_apply(struct lan966x_port *port) { struct lan966x *lan966x = port->lan966x; diff --git a/drivers/net/ethernet/microchip/sparx5/Kconfig b/drivers/net/ethernet/microchip/sparx5/Kconfig index f58c506bda228..15b27fc57aedd 100644 --- a/drivers/net/ethernet/microchip/sparx5/Kconfig +++ b/drivers/net/ethernet/microchip/sparx5/Kconfig @@ -3,7 +3,7 @@ config SPARX5_SWITCH depends on NET_SWITCHDEV depends on HAS_IOMEM depends on OF - depends on ARCH_SPARX5 || COMPILE_TEST + depends on ARCH_SPARX5 || ARCH_LAN969X || COMPILE_TEST depends on PTP_1588_CLOCK_OPTIONAL depends on BRIDGE || BRIDGE=n select PHYLINK diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index ae014e21eb605..d3c9a3020fbf6 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -28,6 +28,9 @@ static void mana_gd_init_pf_regs(struct pci_dev *pdev) gc->db_page_base = gc->bar0_va + mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); + gc->phys_db_page_base = gc->bar0_pa + + mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); + sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF); sriov_base_va = gc->bar0_va + sriov_base_off; @@ -1036,7 +1039,7 @@ static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req, header->inline_oob_size_div4 = client_oob_size / sizeof(u32); if (oob_in_sgl) { - WARN_ON_ONCE(!pad_data || wqe_req->num_sge < 2); + WARN_ON_ONCE(wqe_req->num_sge < 2); header->client_oob_in_sgl = 1; diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index f6aa5d6b6597e..252d8e6f18c3c 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -453,9 +453,158 @@ static u16 ocelot_vlan_unaware_pvid(struct ocelot *ocelot, return VLAN_N_VID - bridge_num - 1; } +/** + * ocelot_update_vlan_reclassify_rule() - Make switch aware only to bridge VLAN TPID + * + * @ocelot: Switch private data structure + * @port: Index of ingress port + * + * IEEE 802.1Q-2018 clauses "5.5 C-VLAN component conformance" and "5.6 S-VLAN + * component conformance" suggest that a C-VLAN component should only recognize + * and filter on C-Tags, and an S-VLAN component should only recognize and + * process based on C-Tags. + * + * In Linux, as per commit 1a0b20b25732 ("Merge branch 'bridge-next'"), C-VLAN + * components are largely represented by a bridge with vlan_protocol 802.1Q, + * and S-VLAN components by a bridge with vlan_protocol 802.1ad. + * + * Currently the driver only offloads vlan_protocol 802.1Q, but the hardware + * design is non-conformant, because the switch assigns each frame to a VLAN + * based on an entirely different question, as detailed in figure "Basic VLAN + * Classification Flow" from its manual and reproduced below. + * + * Set TAG_TYPE, PCP, DEI, VID to port-default values in VLAN_CFG register + * if VLAN_AWARE_ENA[port] and frame has outer tag then: + * if VLAN_INNER_TAG_ENA[port] and frame has inner tag then: + * TAG_TYPE = (Frame.InnerTPID <> 0x8100) + * Set PCP, DEI, VID to values from inner VLAN header + * else: + * TAG_TYPE = (Frame.OuterTPID <> 0x8100) + * Set PCP, DEI, VID to values from outer VLAN header + * if VID == 0 then: + * VID = VLAN_CFG.VLAN_VID + * + * Summarized, the switch will recognize both 802.1Q and 802.1ad TPIDs as VLAN + * "with equal rights", and just set the TAG_TYPE bit to 0 (if 802.1Q) or to 1 + * (if 802.1ad). It will classify based on whichever of the tags is "outer", no + * matter what TPID that may have (or "inner", if VLAN_INNER_TAG_ENA[port]). + * + * In the VLAN Table, the TAG_TYPE information is not accessible - just the + * classified VID is - so it is as if each VLAN Table entry is for 2 VLANs: + * C-VLAN X, and S-VLAN X. + * + * Whereas the Linux bridge behavior is to only filter on frames with a TPID + * equal to the vlan_protocol, and treat everything else as VLAN-untagged. + * + * Consider an ingress packet tagged with 802.1ad VID=3 and 802.1Q VID=5, + * received on a bridge vlan_filtering=1 vlan_protocol=802.1Q port. This frame + * should be treated as 802.1Q-untagged, and classified to the PVID of that + * bridge port. Not to VID=3, and not to VID=5. + * + * The VCAP IS1 TCAM has everything we need to overwrite the choices made in + * the basic VLAN classification pipeline: it can match on TAG_TYPE in the key, + * and it can modify the classified VID in the action. Thus, for each port + * under a vlan_filtering bridge, we can insert a rule in VCAP IS1 lookup 0 to + * match on 802.1ad tagged frames and modify their classified VID to the 802.1Q + * PVID of the port. This effectively makes it appear to the outside world as + * if those packets were processed as VLAN-untagged. + * + * The rule needs to be updated each time the bridge PVID changes, and needs + * to be deleted if the bridge PVID is deleted, or if the port becomes + * VLAN-unaware. + */ +static int ocelot_update_vlan_reclassify_rule(struct ocelot *ocelot, int port) +{ + unsigned long cookie = OCELOT_VCAP_IS1_VLAN_RECLASSIFY(ocelot, port); + struct ocelot_vcap_block *block_vcap_is1 = &ocelot->block[VCAP_IS1]; + struct ocelot_port *ocelot_port = ocelot->ports[port]; + const struct ocelot_bridge_vlan *pvid_vlan; + struct ocelot_vcap_filter *filter; + int err, val, pcp, dei; + bool vid_replace_ena; + u16 vid; + + pvid_vlan = ocelot_port->pvid_vlan; + vid_replace_ena = ocelot_port->vlan_aware && pvid_vlan; + + filter = ocelot_vcap_block_find_filter_by_id(block_vcap_is1, cookie, + false); + if (!vid_replace_ena) { + /* If the reclassification filter doesn't need to exist, delete + * it if it was previously installed, and exit doing nothing + * otherwise. + */ + if (filter) + return ocelot_vcap_filter_del(ocelot, filter); + + return 0; + } + + /* The reclassification rule must apply. See if it already exists + * or if it must be created. + */ + + /* Treating as VLAN-untagged means using as classified VID equal to + * the bridge PVID, and PCP/DEI set to the port default QoS values. + */ + vid = pvid_vlan->vid; + val = ocelot_read_gix(ocelot, ANA_PORT_QOS_CFG, port); + pcp = ANA_PORT_QOS_CFG_QOS_DEFAULT_VAL_X(val); + dei = !!(val & ANA_PORT_QOS_CFG_DP_DEFAULT_VAL); + + if (filter) { + bool changed = false; + + /* Filter exists, just update it */ + if (filter->action.vid != vid) { + filter->action.vid = vid; + changed = true; + } + if (filter->action.pcp != pcp) { + filter->action.pcp = pcp; + changed = true; + } + if (filter->action.dei != dei) { + filter->action.dei = dei; + changed = true; + } + + if (!changed) + return 0; + + return ocelot_vcap_filter_replace(ocelot, filter); + } + + /* Filter doesn't exist, create it */ + filter = kzalloc(sizeof(*filter), GFP_KERNEL); + if (!filter) + return -ENOMEM; + + filter->key_type = OCELOT_VCAP_KEY_ANY; + filter->ingress_port_mask = BIT(port); + filter->vlan.tpid = OCELOT_VCAP_BIT_1; + filter->prio = 1; + filter->id.cookie = cookie; + filter->id.tc_offload = false; + filter->block_id = VCAP_IS1; + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; + filter->lookup = 0; + filter->action.vid_replace_ena = true; + filter->action.pcp_dei_ena = true; + filter->action.vid = vid; + filter->action.pcp = pcp; + filter->action.dei = dei; + + err = ocelot_vcap_filter_add(ocelot, filter, NULL); + if (err) + kfree(filter); + + return err; +} + /* Default vlan to clasify for untagged frames (may be zero) */ -static void ocelot_port_set_pvid(struct ocelot *ocelot, int port, - const struct ocelot_bridge_vlan *pvid_vlan) +static int ocelot_port_set_pvid(struct ocelot *ocelot, int port, + const struct ocelot_bridge_vlan *pvid_vlan) { struct ocelot_port *ocelot_port = ocelot->ports[port]; u16 pvid = ocelot_vlan_unaware_pvid(ocelot, ocelot_port->bridge); @@ -475,15 +624,23 @@ static void ocelot_port_set_pvid(struct ocelot *ocelot, int port, * happens automatically), but also 802.1p traffic which gets * classified to VLAN 0, but that is always in our RX filter, so it * would get accepted were it not for this setting. + * + * Also, we only support the bridge 802.1Q VLAN protocol, so + * 802.1ad-tagged frames (carrying S-Tags) should be considered + * 802.1Q-untagged, and also dropped. */ if (!pvid_vlan && ocelot_port->vlan_aware) val = ANA_PORT_DROP_CFG_DROP_PRIO_S_TAGGED_ENA | - ANA_PORT_DROP_CFG_DROP_PRIO_C_TAGGED_ENA; + ANA_PORT_DROP_CFG_DROP_PRIO_C_TAGGED_ENA | + ANA_PORT_DROP_CFG_DROP_S_TAGGED_ENA; ocelot_rmw_gix(ocelot, val, ANA_PORT_DROP_CFG_DROP_PRIO_S_TAGGED_ENA | - ANA_PORT_DROP_CFG_DROP_PRIO_C_TAGGED_ENA, + ANA_PORT_DROP_CFG_DROP_PRIO_C_TAGGED_ENA | + ANA_PORT_DROP_CFG_DROP_S_TAGGED_ENA, ANA_PORT_DROP_CFG, port); + + return ocelot_update_vlan_reclassify_rule(ocelot, port); } static struct ocelot_bridge_vlan *ocelot_bridge_vlan_find(struct ocelot *ocelot, @@ -631,7 +788,10 @@ int ocelot_port_vlan_filtering(struct ocelot *ocelot, int port, ANA_PORT_VLAN_CFG_VLAN_POP_CNT_M, ANA_PORT_VLAN_CFG, port); - ocelot_port_set_pvid(ocelot, port, ocelot_port->pvid_vlan); + err = ocelot_port_set_pvid(ocelot, port, ocelot_port->pvid_vlan); + if (err) + return err; + ocelot_port_manage_port_tag(ocelot, port); return 0; @@ -670,6 +830,7 @@ EXPORT_SYMBOL(ocelot_vlan_prepare); int ocelot_vlan_add(struct ocelot *ocelot, int port, u16 vid, bool pvid, bool untagged) { + struct ocelot_port *ocelot_port = ocelot->ports[port]; int err; /* Ignore VID 0 added to our RX filter by the 8021q module, since @@ -684,9 +845,17 @@ int ocelot_vlan_add(struct ocelot *ocelot, int port, u16 vid, bool pvid, return err; /* Default ingress vlan classification */ - if (pvid) - ocelot_port_set_pvid(ocelot, port, - ocelot_bridge_vlan_find(ocelot, vid)); + if (pvid) { + err = ocelot_port_set_pvid(ocelot, port, + ocelot_bridge_vlan_find(ocelot, vid)); + if (err) + return err; + } else if (ocelot_port->pvid_vlan && + ocelot_bridge_vlan_find(ocelot, vid) == ocelot_port->pvid_vlan) { + err = ocelot_port_set_pvid(ocelot, port, NULL); + if (err) + return err; + } /* Untagged egress vlan clasification */ ocelot_port_manage_port_tag(ocelot, port); @@ -712,8 +881,11 @@ int ocelot_vlan_del(struct ocelot *ocelot, int port, u16 vid) return err; /* Ingress */ - if (del_pvid) - ocelot_port_set_pvid(ocelot, port, NULL); + if (del_pvid) { + err = ocelot_port_set_pvid(ocelot, port, NULL); + if (err) + return err; + } /* Egress */ ocelot_port_manage_port_tag(ocelot, port); @@ -2607,7 +2779,7 @@ int ocelot_port_set_default_prio(struct ocelot *ocelot, int port, u8 prio) ANA_PORT_QOS_CFG, port); - return 0; + return ocelot_update_vlan_reclassify_rule(ocelot, port); } EXPORT_SYMBOL_GPL(ocelot_port_set_default_prio); diff --git a/drivers/net/ethernet/mscc/ocelot_vcap.c b/drivers/net/ethernet/mscc/ocelot_vcap.c index 73cdec5ca6a34..5734b86aed5b5 100644 --- a/drivers/net/ethernet/mscc/ocelot_vcap.c +++ b/drivers/net/ethernet/mscc/ocelot_vcap.c @@ -695,6 +695,7 @@ static void is1_entry_set(struct ocelot *ocelot, int ix, vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_L2_MC, filter->dmac_mc); vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_L2_BC, filter->dmac_bc); vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_VLAN_TAGGED, tag->tagged); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TPID, tag->tpid); vcap_key_set(vcap, &data, VCAP_IS1_HK_VID, tag->vid.value, tag->vid.mask); vcap_key_set(vcap, &data, VCAP_IS1_HK_PCP, diff --git a/drivers/net/ethernet/natsemi/ns83820.c b/drivers/net/ethernet/natsemi/ns83820.c index 998586872599b..c692d2e878b2e 100644 --- a/drivers/net/ethernet/natsemi/ns83820.c +++ b/drivers/net/ethernet/natsemi/ns83820.c @@ -820,7 +820,7 @@ static void rx_irq(struct net_device *ndev) struct ns83820 *dev = PRIV(ndev); struct rx_info *info = &dev->rx_info; unsigned next_rx; - int rx_rc, len; + int len; u32 cmdsts; __le32 *desc; unsigned long flags; @@ -881,8 +881,10 @@ static void rx_irq(struct net_device *ndev) if (likely(CMDSTS_OK & cmdsts)) { #endif skb_put(skb, len); - if (unlikely(!skb)) + if (unlikely(!skb)) { + ndev->stats.rx_dropped++; goto netdev_mangle_me_harder_failed; + } if (cmdsts & CMDSTS_DEST_MULTI) ndev->stats.multicast++; ndev->stats.rx_packets++; @@ -901,15 +903,12 @@ static void rx_irq(struct net_device *ndev) __vlan_hwaccel_put_tag(skb, htons(ETH_P_IPV6), tag); } #endif - rx_rc = netif_rx(skb); - if (NET_RX_DROP == rx_rc) { -netdev_mangle_me_harder_failed: - ndev->stats.rx_dropped++; - } + netif_rx(skb); } else { dev_kfree_skb_irq(skb); } +netdev_mangle_me_harder_failed: nr++; next_rx = info->next_rx; desc = info->descs + (DESC_SIZE * next_rx); diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c index 5d9db8c2a5b43..45be6954d5aae 100644 --- a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c +++ b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c @@ -256,7 +256,7 @@ nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget, nfp_net_xsk_rx_ring_fill_freelist(r_vec->rx_ring); if (xdp_redir) - xdp_do_flush_map(); + xdp_do_flush(); if (tx_ring->wr_ptr_add) nfp_net_tx_xmit_more_flush(tx_ring); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index e75cbb287625f..87383f9d09ce3 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -1413,7 +1413,7 @@ static u32 nfp_net_get_rxfh_key_size(struct net_device *netdev) struct nfp_net *nn = netdev_priv(netdev); if (!(nn->cap & NFP_NET_CFG_CTRL_RSS_ANY)) - return -EOPNOTSUPP; + return 0; return nfp_net_rss_key_sz(nn); } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index bc7c5cd385969..1ac7a40fcc43e 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -3394,10 +3394,6 @@ void ionic_lif_free(struct ionic_lif *lif) lif->info = NULL; lif->info_pa = 0; - /* unmap doorbell page */ - ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage); - lif->kern_dbpage = NULL; - mutex_destroy(&lif->config_lock); mutex_destroy(&lif->queue_lock); @@ -3423,6 +3419,9 @@ void ionic_lif_deinit(struct ionic_lif *lif) ionic_lif_qcq_deinit(lif, lif->notifyqcq); ionic_lif_qcq_deinit(lif, lif->adminqcq); + ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage); + lif->kern_dbpage = NULL; + ionic_lif_reset(lif); } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c index 3ca6893d1bf26..2869922cffe2e 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_main.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c @@ -464,9 +464,9 @@ static int __ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds, unsigned long start_time; unsigned long max_wait; unsigned long duration; - int done = 0; bool fw_up; int opcode; + bool done; int err; /* Wait for dev cmd to complete, retrying if we get EAGAIN, @@ -474,6 +474,7 @@ static int __ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds, */ max_wait = jiffies + (max_seconds * HZ); try_again: + done = false; opcode = idev->opcode; start_time = jiffies; for (fw_up = ionic_is_fw_running(idev); diff --git a/drivers/net/ethernet/qlogic/qed/qed_debug.c b/drivers/net/ethernet/qlogic/qed/qed_debug.c index cdcead614e9fa..ae421c2707785 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_debug.c +++ b/drivers/net/ethernet/qlogic/qed/qed_debug.c @@ -4461,10 +4461,11 @@ static enum dbg_status qed_protection_override_dump(struct qed_hwfn *p_hwfn, goto out; } - /* Add override window info to buffer */ + /* Add override window info to buffer, preventing buffer overflow */ override_window_dwords = - qed_rd(p_hwfn, p_ptt, GRC_REG_NUMBER_VALID_OVERRIDE_WINDOW) * - PROTECTION_OVERRIDE_ELEMENT_DWORDS; + min(qed_rd(p_hwfn, p_ptt, GRC_REG_NUMBER_VALID_OVERRIDE_WINDOW) * + PROTECTION_OVERRIDE_ELEMENT_DWORDS, + PROTECTION_OVERRIDE_DEPTH_DWORDS); if (override_window_dwords) { addr = BYTES_TO_DWORDS(GRC_REG_PROTECTION_OVERRIDE_WINDOW); offset += qed_grc_dump_addr_range(p_hwfn, diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index cb1746bc0e0c5..273dae622c411 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -4,6 +4,7 @@ * Copyright (c) 2019-2020 Marvell International Ltd. */ +#include #include #include #include @@ -960,7 +961,7 @@ static inline void qede_tpa_cont(struct qede_dev *edev, { int i; - for (i = 0; cqe->len_list[i]; i++) + for (i = 0; cqe->len_list[i] && i < ARRAY_SIZE(cqe->len_list); i++) qede_fill_frag_skb(edev, rxq, cqe->tpa_agg_index, le16_to_cpu(cqe->len_list[i])); @@ -985,7 +986,7 @@ static int qede_tpa_end(struct qede_dev *edev, dma_unmap_page(rxq->dev, tpa_info->buffer.mapping, PAGE_SIZE, rxq->data_direction); - for (i = 0; cqe->len_list[i]; i++) + for (i = 0; cqe->len_list[i] && i < ARRAY_SIZE(cqe->len_list); i++) qede_fill_frag_skb(edev, rxq, cqe->tpa_agg_index, le16_to_cpu(cqe->len_list[i])); if (unlikely(i > 1)) diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 99df00c30b8c6..b5d744d2586f7 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -203,7 +203,7 @@ static struct pci_driver qede_pci_driver = { }; static struct qed_eth_cb_ops qede_ll_ops = { - { + .common = { #ifdef CONFIG_RFS_ACCEL .arfs_filter_op = qede_arfs_filter_op, #endif diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c index 28d24d59efb84..d57b976b90409 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c @@ -1484,8 +1484,11 @@ static int qlcnic_sriov_channel_cfg_cmd(struct qlcnic_adapter *adapter, u8 cmd_o } cmd_op = (cmd.rsp.arg[0] & 0xff); - if (cmd.rsp.arg[0] >> 25 == 2) - return 2; + if (cmd.rsp.arg[0] >> 25 == 2) { + ret = 2; + goto out; + } + if (cmd_op == QLCNIC_BC_CMD_CHANNEL_INIT) set_bit(QLC_BC_VF_STATE, &vf->state); else diff --git a/drivers/net/ethernet/realtek/Kconfig b/drivers/net/ethernet/realtek/Kconfig index 93d9df55b361a..01811924c4db4 100644 --- a/drivers/net/ethernet/realtek/Kconfig +++ b/drivers/net/ethernet/realtek/Kconfig @@ -58,7 +58,7 @@ config 8139TOO config 8139TOO_PIO bool "Use PIO instead of MMIO" default y - depends on 8139TOO + depends on 8139TOO && !NO_IOPORT_MAP help This instructs the driver to use programmed I/O ports (PIO) instead of PCI shared memory (MMIO). This can possibly solve some problems diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 7e5258b2c4290..f4353ccb1b87f 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -3362,7 +3362,7 @@ static void rtl_hw_start_8168h_1(struct rtl8169_private *tp) r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini); } - r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070); + r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0000); r8168_mac_ocp_modify(tp, 0xe052, 0x6000, 0x8008); r8168_mac_ocp_modify(tp, 0xe0d6, 0x01ff, 0x017f); r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f); @@ -3467,7 +3467,7 @@ static void rtl_hw_start_8117(struct rtl8169_private *tp) r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini); } - r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070); + r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0000); r8168_mac_ocp_write(tp, 0xea80, 0x0003); r8168_mac_ocp_modify(tp, 0xe052, 0x0000, 0x0009); r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f); @@ -3660,7 +3660,7 @@ static void rtl_hw_start_8125_common(struct rtl8169_private *tp) r8168_mac_ocp_modify(tp, 0xc0b4, 0x0000, 0x000c); r8168_mac_ocp_modify(tp, 0xeb6a, 0x00ff, 0x0033); r8168_mac_ocp_modify(tp, 0xeb50, 0x03e0, 0x0040); - r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0030); + r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0000); r8168_mac_ocp_modify(tp, 0xe040, 0x1000, 0x0000); r8168_mac_ocp_modify(tp, 0xea1c, 0x0003, 0x0001); r8168_mac_ocp_modify(tp, 0xe0c0, 0x4f0f, 0x4403); @@ -4919,8 +4919,9 @@ static int rtl8169_resume(struct device *device) if (!device_may_wakeup(tp_to_dev(tp))) clk_prepare_enable(tp->clk); - /* Reportedly at least Asus X453MA truncates packets otherwise */ - if (tp->mac_version == RTL_GIGA_MAC_VER_37) + /* Some chip versions may truncate packets without this initialization */ + if (tp->mac_version == RTL_GIGA_MAC_VER_37 || + tp->mac_version == RTL_GIGA_MAC_VER_46) rtl_init_rxcfg(tp); return rtl8169_runtime_resume(device); @@ -5125,6 +5126,7 @@ static int r8169_mdio_register(struct rtl8169_private *tp) new_bus->priv = tp; new_bus->parent = &pdev->dev; new_bus->irq[0] = PHY_MAC_INTERRUPT; + new_bus->phy_mask = GENMASK(31, 1); snprintf(new_bus->id, MII_BUS_ID_SIZE, "r8169-%x-%x", pci_domain_nr(pdev->bus), pci_dev_id(pdev)); diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 58fdc4f8dd483..085d81576f1a4 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -2038,15 +2038,35 @@ static netdev_tx_t ravb_start_xmit(struct sk_buff *skb, struct net_device *ndev) skb_tx_timestamp(skb); } - /* Descriptor type must be set after all the above writes */ - dma_wmb(); + if (num_tx_desc > 1) { desc->die_dt = DT_FEND; desc--; + /* When using multi-descriptors, DT_FEND needs to get written + * before DT_FSTART, but the compiler may reorder the memory + * writes in an attempt to optimize the code. + * Use a dma_wmb() barrier to make sure DT_FEND and DT_FSTART + * are written exactly in the order shown in the code. + * This is particularly important for cases where the DMA engine + * is already running when we are running this code. If the DMA + * sees DT_FSTART without the corresponding DT_FEND it will enter + * an error condition. + */ + dma_wmb(); desc->die_dt = DT_FSTART; } else { + /* Descriptor type must be set after all the above writes */ + dma_wmb(); desc->die_dt = DT_FSINGLE; } + + /* Before ringing the doorbell we need to make sure that the latest + * writes have been committed to memory, otherwise it could delay + * things until the doorbell is rang again. + * This is in replacement of the read operation mentioned in the HW + * manuals. + */ + dma_wmb(); ravb_modify(ndev, TCCR, TCCR_TSRQ0 << q, TCCR_TSRQ0 << q); priv->cur_tx[q] += num_tx_desc; diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c index 0c0fd68ded423..4597c4e9b297b 100644 --- a/drivers/net/ethernet/renesas/sh_eth.c +++ b/drivers/net/ethernet/renesas/sh_eth.c @@ -2360,6 +2360,7 @@ static int sh_eth_set_ringparam(struct net_device *ndev, return 0; } +#ifdef CONFIG_PM_SLEEP static void sh_eth_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) { struct sh_eth_private *mdp = netdev_priv(ndev); @@ -2386,6 +2387,7 @@ static int sh_eth_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) return 0; } +#endif static const struct ethtool_ops sh_eth_ethtool_ops = { .get_regs_len = sh_eth_get_regs_len, @@ -2401,8 +2403,10 @@ static const struct ethtool_ops sh_eth_ethtool_ops = { .set_ringparam = sh_eth_set_ringparam, .get_link_ksettings = phy_ethtool_get_link_ksettings, .set_link_ksettings = phy_ethtool_set_link_ksettings, +#ifdef CONFIG_PM_SLEEP .get_wol = sh_eth_get_wol, .set_wol = sh_eth_set_wol, +#endif }; /* network device open function */ diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index 8d2d7ea2ebefc..c9e17a8208a90 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -1260,7 +1260,7 @@ static int efx_poll(struct napi_struct *napi, int budget) spent = efx_process_channel(channel, budget); - xdp_do_flush_map(); + xdp_do_flush(); if (spent < budget) { if (efx_channel_has_rx_queue(channel) && diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c index c3e2b4a21d105..3b08e36e1ef87 100644 --- a/drivers/net/ethernet/sfc/mae.c +++ b/drivers/net/ethernet/sfc/mae.c @@ -1101,6 +1101,9 @@ void efx_mae_remove_mport(void *desc, void *arg) kfree(mport); } +/* + * Takes ownership of @desc, even if it returns an error + */ static int efx_mae_process_mport(struct efx_nic *efx, struct mae_mport_desc *desc) { @@ -1111,6 +1114,7 @@ static int efx_mae_process_mport(struct efx_nic *efx, if (!IS_ERR_OR_NULL(mport)) { netif_err(efx, drv, efx->net_dev, "mport with id %u does exist!!!\n", desc->mport_id); + kfree(desc); return -EEXIST; } diff --git a/drivers/net/ethernet/sfc/siena/efx_channels.c b/drivers/net/ethernet/sfc/siena/efx_channels.c index 1776f7f8a7a90..a7346e965bfe7 100644 --- a/drivers/net/ethernet/sfc/siena/efx_channels.c +++ b/drivers/net/ethernet/sfc/siena/efx_channels.c @@ -1285,7 +1285,7 @@ static int efx_poll(struct napi_struct *napi, int budget) spent = efx_process_channel(channel, budget); - xdp_do_flush_map(); + xdp_do_flush(); if (spent < budget) { if (efx_channel_has_rx_queue(channel) && diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c index cb590db625e83..90a8eb517033e 100644 --- a/drivers/net/ethernet/smsc/smsc911x.c +++ b/drivers/net/ethernet/smsc/smsc911x.c @@ -2162,10 +2162,20 @@ static const struct net_device_ops smsc911x_netdev_ops = { static void smsc911x_read_mac_address(struct net_device *dev) { struct smsc911x_data *pdata = netdev_priv(dev); - u32 mac_high16 = smsc911x_mac_read(pdata, ADDRH); - u32 mac_low32 = smsc911x_mac_read(pdata, ADDRL); + u32 mac_high16, mac_low32; u8 addr[ETH_ALEN]; + mac_high16 = smsc911x_mac_read(pdata, ADDRH); + mac_low32 = smsc911x_mac_read(pdata, ADDRL); + + /* The first mac_read in some setups can incorrectly read 0. Re-read it + * to get the full MAC if this is observed. + */ + if (mac_high16 == 0) { + SMSC_TRACE(pdata, probe, "Re-read MAC ADDRH\n"); + mac_high16 = smsc911x_mac_read(pdata, ADDRH); + } + addr[0] = (u8)(mac_low32); addr[1] = (u8)(mac_low32 >> 8); addr[2] = (u8)(mac_low32 >> 16); diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index f358ea0031936..b834b129639f0 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -780,7 +780,7 @@ static void netsec_finalize_xdp_rx(struct netsec_priv *priv, u32 xdp_res, u16 pkts) { if (xdp_res & NETSEC_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); if (xdp_res & NETSEC_XDP_TX) netsec_xdp_ring_tx_db(priv, pkts); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index d920a50dd16c7..bab315517bad9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -1565,14 +1565,15 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable) } } else { if (bsp_priv->clk_enabled) { + if (bsp_priv->ops && bsp_priv->ops->set_clock_selection) { + bsp_priv->ops->set_clock_selection(bsp_priv, + bsp_priv->clock_input, false); + } + clk_bulk_disable_unprepare(bsp_priv->num_clks, bsp_priv->clks); clk_disable_unprepare(bsp_priv->clk_phy); - if (bsp_priv->ops && bsp_priv->ops->set_clock_selection) - bsp_priv->ops->set_clock_selection(bsp_priv, - bsp_priv->clock_input, false); - bsp_priv->clk_enabled = false; } } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c index 63998d65fef8e..9377b05bfc71e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c @@ -966,7 +966,7 @@ static int sun8i_dwmac_set_syscon(struct device *dev, /* of_mdio_parse_addr returns a valid (0 ~ 31) PHY * address. No need to mask it again. */ - reg |= 1 << H3_EPHY_ADDR_SHIFT; + reg |= ret << H3_EPHY_ADDR_SHIFT; } else { /* For SoCs without internal PHY the PHY selection bit should be * set to 0 (external PHY). diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index a9837985a483d..bdb4f527289d2 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -69,7 +69,7 @@ static void dwmac4_core_init(struct mac_device_info *hw, init_waitqueue_head(&priv->tstamp_busy_wait); } -static void dwmac4_phylink_get_caps(struct stmmac_priv *priv) +static void dwmac4_update_caps(struct stmmac_priv *priv) { if (priv->plat->tx_queues_to_use > 1) priv->hw->link.caps &= ~(MAC_10HD | MAC_100HD | MAC_1000HD); @@ -1161,7 +1161,7 @@ static int dwmac4_config_l4_filter(struct mac_device_info *hw, u32 filter_no, const struct stmmac_ops dwmac4_ops = { .core_init = dwmac4_core_init, - .phylink_get_caps = dwmac4_phylink_get_caps, + .update_caps = dwmac4_update_caps, .set_mac = stmmac_set_mac, .rx_ipc = dwmac4_rx_ipc_enable, .rx_queue_enable = dwmac4_rx_queue_enable, @@ -1204,7 +1204,7 @@ const struct stmmac_ops dwmac4_ops = { const struct stmmac_ops dwmac410_ops = { .core_init = dwmac4_core_init, - .phylink_get_caps = dwmac4_phylink_get_caps, + .update_caps = dwmac4_update_caps, .set_mac = stmmac_dwmac4_set_mac, .rx_ipc = dwmac4_rx_ipc_enable, .rx_queue_enable = dwmac4_rx_queue_enable, @@ -1253,7 +1253,7 @@ const struct stmmac_ops dwmac410_ops = { const struct stmmac_ops dwmac510_ops = { .core_init = dwmac4_core_init, - .phylink_get_caps = dwmac4_phylink_get_caps, + .update_caps = dwmac4_update_caps, .set_mac = stmmac_dwmac4_set_mac, .rx_ipc = dwmac4_rx_ipc_enable, .rx_queue_enable = dwmac4_rx_queue_enable, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index 052566f5b7f36..0bcb378fa0bc9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -47,6 +47,14 @@ static void dwxgmac2_core_init(struct mac_device_info *hw, writel(XGMAC_INT_DEFAULT_EN, ioaddr + XGMAC_INT_EN); } +static void dwxgmac2_update_caps(struct stmmac_priv *priv) +{ + if (!priv->dma_cap.mbps_10_100) + priv->hw->link.caps &= ~(MAC_10 | MAC_100); + else if (!priv->dma_cap.half_duplex) + priv->hw->link.caps &= ~(MAC_10HD | MAC_100HD); +} + static void dwxgmac2_set_mac(void __iomem *ioaddr, bool enable) { u32 tx = readl(ioaddr + XGMAC_TX_CONFIG); @@ -1583,6 +1591,7 @@ static void dwxgmac3_fpe_configure(void __iomem *ioaddr, struct stmmac_fpe_cfg * const struct stmmac_ops dwxgmac210_ops = { .core_init = dwxgmac2_core_init, + .update_caps = dwxgmac2_update_caps, .set_mac = dwxgmac2_set_mac, .rx_ipc = dwxgmac2_rx_ipc, .rx_queue_enable = dwxgmac2_rx_queue_enable, @@ -1705,8 +1714,8 @@ int dwxgmac2_setup(struct stmmac_priv *priv) mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins); mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | - MAC_1000FD | MAC_2500FD | MAC_5000FD | - MAC_10000FD; + MAC_10 | MAC_100 | MAC_1000FD | + MAC_2500FD | MAC_5000FD | MAC_10000FD; mac->link.duplex = 0; mac->link.speed10 = XGMAC_CONFIG_SS_10_MII; mac->link.speed100 = XGMAC_CONFIG_SS_100_MII; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c index dd2ab6185c40e..b2c03cb65c7cc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c @@ -203,10 +203,6 @@ static void dwxgmac2_dma_rx_mode(struct stmmac_priv *priv, void __iomem *ioaddr, } writel(value, ioaddr + XGMAC_MTL_RXQ_OPMODE(channel)); - - /* Enable MTL RX overflow */ - value = readl(ioaddr + XGMAC_MTL_QINTEN(channel)); - writel(value | XGMAC_RXOIE, ioaddr + XGMAC_MTL_QINTEN(channel)); } static void dwxgmac2_dma_tx_mode(struct stmmac_priv *priv, void __iomem *ioaddr, @@ -364,19 +360,17 @@ static int dwxgmac2_dma_interrupt(struct stmmac_priv *priv, } /* TX/RX NORMAL interrupts */ - if (likely(intr_status & XGMAC_NIS)) { - if (likely(intr_status & XGMAC_RI)) { - u64_stats_update_begin(&stats->syncp); - u64_stats_inc(&stats->rx_normal_irq_n[chan]); - u64_stats_update_end(&stats->syncp); - ret |= handle_rx; - } - if (likely(intr_status & (XGMAC_TI | XGMAC_TBU))) { - u64_stats_update_begin(&stats->syncp); - u64_stats_inc(&stats->tx_normal_irq_n[chan]); - u64_stats_update_end(&stats->syncp); - ret |= handle_tx; - } + if (likely(intr_status & XGMAC_RI)) { + u64_stats_update_begin(&stats->syncp); + u64_stats_inc(&stats->rx_normal_irq_n[chan]); + u64_stats_update_end(&stats->syncp); + ret |= handle_rx; + } + if (likely(intr_status & (XGMAC_TI | XGMAC_TBU))) { + u64_stats_update_begin(&stats->syncp); + u64_stats_inc(&stats->tx_normal_irq_n[chan]); + u64_stats_update_end(&stats->syncp); + ret |= handle_tx; } /* Clear interrupts */ @@ -388,8 +382,11 @@ static int dwxgmac2_dma_interrupt(struct stmmac_priv *priv, static int dwxgmac2_get_hw_feature(void __iomem *ioaddr, struct dma_features *dma_cap) { + struct stmmac_priv *priv; u32 hw_cap; + priv = container_of(dma_cap, struct stmmac_priv, dma_cap); + /* MAC HW feature 0 */ hw_cap = readl(ioaddr + XGMAC_HW_FEATURE0); dma_cap->edma = (hw_cap & XGMAC_HWFEAT_EDMA) >> 31; @@ -412,6 +409,8 @@ static int dwxgmac2_get_hw_feature(void __iomem *ioaddr, dma_cap->vlhash = (hw_cap & XGMAC_HWFEAT_VLHASH) >> 4; dma_cap->half_duplex = (hw_cap & XGMAC_HWFEAT_HDSEL) >> 3; dma_cap->mbps_1000 = (hw_cap & XGMAC_HWFEAT_GMIISEL) >> 1; + if (dma_cap->mbps_1000 && priv->synopsys_id >= DWXGMAC_CORE_2_20) + dma_cap->mbps_10_100 = 1; /* MAC HW feature 1 */ hw_cap = readl(ioaddr + XGMAC_HW_FEATURE1); diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index 47fb8e1646c2e..ee9a7d98648b0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -300,8 +300,8 @@ struct stmmac_est; struct stmmac_ops { /* MAC core initialization */ void (*core_init)(struct mac_device_info *hw, struct net_device *dev); - /* Get phylink capabilities */ - void (*phylink_get_caps)(struct stmmac_priv *priv); + /* Update MAC capabilities */ + void (*update_caps)(struct stmmac_priv *priv); /* Enable the MAC RX/TX */ void (*set_mac)(void __iomem *ioaddr, bool enable); /* Enable and verify that the IPC module is supported */ @@ -423,8 +423,8 @@ struct stmmac_ops { #define stmmac_core_init(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, core_init, __args) -#define stmmac_mac_phylink_get_caps(__priv) \ - stmmac_do_void_callback(__priv, mac, phylink_get_caps, __priv) +#define stmmac_mac_update_caps(__priv) \ + stmmac_do_void_callback(__priv, mac, update_caps, __priv) #define stmmac_mac_set(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, set_mac, __args) #define stmmac_rx_ipc(__priv, __args...) \ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index d3d5c01f6dcba..56a61599d0b6f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -842,6 +842,11 @@ int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) return -EOPNOTSUPP; + if (!priv->plat->clk_ptp_rate) { + netdev_err(priv->dev, "Invalid PTP clock rate"); + return -EINVAL; + } + stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); priv->systime_flags = systime_flags; @@ -1225,8 +1230,8 @@ static int stmmac_phy_setup(struct stmmac_priv *priv) xpcs_get_interfaces(priv->hw->xpcs, priv->phylink_config.supported_interfaces); - /* Get the MAC specific capabilities */ - stmmac_mac_phylink_get_caps(priv); + /* Refresh the MAC-specific capabilities */ + stmmac_mac_update_caps(priv); priv->phylink_config.mac_capabilities = priv->hw->link.caps; @@ -2421,6 +2426,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; + bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; struct xsk_buff_pool *pool = tx_q->xsk_pool; unsigned int entry = tx_q->cur_tx; struct dma_desc *tx_desc = NULL; @@ -2491,7 +2497,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) } stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, - true, priv->mode, true, true, + csum, priv->mode, true, true, xdp_desc.len); stmmac_enable_dma_transmission(priv, priv->ioaddr); @@ -2835,7 +2841,7 @@ static void stmmac_dma_interrupt(struct stmmac_priv *priv) u32 channels_to_check = tx_channel_count > rx_channel_count ? tx_channel_count : rx_channel_count; u32 chan; - int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; + int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; /* Make sure we never check beyond our status buffer. */ if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) @@ -3510,7 +3516,6 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); enum request_irq_err irq_err; - cpumask_t cpu_mask; int irq_idx = 0; char *int_name; int ret; @@ -3622,9 +3627,8 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) irq_idx = i; goto irq_error; } - cpumask_clear(&cpu_mask); - cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); - irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); + irq_set_affinity_hint(priv->rx_irq[i], + cpumask_of(i % num_online_cpus())); } /* Request Tx MSI irq */ @@ -3647,9 +3651,8 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) irq_idx = i; goto irq_error; } - cpumask_clear(&cpu_mask); - cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); - irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); + irq_set_affinity_hint(priv->tx_irq[i], + cpumask_of(i % num_online_cpus())); } return 0; @@ -4784,6 +4787,7 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, { struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; + bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; unsigned int entry = tx_q->cur_tx; struct dma_desc *tx_desc; dma_addr_t dma_addr; @@ -4828,7 +4832,7 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, stmmac_set_desc_addr(priv, tx_desc, dma_addr); stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, - true, priv->mode, true, true, + csum, priv->mode, true, true, xdpf->len); tx_q->tx_count_frames++; @@ -5520,7 +5524,8 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) stmmac_rx_vlan(priv->dev, skb); skb->protocol = eth_type_trans(skb, priv->dev); - if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) + if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) || + (status & csum_none)) skb_checksum_none_assert(skb); else skb->ip_summed = CHECKSUM_UNNECESSARY; @@ -7227,7 +7232,7 @@ int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) priv->rss.table[i] = ethtool_rxfh_indir_default(i, rx_cnt); - stmmac_mac_phylink_get_caps(priv); + stmmac_mac_update_caps(priv); priv->phylink_config.mac_capabilities = priv->hw->link.caps; @@ -7894,7 +7899,14 @@ int stmmac_resume(struct device *dev) stmmac_free_tx_skbufs(priv); stmmac_clear_descriptors(priv, &priv->dma_conf); - stmmac_hw_setup(ndev, false); + ret = stmmac_hw_setup(ndev, false); + if (ret < 0) { + netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); + mutex_unlock(&priv->lock); + rtnl_unlock(); + return ret; + } + stmmac_init_coalesce(priv); stmmac_set_rx_mode(ndev); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 4d570efd9d4bb..6c684f6ee84be 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -419,6 +419,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) struct device_node *np = pdev->dev.of_node; struct plat_stmmacenet_data *plat; struct stmmac_dma_cfg *dma_cfg; + static int bus_id = -ENODEV; int phy_mode; void *ret; int rc; @@ -454,8 +455,14 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) of_property_read_u32(np, "max-speed", &plat->max_speed); plat->bus_id = of_alias_get_id(np, "ethernet"); - if (plat->bus_id < 0) - plat->bus_id = 0; + if (plat->bus_id < 0) { + if (bus_id < 0) + bus_id = of_alias_get_highest_id("ethernet"); + /* No ethernet alias found, init at -1 so first bus_id is 0 */ + if (bus_id < 0) + bus_id = -1; + plat->bus_id = ++bus_id; + } /* Default to phy auto-detection */ plat->phy_addr = -1; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index a04bb2e42c4ee..80ecbd73333d9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -295,7 +295,7 @@ void stmmac_ptp_register(struct stmmac_priv *priv) /* Calculate the clock domain crossing (CDC) error if necessary */ priv->plat->cdc_error_adj = 0; - if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) + if (priv->plat->has_gmac4) priv->plat->cdc_error_adj = (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; stmmac_ptp_clock_ops.n_per_out = priv->dma_cap.pps_out_num; diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c index 011d74087f860..fc6217917fc22 100644 --- a/drivers/net/ethernet/sun/niu.c +++ b/drivers/net/ethernet/sun/niu.c @@ -3336,7 +3336,7 @@ static int niu_rbr_add_page(struct niu *np, struct rx_ring_info *rp, addr = np->ops->map_page(np->device, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); - if (!addr) { + if (np->ops->mapping_error(np->device, addr)) { __free_page(page); return -ENOMEM; } @@ -6672,6 +6672,8 @@ static netdev_tx_t niu_start_xmit(struct sk_buff *skb, len = skb_headlen(skb); mapping = np->ops->map_single(np->device, skb->data, len, DMA_TO_DEVICE); + if (np->ops->mapping_error(np->device, mapping)) + goto out_drop; prod = rp->prod; @@ -6713,6 +6715,8 @@ static netdev_tx_t niu_start_xmit(struct sk_buff *skb, mapping = np->ops->map_page(np->device, skb_frag_page(frag), skb_frag_off(frag), len, DMA_TO_DEVICE); + if (np->ops->mapping_error(np->device, mapping)) + goto out_unmap; rp->tx_buffs[prod].skb = NULL; rp->tx_buffs[prod].mapping = mapping; @@ -6737,6 +6741,19 @@ static netdev_tx_t niu_start_xmit(struct sk_buff *skb, out: return NETDEV_TX_OK; +out_unmap: + while (i--) { + const skb_frag_t *frag; + + prod = PREVIOUS_TX(rp, prod); + frag = &skb_shinfo(skb)->frags[i]; + np->ops->unmap_page(np->device, rp->tx_buffs[prod].mapping, + skb_frag_size(frag), DMA_TO_DEVICE); + } + + np->ops->unmap_single(np->device, rp->tx_buffs[rp->prod].mapping, + skb_headlen(skb), DMA_TO_DEVICE); + out_drop: rp->tx_errors++; kfree_skb(skb); @@ -9636,6 +9653,11 @@ static void niu_pci_unmap_single(struct device *dev, u64 dma_address, dma_unmap_single(dev, dma_address, size, direction); } +static int niu_pci_mapping_error(struct device *dev, u64 addr) +{ + return dma_mapping_error(dev, addr); +} + static const struct niu_ops niu_pci_ops = { .alloc_coherent = niu_pci_alloc_coherent, .free_coherent = niu_pci_free_coherent, @@ -9643,6 +9665,7 @@ static const struct niu_ops niu_pci_ops = { .unmap_page = niu_pci_unmap_page, .map_single = niu_pci_map_single, .unmap_single = niu_pci_unmap_single, + .mapping_error = niu_pci_mapping_error, }; static void niu_driver_version(void) @@ -10009,6 +10032,11 @@ static void niu_phys_unmap_single(struct device *dev, u64 dma_address, /* Nothing to do. */ } +static int niu_phys_mapping_error(struct device *dev, u64 dma_address) +{ + return false; +} + static const struct niu_ops niu_phys_ops = { .alloc_coherent = niu_phys_alloc_coherent, .free_coherent = niu_phys_free_coherent, @@ -10016,6 +10044,7 @@ static const struct niu_ops niu_phys_ops = { .unmap_page = niu_phys_unmap_page, .map_single = niu_phys_map_single, .unmap_single = niu_phys_unmap_single, + .mapping_error = niu_phys_mapping_error, }; static int niu_of_probe(struct platform_device *op) diff --git a/drivers/net/ethernet/sun/niu.h b/drivers/net/ethernet/sun/niu.h index 04c215f91fc08..0b169c08b0f2d 100644 --- a/drivers/net/ethernet/sun/niu.h +++ b/drivers/net/ethernet/sun/niu.h @@ -2879,6 +2879,9 @@ struct tx_ring_info { #define NEXT_TX(tp, index) \ (((index) + 1) < (tp)->pending ? ((index) + 1) : 0) +#define PREVIOUS_TX(tp, index) \ + (((index) - 1) >= 0 ? ((index) - 1) : (((tp)->pending) - 1)) + static inline u32 niu_tx_avail(struct tx_ring_info *tp) { return (tp->pending - @@ -3140,6 +3143,7 @@ struct niu_ops { enum dma_data_direction direction); void (*unmap_single)(struct device *dev, u64 dma_address, size_t size, enum dma_data_direction direction); + int (*mapping_error)(struct device *dev, u64 dma_address); }; struct niu_link_config { diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 9c8376b271891..28cc23736a69b 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -2089,13 +2089,15 @@ static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common) goto of_node_put; ret = of_get_mac_address(port_np, port->slave.mac_addr); - if (ret) { + if (ret == -EPROBE_DEFER) { + goto of_node_put; + } else if (ret) { am65_cpsw_am654_get_efuse_macid(port_np, port->port_id, port->slave.mac_addr); if (!is_valid_ether_addr(port->slave.mac_addr)) { eth_random_addr(port->slave.mac_addr); - dev_err(dev, "Use random MAC address\n"); + dev_info(dev, "Use random MAC address\n"); } } } @@ -2949,6 +2951,16 @@ static int am65_cpsw_nuss_probe(struct platform_device *pdev) return ret; } + am65_cpsw_nuss_get_ver(common); + + ret = am65_cpsw_nuss_init_host_p(common); + if (ret) + goto err_pm_clear; + + ret = am65_cpsw_nuss_init_slave_ports(common); + if (ret) + goto err_pm_clear; + node = of_get_child_by_name(dev->of_node, "mdio"); if (!node) { dev_warn(dev, "MDIO node not found\n"); @@ -2965,16 +2977,6 @@ static int am65_cpsw_nuss_probe(struct platform_device *pdev) } of_node_put(node); - am65_cpsw_nuss_get_ver(common); - - ret = am65_cpsw_nuss_init_host_p(common); - if (ret) - goto err_of_clear; - - ret = am65_cpsw_nuss_init_slave_ports(common); - if (ret) - goto err_of_clear; - /* init common data */ ale_params.dev = dev; ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT; diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 9061dca97fcbf..1c1d4806c119b 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -1416,6 +1416,7 @@ static int cpsw_create_ports(struct cpsw_common *cpsw) ndev->netdev_ops = &cpsw_netdev_ops; ndev->ethtool_ops = &cpsw_ethtool_ops; SET_NETDEV_DEV(ndev, dev); + ndev->dev.of_node = slave_data->slave_node; if (!napi_ndev) { /* CPSW Host port CPDMA interface is shared between diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index 0ec85635dfd60..764ed298b5708 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -1360,7 +1360,7 @@ int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp, * particular hardware is sharing a common queue, so the * incoming device might change per packet. */ - xdp_do_flush_map(); + xdp_do_flush(); break; default: bpf_warn_invalid_xdp_action(ndev, prog, act); diff --git a/drivers/net/ethernet/ti/icssg/icss_iep.c b/drivers/net/ethernet/ti/icssg/icss_iep.c index f3315c6515156..e7306ed529226 100644 --- a/drivers/net/ethernet/ti/icssg/icss_iep.c +++ b/drivers/net/ethernet/ti/icssg/icss_iep.c @@ -638,7 +638,8 @@ static int icss_iep_pps_enable(struct icss_iep *iep, int on) static int icss_iep_extts_enable(struct icss_iep *iep, u32 index, int on) { - u32 val, cap, ret = 0; + u32 val, cap; + int ret = 0; mutex_lock(&iep->ptp_clk_mutex); @@ -702,10 +703,16 @@ struct icss_iep *icss_iep_get_idx(struct device_node *np, int idx) struct platform_device *pdev; struct device_node *iep_np; struct icss_iep *iep; + int ret; iep_np = of_parse_phandle(np, "ti,iep", idx); - if (!iep_np || !of_device_is_available(iep_np)) + if (!iep_np) + return ERR_PTR(-ENODEV); + + if (!of_device_is_available(iep_np)) { + of_node_put(iep_np); return ERR_PTR(-ENODEV); + } pdev = of_find_device_by_node(iep_np); of_node_put(iep_np); @@ -715,21 +722,28 @@ struct icss_iep *icss_iep_get_idx(struct device_node *np, int idx) return ERR_PTR(-EPROBE_DEFER); iep = platform_get_drvdata(pdev); - if (!iep) - return ERR_PTR(-EPROBE_DEFER); + if (!iep) { + ret = -EPROBE_DEFER; + goto err_put_pdev; + } device_lock(iep->dev); if (iep->client_np) { device_unlock(iep->dev); dev_err(iep->dev, "IEP is already acquired by %s", iep->client_np->name); - return ERR_PTR(-EBUSY); + ret = -EBUSY; + goto err_put_pdev; } iep->client_np = np; device_unlock(iep->dev); - get_device(iep->dev); return iep; + +err_put_pdev: + put_device(&pdev->dev); + + return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(icss_iep_get_idx); diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c index d829113c16eee..1c33a9c9ddb5a 100644 --- a/drivers/net/ethernet/ti/netcp_core.c +++ b/drivers/net/ethernet/ti/netcp_core.c @@ -1338,10 +1338,10 @@ int netcp_txpipe_open(struct netcp_tx_pipe *tx_pipe) tx_pipe->dma_channel = knav_dma_open_channel(dev, tx_pipe->dma_chan_name, &config); - if (IS_ERR(tx_pipe->dma_channel)) { + if (!tx_pipe->dma_channel) { dev_err(dev, "failed opening tx chan(%s)\n", tx_pipe->dma_chan_name); - ret = PTR_ERR(tx_pipe->dma_channel); + ret = -EINVAL; goto err; } @@ -1359,7 +1359,7 @@ int netcp_txpipe_open(struct netcp_tx_pipe *tx_pipe) return 0; err: - if (!IS_ERR_OR_NULL(tx_pipe->dma_channel)) + if (tx_pipe->dma_channel) knav_dma_close_channel(tx_pipe->dma_channel); tx_pipe->dma_channel = NULL; return ret; @@ -1678,10 +1678,10 @@ static int netcp_setup_navigator_resources(struct net_device *ndev) netcp->rx_channel = knav_dma_open_channel(netcp->netcp_device->device, netcp->dma_chan_name, &config); - if (IS_ERR(netcp->rx_channel)) { + if (!netcp->rx_channel) { dev_err(netcp->ndev_dev, "failed opening rx chan(%s\n", netcp->dma_chan_name); - ret = PTR_ERR(netcp->rx_channel); + ret = -EINVAL; goto fail; } diff --git a/drivers/net/ethernet/vertexcom/mse102x.c b/drivers/net/ethernet/vertexcom/mse102x.c index 8f67c39f479ee..c902f8761d5d4 100644 --- a/drivers/net/ethernet/vertexcom/mse102x.c +++ b/drivers/net/ethernet/vertexcom/mse102x.c @@ -6,6 +6,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -33,7 +34,7 @@ #define CMD_CTR (0x2 << CMD_SHIFT) #define CMD_MASK GENMASK(15, CMD_SHIFT) -#define LEN_MASK GENMASK(CMD_SHIFT - 1, 0) +#define LEN_MASK GENMASK(CMD_SHIFT - 2, 0) #define DET_CMD_LEN 4 #define DET_SOF_LEN 2 @@ -262,7 +263,7 @@ static int mse102x_tx_frame_spi(struct mse102x_net *mse, struct sk_buff *txp, } static int mse102x_rx_frame_spi(struct mse102x_net *mse, u8 *buff, - unsigned int frame_len) + unsigned int frame_len, bool drop) { struct mse102x_net_spi *mses = to_mse102x_spi(mse); struct spi_transfer *xfer = &mses->spi_xfer; @@ -280,6 +281,9 @@ static int mse102x_rx_frame_spi(struct mse102x_net *mse, u8 *buff, netdev_err(mse->ndev, "%s: spi_sync() failed: %d\n", __func__, ret); mse->stats.xfer_err++; + } else if (drop) { + netdev_dbg(mse->ndev, "%s: Drop frame\n", __func__); + ret = -EINVAL; } else if (*sof != cpu_to_be16(DET_SOF)) { netdev_dbg(mse->ndev, "%s: SPI start of frame is invalid (0x%04x)\n", __func__, *sof); @@ -302,11 +306,12 @@ static void mse102x_dump_packet(const char *msg, int len, const char *data) data, len, true); } -static void mse102x_rx_pkt_spi(struct mse102x_net *mse) +static irqreturn_t mse102x_rx_pkt_spi(struct mse102x_net *mse) { struct sk_buff *skb; unsigned int rxalign; unsigned int rxlen; + bool drop = false; __be16 rx = 0; u16 cmd_resp; u8 *rxpkt; @@ -322,14 +327,15 @@ static void mse102x_rx_pkt_spi(struct mse102x_net *mse) mse102x_tx_cmd_spi(mse, CMD_CTR); ret = mse102x_rx_cmd_spi(mse, (u8 *)&rx); if (ret) - return; + return IRQ_NONE; cmd_resp = be16_to_cpu(rx); if ((cmd_resp & CMD_MASK) != CMD_RTS) { net_dbg_ratelimited("%s: Unexpected response (0x%04x)\n", __func__, cmd_resp); mse->stats.invalid_rts++; - return; + drop = true; + goto drop; } net_dbg_ratelimited("%s: Unexpected response to first CMD\n", @@ -337,26 +343,34 @@ static void mse102x_rx_pkt_spi(struct mse102x_net *mse) } rxlen = cmd_resp & LEN_MASK; - if (!rxlen) { - net_dbg_ratelimited("%s: No frame length defined\n", __func__); + if (rxlen < ETH_ZLEN || rxlen > VLAN_ETH_FRAME_LEN) { + net_dbg_ratelimited("%s: Invalid frame length: %d\n", __func__, + rxlen); mse->stats.invalid_len++; - return; + drop = true; } + /* In case of a invalid CMD_RTS, the frame must be consumed anyway. + * So assume the maximum possible frame length. + */ +drop: + if (drop) + rxlen = VLAN_ETH_FRAME_LEN; + rxalign = ALIGN(rxlen + DET_SOF_LEN + DET_DFT_LEN, 4); skb = netdev_alloc_skb_ip_align(mse->ndev, rxalign); if (!skb) - return; + return IRQ_NONE; /* 2 bytes Start of frame (before ethernet header) * 2 bytes Data frame tail (after ethernet frame) * They are copied, but ignored. */ rxpkt = skb_put(skb, rxlen) - DET_SOF_LEN; - if (mse102x_rx_frame_spi(mse, rxpkt, rxlen)) { + if (mse102x_rx_frame_spi(mse, rxpkt, rxlen, drop)) { mse->ndev->stats.rx_errors++; dev_kfree_skb(skb); - return; + return IRQ_HANDLED; } if (netif_msg_pktdata(mse)) @@ -367,6 +381,8 @@ static void mse102x_rx_pkt_spi(struct mse102x_net *mse) mse->ndev->stats.rx_packets++; mse->ndev->stats.rx_bytes += rxlen; + + return IRQ_HANDLED; } static int mse102x_tx_pkt_spi(struct mse102x_net *mse, struct sk_buff *txb, @@ -498,17 +514,19 @@ static irqreturn_t mse102x_irq(int irq, void *_mse) { struct mse102x_net *mse = _mse; struct mse102x_net_spi *mses = to_mse102x_spi(mse); + irqreturn_t ret; mutex_lock(&mses->lock); - mse102x_rx_pkt_spi(mse); + ret = mse102x_rx_pkt_spi(mse); mutex_unlock(&mses->lock); - return IRQ_HANDLED; + return ret; } static int mse102x_net_open(struct net_device *ndev) { struct mse102x_net *mse = netdev_priv(ndev); + struct mse102x_net_spi *mses = to_mse102x_spi(mse); int ret; ret = request_threaded_irq(ndev->irq, NULL, mse102x_irq, IRQF_ONESHOT, @@ -524,6 +542,13 @@ static int mse102x_net_open(struct net_device *ndev) netif_carrier_on(ndev); + /* The SPI interrupt can stuck in case of pending packet(s). + * So poll for possible packet(s) to re-arm the interrupt. + */ + mutex_lock(&mses->lock); + mse102x_rx_pkt_spi(mse); + mutex_unlock(&mses->lock); + netif_dbg(mse, ifup, ndev, "network device up\n"); return 0; diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index d6bc2309d2a38..7feb991a95924 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -1348,7 +1348,6 @@ static void wx_configure_rx_ring(struct wx *wx, struct wx_ring *ring) { u16 reg_idx = ring->reg_idx; - union wx_rx_desc *rx_desc; u64 rdba = ring->dma; u32 rxdctl; @@ -1378,9 +1377,9 @@ static void wx_configure_rx_ring(struct wx *wx, memset(ring->rx_buffer_info, 0, sizeof(struct wx_rx_buffer) * ring->count); - /* initialize Rx descriptor 0 */ - rx_desc = WX_RX_DESC(ring, 0); - rx_desc->wb.upper.length = 0; + /* reset ntu and ntc to place SW in sync with hardware */ + ring->next_to_clean = 0; + ring->next_to_use = 0; /* enable receive descriptor ring */ wr32m(wx, WX_PX_RR_CFG(reg_idx), @@ -1668,7 +1667,8 @@ int wx_sw_init(struct wx *wx) wx->oem_svid = pdev->subsystem_vendor; wx->oem_ssid = pdev->subsystem_device; wx->bus.device = PCI_SLOT(pdev->devfn); - wx->bus.func = PCI_FUNC(pdev->devfn); + wx->bus.func = FIELD_GET(WX_CFG_PORT_ST_LANID, + rd32(wx, WX_CFG_PORT_ST)); if (wx->oem_svid == PCI_VENDOR_ID_WANGXUN) { wx->subsystem_vendor_id = pdev->subsystem_vendor; diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index c019fe964ecea..23dbe4e4b36c0 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -171,10 +171,6 @@ static void wx_dma_sync_frag(struct wx_ring *rx_ring, skb_frag_off(frag), skb_frag_size(frag), DMA_FROM_DEVICE); - - /* If the page was released, just unmap it. */ - if (unlikely(WX_CB(skb)->page_released)) - page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false); } static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring, @@ -224,10 +220,6 @@ static void wx_put_rx_buffer(struct wx_ring *rx_ring, struct sk_buff *skb, int rx_buffer_pgcnt) { - if (!IS_ERR(skb) && WX_CB(skb)->dma == rx_buffer->dma) - /* the page has been released from the ring */ - WX_CB(skb)->page_released = true; - /* clear contents of rx_buffer */ rx_buffer->page = NULL; rx_buffer->skb = NULL; @@ -315,7 +307,7 @@ static bool wx_alloc_mapped_page(struct wx_ring *rx_ring, return false; dma = page_pool_get_dma_addr(page); - bi->page_dma = dma; + bi->dma = dma; bi->page = page; bi->page_offset = 0; @@ -352,7 +344,7 @@ void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count) DMA_FROM_DEVICE); rx_desc->read.pkt_addr = - cpu_to_le64(bi->page_dma + bi->page_offset); + cpu_to_le64(bi->dma + bi->page_offset); rx_desc++; bi++; @@ -365,6 +357,8 @@ void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count) /* clear the status bits for the next_to_use descriptor */ rx_desc->wb.upper.status_error = 0; + /* clear the length for the next_to_use descriptor */ + rx_desc->wb.upper.length = 0; cleaned_count--; } while (cleaned_count); @@ -2158,9 +2152,6 @@ static void wx_clean_rx_ring(struct wx_ring *rx_ring) if (rx_buffer->skb) { struct sk_buff *skb = rx_buffer->skb; - if (WX_CB(skb)->page_released) - page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false); - dev_kfree_skb(skb); } @@ -2184,6 +2175,9 @@ static void wx_clean_rx_ring(struct wx_ring *rx_ring) } } + /* Zero out the descriptor ring */ + memset(rx_ring->desc, 0, rx_ring->size); + rx_ring->next_to_alloc = 0; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; @@ -2368,7 +2362,7 @@ static int wx_alloc_page_pool(struct wx_ring *rx_ring) struct page_pool_params pp_params = { .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, .order = 0, - .pool_size = rx_ring->size, + .pool_size = rx_ring->count, .nid = dev_to_node(rx_ring->dev), .dev = rx_ring->dev, .dma_dir = DMA_FROM_DEVICE, diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index c555af9ed51b2..ee95706b44630 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -65,6 +65,8 @@ #define WX_CFG_PORT_CTL_DRV_LOAD BIT(3) #define WX_CFG_PORT_CTL_QINQ BIT(2) #define WX_CFG_PORT_CTL_D_VLAN BIT(0) /* double vlan*/ +#define WX_CFG_PORT_ST 0x14404 +#define WX_CFG_PORT_ST_LANID GENMASK(9, 8) #define WX_CFG_TAG_TPID(_i) (0x14430 + ((_i) * 4)) #define WX_CFG_PORT_CTL_NUM_VT_MASK GENMASK(13, 12) /* number of TVs */ @@ -363,8 +365,6 @@ enum WX_MSCA_CMD_value { #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), WX_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -#define WX_CFG_PORT_ST 0x14404 - /******************* Receive Descriptor bit definitions **********************/ #define WX_RXD_STAT_DD BIT(0) /* Done */ #define WX_RXD_STAT_EOP BIT(1) /* End of Packet */ @@ -668,7 +668,6 @@ enum wx_reset_type { struct wx_cb { dma_addr_t dma; u16 append_cnt; /* number of skb's appended */ - bool page_released; bool dma_released; }; @@ -756,7 +755,6 @@ struct wx_tx_buffer { struct wx_rx_buffer { struct sk_buff *skb; dma_addr_t dma; - dma_addr_t page_dma; struct page *page; unsigned int page_offset; }; diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index c10f94d69dad3..3d622634e82aa 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -1309,7 +1309,7 @@ ll_temac_ethtools_set_ringparam(struct net_device *ndev, if (ering->rx_pending > RX_BD_NUM_MAX || ering->rx_mini_pending || ering->rx_jumbo_pending || - ering->rx_pending > TX_BD_NUM_MAX) + ering->tx_pending > TX_BD_NUM_MAX) return -EINVAL; if (netif_running(ndev)) diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c index b358ecc672278..0eff5d4fe35df 100644 --- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c +++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c @@ -285,7 +285,7 @@ static void xemaclite_aligned_read(u32 *src_ptr, u8 *dest_ptr, /* Read the remaining data */ for (; length > 0; length--) - *to_u8_ptr = *from_u8_ptr; + *to_u8_ptr++ = *from_u8_ptr++; } } diff --git a/drivers/net/ethernet/xircom/xirc2ps_cs.c b/drivers/net/ethernet/xircom/xirc2ps_cs.c index 9f505cf02d965..2dc1cfcd7ce99 100644 --- a/drivers/net/ethernet/xircom/xirc2ps_cs.c +++ b/drivers/net/ethernet/xircom/xirc2ps_cs.c @@ -1578,7 +1578,7 @@ do_reset(struct net_device *dev, int full) msleep(40); /* wait 40 msec to let it complete */ } if (full_duplex) - PutByte(XIRCREG1_ECR, GetByte(XIRCREG1_ECR | FullDuplex)); + PutByte(XIRCREG1_ECR, GetByte(XIRCREG1_ECR) | FullDuplex); } else { /* No MII */ SelectPage(0); value = GetByte(XIRCREG_ESR); /* read the ESR */ diff --git a/drivers/net/fjes/fjes_main.c b/drivers/net/fjes/fjes_main.c index 2513be6d4e11d..cfcf0f7d9d903 100644 --- a/drivers/net/fjes/fjes_main.c +++ b/drivers/net/fjes/fjes_main.c @@ -14,9 +14,7 @@ #include "fjes.h" #include "fjes_trace.h" -#define MAJ 1 -#define MIN 2 -#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) +#define DRV_VERSION "1.2" #define DRV_NAME "fjes" char fjes_driver_name[] = DRV_NAME; char fjes_driver_version[] = DRV_VERSION; diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c index 6ed38a3cdd734..6cecdfa8631d7 100644 --- a/drivers/net/hamradio/6pack.c +++ b/drivers/net/hamradio/6pack.c @@ -121,8 +121,6 @@ struct sixpack { struct timer_list tx_t; struct timer_list resync_t; - refcount_t refcnt; - struct completion dead; spinlock_t lock; }; @@ -359,42 +357,13 @@ static void sp_bump(struct sixpack *sp, char cmd) /* ----------------------------------------------------------------------- */ -/* - * We have a potential race on dereferencing tty->disc_data, because the tty - * layer provides no locking at all - thus one cpu could be running - * sixpack_receive_buf while another calls sixpack_close, which zeroes - * tty->disc_data and frees the memory that sixpack_receive_buf is using. The - * best way to fix this is to use a rwlock in the tty struct, but for now we - * use a single global rwlock for all ttys in ppp line discipline. - */ -static DEFINE_RWLOCK(disc_data_lock); - -static struct sixpack *sp_get(struct tty_struct *tty) -{ - struct sixpack *sp; - - read_lock(&disc_data_lock); - sp = tty->disc_data; - if (sp) - refcount_inc(&sp->refcnt); - read_unlock(&disc_data_lock); - - return sp; -} - -static void sp_put(struct sixpack *sp) -{ - if (refcount_dec_and_test(&sp->refcnt)) - complete(&sp->dead); -} - /* * Called by the TTY driver when there's room for more data. If we have * more packets to send, we send them here. */ static void sixpack_write_wakeup(struct tty_struct *tty) { - struct sixpack *sp = sp_get(tty); + struct sixpack *sp = tty->disc_data; int actual; if (!sp) @@ -406,7 +375,7 @@ static void sixpack_write_wakeup(struct tty_struct *tty) clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); sp->tx_enable = 0; netif_wake_queue(sp->dev); - goto out; + return; } if (sp->tx_enable) { @@ -414,9 +383,6 @@ static void sixpack_write_wakeup(struct tty_struct *tty) sp->xleft -= actual; sp->xhead += actual; } - -out: - sp_put(sp); } /* ----------------------------------------------------------------------- */ @@ -436,7 +402,7 @@ static void sixpack_receive_buf(struct tty_struct *tty, const u8 *cp, if (!count) return; - sp = sp_get(tty); + sp = tty->disc_data; if (!sp) return; @@ -452,7 +418,6 @@ static void sixpack_receive_buf(struct tty_struct *tty, const u8 *cp, } sixpack_decode(sp, cp, count1); - sp_put(sp); tty_unthrottle(tty); } @@ -567,8 +532,6 @@ static int sixpack_open(struct tty_struct *tty) spin_lock_init(&sp->lock); spin_lock_init(&sp->rxlock); - refcount_set(&sp->refcnt, 1); - init_completion(&sp->dead); /* !!! length of the buffers. MTU is IP MTU, not PACLEN! */ @@ -650,19 +613,11 @@ static void sixpack_close(struct tty_struct *tty) { struct sixpack *sp; - write_lock_irq(&disc_data_lock); sp = tty->disc_data; - tty->disc_data = NULL; - write_unlock_irq(&disc_data_lock); if (!sp) return; - /* - * We have now ensured that nobody can start using ap from now on, but - * we have to wait for all existing users to finish. - */ - if (!refcount_dec_and_test(&sp->refcnt)) - wait_for_completion(&sp->dead); + tty->disc_data = NULL; /* We must stop the queue to avoid potentially scribbling * on the free buffers. The sp->dead completion is not sufficient @@ -686,7 +641,7 @@ static void sixpack_close(struct tty_struct *tty) static int sixpack_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { - struct sixpack *sp = sp_get(tty); + struct sixpack *sp = tty->disc_data; struct net_device *dev; unsigned int tmp, err; @@ -738,8 +693,6 @@ static int sixpack_ioctl(struct tty_struct *tty, unsigned int cmd, err = tty_mode_ioctl(tty, cmd, arg); } - sp_put(sp); - return err; } diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h index 810977952f950..f07bde6695c7b 100644 --- a/drivers/net/hyperv/hyperv_net.h +++ b/drivers/net/hyperv/hyperv_net.h @@ -158,7 +158,6 @@ struct hv_netvsc_packet { u8 cp_partial; /* partial copy into send buffer */ u8 rmsg_size; /* RNDIS header and PPI size */ - u8 rmsg_pgcnt; /* page count of RNDIS header and PPI */ u8 page_buf_cnt; u16 q_idx; @@ -893,6 +892,18 @@ struct nvsp_message { sizeof(struct nvsp_message)) #define NETVSC_MIN_IN_MSG_SIZE sizeof(struct vmpacket_descriptor) +/* Maximum # of contiguous data ranges that can make up a trasmitted packet. + * Typically it's the max SKB fragments plus 2 for the rndis packet and the + * linear portion of the SKB. But if MAX_SKB_FRAGS is large, the value may + * need to be limited to MAX_PAGE_BUFFER_COUNT, which is the max # of entries + * in a GPA direct packet sent to netvsp over VMBus. + */ +#if MAX_SKB_FRAGS + 2 < MAX_PAGE_BUFFER_COUNT +#define MAX_DATA_RANGES (MAX_SKB_FRAGS + 2) +#else +#define MAX_DATA_RANGES MAX_PAGE_BUFFER_COUNT +#endif + /* Estimated requestor size: * out_ring_size/min_out_msg_size + in_ring_size/min_in_msg_size */ @@ -1050,6 +1061,7 @@ struct net_device_context { struct net_device __rcu *vf_netdev; struct netvsc_vf_pcpu_stats __percpu *vf_stats; struct delayed_work vf_takeover; + struct delayed_work vfns_work; /* 1: allocated, serial number is valid. 0: not allocated */ u32 vf_alloc; @@ -1064,6 +1076,8 @@ struct net_device_context { struct netvsc_device_info *saved_netvsc_dev_info; }; +void netvsc_vfns_work(struct work_struct *w); + /* Azure hosts don't support non-TCP port numbers in hashing for fragmented * packets. We can use ethtool to change UDP hash level when necessary. */ diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index b2f27e505f76c..61584b40cb038 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -945,8 +945,7 @@ static void netvsc_copy_to_send_buf(struct netvsc_device *net_device, + pend_size; int i; u32 padding = 0; - u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : - packet->page_buf_cnt; + u32 page_count = packet->cp_partial ? 1 : packet->page_buf_cnt; u32 remain; /* Add padding */ @@ -1047,6 +1046,42 @@ static int netvsc_dma_map(struct hv_device *hv_dev, return 0; } +/* Build an "array" of mpb entries describing the data to be transferred + * over VMBus. After the desc header fields, each "array" entry is variable + * size, and each entry starts after the end of the previous entry. The + * "offset" and "len" fields for each entry imply the size of the entry. + * + * The pfns are in HV_HYP_PAGE_SIZE, because all communication with Hyper-V + * uses that granularity, even if the system page size of the guest is larger. + * Each entry in the input "pb" array must describe a contiguous range of + * guest physical memory so that the pfns are sequential if the range crosses + * a page boundary. The offset field must be < HV_HYP_PAGE_SIZE. + */ +static inline void netvsc_build_mpb_array(struct hv_page_buffer *pb, + u32 page_buffer_count, + struct vmbus_packet_mpb_array *desc, + u32 *desc_size) +{ + struct hv_mpb_array *mpb_entry = &desc->range; + int i, j; + + for (i = 0; i < page_buffer_count; i++) { + u32 offset = pb[i].offset; + u32 len = pb[i].len; + + mpb_entry->offset = offset; + mpb_entry->len = len; + + for (j = 0; j < HVPFN_UP(offset + len); j++) + mpb_entry->pfn_array[j] = pb[i].pfn + j; + + mpb_entry = (struct hv_mpb_array *)&mpb_entry->pfn_array[j]; + } + + desc->rangecount = page_buffer_count; + *desc_size = (char *)mpb_entry - (char *)desc; +} + static inline int netvsc_send_pkt( struct hv_device *device, struct hv_netvsc_packet *packet, @@ -1089,8 +1124,11 @@ static inline int netvsc_send_pkt( packet->dma_range = NULL; if (packet->page_buf_cnt) { + struct vmbus_channel_packet_page_buffer desc; + u32 desc_size; + if (packet->cp_partial) - pb += packet->rmsg_pgcnt; + pb++; ret = netvsc_dma_map(ndev_ctx->device_ctx, packet, pb); if (ret) { @@ -1098,11 +1136,12 @@ static inline int netvsc_send_pkt( goto exit; } - ret = vmbus_sendpacket_pagebuffer(out_channel, - pb, packet->page_buf_cnt, - &nvmsg, sizeof(nvmsg), - req_id); - + netvsc_build_mpb_array(pb, packet->page_buf_cnt, + (struct vmbus_packet_mpb_array *)&desc, + &desc_size); + ret = vmbus_sendpacket_mpb_desc(out_channel, + (struct vmbus_packet_mpb_array *)&desc, + desc_size, &nvmsg, sizeof(nvmsg), req_id); if (ret) netvsc_dma_unmap(ndev_ctx->device_ctx, packet); } else { @@ -1251,7 +1290,7 @@ int netvsc_send(struct net_device *ndev, packet->send_buf_index = section_index; if (packet->cp_partial) { - packet->page_buf_cnt -= packet->rmsg_pgcnt; + packet->page_buf_cnt--; packet->total_data_buflen = msd_len + packet->rmsg_size; } else { packet->page_buf_cnt = 0; diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 8698d2db3dc8e..aa114240e340d 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -325,43 +325,10 @@ static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, return txq; } -static u32 fill_pg_buf(unsigned long hvpfn, u32 offset, u32 len, - struct hv_page_buffer *pb) -{ - int j = 0; - - hvpfn += offset >> HV_HYP_PAGE_SHIFT; - offset = offset & ~HV_HYP_PAGE_MASK; - - while (len > 0) { - unsigned long bytes; - - bytes = HV_HYP_PAGE_SIZE - offset; - if (bytes > len) - bytes = len; - pb[j].pfn = hvpfn; - pb[j].offset = offset; - pb[j].len = bytes; - - offset += bytes; - len -= bytes; - - if (offset == HV_HYP_PAGE_SIZE && len) { - hvpfn++; - offset = 0; - j++; - } - } - - return j + 1; -} - static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, struct hv_netvsc_packet *packet, struct hv_page_buffer *pb) { - u32 slots_used = 0; - char *data = skb->data; int frags = skb_shinfo(skb)->nr_frags; int i; @@ -370,28 +337,27 @@ static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, * 2. skb linear data * 3. skb fragment data */ - slots_used += fill_pg_buf(virt_to_hvpfn(hdr), - offset_in_hvpage(hdr), - len, - &pb[slots_used]); + pb[0].offset = offset_in_hvpage(hdr); + pb[0].len = len; + pb[0].pfn = virt_to_hvpfn(hdr); packet->rmsg_size = len; - packet->rmsg_pgcnt = slots_used; - slots_used += fill_pg_buf(virt_to_hvpfn(data), - offset_in_hvpage(data), - skb_headlen(skb), - &pb[slots_used]); + pb[1].offset = offset_in_hvpage(skb->data); + pb[1].len = skb_headlen(skb); + pb[1].pfn = virt_to_hvpfn(skb->data); for (i = 0; i < frags; i++) { skb_frag_t *frag = skb_shinfo(skb)->frags + i; + struct hv_page_buffer *cur_pb = &pb[i + 2]; + u64 pfn = page_to_hvpfn(skb_frag_page(frag)); + u32 offset = skb_frag_off(frag); - slots_used += fill_pg_buf(page_to_hvpfn(skb_frag_page(frag)), - skb_frag_off(frag), - skb_frag_size(frag), - &pb[slots_used]); + cur_pb->offset = offset_in_hvpage(offset); + cur_pb->len = skb_frag_size(frag); + cur_pb->pfn = pfn + (offset >> HV_HYP_PAGE_SHIFT); } - return slots_used; + return frags + 2; } static int count_skb_frag_slots(struct sk_buff *skb) @@ -482,7 +448,7 @@ static int netvsc_xmit(struct sk_buff *skb, struct net_device *net, bool xdp_tx) struct net_device *vf_netdev; u32 rndis_msg_size; u32 hash; - struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT]; + struct hv_page_buffer pb[MAX_DATA_RANGES]; /* If VF is present and up then redirect packets to it. * Skip the VF if it is marked down or has no carrier. @@ -2347,8 +2313,11 @@ static int netvsc_prepare_bonding(struct net_device *vf_netdev) if (!ndev) return NOTIFY_DONE; - /* set slave flag before open to prevent IPv6 addrconf */ + /* Set slave flag and no addrconf flag before open + * to prevent IPv6 addrconf. + */ vf_netdev->flags |= IFF_SLAVE; + vf_netdev->priv_flags |= IFF_NO_ADDRCONF; return NOTIFY_DONE; } @@ -2544,6 +2513,7 @@ static int netvsc_probe(struct hv_device *dev, spin_lock_init(&net_device_ctx->lock); INIT_LIST_HEAD(&net_device_ctx->reconfig_events); INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup); + INIT_DELAYED_WORK(&net_device_ctx->vfns_work, netvsc_vfns_work); net_device_ctx->vf_stats = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats); @@ -2686,6 +2656,8 @@ static void netvsc_remove(struct hv_device *dev) cancel_delayed_work_sync(&ndev_ctx->dwork); rtnl_lock(); + cancel_delayed_work_sync(&ndev_ctx->vfns_work); + nvdev = rtnl_dereference(ndev_ctx->nvdev); if (nvdev) { cancel_work_sync(&nvdev->subchan_work); @@ -2727,6 +2699,7 @@ static int netvsc_suspend(struct hv_device *dev) cancel_delayed_work_sync(&ndev_ctx->dwork); rtnl_lock(); + cancel_delayed_work_sync(&ndev_ctx->vfns_work); nvdev = rtnl_dereference(ndev_ctx->nvdev); if (nvdev == NULL) { @@ -2820,6 +2793,27 @@ static void netvsc_event_set_vf_ns(struct net_device *ndev) } } +void netvsc_vfns_work(struct work_struct *w) +{ + struct net_device_context *ndev_ctx = + container_of(w, struct net_device_context, vfns_work.work); + struct net_device *ndev; + + if (!rtnl_trylock()) { + schedule_delayed_work(&ndev_ctx->vfns_work, 1); + return; + } + + ndev = hv_get_drvdata(ndev_ctx->device_ctx); + if (!ndev) + goto out; + + netvsc_event_set_vf_ns(ndev); + +out: + rtnl_unlock(); +} + /* * On Hyper-V, every VF interface is matched with a corresponding * synthetic interface. The synthetic interface is presented first @@ -2830,10 +2824,12 @@ static int netvsc_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *event_dev = netdev_notifier_info_to_dev(ptr); + struct net_device_context *ndev_ctx; int ret = 0; if (event_dev->netdev_ops == &device_ops && event == NETDEV_REGISTER) { - netvsc_event_set_vf_ns(event_dev); + ndev_ctx = netdev_priv(event_dev); + schedule_delayed_work(&ndev_ctx->vfns_work, 0); return NOTIFY_DONE; } diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c index af95947a87c55..09144f0ec2aa4 100644 --- a/drivers/net/hyperv/rndis_filter.c +++ b/drivers/net/hyperv/rndis_filter.c @@ -226,8 +226,7 @@ static int rndis_filter_send_request(struct rndis_device *dev, struct rndis_request *req) { struct hv_netvsc_packet *packet; - struct hv_page_buffer page_buf[2]; - struct hv_page_buffer *pb = page_buf; + struct hv_page_buffer pb; int ret; /* Setup the packet to send it */ @@ -236,27 +235,14 @@ static int rndis_filter_send_request(struct rndis_device *dev, packet->total_data_buflen = req->request_msg.msg_len; packet->page_buf_cnt = 1; - pb[0].pfn = virt_to_phys(&req->request_msg) >> - HV_HYP_PAGE_SHIFT; - pb[0].len = req->request_msg.msg_len; - pb[0].offset = offset_in_hvpage(&req->request_msg); - - /* Add one page_buf when request_msg crossing page boundary */ - if (pb[0].offset + pb[0].len > HV_HYP_PAGE_SIZE) { - packet->page_buf_cnt++; - pb[0].len = HV_HYP_PAGE_SIZE - - pb[0].offset; - pb[1].pfn = virt_to_phys((void *)&req->request_msg - + pb[0].len) >> HV_HYP_PAGE_SHIFT; - pb[1].offset = 0; - pb[1].len = req->request_msg.msg_len - - pb[0].len; - } + pb.pfn = virt_to_phys(&req->request_msg) >> HV_HYP_PAGE_SHIFT; + pb.len = req->request_msg.msg_len; + pb.offset = offset_in_hvpage(&req->request_msg); trace_rndis_send(dev->ndev, 0, &req->request_msg); rcu_read_lock_bh(); - ret = netvsc_send(dev->ndev, packet, NULL, pb, NULL, false); + ret = netvsc_send(dev->ndev, packet, NULL, &pb, NULL, false); rcu_read_unlock_bh(); return ret; diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c index 0a0ad3d77557f..587643a371de3 100644 --- a/drivers/net/ieee802154/ca8210.c +++ b/drivers/net/ieee802154/ca8210.c @@ -1446,8 +1446,7 @@ static u8 mcps_data_request( command.pdata.data_req.src_addr_mode = src_addr_mode; command.pdata.data_req.dst.mode = dst_address_mode; if (dst_address_mode != MAC_MODE_NO_ADDR) { - command.pdata.data_req.dst.pan_id[0] = LS_BYTE(dst_pan_id); - command.pdata.data_req.dst.pan_id[1] = MS_BYTE(dst_pan_id); + put_unaligned_le16(dst_pan_id, command.pdata.data_req.dst.pan_id); if (dst_address_mode == MAC_MODE_SHORT_ADDR) { command.pdata.data_req.dst.address[0] = LS_BYTE( dst_addr->short_address @@ -1795,12 +1794,12 @@ static int ca8210_skb_rx( } hdr.source.mode = data_ind[0]; dev_dbg(&priv->spi->dev, "srcAddrMode: %#03x\n", hdr.source.mode); - hdr.source.pan_id = *(u16 *)&data_ind[1]; + hdr.source.pan_id = cpu_to_le16(get_unaligned_le16(&data_ind[1])); dev_dbg(&priv->spi->dev, "srcPanId: %#06x\n", hdr.source.pan_id); memcpy(&hdr.source.extended_addr, &data_ind[3], 8); hdr.dest.mode = data_ind[11]; dev_dbg(&priv->spi->dev, "dstAddrMode: %#03x\n", hdr.dest.mode); - hdr.dest.pan_id = *(u16 *)&data_ind[12]; + hdr.dest.pan_id = cpu_to_le16(get_unaligned_le16(&data_ind[12])); dev_dbg(&priv->spi->dev, "dstPanId: %#06x\n", hdr.dest.pan_id); memcpy(&hdr.dest.extended_addr, &data_ind[14], 8); @@ -1927,7 +1926,7 @@ static int ca8210_skb_tx( status = mcps_data_request( header.source.mode, header.dest.mode, - header.dest.pan_id, + le16_to_cpu(header.dest.pan_id), (union macaddr *)&header.dest.extended_addr, skb->len - mac_len, &skb->data[mac_len], diff --git a/drivers/net/ipa/ipa_sysfs.c b/drivers/net/ipa/ipa_sysfs.c index 2ff09ce343b73..2e676b9d4042d 100644 --- a/drivers/net/ipa/ipa_sysfs.c +++ b/drivers/net/ipa/ipa_sysfs.c @@ -38,8 +38,12 @@ static const char *ipa_version_string(struct ipa *ipa) return "4.11"; case IPA_VERSION_5_0: return "5.0"; + case IPA_VERSION_5_1: + return "5.1"; + case IPA_VERSION_5_5: + return "5.5"; default: - return "0.0"; /* Won't happen (checked at probe time) */ + return "0.0"; /* Should not happen */ } } diff --git a/drivers/net/ipvlan/ipvlan_l3s.c b/drivers/net/ipvlan/ipvlan_l3s.c index d5b05e8032199..ca35a50bb6405 100644 --- a/drivers/net/ipvlan/ipvlan_l3s.c +++ b/drivers/net/ipvlan/ipvlan_l3s.c @@ -224,5 +224,4 @@ void ipvlan_l3s_unregister(struct ipvl_port *port) dev->priv_flags &= ~IFF_L3MDEV_RX_HANDLER; ipvlan_unregister_nf_hook(read_pnet(&port->pnet)); - dev->l3mdev_ops = NULL; } diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 7c96493a367bf..af6cc3e90ef7c 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -243,15 +243,39 @@ static sci_t make_sci(const u8 *addr, __be16 port) return sci; } -static sci_t macsec_frame_sci(struct macsec_eth_header *hdr, bool sci_present) +static sci_t macsec_active_sci(struct macsec_secy *secy) { - sci_t sci; + struct macsec_rx_sc *rx_sc = rcu_dereference_bh(secy->rx_sc); + + /* Case single RX SC */ + if (rx_sc && !rcu_dereference_bh(rx_sc->next)) + return (rx_sc->active) ? rx_sc->sci : 0; + /* Case no RX SC or multiple */ + else + return 0; +} + +static sci_t macsec_frame_sci(struct macsec_eth_header *hdr, bool sci_present, + struct macsec_rxh_data *rxd) +{ + struct macsec_dev *macsec; + sci_t sci = 0; - if (sci_present) + /* SC = 1 */ + if (sci_present) { memcpy(&sci, hdr->secure_channel_id, sizeof(hdr->secure_channel_id)); - else + /* SC = 0; ES = 0 */ + } else if ((!(hdr->tci_an & (MACSEC_TCI_ES | MACSEC_TCI_SC))) && + (list_is_singular(&rxd->secys))) { + /* Only one SECY should exist on this scenario */ + macsec = list_first_or_null_rcu(&rxd->secys, struct macsec_dev, + secys); + if (macsec) + return macsec_active_sci(&macsec->secy); + } else { sci = make_sci(hdr->eth.h_source, MACSEC_PORT_ES); + } return sci; } @@ -1105,7 +1129,7 @@ static rx_handler_result_t macsec_handle_frame(struct sk_buff **pskb) struct macsec_rxh_data *rxd; struct macsec_dev *macsec; unsigned int len; - sci_t sci; + sci_t sci = 0; u32 hdr_pn; bool cbit; struct pcpu_rx_sc_stats *rxsc_stats; @@ -1152,11 +1176,14 @@ static rx_handler_result_t macsec_handle_frame(struct sk_buff **pskb) macsec_skb_cb(skb)->has_sci = !!(hdr->tci_an & MACSEC_TCI_SC); macsec_skb_cb(skb)->assoc_num = hdr->tci_an & MACSEC_AN_MASK; - sci = macsec_frame_sci(hdr, macsec_skb_cb(skb)->has_sci); rcu_read_lock(); rxd = macsec_data_rcu(skb->dev); + sci = macsec_frame_sci(hdr, macsec_skb_cb(skb)->has_sci, rxd); + if (!sci) + goto drop_nosc; + list_for_each_entry_rcu(macsec, &rxd->secys, secys) { struct macsec_rx_sc *sc = find_rx_sc(&macsec->secy, sci); @@ -1279,6 +1306,7 @@ static rx_handler_result_t macsec_handle_frame(struct sk_buff **pskb) macsec_rxsa_put(rx_sa); drop_nosa: macsec_rxsc_put(rx_sc); +drop_nosc: rcu_read_unlock(); drop_direct: kfree_skb(skb); @@ -1812,7 +1840,7 @@ static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info) if (tb_sa[MACSEC_SA_ATTR_PN]) { spin_lock_bh(&rx_sa->lock); - rx_sa->next_pn = nla_get_u64(tb_sa[MACSEC_SA_ATTR_PN]); + rx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]); spin_unlock_bh(&rx_sa->lock); } @@ -2054,7 +2082,7 @@ static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info) } spin_lock_bh(&tx_sa->lock); - tx_sa->next_pn = nla_get_u64(tb_sa[MACSEC_SA_ATTR_PN]); + tx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]); spin_unlock_bh(&tx_sa->lock); if (tb_sa[MACSEC_SA_ATTR_ACTIVE]) @@ -2366,7 +2394,7 @@ static int macsec_upd_txsa(struct sk_buff *skb, struct genl_info *info) spin_lock_bh(&tx_sa->lock); prev_pn = tx_sa->next_pn_halves; - tx_sa->next_pn = nla_get_u64(tb_sa[MACSEC_SA_ATTR_PN]); + tx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]); spin_unlock_bh(&tx_sa->lock); } @@ -2464,7 +2492,7 @@ static int macsec_upd_rxsa(struct sk_buff *skb, struct genl_info *info) spin_lock_bh(&rx_sa->lock); prev_pn = rx_sa->next_pn_halves; - rx_sa->next_pn = nla_get_u64(tb_sa[MACSEC_SA_ATTR_PN]); + rx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]); spin_unlock_bh(&rx_sa->lock); } diff --git a/drivers/net/mdio/mdio-mux-meson-gxl.c b/drivers/net/mdio/mdio-mux-meson-gxl.c index 76188575ca1fc..19153d44800a9 100644 --- a/drivers/net/mdio/mdio-mux-meson-gxl.c +++ b/drivers/net/mdio/mdio-mux-meson-gxl.c @@ -17,6 +17,7 @@ #define REG2_LEDACT GENMASK(23, 22) #define REG2_LEDLINK GENMASK(25, 24) #define REG2_DIV4SEL BIT(27) +#define REG2_REVERSED BIT(28) #define REG2_ADCBYPASS BIT(30) #define REG2_CLKINSEL BIT(31) #define ETH_REG3 0x4 @@ -65,7 +66,7 @@ static void gxl_enable_internal_mdio(struct gxl_mdio_mux *priv) * The only constraint is that it must match the one in * drivers/net/phy/meson-gxl.c to properly match the PHY. */ - writel(FIELD_PREP(REG2_PHYID, EPHY_GXL_ID), + writel(REG2_REVERSED | FIELD_PREP(REG2_PHYID, EPHY_GXL_ID), priv->regs + ETH_REG2); /* Enable the internal phy */ diff --git a/drivers/net/mdio/of_mdio.c b/drivers/net/mdio/of_mdio.c index 7eb32ebb846d8..15b72203a2584 100644 --- a/drivers/net/mdio/of_mdio.c +++ b/drivers/net/mdio/of_mdio.c @@ -440,6 +440,5 @@ void of_phy_deregister_fixed_link(struct device_node *np) fixed_phy_unregister(phydev); put_device(&phydev->mdio.dev); /* of_phy_find_device() */ - phy_device_free(phydev); /* fixed_phy_register() */ } EXPORT_SYMBOL(of_phy_deregister_fixed_link); diff --git a/drivers/net/pcs/pcs-rzn1-miic.c b/drivers/net/pcs/pcs-rzn1-miic.c index 97139c07130fc..b65682b8b6cd9 100644 --- a/drivers/net/pcs/pcs-rzn1-miic.c +++ b/drivers/net/pcs/pcs-rzn1-miic.c @@ -19,7 +19,7 @@ #define MIIC_PRCMD 0x0 #define MIIC_ESID_CODE 0x4 -#define MIIC_MODCTRL 0x20 +#define MIIC_MODCTRL 0x8 #define MIIC_MODCTRL_SW_MODE GENMASK(4, 0) #define MIIC_CONVCTRL(port) (0x100 + (port) * 4) diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c index e397e7d642d92..a778379108ef1 100644 --- a/drivers/net/phy/dp83867.c +++ b/drivers/net/phy/dp83867.c @@ -774,6 +774,14 @@ static int dp83867_config_init(struct phy_device *phydev) return ret; } + /* Although the DP83867 reports EEE capability through the + * MDIO_PCS_EEE_ABLE and MDIO_AN_EEE_ADV registers, the feature + * is not actually implemented in hardware. + */ + linkmode_zero(phydev->supported_eee); + linkmode_zero(phydev->advertising_eee); + phydev->eee_enabled = false; + if (phy_interface_is_rgmii(phydev) || phydev->interface == PHY_INTERFACE_MODE_SGMII) { val = phy_read(phydev, MII_DP83867_PHYCTRL); diff --git a/drivers/net/phy/fixed_phy.c b/drivers/net/phy/fixed_phy.c index aef739c20ac4d..4694fb3eaa2ff 100644 --- a/drivers/net/phy/fixed_phy.c +++ b/drivers/net/phy/fixed_phy.c @@ -329,6 +329,7 @@ void fixed_phy_unregister(struct phy_device *phy) phy_device_remove(phy); of_node_put(phy->mdio.dev.of_node); fixed_phy_del(phy->mdio.addr); + phy_device_free(phy); } EXPORT_SYMBOL_GPL(fixed_phy_unregister); diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index eba652a4c1d88..760d0d2f791ca 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -1802,6 +1802,43 @@ static int marvell_resume(struct phy_device *phydev) return err; } +/* m88e1510_resume + * + * The 88e1510 PHY has an erratum where the phy downshift counter is not cleared + * after phy being suspended(BMCR_PDOWN set) and then later resumed(BMCR_PDOWN + * cleared). This can cause the link to intermittently downshift to a lower speed. + * + * Disabling and re-enabling the downshift feature clears the counter, allowing + * the PHY to retry gigabit link negotiation up to the programmed retry count + * before downshifting. This behavior has been observed on copper links. + */ +static int m88e1510_resume(struct phy_device *phydev) +{ + int err; + u8 cnt = 0; + + err = marvell_resume(phydev); + if (err < 0) + return err; + + /* read downshift counter value */ + err = m88e1011_get_downshift(phydev, &cnt); + if (err < 0) + return err; + + if (cnt) { + /* downshift disabled */ + err = m88e1011_set_downshift(phydev, 0); + if (err < 0) + return err; + + /* downshift enabled, with previous counter value */ + err = m88e1011_set_downshift(phydev, cnt); + } + + return err; +} + static int marvell_aneg_done(struct phy_device *phydev) { int retval = phy_read(phydev, MII_M1011_PHY_STATUS); @@ -3506,7 +3543,7 @@ static struct phy_driver marvell_drivers[] = { .handle_interrupt = marvell_handle_interrupt, .get_wol = m88e1318_get_wol, .set_wol = m88e1318_set_wol, - .resume = marvell_resume, + .resume = m88e1510_resume, .suspend = marvell_suspend, .read_page = marvell_read_page, .write_page = marvell_write_page, diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index 25dcaa49ab8be..7da30a6752bee 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -81,8 +81,11 @@ int mdiobus_register_device(struct mdio_device *mdiodev) return err; err = mdiobus_register_reset(mdiodev); - if (err) + if (err) { + gpiod_put(mdiodev->reset_gpio); + mdiodev->reset_gpio = NULL; return err; + } /* Assert the reset signal */ mdio_device_reset(mdiodev, 1); @@ -99,6 +102,7 @@ int mdiobus_unregister_device(struct mdio_device *mdiodev) if (mdiodev->bus->mdio_map[mdiodev->addr] != mdiodev) return -EINVAL; + gpiod_put(mdiodev->reset_gpio); reset_control_put(mdiodev->reset_ctrl); mdiodev->bus->mdio_map[mdiodev->addr] = NULL; @@ -775,9 +779,6 @@ void mdiobus_unregister(struct mii_bus *bus) if (!mdiodev) continue; - if (mdiodev->reset_gpio) - gpiod_put(mdiodev->reset_gpio); - mdiodev->device_remove(mdiodev); mdiodev->device_free(mdiodev); } @@ -850,6 +851,9 @@ int __mdiobus_read(struct mii_bus *bus, int addr, u32 regnum) lockdep_assert_held_once(&bus->mdio_lock); + if (addr >= PHY_MAX_ADDR) + return -ENXIO; + if (bus->read) retval = bus->read(bus, addr, regnum); else @@ -879,6 +883,9 @@ int __mdiobus_write(struct mii_bus *bus, int addr, u32 regnum, u16 val) lockdep_assert_held_once(&bus->mdio_lock); + if (addr >= PHY_MAX_ADDR) + return -ENXIO; + if (bus->write) err = bus->write(bus, addr, regnum, val); else @@ -940,6 +947,9 @@ int __mdiobus_c45_read(struct mii_bus *bus, int addr, int devad, u32 regnum) lockdep_assert_held_once(&bus->mdio_lock); + if (addr >= PHY_MAX_ADDR) + return -ENXIO; + if (bus->read_c45) retval = bus->read_c45(bus, addr, devad, regnum); else @@ -971,6 +981,9 @@ int __mdiobus_c45_write(struct mii_bus *bus, int addr, int devad, u32 regnum, lockdep_assert_held_once(&bus->mdio_lock); + if (addr >= PHY_MAX_ADDR) + return -ENXIO; + if (bus->write_c45) err = bus->write_c45(bus, addr, devad, regnum, val); else diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 9a0432145645f..6a114883ed8c5 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -372,6 +372,8 @@ static const struct kszphy_type ksz8051_type = { static const struct kszphy_type ksz8081_type = { .led_mode_reg = MII_KSZPHY_CTRL_2, + .cable_diag_reg = KSZ8081_LMD, + .pair_mask = KSZPHY_WIRE_PAIR_MASK, .has_broadcast_disable = true, .has_nand_tree_disable = true, .has_rmii_ref_clk_sel = true, @@ -4720,6 +4722,14 @@ static int lan8841_suspend(struct phy_device *phydev) return genphy_suspend(phydev); } +static int ksz9131_resume(struct phy_device *phydev) +{ + if (phydev->suspended && phy_interface_is_rgmii(phydev)) + ksz9131_config_rgmii_delay(phydev); + + return kszphy_resume(phydev); +} + static struct phy_driver ksphy_driver[] = { { .phy_id = PHY_ID_KS8737, @@ -4966,7 +4976,7 @@ static struct phy_driver ksphy_driver[] = { .get_strings = kszphy_get_strings, .get_stats = kszphy_get_stats, .suspend = kszphy_suspend, - .resume = kszphy_resume, + .resume = ksz9131_resume, .cable_test_start = ksz9x31_cable_test_start, .cable_test_get_status = ksz9x31_cable_test_get_status, .get_features = ksz9477_get_features, diff --git a/drivers/net/phy/microchip.c b/drivers/net/phy/microchip.c index 623607fd2cefd..cb8306cd98260 100644 --- a/drivers/net/phy/microchip.c +++ b/drivers/net/phy/microchip.c @@ -310,7 +310,7 @@ static void lan88xx_link_change_notify(struct phy_device *phydev) * As workaround, set to 10 before setting to 100 * at forced 100 F/H mode. */ - if (!phydev->autoneg && phydev->speed == 100) { + if (phydev->state == PHY_NOLINK && !phydev->autoneg && phydev->speed == 100) { /* disable phy interrupt */ temp = phy_read(phydev, LAN88XX_INT_MASK); temp &= ~LAN88XX_INT_MASK_MDINTPIN_EN_; diff --git a/drivers/net/phy/mscc/mscc.h b/drivers/net/phy/mscc/mscc.h index 7a962050a4d45..4ba6e32cf6d8d 100644 --- a/drivers/net/phy/mscc/mscc.h +++ b/drivers/net/phy/mscc/mscc.h @@ -362,6 +362,13 @@ struct vsc85xx_hw_stat { u16 mask; }; +struct vsc8531_skb_cb { + u32 ns; +}; + +#define VSC8531_SKB_CB(skb) \ + ((struct vsc8531_skb_cb *)((skb)->cb)) + struct vsc8531_private { int rate_magic; u16 supp_led_modes; @@ -410,6 +417,11 @@ struct vsc8531_private { */ struct mutex ts_lock; struct mutex phc_lock; + + /* list of skbs that were received and need timestamp information but it + * didn't received it yet + */ + struct sk_buff_head rx_skbs_list; }; /* Shared structure between the PHYs of the same package. @@ -464,6 +476,7 @@ static inline void vsc8584_config_macsec_intr(struct phy_device *phydev) void vsc85xx_link_change_notify(struct phy_device *phydev); void vsc8584_config_ts_intr(struct phy_device *phydev); int vsc8584_ptp_init(struct phy_device *phydev); +void vsc8584_ptp_deinit(struct phy_device *phydev); int vsc8584_ptp_probe_once(struct phy_device *phydev); int vsc8584_ptp_probe(struct phy_device *phydev); irqreturn_t vsc8584_handle_ts_interrupt(struct phy_device *phydev); @@ -478,6 +491,9 @@ static inline int vsc8584_ptp_init(struct phy_device *phydev) { return 0; } +static inline void vsc8584_ptp_deinit(struct phy_device *phydev) +{ +} static inline int vsc8584_ptp_probe_once(struct phy_device *phydev) { return 0; diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c index 4171f01d34e57..3a932b30f4358 100644 --- a/drivers/net/phy/mscc/mscc_main.c +++ b/drivers/net/phy/mscc/mscc_main.c @@ -2335,6 +2335,11 @@ static int vsc85xx_probe(struct phy_device *phydev) return vsc85xx_dt_led_modes_get(phydev, default_mode); } +static void vsc85xx_remove(struct phy_device *phydev) +{ + vsc8584_ptp_deinit(phydev); +} + /* Microsemi VSC85xx PHYs */ static struct phy_driver vsc85xx_driver[] = { { @@ -2589,6 +2594,7 @@ static struct phy_driver vsc85xx_driver[] = { .config_intr = &vsc85xx_config_intr, .suspend = &genphy_suspend, .resume = &genphy_resume, + .remove = &vsc85xx_remove, .probe = &vsc8574_probe, .set_wol = &vsc85xx_wol_set, .get_wol = &vsc85xx_wol_get, @@ -2614,6 +2620,7 @@ static struct phy_driver vsc85xx_driver[] = { .config_intr = &vsc85xx_config_intr, .suspend = &genphy_suspend, .resume = &genphy_resume, + .remove = &vsc85xx_remove, .probe = &vsc8574_probe, .set_wol = &vsc85xx_wol_set, .get_wol = &vsc85xx_wol_get, @@ -2639,6 +2646,7 @@ static struct phy_driver vsc85xx_driver[] = { .config_intr = &vsc85xx_config_intr, .suspend = &genphy_suspend, .resume = &genphy_resume, + .remove = &vsc85xx_remove, .probe = &vsc8584_probe, .get_tunable = &vsc85xx_get_tunable, .set_tunable = &vsc85xx_set_tunable, @@ -2662,6 +2670,7 @@ static struct phy_driver vsc85xx_driver[] = { .config_intr = &vsc85xx_config_intr, .suspend = &genphy_suspend, .resume = &genphy_resume, + .remove = &vsc85xx_remove, .probe = &vsc8584_probe, .get_tunable = &vsc85xx_get_tunable, .set_tunable = &vsc85xx_set_tunable, @@ -2685,6 +2694,7 @@ static struct phy_driver vsc85xx_driver[] = { .config_intr = &vsc85xx_config_intr, .suspend = &genphy_suspend, .resume = &genphy_resume, + .remove = &vsc85xx_remove, .probe = &vsc8584_probe, .get_tunable = &vsc85xx_get_tunable, .set_tunable = &vsc85xx_set_tunable, diff --git a/drivers/net/phy/mscc/mscc_ptp.c b/drivers/net/phy/mscc/mscc_ptp.c index cf728bfd83e22..939a8a17595ef 100644 --- a/drivers/net/phy/mscc/mscc_ptp.c +++ b/drivers/net/phy/mscc/mscc_ptp.c @@ -455,12 +455,12 @@ static void vsc85xx_dequeue_skb(struct vsc85xx_ptp *ptp) *p++ = (reg >> 24) & 0xff; } - len = skb_queue_len(&ptp->tx_queue); + len = skb_queue_len_lockless(&ptp->tx_queue); if (len < 1) return; while (len--) { - skb = __skb_dequeue(&ptp->tx_queue); + skb = skb_dequeue(&ptp->tx_queue); if (!skb) return; @@ -485,7 +485,7 @@ static void vsc85xx_dequeue_skb(struct vsc85xx_ptp *ptp) * packet in the FIFO right now, reschedule it for later * packets. */ - __skb_queue_tail(&ptp->tx_queue, skb); + skb_queue_tail(&ptp->tx_queue, skb); } } @@ -897,6 +897,7 @@ static int vsc85xx_eth1_conf(struct phy_device *phydev, enum ts_blk blk, get_unaligned_be32(ptp_multicast)); } else { val |= ANA_ETH1_FLOW_ADDR_MATCH2_ANY_MULTICAST; + val |= ANA_ETH1_FLOW_ADDR_MATCH2_ANY_UNICAST; vsc85xx_ts_write_csr(phydev, blk, MSCC_ANA_ETH1_FLOW_ADDR_MATCH2(0), val); vsc85xx_ts_write_csr(phydev, blk, @@ -943,7 +944,9 @@ static int vsc85xx_ip1_conf(struct phy_device *phydev, enum ts_blk blk, /* UDP checksum offset in IPv4 packet * according to: https://tools.ietf.org/html/rfc768 */ - val |= IP1_NXT_PROT_UDP_CHKSUM_OFF(26) | IP1_NXT_PROT_UDP_CHKSUM_CLEAR; + val |= IP1_NXT_PROT_UDP_CHKSUM_OFF(26); + if (enable) + val |= IP1_NXT_PROT_UDP_CHKSUM_CLEAR; vsc85xx_ts_write_csr(phydev, blk, MSCC_ANA_IP1_NXT_PROT_UDP_CHKSUM, val); @@ -1064,6 +1067,7 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) case HWTSTAMP_TX_ON: break; case HWTSTAMP_TX_OFF: + skb_queue_purge(&vsc8531->ptp->tx_queue); break; default: return -ERANGE; @@ -1088,9 +1092,6 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) mutex_lock(&vsc8531->ts_lock); - __skb_queue_purge(&vsc8531->ptp->tx_queue); - __skb_queue_head_init(&vsc8531->ptp->tx_queue); - /* Disable predictor while configuring the 1588 block */ val = vsc85xx_ts_read_csr(phydev, PROCESSOR, MSCC_PHY_PTP_INGR_PREDICTOR); @@ -1165,18 +1166,22 @@ static void vsc85xx_txtstamp(struct mii_timestamper *mii_ts, container_of(mii_ts, struct vsc8531_private, mii_ts); if (!vsc8531->ptp->configured) - return; + goto out; - if (vsc8531->ptp->tx_type == HWTSTAMP_TX_OFF) { - kfree_skb(skb); - return; - } + if (vsc8531->ptp->tx_type == HWTSTAMP_TX_OFF) + goto out; + + if (vsc8531->ptp->tx_type == HWTSTAMP_TX_ONESTEP_SYNC) + if (ptp_msg_is_sync(skb, type)) + goto out; skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; - mutex_lock(&vsc8531->ts_lock); - __skb_queue_tail(&vsc8531->ptp->tx_queue, skb); - mutex_unlock(&vsc8531->ts_lock); + skb_queue_tail(&vsc8531->ptp->tx_queue, skb); + return; + +out: + kfree_skb(skb); } static bool vsc85xx_rxtstamp(struct mii_timestamper *mii_ts, @@ -1184,9 +1189,7 @@ static bool vsc85xx_rxtstamp(struct mii_timestamper *mii_ts, { struct vsc8531_private *vsc8531 = container_of(mii_ts, struct vsc8531_private, mii_ts); - struct skb_shared_hwtstamps *shhwtstamps = NULL; struct vsc85xx_ptphdr *ptphdr; - struct timespec64 ts; unsigned long ns; if (!vsc8531->ptp->configured) @@ -1196,27 +1199,52 @@ static bool vsc85xx_rxtstamp(struct mii_timestamper *mii_ts, type == PTP_CLASS_NONE) return false; - vsc85xx_gettime(&vsc8531->ptp->caps, &ts); - ptphdr = get_ptp_header_rx(skb, vsc8531->ptp->rx_filter); if (!ptphdr) return false; - shhwtstamps = skb_hwtstamps(skb); - memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps)); - ns = ntohl(ptphdr->rsrvd2); - /* nsec is in reserved field */ - if (ts.tv_nsec < ns) - ts.tv_sec--; + VSC8531_SKB_CB(skb)->ns = ns; + skb_queue_tail(&vsc8531->rx_skbs_list, skb); - shhwtstamps->hwtstamp = ktime_set(ts.tv_sec, ns); - netif_rx(skb); + ptp_schedule_worker(vsc8531->ptp->ptp_clock, 0); return true; } +static long vsc85xx_do_aux_work(struct ptp_clock_info *info) +{ + struct vsc85xx_ptp *ptp = container_of(info, struct vsc85xx_ptp, caps); + struct skb_shared_hwtstamps *shhwtstamps = NULL; + struct phy_device *phydev = ptp->phydev; + struct vsc8531_private *priv = phydev->priv; + struct sk_buff_head received; + struct sk_buff *rx_skb; + struct timespec64 ts; + unsigned long flags; + + __skb_queue_head_init(&received); + spin_lock_irqsave(&priv->rx_skbs_list.lock, flags); + skb_queue_splice_tail_init(&priv->rx_skbs_list, &received); + spin_unlock_irqrestore(&priv->rx_skbs_list.lock, flags); + + vsc85xx_gettime(info, &ts); + while ((rx_skb = __skb_dequeue(&received)) != NULL) { + shhwtstamps = skb_hwtstamps(rx_skb); + memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps)); + + if (ts.tv_nsec < VSC8531_SKB_CB(rx_skb)->ns) + ts.tv_sec--; + + shhwtstamps->hwtstamp = ktime_set(ts.tv_sec, + VSC8531_SKB_CB(rx_skb)->ns); + netif_rx(rx_skb); + } + + return -1; +} + static const struct ptp_clock_info vsc85xx_clk_caps = { .owner = THIS_MODULE, .name = "VSC85xx timer", @@ -1230,6 +1258,7 @@ static const struct ptp_clock_info vsc85xx_clk_caps = { .adjfine = &vsc85xx_adjfine, .gettime64 = &vsc85xx_gettime, .settime64 = &vsc85xx_settime, + .do_aux_work = &vsc85xx_do_aux_work, }; static struct vsc8531_private *vsc8584_base_priv(struct phy_device *phydev) @@ -1264,7 +1293,6 @@ static void vsc8584_set_input_clk_configured(struct phy_device *phydev) static int __vsc8584_init_ptp(struct phy_device *phydev) { - struct vsc8531_private *vsc8531 = phydev->priv; static const u32 ltc_seq_e[] = { 0, 400000, 0, 0, 0 }; static const u8 ltc_seq_a[] = { 8, 6, 5, 4, 2 }; u32 val; @@ -1481,17 +1509,7 @@ static int __vsc8584_init_ptp(struct phy_device *phydev) vsc85xx_ts_eth_cmp1_sig(phydev); - vsc8531->mii_ts.rxtstamp = vsc85xx_rxtstamp; - vsc8531->mii_ts.txtstamp = vsc85xx_txtstamp; - vsc8531->mii_ts.hwtstamp = vsc85xx_hwtstamp; - vsc8531->mii_ts.ts_info = vsc85xx_ts_info; - phydev->mii_ts = &vsc8531->mii_ts; - - memcpy(&vsc8531->ptp->caps, &vsc85xx_clk_caps, sizeof(vsc85xx_clk_caps)); - - vsc8531->ptp->ptp_clock = ptp_clock_register(&vsc8531->ptp->caps, - &phydev->mdio.dev); - return PTR_ERR_OR_ZERO(vsc8531->ptp->ptp_clock); + return 0; } void vsc8584_config_ts_intr(struct phy_device *phydev) @@ -1518,6 +1536,17 @@ int vsc8584_ptp_init(struct phy_device *phydev) return 0; } +void vsc8584_ptp_deinit(struct phy_device *phydev) +{ + struct vsc8531_private *vsc8531 = phydev->priv; + + if (vsc8531->ptp->ptp_clock) { + ptp_clock_unregister(vsc8531->ptp->ptp_clock); + skb_queue_purge(&vsc8531->rx_skbs_list); + skb_queue_purge(&vsc8531->ptp->tx_queue); + } +} + irqreturn_t vsc8584_handle_ts_interrupt(struct phy_device *phydev) { struct vsc8531_private *priv = phydev->priv; @@ -1538,7 +1567,7 @@ irqreturn_t vsc8584_handle_ts_interrupt(struct phy_device *phydev) if (rc & VSC85XX_1588_INT_FIFO_ADD) { vsc85xx_get_tx_ts(priv->ptp); } else if (rc & VSC85XX_1588_INT_FIFO_OVERFLOW) { - __skb_queue_purge(&priv->ptp->tx_queue); + skb_queue_purge(&priv->ptp->tx_queue); vsc85xx_ts_reset_fifo(phydev); } @@ -1557,6 +1586,8 @@ int vsc8584_ptp_probe(struct phy_device *phydev) mutex_init(&vsc8531->phc_lock); mutex_init(&vsc8531->ts_lock); + skb_queue_head_init(&vsc8531->rx_skbs_list); + skb_queue_head_init(&vsc8531->ptp->tx_queue); /* Retrieve the shared load/save GPIO. Request it as non exclusive as * the same GPIO can be requested by all the PHYs of the same package. @@ -1574,7 +1605,16 @@ int vsc8584_ptp_probe(struct phy_device *phydev) vsc8531->ptp->phydev = phydev; - return 0; + vsc8531->mii_ts.rxtstamp = vsc85xx_rxtstamp; + vsc8531->mii_ts.txtstamp = vsc85xx_txtstamp; + vsc8531->mii_ts.hwtstamp = vsc85xx_hwtstamp; + vsc8531->mii_ts.ts_info = vsc85xx_ts_info; + phydev->mii_ts = &vsc8531->mii_ts; + + memcpy(&vsc8531->ptp->caps, &vsc85xx_clk_caps, sizeof(vsc85xx_clk_caps)); + vsc8531->ptp->ptp_clock = ptp_clock_register(&vsc8531->ptp->caps, + &phydev->mdio.dev); + return PTR_ERR_OR_ZERO(vsc8531->ptp->ptp_clock); } int vsc8584_ptp_probe_once(struct phy_device *phydev) diff --git a/drivers/net/phy/mscc/mscc_ptp.h b/drivers/net/phy/mscc/mscc_ptp.h index da3465360e901..ae9ad925bfa8c 100644 --- a/drivers/net/phy/mscc/mscc_ptp.h +++ b/drivers/net/phy/mscc/mscc_ptp.h @@ -98,6 +98,7 @@ #define MSCC_ANA_ETH1_FLOW_ADDR_MATCH2(x) (MSCC_ANA_ETH1_FLOW_ENA(x) + 3) #define ANA_ETH1_FLOW_ADDR_MATCH2_MASK_MASK GENMASK(22, 20) #define ANA_ETH1_FLOW_ADDR_MATCH2_ANY_MULTICAST 0x400000 +#define ANA_ETH1_FLOW_ADDR_MATCH2_ANY_UNICAST 0x200000 #define ANA_ETH1_FLOW_ADDR_MATCH2_FULL_ADDR 0x100000 #define ANA_ETH1_FLOW_ADDR_MATCH2_SRC_DEST_MASK GENMASK(17, 16) #define ANA_ETH1_FLOW_ADDR_MATCH2_SRC_DEST 0x020000 diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index df54c137c5f5f..cf171bdd667aa 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -1482,6 +1482,19 @@ void phy_state_machine(struct work_struct *work) } break; case PHY_HALTED: + if (phydev->link) { + if (phydev->autoneg == AUTONEG_ENABLE) { + phydev->speed = SPEED_UNKNOWN; + phydev->duplex = DUPLEX_UNKNOWN; + } + if (phydev->master_slave_state != + MASTER_SLAVE_STATE_UNSUPPORTED) + phydev->master_slave_state = + MASTER_SLAVE_STATE_UNKNOWN; + phydev->mdix = ETH_TP_MDI_INVALID; + linkmode_zero(phydev->lp_advertising); + } + fallthrough; case PHY_ERROR: if (phydev->link) { phydev->link = 0; diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index ec2a3d16b1a2d..875788918bcb3 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1806,8 +1806,10 @@ void phy_detach(struct phy_device *phydev) struct module *ndev_owner = NULL; struct mii_bus *bus; - if (phydev->devlink) + if (phydev->devlink) { device_link_del(phydev->devlink); + phydev->devlink = NULL; + } if (phydev->sysfs_links) { if (dev) @@ -3375,7 +3377,8 @@ static int phy_probe(struct device *dev) /* Get the LEDs from the device tree, and instantiate standard * LEDs for them. */ - if (IS_ENABLED(CONFIG_PHYLIB_LEDS)) + if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev) && + !phy_driver_is_genphy_10g(phydev)) err = of_phy_leds(phydev); out: @@ -3392,7 +3395,8 @@ static int phy_remove(struct device *dev) cancel_delayed_work_sync(&phydev->state_queue); - if (IS_ENABLED(CONFIG_PHYLIB_LEDS)) + if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev) && + !phy_driver_is_genphy_10g(phydev)) phy_leds_unregister(phydev); phydev->state = PHY_DOWN; diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index b5f012619e42d..800e8b9eb4532 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -1718,7 +1718,7 @@ bool phylink_expects_phy(struct phylink *pl) { if (pl->cfg_link_an_mode == MLO_AN_FIXED || (pl->cfg_link_an_mode == MLO_AN_INBAND && - phy_interface_mode_is_8023z(pl->link_config.interface))) + phy_interface_mode_is_8023z(pl->link_interface))) return false; return true; } diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c index c88edb19d2e71..5f9bb0ebe455d 100644 --- a/drivers/net/phy/smsc.c +++ b/drivers/net/phy/smsc.c @@ -155,10 +155,29 @@ static int smsc_phy_reset(struct phy_device *phydev) static int lan87xx_config_aneg(struct phy_device *phydev) { - int rc; + u8 mdix_ctrl; int val; + int rc; - switch (phydev->mdix_ctrl) { + /* When auto-negotiation is disabled (forced mode), the PHY's + * Auto-MDIX will continue toggling the TX/RX pairs. + * + * To establish a stable link, we must select a fixed MDI mode. + * If the user has not specified a fixed MDI mode (i.e., mdix_ctrl is + * 'auto'), we default to ETH_TP_MDI. This choice of a ETH_TP_MDI mode + * mirrors the behavior the hardware would exhibit if the AUTOMDIX_EN + * strap were configured for a fixed MDI connection. + */ + if (phydev->autoneg == AUTONEG_DISABLE) { + if (phydev->mdix_ctrl == ETH_TP_MDI_AUTO) + mdix_ctrl = ETH_TP_MDI; + else + mdix_ctrl = phydev->mdix_ctrl; + } else { + mdix_ctrl = phydev->mdix_ctrl; + } + + switch (mdix_ctrl) { case ETH_TP_MDI: val = SPECIAL_CTRL_STS_OVRRD_AMDIX_; break; @@ -167,7 +186,8 @@ static int lan87xx_config_aneg(struct phy_device *phydev) SPECIAL_CTRL_STS_AMDIX_STATE_; break; case ETH_TP_MDI_AUTO: - val = SPECIAL_CTRL_STS_AMDIX_ENABLE_; + val = SPECIAL_CTRL_STS_OVRRD_AMDIX_ | + SPECIAL_CTRL_STS_AMDIX_ENABLE_; break; default: return genphy_config_aneg(phydev); @@ -183,7 +203,7 @@ static int lan87xx_config_aneg(struct phy_device *phydev) rc |= val; phy_write(phydev, SPECIAL_CTRL_STS, rc); - phydev->mdix = phydev->mdix_ctrl; + phydev->mdix = mdix_ctrl; return genphy_config_aneg(phydev); } @@ -261,6 +281,33 @@ int lan87xx_read_status(struct phy_device *phydev) } EXPORT_SYMBOL_GPL(lan87xx_read_status); +static int lan87xx_phy_config_init(struct phy_device *phydev) +{ + int rc; + + /* The LAN87xx PHY's initial MDI-X mode is determined by the AUTOMDIX_EN + * hardware strap, but the driver cannot read the strap's status. This + * creates an unpredictable initial state. + * + * To ensure consistent and reliable behavior across all boards, + * override the strap configuration on initialization and force the PHY + * into a known state with Auto-MDIX enabled, which is the expected + * default for modern hardware. + */ + rc = phy_modify(phydev, SPECIAL_CTRL_STS, + SPECIAL_CTRL_STS_OVRRD_AMDIX_ | + SPECIAL_CTRL_STS_AMDIX_ENABLE_ | + SPECIAL_CTRL_STS_AMDIX_STATE_, + SPECIAL_CTRL_STS_OVRRD_AMDIX_ | + SPECIAL_CTRL_STS_AMDIX_ENABLE_); + if (rc < 0) + return rc; + + phydev->mdix_ctrl = ETH_TP_MDI_AUTO; + + return smsc_phy_config_init(phydev); +} + static int lan874x_phy_config_init(struct phy_device *phydev) { u16 val; @@ -696,7 +743,7 @@ static struct phy_driver smsc_phy_driver[] = { /* basic functions */ .read_status = lan87xx_read_status, - .config_init = smsc_phy_config_init, + .config_init = lan87xx_phy_config_init, .soft_reset = smsc_phy_reset, .config_aneg = lan87xx_config_aneg, @@ -739,6 +786,7 @@ static struct phy_driver smsc_phy_driver[] = { /* PHY_BASIC_FEATURES */ + .flags = PHY_RST_AFTER_CLK_EN, .probe = smsc_phy_probe, /* basic functions */ diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c index ee1527cf3d0c4..46ac51217114b 100644 --- a/drivers/net/ppp/ppp_generic.c +++ b/drivers/net/ppp/ppp_generic.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1613,11 +1614,14 @@ static int ppp_fill_forward_path(struct net_device_path_ctx *ctx, if (ppp->flags & SC_MULTILINK) return -EOPNOTSUPP; - if (list_empty(&ppp->channels)) + pch = list_first_or_null_rcu(&ppp->channels, struct channel, clist); + if (!pch) + return -ENODEV; + + chan = READ_ONCE(pch->chan); + if (!chan) return -ENODEV; - pch = list_first_entry(&ppp->channels, struct channel, clist); - chan = pch->chan; if (!chan->ops->fill_forward_path) return -EOPNOTSUPP; @@ -1749,7 +1753,6 @@ pad_compress_skb(struct ppp *ppp, struct sk_buff *skb) */ if (net_ratelimit()) netdev_err(ppp->dev, "ppp: compressor dropped pkt\n"); - kfree_skb(skb); consume_skb(new_skb); new_skb = NULL; } @@ -1851,9 +1854,10 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb) "down - pkt dropped.\n"); goto drop; } - skb = pad_compress_skb(ppp, skb); - if (!skb) + new_skb = pad_compress_skb(ppp, skb); + if (!new_skb) goto drop; + skb = new_skb; } /* @@ -3000,7 +3004,7 @@ ppp_unregister_channel(struct ppp_channel *chan) */ down_write(&pch->chan_sem); spin_lock_bh(&pch->downl); - pch->chan = NULL; + WRITE_ONCE(pch->chan, NULL); spin_unlock_bh(&pch->downl); up_write(&pch->chan_sem); ppp_disconnect_channel(pch); @@ -3506,7 +3510,7 @@ ppp_connect_channel(struct channel *pch, int unit) hdrlen = pch->file.hdrlen + 2; /* for protocol bytes */ if (hdrlen > ppp->dev->hard_header_len) ppp->dev->hard_header_len = hdrlen; - list_add_tail(&pch->clist, &ppp->channels); + list_add_tail_rcu(&pch->clist, &ppp->channels); ++ppp->n_channels; pch->ppp = ppp; refcount_inc(&ppp->file.refcnt); @@ -3536,10 +3540,11 @@ ppp_disconnect_channel(struct channel *pch) if (ppp) { /* remove it from the ppp unit's list */ ppp_lock(ppp); - list_del(&pch->clist); + list_del_rcu(&pch->clist); if (--ppp->n_channels == 0) wake_up_interruptible(&ppp->file.rwait); ppp_unlock(ppp); + synchronize_net(); if (refcount_dec_and_test(&ppp->file.refcnt)) ppp_destroy_interface(ppp); err = 0; diff --git a/drivers/net/ppp/pptp.c b/drivers/net/ppp/pptp.c index 6833ef0c79305..3a10303eb756a 100644 --- a/drivers/net/ppp/pptp.c +++ b/drivers/net/ppp/pptp.c @@ -159,19 +159,17 @@ static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb) int len; unsigned char *data; __u32 seq_recv; - - struct rtable *rt; struct net_device *tdev; struct iphdr *iph; int max_headroom; if (sk_pppox(po)->sk_state & PPPOX_DEAD) - goto tx_error; + goto tx_drop; rt = pptp_route_output(po, &fl4); if (IS_ERR(rt)) - goto tx_error; + goto tx_drop; tdev = rt->dst.dev; @@ -179,16 +177,20 @@ static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb) if (skb_headroom(skb) < max_headroom || skb_cloned(skb) || skb_shared(skb)) { struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom); - if (!new_skb) { - ip_rt_put(rt); + + if (!new_skb) goto tx_error; - } + if (skb->sk) skb_set_owner_w(new_skb, skb->sk); consume_skb(skb); skb = new_skb; } + /* Ensure we can safely access protocol field and LCP code */ + if (!pskb_may_pull(skb, 3)) + goto tx_error; + data = skb->data; islcp = ((data[0] << 8) + data[1]) == PPP_LCP && 1 <= data[2] && data[2] <= 7; @@ -262,6 +264,8 @@ static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb) return 1; tx_error: + ip_rt_put(rt); +tx_drop: kfree_skb(skb); return 1; } diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index 0a53ec293d040..dcaa62377808c 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -396,9 +396,9 @@ static void tbnet_tear_down(struct tbnet *net, bool send_logout) ret = tb_xdomain_disable_paths(net->xd, net->local_transmit_path, - net->rx_ring.ring->hop, + net->tx_ring.ring->hop, net->remote_transmit_path, - net->tx_ring.ring->hop); + net->rx_ring.ring->hop); if (ret) netdev_warn(net->dev, "failed to disable DMA paths\n"); @@ -662,9 +662,9 @@ static void tbnet_connected_work(struct work_struct *work) goto err_free_rx_buffers; ret = tb_xdomain_enable_paths(net->xd, net->local_transmit_path, - net->rx_ring.ring->hop, + net->tx_ring.ring->hop, net->remote_transmit_path, - net->tx_ring.ring->hop); + net->rx_ring.ring->hop); if (ret) { netdev_err(net->dev, "failed to enable DMA paths\n"); goto err_free_tx_buffers; @@ -924,8 +924,12 @@ static int tbnet_open(struct net_device *dev) netif_carrier_off(dev); - ring = tb_ring_alloc_tx(xd->tb->nhi, -1, TBNET_RING_SIZE, - RING_FLAG_FRAME); + flags = RING_FLAG_FRAME; + /* Only enable full E2E if the other end supports it too */ + if (tbnet_e2e && net->svc->prtcstns & TBNET_E2E) + flags |= RING_FLAG_E2E; + + ring = tb_ring_alloc_tx(xd->tb->nhi, -1, TBNET_RING_SIZE, flags); if (!ring) { netdev_err(dev, "failed to allocate Tx ring\n"); return -ENOMEM; @@ -944,11 +948,6 @@ static int tbnet_open(struct net_device *dev) sof_mask = BIT(TBIP_PDF_FRAME_START); eof_mask = BIT(TBIP_PDF_FRAME_END); - flags = RING_FLAG_FRAME; - /* Only enable full E2E if the other end supports it too */ - if (tbnet_e2e && net->svc->prtcstns & TBNET_E2E) - flags |= RING_FLAG_E2E; - ring = tb_ring_alloc_rx(xd->tb->nhi, -1, TBNET_RING_SIZE, flags, net->tx_ring.ring->hop, sof_mask, eof_mask, tbnet_start_poll, net); diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c index 284375f662f1e..3ebb1f84d3025 100644 --- a/drivers/net/usb/aqc111.c +++ b/drivers/net/usb/aqc111.c @@ -30,11 +30,14 @@ static int aqc111_read_cmd_nopm(struct usbnet *dev, u8 cmd, u16 value, ret = usbnet_read_cmd_nopm(dev, cmd, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, value, index, data, size); - if (unlikely(ret < 0)) + if (unlikely(ret < size)) { netdev_warn(dev->net, "Failed to read(0x%x) reg index 0x%04x: %d\n", cmd, index, ret); + ret = ret < 0 ? ret : -ENODATA; + } + return ret; } @@ -46,11 +49,14 @@ static int aqc111_read_cmd(struct usbnet *dev, u8 cmd, u16 value, ret = usbnet_read_cmd(dev, cmd, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, value, index, data, size); - if (unlikely(ret < 0)) + if (unlikely(ret < size)) { netdev_warn(dev->net, "Failed to read(0x%x) reg index 0x%04x: %d\n", cmd, index, ret); + ret = ret < 0 ? ret : -ENODATA; + } + return ret; } diff --git a/drivers/net/usb/asix.h b/drivers/net/usb/asix.h index 74162190bccc1..8531b804021aa 100644 --- a/drivers/net/usb/asix.h +++ b/drivers/net/usb/asix.h @@ -224,7 +224,6 @@ int asix_write_rx_ctl(struct usbnet *dev, u16 mode, int in_pm); u16 asix_read_medium_status(struct usbnet *dev, int in_pm); int asix_write_medium_mode(struct usbnet *dev, u16 mode, int in_pm); -void asix_adjust_link(struct net_device *netdev); int asix_write_gpio(struct usbnet *dev, u16 value, int sleep, int in_pm); diff --git a/drivers/net/usb/asix_common.c b/drivers/net/usb/asix_common.c index 72ffc89b477ad..7fd763917ae2c 100644 --- a/drivers/net/usb/asix_common.c +++ b/drivers/net/usb/asix_common.c @@ -414,28 +414,6 @@ int asix_write_medium_mode(struct usbnet *dev, u16 mode, int in_pm) return ret; } -/* set MAC link settings according to information from phylib */ -void asix_adjust_link(struct net_device *netdev) -{ - struct phy_device *phydev = netdev->phydev; - struct usbnet *dev = netdev_priv(netdev); - u16 mode = 0; - - if (phydev->link) { - mode = AX88772_MEDIUM_DEFAULT; - - if (phydev->duplex == DUPLEX_HALF) - mode &= ~AX_MEDIUM_FD; - - if (phydev->speed != SPEED_100) - mode &= ~AX_MEDIUM_PS; - } - - asix_write_medium_mode(dev, mode, 0); - phy_print_status(phydev); - usbnet_link_change(dev, phydev->link, 0); -} - int asix_write_gpio(struct usbnet *dev, u16 value, int sleep, int in_pm) { int ret; diff --git a/drivers/net/usb/asix_devices.c b/drivers/net/usb/asix_devices.c index ec4dcf89cbedd..f70b8ab0ccd26 100644 --- a/drivers/net/usb/asix_devices.c +++ b/drivers/net/usb/asix_devices.c @@ -230,7 +230,9 @@ static int ax88172_bind(struct usbnet *dev, struct usb_interface *intf) int i; unsigned long gpio_bits = dev->driver_info->data; - usbnet_get_endpoints(dev,intf); + ret = usbnet_get_endpoints(dev, intf); + if (ret) + goto out; /* Toggle the GPIOs in a manufacturer/model specific way */ for (i = 2; i >= 0; i--) { @@ -625,6 +627,21 @@ static void ax88772_suspend(struct usbnet *dev) asix_read_medium_status(dev, 1)); } +/* Notes on PM callbacks and locking context: + * + * - asix_suspend()/asix_resume() are invoked for both runtime PM and + * system-wide suspend/resume. For struct usb_driver the ->resume() + * callback does not receive pm_message_t, so the resume type cannot + * be distinguished here. + * + * - The MAC driver must hold RTNL when calling phylink interfaces such as + * phylink_suspend()/resume(). Those calls will also perform MDIO I/O. + * + * - Taking RTNL and doing MDIO from a runtime-PM resume callback (while + * the USB PM lock is held) is fragile. Since autosuspend brings no + * measurable power saving here, we block it by holding a PM usage + * reference in ax88772_bind(). + */ static int asix_suspend(struct usb_interface *intf, pm_message_t message) { struct usbnet *dev = usb_get_intfdata(intf); @@ -676,6 +693,7 @@ static int ax88772_init_mdio(struct usbnet *dev) priv->mdio->read = &asix_mdio_bus_read; priv->mdio->write = &asix_mdio_bus_write; priv->mdio->name = "Asix MDIO Bus"; + priv->mdio->phy_mask = ~(BIT(priv->phy_addr & 0x1f) | BIT(AX_EMBD_PHY_ADDR)); /* mii bus name is usb-- */ snprintf(priv->mdio->id, MII_BUS_ID_SIZE, "usb-%03d:%03d", dev->udev->bus->busnum, dev->udev->devnum); @@ -752,7 +770,6 @@ static void ax88772_mac_link_down(struct phylink_config *config, struct usbnet *dev = netdev_priv(to_net_dev(config->dev)); asix_write_medium_mode(dev, 0, 0); - usbnet_link_change(dev, false, false); } static void ax88772_mac_link_up(struct phylink_config *config, @@ -783,7 +800,6 @@ static void ax88772_mac_link_up(struct phylink_config *config, m |= AX_MEDIUM_RFC; asix_write_medium_mode(dev, m, 0); - usbnet_link_change(dev, true, false); } static const struct phylink_mac_ops ax88772_phylink_mac_ops = { @@ -834,7 +850,9 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf) dev->driver_priv = priv; - usbnet_get_endpoints(dev, intf); + ret = usbnet_get_endpoints(dev, intf); + if (ret) + return ret; /* Maybe the boot loader passed the MAC address via device tree */ if (!eth_platform_get_mac_address(&dev->udev->dev, buf)) { @@ -920,6 +938,13 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf) if (ret) goto initphy_err; + /* Keep this interface runtime-PM active by taking a usage ref. + * Prevents runtime suspend while bound and avoids resume paths + * that could deadlock (autoresume under RTNL while USB PM lock + * is held, phylink/MDIO wants RTNL). + */ + pm_runtime_get_noresume(&intf->dev); + return 0; initphy_err: @@ -949,6 +974,8 @@ static void ax88772_unbind(struct usbnet *dev, struct usb_interface *intf) phylink_destroy(priv->phylink); ax88772_mdio_unregister(priv); asix_rx_fixup_common_free(dev->driver_priv); + /* Drop the PM usage ref taken in bind() */ + pm_runtime_put(&intf->dev); } static void ax88178_unbind(struct usbnet *dev, struct usb_interface *intf) @@ -1258,7 +1285,9 @@ static int ax88178_bind(struct usbnet *dev, struct usb_interface *intf) int ret; u8 buf[ETH_ALEN] = {0}; - usbnet_get_endpoints(dev,intf); + ret = usbnet_get_endpoints(dev, intf); + if (ret) + return ret; /* Get the MAC address */ ret = asix_read_cmd(dev, AX_CMD_READ_NODE_ID, 0, 0, ETH_ALEN, buf, 0); @@ -1350,10 +1379,9 @@ static const struct driver_info ax88772_info = { .description = "ASIX AX88772 USB 2.0 Ethernet", .bind = ax88772_bind, .unbind = ax88772_unbind, - .status = asix_status, .reset = ax88772_reset, .stop = ax88772_stop, - .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_LINK_INTR | FLAG_MULTI_PACKET, + .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_MULTI_PACKET, .rx_fixup = asix_rx_fixup_common, .tx_fixup = asix_tx_fixup, }; @@ -1362,11 +1390,9 @@ static const struct driver_info ax88772b_info = { .description = "ASIX AX88772B USB 2.0 Ethernet", .bind = ax88772_bind, .unbind = ax88772_unbind, - .status = asix_status, .reset = ax88772_reset, .stop = ax88772_stop, - .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_LINK_INTR | - FLAG_MULTI_PACKET, + .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_MULTI_PACKET, .rx_fixup = asix_rx_fixup_common, .tx_fixup = asix_tx_fixup, .data = FLAG_EEPROM_MAC, @@ -1376,11 +1402,9 @@ static const struct driver_info lxausb_t1l_info = { .description = "Linux Automation GmbH USB 10Base-T1L", .bind = ax88772_bind, .unbind = ax88772_unbind, - .status = asix_status, .reset = ax88772_reset, .stop = ax88772_stop, - .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_LINK_INTR | - FLAG_MULTI_PACKET, + .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_MULTI_PACKET, .rx_fixup = asix_rx_fixup_common, .tx_fixup = asix_tx_fixup, .data = FLAG_EEPROM_MAC, @@ -1412,10 +1436,8 @@ static const struct driver_info hg20f9_info = { .description = "HG20F9 USB 2.0 Ethernet", .bind = ax88772_bind, .unbind = ax88772_unbind, - .status = asix_status, .reset = ax88772_reset, - .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_LINK_INTR | - FLAG_MULTI_PACKET, + .flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_MULTI_PACKET, .rx_fixup = asix_rx_fixup_common, .tx_fixup = asix_tx_fixup, .data = FLAG_EEPROM_MAC, @@ -1608,6 +1630,11 @@ static struct usb_driver asix_driver = { .resume = asix_resume, .reset_resume = asix_resume, .disconnect = usbnet_disconnect, + /* usbnet enables autosuspend by default (supports_autosuspend=1). + * We keep runtime-PM active for AX88772* by taking a PM usage + * reference in ax88772_bind() (pm_runtime_get_noresume()) and + * dropping it in unbind(), which effectively blocks autosuspend. + */ .supports_autosuspend = 1, .disable_hub_initiated_lpm = 1, }; diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c index db05622f1f703..22554daaf6ff1 100644 --- a/drivers/net/usb/cdc_ncm.c +++ b/drivers/net/usb/cdc_ncm.c @@ -893,6 +893,10 @@ int cdc_ncm_bind_common(struct usbnet *dev, struct usb_interface *intf, u8 data_ } } + if (ctx->func_desc) + ctx->filtering_supported = !!(ctx->func_desc->bmNetworkCapabilities + & USB_CDC_NCM_NCAP_ETH_FILTER); + iface_no = ctx->data->cur_altsetting->desc.bInterfaceNumber; /* Device-specific flags */ @@ -1898,6 +1902,14 @@ static void cdc_ncm_status(struct usbnet *dev, struct urb *urb) } } +static void cdc_ncm_update_filter(struct usbnet *dev) +{ + struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0]; + + if (ctx->filtering_supported) + usbnet_cdc_update_filter(dev); +} + static const struct driver_info cdc_ncm_info = { .description = "CDC NCM (NO ZLP)", .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET @@ -1908,7 +1920,7 @@ static const struct driver_info cdc_ncm_info = { .status = cdc_ncm_status, .rx_fixup = cdc_ncm_rx_fixup, .tx_fixup = cdc_ncm_tx_fixup, - .set_rx_mode = usbnet_cdc_update_filter, + .set_rx_mode = cdc_ncm_update_filter, }; /* Same as cdc_ncm_info, but with FLAG_SEND_ZLP */ @@ -1922,7 +1934,7 @@ static const struct driver_info cdc_ncm_zlp_info = { .status = cdc_ncm_status, .rx_fixup = cdc_ncm_rx_fixup, .tx_fixup = cdc_ncm_tx_fixup, - .set_rx_mode = usbnet_cdc_update_filter, + .set_rx_mode = cdc_ncm_update_filter, }; /* Same as cdc_ncm_info, but with FLAG_WWAN */ @@ -1936,7 +1948,7 @@ static const struct driver_info wwan_info = { .status = cdc_ncm_status, .rx_fixup = cdc_ncm_rx_fixup, .tx_fixup = cdc_ncm_tx_fixup, - .set_rx_mode = usbnet_cdc_update_filter, + .set_rx_mode = cdc_ncm_update_filter, }; /* Same as wwan_info, but with FLAG_NOARP */ @@ -1950,7 +1962,7 @@ static const struct driver_info wwan_noarp_info = { .status = cdc_ncm_status, .rx_fixup = cdc_ncm_rx_fixup, .tx_fixup = cdc_ncm_tx_fixup, - .set_rx_mode = usbnet_cdc_update_filter, + .set_rx_mode = cdc_ncm_update_filter, }; static const struct usb_device_id cdc_devs[] = { @@ -2031,6 +2043,13 @@ static const struct usb_device_id cdc_devs[] = { .driver_info = (unsigned long)&wwan_info, }, + /* Intel modem (label from OEM reads Fibocom L850-GL) */ + { USB_DEVICE_AND_INTERFACE_INFO(0x8087, 0x095a, + USB_CLASS_COMM, + USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE), + .driver_info = (unsigned long)&wwan_info, + }, + /* DisplayLink docking stations */ { .match_flags = USB_DEVICE_ID_MATCH_INT_INFO | USB_DEVICE_ID_MATCH_VENDOR, diff --git a/drivers/net/usb/ch9200.c b/drivers/net/usb/ch9200.c index f69d9b902da04..a206ffa76f1b9 100644 --- a/drivers/net/usb/ch9200.c +++ b/drivers/net/usb/ch9200.c @@ -178,6 +178,7 @@ static int ch9200_mdio_read(struct net_device *netdev, int phy_id, int loc) { struct usbnet *dev = netdev_priv(netdev); unsigned char buff[2]; + int ret; netdev_dbg(netdev, "%s phy_id:%02x loc:%02x\n", __func__, phy_id, loc); @@ -185,8 +186,10 @@ static int ch9200_mdio_read(struct net_device *netdev, int phy_id, int loc) if (phy_id != 0) return -ENODEV; - control_read(dev, REQUEST_READ, 0, loc * 2, buff, 0x02, - CONTROL_TIMEOUT_MS); + ret = control_read(dev, REQUEST_READ, 0, loc * 2, buff, 0x02, + CONTROL_TIMEOUT_MS); + if (ret < 0) + return ret; return (buff[0] | buff[1] << 8); } diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index 09173d7b87ed5..121f1c15c6793 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -1940,13 +1940,19 @@ static const struct ethtool_ops lan78xx_ethtool_ops = { .get_regs = lan78xx_get_regs, }; -static void lan78xx_init_mac_address(struct lan78xx_net *dev) +static int lan78xx_init_mac_address(struct lan78xx_net *dev) { u32 addr_lo, addr_hi; u8 addr[6]; + int ret; - lan78xx_read_reg(dev, RX_ADDRL, &addr_lo); - lan78xx_read_reg(dev, RX_ADDRH, &addr_hi); + ret = lan78xx_read_reg(dev, RX_ADDRL, &addr_lo); + if (ret < 0) + return ret; + + ret = lan78xx_read_reg(dev, RX_ADDRH, &addr_hi); + if (ret < 0) + return ret; addr[0] = addr_lo & 0xFF; addr[1] = (addr_lo >> 8) & 0xFF; @@ -1979,14 +1985,26 @@ static void lan78xx_init_mac_address(struct lan78xx_net *dev) (addr[2] << 16) | (addr[3] << 24); addr_hi = addr[4] | (addr[5] << 8); - lan78xx_write_reg(dev, RX_ADDRL, addr_lo); - lan78xx_write_reg(dev, RX_ADDRH, addr_hi); + ret = lan78xx_write_reg(dev, RX_ADDRL, addr_lo); + if (ret < 0) + return ret; + + ret = lan78xx_write_reg(dev, RX_ADDRH, addr_hi); + if (ret < 0) + return ret; } - lan78xx_write_reg(dev, MAF_LO(0), addr_lo); - lan78xx_write_reg(dev, MAF_HI(0), addr_hi | MAF_HI_VALID_); + ret = lan78xx_write_reg(dev, MAF_LO(0), addr_lo); + if (ret < 0) + return ret; + + ret = lan78xx_write_reg(dev, MAF_HI(0), addr_hi | MAF_HI_VALID_); + if (ret < 0) + return ret; eth_hw_addr_set(dev->net, addr); + + return 0; } /* MDIO read and write wrappers for phylib */ @@ -2910,8 +2928,6 @@ static int lan78xx_reset(struct lan78xx_net *dev) } } while (buf & HW_CFG_LRST_); - lan78xx_init_mac_address(dev); - /* save DEVID for later usage */ ret = lan78xx_read_reg(dev, ID_REV, &buf); if (ret < 0) @@ -2920,6 +2936,10 @@ static int lan78xx_reset(struct lan78xx_net *dev) dev->chipid = (buf & ID_REV_CHIP_ID_MASK_) >> 16; dev->chiprev = buf & ID_REV_CHIP_REV_MASK_; + ret = lan78xx_init_mac_address(dev); + if (ret < 0) + return ret; + /* Respond to the IN token with a NAK */ ret = lan78xx_read_reg(dev, USB_CFG0, &buf); if (ret < 0) @@ -4229,8 +4249,6 @@ static void lan78xx_disconnect(struct usb_interface *intf) if (!dev) return; - netif_napi_del(&dev->napi); - udev = interface_to_usbdev(intf); net = dev->net; diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index dc84d9029c2c7..73df808978b20 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -192,6 +192,12 @@ static int qmimux_rx_fixup(struct usbnet *dev, struct sk_buff *skb) if (!skbn) return 0; + /* Raw IP packets don't have a MAC header, but other subsystems + * (like xfrm) may still access MAC header offsets, so they must + * be initialized. + */ + skb_reset_mac_header(skbn); + switch (skb->data[offset + qmimux_hdr_sz] & 0xf0) { case 0x40: skbn->protocol = htons(ETH_P_IP); @@ -1363,6 +1369,9 @@ static const struct usb_device_id products[] = { {QMI_FIXED_INTF(0x2357, 0x0201, 4)}, /* TP-LINK HSUPA Modem MA180 */ {QMI_FIXED_INTF(0x2357, 0x9000, 4)}, /* TP-LINK MA260 */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1031, 3)}, /* Telit LE910C1-EUX */ + {QMI_QUIRK_SET_DTR(0x1bc7, 0x1034, 2)}, /* Telit LE910C4-WWX */ + {QMI_QUIRK_SET_DTR(0x1bc7, 0x1037, 4)}, /* Telit LE910C4-WWX */ + {QMI_QUIRK_SET_DTR(0x1bc7, 0x1038, 3)}, /* Telit LE910C4-WWX */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x103a, 0)}, /* Telit LE910C4-WWX */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1040, 2)}, /* Telit LE922A */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1050, 2)}, /* Telit FN980 */ @@ -1432,6 +1441,7 @@ static const struct usb_device_id products[] = { {QMI_QUIRK_SET_DTR(0x22de, 0x9051, 2)}, /* Hucom Wireless HM-211S/K */ {QMI_FIXED_INTF(0x22de, 0x9061, 3)}, /* WeTelecom WPD-600N */ {QMI_QUIRK_SET_DTR(0x1e0e, 0x9001, 5)}, /* SIMCom 7100E, 7230E, 7600E ++ */ + {QMI_QUIRK_SET_DTR(0x1e0e, 0x9071, 3)}, /* SIMCom 8230C ++ */ {QMI_QUIRK_SET_DTR(0x2c7c, 0x0121, 4)}, /* Quectel EC21 Mini PCIe */ {QMI_QUIRK_SET_DTR(0x2c7c, 0x0191, 4)}, /* Quectel EG91 */ {QMI_QUIRK_SET_DTR(0x2c7c, 0x0195, 4)}, /* Quectel EG95 */ diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index bbcefcc7ef8f0..386376ceeda25 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -10032,6 +10032,7 @@ static const struct usb_device_id rtl8152_table[] = { { USB_DEVICE(VENDOR_ID_NVIDIA, 0x09ff) }, { USB_DEVICE(VENDOR_ID_TPLINK, 0x0601) }, { USB_DEVICE(VENDOR_ID_DLINK, 0xb301) }, + { USB_DEVICE(VENDOR_ID_DELL, 0xb097) }, { USB_DEVICE(VENDOR_ID_ASUS, 0x1976) }, {} }; @@ -10103,7 +10104,12 @@ static int __init rtl8152_driver_init(void) ret = usb_register_device_driver(&rtl8152_cfgselector_driver, THIS_MODULE); if (ret) return ret; - return usb_register(&rtl8152_driver); + + ret = usb_register(&rtl8152_driver); + if (ret) + usb_deregister_device_driver(&rtl8152_cfgselector_driver); + + return ret; } static void __exit rtl8152_driver_exit(void) diff --git a/drivers/net/usb/rndis_host.c b/drivers/net/usb/rndis_host.c index bb0bf14158727..7b3739b29c8f7 100644 --- a/drivers/net/usb/rndis_host.c +++ b/drivers/net/usb/rndis_host.c @@ -630,16 +630,6 @@ static const struct driver_info zte_rndis_info = { .tx_fixup = rndis_tx_fixup, }; -static const struct driver_info wwan_rndis_info = { - .description = "Mobile Broadband RNDIS device", - .flags = FLAG_WWAN | FLAG_POINTTOPOINT | FLAG_FRAMING_RN | FLAG_NO_SETINT, - .bind = rndis_bind, - .unbind = rndis_unbind, - .status = rndis_status, - .rx_fixup = rndis_rx_fixup, - .tx_fixup = rndis_tx_fixup, -}; - /*-------------------------------------------------------------------------*/ static const struct usb_device_id products [] = { @@ -676,11 +666,9 @@ static const struct usb_device_id products [] = { USB_INTERFACE_INFO(USB_CLASS_WIRELESS_CONTROLLER, 1, 3), .driver_info = (unsigned long) &rndis_info, }, { - /* Mobile Broadband Modem, seen in Novatel Verizon USB730L and - * Telit FN990A (RNDIS) - */ + /* Novatel Verizon USB730L */ USB_INTERFACE_INFO(USB_CLASS_MISC, 4, 1), - .driver_info = (unsigned long)&wwan_rndis_info, + .driver_info = (unsigned long) &rndis_info, }, { }, // END }; diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c index ddff6f19ff98e..278e6cb6f4d99 100644 --- a/drivers/net/usb/rtl8150.c +++ b/drivers/net/usb/rtl8150.c @@ -664,7 +664,6 @@ static void rtl8150_set_multicast(struct net_device *netdev) rtl8150_t *dev = netdev_priv(netdev); u16 rx_creg = 0x9e; - netif_stop_queue(netdev); if (netdev->flags & IFF_PROMISC) { rx_creg |= 0x0001; dev_info(&netdev->dev, "%s: promiscuous mode\n", netdev->name); @@ -678,7 +677,6 @@ static void rtl8150_set_multicast(struct net_device *netdev) rx_creg &= 0x00fc; } async_set_registers(dev, RCR, sizeof(rx_creg), rx_creg); - netif_wake_queue(netdev); } static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb, @@ -687,9 +685,16 @@ static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb, rtl8150_t *dev = netdev_priv(netdev); int count, res; + /* pad the frame and ensure terminating USB packet, datasheet 9.2.3 */ + count = max(skb->len, ETH_ZLEN); + if (count % 64 == 0) + count++; + if (skb_padto(skb, count)) { + netdev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + netif_stop_queue(netdev); - count = (skb->len < 60) ? 60 : skb->len; - count = (count & 0x3f) ? count : count + 1; dev->tx_skb = skb; usb_fill_bulk_urb(dev->tx_urb, dev->udev, usb_sndbulkpipe(dev->udev, 2), skb->data, count, write_bulk_callback, dev); diff --git a/drivers/net/usb/sierra_net.c b/drivers/net/usb/sierra_net.c index 673d3aa837926..42b66adb35f1b 100644 --- a/drivers/net/usb/sierra_net.c +++ b/drivers/net/usb/sierra_net.c @@ -689,6 +689,10 @@ static int sierra_net_bind(struct usbnet *dev, struct usb_interface *intf) status); return -ENODEV; } + if (!dev->status) { + dev_err(&dev->udev->dev, "No status endpoint found"); + return -ENODEV; + } /* Initialize sierra private data */ priv = kzalloc(sizeof *priv, GFP_KERNEL); if (!priv) diff --git a/drivers/net/usb/smsc75xx.c b/drivers/net/usb/smsc75xx.c old mode 100644 new mode 100755 index 78ad2da3ee29b..e8be3d8d46f9a --- a/drivers/net/usb/smsc75xx.c +++ b/drivers/net/usb/smsc75xx.c @@ -2221,7 +2221,8 @@ static int smsc75xx_rx_fixup(struct usbnet *dev, struct sk_buff *skb) dev->net->stats.rx_frame_errors++; } else { /* MAX_SINGLE_PACKET_SIZE + 4(CRC) + 2(COE) + 4(Vlan) */ - if (unlikely(size > (MAX_SINGLE_PACKET_SIZE + ETH_HLEN + 12))) { + if (unlikely(size > (MAX_SINGLE_PACKET_SIZE + ETH_HLEN + 12) || + size > skb->len)) { netif_dbg(dev, rx_err, dev->net, "size err rx_cmd_a=0x%08x\n", rx_cmd_a); diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index ac0458b96738c..e6a1864f03f94 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -1113,6 +1113,9 @@ static void __handle_link_change(struct usbnet *dev) if (!test_bit(EVENT_DEV_OPEN, &dev->flags)) return; + if (test_and_clear_bit(EVENT_LINK_CARRIER_ON, &dev->flags)) + netif_carrier_on(dev->net); + if (!netif_carrier_ok(dev->net)) { /* kill URBs for reading packets to save bus bandwidth */ unlink_urbs(dev, &dev->rxq); @@ -1647,6 +1650,8 @@ void usbnet_disconnect (struct usb_interface *intf) net = dev->net; unregister_netdev (net); + cancel_work_sync(&dev->kevent); + while ((urb = usb_get_from_anchor(&dev->deferred))) { dev_kfree_skb(urb->context); kfree(urb->sg); @@ -2015,10 +2020,12 @@ EXPORT_SYMBOL(usbnet_manage_power); void usbnet_link_change(struct usbnet *dev, bool link, bool need_reset) { /* update link after link is reseted */ - if (link && !need_reset) - netif_carrier_on(dev->net); - else + if (link && !need_reset) { + set_bit(EVENT_LINK_CARRIER_ON, &dev->flags); + } else { + clear_bit(EVENT_LINK_CARRIER_ON, &dev->flags); netif_carrier_off(dev->net); + } if (need_reset && link) usbnet_defer_kevent(dev, EVENT_LINK_RESET); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 3bf394b24d971..caae11ba2f8be 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -487,6 +487,26 @@ static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); } +static int check_mergeable_len(struct net_device *dev, void *mrg_ctx, + unsigned int len) +{ + unsigned int headroom, tailroom, room, truesize; + + truesize = mergeable_ctx_to_truesize(mrg_ctx); + headroom = mergeable_ctx_to_headroom(mrg_ctx); + tailroom = headroom ? sizeof(struct skb_shared_info) : 0; + room = SKB_DATA_ALIGN(headroom + tailroom); + + if (len > truesize - room) { + pr_debug("%s: rx error: len %u exceeds truesize %lu\n", + dev->name, len, (unsigned long)(truesize - room)); + DEV_STATS_INC(dev, rx_length_errors); + return -1; + } + + return 0; +} + static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen, unsigned int headroom, unsigned int len) @@ -571,17 +591,6 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, goto ok; } - /* - * Verify that we can indeed put this data into a skb. - * This is here to handle cases when the device erroneously - * tries to receive more than is possible. This is usually - * the case of a broken device. - */ - if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) { - net_dbg_ratelimited("%s: too much data\n", skb->dev->name); - dev_kfree_skb(skb); - return NULL; - } BUG_ON(offset >= PAGE_SIZE); while (len) { unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); @@ -1084,7 +1093,8 @@ static unsigned int virtnet_get_headroom(struct virtnet_info *vi) * across multiple buffers (num_buf > 1), and we make sure buffers * have enough headroom. */ -static struct page *xdp_linearize_page(struct receive_queue *rq, +static struct page *xdp_linearize_page(struct net_device *dev, + struct receive_queue *rq, int *num_buf, struct page *p, int offset, @@ -1104,18 +1114,27 @@ static struct page *xdp_linearize_page(struct receive_queue *rq, memcpy(page_address(page) + page_off, page_address(p) + offset, *len); page_off += *len; + /* Only mergeable mode can go inside this while loop. In small mode, + * *num_buf == 1, so it cannot go inside. + */ while (--*num_buf) { unsigned int buflen; void *buf; + void *ctx; int off; - buf = virtnet_rq_get_buf(rq, &buflen, NULL); + buf = virtnet_rq_get_buf(rq, &buflen, &ctx); if (unlikely(!buf)) goto err_buf; p = virt_to_head_page(buf); off = buf - page_address(p); + if (check_mergeable_len(dev, ctx, buflen)) { + put_page(p); + goto err_buf; + } + /* guard against a misconfigured or uncooperative backend that * is sending packet larger than the MTU. */ @@ -1204,7 +1223,7 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev, headroom = vi->hdr_len + header_offset; buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - xdp_page = xdp_linearize_page(rq, &num_buf, page, + xdp_page = xdp_linearize_page(dev, rq, &num_buf, page, offset, header_offset, &tlen); if (!xdp_page) @@ -1314,9 +1333,19 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) { struct page *page = buf; - struct sk_buff *skb = - page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0); + struct sk_buff *skb; + + /* Make sure that len does not exceed the size allocated in + * add_recvbuf_big. + */ + if (unlikely(len > (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE)) { + pr_debug("%s: rx error: len %u exceeds allocated size %lu\n", + dev->name, len, + (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE); + goto err; + } + skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0); u64_stats_add(&stats->bytes, len - vi->hdr_len); if (unlikely(!skb)) goto err; @@ -1539,7 +1568,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info *vi, */ if (!xdp_prog->aux->xdp_has_frags) { /* linearize data for XDP */ - xdp_page = xdp_linearize_page(rq, num_buf, + xdp_page = xdp_linearize_page(vi->dev, rq, num_buf, *page, offset, VIRTIO_XDP_HEADROOM, len); @@ -1792,22 +1821,28 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, return; } - /* 1. Save the flags early, as the XDP program might overwrite them. + /* About the flags below: + * 1. Save the flags early, as the XDP program might overwrite them. * These flags ensure packets marked as VIRTIO_NET_HDR_F_DATA_VALID * stay valid after XDP processing. * 2. XDP doesn't work with partially checksummed packets (refer to * virtnet_xdp_set()), so packets marked as * VIRTIO_NET_HDR_F_NEEDS_CSUM get dropped during XDP processing. */ - flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags; - if (vi->mergeable_rx_bufs) + if (vi->mergeable_rx_bufs) { + flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags; skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); - else if (vi->big_packets) + } else if (vi->big_packets) { + void *p = page_address((struct page *)buf); + + flags = ((struct virtio_net_common_hdr *)p)->hdr.flags; skb = receive_big(dev, vi, rq, buf, len, stats); - else + } else { + flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags; skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); + } if (unlikely(!skb)) return; diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index beebe09eb88ff..68b8e458a88f6 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -1499,6 +1499,30 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, return (hlen + (hdr.tcp->doff << 2)); } +static void +vmxnet3_lro_tunnel(struct sk_buff *skb, __be16 ip_proto) +{ + struct udphdr *uh = NULL; + + if (ip_proto == htons(ETH_P_IP)) { + struct iphdr *iph = (struct iphdr *)skb->data; + + if (iph->protocol == IPPROTO_UDP) + uh = (struct udphdr *)(iph + 1); + } else { + struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; + + if (iph->nexthdr == IPPROTO_UDP) + uh = (struct udphdr *)(iph + 1); + } + if (uh) { + if (uh->check) + skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; + else + skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; + } +} + static int vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter, int quota) @@ -1803,6 +1827,8 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, if (segCnt != 0 && mss != 0) { skb_shinfo(skb)->gso_type = rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; + if (encap_lro) + vmxnet3_lro_tunnel(skb, skb->protocol); skb_shinfo(skb)->gso_size = mss; skb_shinfo(skb)->gso_segs = segCnt; } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { @@ -1955,6 +1981,11 @@ vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, rq->comp_ring.gen = VMXNET3_INIT_GEN; rq->comp_ring.next2proc = 0; + + if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) + xdp_rxq_info_unreg(&rq->xdp_rxq); + page_pool_destroy(rq->page_pool); + rq->page_pool = NULL; } @@ -1995,11 +2026,6 @@ static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, } } - if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) - xdp_rxq_info_unreg(&rq->xdp_rxq); - page_pool_destroy(rq->page_pool); - rq->page_pool = NULL; - if (rq->data_ring.base) { dma_free_coherent(&adapter->pdev->dev, rq->rx_ring[0].size * rq->data_ring.desc_size, @@ -3457,8 +3483,6 @@ vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) struct vmxnet3_adapter *adapter = netdev_priv(netdev); int err = 0; - netdev->mtu = new_mtu; - /* * Reset_work may be in the middle of resetting the device, wait for its * completion. @@ -3472,6 +3496,7 @@ vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) /* we need to re-create the rx queue based on the new mtu */ vmxnet3_rq_destroy_all(adapter); + netdev->mtu = new_mtu; vmxnet3_adjust_rx_ring_size(adapter); err = vmxnet3_rq_create_all(adapter); if (err) { @@ -3488,6 +3513,8 @@ vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) "Closing it\n", err); goto out; } + } else { + netdev->mtu = new_mtu; } out: diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 5968a3ab81770..64114e98d75d0 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1345,6 +1345,8 @@ static void vrf_ip6_input_dst(struct sk_buff *skb, struct net_device *vrf_dev, struct net *net = dev_net(vrf_dev); struct rt6_info *rt6; + skb_dst_drop(skb); + rt6 = vrf_ip6_route_lookup(net, vrf_dev, &fl6, ifindex, skb, RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_IFACE); if (unlikely(!rt6)) diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index 64db3e98a1b66..1b6b6acd34894 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -227,9 +227,9 @@ static int vxlan_fdb_info(struct sk_buff *skb, struct vxlan_dev *vxlan, be32_to_cpu(fdb->vni))) goto nla_put_failure; - ci.ndm_used = jiffies_to_clock_t(now - fdb->used); + ci.ndm_used = jiffies_to_clock_t(now - READ_ONCE(fdb->used)); ci.ndm_confirmed = 0; - ci.ndm_updated = jiffies_to_clock_t(now - fdb->updated); + ci.ndm_updated = jiffies_to_clock_t(now - READ_ONCE(fdb->updated)); ci.ndm_refcnt = 0; if (nla_put(skb, NDA_CACHEINFO, sizeof(ci), &ci)) @@ -435,8 +435,8 @@ static struct vxlan_fdb *vxlan_find_mac(struct vxlan_dev *vxlan, struct vxlan_fdb *f; f = __vxlan_find_mac(vxlan, mac, vni); - if (f && f->used != jiffies) - f->used = jiffies; + if (f && READ_ONCE(f->used) != jiffies) + WRITE_ONCE(f->used, jiffies); return f; } @@ -606,10 +606,10 @@ static int vxlan_fdb_append(struct vxlan_fdb *f, if (rd == NULL) return -ENOMEM; - if (dst_cache_init(&rd->dst_cache, GFP_ATOMIC)) { - kfree(rd); - return -ENOMEM; - } + /* The driver can work correctly without a dst cache, so do not treat + * dst cache initialization errors as fatal. + */ + dst_cache_init(&rd->dst_cache, GFP_ATOMIC | __GFP_NOWARN); rd->remote_ip = *ip; rd->remote_port = port; @@ -1010,12 +1010,12 @@ static int vxlan_fdb_update_existing(struct vxlan_dev *vxlan, !(f->flags & NTF_VXLAN_ADDED_BY_USER)) { if (f->state != state) { f->state = state; - f->updated = jiffies; + WRITE_ONCE(f->updated, jiffies); notify = 1; } if (f->flags != fdb_flags) { f->flags = fdb_flags; - f->updated = jiffies; + WRITE_ONCE(f->updated, jiffies); notify = 1; } } @@ -1049,7 +1049,7 @@ static int vxlan_fdb_update_existing(struct vxlan_dev *vxlan, } if (ndm_flags & NTF_USE) - f->used = jiffies; + WRITE_ONCE(f->used, jiffies); if (notify) { if (rd == NULL) @@ -1478,7 +1478,7 @@ static bool vxlan_snoop(struct net_device *dev, src_mac, &rdst->remote_ip.sa, &src_ip->sa); rdst->remote_ip = *src_ip; - f->updated = jiffies; + WRITE_ONCE(f->updated, jiffies); vxlan_fdb_notify(vxlan, f, rdst, RTM_NEWNEIGH, true, NULL); } else { u32 hash_index = fdb_head_index(vxlan, src_mac, vni); @@ -2920,7 +2920,7 @@ static void vxlan_cleanup(struct timer_list *t) if (f->flags & NTF_EXT_LEARNED) continue; - timeout = f->used + vxlan->cfg.age_interval * HZ; + timeout = READ_ONCE(f->used) + vxlan->cfg.age_interval * HZ; if (time_before_eq(timeout, jiffies)) { netdev_dbg(vxlan->dev, "garbage collect %pM\n", @@ -4240,6 +4240,7 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], struct netlink_ext_ack *extack) { struct vxlan_dev *vxlan = netdev_priv(dev); + bool rem_ip_changed, change_igmp; struct net_device *lowerdev; struct vxlan_config conf; struct vxlan_rdst *dst; @@ -4263,8 +4264,13 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], if (err) return err; + rem_ip_changed = !vxlan_addr_equal(&conf.remote_ip, &dst->remote_ip); + change_igmp = vxlan->dev->flags & IFF_UP && + (rem_ip_changed || + dst->remote_ifindex != conf.remote_ifindex); + /* handle default dst entry */ - if (!vxlan_addr_equal(&conf.remote_ip, &dst->remote_ip)) { + if (rem_ip_changed) { u32 hash_index = fdb_head_index(vxlan, all_zeros_mac, conf.vni); spin_lock_bh(&vxlan->hash_lock[hash_index]); @@ -4308,6 +4314,9 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], } } + if (change_igmp && vxlan_addr_multicast(&dst->remote_ip)) + err = vxlan_multicast_leave(vxlan); + if (conf.age_interval != vxlan->cfg.age_interval) mod_timer(&vxlan->age_timer, jiffies); @@ -4315,7 +4324,12 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], if (lowerdev && lowerdev != dst->remote_dev) dst->remote_dev = lowerdev; vxlan_config_apply(dev, &conf, lowerdev, vxlan->net, true); - return 0; + + if (!err && change_igmp && + vxlan_addr_multicast(&dst->remote_ip)) + err = vxlan_multicast_join(vxlan); + + return err; } static void vxlan_dellink(struct net_device *dev, struct list_head *head) diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c index 6e6e9f05509ab..06d19e90eadb5 100644 --- a/drivers/net/vxlan/vxlan_vnifilter.c +++ b/drivers/net/vxlan/vxlan_vnifilter.c @@ -627,7 +627,11 @@ static void vxlan_vni_delete_group(struct vxlan_dev *vxlan, * default dst remote_ip previously added for this vni */ if (!vxlan_addr_any(&vninode->remote_ip) || - !vxlan_addr_any(&dst->remote_ip)) + !vxlan_addr_any(&dst->remote_ip)) { + u32 hash_index = fdb_head_index(vxlan, all_zeros_mac, + vninode->vni); + + spin_lock_bh(&vxlan->hash_lock[hash_index]); __vxlan_fdb_delete(vxlan, all_zeros_mac, (vxlan_addr_any(&vninode->remote_ip) ? dst->remote_ip : vninode->remote_ip), @@ -635,6 +639,8 @@ static void vxlan_vni_delete_group(struct vxlan_dev *vxlan, vninode->vni, vninode->vni, dst->remote_ifindex, true); + spin_unlock_bh(&vxlan->hash_lock[hash_index]); + } if (vxlan->dev->flags & IFF_UP) { if (vxlan_addr_multicast(&vninode->remote_ip) && diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c index deb9636b0ecf8..f98e0f027a054 100644 --- a/drivers/net/wireguard/device.c +++ b/drivers/net/wireguard/device.c @@ -369,6 +369,7 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, if (ret < 0) goto err_free_handshake_queue; + dev_set_threaded(dev, true); ret = register_netdevice(dev); if (ret < 0) goto err_uninit_ratelimiter; diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c index 655fb5cdf01f8..233e5c619fa4e 100644 --- a/drivers/net/wireless/ath/ath10k/mac.c +++ b/drivers/net/wireless/ath/ath10k/mac.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "hif.h" #include "core.h" @@ -287,8 +288,15 @@ static int ath10k_send_key(struct ath10k_vif *arvif, key->flags |= IEEE80211_KEY_FLAG_GENERATE_IV; if (cmd == DISABLE_KEY) { - arg.key_cipher = ar->wmi_key_cipher[WMI_CIPHER_NONE]; - arg.key_data = NULL; + if (flags & WMI_KEY_GROUP) { + /* Not all hardware handles group-key deletion operation + * correctly. Replace the key with a junk value to invalidate it. + */ + get_random_bytes(key->key, key->keylen); + } else { + arg.key_cipher = ar->wmi_key_cipher[WMI_CIPHER_NONE]; + arg.key_data = NULL; + } } return ath10k_wmi_vdev_install_key(arvif->ar, &arg); diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c index 2c39bad7ebfb9..1d06d4125992d 100644 --- a/drivers/net/wireless/ath/ath10k/snoc.c +++ b/drivers/net/wireless/ath/ath10k/snoc.c @@ -937,7 +937,9 @@ static int ath10k_snoc_hif_start(struct ath10k *ar) dev_set_threaded(&ar->napi_dev, true); ath10k_core_napi_enable(ar); - ath10k_snoc_irq_enable(ar); + /* IRQs are left enabled when we restart due to a firmware crash */ + if (!test_bit(ATH10K_SNOC_FLAG_RECOVERY, &ar_snoc->flags)) + ath10k_snoc_irq_enable(ar); ath10k_snoc_rx_post(ar); clear_bit(ATH10K_SNOC_FLAG_RECOVERY, &ar_snoc->flags); diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c index 818aea99f85eb..c7c96d210061d 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.c +++ b/drivers/net/wireless/ath/ath10k/wmi.c @@ -1937,6 +1937,7 @@ int ath10k_wmi_cmd_send(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id) if (cmd_id == WMI_CMD_UNSUPPORTED) { ath10k_warn(ar, "wmi command %d is not supported by firmware\n", cmd_id); + dev_kfree_skb_any(skb); return ret; } diff --git a/drivers/net/wireless/ath/ath11k/ce.c b/drivers/net/wireless/ath/ath11k/ce.c index e66e86bdec20f..39d9aad33bc69 100644 --- a/drivers/net/wireless/ath/ath11k/ce.c +++ b/drivers/net/wireless/ath/ath11k/ce.c @@ -394,10 +394,6 @@ static int ath11k_ce_completed_recv_next(struct ath11k_ce_pipe *pipe, } *nbytes = ath11k_hal_ce_dst_status_get_length(desc); - if (*nbytes == 0) { - ret = -EIO; - goto err; - } *skb = pipe->dest_ring->skb[sw_index]; pipe->dest_ring->skb[sw_index] = NULL; @@ -430,8 +426,8 @@ static void ath11k_ce_recv_process_cb(struct ath11k_ce_pipe *pipe) dma_unmap_single(ab->dev, ATH11K_SKB_RXCB(skb)->paddr, max_nbytes, DMA_FROM_DEVICE); - if (unlikely(max_nbytes < nbytes)) { - ath11k_warn(ab, "rxed more than expected (nbytes %d, max %d)", + if (unlikely(max_nbytes < nbytes || nbytes == 0)) { + ath11k_warn(ab, "unexpected rx length (nbytes %d, max %d)", nbytes, max_nbytes); dev_kfree_skb_any(skb); continue; diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index fc7c4564a715c..355424baeedde 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -704,6 +704,94 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { }, }; +static const struct dmi_system_id ath11k_pm_quirk_table[] = { + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* X13 G4 AMD #1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21J3"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* X13 G4 AMD #2 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21J4"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* T14 G4 AMD #1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21K3"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* T14 G4 AMD #2 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21K4"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* P14s G4 AMD #1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21K5"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* P14s G4 AMD #2 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21K6"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* T16 G2 AMD #1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21K7"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* T16 G2 AMD #2 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21K8"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* P16s G2 AMD #1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21K9"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* P16s G2 AMD #2 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21KA"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* T14s G4 AMD #1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21F8"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* T14s G4 AMD #2 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21F9"), + }, + }, + {} +}; + static inline struct ath11k_pdev *ath11k_core_get_single_pdev(struct ath11k_base *ab) { WARN_ON(!ab->hw_params.single_pdev_only); @@ -748,6 +836,7 @@ void ath11k_fw_stats_init(struct ath11k *ar) INIT_LIST_HEAD(&ar->fw_stats.bcn); init_completion(&ar->fw_stats_complete); + init_completion(&ar->fw_stats_done); } void ath11k_fw_stats_free(struct ath11k_fw_stats *stats) @@ -1637,6 +1726,20 @@ int ath11k_core_qmi_firmware_ready(struct ath11k_base *ab) { int ret; + switch (ath11k_crypto_mode) { + case ATH11K_CRYPT_MODE_SW: + set_bit(ATH11K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags); + set_bit(ATH11K_FLAG_RAW_MODE, &ab->dev_flags); + break; + case ATH11K_CRYPT_MODE_HW: + clear_bit(ATH11K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags); + clear_bit(ATH11K_FLAG_RAW_MODE, &ab->dev_flags); + break; + default: + ath11k_info(ab, "invalid crypto_mode: %d\n", ath11k_crypto_mode); + return -EINVAL; + } + ret = ath11k_core_start_firmware(ab, ab->fw_mode); if (ret) { ath11k_err(ab, "failed to start firmware: %d\n", ret); @@ -1655,20 +1758,6 @@ int ath11k_core_qmi_firmware_ready(struct ath11k_base *ab) goto err_firmware_stop; } - switch (ath11k_crypto_mode) { - case ATH11K_CRYPT_MODE_SW: - set_bit(ATH11K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags); - set_bit(ATH11K_FLAG_RAW_MODE, &ab->dev_flags); - break; - case ATH11K_CRYPT_MODE_HW: - clear_bit(ATH11K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags); - clear_bit(ATH11K_FLAG_RAW_MODE, &ab->dev_flags); - break; - default: - ath11k_info(ab, "invalid crypto_mode: %d\n", ath11k_crypto_mode); - return -EINVAL; - } - if (ath11k_frame_mode == ATH11K_HW_TXRX_RAW) set_bit(ATH11K_FLAG_RAW_MODE, &ab->dev_flags); @@ -1716,14 +1805,10 @@ static int ath11k_core_reconfigure_on_crash(struct ath11k_base *ab) mutex_unlock(&ab->core_lock); ath11k_dp_free(ab); - ath11k_hal_srng_deinit(ab); + ath11k_hal_srng_clear(ab); ab->free_vdev_map = (1LL << (ab->num_radios * TARGET_NUM_VDEVS(ab))) - 1; - ret = ath11k_hal_srng_init(ab); - if (ret) - return ret; - clear_bit(ATH11K_FLAG_CRASH_FLUSH, &ab->dev_flags); ret = ath11k_core_qmi_firmware_ready(ab); @@ -1742,6 +1827,7 @@ static int ath11k_core_reconfigure_on_crash(struct ath11k_base *ab) void ath11k_core_halt(struct ath11k *ar) { struct ath11k_base *ab = ar->ab; + struct list_head *pos, *n; lockdep_assert_held(&ar->conf_mutex); @@ -1756,7 +1842,12 @@ void ath11k_core_halt(struct ath11k *ar) rcu_assign_pointer(ab->pdevs_active[ar->pdev_idx], NULL); synchronize_rcu(); - INIT_LIST_HEAD(&ar->arvifs); + + spin_lock_bh(&ar->data_lock); + list_for_each_safe(pos, n, &ar->arvifs) + list_del_init(pos); + spin_unlock_bh(&ar->data_lock); + idr_init(&ar->txmgmt_idr); } @@ -2011,8 +2102,17 @@ EXPORT_SYMBOL(ath11k_core_pre_init); int ath11k_core_init(struct ath11k_base *ab) { + const struct dmi_system_id *dmi_id; int ret; + dmi_id = dmi_first_match(ath11k_pm_quirk_table); + if (dmi_id) + ab->pm_policy = (kernel_ulong_t)dmi_id->driver_data; + else + ab->pm_policy = ATH11K_PM_DEFAULT; + + ath11k_dbg(ab, ATH11K_DBG_BOOT, "pm policy %u\n", ab->pm_policy); + ret = ath11k_core_soc_create(ab); if (ret) { ath11k_err(ab, "failed to create soc core: %d\n", ret); diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index b044477624837..4bb36dc6ae08b 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -365,6 +365,8 @@ struct ath11k_vif { struct ieee80211_chanctx_conf chanctx; struct ath11k_arp_ns_offload arp_ns_offload; struct ath11k_rekey_data rekey_data; + u32 num_stations; + bool reinstall_group_keys; #ifdef CONFIG_ATH11K_DEBUGFS struct dentry *debugfs_twt; @@ -550,6 +552,8 @@ struct ath11k_fw_stats { struct list_head pdevs; struct list_head vdevs; struct list_head bcn; + u32 num_vdev_recvd; + u32 num_bcn_recvd; }; struct ath11k_dbg_htt_stats { @@ -732,7 +736,7 @@ struct ath11k { u8 alpha2[REG_ALPHA2_LEN + 1]; struct ath11k_fw_stats fw_stats; struct completion fw_stats_complete; - bool fw_stats_done; + struct completion fw_stats_done; /* protected by conf_mutex */ bool ps_state_enable; @@ -840,6 +844,11 @@ struct ath11k_msi_config { u16 hw_rev; }; +enum ath11k_pm_policy { + ATH11K_PM_DEFAULT, + ATH11K_PM_WOW, +}; + /* Master structure to hold the hw data which may be used in core module */ struct ath11k_base { enum ath11k_hw_rev hw_rev; @@ -992,6 +1001,8 @@ struct ath11k_base { } testmode; #endif + enum ath11k_pm_policy pm_policy; + /* must be last */ u8 drv_priv[] __aligned(sizeof(void *)); }; @@ -1225,6 +1236,11 @@ static inline struct ath11k_vif *ath11k_vif_to_arvif(struct ieee80211_vif *vif) return (struct ath11k_vif *)vif->drv_priv; } +static inline struct ath11k_sta *ath11k_sta_to_arsta(struct ieee80211_sta *sta) +{ + return (struct ath11k_sta *)sta->drv_priv; +} + static inline struct ath11k *ath11k_ab_to_ar(struct ath11k_base *ab, int mac_id) { diff --git a/drivers/net/wireless/ath/ath11k/debugfs.c b/drivers/net/wireless/ath/ath11k/debugfs.c index 8cda73b78ebf4..4304fed44d583 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs.c +++ b/drivers/net/wireless/ath/ath11k/debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -96,9 +96,10 @@ void ath11k_debugfs_add_dbring_entry(struct ath11k *ar, static void ath11k_debugfs_fw_stats_reset(struct ath11k *ar) { spin_lock_bh(&ar->data_lock); - ar->fw_stats_done = false; ath11k_fw_stats_pdevs_free(&ar->fw_stats.pdevs); ath11k_fw_stats_vdevs_free(&ar->fw_stats.vdevs); + ar->fw_stats.num_vdev_recvd = 0; + ar->fw_stats.num_bcn_recvd = 0; spin_unlock_bh(&ar->data_lock); } @@ -106,15 +107,14 @@ void ath11k_debugfs_fw_stats_process(struct ath11k *ar, struct ath11k_fw_stats * { struct ath11k_base *ab = ar->ab; struct ath11k_pdev *pdev; - bool is_end; - static unsigned int num_vdev, num_bcn; + bool is_end = true; size_t total_vdevs_started = 0; int i; /* WMI_REQUEST_PDEV_STAT request has been already processed */ if (stats->stats_id == WMI_REQUEST_RSSI_PER_CHAIN_STAT) { - ar->fw_stats_done = true; + complete(&ar->fw_stats_done); return; } @@ -132,15 +132,16 @@ void ath11k_debugfs_fw_stats_process(struct ath11k *ar, struct ath11k_fw_stats * total_vdevs_started += ar->num_started_vdevs; } - is_end = ((++num_vdev) == total_vdevs_started); + if (total_vdevs_started) + is_end = ((++ar->fw_stats.num_vdev_recvd) == + total_vdevs_started); list_splice_tail_init(&stats->vdevs, &ar->fw_stats.vdevs); - if (is_end) { - ar->fw_stats_done = true; - num_vdev = 0; - } + if (is_end) + complete(&ar->fw_stats_done); + return; } @@ -152,15 +153,15 @@ void ath11k_debugfs_fw_stats_process(struct ath11k *ar, struct ath11k_fw_stats * /* Mark end until we reached the count of all started VDEVs * within the PDEV */ - is_end = ((++num_bcn) == ar->num_started_vdevs); + if (ar->num_started_vdevs) + is_end = ((++ar->fw_stats.num_bcn_recvd) == + ar->num_started_vdevs); list_splice_tail_init(&stats->bcn, &ar->fw_stats.bcn); - if (is_end) { - ar->fw_stats_done = true; - num_bcn = 0; - } + if (is_end) + complete(&ar->fw_stats_done); } } @@ -168,21 +169,15 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar, struct stats_request_params *req_param) { struct ath11k_base *ab = ar->ab; - unsigned long timeout, time_left; + unsigned long time_left; int ret; lockdep_assert_held(&ar->conf_mutex); - /* FW stats can get split when exceeding the stats data buffer limit. - * In that case, since there is no end marking for the back-to-back - * received 'update stats' event, we keep a 3 seconds timeout in case, - * fw_stats_done is not marked yet - */ - timeout = jiffies + msecs_to_jiffies(3 * 1000); - ath11k_debugfs_fw_stats_reset(ar); reinit_completion(&ar->fw_stats_complete); + reinit_completion(&ar->fw_stats_done); ret = ath11k_wmi_send_stats_request_cmd(ar, req_param); @@ -193,21 +188,18 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar, } time_left = wait_for_completion_timeout(&ar->fw_stats_complete, 1 * HZ); - if (!time_left) return -ETIMEDOUT; - for (;;) { - if (time_after(jiffies, timeout)) - break; + /* FW stats can get split when exceeding the stats data buffer limit. + * In that case, since there is no end marking for the back-to-back + * received 'update stats' event, we keep a 3 seconds timeout in case, + * fw_stats_done is not marked yet + */ + time_left = wait_for_completion_timeout(&ar->fw_stats_done, 3 * HZ); + if (!time_left) + return -ETIMEDOUT; - spin_lock_bh(&ar->data_lock); - if (ar->fw_stats_done) { - spin_unlock_bh(&ar->data_lock); - break; - } - spin_unlock_bh(&ar->data_lock); - } return 0; } @@ -980,7 +972,7 @@ int ath11k_debugfs_pdev_create(struct ath11k_base *ab) debugfs_create_file("simulate_fw_crash", 0600, ab->debugfs_soc, ab, &fops_simulate_fw_crash); - debugfs_create_file("soc_dp_stats", 0600, ab->debugfs_soc, ab, + debugfs_create_file("soc_dp_stats", 0400, ab->debugfs_soc, ab, &fops_soc_dp_stats); if (ab->hw_params.sram_dump.start != 0) @@ -1460,7 +1452,7 @@ static void ath11k_reset_peer_ps_duration(void *data, struct ieee80211_sta *sta) { struct ath11k *ar = data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); spin_lock_bh(&ar->data_lock); arsta->ps_total_duration = 0; @@ -1511,7 +1503,7 @@ static void ath11k_peer_ps_state_disable(void *data, struct ieee80211_sta *sta) { struct ath11k *ar = data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); spin_lock_bh(&ar->data_lock); arsta->peer_ps_state = WMI_PEER_PS_STATE_DISABLED; diff --git a/drivers/net/wireless/ath/ath11k/debugfs_sta.c b/drivers/net/wireless/ath/ath11k/debugfs_sta.c index 168879a380cb2..f56a24b6c8da2 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_sta.c +++ b/drivers/net/wireless/ath/ath11k/debugfs_sta.c @@ -137,7 +137,7 @@ static ssize_t ath11k_dbg_sta_dump_tx_stats(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; struct ath11k_htt_data_stats *stats; static const char *str_name[ATH11K_STATS_TYPE_MAX] = {"succ", "fail", @@ -244,7 +244,7 @@ static ssize_t ath11k_dbg_sta_dump_rx_stats(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; struct ath11k_rx_peer_stats *rx_stats = arsta->rx_stats; int len = 0, i, retval = 0; @@ -341,7 +341,7 @@ static int ath11k_dbg_sta_open_htt_peer_stats(struct inode *inode, struct file *file) { struct ieee80211_sta *sta = inode->i_private; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; struct debug_htt_stats_req *stats_req; int type = ar->debug.htt_stats.type; @@ -377,7 +377,7 @@ static int ath11k_dbg_sta_release_htt_peer_stats(struct inode *inode, struct file *file) { struct ieee80211_sta *sta = inode->i_private; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; mutex_lock(&ar->conf_mutex); @@ -414,7 +414,7 @@ static ssize_t ath11k_dbg_sta_write_peer_pktlog(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; int ret, enable; @@ -454,7 +454,7 @@ static ssize_t ath11k_dbg_sta_read_peer_pktlog(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; char buf[32] = {0}; int len; @@ -481,7 +481,7 @@ static ssize_t ath11k_dbg_sta_write_delba(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; u32 tid, initiator, reason; int ret; @@ -532,7 +532,7 @@ static ssize_t ath11k_dbg_sta_write_addba_resp(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; u32 tid, status; int ret; @@ -582,7 +582,7 @@ static ssize_t ath11k_dbg_sta_write_addba(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; u32 tid, buf_size; int ret; @@ -633,7 +633,7 @@ static ssize_t ath11k_dbg_sta_read_aggr_mode(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; char buf[64]; int len = 0; @@ -653,7 +653,7 @@ static ssize_t ath11k_dbg_sta_write_aggr_mode(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; u32 aggr_mode; int ret; @@ -698,7 +698,7 @@ ath11k_write_htt_peer_stats_reset(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; struct htt_ext_stats_cfg_params cfg_params = { 0 }; int ret; @@ -757,7 +757,7 @@ static ssize_t ath11k_dbg_sta_read_peer_ps_state(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; char buf[20]; int len; @@ -784,7 +784,7 @@ static ssize_t ath11k_dbg_sta_read_current_ps_duration(struct file *file, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; u64 time_since_station_in_power_save; char buf[20]; @@ -818,7 +818,7 @@ static ssize_t ath11k_dbg_sta_read_total_ps_duration(struct file *file, size_t count, loff_t *ppos) { struct ieee80211_sta *sta = file->private_data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; char buf[20]; u64 power_save_duration; diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index 4c70366ac56eb..8cc51ab699de7 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -1100,7 +1100,7 @@ int ath11k_dp_rx_ampdu_start(struct ath11k *ar, struct ieee80211_ampdu_params *params) { struct ath11k_base *ab = ar->ab; - struct ath11k_sta *arsta = (void *)params->sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(params->sta); int vdev_id = arsta->arvif->vdev_id; int ret; @@ -1118,7 +1118,7 @@ int ath11k_dp_rx_ampdu_stop(struct ath11k *ar, { struct ath11k_base *ab = ar->ab; struct ath11k_peer *peer; - struct ath11k_sta *arsta = (void *)params->sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(params->sta); int vdev_id = arsta->arvif->vdev_id; dma_addr_t paddr; bool active; @@ -1460,7 +1460,7 @@ ath11k_update_per_peer_tx_stats(struct ath11k *ar, } sta = peer->sta; - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); memset(&arsta->txrate, 0, sizeof(arsta->txrate)); @@ -2649,7 +2649,7 @@ int ath11k_dp_process_rx(struct ath11k_base *ab, int ring_id, struct ath11k *ar; struct hal_reo_dest_ring *desc; enum hal_reo_dest_ring_push_reason push_reason; - u32 cookie; + u32 cookie, info0, rx_msdu_info0, rx_mpdu_info0; int i; for (i = 0; i < MAX_RADIOS; i++) @@ -2666,7 +2666,7 @@ int ath11k_dp_process_rx(struct ath11k_base *ab, int ring_id, (struct hal_reo_dest_ring *)ath11k_hal_srng_dst_get_next_entry(ab, srng))) { cookie = FIELD_GET(BUFFER_ADDR_INFO1_SW_COOKIE, - desc->buf_addr_info.info1); + READ_ONCE(desc->buf_addr_info.info1)); buf_id = FIELD_GET(DP_RXDMA_BUF_COOKIE_BUF_ID, cookie); mac_id = FIELD_GET(DP_RXDMA_BUF_COOKIE_PDEV_ID, cookie); @@ -2695,8 +2695,9 @@ int ath11k_dp_process_rx(struct ath11k_base *ab, int ring_id, num_buffs_reaped[mac_id]++; + info0 = READ_ONCE(desc->info0); push_reason = FIELD_GET(HAL_REO_DEST_RING_INFO0_PUSH_REASON, - desc->info0); + info0); if (unlikely(push_reason != HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION)) { dev_kfree_skb_any(msdu); @@ -2704,18 +2705,21 @@ int ath11k_dp_process_rx(struct ath11k_base *ab, int ring_id, continue; } - rxcb->is_first_msdu = !!(desc->rx_msdu_info.info0 & + rx_msdu_info0 = READ_ONCE(desc->rx_msdu_info.info0); + rx_mpdu_info0 = READ_ONCE(desc->rx_mpdu_info.info0); + + rxcb->is_first_msdu = !!(rx_msdu_info0 & RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU); - rxcb->is_last_msdu = !!(desc->rx_msdu_info.info0 & + rxcb->is_last_msdu = !!(rx_msdu_info0 & RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU); - rxcb->is_continuation = !!(desc->rx_msdu_info.info0 & + rxcb->is_continuation = !!(rx_msdu_info0 & RX_MSDU_DESC_INFO0_MSDU_CONTINUATION); rxcb->peer_id = FIELD_GET(RX_MPDU_DESC_META_DATA_PEER_ID, - desc->rx_mpdu_info.meta_data); + READ_ONCE(desc->rx_mpdu_info.meta_data)); rxcb->seq_no = FIELD_GET(RX_MPDU_DESC_INFO0_SEQ_NUM, - desc->rx_mpdu_info.info0); + rx_mpdu_info0); rxcb->tid = FIELD_GET(HAL_REO_DEST_RING_INFO0_RX_QUEUE_NUM, - desc->info0); + info0); rxcb->mac_id = mac_id; __skb_queue_tail(&msdu_list[mac_id], msdu); @@ -5265,7 +5269,7 @@ int ath11k_dp_rx_process_mon_status(struct ath11k_base *ab, int mac_id, goto next_skb; } - arsta = (struct ath11k_sta *)peer->sta->drv_priv; + arsta = ath11k_sta_to_arsta(peer->sta); ath11k_dp_rx_update_peer_stats(arsta, ppdu_info); if (ath11k_debugfs_is_pktlog_peer_valid(ar, peer->addr)) diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.c b/drivers/net/wireless/ath/ath11k/dp_tx.c index 7dd1ee5898017..c1072e66e3e8f 100644 --- a/drivers/net/wireless/ath/ath11k/dp_tx.c +++ b/drivers/net/wireless/ath/ath11k/dp_tx.c @@ -467,7 +467,7 @@ void ath11k_dp_tx_update_txcompl(struct ath11k *ar, struct hal_tx_status *ts) } sta = peer->sta; - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); memset(&arsta->txrate, 0, sizeof(arsta->txrate)); pkt_type = FIELD_GET(HAL_TX_RATE_STATS_INFO0_PKT_TYPE, @@ -627,7 +627,7 @@ static void ath11k_dp_tx_complete_msdu(struct ath11k *ar, ieee80211_free_txskb(ar->hw, msdu); return; } - arsta = (struct ath11k_sta *)peer->sta->drv_priv; + arsta = ath11k_sta_to_arsta(peer->sta); status.sta = peer->sta; status.skb = msdu; status.info = info; diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c index ae5f7e401e21b..79cf65bc2e172 100644 --- a/drivers/net/wireless/ath/ath11k/hal.c +++ b/drivers/net/wireless/ath/ath11k/hal.c @@ -601,7 +601,7 @@ u32 ath11k_hal_ce_dst_status_get_length(void *buf) (struct hal_ce_srng_dst_status_desc *)buf; u32 len; - len = FIELD_GET(HAL_CE_DST_STATUS_DESC_FLAGS_LEN, desc->flags); + len = FIELD_GET(HAL_CE_DST_STATUS_DESC_FLAGS_LEN, READ_ONCE(desc->flags)); desc->flags &= ~HAL_CE_DST_STATUS_DESC_FLAGS_LEN; return len; @@ -796,13 +796,23 @@ u32 *ath11k_hal_srng_src_peek(struct ath11k_base *ab, struct hal_srng *srng) void ath11k_hal_srng_access_begin(struct ath11k_base *ab, struct hal_srng *srng) { + u32 hp; + lockdep_assert_held(&srng->lock); if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.cached_tp = *(volatile u32 *)srng->u.src_ring.tp_addr; } else { - srng->u.dst_ring.cached_hp = *srng->u.dst_ring.hp_addr; + hp = READ_ONCE(*srng->u.dst_ring.hp_addr); + + if (hp != srng->u.dst_ring.cached_hp) { + srng->u.dst_ring.cached_hp = hp; + /* Make sure descriptor is read after the head + * pointer. + */ + dma_rmb(); + } /* Try to prefetch the next descriptor in the ring */ if (srng->flags & HAL_SRNG_FLAGS_CACHED) @@ -817,7 +827,6 @@ void ath11k_hal_srng_access_end(struct ath11k_base *ab, struct hal_srng *srng) { lockdep_assert_held(&srng->lock); - /* TODO: See if we need a write memory barrier here */ if (srng->flags & HAL_SRNG_FLAGS_LMAC_RING) { /* For LMAC rings, ring pointer updates are done through FW and * hence written to a shared memory location that is read by FW @@ -825,21 +834,37 @@ void ath11k_hal_srng_access_end(struct ath11k_base *ab, struct hal_srng *srng) if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.last_tp = *(volatile u32 *)srng->u.src_ring.tp_addr; - *srng->u.src_ring.hp_addr = srng->u.src_ring.hp; + /* Make sure descriptor is written before updating the + * head pointer. + */ + dma_wmb(); + WRITE_ONCE(*srng->u.src_ring.hp_addr, srng->u.src_ring.hp); } else { srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr; - *srng->u.dst_ring.tp_addr = srng->u.dst_ring.tp; + /* Make sure descriptor is read before updating the + * tail pointer. + */ + dma_mb(); + WRITE_ONCE(*srng->u.dst_ring.tp_addr, srng->u.dst_ring.tp); } } else { if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.last_tp = *(volatile u32 *)srng->u.src_ring.tp_addr; + /* Assume implementation use an MMIO write accessor + * which has the required wmb() so that the descriptor + * is written before the updating the head pointer. + */ ath11k_hif_write32(ab, (unsigned long)srng->u.src_ring.hp_addr - (unsigned long)ab->mem, srng->u.src_ring.hp); } else { srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr; + /* Make sure descriptor is read before updating the + * tail pointer. + */ + mb(); ath11k_hif_write32(ab, (unsigned long)srng->u.dst_ring.tp_addr - (unsigned long)ab->mem, @@ -1319,6 +1344,10 @@ EXPORT_SYMBOL(ath11k_hal_srng_init); void ath11k_hal_srng_deinit(struct ath11k_base *ab) { struct ath11k_hal *hal = &ab->hal; + int i; + + for (i = 0; i < HAL_SRNG_RING_ID_MAX; i++) + ab->hal.srng_list[i].initialized = 0; ath11k_hal_unregister_srng_key(ab); ath11k_hal_free_cont_rdp(ab); @@ -1327,6 +1356,22 @@ void ath11k_hal_srng_deinit(struct ath11k_base *ab) } EXPORT_SYMBOL(ath11k_hal_srng_deinit); +void ath11k_hal_srng_clear(struct ath11k_base *ab) +{ + /* No need to memset rdp and wrp memory since each individual + * segment would get cleared in ath11k_hal_srng_src_hw_init() + * and ath11k_hal_srng_dst_hw_init(). + */ + memset(ab->hal.srng_list, 0, + sizeof(ab->hal.srng_list)); + memset(ab->hal.shadow_reg_addr, 0, + sizeof(ab->hal.shadow_reg_addr)); + ab->hal.avail_blk_resource = 0; + ab->hal.current_blk_index = 0; + ab->hal.num_shadow_reg_configured = 0; +} +EXPORT_SYMBOL(ath11k_hal_srng_clear); + void ath11k_hal_dump_srng_stats(struct ath11k_base *ab) { struct hal_srng *srng; diff --git a/drivers/net/wireless/ath/ath11k/hal.h b/drivers/net/wireless/ath/ath11k/hal.h index 80447f488954a..aa21eb1fdce15 100644 --- a/drivers/net/wireless/ath/ath11k/hal.h +++ b/drivers/net/wireless/ath/ath11k/hal.h @@ -962,6 +962,7 @@ int ath11k_hal_srng_setup(struct ath11k_base *ab, enum hal_ring_type type, struct hal_srng_params *params); int ath11k_hal_srng_init(struct ath11k_base *ath11k); void ath11k_hal_srng_deinit(struct ath11k_base *ath11k); +void ath11k_hal_srng_clear(struct ath11k_base *ab); void ath11k_hal_dump_srng_stats(struct ath11k_base *ab); void ath11k_hal_srng_get_shadow_config(struct ath11k_base *ab, u32 **cfg, u32 *len); diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index 4247c0f840a48..2921be9bd530c 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -254,9 +254,6 @@ static const u32 ath11k_smps_map[] = { [WLAN_HT_CAP_SM_PS_DISABLED] = WMI_PEER_SMPS_PS_NONE, }; -static int ath11k_start_vdev_delay(struct ieee80211_hw *hw, - struct ieee80211_vif *vif); - enum nl80211_he_ru_alloc ath11k_mac_phy_he_ru_to_nl80211_he_ru_alloc(u16 ru_phy) { enum nl80211_he_ru_alloc ret; @@ -2828,7 +2825,7 @@ static void ath11k_peer_assoc_prepare(struct ath11k *ar, lockdep_assert_held(&ar->conf_mutex); - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); memset(arg, 0, sizeof(*arg)); @@ -4208,6 +4205,40 @@ static int ath11k_clear_peer_keys(struct ath11k_vif *arvif, return first_errno; } +static int ath11k_set_group_keys(struct ath11k_vif *arvif) +{ + struct ath11k *ar = arvif->ar; + struct ath11k_base *ab = ar->ab; + const u8 *addr = arvif->bssid; + int i, ret, first_errno = 0; + struct ath11k_peer *peer; + + spin_lock_bh(&ab->base_lock); + peer = ath11k_peer_find(ab, arvif->vdev_id, addr); + spin_unlock_bh(&ab->base_lock); + + if (!peer) + return -ENOENT; + + for (i = 0; i < ARRAY_SIZE(peer->keys); i++) { + struct ieee80211_key_conf *key = peer->keys[i]; + + if (!key || (key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) + continue; + + ret = ath11k_install_key(arvif, key, SET_KEY, addr, + WMI_KEY_GROUP); + if (ret < 0 && first_errno == 0) + first_errno = ret; + + if (ret < 0) + ath11k_warn(ab, "failed to set group key of idx %d for vdev %d: %d\n", + i, arvif->vdev_id, ret); + } + + return first_errno; +} + static int ath11k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct ieee80211_key_conf *key) @@ -4217,6 +4248,7 @@ static int ath11k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct ath11k_vif *arvif = ath11k_vif_to_arvif(vif); struct ath11k_peer *peer; struct ath11k_sta *arsta; + bool is_ap_with_no_sta; const u8 *peer_addr; int ret = 0; u32 flags = 0; @@ -4277,16 +4309,57 @@ static int ath11k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, else flags |= WMI_KEY_GROUP; - ret = ath11k_install_key(arvif, key, cmd, peer_addr, flags); - if (ret) { - ath11k_warn(ab, "ath11k_install_key failed (%d)\n", ret); - goto exit; - } + ath11k_dbg(ar->ab, ATH11K_DBG_MAC, + "%s for peer %pM on vdev %d flags 0x%X, type = %d, num_sta %d\n", + cmd == SET_KEY ? "SET_KEY" : "DEL_KEY", peer_addr, arvif->vdev_id, + flags, arvif->vdev_type, arvif->num_stations); + + /* Allow group key clearing only in AP mode when no stations are + * associated. There is a known race condition in firmware where + * group addressed packets may be dropped if the key is cleared + * and immediately set again during rekey. + * + * During GTK rekey, mac80211 issues a clear key (if the old key + * exists) followed by an install key operation for same key + * index. This causes ath11k to send two WMI commands in quick + * succession: one to clear the old key and another to install the + * new key in the same slot. + * + * Under certain conditions—especially under high load or time + * sensitive scenarios, firmware may process these commands + * asynchronously in a way that firmware assumes the key is + * cleared whereas hardware has a valid key. This inconsistency + * between hardware and firmware leads to group addressed packet + * drops after rekey. + * Only setting the same key again can restore a valid key in + * firmware and allow packets to be transmitted. + * + * There is a use case where an AP can transition from Secure mode + * to open mode without a vdev restart by just deleting all + * associated peers and clearing key, Hence allow clear key for + * that case alone. Mark arvif->reinstall_group_keys in such cases + * and reinstall the same key when the first peer is added, + * allowing firmware to recover from the race if it had occurred. + */ - ret = ath11k_dp_peer_rx_pn_replay_config(arvif, peer_addr, cmd, key); - if (ret) { - ath11k_warn(ab, "failed to offload PN replay detection %d\n", ret); - goto exit; + is_ap_with_no_sta = (vif->type == NL80211_IFTYPE_AP && + !arvif->num_stations); + if ((flags & WMI_KEY_PAIRWISE) || cmd == SET_KEY || is_ap_with_no_sta) { + ret = ath11k_install_key(arvif, key, cmd, peer_addr, flags); + if (ret) { + ath11k_warn(ab, "ath11k_install_key failed (%d)\n", ret); + goto exit; + } + + ret = ath11k_dp_peer_rx_pn_replay_config(arvif, peer_addr, cmd, key); + if (ret) { + ath11k_warn(ab, "failed to offload PN replay detection %d\n", + ret); + goto exit; + } + + if ((flags & WMI_KEY_GROUP) && cmd == SET_KEY && is_ap_with_no_sta) + arvif->reinstall_group_keys = true; } spin_lock_bh(&ab->base_lock); @@ -4311,7 +4384,7 @@ static int ath11k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, ath11k_warn(ab, "peer %pM disappeared!\n", peer_addr); if (sta) { - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); switch (key->cipher) { case WLAN_CIPHER_SUITE_TKIP: @@ -4879,6 +4952,7 @@ static int ath11k_mac_inc_num_stations(struct ath11k_vif *arvif, return -ENOBUFS; ar->num_stations++; + arvif->num_stations++; return 0; } @@ -4894,100 +4968,7 @@ static void ath11k_mac_dec_num_stations(struct ath11k_vif *arvif, return; ar->num_stations--; -} - -static int ath11k_mac_station_add(struct ath11k *ar, - struct ieee80211_vif *vif, - struct ieee80211_sta *sta) -{ - struct ath11k_base *ab = ar->ab; - struct ath11k_vif *arvif = ath11k_vif_to_arvif(vif); - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; - struct peer_create_params peer_param; - int ret; - - lockdep_assert_held(&ar->conf_mutex); - - ret = ath11k_mac_inc_num_stations(arvif, sta); - if (ret) { - ath11k_warn(ab, "refusing to associate station: too many connected already (%d)\n", - ar->max_num_stations); - goto exit; - } - - arsta->rx_stats = kzalloc(sizeof(*arsta->rx_stats), GFP_KERNEL); - if (!arsta->rx_stats) { - ret = -ENOMEM; - goto dec_num_station; - } - - peer_param.vdev_id = arvif->vdev_id; - peer_param.peer_addr = sta->addr; - peer_param.peer_type = WMI_PEER_TYPE_DEFAULT; - - ret = ath11k_peer_create(ar, arvif, sta, &peer_param); - if (ret) { - ath11k_warn(ab, "Failed to add peer: %pM for VDEV: %d\n", - sta->addr, arvif->vdev_id); - goto free_rx_stats; - } - - ath11k_dbg(ab, ATH11K_DBG_MAC, "Added peer: %pM for VDEV: %d\n", - sta->addr, arvif->vdev_id); - - if (ath11k_debugfs_is_extd_tx_stats_enabled(ar)) { - arsta->tx_stats = kzalloc(sizeof(*arsta->tx_stats), GFP_KERNEL); - if (!arsta->tx_stats) { - ret = -ENOMEM; - goto free_peer; - } - } - - if (ieee80211_vif_is_mesh(vif)) { - ath11k_dbg(ab, ATH11K_DBG_MAC, - "setting USE_4ADDR for mesh STA %pM\n", sta->addr); - ret = ath11k_wmi_set_peer_param(ar, sta->addr, - arvif->vdev_id, - WMI_PEER_USE_4ADDR, 1); - if (ret) { - ath11k_warn(ab, "failed to set mesh STA %pM 4addr capability: %d\n", - sta->addr, ret); - goto free_tx_stats; - } - } - - ret = ath11k_dp_peer_setup(ar, arvif->vdev_id, sta->addr); - if (ret) { - ath11k_warn(ab, "failed to setup dp for peer %pM on vdev %i (%d)\n", - sta->addr, arvif->vdev_id, ret); - goto free_tx_stats; - } - - if (ab->hw_params.vdev_start_delay && - !arvif->is_started && - arvif->vdev_type != WMI_VDEV_TYPE_AP) { - ret = ath11k_start_vdev_delay(ar->hw, vif); - if (ret) { - ath11k_warn(ab, "failed to delay vdev start: %d\n", ret); - goto free_tx_stats; - } - } - - ewma_avg_rssi_init(&arsta->avg_rssi); - return 0; - -free_tx_stats: - kfree(arsta->tx_stats); - arsta->tx_stats = NULL; -free_peer: - ath11k_peer_delete(ar, arvif->vdev_id, sta->addr); -free_rx_stats: - kfree(arsta->rx_stats); - arsta->rx_stats = NULL; -dec_num_station: - ath11k_mac_dec_num_stations(arvif, sta); -exit: - return ret; + arvif->num_stations--; } static u32 ath11k_mac_ieee80211_sta_bw_to_wmi(struct ath11k *ar, @@ -5018,140 +4999,6 @@ static u32 ath11k_mac_ieee80211_sta_bw_to_wmi(struct ath11k *ar, return bw; } -static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw, - struct ieee80211_vif *vif, - struct ieee80211_sta *sta, - enum ieee80211_sta_state old_state, - enum ieee80211_sta_state new_state) -{ - struct ath11k *ar = hw->priv; - struct ath11k_vif *arvif = ath11k_vif_to_arvif(vif); - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; - struct ath11k_peer *peer; - int ret = 0; - - /* cancel must be done outside the mutex to avoid deadlock */ - if ((old_state == IEEE80211_STA_NONE && - new_state == IEEE80211_STA_NOTEXIST)) { - cancel_work_sync(&arsta->update_wk); - cancel_work_sync(&arsta->set_4addr_wk); - } - - mutex_lock(&ar->conf_mutex); - - if (old_state == IEEE80211_STA_NOTEXIST && - new_state == IEEE80211_STA_NONE) { - memset(arsta, 0, sizeof(*arsta)); - arsta->arvif = arvif; - arsta->peer_ps_state = WMI_PEER_PS_STATE_DISABLED; - INIT_WORK(&arsta->update_wk, ath11k_sta_rc_update_wk); - INIT_WORK(&arsta->set_4addr_wk, ath11k_sta_set_4addr_wk); - - ret = ath11k_mac_station_add(ar, vif, sta); - if (ret) - ath11k_warn(ar->ab, "Failed to add station: %pM for VDEV: %d\n", - sta->addr, arvif->vdev_id); - } else if ((old_state == IEEE80211_STA_NONE && - new_state == IEEE80211_STA_NOTEXIST)) { - bool skip_peer_delete = ar->ab->hw_params.vdev_start_delay && - vif->type == NL80211_IFTYPE_STATION; - - ath11k_dp_peer_cleanup(ar, arvif->vdev_id, sta->addr); - - if (!skip_peer_delete) { - ret = ath11k_peer_delete(ar, arvif->vdev_id, sta->addr); - if (ret) - ath11k_warn(ar->ab, - "Failed to delete peer: %pM for VDEV: %d\n", - sta->addr, arvif->vdev_id); - else - ath11k_dbg(ar->ab, - ATH11K_DBG_MAC, - "Removed peer: %pM for VDEV: %d\n", - sta->addr, arvif->vdev_id); - } - - ath11k_mac_dec_num_stations(arvif, sta); - mutex_lock(&ar->ab->tbl_mtx_lock); - spin_lock_bh(&ar->ab->base_lock); - peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); - if (skip_peer_delete && peer) { - peer->sta = NULL; - } else if (peer && peer->sta == sta) { - ath11k_warn(ar->ab, "Found peer entry %pM n vdev %i after it was supposedly removed\n", - vif->addr, arvif->vdev_id); - ath11k_peer_rhash_delete(ar->ab, peer); - peer->sta = NULL; - list_del(&peer->list); - kfree(peer); - ar->num_peers--; - } - spin_unlock_bh(&ar->ab->base_lock); - mutex_unlock(&ar->ab->tbl_mtx_lock); - - kfree(arsta->tx_stats); - arsta->tx_stats = NULL; - - kfree(arsta->rx_stats); - arsta->rx_stats = NULL; - } else if (old_state == IEEE80211_STA_AUTH && - new_state == IEEE80211_STA_ASSOC && - (vif->type == NL80211_IFTYPE_AP || - vif->type == NL80211_IFTYPE_MESH_POINT || - vif->type == NL80211_IFTYPE_ADHOC)) { - ret = ath11k_station_assoc(ar, vif, sta, false); - if (ret) - ath11k_warn(ar->ab, "Failed to associate station: %pM\n", - sta->addr); - - spin_lock_bh(&ar->data_lock); - /* Set arsta bw and prev bw */ - arsta->bw = ath11k_mac_ieee80211_sta_bw_to_wmi(ar, sta); - arsta->bw_prev = arsta->bw; - spin_unlock_bh(&ar->data_lock); - } else if (old_state == IEEE80211_STA_ASSOC && - new_state == IEEE80211_STA_AUTHORIZED) { - spin_lock_bh(&ar->ab->base_lock); - - peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); - if (peer) - peer->is_authorized = true; - - spin_unlock_bh(&ar->ab->base_lock); - - if (vif->type == NL80211_IFTYPE_STATION && arvif->is_up) { - ret = ath11k_wmi_set_peer_param(ar, sta->addr, - arvif->vdev_id, - WMI_PEER_AUTHORIZE, - 1); - if (ret) - ath11k_warn(ar->ab, "Unable to authorize peer %pM vdev %d: %d\n", - sta->addr, arvif->vdev_id, ret); - } - } else if (old_state == IEEE80211_STA_AUTHORIZED && - new_state == IEEE80211_STA_ASSOC) { - spin_lock_bh(&ar->ab->base_lock); - - peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); - if (peer) - peer->is_authorized = false; - - spin_unlock_bh(&ar->ab->base_lock); - } else if (old_state == IEEE80211_STA_ASSOC && - new_state == IEEE80211_STA_AUTH && - (vif->type == NL80211_IFTYPE_AP || - vif->type == NL80211_IFTYPE_MESH_POINT || - vif->type == NL80211_IFTYPE_ADHOC)) { - ret = ath11k_station_disassoc(ar, vif, sta); - if (ret) - ath11k_warn(ar->ab, "Failed to disassociate station: %pM\n", - sta->addr); - } - - mutex_unlock(&ar->conf_mutex); - return ret; -} - static int ath11k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta) @@ -5192,7 +5039,7 @@ static void ath11k_mac_op_sta_set_4addr(struct ieee80211_hw *hw, struct ieee80211_sta *sta, bool enabled) { struct ath11k *ar = hw->priv; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); if (enabled && !arsta->use_4addr_set) { ieee80211_queue_work(ar->hw, &arsta->set_4addr_wk); @@ -5206,7 +5053,7 @@ static void ath11k_mac_op_sta_rc_update(struct ieee80211_hw *hw, u32 changed) { struct ath11k *ar = hw->priv; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k_vif *arvif = ath11k_vif_to_arvif(vif); struct ath11k_peer *peer; u32 bw, smps; @@ -6204,7 +6051,7 @@ static void ath11k_mac_op_tx(struct ieee80211_hw *hw, } if (control->sta) - arsta = (struct ath11k_sta *)control->sta->drv_priv; + arsta = ath11k_sta_to_arsta(control->sta); ret = ath11k_dp_tx(ar, arvif, arsta, skb); if (unlikely(ret)) { @@ -7546,8 +7393,8 @@ static void ath11k_mac_op_change_chanctx(struct ieee80211_hw *hw, mutex_unlock(&ar->conf_mutex); } -static int ath11k_start_vdev_delay(struct ieee80211_hw *hw, - struct ieee80211_vif *vif) +static int ath11k_mac_start_vdev_delay(struct ieee80211_hw *hw, + struct ieee80211_vif *vif) { struct ath11k *ar = hw->priv; struct ath11k_base *ab = ar->ab; @@ -8228,7 +8075,7 @@ static void ath11k_mac_set_bitrate_mask_iter(void *data, struct ieee80211_sta *sta) { struct ath11k_vif *arvif = data; - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arvif->ar; spin_lock_bh(&ar->data_lock); @@ -8632,7 +8479,7 @@ static void ath11k_mac_op_sta_statistics(struct ieee80211_hw *hw, struct ieee80211_sta *sta, struct station_info *sinfo) { - struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); struct ath11k *ar = arsta->arvif->ar; s8 signal; bool db2dbm = test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT, @@ -9010,11 +8857,13 @@ static int ath11k_fw_stats_request(struct ath11k *ar, lockdep_assert_held(&ar->conf_mutex); spin_lock_bh(&ar->data_lock); - ar->fw_stats_done = false; ath11k_fw_stats_pdevs_free(&ar->fw_stats.pdevs); + ar->fw_stats.num_vdev_recvd = 0; + ar->fw_stats.num_bcn_recvd = 0; spin_unlock_bh(&ar->data_lock); reinit_completion(&ar->fw_stats_complete); + reinit_completion(&ar->fw_stats_done); ret = ath11k_wmi_send_stats_request_cmd(ar, req_param); if (ret) { @@ -9097,6 +8946,249 @@ static int ath11k_mac_op_get_txpower(struct ieee80211_hw *hw, return 0; } +static int ath11k_mac_station_add(struct ath11k *ar, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct ath11k_base *ab = ar->ab; + struct ath11k_vif *arvif = ath11k_vif_to_arvif(vif); + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); + struct peer_create_params peer_param; + int ret; + + lockdep_assert_held(&ar->conf_mutex); + + ret = ath11k_mac_inc_num_stations(arvif, sta); + if (ret) { + ath11k_warn(ab, "refusing to associate station: too many connected already (%d)\n", + ar->max_num_stations); + goto exit; + } + + /* Driver allows the DEL KEY followed by SET KEY sequence for + * group keys for only when there is no clients associated, if at + * all firmware has entered the race during that window, + * reinstalling the same key when the first sta connects will allow + * firmware to recover from the race. + */ + if (arvif->num_stations == 1 && arvif->reinstall_group_keys) { + ath11k_dbg(ab, ATH11K_DBG_MAC, "set group keys on 1st station add for vdev %d\n", + arvif->vdev_id); + ret = ath11k_set_group_keys(arvif); + if (ret) + goto dec_num_station; + arvif->reinstall_group_keys = false; + } + + arsta->rx_stats = kzalloc(sizeof(*arsta->rx_stats), GFP_KERNEL); + if (!arsta->rx_stats) { + ret = -ENOMEM; + goto dec_num_station; + } + + peer_param.vdev_id = arvif->vdev_id; + peer_param.peer_addr = sta->addr; + peer_param.peer_type = WMI_PEER_TYPE_DEFAULT; + + ret = ath11k_peer_create(ar, arvif, sta, &peer_param); + if (ret) { + ath11k_warn(ab, "Failed to add peer: %pM for VDEV: %d\n", + sta->addr, arvif->vdev_id); + goto free_rx_stats; + } + + ath11k_dbg(ab, ATH11K_DBG_MAC, "Added peer: %pM for VDEV: %d\n", + sta->addr, arvif->vdev_id); + + if (ath11k_debugfs_is_extd_tx_stats_enabled(ar)) { + arsta->tx_stats = kzalloc(sizeof(*arsta->tx_stats), GFP_KERNEL); + if (!arsta->tx_stats) { + ret = -ENOMEM; + goto free_peer; + } + } + + if (ieee80211_vif_is_mesh(vif)) { + ath11k_dbg(ab, ATH11K_DBG_MAC, + "setting USE_4ADDR for mesh STA %pM\n", sta->addr); + ret = ath11k_wmi_set_peer_param(ar, sta->addr, + arvif->vdev_id, + WMI_PEER_USE_4ADDR, 1); + if (ret) { + ath11k_warn(ab, "failed to set mesh STA %pM 4addr capability: %d\n", + sta->addr, ret); + goto free_tx_stats; + } + } + + ret = ath11k_dp_peer_setup(ar, arvif->vdev_id, sta->addr); + if (ret) { + ath11k_warn(ab, "failed to setup dp for peer %pM on vdev %i (%d)\n", + sta->addr, arvif->vdev_id, ret); + goto free_tx_stats; + } + + if (ab->hw_params.vdev_start_delay && + !arvif->is_started && + arvif->vdev_type != WMI_VDEV_TYPE_AP) { + ret = ath11k_mac_start_vdev_delay(ar->hw, vif); + if (ret) { + ath11k_warn(ab, "failed to delay vdev start: %d\n", ret); + goto free_tx_stats; + } + } + + ewma_avg_rssi_init(&arsta->avg_rssi); + return 0; + +free_tx_stats: + kfree(arsta->tx_stats); + arsta->tx_stats = NULL; +free_peer: + ath11k_peer_delete(ar, arvif->vdev_id, sta->addr); +free_rx_stats: + kfree(arsta->rx_stats); + arsta->rx_stats = NULL; +dec_num_station: + ath11k_mac_dec_num_stations(arvif, sta); +exit: + return ret; +} + +static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + enum ieee80211_sta_state old_state, + enum ieee80211_sta_state new_state) +{ + struct ath11k *ar = hw->priv; + struct ath11k_vif *arvif = ath11k_vif_to_arvif(vif); + struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); + struct ath11k_peer *peer; + int ret = 0; + + /* cancel must be done outside the mutex to avoid deadlock */ + if ((old_state == IEEE80211_STA_NONE && + new_state == IEEE80211_STA_NOTEXIST)) { + cancel_work_sync(&arsta->update_wk); + cancel_work_sync(&arsta->set_4addr_wk); + } + + mutex_lock(&ar->conf_mutex); + + if (old_state == IEEE80211_STA_NOTEXIST && + new_state == IEEE80211_STA_NONE) { + memset(arsta, 0, sizeof(*arsta)); + arsta->arvif = arvif; + arsta->peer_ps_state = WMI_PEER_PS_STATE_DISABLED; + INIT_WORK(&arsta->update_wk, ath11k_sta_rc_update_wk); + INIT_WORK(&arsta->set_4addr_wk, ath11k_sta_set_4addr_wk); + + ret = ath11k_mac_station_add(ar, vif, sta); + if (ret) + ath11k_warn(ar->ab, "Failed to add station: %pM for VDEV: %d\n", + sta->addr, arvif->vdev_id); + } else if ((old_state == IEEE80211_STA_NONE && + new_state == IEEE80211_STA_NOTEXIST)) { + bool skip_peer_delete = ar->ab->hw_params.vdev_start_delay && + vif->type == NL80211_IFTYPE_STATION; + + ath11k_dp_peer_cleanup(ar, arvif->vdev_id, sta->addr); + + if (!skip_peer_delete) { + ret = ath11k_peer_delete(ar, arvif->vdev_id, sta->addr); + if (ret) + ath11k_warn(ar->ab, + "Failed to delete peer: %pM for VDEV: %d\n", + sta->addr, arvif->vdev_id); + else + ath11k_dbg(ar->ab, + ATH11K_DBG_MAC, + "Removed peer: %pM for VDEV: %d\n", + sta->addr, arvif->vdev_id); + } + + ath11k_mac_dec_num_stations(arvif, sta); + mutex_lock(&ar->ab->tbl_mtx_lock); + spin_lock_bh(&ar->ab->base_lock); + peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); + if (skip_peer_delete && peer) { + peer->sta = NULL; + } else if (peer && peer->sta == sta) { + ath11k_warn(ar->ab, "Found peer entry %pM n vdev %i after it was supposedly removed\n", + vif->addr, arvif->vdev_id); + ath11k_peer_rhash_delete(ar->ab, peer); + peer->sta = NULL; + list_del(&peer->list); + kfree(peer); + ar->num_peers--; + } + spin_unlock_bh(&ar->ab->base_lock); + mutex_unlock(&ar->ab->tbl_mtx_lock); + + kfree(arsta->tx_stats); + arsta->tx_stats = NULL; + + kfree(arsta->rx_stats); + arsta->rx_stats = NULL; + } else if (old_state == IEEE80211_STA_AUTH && + new_state == IEEE80211_STA_ASSOC && + (vif->type == NL80211_IFTYPE_AP || + vif->type == NL80211_IFTYPE_MESH_POINT || + vif->type == NL80211_IFTYPE_ADHOC)) { + ret = ath11k_station_assoc(ar, vif, sta, false); + if (ret) + ath11k_warn(ar->ab, "Failed to associate station: %pM\n", + sta->addr); + + spin_lock_bh(&ar->data_lock); + /* Set arsta bw and prev bw */ + arsta->bw = ath11k_mac_ieee80211_sta_bw_to_wmi(ar, sta); + arsta->bw_prev = arsta->bw; + spin_unlock_bh(&ar->data_lock); + } else if (old_state == IEEE80211_STA_ASSOC && + new_state == IEEE80211_STA_AUTHORIZED) { + spin_lock_bh(&ar->ab->base_lock); + + peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); + if (peer) + peer->is_authorized = true; + + spin_unlock_bh(&ar->ab->base_lock); + + if (vif->type == NL80211_IFTYPE_STATION && arvif->is_up) { + ret = ath11k_wmi_set_peer_param(ar, sta->addr, + arvif->vdev_id, + WMI_PEER_AUTHORIZE, + 1); + if (ret) + ath11k_warn(ar->ab, "Unable to authorize peer %pM vdev %d: %d\n", + sta->addr, arvif->vdev_id, ret); + } + } else if (old_state == IEEE80211_STA_AUTHORIZED && + new_state == IEEE80211_STA_ASSOC) { + spin_lock_bh(&ar->ab->base_lock); + + peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); + if (peer) + peer->is_authorized = false; + + spin_unlock_bh(&ar->ab->base_lock); + } else if (old_state == IEEE80211_STA_ASSOC && + new_state == IEEE80211_STA_AUTH && + (vif->type == NL80211_IFTYPE_AP || + vif->type == NL80211_IFTYPE_MESH_POINT || + vif->type == NL80211_IFTYPE_ADHOC)) { + ret = ath11k_station_disassoc(ar, vif, sta); + if (ret) + ath11k_warn(ar->ab, "Failed to disassociate station: %pM\n", + sta->addr); + } + + mutex_unlock(&ar->conf_mutex); + return ret; +} + static const struct ieee80211_ops ath11k_ops = { .tx = ath11k_mac_op_tx, .wake_tx_queue = ieee80211_handle_wake_tx_queue, diff --git a/drivers/net/wireless/ath/ath11k/peer.c b/drivers/net/wireless/ath/ath11k/peer.c index ca719eb3f7f82..6d0126c393018 100644 --- a/drivers/net/wireless/ath/ath11k/peer.c +++ b/drivers/net/wireless/ath/ath11k/peer.c @@ -446,7 +446,7 @@ int ath11k_peer_create(struct ath11k *ar, struct ath11k_vif *arvif, peer->sec_type_grp = HAL_ENCRYPT_TYPE_OPEN; if (sta) { - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); arsta->tcl_metadata |= FIELD_PREP(HTT_TCL_META_DATA_TYPE, 0) | FIELD_PREP(HTT_TCL_META_DATA_PEER_ID, peer->peer_id); diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c index fa46e645009cf..91e31f30d2c80 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.c +++ b/drivers/net/wireless/ath/ath11k/qmi.c @@ -1989,6 +1989,15 @@ static int ath11k_qmi_alloc_target_mem_chunk(struct ath11k_base *ab) chunk->prev_size == chunk->size) continue; + if (ab->qmi.mem_seg_count <= ATH11K_QMI_FW_MEM_REQ_SEGMENT_CNT) { + ath11k_dbg(ab, ATH11K_DBG_QMI, + "size/type mismatch (current %d %u) (prev %d %u), try later with small size\n", + chunk->size, chunk->type, + chunk->prev_size, chunk->prev_type); + ab->qmi.target_mem_delayed = true; + return 0; + } + /* cannot reuse the existing chunk */ dma_free_coherent(ab->dev, chunk->prev_size, chunk->vaddr, chunk->paddr); diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index 2cc13e60f422f..16687223bdcba 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -5841,6 +5841,9 @@ static int wmi_process_mgmt_tx_comp(struct ath11k *ar, dma_unmap_single(ar->ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); info = IEEE80211_SKB_CB(msdu); + memset(&info->status, 0, sizeof(info->status)); + info->status.rates[0].idx = -1; + if ((!(info->flags & IEEE80211_TX_CTL_NO_ACK)) && !tx_compl_param->status) { info->flags |= IEEE80211_TX_STAT_ACK; @@ -6452,7 +6455,7 @@ static int ath11k_wmi_tlv_rssi_chain_parse(struct ath11k_base *ab, goto exit; } - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); BUILD_BUG_ON(ARRAY_SIZE(arsta->chain_signal) > ARRAY_SIZE(stats_rssi->rssi_avg_beacon)); @@ -6540,7 +6543,7 @@ static int ath11k_wmi_tlv_fw_stats_data_parse(struct ath11k_base *ab, arvif->bssid, NULL); if (sta) { - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); arsta->rssi_beacon = src->beacon_snr; ath11k_dbg(ab, ATH11K_DBG_WMI, "stats vdev id %d snr %d\n", @@ -7469,7 +7472,7 @@ static void ath11k_wmi_event_peer_sta_ps_state_chg(struct ath11k_base *ab, goto exit; } - arsta = (struct ath11k_sta *)sta->drv_priv; + arsta = ath11k_sta_to_arsta(sta); spin_lock_bh(&ar->data_lock); @@ -8183,7 +8186,7 @@ static void ath11k_update_stats_event(struct ath11k_base *ab, struct sk_buff *sk */ if (stats.stats_id == WMI_REQUEST_PDEV_STAT) { list_splice_tail_init(&stats.pdevs, &ar->fw_stats.pdevs); - ar->fw_stats_done = true; + complete(&ar->fw_stats_done); goto complete; } diff --git a/drivers/net/wireless/ath/ath12k/ce.c b/drivers/net/wireless/ath/ath12k/ce.c index be0d669d31fcc..b66d23d6b2bd9 100644 --- a/drivers/net/wireless/ath/ath12k/ce.c +++ b/drivers/net/wireless/ath/ath12k/ce.c @@ -344,10 +344,6 @@ static int ath12k_ce_completed_recv_next(struct ath12k_ce_pipe *pipe, } *nbytes = ath12k_hal_ce_dst_status_get_length(desc); - if (*nbytes == 0) { - ret = -EIO; - goto err; - } *skb = pipe->dest_ring->skb[sw_index]; pipe->dest_ring->skb[sw_index] = NULL; @@ -380,8 +376,8 @@ static void ath12k_ce_recv_process_cb(struct ath12k_ce_pipe *pipe) dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr, max_nbytes, DMA_FROM_DEVICE); - if (unlikely(max_nbytes < nbytes)) { - ath12k_warn(ab, "rxed more than expected (nbytes %d, max %d)", + if (unlikely(max_nbytes < nbytes || nbytes == 0)) { + ath12k_warn(ab, "unexpected rx length (nbytes %d, max %d)", nbytes, max_nbytes); dev_kfree_skb_any(skb); continue; diff --git a/drivers/net/wireless/ath/ath12k/ce.h b/drivers/net/wireless/ath/ath12k/ce.h index 857bc5f9e946a..f9547a3945e44 100644 --- a/drivers/net/wireless/ath/ath12k/ce.h +++ b/drivers/net/wireless/ath/ath12k/ce.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_CE_H @@ -39,8 +39,8 @@ #define PIPEDIR_INOUT_H2H 4 /* bidirectional, host to host */ /* CE address/mask */ -#define CE_HOST_IE_ADDRESS 0x00A1803C -#define CE_HOST_IE_2_ADDRESS 0x00A18040 +#define CE_HOST_IE_ADDRESS 0x75804C +#define CE_HOST_IE_2_ADDRESS 0x758050 #define CE_HOST_IE_3_ADDRESS CE_HOST_IE_ADDRESS #define CE_HOST_IE_3_SHIFT 0xC diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c index 3df8059d55129..1b07a183aaedc 100644 --- a/drivers/net/wireless/ath/ath12k/core.c +++ b/drivers/net/wireless/ath/ath12k/core.c @@ -657,6 +657,7 @@ static int ath12k_core_reconfigure_on_crash(struct ath12k_base *ab) void ath12k_core_halt(struct ath12k *ar) { + struct list_head *pos, *n; struct ath12k_base *ab = ar->ab; lockdep_assert_held(&ar->conf_mutex); @@ -671,7 +672,12 @@ void ath12k_core_halt(struct ath12k *ar) rcu_assign_pointer(ab->pdevs_active[ar->pdev_idx], NULL); synchronize_rcu(); - INIT_LIST_HEAD(&ar->arvifs); + + spin_lock_bh(&ar->data_lock); + list_for_each_safe(pos, n, &ar->arvifs) + list_del_init(pos); + spin_unlock_bh(&ar->data_lock); + idr_init(&ar->txmgmt_idr); } diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h index 33f4706af880d..18dfd7aab610c 100644 --- a/drivers/net/wireless/ath/ath12k/core.h +++ b/drivers/net/wireless/ath/ath12k/core.h @@ -125,6 +125,7 @@ struct ath12k_ext_irq_grp { u32 num_irq; u32 grp_id; u64 timestamp; + bool napi_enabled; struct napi_struct napi; struct net_device napi_ndev; }; diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c index c663ff990b479..c918f5d12975c 100644 --- a/drivers/net/wireless/ath/ath12k/dp.c +++ b/drivers/net/wireless/ath/ath12k/dp.c @@ -74,6 +74,7 @@ int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) ret = ath12k_dp_rx_peer_frag_setup(ar, addr, vdev_id); if (ret) { ath12k_warn(ab, "failed to setup rx defrag context\n"); + tid--; goto peer_clean; } @@ -91,7 +92,7 @@ int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) return -ENOENT; } - for (; tid >= 0; tid--) + for (tid--; tid >= 0; tid--) ath12k_dp_rx_peer_tid_delete(ar, peer, tid); spin_unlock_bh(&ab->base_lock); diff --git a/drivers/net/wireless/ath/ath12k/dp.h b/drivers/net/wireless/ath/ath12k/dp.h index 61f765432516b..284032db0b98d 100644 --- a/drivers/net/wireless/ath/ath12k/dp.h +++ b/drivers/net/wireless/ath/ath12k/dp.h @@ -162,7 +162,7 @@ struct ath12k_pdev_dp { #define DP_REO_REINJECT_RING_SIZE 32 #define DP_RX_RELEASE_RING_SIZE 1024 #define DP_REO_EXCEPTION_RING_SIZE 128 -#define DP_REO_CMD_RING_SIZE 128 +#define DP_REO_CMD_RING_SIZE 256 #define DP_REO_STATUS_RING_SIZE 2048 #define DP_RXDMA_BUF_RING_SIZE 4096 #define DP_RXDMA_REFILL_RING_SIZE 2048 diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c index 35f22a4a16cf2..69bf75ebd7518 100644 --- a/drivers/net/wireless/ath/ath12k/dp_mon.c +++ b/drivers/net/wireless/ath/ath12k/dp_mon.c @@ -1077,6 +1077,8 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct bool is_mcbc = rxcb->is_mcbc; bool is_eapol_tkip = rxcb->is_eapol; + status->link_valid = 0; + if ((status->encoding == RX_ENC_HE) && !(status->flag & RX_FLAG_RADIOTAP_HE) && !(status->flag & RX_FLAG_SKIP_MONITOR)) { he = skb_push(msdu, sizeof(known)); diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c index 8d9315038a75e..56dda76d066c3 100644 --- a/drivers/net/wireless/ath/ath12k/dp_rx.c +++ b/drivers/net/wireless/ath/ath12k/dp_rx.c @@ -3683,6 +3683,15 @@ static bool ath12k_dp_rx_h_tkip_mic_err(struct ath12k *ar, struct sk_buff *msdu, l3pad_bytes = ath12k_dp_rx_h_l3pad(ab, desc); msdu_len = ath12k_dp_rx_h_msdu_len(ab, desc); + + if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE) { + ath12k_dbg(ab, ATH12K_DBG_DATA, + "invalid msdu len in tkip mic err %u\n", msdu_len); + ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "", desc, + sizeof(*desc)); + return true; + } + skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.c b/drivers/net/wireless/ath/ath12k/dp_tx.c index e025e4d0e7678..474e0d4d406ea 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.c +++ b/drivers/net/wireless/ath/ath12k/dp_tx.c @@ -118,7 +118,7 @@ static void ath12k_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab, void *cmd, le32_encode_bits(ti->data_len, HAL_TX_MSDU_EXT_INFO1_BUF_LEN); - tcl_ext_cmd->info1 = le32_encode_bits(1, HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE) | + tcl_ext_cmd->info1 |= le32_encode_bits(1, HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE) | le32_encode_bits(ti->encap_type, HAL_TX_MSDU_EXT_INFO1_ENCAP_TYPE) | le32_encode_bits(ti->encrypt_type, @@ -422,13 +422,13 @@ ath12k_dp_tx_process_htt_tx_complete(struct ath12k_base *ab, switch (wbm_status) { case HAL_WBM_REL_HTT_TX_COMP_STATUS_OK: - case HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP: - case HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL: ts.acked = (wbm_status == HAL_WBM_REL_HTT_TX_COMP_STATUS_OK); ts.ack_rssi = le32_get_bits(status_desc->info2, HTT_TX_WBM_COMP_INFO2_ACK_RSSI); ath12k_dp_tx_htt_tx_complete_buf(ab, msdu, tx_ring, &ts); break; + case HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP: + case HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL: case HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ: case HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT: ath12k_dp_tx_free_txbuf(ab, msdu, mac_id, tx_ring); diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c index 0b5a91ab0df49..2d80cb9f0e7b7 100644 --- a/drivers/net/wireless/ath/ath12k/hal.c +++ b/drivers/net/wireless/ath/ath12k/hal.c @@ -1565,7 +1565,7 @@ u32 ath12k_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc { u32 len; - len = le32_get_bits(desc->flags, HAL_CE_DST_STATUS_DESC_FLAGS_LEN); + len = le32_get_bits(READ_ONCE(desc->flags), HAL_CE_DST_STATUS_DESC_FLAGS_LEN); desc->flags &= ~cpu_to_le32(HAL_CE_DST_STATUS_DESC_FLAGS_LEN); return len; @@ -1728,13 +1728,24 @@ void *ath12k_hal_srng_src_get_next_reaped(struct ath12k_base *ab, void ath12k_hal_srng_access_begin(struct ath12k_base *ab, struct hal_srng *srng) { + u32 hp; + lockdep_assert_held(&srng->lock); - if (srng->ring_dir == HAL_SRNG_DIR_SRC) + if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.cached_tp = *(volatile u32 *)srng->u.src_ring.tp_addr; - else - srng->u.dst_ring.cached_hp = *srng->u.dst_ring.hp_addr; + } else { + hp = READ_ONCE(*srng->u.dst_ring.hp_addr); + + if (hp != srng->u.dst_ring.cached_hp) { + srng->u.dst_ring.cached_hp = hp; + /* Make sure descriptor is read after the head + * pointer. + */ + dma_rmb(); + } + } } /* Update cached ring head/tail pointers to HW. ath12k_hal_srng_access_begin() @@ -1744,7 +1755,6 @@ void ath12k_hal_srng_access_end(struct ath12k_base *ab, struct hal_srng *srng) { lockdep_assert_held(&srng->lock); - /* TODO: See if we need a write memory barrier here */ if (srng->flags & HAL_SRNG_FLAGS_LMAC_RING) { /* For LMAC rings, ring pointer updates are done through FW and * hence written to a shared memory location that is read by FW @@ -1752,21 +1762,37 @@ void ath12k_hal_srng_access_end(struct ath12k_base *ab, struct hal_srng *srng) if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.last_tp = *(volatile u32 *)srng->u.src_ring.tp_addr; - *srng->u.src_ring.hp_addr = srng->u.src_ring.hp; + /* Make sure descriptor is written before updating the + * head pointer. + */ + dma_wmb(); + WRITE_ONCE(*srng->u.src_ring.hp_addr, srng->u.src_ring.hp); } else { srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr; - *srng->u.dst_ring.tp_addr = srng->u.dst_ring.tp; + /* Make sure descriptor is read before updating the + * tail pointer. + */ + dma_mb(); + WRITE_ONCE(*srng->u.dst_ring.tp_addr, srng->u.dst_ring.tp); } } else { if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.last_tp = *(volatile u32 *)srng->u.src_ring.tp_addr; + /* Assume implementation use an MMIO write accessor + * which has the required wmb() so that the descriptor + * is written before the updating the head pointer. + */ ath12k_hif_write32(ab, (unsigned long)srng->u.src_ring.hp_addr - (unsigned long)ab->mem, srng->u.src_ring.hp); } else { srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr; + /* Make sure descriptor is read before updating the + * tail pointer. + */ + mb(); ath12k_hif_write32(ab, (unsigned long)srng->u.dst_ring.tp_addr - (unsigned long)ab->mem, diff --git a/drivers/net/wireless/ath/ath12k/hal_desc.h b/drivers/net/wireless/ath/ath12k/hal_desc.h index 6c17adc6d60b5..5fd9232ad101e 100644 --- a/drivers/net/wireless/ath/ath12k/hal_desc.h +++ b/drivers/net/wireless/ath/ath12k/hal_desc.h @@ -683,7 +683,7 @@ enum hal_rx_msdu_desc_reo_dest_ind { #define RX_MSDU_DESC_INFO0_DECAP_FORMAT GENMASK(30, 29) #define HAL_RX_MSDU_PKT_LENGTH_GET(val) \ - (u32_get_bits((val), RX_MSDU_DESC_INFO0_MSDU_LENGTH)) + (le32_get_bits((val), RX_MSDU_DESC_INFO0_MSDU_LENGTH)) struct rx_msdu_desc { __le32 info0; @@ -2918,7 +2918,7 @@ struct hal_mon_buf_ring { #define HAL_MON_DEST_COOKIE_BUF_ID GENMASK(17, 0) -#define HAL_MON_DEST_INFO0_END_OFFSET GENMASK(15, 0) +#define HAL_MON_DEST_INFO0_END_OFFSET GENMASK(11, 0) #define HAL_MON_DEST_INFO0_FLUSH_DETECTED BIT(16) #define HAL_MON_DEST_INFO0_END_OF_PPDU BIT(17) #define HAL_MON_DEST_INFO0_INITIATOR BIT(18) diff --git a/drivers/net/wireless/ath/ath12k/hw.c b/drivers/net/wireless/ath/ath12k/hw.c index dafd7c34d7465..97ed179be2283 100644 --- a/drivers/net/wireless/ath/ath12k/hw.c +++ b/drivers/net/wireless/ath/ath12k/hw.c @@ -1002,7 +1002,7 @@ static const struct ath12k_hw_params ath12k_hw_params[] = { .download_calib = true, .supports_suspend = false, .tcl_ring_retry = true, - .reoq_lut_support = false, + .reoq_lut_support = true, .supports_shadow_regs = false, .hal_desc_sz = sizeof(struct hal_rx_desc_qcn9274), diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index e1db6e69d2207..010413bfdb141 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -4743,23 +4743,32 @@ static void ath12k_mgmt_over_wmi_tx_drop(struct ath12k *ar, struct sk_buff *skb) wake_up(&ar->txmgmt_empty_waitq); } -int ath12k_mac_tx_mgmt_pending_free(int buf_id, void *skb, void *ctx) +static void ath12k_mac_tx_mgmt_free(struct ath12k *ar, int buf_id) { - struct sk_buff *msdu = skb; + struct sk_buff *msdu; struct ieee80211_tx_info *info; - struct ath12k *ar = ctx; - struct ath12k_base *ab = ar->ab; spin_lock_bh(&ar->txmgmt_idr_lock); - idr_remove(&ar->txmgmt_idr, buf_id); + msdu = idr_remove(&ar->txmgmt_idr, buf_id); spin_unlock_bh(&ar->txmgmt_idr_lock); - dma_unmap_single(ab->dev, ATH12K_SKB_CB(msdu)->paddr, msdu->len, + + if (!msdu) + return; + + dma_unmap_single(ar->ab->dev, ATH12K_SKB_CB(msdu)->paddr, msdu->len, DMA_TO_DEVICE); info = IEEE80211_SKB_CB(msdu); memset(&info->status, 0, sizeof(info->status)); - ath12k_mgmt_over_wmi_tx_drop(ar, skb); + ath12k_mgmt_over_wmi_tx_drop(ar, msdu); +} + +int ath12k_mac_tx_mgmt_pending_free(int buf_id, void *skb, void *ctx) +{ + struct ath12k *ar = ctx; + + ath12k_mac_tx_mgmt_free(ar, buf_id); return 0; } @@ -4768,17 +4777,10 @@ static int ath12k_mac_vif_txmgmt_idr_remove(int buf_id, void *skb, void *ctx) { struct ieee80211_vif *vif = ctx; struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb); - struct sk_buff *msdu = skb; struct ath12k *ar = skb_cb->ar; - struct ath12k_base *ab = ar->ab; - if (skb_cb->vif == vif) { - spin_lock_bh(&ar->txmgmt_idr_lock); - idr_remove(&ar->txmgmt_idr, buf_id); - spin_unlock_bh(&ar->txmgmt_idr_lock); - dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, - DMA_TO_DEVICE); - } + if (skb_cb->vif == vif) + ath12k_mac_tx_mgmt_free(ar, buf_id); return 0; } diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c index 041a9602f0e15..7dfbabf0637d2 100644 --- a/drivers/net/wireless/ath/ath12k/pci.c +++ b/drivers/net/wireless/ath/ath12k/pci.c @@ -442,8 +442,11 @@ static void __ath12k_pci_ext_irq_disable(struct ath12k_base *ab) ath12k_pci_ext_grp_disable(irq_grp); - napi_synchronize(&irq_grp->napi); - napi_disable(&irq_grp->napi); + if (irq_grp->napi_enabled) { + napi_synchronize(&irq_grp->napi); + napi_disable(&irq_grp->napi); + irq_grp->napi_enabled = false; + } } } @@ -976,7 +979,11 @@ void ath12k_pci_ext_irq_enable(struct ath12k_base *ab) for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) { struct ath12k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i]; - napi_enable(&irq_grp->napi); + if (!irq_grp->napi_enabled) { + napi_enable(&irq_grp->napi); + irq_grp->napi_enabled = true; + } + ath12k_pci_ext_grp_enable(irq_grp); } } @@ -1146,6 +1153,9 @@ void ath12k_pci_power_down(struct ath12k_base *ab) { struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); + if (!test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags)) + return; + /* restore aspm in case firmware bootup fails */ ath12k_pci_aspm_restore(ab_pci); diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c index c977dfbae0a46..7e400a0e0eb11 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.c +++ b/drivers/net/wireless/ath/ath12k/wmi.c @@ -951,14 +951,24 @@ int ath12k_wmi_vdev_down(struct ath12k *ar, u8 vdev_id) static void ath12k_wmi_put_wmi_channel(struct ath12k_wmi_channel_params *chan, struct wmi_vdev_start_req_arg *arg) { + u32 center_freq1 = arg->band_center_freq1; + memset(chan, 0, sizeof(*chan)); chan->mhz = cpu_to_le32(arg->freq); - chan->band_center_freq1 = cpu_to_le32(arg->band_center_freq1); - if (arg->mode == MODE_11AC_VHT80_80) + chan->band_center_freq1 = cpu_to_le32(center_freq1); + if (arg->mode == MODE_11BE_EHT160) { + if (arg->freq > center_freq1) + chan->band_center_freq1 = cpu_to_le32(center_freq1 + 40); + else + chan->band_center_freq1 = cpu_to_le32(center_freq1 - 40); + + chan->band_center_freq2 = cpu_to_le32(center_freq1); + } else if (arg->mode == MODE_11BE_EHT80_80) { chan->band_center_freq2 = cpu_to_le32(arg->band_center_freq2); - else + } else { chan->band_center_freq2 = 0; + } chan->info |= le32_encode_bits(arg->mode, WMI_CHAN_INFO_MODE); if (arg->passive) @@ -2066,7 +2076,7 @@ int ath12k_wmi_send_peer_assoc_cmd(struct ath12k *ar, for (i = 0; i < arg->peer_eht_mcs_count; i++) { eht_mcs = ptr; - eht_mcs->tlv_header = ath12k_wmi_tlv_cmd_hdr(WMI_TAG_HE_RATE_SET, + eht_mcs->tlv_header = ath12k_wmi_tlv_cmd_hdr(WMI_TAG_EHT_RATE_SET, sizeof(*eht_mcs)); eht_mcs->rx_mcs_set = cpu_to_le32(arg->peer_eht_rx_mcs_set[i]); @@ -2115,8 +2125,8 @@ void ath12k_wmi_start_scan_init(struct ath12k *ar, arg->dwell_time_active = 50; arg->dwell_time_active_2g = 0; arg->dwell_time_passive = 150; - arg->dwell_time_active_6g = 40; - arg->dwell_time_passive_6g = 30; + arg->dwell_time_active_6g = 70; + arg->dwell_time_passive_6g = 70; arg->min_rest_time = 50; arg->max_rest_time = 500; arg->repeat_probe_time = 0; @@ -4128,6 +4138,7 @@ static int ath12k_service_ready_ext_event(struct ath12k_base *ab, return 0; err: + kfree(svc_rdy_ext.mac_phy_caps); ath12k_wmi_free_dbring_caps(ab); return ret; } @@ -4954,6 +4965,11 @@ static int wmi_process_mgmt_tx_comp(struct ath12k *ar, u32 desc_id, dma_unmap_single(ar->ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); info = IEEE80211_SKB_CB(msdu); + memset(&info->status, 0, sizeof(info->status)); + + /* skip tx rate update from ieee80211_status*/ + info->status.rates[0].idx = -1; + if ((!(info->flags & IEEE80211_TX_CTL_NO_ACK)) && !status) info->flags |= IEEE80211_TX_STAT_ACK; @@ -5502,7 +5518,7 @@ static int ath12k_reg_chan_list_event(struct ath12k_base *ab, struct sk_buff *sk goto fallback; } - spin_lock(&ab->base_lock); + spin_lock_bh(&ab->base_lock); if (test_bit(ATH12K_FLAG_REGISTERED, &ab->dev_flags)) { /* Once mac is registered, ar is valid and all CC events from * fw is considered to be received due to user requests @@ -5526,7 +5542,7 @@ static int ath12k_reg_chan_list_event(struct ath12k_base *ab, struct sk_buff *sk ab->default_regd[pdev_idx] = regd; } ab->dfs_region = reg_info->dfs_region; - spin_unlock(&ab->base_lock); + spin_unlock_bh(&ab->base_lock); goto mem_free; @@ -6292,7 +6308,7 @@ static int ath12k_wmi_tlv_services_parser(struct ath12k_base *ab, void *data) { const struct wmi_service_available_event *ev; - u32 *wmi_ext2_service_bitmap; + __le32 *wmi_ext2_service_bitmap; int i, j; u16 expected_len; @@ -6324,12 +6340,12 @@ static int ath12k_wmi_tlv_services_parser(struct ath12k_base *ab, ev->wmi_service_segment_bitmap[3]); break; case WMI_TAG_ARRAY_UINT32: - wmi_ext2_service_bitmap = (u32 *)ptr; + wmi_ext2_service_bitmap = (__le32 *)ptr; for (i = 0, j = WMI_MAX_EXT_SERVICE; i < WMI_SERVICE_SEGMENT_BM_SIZE32 && j < WMI_MAX_EXT2_SERVICE; i++) { do { - if (wmi_ext2_service_bitmap[i] & + if (__le32_to_cpu(wmi_ext2_service_bitmap[i]) & BIT(j % WMI_AVAIL_SERVICE_BITS_IN_SIZE32)) set_bit(j, ab->wmi_ab.svc_map); } while (++j % WMI_AVAIL_SERVICE_BITS_IN_SIZE32); @@ -6337,8 +6353,10 @@ static int ath12k_wmi_tlv_services_parser(struct ath12k_base *ab, ath12k_dbg(ab, ATH12K_DBG_WMI, "wmi_ext2_service_bitmap 0x%04x 0x%04x 0x%04x 0x%04x", - wmi_ext2_service_bitmap[0], wmi_ext2_service_bitmap[1], - wmi_ext2_service_bitmap[2], wmi_ext2_service_bitmap[3]); + __le32_to_cpu(wmi_ext2_service_bitmap[0]), + __le32_to_cpu(wmi_ext2_service_bitmap[1]), + __le32_to_cpu(wmi_ext2_service_bitmap[2]), + __le32_to_cpu(wmi_ext2_service_bitmap[3])); break; } return 0; diff --git a/drivers/net/wireless/ath/ath6kl/bmi.c b/drivers/net/wireless/ath/ath6kl/bmi.c index af98e871199d3..5a9e93fd1ef42 100644 --- a/drivers/net/wireless/ath/ath6kl/bmi.c +++ b/drivers/net/wireless/ath/ath6kl/bmi.c @@ -87,7 +87,9 @@ int ath6kl_bmi_get_target_info(struct ath6kl *ar, * We need to do some backwards compatibility to make this work. */ if (le32_to_cpu(targ_info->byte_count) != sizeof(*targ_info)) { - WARN_ON(1); + ath6kl_err("mismatched byte count %d vs. expected %zd\n", + le32_to_cpu(targ_info->byte_count), + sizeof(*targ_info)); return -EINVAL; } diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_beacon.c b/drivers/net/wireless/ath/ath9k/htc_drv_beacon.c index 533471e694007..18c7654bc539d 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_beacon.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_beacon.c @@ -290,6 +290,9 @@ void ath9k_htc_swba(struct ath9k_htc_priv *priv, struct ath_common *common = ath9k_hw_common(priv->ah); int slot; + if (!priv->cur_beacon_conf.enable_beacon) + return; + if (swba->beacon_pending != 0) { priv->beacon.bmisscnt++; if (priv->beacon.bmisscnt > BSTUCK_THRESHOLD) { diff --git a/drivers/net/wireless/ath/ath9k/init.c b/drivers/net/wireless/ath/ath9k/init.c index 4f00400c7ffb8..58386906598a7 100644 --- a/drivers/net/wireless/ath/ath9k/init.c +++ b/drivers/net/wireless/ath/ath9k/init.c @@ -691,7 +691,9 @@ static int ath9k_of_init(struct ath_softc *sc) ah->ah_flags |= AH_NO_EEP_SWAP; } - of_get_mac_address(np, common->macaddr); + ret = of_get_mac_address(np, common->macaddr); + if (ret == -EPROBE_DEFER) + return ret; return 0; } diff --git a/drivers/net/wireless/ath/carl9170/usb.c b/drivers/net/wireless/ath/carl9170/usb.c index a5265997b5767..debac4699687e 100644 --- a/drivers/net/wireless/ath/carl9170/usb.c +++ b/drivers/net/wireless/ath/carl9170/usb.c @@ -438,14 +438,21 @@ static void carl9170_usb_rx_complete(struct urb *urb) if (atomic_read(&ar->rx_anch_urbs) == 0) { /* - * The system is too slow to cope with - * the enormous workload. We have simply - * run out of active rx urbs and this - * unfortunately leads to an unpredictable - * device. + * At this point, either the system is too slow to + * cope with the enormous workload (so we have simply + * run out of active rx urbs and this unfortunately + * leads to an unpredictable device), or the device + * is not fully functional after an unsuccessful + * firmware loading attempts (so it doesn't pass + * ieee80211_register_hw() and there is no internal + * workqueue at all). */ - ieee80211_queue_work(ar->hw, &ar->ping_work); + if (ar->registered) + ieee80211_queue_work(ar->hw, &ar->ping_work); + else + pr_warn_once("device %s is not registered\n", + dev_name(&ar->udev->dev)); } } else { /* diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c index 00794086cc7c9..bf80675667ba3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c @@ -392,10 +392,8 @@ void brcmf_btcoex_detach(struct brcmf_cfg80211_info *cfg) if (!cfg->btcoex) return; - if (cfg->btcoex->timer_on) { - cfg->btcoex->timer_on = false; - timer_shutdown_sync(&cfg->btcoex->timer); - } + timer_shutdown_sync(&cfg->btcoex->timer); + cfg->btcoex->timer_on = false; cancel_work_sync(&cfg->btcoex->work); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 80833abc47dd1..c371d8014297c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1541,10 +1541,6 @@ brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) return -EAGAIN; } - /* If scan req comes for p2p0, send it over primary I/F */ - if (vif == cfg->p2p.bss_idx[P2PAPI_BSSCFG_DEVICE].vif) - vif = cfg->p2p.bss_idx[P2PAPI_BSSCFG_PRIMARY].vif; - brcmf_dbg(SCAN, "START ESCAN\n"); cfg->scan_request = request; @@ -1560,6 +1556,10 @@ brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (err) goto scan_out; + /* If scan req comes for p2p0, send it over primary I/F */ + if (vif == cfg->p2p.bss_idx[P2PAPI_BSSCFG_DEVICE].vif) + vif = cfg->p2p.bss_idx[P2PAPI_BSSCFG_PRIMARY].vif; + err = brcmf_do_escan(vif->ifp, request); if (err) goto scan_out; @@ -5593,8 +5593,7 @@ brcmf_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev, brcmf_dbg(TRACE, "Action frame, cookie=%lld, len=%d, freq=%d\n", *cookie, le16_to_cpu(action_frame->len), freq); - ack = brcmf_p2p_send_action_frame(cfg, cfg_to_ndev(cfg), - af_params); + ack = brcmf_p2p_send_action_frame(vif->ifp, af_params); cfg80211_mgmt_tx_status(wdev, *cookie, buf, len, ack, GFP_KERNEL); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index d4492d02e4ea1..33f27c83c33eb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1529,6 +1529,7 @@ int brcmf_p2p_notify_action_tx_complete(struct brcmf_if *ifp, /** * brcmf_p2p_tx_action_frame() - send action frame over fil. * + * @ifp: interface to transmit on. * @p2p: p2p info struct for vif. * @af_params: action frame data/info. * @@ -1538,12 +1539,11 @@ int brcmf_p2p_notify_action_tx_complete(struct brcmf_if *ifp, * The WLC_E_ACTION_FRAME_COMPLETE event will be received when the action * frame is transmitted. */ -static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, +static s32 brcmf_p2p_tx_action_frame(struct brcmf_if *ifp, + struct brcmf_p2p_info *p2p, struct brcmf_fil_af_params_le *af_params) { struct brcmf_pub *drvr = p2p->cfg->pub; - struct brcmf_cfg80211_vif *vif; - struct brcmf_p2p_action_frame *p2p_af; s32 err = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -1552,14 +1552,7 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, clear_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status); clear_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status); - /* check if it is a p2p_presence response */ - p2p_af = (struct brcmf_p2p_action_frame *)af_params->action_frame.data; - if (p2p_af->subtype == P2P_AF_PRESENCE_RSP) - vif = p2p->bss_idx[P2PAPI_BSSCFG_CONNECTION].vif; - else - vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; - - err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe", af_params, + err = brcmf_fil_bsscfg_data_set(ifp, "actframe", af_params, sizeof(*af_params)); if (err) { bphy_err(drvr, " sending action frame has failed\n"); @@ -1711,16 +1704,14 @@ static bool brcmf_p2p_check_dwell_overflow(u32 requested_dwell, /** * brcmf_p2p_send_action_frame() - send action frame . * - * @cfg: driver private data for cfg80211 interface. - * @ndev: net device to transmit on. + * @ifp: interface to transmit on. * @af_params: configuration data for action frame. */ -bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, - struct net_device *ndev, +bool brcmf_p2p_send_action_frame(struct brcmf_if *ifp, struct brcmf_fil_af_params_le *af_params) { + struct brcmf_cfg80211_info *cfg = ifp->drvr->config; struct brcmf_p2p_info *p2p = &cfg->p2p; - struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_fil_action_frame_le *action_frame; struct brcmf_config_af_params config_af_params; struct afx_hdl *afx_hdl = &p2p->afx_hdl; @@ -1857,7 +1848,7 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, if (af_params->channel) msleep(P2P_AF_RETRY_DELAY_TIME); - ack = !brcmf_p2p_tx_action_frame(p2p, af_params); + ack = !brcmf_p2p_tx_action_frame(ifp, p2p, af_params); tx_retry++; dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, dwell_jiffies); @@ -2217,7 +2208,6 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p, WARN_ON(p2p_ifp->bsscfgidx != bsscfgidx); - init_completion(&p2p->send_af_done); INIT_WORK(&p2p->afx_hdl.afx_work, brcmf_p2p_afx_handler); init_completion(&p2p->afx_hdl.act_frm_scan); init_completion(&p2p->wait_next_af); @@ -2513,6 +2503,8 @@ s32 brcmf_p2p_attach(struct brcmf_cfg80211_info *cfg, bool p2pdev_forced) pri_ifp = brcmf_get_ifp(cfg->pub, 0); p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif = pri_ifp->vif; + init_completion(&p2p->send_af_done); + if (p2pdev_forced) { err_ptr = brcmf_p2p_create_p2pdev(p2p, NULL, NULL); if (IS_ERR(err_ptr)) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h index d2ecee565bf2e..d3137ebd71582 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h @@ -168,8 +168,7 @@ int brcmf_p2p_notify_action_frame_rx(struct brcmf_if *ifp, int brcmf_p2p_notify_action_tx_complete(struct brcmf_if *ifp, const struct brcmf_event_msg *e, void *data); -bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, - struct net_device *ndev, +bool brcmf_p2p_send_action_frame(struct brcmf_if *ifp, struct brcmf_fil_af_params_le *af_params); bool brcmf_p2p_scan_finding_common_channel(struct brcmf_cfg80211_info *cfg, struct brcmf_bss_info_le *bi); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 2178675ae1a44..6f64a05debd2c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -903,14 +903,16 @@ brcmf_usb_dl_writeimage(struct brcmf_usbdev_info *devinfo, u8 *fw, int fwlen) } /* 1) Prepare USB boot loader for runtime image */ - brcmf_usb_dl_cmd(devinfo, DL_START, &state, sizeof(state)); + err = brcmf_usb_dl_cmd(devinfo, DL_START, &state, sizeof(state)); + if (err) + goto fail; rdlstate = le32_to_cpu(state.state); rdlbytes = le32_to_cpu(state.bytes); /* 2) Check we are in the Waiting state */ if (rdlstate != DL_WAITING) { - brcmf_err("Failed to DL_START\n"); + brcmf_err("Invalid DL state: %u\n", rdlstate); err = -EINVAL; goto fail; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c index 47c0e8e429e54..3064e603e7e3e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c @@ -919,7 +919,7 @@ void wlc_lcnphy_read_table(struct brcms_phy *pi, struct phytbl_info *pti) static void wlc_lcnphy_common_read_table(struct brcms_phy *pi, u32 tbl_id, - const u16 *tbl_ptr, u32 tbl_len, + u16 *tbl_ptr, u32 tbl_len, u32 tbl_width, u32 tbl_offset) { struct phytbl_info tab; diff --git a/drivers/net/wireless/intel/iwlegacy/4965-mac.c b/drivers/net/wireless/intel/iwlegacy/4965-mac.c index 0a4aa3c678c10..75118e2406191 100644 --- a/drivers/net/wireless/intel/iwlegacy/4965-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/4965-mac.c @@ -1575,8 +1575,11 @@ il4965_tx_cmd_build_rate(struct il_priv *il, || rate_idx > RATE_COUNT_LEGACY) rate_idx = rate_lowest_index(&il->bands[info->band], sta); /* For 5 GHZ band, remap mac80211 rate indices into driver indices */ - if (info->band == NL80211_BAND_5GHZ) + if (info->band == NL80211_BAND_5GHZ) { rate_idx += IL_FIRST_OFDM_RATE; + if (rate_idx > IL_LAST_OFDM_RATE) + rate_idx = IL_LAST_OFDM_RATE; + } /* Get PLCP rate for tx_cmd->rate_n_flags */ rate_plcp = il_rates[rate_idx].plcp; /* Zero out flags for this packet */ diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c index d594694206b33..906f2790f5619 100644 --- a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c +++ b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c @@ -44,6 +44,8 @@ IWL_QU_C_HR_B_FW_PRE "-" __stringify(api) ".ucode" #define IWL_QU_B_JF_B_MODULE_FIRMWARE(api) \ IWL_QU_B_JF_B_FW_PRE "-" __stringify(api) ".ucode" +#define IWL_QU_C_JF_B_MODULE_FIRMWARE(api) \ + IWL_QU_C_JF_B_FW_PRE "-" __stringify(api) ".ucode" #define IWL_CC_A_MODULE_FIRMWARE(api) \ IWL_CC_A_FW_PRE "-" __stringify(api) ".ucode" @@ -423,6 +425,7 @@ const struct iwl_cfg iwl_cfg_quz_a0_hr_b0 = { MODULE_FIRMWARE(IWL_QU_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_QU_C_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_QU_B_JF_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); +MODULE_FIRMWARE(IWL_QU_C_JF_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_QUZ_A_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_QUZ_A_JF_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_CC_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/main.c b/drivers/net/wireless/intel/iwlwifi/dvm/main.c index a873be109f439..b490a88b97ca7 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/main.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/main.c @@ -1048,9 +1048,11 @@ static void iwl_bg_restart(struct work_struct *data) * *****************************************************************************/ -static void iwl_setup_deferred_work(struct iwl_priv *priv) +static int iwl_setup_deferred_work(struct iwl_priv *priv) { priv->workqueue = alloc_ordered_workqueue(DRV_NAME, 0); + if (!priv->workqueue) + return -ENOMEM; INIT_WORK(&priv->restart, iwl_bg_restart); INIT_WORK(&priv->beacon_update, iwl_bg_beacon_update); @@ -1067,6 +1069,8 @@ static void iwl_setup_deferred_work(struct iwl_priv *priv) timer_setup(&priv->statistics_periodic, iwl_bg_statistics_periodic, 0); timer_setup(&priv->ucode_trace, iwl_bg_ucode_trace, 0); + + return 0; } void iwl_cancel_deferred_work(struct iwl_priv *priv) @@ -1456,7 +1460,9 @@ static struct iwl_op_mode *iwl_op_mode_dvm_start(struct iwl_trans *trans, /******************** * 6. Setup services ********************/ - iwl_setup_deferred_work(priv); + if (iwl_setup_deferred_work(priv)) + goto out_uninit_drv; + iwl_setup_rx_handlers(priv); iwl_power_initialize(priv); @@ -1494,6 +1500,7 @@ static struct iwl_op_mode *iwl_op_mode_dvm_start(struct iwl_trans *trans, iwl_cancel_deferred_work(priv); destroy_workqueue(priv->workqueue); priv->workqueue = NULL; +out_uninit_drv: iwl_uninit_drv(priv); out_free_eeprom_blob: kfree(priv->eeprom_blob); diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/rs.c b/drivers/net/wireless/intel/iwlwifi/dvm/rs.c index f4a6f76cf193a..e70024525eb90 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/rs.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/rs.c @@ -2904,7 +2904,7 @@ static void rs_fill_link_cmd(struct iwl_priv *priv, /* Repeat initial/next rate. * For legacy IWL_NUMBER_TRY == 1, this loop will not execute. * For HT IWL_HT_NUMBER_TRY == 3, this executes twice. */ - while (repeat_rate > 0 && (index < LINK_QUAL_MAX_RETRY_NUM)) { + while (repeat_rate > 0 && index < (LINK_QUAL_MAX_RETRY_NUM - 1)) { if (is_legacy(tbl_type.lq_type)) { if (ant_toggle_cnt < NUM_TRY_BEFORE_ANT_TOGGLE) ant_toggle_cnt++; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c index 0a1f302ad6d3f..2deb259615d9c 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c @@ -2816,6 +2816,7 @@ int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt, struct iwl_fw_dump_desc *desc; unsigned int delay = 0; bool monitor_only = false; + int ret; if (trigger) { u16 occurrences = le16_to_cpu(trigger->occurrences) - 1; @@ -2846,7 +2847,11 @@ int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt, desc->trig_desc.type = cpu_to_le32(trig); memcpy(desc->trig_desc.data, str, len); - return iwl_fw_dbg_collect_desc(fwrt, desc, monitor_only, delay); + ret = iwl_fw_dbg_collect_desc(fwrt, desc, monitor_only, delay); + if (ret) + kfree(desc); + + return ret; } IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c index a97ed7cbe4d14..d588e4cd808d8 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation */ #include #include "iwl-drv.h" @@ -1382,15 +1382,15 @@ void _iwl_dbg_tlv_time_point(struct iwl_fw_runtime *fwrt, switch (tp_id) { case IWL_FW_INI_TIME_POINT_EARLY: iwl_dbg_tlv_init_cfg(fwrt); - iwl_dbg_tlv_apply_config(fwrt, conf_list); iwl_dbg_tlv_update_drams(fwrt); iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data, NULL); + iwl_dbg_tlv_apply_config(fwrt, conf_list); break; case IWL_FW_INI_TIME_POINT_AFTER_ALIVE: iwl_dbg_tlv_apply_buffers(fwrt); iwl_dbg_tlv_send_hcmds(fwrt, hcmd_list); - iwl_dbg_tlv_apply_config(fwrt, conf_list); iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data, NULL); + iwl_dbg_tlv_apply_config(fwrt, conf_list); break; case IWL_FW_INI_TIME_POINT_PERIODIC: iwl_dbg_tlv_set_periodic_trigs(fwrt); @@ -1400,14 +1400,14 @@ void _iwl_dbg_tlv_time_point(struct iwl_fw_runtime *fwrt, case IWL_FW_INI_TIME_POINT_MISSED_BEACONS: case IWL_FW_INI_TIME_POINT_FW_DHC_NOTIFICATION: iwl_dbg_tlv_send_hcmds(fwrt, hcmd_list); - iwl_dbg_tlv_apply_config(fwrt, conf_list); iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data, iwl_dbg_tlv_check_fw_pkt); + iwl_dbg_tlv_apply_config(fwrt, conf_list); break; default: iwl_dbg_tlv_send_hcmds(fwrt, hcmd_list); - iwl_dbg_tlv_apply_config(fwrt, conf_list); iwl_dbg_tlv_tp_trigger(fwrt, sync, trig_list, tp_data, NULL); + iwl_dbg_tlv_apply_config(fwrt, conf_list); break; } } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c index a82cdd897173f..6c108dbbbc540 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c @@ -2143,6 +2143,7 @@ static void iwl_mvm_convert_gtk_v2(struct iwl_wowlan_status_data *status, status->gtk[0].len = data->key_len; status->gtk[0].flags = data->key_flags; + status->gtk[0].id = status->gtk[0].flags & IWL_WOWLAN_GTK_IDX_MASK; memcpy(status->gtk[0].key, data->key, sizeof(data->key)); @@ -2369,6 +2370,7 @@ iwl_mvm_send_wowlan_get_status(struct iwl_mvm *mvm, u8 sta_id) * currently used key. */ status->gtk[0].flags = v6->gtk.key_index | BIT(7); + status->gtk[0].id = v6->gtk.key_index; } else if (notif_ver == 7) { struct iwl_wowlan_status_v7 *v7 = (void *)cmd.resp_pkt->data; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index b2cf5aeff7e3c..d2dbbc9fe3844 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -65,8 +65,10 @@ static int __init iwl_mvm_init(void) } ret = iwl_opmode_register("iwlmvm", &iwl_mvm_ops); - if (ret) + if (ret) { pr_err("Unable to register MVM op_mode: %d\n", ret); + iwl_mvm_rate_control_unregister(); + } return ret; } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c index cc866401aad0b..8b22779e5b3e5 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c @@ -828,7 +828,7 @@ static inline bool iwl_mvm_scan_fits(struct iwl_mvm *mvm, int n_ssids, int n_channels) { return ((n_ssids <= PROBE_OPTION_MAX) && - (n_channels <= mvm->fw->ucode_capa.n_scan_channels) & + (n_channels <= mvm->fw->ucode_capa.n_scan_channels) && (ies->common_ie_len + ies->len[NL80211_BAND_2GHZ] + ies->len[NL80211_BAND_5GHZ] + ies->len[NL80211_BAND_6GHZ] <= diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index 4a2de79f2e864..c01a9a6f06a4d 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -580,6 +580,8 @@ static const struct iwl_dev_info iwl_dev_info_table[] = { IWL_DEV_INFO(0x7A70, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name), IWL_DEV_INFO(0x7AF0, 0x1691, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690s_name), IWL_DEV_INFO(0x7AF0, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name), + IWL_DEV_INFO(0x7F70, 0x1691, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690s_name), + IWL_DEV_INFO(0x7F70, 0x1692, iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_killer_1690i_name), IWL_DEV_INFO(0x271C, 0x0214, iwl9260_2ac_cfg, iwl9260_1_name), IWL_DEV_INFO(0x7E40, 0x1691, iwl_cfg_ma, iwl_ax411_killer_1690s_name), diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c index e9807fcca6ad1..5c2e8d2883976 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c @@ -2701,6 +2701,8 @@ static ssize_t iwl_dbgfs_rx_queue_read(struct file *file, for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) { struct iwl_rxq *rxq = &trans_pcie->rxq[i]; + spin_lock_bh(&rxq->lock); + pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n", i); pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n", @@ -2721,6 +2723,7 @@ static ssize_t iwl_dbgfs_rx_queue_read(struct file *file, pos += scnprintf(buf + pos, bufsz - pos, "\tclosed_rb_num: Not Allocated\n"); } + spin_unlock_bh(&rxq->lock); } ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos); kfree(buf); @@ -3385,8 +3388,11 @@ iwl_trans_pcie_dump_data(struct iwl_trans *trans, /* Dump RBs is supported only for pre-9000 devices (1 queue) */ struct iwl_rxq *rxq = &trans_pcie->rxq[0]; /* RBs */ + spin_lock_bh(&rxq->lock); num_rbs = iwl_get_closed_rb_stts(trans, rxq); num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK; + spin_unlock_bh(&rxq->lock); + len += num_rbs * (sizeof(*data) + sizeof(struct iwl_fw_error_dump_rb) + (PAGE_SIZE << trans_pcie->rx_page_order)); diff --git a/drivers/net/wireless/intersil/p54/fwio.c b/drivers/net/wireless/intersil/p54/fwio.c index b52cce38115d0..06e5df90b43ec 100644 --- a/drivers/net/wireless/intersil/p54/fwio.c +++ b/drivers/net/wireless/intersil/p54/fwio.c @@ -231,6 +231,7 @@ int p54_download_eeprom(struct p54_common *priv, void *buf, mutex_lock(&priv->eeprom_mutex); priv->eeprom = buf; + priv->eeprom_slice_size = len; eeprom_hdr = skb_put(skb, eeprom_hdr_size + len); if (priv->fw_var < 0x509) { @@ -253,6 +254,7 @@ int p54_download_eeprom(struct p54_common *priv, void *buf, ret = -EBUSY; } priv->eeprom = NULL; + priv->eeprom_slice_size = 0; mutex_unlock(&priv->eeprom_mutex); return ret; } diff --git a/drivers/net/wireless/intersil/p54/p54.h b/drivers/net/wireless/intersil/p54/p54.h index 3356ea708d816..97fc863fef810 100644 --- a/drivers/net/wireless/intersil/p54/p54.h +++ b/drivers/net/wireless/intersil/p54/p54.h @@ -258,6 +258,7 @@ struct p54_common { /* eeprom handling */ void *eeprom; + size_t eeprom_slice_size; struct completion eeprom_comp; struct mutex eeprom_mutex; }; diff --git a/drivers/net/wireless/intersil/p54/txrx.c b/drivers/net/wireless/intersil/p54/txrx.c index 8414aa208655f..2deb1bb54f24b 100644 --- a/drivers/net/wireless/intersil/p54/txrx.c +++ b/drivers/net/wireless/intersil/p54/txrx.c @@ -496,14 +496,19 @@ static void p54_rx_eeprom_readback(struct p54_common *priv, return ; if (priv->fw_var >= 0x509) { - memcpy(priv->eeprom, eeprom->v2.data, - le16_to_cpu(eeprom->v2.len)); + if (le16_to_cpu(eeprom->v2.len) != priv->eeprom_slice_size) + return; + + memcpy(priv->eeprom, eeprom->v2.data, priv->eeprom_slice_size); } else { - memcpy(priv->eeprom, eeprom->v1.data, - le16_to_cpu(eeprom->v1.len)); + if (le16_to_cpu(eeprom->v1.len) != priv->eeprom_slice_size) + return; + + memcpy(priv->eeprom, eeprom->v1.data, priv->eeprom_slice_size); } priv->eeprom = NULL; + priv->eeprom_slice_size = 0; tmp = p54_find_and_unlink_skb(priv, hdr->req_id); dev_kfree_skb_any(tmp); complete(&priv->eeprom_comp); diff --git a/drivers/net/wireless/marvell/libertas/cfg.c b/drivers/net/wireless/marvell/libertas/cfg.c index b700c213d10c4..38ad49033d0ba 100644 --- a/drivers/net/wireless/marvell/libertas/cfg.c +++ b/drivers/net/wireless/marvell/libertas/cfg.c @@ -1150,10 +1150,13 @@ static int lbs_associate(struct lbs_private *priv, /* add SSID TLV */ rcu_read_lock(); ssid_eid = ieee80211_bss_get_ie(bss, WLAN_EID_SSID); - if (ssid_eid) - pos += lbs_add_ssid_tlv(pos, ssid_eid + 2, ssid_eid[1]); - else + if (ssid_eid) { + u32 ssid_len = min(ssid_eid[1], IEEE80211_MAX_SSID_LEN); + + pos += lbs_add_ssid_tlv(pos, ssid_eid + 2, ssid_len); + } else { lbs_deb_assoc("no SSID\n"); + } rcu_read_unlock(); /* add DS param TLV */ diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c index b7ead0cd00450..df6e4a112ec48 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -659,10 +659,9 @@ static void mwifiex_reg_notifier(struct wiphy *wiphy, return; } - /* Don't send world or same regdom info to firmware */ - if (strncmp(request->alpha2, "00", 2) && - strncmp(request->alpha2, adapter->country_code, - sizeof(request->alpha2))) { + /* Don't send same regdom info to firmware */ + if (strncmp(request->alpha2, adapter->country_code, + sizeof(request->alpha2)) != 0) { memcpy(adapter->country_code, request->alpha2, sizeof(request->alpha2)); mwifiex_send_domain_info_cmd_fw(wiphy); @@ -4316,8 +4315,9 @@ int mwifiex_init_channel_scan_gap(struct mwifiex_adapter *adapter) * additional active scan request for hidden SSIDs on passive channels. */ adapter->num_in_chan_stats = 2 * (n_channels_bg + n_channels_a); - adapter->chan_stats = vmalloc(array_size(sizeof(*adapter->chan_stats), - adapter->num_in_chan_stats)); + adapter->chan_stats = kcalloc(adapter->num_in_chan_stats, + sizeof(*adapter->chan_stats), + GFP_KERNEL); if (!adapter->chan_stats) return -ENOMEM; diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c index 6c60a4c21a312..685dcab11a488 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.c +++ b/drivers/net/wireless/marvell/mwifiex/main.c @@ -664,7 +664,7 @@ static int _mwifiex_fw_dpc(const struct firmware *firmware, void *context) goto done; err_add_intf: - vfree(adapter->chan_stats); + kfree(adapter->chan_stats); err_init_chan_scan: wiphy_unregister(adapter->wiphy); wiphy_free(adapter->wiphy); @@ -1481,7 +1481,7 @@ static void mwifiex_uninit_sw(struct mwifiex_adapter *adapter) wiphy_free(adapter->wiphy); adapter->wiphy = NULL; - vfree(adapter->chan_stats); + kfree(adapter->chan_stats); mwifiex_free_cmd_buffers(adapter); } diff --git a/drivers/net/wireless/marvell/mwl8k.c b/drivers/net/wireless/marvell/mwl8k.c index c0ecd769ada76..1a48914e6f828 100644 --- a/drivers/net/wireless/marvell/mwl8k.c +++ b/drivers/net/wireless/marvell/mwl8k.c @@ -1222,6 +1222,10 @@ static int rxq_refill(struct ieee80211_hw *hw, int index, int limit) addr = dma_map_single(&priv->pdev->dev, skb->data, MWL8K_RX_MAXSZ, DMA_FROM_DEVICE); + if (dma_mapping_error(&priv->pdev->dev, addr)) { + kfree_skb(skb); + break; + } rxq->rxd_count++; rx = rxq->tail++; diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index cd04865970643..b7e100710601a 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -957,6 +957,7 @@ void mt76_dma_cleanup(struct mt76_dev *dev) int i; mt76_worker_disable(&dev->tx_worker); + napi_disable(&dev->tx_napi); netif_napi_del(&dev->tx_napi); for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 8b620d4fed439..df0ea638370b5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -439,6 +439,7 @@ struct mt76_hw_cap { #define MT_DRV_RX_DMA_HDR BIT(3) #define MT_DRV_HW_MGMT_TXQ BIT(4) #define MT_DRV_AMSDU_OFFLOAD BIT(5) +#define MT_DRV_IGNORE_TXS_FAILED BIT(6) struct mt76_driver_ops { u32 drv_flags; diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/soc.c b/drivers/net/wireless/mediatek/mt76/mt7603/soc.c index ba927033bbe8c..1206769cdc7fd 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/soc.c @@ -48,7 +48,7 @@ mt76_wmac_probe(struct platform_device *pdev) return 0; error: - ieee80211_free_hw(mt76_hw(dev)); + mt76_free_device(mdev); return ret; } diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h index 87bfa441a9374..4979012d5abfa 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h @@ -280,6 +280,9 @@ enum tx_mgnt_type { #define MT_TXFREE_INFO_COUNT GENMASK(27, 24) #define MT_TXFREE_INFO_STAT GENMASK(29, 28) +#define MT_TXS_HDR_SIZE 4 /* Unit: DW */ +#define MT_TXS_SIZE 12 /* Unit: DW */ + #define MT_TXS0_BW GENMASK(31, 29) #define MT_TXS0_TID GENMASK(28, 26) #define MT_TXS0_AMPDU BIT(25) diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/pci.c b/drivers/net/wireless/mediatek/mt76/mt76x0/pci.c index 9277ff38b7a22..57ae362dad50b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/pci.c @@ -152,7 +152,8 @@ mt76x0e_probe(struct pci_dev *pdev, const struct pci_device_id *id) static const struct mt76_driver_ops drv_ops = { .txwi_size = sizeof(struct mt76x02_txwi), .drv_flags = MT_DRV_TX_ALIGNED4_SKBS | - MT_DRV_SW_RX_AIRTIME, + MT_DRV_SW_RX_AIRTIME | + MT_DRV_IGNORE_TXS_FAILED, .survey_flags = SURVEY_INFO_TIME_TX, .update_survey = mt76x02_update_channel, .tx_prepare_skb = mt76x02_tx_prepare_skb, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x0/usb.c b/drivers/net/wireless/mediatek/mt76/mt76x0/usb.c index 0422c332354a1..520fd46227a7b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x0/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x0/usb.c @@ -210,7 +210,8 @@ static int mt76x0u_probe(struct usb_interface *usb_intf, const struct usb_device_id *id) { static const struct mt76_driver_ops drv_ops = { - .drv_flags = MT_DRV_SW_RX_AIRTIME, + .drv_flags = MT_DRV_SW_RX_AIRTIME | + MT_DRV_IGNORE_TXS_FAILED, .survey_flags = SURVEY_INFO_TIME_TX, .update_survey = mt76x02_update_channel, .tx_prepare_skb = mt76x02u_tx_prepare_skb, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c b/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c index df85ebc6e1df0..7e2475b3c278e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c @@ -22,7 +22,8 @@ mt76x2e_probe(struct pci_dev *pdev, const struct pci_device_id *id) static const struct mt76_driver_ops drv_ops = { .txwi_size = sizeof(struct mt76x02_txwi), .drv_flags = MT_DRV_TX_ALIGNED4_SKBS | - MT_DRV_SW_RX_AIRTIME, + MT_DRV_SW_RX_AIRTIME | + MT_DRV_IGNORE_TXS_FAILED, .survey_flags = SURVEY_INFO_TIME_TX, .update_survey = mt76x02_update_channel, .tx_prepare_skb = mt76x02_tx_prepare_skb, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c b/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c index d804309992196..00248e2b21ea7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/usb.c @@ -17,6 +17,8 @@ static const struct usb_device_id mt76x2u_device_table[] = { { USB_DEVICE(0x057c, 0x8503) }, /* Avm FRITZ!WLAN AC860 */ { USB_DEVICE(0x7392, 0xb711) }, /* Edimax EW 7722 UAC */ { USB_DEVICE(0x0e8d, 0x7632) }, /* HC-M7662BU1 */ + { USB_DEVICE(0x0471, 0x2126) }, /* LiteOn WN4516R module, nonstandard USB connector */ + { USB_DEVICE(0x0471, 0x7600) }, /* LiteOn WN4519R module, nonstandard USB connector */ { USB_DEVICE(0x2c4e, 0x0103) }, /* Mercury UD13 */ { USB_DEVICE(0x0846, 0x9053) }, /* Netgear A6210 */ { USB_DEVICE(0x045e, 0x02e6) }, /* XBox One Wireless Adapter */ @@ -29,7 +31,8 @@ static int mt76x2u_probe(struct usb_interface *intf, const struct usb_device_id *id) { static const struct mt76_driver_ops drv_ops = { - .drv_flags = MT_DRV_SW_RX_AIRTIME, + .drv_flags = MT_DRV_SW_RX_AIRTIME | + MT_DRV_IGNORE_TXS_FAILED, .survey_flags = SURVEY_INFO_TIME_TX, .update_survey = mt76x02_update_channel, .tx_prepare_skb = mt76x02u_tx_prepare_skb, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/usb_init.c b/drivers/net/wireless/mediatek/mt76/mt76x2/usb_init.c index 33a14365ec9b9..3b55628115115 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x2/usb_init.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x2/usb_init.c @@ -191,6 +191,7 @@ int mt76x2u_register_device(struct mt76x02_dev *dev) { struct ieee80211_hw *hw = mt76_hw(dev); struct mt76_usb *usb = &dev->mt76.usb; + bool vht; int err; INIT_DELAYED_WORK(&dev->cal_work, mt76x2u_phy_calibrate); @@ -217,7 +218,17 @@ int mt76x2u_register_device(struct mt76x02_dev *dev) /* check hw sg support in order to enable AMSDU */ hw->max_tx_fragments = dev->mt76.usb.sg_en ? MT_TX_SG_MAX_SIZE : 1; - err = mt76_register_device(&dev->mt76, true, mt76x02_rates, + switch (dev->mt76.rev) { + case 0x76320044: + /* these ASIC revisions do not support VHT */ + vht = false; + break; + default: + vht = true; + break; + } + + err = mt76_register_device(&dev->mt76, vht, mt76x02_rates, ARRAY_SIZE(mt76x02_rates)); if (err) goto fail; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index f0226db2e57c7..fae9ec98da3b9 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -2060,16 +2060,21 @@ static int mt7915_load_firmware(struct mt7915_dev *dev) { int ret; - /* make sure fw is download state */ - if (mt7915_firmware_state(dev, false)) { - /* restart firmware once */ - mt76_connac_mcu_restart(&dev->mt76); - ret = mt7915_firmware_state(dev, false); - if (ret) { - dev_err(dev->mt76.dev, - "Firmware is not ready for download\n"); - return ret; - } + /* Release Semaphore if taken by previous failed attempt */ + ret = mt76_connac_mcu_patch_sem_ctrl(&dev->mt76, false); + if (ret != PATCH_REL_SEM_SUCCESS) { + dev_err(dev->mt76.dev, "Could not release semaphore\n"); + /* Continue anyways */ + } + + /* Always restart MCU firmware */ + mt76_connac_mcu_restart(&dev->mt76); + + /* Check if MCU is ready */ + ret = mt7915_firmware_state(dev, false); + if (ret) { + dev_err(dev->mt76.dev, "Firmware did not enter download state\n"); + return ret; } ret = mt76_connac2_load_patch(&dev->mt76, fw_name_var(dev, ROM_PATCH)); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c index 7db436d908a39..f4850c6daeb72 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c @@ -755,6 +755,9 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, wed->wlan.base = devm_ioremap(dev->mt76.dev, pci_resource_start(pci_dev, 0), pci_resource_len(pci_dev, 0)); + if (!wed->wlan.base) + return -ENOMEM; + wed->wlan.phy_base = pci_resource_start(pci_dev, 0); wed->wlan.wpdma_int = pci_resource_start(pci_dev, 0) + MT_INT_WED_SOURCE_CSR; @@ -782,6 +785,9 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, wed->wlan.bus_type = MTK_WED_BUS_AXI; wed->wlan.base = devm_ioremap(dev->mt76.dev, res->start, resource_size(res)); + if (!wed->wlan.base) + return -ENOMEM; + wed->wlan.phy_base = res->start; wed->wlan.wpdma_int = res->start + MT_INT_SOURCE_CSR; wed->wlan.wpdma_mask = res->start + MT_INT_MASK_CSR; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 31ef58e2a3d2a..ae7a01c7ce36d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -83,6 +83,11 @@ mt7921_init_he_caps(struct mt792x_phy *phy, enum nl80211_band band, he_cap_elem->phy_cap_info[9] |= IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU | IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU; + + if (is_mt7922(phy->mt76->dev)) { + he_cap_elem->phy_cap_info[0] |= + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + } break; case NL80211_IFTYPE_STATION: he_cap_elem->mac_cap_info[1] |= @@ -130,6 +135,8 @@ mt7921_init_he_caps(struct mt792x_phy *phy, enum nl80211_band band, if (is_mt7922(phy->mt76->dev)) { he_cap_elem->phy_cap_info[0] |= IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + he_cap_elem->phy_cap_info[4] |= + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4; he_cap_elem->phy_cap_info[8] |= IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU; @@ -1082,6 +1089,9 @@ static void mt7921_sta_set_decap_offload(struct ieee80211_hw *hw, struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; struct mt792x_dev *dev = mt792x_hw_dev(hw); + if (!msta->wcid.sta) + return; + mt792x_mutex_acquire(dev); if (enabled) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c index 13e892d788b27..92281811103ef 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c @@ -21,6 +21,9 @@ static const struct usb_device_id mt7921u_device_table[] = { /* Netgear, Inc. [A8000,AXE3000] */ { USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9060, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM }, + /* Netgear, Inc. A7500 */ + { USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9065, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM }, /* TP-Link TXE50UH */ { USB_DEVICE_AND_INTERFACE_INFO(0x35bc, 0x0107, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM }, diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c index 586e247a1e064..04c9fd0e6b002 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c @@ -300,7 +300,7 @@ int mt7996_dma_init(struct mt7996_dev *dev) ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU], MT_RXQ_ID(MT_RXQ_MCU), MT7996_RX_MCU_RING_SIZE, - MT_RX_BUF_SIZE, + MT7996_RX_MCU_BUF_SIZE, MT_RXQ_RING_BASE(MT_RXQ_MCU)); if (ret) return ret; @@ -309,7 +309,7 @@ int mt7996_dma_init(struct mt7996_dev *dev) ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU_WA], MT_RXQ_ID(MT_RXQ_MCU_WA), MT7996_RX_MCU_RING_SIZE_WA, - MT_RX_BUF_SIZE, + MT7996_RX_MCU_BUF_SIZE, MT_RXQ_RING_BASE(MT_RXQ_MCU_WA)); if (ret) return ret; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index 0a701dcb8a92c..fa81a1c30704b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -730,11 +730,13 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, eht_cap->has_eht = true; eht_cap_elem->mac_cap_info[0] = - IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | IEEE80211_EHT_MAC_CAP0_OM_CONTROL | u8_encode_bits(IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_11454, IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK); + eht_cap_elem->mac_cap_info[1] |= + IEEE80211_EHT_MAC_CAP1_MAX_AMPDU_LEN_MASK; + eht_cap_elem->phy_cap_info[0] = IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI | IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER | diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 73d46ec1181ae..8fa16f95e6a7b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -650,6 +650,14 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb) status->last_amsdu = amsdu_info == MT_RXD4_LAST_AMSDU_FRAME; } + /* IEEE 802.11 fragmentation can only be applied to unicast frames. + * Hence, drop fragments with multicast/broadcast RA. + * This check fixes vulnerabilities, like CVE-2020-26145. + */ + if ((ieee80211_has_morefrags(fc) || seq_ctrl & IEEE80211_SCTL_FRAG) && + FIELD_GET(MT_RXD3_NORMAL_ADDR_TYPE, rxd3) != MT_RXD3_NORMAL_U2M) + return -EINVAL; + hdr_gap = (u8 *)rxd - skb->data + 2 * remove_pad; if (hdr_trans && ieee80211_has_morefrags(fc)) { if (mt7996_reverse_frag0_hdr_trans(skb, hdr_gap)) @@ -1354,7 +1362,7 @@ bool mt7996_rx_check(struct mt76_dev *mdev, void *data, int len) mt7996_mac_tx_free(dev, data, len); return false; case PKT_TYPE_TXS: - for (rxd += 4; rxd + 8 <= end; rxd += 8) + for (rxd += MT_TXS_HDR_SIZE; rxd + MT_TXS_SIZE <= end; rxd += MT_TXS_SIZE) mt7996_mac_add_txs(dev, rxd); return false; case PKT_TYPE_RX_FW_MONITOR: @@ -1391,7 +1399,7 @@ void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, mt7996_mcu_rx_event(dev, skb); break; case PKT_TYPE_TXS: - for (rxd += 4; rxd + 8 <= end; rxd += 8) + for (rxd += MT_TXS_HDR_SIZE; rxd + MT_TXS_SIZE <= end; rxd += MT_TXS_SIZE) mt7996_mac_add_txs(dev, rxd); dev_kfree_skb(skb); break; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index 65a5f24e53136..8ab55fc705f07 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -1616,8 +1616,8 @@ mt7996_mcu_get_mmps_mode(enum ieee80211_smps_mode smps) int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev, void *data, u16 version) { + struct uni_header hdr = {}; struct ra_fixed_rate *req; - struct uni_header hdr; struct sk_buff *skb; struct tlv *tlv; int len; @@ -2638,7 +2638,7 @@ int mt7996_mcu_set_hdr_trans(struct mt7996_dev *dev, bool hdr_trans) { struct { u8 __rsv[4]; - } __packed hdr; + } __packed hdr = {}; struct hdr_trans_blacklist *req_blacklist; struct hdr_trans_en *req_en; struct sk_buff *skb; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index 25bb365612314..7d2074e2b635e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -27,6 +27,9 @@ #define MT7996_RX_RING_SIZE 1536 #define MT7996_RX_MCU_RING_SIZE 512 #define MT7996_RX_MCU_RING_SIZE_WA 1024 +/* scatter-gather of mcu event is not supported in connac3 */ +#define MT7996_RX_MCU_BUF_SIZE (2048 + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) #define MT7996_FIRMWARE_WA "mediatek/mt7996/mt7996_wa.bin" #define MT7996_FIRMWARE_WM "mediatek/mt7996/mt7996_wm.bin" diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index 1809b03292c3d..47cdccdbed6aa 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -100,7 +100,8 @@ __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags, return; /* Tx status can be unreliable. if it fails, mark the frame as ACKed */ - if (flags & MT_TX_CB_TXS_FAILED) { + if (flags & MT_TX_CB_TXS_FAILED && + (dev->drv->drv_flags & MT_DRV_IGNORE_TXS_FAILED)) { info->status.rates[0].count = 0; info->status.rates[0].idx = -1; info->flags |= IEEE80211_TX_STAT_ACK; diff --git a/drivers/net/wireless/microchip/wilc1000/wlan_cfg.c b/drivers/net/wireless/microchip/wilc1000/wlan_cfg.c index 131388886acbf..cfabd5aebb540 100644 --- a/drivers/net/wireless/microchip/wilc1000/wlan_cfg.c +++ b/drivers/net/wireless/microchip/wilc1000/wlan_cfg.c @@ -41,10 +41,10 @@ static const struct wilc_cfg_word g_cfg_word[] = { }; static const struct wilc_cfg_str g_cfg_str[] = { - {WID_FIRMWARE_VERSION, NULL}, - {WID_MAC_ADDR, NULL}, - {WID_ASSOC_RES_INFO, NULL}, - {WID_NIL, NULL} + {WID_FIRMWARE_VERSION, 0, NULL}, + {WID_MAC_ADDR, 0, NULL}, + {WID_ASSOC_RES_INFO, 0, NULL}, + {WID_NIL, 0, NULL} }; #define WILC_RESP_MSG_TYPE_CONFIG_REPLY 'R' @@ -147,44 +147,58 @@ static void wilc_wlan_parse_response_frame(struct wilc *wl, u8 *info, int size) switch (FIELD_GET(WILC_WID_TYPE, wid)) { case WID_CHAR: + len = 3; + if (len + 2 > size) + return; + while (cfg->b[i].id != WID_NIL && cfg->b[i].id != wid) i++; if (cfg->b[i].id == wid) cfg->b[i].val = info[4]; - len = 3; break; case WID_SHORT: + len = 4; + if (len + 2 > size) + return; + while (cfg->hw[i].id != WID_NIL && cfg->hw[i].id != wid) i++; if (cfg->hw[i].id == wid) cfg->hw[i].val = get_unaligned_le16(&info[4]); - len = 4; break; case WID_INT: + len = 6; + if (len + 2 > size) + return; + while (cfg->w[i].id != WID_NIL && cfg->w[i].id != wid) i++; if (cfg->w[i].id == wid) cfg->w[i].val = get_unaligned_le32(&info[4]); - len = 6; break; case WID_STR: + len = 2 + get_unaligned_le16(&info[2]); + while (cfg->s[i].id != WID_NIL && cfg->s[i].id != wid) i++; - if (cfg->s[i].id == wid) + if (cfg->s[i].id == wid) { + if (len > cfg->s[i].len || (len + 2 > size)) + return; + memcpy(cfg->s[i].str, &info[2], - get_unaligned_le16(&info[2]) + 2); + len); + } - len = 2 + get_unaligned_le16(&info[2]); break; default: @@ -384,12 +398,15 @@ int wilc_wlan_cfg_init(struct wilc *wl) /* store the string cfg parameters */ wl->cfg.s[i].id = WID_FIRMWARE_VERSION; wl->cfg.s[i].str = str_vals->firmware_version; + wl->cfg.s[i].len = sizeof(str_vals->firmware_version); i++; wl->cfg.s[i].id = WID_MAC_ADDR; wl->cfg.s[i].str = str_vals->mac_address; + wl->cfg.s[i].len = sizeof(str_vals->mac_address); i++; wl->cfg.s[i].id = WID_ASSOC_RES_INFO; wl->cfg.s[i].str = str_vals->assoc_rsp; + wl->cfg.s[i].len = sizeof(str_vals->assoc_rsp); i++; wl->cfg.s[i].id = WID_NIL; wl->cfg.s[i].str = NULL; diff --git a/drivers/net/wireless/microchip/wilc1000/wlan_cfg.h b/drivers/net/wireless/microchip/wilc1000/wlan_cfg.h index 7038b74f8e8ff..5ae74bced7d74 100644 --- a/drivers/net/wireless/microchip/wilc1000/wlan_cfg.h +++ b/drivers/net/wireless/microchip/wilc1000/wlan_cfg.h @@ -24,12 +24,13 @@ struct wilc_cfg_word { struct wilc_cfg_str { u16 id; + u16 len; u8 *str; }; struct wilc_cfg_str_vals { - u8 mac_address[7]; - u8 firmware_version[129]; + u8 mac_address[8]; + u8 firmware_version[130]; u8 assoc_rsp[WILC_MAX_ASSOC_RESP_FRAME_SIZE]; }; diff --git a/drivers/net/wireless/purelifi/plfxlc/mac.c b/drivers/net/wireless/purelifi/plfxlc/mac.c index 506d2f31efb5a..f9f7532c32fde 100644 --- a/drivers/net/wireless/purelifi/plfxlc/mac.c +++ b/drivers/net/wireless/purelifi/plfxlc/mac.c @@ -100,12 +100,6 @@ int plfxlc_mac_init_hw(struct ieee80211_hw *hw) return r; } -void plfxlc_mac_release(struct plfxlc_mac *mac) -{ - plfxlc_chip_release(&mac->chip); - lockdep_assert_held(&mac->lock); -} - int plfxlc_op_start(struct ieee80211_hw *hw) { plfxlc_hw_mac(hw)->chip.usb.initialized = 1; @@ -753,3 +747,9 @@ struct ieee80211_hw *plfxlc_mac_alloc_hw(struct usb_interface *intf) SET_IEEE80211_DEV(hw, &intf->dev); return hw; } + +void plfxlc_mac_release_hw(struct ieee80211_hw *hw) +{ + plfxlc_chip_release(&plfxlc_hw_mac(hw)->chip); + ieee80211_free_hw(hw); +} diff --git a/drivers/net/wireless/purelifi/plfxlc/mac.h b/drivers/net/wireless/purelifi/plfxlc/mac.h index 49b92413729bf..c0445932b2e8a 100644 --- a/drivers/net/wireless/purelifi/plfxlc/mac.h +++ b/drivers/net/wireless/purelifi/plfxlc/mac.h @@ -168,7 +168,7 @@ static inline u8 *plfxlc_mac_get_perm_addr(struct plfxlc_mac *mac) } struct ieee80211_hw *plfxlc_mac_alloc_hw(struct usb_interface *intf); -void plfxlc_mac_release(struct plfxlc_mac *mac); +void plfxlc_mac_release_hw(struct ieee80211_hw *hw); int plfxlc_mac_preinit_hw(struct ieee80211_hw *hw, const u8 *hw_address); int plfxlc_mac_init_hw(struct ieee80211_hw *hw); diff --git a/drivers/net/wireless/purelifi/plfxlc/usb.c b/drivers/net/wireless/purelifi/plfxlc/usb.c index 311676c1ece0a..901e0139969e8 100644 --- a/drivers/net/wireless/purelifi/plfxlc/usb.c +++ b/drivers/net/wireless/purelifi/plfxlc/usb.c @@ -503,8 +503,10 @@ int plfxlc_usb_wreq_async(struct plfxlc_usb *usb, const u8 *buffer, (void *)buffer, buffer_len, complete_fn, context); r = usb_submit_urb(urb, GFP_ATOMIC); - if (r) + if (r) { + usb_free_urb(urb); dev_err(&udev->dev, "Async write submit failed (%d)\n", r); + } return r; } @@ -602,7 +604,7 @@ static int probe(struct usb_interface *intf, r = plfxlc_upload_mac_and_serial(intf, hw_address, serial_number); if (r) { dev_err(&intf->dev, "MAC and Serial upload failed (%d)\n", r); - goto error; + goto error_free_hw; } chip->unit_type = STA; @@ -611,13 +613,13 @@ static int probe(struct usb_interface *intf, r = plfxlc_mac_preinit_hw(hw, hw_address); if (r) { dev_err(&intf->dev, "Init mac failed (%d)\n", r); - goto error; + goto error_free_hw; } r = ieee80211_register_hw(hw); if (r) { dev_err(&intf->dev, "Register device failed (%d)\n", r); - goto error; + goto error_free_hw; } if ((le16_to_cpu(interface_to_usbdev(intf)->descriptor.idVendor) == @@ -630,7 +632,7 @@ static int probe(struct usb_interface *intf, } if (r != 0) { dev_err(&intf->dev, "FPGA download failed (%d)\n", r); - goto error; + goto error_unreg_hw; } tx->mac_fifo_full = 0; @@ -640,21 +642,21 @@ static int probe(struct usb_interface *intf, r = plfxlc_usb_init_hw(usb); if (r < 0) { dev_err(&intf->dev, "usb_init_hw failed (%d)\n", r); - goto error; + goto error_unreg_hw; } msleep(PLF_MSLEEP_TIME); r = plfxlc_chip_switch_radio(chip, PLFXLC_RADIO_ON); if (r < 0) { dev_dbg(&intf->dev, "chip_switch_radio_on failed (%d)\n", r); - goto error; + goto error_unreg_hw; } msleep(PLF_MSLEEP_TIME); r = plfxlc_chip_set_rate(chip, 8); if (r < 0) { dev_dbg(&intf->dev, "chip_set_rate failed (%d)\n", r); - goto error; + goto error_unreg_hw; } msleep(PLF_MSLEEP_TIME); @@ -662,7 +664,7 @@ static int probe(struct usb_interface *intf, hw_address, ETH_ALEN, USB_REQ_MAC_WR); if (r < 0) { dev_dbg(&intf->dev, "MAC_WR failure (%d)\n", r); - goto error; + goto error_unreg_hw; } plfxlc_chip_enable_rxtx(chip); @@ -689,12 +691,12 @@ static int probe(struct usb_interface *intf, plfxlc_mac_init_hw(hw); usb->initialized = true; return 0; + +error_unreg_hw: + ieee80211_unregister_hw(hw); +error_free_hw: + plfxlc_mac_release_hw(hw); error: - if (hw) { - plfxlc_mac_release(plfxlc_hw_mac(hw)); - ieee80211_unregister_hw(hw); - ieee80211_free_hw(hw); - } dev_err(&intf->dev, "pureLifi:Device error"); return r; } @@ -728,8 +730,7 @@ static void disconnect(struct usb_interface *intf) */ usb_reset_device(interface_to_usbdev(intf)); - plfxlc_mac_release(mac); - ieee80211_free_hw(hw); + plfxlc_mac_release_hw(hw); } static void plfxlc_usb_resume(struct plfxlc_usb *usb) diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c index 04945f905d6d0..f6528469022bf 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c @@ -1041,10 +1041,11 @@ static void rtl8187_stop(struct ieee80211_hw *dev) rtl818x_iowrite8(priv, &priv->map->CONFIG4, reg | RTL818X_CONFIG4_VCOOFF); rtl818x_iowrite8(priv, &priv->map->EEPROM_CMD, RTL818X_EEPROM_CMD_NORMAL); + usb_kill_anchored_urbs(&priv->anchored); + while ((skb = skb_dequeue(&priv->b_tx_status.queue))) dev_kfree_skb_any(skb); - usb_kill_anchored_urbs(&priv->anchored); mutex_unlock(&priv->conf_mutex); if (!priv->is_rtl8187b) diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c index 6e47dde938909..03aacb7a43171 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c @@ -900,9 +900,10 @@ rtl8xxxu_writeN(struct rtl8xxxu_priv *priv, u16 addr, u8 *buf, u16 len) return len; write_error: - dev_info(&udev->dev, - "%s: Failed to write block at addr: %04x size: %04x\n", - __func__, addr, blocksize); + if (rtl8xxxu_debug & RTL8XXXU_DEBUG_REG_WRITE) + dev_info(&udev->dev, + "%s: Failed to write block at addr: %04x size: %04x\n", + __func__, addr, blocksize); return -EAGAIN; } @@ -4073,8 +4074,14 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw) */ rtl8xxxu_write16(priv, REG_TRXFF_BNDY + 2, fops->trxff_boundary); - ret = rtl8xxxu_download_firmware(priv); - dev_dbg(dev, "%s: download_firmware %i\n", __func__, ret); + for (int retry = 5; retry >= 0 ; retry--) { + ret = rtl8xxxu_download_firmware(priv); + dev_dbg(dev, "%s: download_firmware %i\n", __func__, ret); + if (ret != -EAGAIN) + break; + if (retry) + dev_dbg(dev, "%s: retry firmware download\n", __func__); + } if (ret) goto exit; ret = rtl8xxxu_start_firmware(priv); @@ -6494,7 +6501,7 @@ static int rtl8xxxu_submit_rx_urb(struct rtl8xxxu_priv *priv, skb_size = fops->rx_agg_buf_size; skb_size += (rx_desc_sz + sizeof(struct rtl8723au_phy_stats)); } else { - skb_size = IEEE80211_MAX_FRAME_LEN; + skb_size = IEEE80211_MAX_FRAME_LEN + rx_desc_sz; } skb = __netdev_alloc_skb(NULL, skb_size, GFP_KERNEL); diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index 3abd0c4c954bc..40112b2c37775 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -155,6 +155,16 @@ static void _rtl_pci_update_default_setting(struct ieee80211_hw *hw) if (rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8192SE && init_aspm == 0x43) ppsc->support_aspm = false; + + /* RTL8723BE found on some ASUSTek laptops, such as F441U and + * X555UQ with subsystem ID 11ad:1723 are known to output large + * amounts of PCIe AER errors during and after boot up, causing + * heavy lags, poor network throughput, and occasional lock-ups. + */ + if (rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8723BE && + (rtlpci->pdev->subsystem_vendor == 0x11ad && + rtlpci->pdev->subsystem_device == 0x1723)) + ppsc->support_aspm = false; } static bool _rtl_pci_platform_switch_device_pci_aspm( @@ -563,8 +573,11 @@ static int _rtl_pci_init_one_rxdesc(struct ieee80211_hw *hw, dma_map_single(&rtlpci->pdev->dev, skb_tail_pointer(skb), rtlpci->rxbuffersize, DMA_FROM_DEVICE); bufferaddress = *((dma_addr_t *)skb->cb); - if (dma_mapping_error(&rtlpci->pdev->dev, bufferaddress)) + if (dma_mapping_error(&rtlpci->pdev->dev, bufferaddress)) { + if (!new_skb) + kfree_skb(skb); return 0; + } rtlpci->rx_ring[rxring_idx].rx_buf[desc_idx] = skb; if (rtlpriv->use_new_trx_flow) { /* skb->cb may be 64 bit address */ @@ -793,13 +806,19 @@ static void _rtl_pci_rx_interrupt(struct ieee80211_hw *hw) skb = new_skb; no_new: if (rtlpriv->use_new_trx_flow) { - _rtl_pci_init_one_rxdesc(hw, skb, (u8 *)buffer_desc, - rxring_idx, - rtlpci->rx_ring[rxring_idx].idx); + if (!_rtl_pci_init_one_rxdesc(hw, skb, (u8 *)buffer_desc, + rxring_idx, + rtlpci->rx_ring[rxring_idx].idx)) { + if (new_skb) + dev_kfree_skb_any(skb); + } } else { - _rtl_pci_init_one_rxdesc(hw, skb, (u8 *)pdesc, - rxring_idx, - rtlpci->rx_ring[rxring_idx].idx); + if (!_rtl_pci_init_one_rxdesc(hw, skb, (u8 *)pdesc, + rxring_idx, + rtlpci->rx_ring[rxring_idx].idx)) { + if (new_skb) + dev_kfree_skb_any(skb); + } if (rtlpci->rx_ring[rxring_idx].idx == rtlpci->rxringcount - 1) rtlpriv->cfg->ops->set_desc(hw, (u8 *)pdesc, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c index e6403d4c937c8..a0c0932a85d1e 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c @@ -294,7 +294,6 @@ static const struct usb_device_id rtl8192c_usb_ids[] = { {RTL_USB_DEVICE(0x050d, 0x1102, rtl92cu_hal_cfg)}, /*Belkin - Edimax*/ {RTL_USB_DEVICE(0x050d, 0x11f2, rtl92cu_hal_cfg)}, /*Belkin - ISY*/ {RTL_USB_DEVICE(0x06f8, 0xe033, rtl92cu_hal_cfg)}, /*Hercules - Edimax*/ - {RTL_USB_DEVICE(0x07b8, 0x8188, rtl92cu_hal_cfg)}, /*Abocom - Abocom*/ {RTL_USB_DEVICE(0x07b8, 0x8189, rtl92cu_hal_cfg)}, /*Funai - Abocom*/ {RTL_USB_DEVICE(0x0846, 0x9041, rtl92cu_hal_cfg)}, /*NetGear WNA1000M*/ {RTL_USB_DEVICE(0x0846, 0x9043, rtl92cu_hal_cfg)}, /*NG WNA1000Mv2*/ diff --git a/drivers/net/wireless/realtek/rtw88/coex.c b/drivers/net/wireless/realtek/rtw88/coex.c index d35f26919806a..c45c7b596ffe9 100644 --- a/drivers/net/wireless/realtek/rtw88/coex.c +++ b/drivers/net/wireless/realtek/rtw88/coex.c @@ -309,7 +309,7 @@ static void rtw_coex_tdma_timer_base(struct rtw_dev *rtwdev, u8 type) { struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; - u8 para[2] = {0}; + u8 para[6] = {}; u8 times; u16 tbtt_interval = coex_stat->wl_beacon_interval; diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c index 0c1c1ff31085c..929182424b8b8 100644 --- a/drivers/net/wireless/realtek/rtw88/mac.c +++ b/drivers/net/wireless/realtek/rtw88/mac.c @@ -783,7 +783,8 @@ static int __rtw_download_firmware(struct rtw_dev *rtwdev, if (!check_firmware_size(data, size)) return -EINVAL; - if (!ltecoex_read_reg(rtwdev, 0x38, <ecoex_bckp)) + if (rtwdev->chip->ltecoex_addr && + !ltecoex_read_reg(rtwdev, 0x38, <ecoex_bckp)) return -EBUSY; wlan_cpu_enable(rtwdev, false); @@ -801,7 +802,8 @@ static int __rtw_download_firmware(struct rtw_dev *rtwdev, wlan_cpu_enable(rtwdev, true); - if (!ltecoex_reg_write(rtwdev, 0x38, ltecoex_bckp)) { + if (rtwdev->chip->ltecoex_addr && + !ltecoex_reg_write(rtwdev, 0x38, ltecoex_bckp)) { ret = -EBUSY; goto dlfw_fail; } diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index b90ea6c88b15d..0d0b5123b5fe2 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -1544,6 +1544,7 @@ static void rtw_init_ht_cap(struct rtw_dev *rtwdev, { const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; + int i; ht_cap->ht_supported = true; ht_cap->cap = 0; @@ -1563,25 +1564,20 @@ static void rtw_init_ht_cap(struct rtw_dev *rtwdev, ht_cap->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; ht_cap->ampdu_density = chip->ampdu_density; ht_cap->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; - if (efuse->hw_cap.nss > 1) { - ht_cap->mcs.rx_mask[0] = 0xFF; - ht_cap->mcs.rx_mask[1] = 0xFF; - ht_cap->mcs.rx_mask[4] = 0x01; - ht_cap->mcs.rx_highest = cpu_to_le16(300); - } else { - ht_cap->mcs.rx_mask[0] = 0xFF; - ht_cap->mcs.rx_mask[1] = 0x00; - ht_cap->mcs.rx_mask[4] = 0x01; - ht_cap->mcs.rx_highest = cpu_to_le16(150); - } + + for (i = 0; i < efuse->hw_cap.nss; i++) + ht_cap->mcs.rx_mask[i] = 0xFF; + ht_cap->mcs.rx_mask[4] = 0x01; + ht_cap->mcs.rx_highest = cpu_to_le16(150 * efuse->hw_cap.nss); } static void rtw_init_vht_cap(struct rtw_dev *rtwdev, struct ieee80211_sta_vht_cap *vht_cap) { struct rtw_efuse *efuse = &rtwdev->efuse; - u16 mcs_map; + u16 mcs_map = 0; __le16 highest; + int i; if (efuse->hw_cap.ptcl != EFUSE_HW_CAP_IGNORE && efuse->hw_cap.ptcl != EFUSE_HW_CAP_PTCL_VHT) @@ -1604,21 +1600,15 @@ static void rtw_init_vht_cap(struct rtw_dev *rtwdev, if (rtw_chip_has_rx_ldpc(rtwdev)) vht_cap->cap |= IEEE80211_VHT_CAP_RXLDPC; - mcs_map = IEEE80211_VHT_MCS_SUPPORT_0_9 << 0 | - IEEE80211_VHT_MCS_NOT_SUPPORTED << 4 | - IEEE80211_VHT_MCS_NOT_SUPPORTED << 6 | - IEEE80211_VHT_MCS_NOT_SUPPORTED << 8 | - IEEE80211_VHT_MCS_NOT_SUPPORTED << 10 | - IEEE80211_VHT_MCS_NOT_SUPPORTED << 12 | - IEEE80211_VHT_MCS_NOT_SUPPORTED << 14; - if (efuse->hw_cap.nss > 1) { - highest = cpu_to_le16(780); - mcs_map |= IEEE80211_VHT_MCS_SUPPORT_0_9 << 2; - } else { - highest = cpu_to_le16(390); - mcs_map |= IEEE80211_VHT_MCS_NOT_SUPPORTED << 2; + for (i = 0; i < 8; i++) { + if (i < efuse->hw_cap.nss) + mcs_map |= IEEE80211_VHT_MCS_SUPPORT_0_9 << (i * 2); + else + mcs_map |= IEEE80211_VHT_MCS_NOT_SUPPORTED << (i * 2); } + highest = cpu_to_le16(390 * efuse->hw_cap.nss); + vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(mcs_map); vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(mcs_map); vht_cap->vht_mcs.rx_highest = highest; diff --git a/drivers/net/wireless/realtek/rtw88/reg.h b/drivers/net/wireless/realtek/rtw88/reg.h index 7c6c11d50ff30..0e76bc07bddef 100644 --- a/drivers/net/wireless/realtek/rtw88/reg.h +++ b/drivers/net/wireless/realtek/rtw88/reg.h @@ -108,6 +108,7 @@ #define BIT_SHIFT_ROM_PGE 16 #define BIT_FW_INIT_RDY BIT(15) #define BIT_FW_DW_RDY BIT(14) +#define BIT_CPU_CLK_SEL (BIT(12) | BIT(13)) #define BIT_RPWM_TOGGLE BIT(7) #define BIT_RAM_DL_SEL BIT(7) /* legacy only */ #define BIT_DMEM_CHKSUM_OK BIT(6) @@ -125,7 +126,7 @@ BIT_CHECK_SUM_OK) #define FW_READY_LEGACY (BIT_MCUFWDL_RDY | BIT_FWDL_CHK_RPT | \ BIT_WINTINI_RDY | BIT_RAM_DL_SEL) -#define FW_READY_MASK 0xffff +#define FW_READY_MASK (0xffff & ~BIT_CPU_CLK_SEL) #define REG_MCU_TST_CFG 0x84 #define VAL_FW_TRIGGER 0x1 diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c index 3017a9760da8d..99318a82b43f4 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c @@ -975,11 +975,11 @@ static void rtw8822b_query_rx_desc(struct rtw_dev *rtwdev, u8 *rx_desc, } static void -rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs) +rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, + u8 rs, u32 *phy_pwr_idx) { struct rtw_hal *hal = &rtwdev->hal; static const u32 offset_txagc[2] = {0x1d00, 0x1d80}; - static u32 phy_pwr_idx; u8 rate, rate_idx, pwr_index, shift; int j; @@ -987,12 +987,12 @@ rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs) rate = rtw_rate_section[rs][j]; pwr_index = hal->tx_pwr_tbl[path][rate]; shift = rate & 0x3; - phy_pwr_idx |= ((u32)pwr_index << (shift * 8)); + *phy_pwr_idx |= ((u32)pwr_index << (shift * 8)); if (shift == 0x3) { rate_idx = rate & 0xfc; rtw_write32(rtwdev, offset_txagc[path] + rate_idx, - phy_pwr_idx); - phy_pwr_idx = 0; + *phy_pwr_idx); + *phy_pwr_idx = 0; } } } @@ -1000,11 +1000,13 @@ rtw8822b_set_tx_power_index_by_rate(struct rtw_dev *rtwdev, u8 path, u8 rs) static void rtw8822b_set_tx_power_index(struct rtw_dev *rtwdev) { struct rtw_hal *hal = &rtwdev->hal; + u32 phy_pwr_idx = 0; int rs, path; for (path = 0; path < hal->rf_path_num; path++) { for (rs = 0; rs < RTW_RATE_SECTION_MAX; rs++) - rtw8822b_set_tx_power_index_by_rate(rtwdev, path, rs); + rtw8822b_set_tx_power_index_by_rate(rtwdev, path, rs, + &phy_pwr_idx); } } diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c index 3fe5c70ce731b..f9b2527fbeee5 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c @@ -3991,7 +3991,8 @@ static void rtw8822c_dpk_cal_coef1(struct rtw_dev *rtwdev) rtw_write32(rtwdev, REG_NCTL0, 0x00001148); rtw_write32(rtwdev, REG_NCTL0, 0x00001149); - check_hw_ready(rtwdev, 0x2d9c, MASKBYTE0, 0x55); + if (!check_hw_ready(rtwdev, 0x2d9c, MASKBYTE0, 0x55)) + rtw_warn(rtwdev, "DPK stuck, performance may be suboptimal"); rtw_write8(rtwdev, 0x1b10, 0x0); rtw_write32_mask(rtwdev, REG_NCTL0, BIT_SUBPAGE, 0x0000000c); diff --git a/drivers/net/wireless/realtek/rtw88/sdio.c b/drivers/net/wireless/realtek/rtw88/sdio.c index 9043569935796..df4248744d87a 100644 --- a/drivers/net/wireless/realtek/rtw88/sdio.c +++ b/drivers/net/wireless/realtek/rtw88/sdio.c @@ -143,6 +143,10 @@ static u32 rtw_sdio_to_io_address(struct rtw_dev *rtwdev, u32 addr, static bool rtw_sdio_use_direct_io(struct rtw_dev *rtwdev, u32 addr) { + if (!test_bit(RTW_FLAG_POWERON, rtwdev->flags) && + !rtw_sdio_is_bus_addr(addr)) + return false; + return !rtw_sdio_is_sdio30_supported(rtwdev) || rtw_sdio_is_bus_addr(addr); } @@ -718,10 +722,7 @@ static u8 rtw_sdio_get_tx_qsel(struct rtw_dev *rtwdev, struct sk_buff *skb, case RTW_TX_QUEUE_H2C: return TX_DESC_QSEL_H2C; case RTW_TX_QUEUE_MGMT: - if (rtw_chip_wcpu_11n(rtwdev)) - return TX_DESC_QSEL_HIGH; - else - return TX_DESC_QSEL_MGMT; + return TX_DESC_QSEL_MGMT; case RTW_TX_QUEUE_HI0: return TX_DESC_QSEL_HIGH; default: @@ -1226,10 +1227,7 @@ static void rtw_sdio_process_tx_queue(struct rtw_dev *rtwdev, return; } - if (queue <= RTW_TX_QUEUE_VO) - rtw_sdio_indicate_tx_status(rtwdev, skb); - else - dev_kfree_skb_any(skb); + rtw_sdio_indicate_tx_status(rtwdev, skb); } static void rtw_sdio_tx_handler(struct work_struct *work) diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c index 8f1d653282b7e..eba3a2ee747f9 100644 --- a/drivers/net/wireless/realtek/rtw88/usb.c +++ b/drivers/net/wireless/realtek/rtw88/usb.c @@ -133,7 +133,7 @@ static void rtw_usb_write(struct rtw_dev *rtwdev, u32 addr, u32 val, int len) ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0), RTW_USB_CMD_REQ, RTW_USB_CMD_WRITE, - addr, 0, data, len, 30000); + addr, 0, data, len, 500); if (ret < 0 && ret != -ENODEV && count++ < 4) rtw_err(rtwdev, "write register 0x%x failed with %d\n", addr, ret); diff --git a/drivers/net/wireless/realtek/rtw88/util.c b/drivers/net/wireless/realtek/rtw88/util.c index e222d3c01a77e..66819f6944055 100644 --- a/drivers/net/wireless/realtek/rtw88/util.c +++ b/drivers/net/wireless/realtek/rtw88/util.c @@ -101,7 +101,8 @@ void rtw_desc_to_mcsrate(u16 rate, u8 *mcs, u8 *nss) *nss = 4; *mcs = rate - DESC_RATEVHT4SS_MCS0; } else if (rate >= DESC_RATEMCS0 && - rate <= DESC_RATEMCS15) { + rate <= DESC_RATEMCS31) { + *nss = 0; *mcs = rate - DESC_RATEMCS0; } } diff --git a/drivers/net/wireless/realtek/rtw89/cam.c b/drivers/net/wireless/realtek/rtw89/cam.c index f5301c2bbf133..9a0ffaddb8360 100644 --- a/drivers/net/wireless/realtek/rtw89/cam.c +++ b/drivers/net/wireless/realtek/rtw89/cam.c @@ -6,6 +6,7 @@ #include "debug.h" #include "fw.h" #include "mac.h" +#include "ps.h" static struct sk_buff * rtw89_cam_get_sec_key_cmd(struct rtw89_dev *rtwdev, @@ -333,9 +334,11 @@ int rtw89_cam_sec_key_add(struct rtw89_dev *rtwdev, switch (key->cipher) { case WLAN_CIPHER_SUITE_WEP40: + rtw89_leave_ips_by_hwflags(rtwdev); hw_key_type = RTW89_SEC_KEY_TYPE_WEP40; break; case WLAN_CIPHER_SUITE_WEP104: + rtw89_leave_ips_by_hwflags(rtwdev); hw_key_type = RTW89_SEC_KEY_TYPE_WEP104; break; case WLAN_CIPHER_SUITE_CCMP: diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index d1d8fd812cbf6..c172ef13c9540 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -1597,6 +1597,11 @@ static void rtw89_core_cancel_6ghz_probe_tx(struct rtw89_dev *rtwdev, if (rx_status->band != NL80211_BAND_6GHZ) return; + if (unlikely(!(rtwdev->chip->support_bands & BIT(NL80211_BAND_6GHZ)))) { + rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "invalid rx on unsupported 6 GHz\n"); + return; + } + ssid_ie = cfg80211_find_ie(WLAN_EID_SSID, ies, skb->len); list_for_each_entry(info, &pkt_list[NL80211_BAND_6GHZ], list) { @@ -1991,6 +1996,9 @@ static enum rtw89_ps_mode rtw89_update_ps_mode(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; + if (rtwdev->hci.type != RTW89_HCI_TYPE_PCIE) + return RTW89_PS_MODE_NONE; + if (rtw89_disable_ps_mode || !chip->ps_mode_supported || RTW89_CHK_FW_FEATURE(NO_DEEP_PS, &rtwdev->fw)) return RTW89_PS_MODE_NONE; diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index a8e2efae6e526..539537360914c 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -755,7 +755,6 @@ static int __rtw89_fw_download_hdr(struct rtw89_dev *rtwdev, const u8 *fw, u32 l ret = rtw89_h2c_tx(rtwdev, skb, false); if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); - ret = -1; goto fail; } @@ -816,7 +815,6 @@ static int __rtw89_fw_download_main(struct rtw89_dev *rtwdev, ret = rtw89_h2c_tx(rtwdev, skb, true); if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); - ret = -1; goto fail; } @@ -3429,13 +3427,18 @@ static int rtw89_fw_read_c2h_reg(struct rtw89_dev *rtwdev, const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_fw_info *fw_info = &rtwdev->fw; const u32 *c2h_reg = chip->c2h_regs; - u32 ret; + u32 ret, timeout; u8 i, val; info->id = RTW89_FWCMD_C2HREG_FUNC_NULL; + if (rtwdev->hci.type == RTW89_HCI_TYPE_USB) + timeout = RTW89_C2H_TIMEOUT_USB; + else + timeout = RTW89_C2H_TIMEOUT; + ret = read_poll_timeout_atomic(rtw89_read8, val, val, 1, - RTW89_C2H_TIMEOUT, false, rtwdev, + timeout, false, rtwdev, chip->c2h_ctrl_reg); if (ret) { rtw89_warn(rtwdev, "c2h reg timeout\n"); diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index 775f4e8fbda4d..bc6a9ea9352e0 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -69,6 +69,8 @@ struct rtw89_h2creg_sch_tx_en { #define RTW89_C2HREG_HDR_LEN 2 #define RTW89_H2CREG_HDR_LEN 2 #define RTW89_C2H_TIMEOUT 1000000 +#define RTW89_C2H_TIMEOUT_USB 4000 + struct rtw89_mac_c2h_info { u8 id; u8 content_len; diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index 3c818c4b4653a..3d63f8b2770e2 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -1301,6 +1301,23 @@ void rtw89_mac_notify_wake(struct rtw89_dev *rtwdev) rtw89_mac_send_rpwm(rtwdev, state, true); } +static void rtw89_mac_power_switch_boot_mode(struct rtw89_dev *rtwdev) +{ + u32 boot_mode; + + if (rtwdev->hci.type != RTW89_HCI_TYPE_USB) + return; + + boot_mode = rtw89_read32_mask(rtwdev, R_AX_GPIO_MUXCFG, B_AX_BOOT_MODE); + if (!boot_mode) + return; + + rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_APFN_ONMAC); + rtw89_write32_clr(rtwdev, R_AX_SYS_STATUS1, B_AX_AUTO_WLPON); + rtw89_write32_clr(rtwdev, R_AX_GPIO_MUXCFG, B_AX_BOOT_MODE); + rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST); +} + static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on) { #define PWR_ACT 1 @@ -1310,6 +1327,8 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on) int ret; u8 val; + rtw89_mac_power_switch_boot_mode(rtwdev); + if (on) { cfg_seq = chip->pwr_on_seq; cfg_func = chip->ops->pwr_on_func; diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 30cc6e03c355e..33b2543ee4d23 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -1822,22 +1822,87 @@ static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u return 0; } +static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data) +{ + u16 addr_2lsb = addr & B_AX_DBI_2LSB; + u16 write_addr; + u8 flag; + int ret; + + write_addr = addr & B_AX_DBI_ADDR_MSK; + write_addr |= u16_encode_bits(BIT(addr_2lsb), B_AX_DBI_WREN_MSK); + rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data); + rtw89_write16(rtwdev, R_AX_DBI_FLAG, write_addr); + rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16); + + ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10, + 10 * RTW89_PCI_WR_RETRY_CNT, false, + rtwdev, R_AX_DBI_FLAG + 2); + if (ret) + rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n", + addr); + + return ret; +} + +static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value) +{ + u16 read_addr = addr & B_AX_DBI_ADDR_MSK; + u8 flag; + int ret; + + rtw89_write16(rtwdev, R_AX_DBI_FLAG, read_addr); + rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16); + + ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10, + 10 * RTW89_PCI_WR_RETRY_CNT, false, + rtwdev, R_AX_DBI_FLAG + 2); + if (ret) { + rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n", + addr); + return ret; + } + + read_addr = R_AX_DBI_RDATA + (addr & 3); + *value = rtw89_read8(rtwdev, read_addr); + + return 0; +} + static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr, u8 data) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; struct pci_dev *pdev = rtwpci->pdev; + int ret; + + ret = pci_write_config_byte(pdev, addr, data); + if (!ret) + return 0; - return pci_write_config_byte(pdev, addr, data); + if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) + ret = rtw89_dbi_write8(rtwdev, addr, data); + + return ret; } static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr, u8 *value) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; struct pci_dev *pdev = rtwpci->pdev; + int ret; - return pci_read_config_byte(pdev, addr, value); + ret = pci_read_config_byte(pdev, addr, value); + if (!ret) + return 0; + + if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) + ret = rtw89_dbi_read8(rtwdev, addr, value); + + return ret; } static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr, diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h index 4259b79b138fb..119c0608b91a8 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.h +++ b/drivers/net/wireless/realtek/rtw89/pci.h @@ -42,6 +42,7 @@ #define B_AX_DBI_WFLAG BIT(16) #define B_AX_DBI_WREN_MSK GENMASK(15, 12) #define B_AX_DBI_ADDR_MSK GENMASK(11, 2) +#define B_AX_DBI_2LSB GENMASK(1, 0) #define R_AX_DBI_WDATA 0x1094 #define R_AX_DBI_RDATA 0x1098 diff --git a/drivers/net/wireless/realtek/rtw89/reg.h b/drivers/net/wireless/realtek/rtw89/reg.h index c0aac4d3678a3..ef11625950423 100644 --- a/drivers/net/wireless/realtek/rtw89/reg.h +++ b/drivers/net/wireless/realtek/rtw89/reg.h @@ -172,6 +172,7 @@ #define R_AX_SYS_STATUS1 0x00F4 #define B_AX_SEL_0XC0_MASK GENMASK(17, 16) +#define B_AX_AUTO_WLPON BIT(10) #define B_AX_PAD_HCI_SEL_V2_MASK GENMASK(5, 3) #define MAC_AX_HCI_SEL_SDIO_UART 0 #define MAC_AX_HCI_SEL_MULTI_USB 1 diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c index 9e2328db18656..91f0895d9f540 100644 --- a/drivers/net/wireless/realtek/rtw89/regd.c +++ b/drivers/net/wireless/realtek/rtw89/regd.c @@ -451,6 +451,7 @@ void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); struct rtw89_dev *rtwdev = hw->priv; + wiphy_lock(wiphy); mutex_lock(&rtwdev->mutex); rtw89_leave_ps_mode(rtwdev); @@ -468,6 +469,7 @@ void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request exit: mutex_unlock(&rtwdev->mutex); + wiphy_unlock(wiphy); } static void __rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev) diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index 01b17b8f4ff9d..ae3e02efcc5d7 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -156,9 +156,11 @@ static void ser_state_run(struct rtw89_ser *ser, u8 evt) rtw89_debug(rtwdev, RTW89_DBG_SER, "ser: %s receive %s\n", ser_st_name(ser), ser_ev_name(ser, evt)); + wiphy_lock(rtwdev->hw->wiphy); mutex_lock(&rtwdev->mutex); rtw89_leave_lps(rtwdev); mutex_unlock(&rtwdev->mutex); + wiphy_unlock(rtwdev->hw->wiphy); ser->st_tbl[ser->state].st_func(ser, evt); } @@ -205,7 +207,6 @@ static void rtw89_ser_hdl_work(struct work_struct *work) static int ser_send_msg(struct rtw89_ser *ser, u8 event) { - struct rtw89_dev *rtwdev = container_of(ser, struct rtw89_dev, ser); struct ser_msg *msg = NULL; if (test_bit(RTW89_SER_DRV_STOP_RUN, ser->flags)) @@ -221,7 +222,7 @@ static int ser_send_msg(struct rtw89_ser *ser, u8 event) list_add(&msg->list, &ser->msg_q); spin_unlock_irq(&ser->msg_q_lock); - ieee80211_queue_work(rtwdev->hw, &ser->ser_hdl_work); + schedule_work(&ser->ser_hdl_work); return 0; } @@ -676,9 +677,11 @@ static void ser_l2_reset_st_hdl(struct rtw89_ser *ser, u8 evt) switch (evt) { case SER_EV_STATE_IN: + wiphy_lock(rtwdev->hw->wiphy); mutex_lock(&rtwdev->mutex); ser_l2_reset_st_pre_hdl(ser); mutex_unlock(&rtwdev->mutex); + wiphy_unlock(rtwdev->hw->wiphy); ieee80211_restart_hw(rtwdev->hw); ser_set_alarm(ser, SER_RECFG_TIMEOUT, SER_EV_L2_RECFG_TIMEOUT); diff --git a/drivers/net/wireless/st/cw1200/sta.c b/drivers/net/wireless/st/cw1200/sta.c index 8ef1d06b9bbdd..121d810c8839e 100644 --- a/drivers/net/wireless/st/cw1200/sta.c +++ b/drivers/net/wireless/st/cw1200/sta.c @@ -1290,7 +1290,7 @@ static void cw1200_do_join(struct cw1200_common *priv) rcu_read_lock(); ssidie = ieee80211_bss_get_ie(bss, WLAN_EID_SSID); if (ssidie) { - join.ssid_len = ssidie[1]; + join.ssid_len = min(ssidie[1], IEEE80211_MAX_SSID_LEN); memcpy(join.ssid, &ssidie[2], join.ssid_len); } rcu_read_unlock(); diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c index d86a1bd7aab08..1214e7dcc8124 100644 --- a/drivers/net/wireless/virtual/mac80211_hwsim.c +++ b/drivers/net/wireless/virtual/mac80211_hwsim.c @@ -1201,6 +1201,11 @@ static void mac80211_hwsim_set_tsf(struct ieee80211_hw *hw, /* MLD not supported here */ u32 bcn_int = data->link_data[0].beacon_int; u64 delta = abs(tsf - now); + struct ieee80211_bss_conf *conf; + + conf = link_conf_dereference_protected(vif, data->link_data[0].link_id); + if (conf && !conf->enable_beacon) + return; /* adjust after beaconing with new timestamp at old TBTT */ if (tsf > now) { @@ -6184,14 +6189,15 @@ static struct genl_family hwsim_genl_family __ro_after_init = { .n_mcgrps = ARRAY_SIZE(hwsim_mcgrps), }; -static void remove_user_radios(u32 portid) +static void remove_user_radios(u32 portid, int netgroup) { struct mac80211_hwsim_data *entry, *tmp; LIST_HEAD(list); spin_lock_bh(&hwsim_radio_lock); list_for_each_entry_safe(entry, tmp, &hwsim_radios, list) { - if (entry->destroy_on_close && entry->portid == portid) { + if (entry->destroy_on_close && entry->portid == portid && + entry->netgroup == netgroup) { list_move(&entry->list, &list); rhashtable_remove_fast(&hwsim_radios_rht, &entry->rht, hwsim_rht_params); @@ -6216,7 +6222,7 @@ static int mac80211_hwsim_netlink_notify(struct notifier_block *nb, if (state != NETLINK_URELEASE) return NOTIFY_DONE; - remove_user_radios(notify->portid); + remove_user_radios(notify->portid, hwsim_net_get_netgroup(notify->net)); if (notify->portid == hwsim_net_get_wmediumd(notify->net)) { printk(KERN_INFO "mac80211_hwsim: wmediumd released netlink" diff --git a/drivers/net/wireless/virtual/virt_wifi.c b/drivers/net/wireless/virtual/virt_wifi.c index fb4d95a027fef..2977b30c6d593 100644 --- a/drivers/net/wireless/virtual/virt_wifi.c +++ b/drivers/net/wireless/virtual/virt_wifi.c @@ -277,7 +277,9 @@ static void virt_wifi_connect_complete(struct work_struct *work) priv->is_connected = true; /* Schedules an event that acquires the rtnl lock. */ - cfg80211_connect_result(priv->upperdev, requested_bss, NULL, 0, NULL, 0, + cfg80211_connect_result(priv->upperdev, + priv->is_connected ? fake_router_bssid : NULL, + NULL, 0, NULL, 0, status, GFP_KERNEL); netif_carrier_on(priv->upperdev); } diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_mac.c b/drivers/net/wireless/zydas/zd1211rw/zd_mac.c index 5d534e15a844f..278875c02f41f 100644 --- a/drivers/net/wireless/zydas/zd1211rw/zd_mac.c +++ b/drivers/net/wireless/zydas/zd1211rw/zd_mac.c @@ -583,7 +583,11 @@ void zd_mac_tx_to_dev(struct sk_buff *skb, int error) skb_queue_tail(q, skb); while (skb_queue_len(q) > ZD_MAC_MAX_ACK_WAITERS) { - zd_mac_tx_status(hw, skb_dequeue(q), + skb = skb_dequeue(q); + if (!skb) + break; + + zd_mac_tx_status(hw, skb, mac->ack_pending ? mac->ack_signal : 0, NULL); mac->ack_pending = 0; diff --git a/drivers/net/wwan/t7xx/t7xx_netdev.c b/drivers/net/wwan/t7xx/t7xx_netdev.c index 3ef4a8a4f8fdb..d879424093b78 100644 --- a/drivers/net/wwan/t7xx/t7xx_netdev.c +++ b/drivers/net/wwan/t7xx/t7xx_netdev.c @@ -296,7 +296,7 @@ static int t7xx_ccmni_wwan_newlink(void *ctxt, struct net_device *dev, u32 if_id ccmni->ctlb = ctlb; ccmni->dev = dev; atomic_set(&ccmni->usage, 0); - ctlb->ccmni_inst[if_id] = ccmni; + WRITE_ONCE(ctlb->ccmni_inst[if_id], ccmni); ret = register_netdevice(dev); if (ret) @@ -318,6 +318,7 @@ static void t7xx_ccmni_wwan_dellink(void *ctxt, struct net_device *dev, struct l if (WARN_ON(ctlb->ccmni_inst[if_id] != ccmni)) return; + WRITE_ONCE(ctlb->ccmni_inst[if_id], NULL); unregister_netdevice(dev); } @@ -413,7 +414,7 @@ static void t7xx_ccmni_recv_skb(struct t7xx_ccmni_ctrl *ccmni_ctlb, struct sk_bu skb_cb = T7XX_SKB_CB(skb); netif_id = skb_cb->netif_idx; - ccmni = ccmni_ctlb->ccmni_inst[netif_id]; + ccmni = READ_ONCE(ccmni_ctlb->ccmni_inst[netif_id]); if (!ccmni) { dev_kfree_skb(skb); return; @@ -435,7 +436,7 @@ static void t7xx_ccmni_recv_skb(struct t7xx_ccmni_ctrl *ccmni_ctlb, struct sk_bu static void t7xx_ccmni_queue_tx_irq_notify(struct t7xx_ccmni_ctrl *ctlb, int qno) { - struct t7xx_ccmni *ccmni = ctlb->ccmni_inst[0]; + struct t7xx_ccmni *ccmni = READ_ONCE(ctlb->ccmni_inst[0]); struct netdev_queue *net_queue; if (netif_running(ccmni->dev) && atomic_read(&ccmni->usage) > 0) { @@ -447,7 +448,7 @@ static void t7xx_ccmni_queue_tx_irq_notify(struct t7xx_ccmni_ctrl *ctlb, int qno static void t7xx_ccmni_queue_tx_full_notify(struct t7xx_ccmni_ctrl *ctlb, int qno) { - struct t7xx_ccmni *ccmni = ctlb->ccmni_inst[0]; + struct t7xx_ccmni *ccmni = READ_ONCE(ctlb->ccmni_inst[0]); struct netdev_queue *net_queue; if (atomic_read(&ccmni->usage) > 0) { @@ -465,7 +466,7 @@ static void t7xx_ccmni_queue_state_notify(struct t7xx_pci_dev *t7xx_dev, if (ctlb->md_sta != MD_STATE_READY) return; - if (!ctlb->ccmni_inst[0]) { + if (!READ_ONCE(ctlb->ccmni_inst[0])) { dev_warn(&t7xx_dev->pdev->dev, "No netdev registered yet\n"); return; } diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 0115f8f5b7245..5b59c0ee6c666 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -637,8 +637,6 @@ static int xennet_xdp_xmit_one(struct net_device *dev, tx_stats->packets++; u64_stats_update_end(&tx_stats->syncp); - xennet_tx_buf_gc(queue); - return 0; } @@ -848,9 +846,6 @@ static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev tx_stats->packets++; u64_stats_update_end(&tx_stats->syncp); - /* Note: It is not safe to access skb after xennet_tx_buf_gc()! */ - xennet_tx_buf_gc(queue); - if (!netfront_tx_slot_available(queue)) netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); diff --git a/drivers/nfc/pn544/i2c.c b/drivers/nfc/pn544/i2c.c index 3f6d74832bac3..be566ca7bae94 100644 --- a/drivers/nfc/pn544/i2c.c +++ b/drivers/nfc/pn544/i2c.c @@ -126,8 +126,6 @@ struct pn544_i2c_fw_secure_blob { #define PN544_FW_CMD_RESULT_COMMAND_REJECTED 0xE0 #define PN544_FW_CMD_RESULT_CHUNK_ERROR 0xE6 -#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) - #define PN544_FW_WRITE_BUFFER_MAX_LEN 0x9f7 #define PN544_FW_I2C_MAX_PAYLOAD PN544_HCI_I2C_LLC_MAX_SIZE #define PN544_FW_I2C_WRITE_FRAME_HEADER_LEN 8 diff --git a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c index b640aa0bf45e6..5e79cfce8649c 100644 --- a/drivers/ntb/hw/epf/ntb_hw_epf.c +++ b/drivers/ntb/hw/epf/ntb_hw_epf.c @@ -49,6 +49,7 @@ #define NTB_EPF_COMMAND_TIMEOUT 1000 /* 1 Sec */ enum pci_barno { + NO_BAR = -1, BAR_0, BAR_1, BAR_2, @@ -57,16 +58,26 @@ enum pci_barno { BAR_5, }; +enum epf_ntb_bar { + BAR_CONFIG, + BAR_PEER_SPAD, + BAR_DB, + BAR_MW1, + BAR_MW2, + BAR_MW3, + BAR_MW4, + NTB_BAR_NUM, +}; + +#define NTB_EPF_MAX_MW_COUNT (NTB_BAR_NUM - BAR_MW1) + struct ntb_epf_dev { struct ntb_dev ntb; struct device *dev; /* Mutex to protect providing commands to NTB EPF */ struct mutex cmd_lock; - enum pci_barno ctrl_reg_bar; - enum pci_barno peer_spad_reg_bar; - enum pci_barno db_reg_bar; - enum pci_barno mw_bar; + const enum pci_barno *barno_map; unsigned int mw_count; unsigned int spad_count; @@ -85,17 +96,6 @@ struct ntb_epf_dev { #define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb) -struct ntb_epf_data { - /* BAR that contains both control region and self spad region */ - enum pci_barno ctrl_reg_bar; - /* BAR that contains peer spad region */ - enum pci_barno peer_spad_reg_bar; - /* BAR that contains Doorbell region and Memory window '1' */ - enum pci_barno db_reg_bar; - /* BAR that contains memory windows*/ - enum pci_barno mw_bar; -}; - static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command, u32 argument) { @@ -144,7 +144,7 @@ static int ntb_epf_mw_to_bar(struct ntb_epf_dev *ndev, int idx) return -EINVAL; } - return idx + 2; + return ndev->barno_map[BAR_MW1 + idx]; } static int ntb_epf_mw_count(struct ntb_dev *ntb, int pidx) @@ -413,7 +413,9 @@ static int ntb_epf_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, return -EINVAL; } - bar = idx + ndev->mw_bar; + bar = ntb_epf_mw_to_bar(ndev, idx); + if (bar < 0) + return bar; mw_size = pci_resource_len(ntb->pdev, bar); @@ -455,7 +457,9 @@ static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx, if (idx == 0) offset = readl(ndev->ctrl_reg + NTB_EPF_MW1_OFFSET); - bar = idx + ndev->mw_bar; + bar = ntb_epf_mw_to_bar(ndev, idx); + if (bar < 0) + return bar; if (base) *base = pci_resource_start(ndev->ntb.pdev, bar) + offset; @@ -560,6 +564,11 @@ static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT); ndev->spad_count = readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); + if (ndev->mw_count > NTB_EPF_MAX_MW_COUNT) { + dev_err(dev, "Unsupported MW count: %u\n", ndev->mw_count); + return -EINVAL; + } + return 0; } @@ -596,14 +605,15 @@ static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, dev_warn(&pdev->dev, "Cannot DMA highmem\n"); } - ndev->ctrl_reg = pci_iomap(pdev, ndev->ctrl_reg_bar, 0); + ndev->ctrl_reg = pci_iomap(pdev, ndev->barno_map[BAR_CONFIG], 0); if (!ndev->ctrl_reg) { ret = -EIO; goto err_pci_regions; } - if (ndev->peer_spad_reg_bar) { - ndev->peer_spad_reg = pci_iomap(pdev, ndev->peer_spad_reg_bar, 0); + if (ndev->barno_map[BAR_PEER_SPAD] != ndev->barno_map[BAR_CONFIG]) { + ndev->peer_spad_reg = pci_iomap(pdev, + ndev->barno_map[BAR_PEER_SPAD], 0); if (!ndev->peer_spad_reg) { ret = -EIO; goto err_pci_regions; @@ -614,7 +624,7 @@ static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, ndev->peer_spad_reg = ndev->ctrl_reg + spad_off + spad_sz; } - ndev->db_reg = pci_iomap(pdev, ndev->db_reg_bar, 0); + ndev->db_reg = pci_iomap(pdev, ndev->barno_map[BAR_DB], 0); if (!ndev->db_reg) { ret = -EIO; goto err_pci_regions; @@ -659,12 +669,7 @@ static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev) static int ntb_epf_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - enum pci_barno peer_spad_reg_bar = BAR_1; - enum pci_barno ctrl_reg_bar = BAR_0; - enum pci_barno db_reg_bar = BAR_2; - enum pci_barno mw_bar = BAR_2; struct device *dev = &pdev->dev; - struct ntb_epf_data *data; struct ntb_epf_dev *ndev; int ret; @@ -675,18 +680,10 @@ static int ntb_epf_pci_probe(struct pci_dev *pdev, if (!ndev) return -ENOMEM; - data = (struct ntb_epf_data *)id->driver_data; - if (data) { - peer_spad_reg_bar = data->peer_spad_reg_bar; - ctrl_reg_bar = data->ctrl_reg_bar; - db_reg_bar = data->db_reg_bar; - mw_bar = data->mw_bar; - } + ndev->barno_map = (const enum pci_barno *)id->driver_data; + if (!ndev->barno_map) + return -EINVAL; - ndev->peer_spad_reg_bar = peer_spad_reg_bar; - ndev->ctrl_reg_bar = ctrl_reg_bar; - ndev->db_reg_bar = db_reg_bar; - ndev->mw_bar = mw_bar; ndev->dev = dev; ntb_epf_init_struct(ndev, pdev); @@ -730,30 +727,36 @@ static void ntb_epf_pci_remove(struct pci_dev *pdev) ntb_epf_deinit_pci(ndev); } -static const struct ntb_epf_data j721e_data = { - .ctrl_reg_bar = BAR_0, - .peer_spad_reg_bar = BAR_1, - .db_reg_bar = BAR_2, - .mw_bar = BAR_2, +static const enum pci_barno j721e_map[NTB_BAR_NUM] = { + [BAR_CONFIG] = BAR_0, + [BAR_PEER_SPAD] = BAR_1, + [BAR_DB] = BAR_2, + [BAR_MW1] = BAR_2, + [BAR_MW2] = BAR_3, + [BAR_MW3] = BAR_4, + [BAR_MW4] = BAR_5 }; -static const struct ntb_epf_data mx8_data = { - .ctrl_reg_bar = BAR_0, - .peer_spad_reg_bar = BAR_0, - .db_reg_bar = BAR_2, - .mw_bar = BAR_4, +static const enum pci_barno mx8_map[NTB_BAR_NUM] = { + [BAR_CONFIG] = BAR_0, + [BAR_PEER_SPAD] = BAR_0, + [BAR_DB] = BAR_2, + [BAR_MW1] = BAR_4, + [BAR_MW2] = BAR_5, + [BAR_MW3] = NO_BAR, + [BAR_MW4] = NO_BAR }; static const struct pci_device_id ntb_epf_pci_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E), .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, - .driver_data = (kernel_ulong_t)&j721e_data, + .driver_data = (kernel_ulong_t)j721e_map, }, { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x0809), .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, - .driver_data = (kernel_ulong_t)&mx8_data, + .driver_data = (kernel_ulong_t)mx8_map, }, { }, }; diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c index 082253a3a9560..04f4a049599a1 100644 --- a/drivers/nvdimm/label.c +++ b/drivers/nvdimm/label.c @@ -442,7 +442,8 @@ int nd_label_data_init(struct nvdimm_drvdata *ndd) if (ndd->data) return 0; - if (ndd->nsarea.status || ndd->nsarea.max_xfer == 0) { + if (ndd->nsarea.status || ndd->nsarea.max_xfer == 0 || + ndd->nsarea.config_size == 0) { dev_dbg(ndd->dev, "failed to init config data area: (%u:%u)\n", ndd->nsarea.max_xfer, ndd->nsarea.config_size); return -ENXIO; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index c6b0637e61deb..78d00e25a1790 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -689,6 +689,10 @@ blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl, !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) && !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH)) return BLK_STS_RESOURCE; + + if (!(rq->rq_flags & RQF_DONTPREP)) + nvme_clear_nvme_request(rq); + return nvme_host_path_error(rq); } EXPORT_SYMBOL_GPL(nvme_fail_nonready_command); @@ -3596,7 +3600,7 @@ static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns) return; } } - list_add(&ns->list, &ns->ctrl->namespaces); + list_add_rcu(&ns->list, &ns->ctrl->namespaces); } static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) @@ -4156,7 +4160,8 @@ static void nvme_fw_act_work(struct work_struct *work) msleep(100); } - if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) + if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING) || + !nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) return; nvme_unquiesce_io_queues(ctrl); @@ -4409,8 +4414,14 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl) * checking that they started once before, hence are reconnecting back. */ if (test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) && - nvme_discovery_ctrl(ctrl)) + nvme_discovery_ctrl(ctrl)) { + if (!ctrl->kato) { + nvme_stop_keep_alive(ctrl); + ctrl->kato = NVME_DEFAULT_KATO; + nvme_start_keep_alive(ctrl); + } nvme_change_uevent(ctrl, "NVME_EVENT=rediscover"); + } if (ctrl->queue_count > 1) { nvme_queue_scan(ctrl); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 3e0da2422b334..37fede155b920 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2349,17 +2349,11 @@ nvme_fc_ctrl_free(struct kref *ref) container_of(ref, struct nvme_fc_ctrl, ref); unsigned long flags; - if (ctrl->ctrl.tagset) - nvme_remove_io_tag_set(&ctrl->ctrl); - /* remove from rport list */ spin_lock_irqsave(&ctrl->rport->lock, flags); list_del(&ctrl->ctrl_list); spin_unlock_irqrestore(&ctrl->rport->lock, flags); - nvme_unquiesce_admin_queue(&ctrl->ctrl); - nvme_remove_admin_tag_set(&ctrl->ctrl); - kfree(ctrl->queues); put_device(ctrl->dev); @@ -3024,11 +3018,17 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) ++ctrl->ctrl.nr_reconnects; - if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) + spin_lock_irqsave(&ctrl->rport->lock, flags); + if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) { + spin_unlock_irqrestore(&ctrl->rport->lock, flags); return -ENODEV; + } - if (nvme_fc_ctlr_active_on_rport(ctrl)) + if (nvme_fc_ctlr_active_on_rport(ctrl)) { + spin_unlock_irqrestore(&ctrl->rport->lock, flags); return -ENOTUNIQ; + } + spin_unlock_irqrestore(&ctrl->rport->lock, flags); dev_info(ctrl->ctrl.device, "NVME-FC{%d}: create association : host wwpn 0x%016llx " @@ -3242,13 +3242,20 @@ nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl) { struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); - cancel_work_sync(&ctrl->ioerr_work); cancel_delayed_work_sync(&ctrl->connect_work); + /* * kill the association on the link side. this will block * waiting for io to terminate */ nvme_fc_delete_association(ctrl); + cancel_work_sync(&ctrl->ioerr_work); + + if (ctrl->ctrl.tagset) + nvme_remove_io_tag_set(&ctrl->ctrl); + + nvme_unquiesce_admin_queue(&ctrl->ctrl); + nvme_remove_admin_tag_set(&ctrl->ctrl); } static void diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 4ce31f9f06947..5cf050e562b73 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -526,16 +526,14 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req, pdu->u.result = le64_to_cpu(nvme_req(req)->result.u64); /* - * For iopoll, complete it directly. - * Otherwise, move the completion to task work. + * IOPOLL could potentially complete this request directly, but + * if multiple rings are polling on the same queue, then it's possible + * for one ring to find completions for another ring. Punting the + * completion via task_work will always direct it to the right + * location, rather than potentially complete requests for ringA + * under iopoll invocations from ringB. */ - if (blk_rq_is_poll(req)) { - WRITE_ONCE(ioucmd->cookie, NULL); - nvme_uring_task_cb(ioucmd, IO_URING_F_UNLOCKED); - } else { - io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb); - } - + io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb); return RQ_END_IO_FREE; } diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 119afdfe4b91e..578f4f29eacfe 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -131,12 +131,14 @@ void nvme_mpath_start_request(struct request *rq) struct nvme_ns *ns = rq->q->queuedata; struct gendisk *disk = ns->head->disk; - if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) { + if ((READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) && + !(nvme_req(rq)->flags & NVME_MPATH_CNT_ACTIVE)) { atomic_inc(&ns->ctrl->nr_active); nvme_req(rq)->flags |= NVME_MPATH_CNT_ACTIVE; } - if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq)) + if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq) || + (nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) return; nvme_req(rq)->flags |= NVME_MPATH_IO_STATS; @@ -684,7 +686,7 @@ static void nvme_mpath_set_live(struct nvme_ns *ns) return; } nvme_add_ns_head_cdev(head); - kblockd_schedule_work(&head->partition_scan_work); + queue_work(nvme_wq, &head->partition_scan_work); } mutex_lock(&head->lock); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index a763df0200ab4..053385c84bf84 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -386,7 +386,7 @@ static bool nvme_dbbuf_update_and_check_event(u16 value, __le32 *dbbuf_db, * as it only leads to a small amount of wasted memory for the lifetime of * the I/O. */ -static int nvme_pci_npages_prp(void) +static __always_inline int nvme_pci_npages_prp(void) { unsigned max_bytes = (NVME_MAX_KB_SZ * 1024) + NVME_CTRL_PAGE_SIZE; unsigned nprps = DIV_ROUND_UP(max_bytes, NVME_CTRL_PAGE_SIZE); @@ -1107,7 +1107,9 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq) WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags)); disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); + spin_lock(&nvmeq->cq_poll_lock); nvme_poll_cq(nvmeq, NULL); + spin_unlock(&nvmeq->cq_poll_lock); enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); } @@ -1753,8 +1755,28 @@ static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) * might be pointing at! */ result = nvme_disable_ctrl(&dev->ctrl, false); - if (result < 0) - return result; + if (result < 0) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + + /* + * The NVMe Controller Reset method did not get an expected + * CSTS.RDY transition, so something with the device appears to + * be stuck. Use the lower level and bigger hammer PCIe + * Function Level Reset to attempt restoring the device to its + * initial state, and try again. + */ + result = pcie_reset_flr(pdev, false); + if (result < 0) + return result; + + pci_restore_state(pdev); + result = nvme_disable_ctrl(&dev->ctrl, false); + if (result < 0) + return result; + + dev_info(dev->ctrl.device, + "controller reset completed after pcie flr\n"); + } result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH); if (result) @@ -2964,10 +2986,12 @@ static unsigned long check_vendor_combination_bug(struct pci_dev *pdev) * Exclude Samsung 990 Evo from NVME_QUIRK_SIMPLE_SUSPEND * because of high power consumption (> 2 Watt) in s2idle * sleep. Only some boards with Intel CPU are affected. + * (Note for testing: Samsung 990 Evo Plus has same PCI ID) */ if (dmi_match(DMI_BOARD_NAME, "DN50Z-140HC-YD") || dmi_match(DMI_BOARD_NAME, "GMxPXxx") || dmi_match(DMI_BOARD_NAME, "GXxMRXx") || + dmi_match(DMI_BOARD_NAME, "NS5X_NS7XAU") || dmi_match(DMI_BOARD_NAME, "PH4PG31") || dmi_match(DMI_BOARD_NAME, "PH4PRX1_PH6PRX1") || dmi_match(DMI_BOARD_NAME, "PH6PG01_PH6PG71")) @@ -3377,7 +3401,7 @@ static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev) dev_info(dev->ctrl.device, "restart after slot reset\n"); pci_restore_state(pdev); - if (!nvme_try_sched_reset(&dev->ctrl)) + if (nvme_try_sched_reset(&dev->ctrl)) nvme_unquiesce_io_queues(&dev->ctrl); return PCI_ERS_RESULT_RECOVERED; } @@ -3427,6 +3451,9 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1217, 0x8760), /* O2 Micro 64GB Steam Deck */ .driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, }, + { PCI_DEVICE(0x126f, 0x1001), /* Silicon Motion generic */ + .driver_data = NVME_QUIRK_NO_DEEPEST_PS | + NVME_QUIRK_IGNORE_DEV_SUBNQN, }, { PCI_DEVICE(0x126f, 0x2262), /* Silicon Motion generic */ .driver_data = NVME_QUIRK_NO_DEEPEST_PS | NVME_QUIRK_BOGUS_NID, }, @@ -3450,6 +3477,9 @@ static const struct pci_device_id nvme_id_table[] = { NVME_QUIRK_IGNORE_DEV_SUBNQN, }, { PCI_DEVICE(0x15b7, 0x5008), /* Sandisk SN530 */ .driver_data = NVME_QUIRK_BROKEN_MSI }, + { PCI_DEVICE(0x15b7, 0x5009), /* Sandisk SN550 */ + .driver_data = NVME_QUIRK_BROKEN_MSI | + NVME_QUIRK_NO_DEEPEST_PS }, { PCI_DEVICE(0x1987, 0x5012), /* Phison E12 */ .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1987, 0x5016), /* Phison E16 */ @@ -3533,6 +3563,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(0x1e49, 0x0041), /* ZHITAI TiPro7000 NVMe SSD */ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, + { PCI_DEVICE(0x025e, 0xf1ac), /* SOLIDIGM P44 pro SSDPFKKW020X7 */ + .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */ .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1d97, 0x2263), /* Lexar NM610 */ diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 84db7f4f861cb..5b76670f34be2 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -1710,7 +1710,7 @@ static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) cancel_work_sync(&queue->io_work); } -static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) +static void nvme_tcp_stop_queue_nowait(struct nvme_ctrl *nctrl, int qid) { struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); struct nvme_tcp_queue *queue = &ctrl->queues[qid]; @@ -1724,6 +1724,31 @@ static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) mutex_unlock(&queue->queue_lock); } +static void nvme_tcp_wait_queue(struct nvme_ctrl *nctrl, int qid) +{ + struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); + struct nvme_tcp_queue *queue = &ctrl->queues[qid]; + int timeout = 100; + + while (timeout > 0) { + if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags) || + !sk_wmem_alloc_get(queue->sock->sk)) + return; + msleep(2); + timeout -= 2; + } + dev_warn(nctrl->device, + "qid %d: timeout draining sock wmem allocation expired\n", + qid); +} + +static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) +{ + nvme_tcp_stop_queue_nowait(nctrl, qid); + nvme_tcp_wait_queue(nctrl, qid); +} + + static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue) { write_lock_bh(&queue->sock->sk->sk_callback_lock); @@ -1790,7 +1815,9 @@ static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) int i; for (i = 1; i < ctrl->queue_count; i++) - nvme_tcp_stop_queue(ctrl, i); + nvme_tcp_stop_queue_nowait(ctrl, i); + for (i = 1; i < ctrl->queue_count; i++) + nvme_tcp_wait_queue(ctrl, i); } static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl, diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 570c58d2b5a58..188b9f1bdaca1 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -54,6 +54,8 @@ struct nvmet_fc_ls_req_op { /* for an LS RQST XMT */ int ls_error; struct list_head lsreq_list; /* tgtport->ls_req_list */ bool req_queued; + + struct work_struct put_work; }; @@ -111,8 +113,6 @@ struct nvmet_fc_tgtport { struct nvmet_fc_port_entry *pe; struct kref ref; u32 max_sg_cnt; - - struct work_struct put_work; }; struct nvmet_fc_port_entry { @@ -236,12 +236,13 @@ static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc); static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue); static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue); static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport); -static void nvmet_fc_put_tgtport_work(struct work_struct *work) +static void nvmet_fc_put_lsop_work(struct work_struct *work) { - struct nvmet_fc_tgtport *tgtport = - container_of(work, struct nvmet_fc_tgtport, put_work); + struct nvmet_fc_ls_req_op *lsop = + container_of(work, struct nvmet_fc_ls_req_op, put_work); - nvmet_fc_tgtport_put(tgtport); + nvmet_fc_tgtport_put(lsop->tgtport); + kfree(lsop); } static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport); static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, @@ -368,7 +369,7 @@ __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop) DMA_BIDIRECTIONAL); out_putwork: - queue_work(nvmet_wq, &tgtport->put_work); + queue_work(nvmet_wq, &lsop->put_work); } static int @@ -389,6 +390,7 @@ __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport, lsreq->done = done; lsop->req_queued = false; INIT_LIST_HEAD(&lsop->lsreq_list); + INIT_WORK(&lsop->put_work, nvmet_fc_put_lsop_work); lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr, lsreq->rqstlen + lsreq->rsplen, @@ -448,8 +450,6 @@ nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) __nvmet_fc_finish_ls_req(lsop); /* fc-nvme target doesn't care about success or failure of cmd */ - - kfree(lsop); } /* @@ -1090,6 +1090,14 @@ nvmet_fc_delete_assoc_work(struct work_struct *work) static void nvmet_fc_schedule_delete_assoc(struct nvmet_fc_tgt_assoc *assoc) { + int terminating; + + terminating = atomic_xchg(&assoc->terminating, 1); + + /* if already terminating, do nothing */ + if (terminating) + return; + nvmet_fc_tgtport_get(assoc->tgtport); if (!queue_work(nvmet_wq, &assoc->del_work)) nvmet_fc_tgtport_put(assoc->tgtport); @@ -1209,13 +1217,7 @@ nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) { struct nvmet_fc_tgtport *tgtport = assoc->tgtport; unsigned long flags; - int i, terminating; - - terminating = atomic_xchg(&assoc->terminating, 1); - - /* if already terminating, do nothing */ - if (terminating) - return; + int i; spin_lock_irqsave(&tgtport->lock, flags); list_del_rcu(&assoc->a_list); @@ -1407,7 +1409,6 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, kref_init(&newrec->ref); ida_init(&newrec->assoc_cnt); newrec->max_sg_cnt = template->max_sgl_segments; - INIT_WORK(&newrec->put_work, nvmet_fc_put_tgtport_work); ret = nvmet_fc_alloc_ls_iodlist(newrec); if (ret) { diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index 4b35bdcac185f..aeeb7455fc2e7 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -613,12 +613,13 @@ fcloop_fcp_recv_work(struct work_struct *work) { struct fcloop_fcpreq *tfcp_req = container_of(work, struct fcloop_fcpreq, fcp_rcv_work); - struct nvmefc_fcp_req *fcpreq = tfcp_req->fcpreq; + struct nvmefc_fcp_req *fcpreq; unsigned long flags; int ret = 0; bool aborted = false; spin_lock_irqsave(&tfcp_req->reqlock, flags); + fcpreq = tfcp_req->fcpreq; switch (tfcp_req->inistate) { case INI_IO_START: tfcp_req->inistate = INI_IO_ACTIVE; @@ -633,16 +634,19 @@ fcloop_fcp_recv_work(struct work_struct *work) } spin_unlock_irqrestore(&tfcp_req->reqlock, flags); - if (unlikely(aborted)) - ret = -ECANCELED; - else { - if (likely(!check_for_drop(tfcp_req))) - ret = nvmet_fc_rcv_fcp_req(tfcp_req->tport->targetport, - &tfcp_req->tgt_fcp_req, - fcpreq->cmdaddr, fcpreq->cmdlen); - else - pr_info("%s: dropped command ********\n", __func__); + if (unlikely(aborted)) { + /* the abort handler will call fcloop_call_host_done */ + return; + } + + if (unlikely(check_for_drop(tfcp_req))) { + pr_info("%s: dropped command ********\n", __func__); + return; } + + ret = nvmet_fc_rcv_fcp_req(tfcp_req->tport->targetport, + &tfcp_req->tgt_fcp_req, + fcpreq->cmdaddr, fcpreq->cmdlen); if (ret) fcloop_call_host_done(fcpreq, tfcp_req, ret); } @@ -657,9 +661,10 @@ fcloop_fcp_abort_recv_work(struct work_struct *work) unsigned long flags; spin_lock_irqsave(&tfcp_req->reqlock, flags); - fcpreq = tfcp_req->fcpreq; switch (tfcp_req->inistate) { case INI_IO_ABORTED: + fcpreq = tfcp_req->fcpreq; + tfcp_req->fcpreq = NULL; break; case INI_IO_COMPLETED: completed = true; @@ -681,10 +686,6 @@ fcloop_fcp_abort_recv_work(struct work_struct *work) nvmet_fc_rcv_fcp_abort(tfcp_req->tport->targetport, &tfcp_req->tgt_fcp_req); - spin_lock_irqsave(&tfcp_req->reqlock, flags); - tfcp_req->fcpreq = NULL; - spin_unlock_irqrestore(&tfcp_req->reqlock, flags); - fcloop_call_host_done(fcpreq, tfcp_req, -ECANCELED); /* call_host_done releases reference for abort downcall */ } diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index a0af659a4c4a2..6a539c3b8b530 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1456,6 +1456,9 @@ static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue) { struct socket *sock = queue->sock; + if (!queue->state_change) + return; + write_lock_bh(&sock->sk->sk_callback_lock); sock->sk->sk_data_ready = queue->data_ready; sock->sk->sk_state_change = queue->state_change; diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index fd11d3825cf85..32af356be7d6d 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -478,9 +478,11 @@ static int nvmem_cell_info_to_nvmem_cell_entry_nodup(struct nvmem_device *nvmem, cell->nbits = info->nbits; cell->np = info->np; - if (cell->nbits) + if (cell->nbits) { cell->bytes = DIV_ROUND_UP(cell->nbits + cell->bit_offset, BITS_PER_BYTE); + cell->raw_len = ALIGN(cell->bytes, nvmem->word_size); + } if (!IS_ALIGNED(cell->offset, nvmem->stride)) { dev_err(&nvmem->dev, @@ -489,6 +491,18 @@ static int nvmem_cell_info_to_nvmem_cell_entry_nodup(struct nvmem_device *nvmem, return -EINVAL; } + if (!IS_ALIGNED(cell->raw_len, nvmem->word_size)) { + dev_err(&nvmem->dev, + "cell %s raw len %zd unaligned to nvmem word size %d\n", + cell->name ?: "", cell->raw_len, + nvmem->word_size); + + if (info->raw_len) + return -EINVAL; + + cell->raw_len = ALIGN(cell->raw_len, nvmem->word_size); + } + return 0; } diff --git a/drivers/nvmem/imx-ocotp.c b/drivers/nvmem/imx-ocotp.c index f1e202efaa497..c582898179d30 100644 --- a/drivers/nvmem/imx-ocotp.c +++ b/drivers/nvmem/imx-ocotp.c @@ -23,6 +23,7 @@ #include #include #include +#include /* ETH_ALEN */ #define IMX_OCOTP_OFFSET_B0W0 0x400 /* Offset from base address of the * OTP Bank0 Word0 @@ -227,9 +228,11 @@ static int imx_ocotp_cell_pp(void *context, const char *id, int index, int i; /* Deal with some post processing of nvmem cell data */ - if (id && !strcmp(id, "mac-address")) + if (id && !strcmp(id, "mac-address")) { + bytes = min(bytes, ETH_ALEN); for (i = 0; i < bytes / 2; i++) swap(buf[i], buf[bytes - i - 1]); + } return 0; } diff --git a/drivers/nvmem/rockchip-otp.c b/drivers/nvmem/rockchip-otp.c index 7107d68a2f8c7..c6684ab14e742 100644 --- a/drivers/nvmem/rockchip-otp.c +++ b/drivers/nvmem/rockchip-otp.c @@ -59,7 +59,6 @@ #define RK3588_OTPC_AUTO_EN 0x08 #define RK3588_OTPC_INT_ST 0x84 #define RK3588_OTPC_DOUT0 0x20 -#define RK3588_NO_SECURE_OFFSET 0x300 #define RK3588_NBYTES 4 #define RK3588_BURST_NUM 1 #define RK3588_BURST_SHIFT 8 @@ -69,6 +68,7 @@ struct rockchip_data { int size; + int read_offset; const char * const *clks; int num_clks; nvmem_reg_read_t reg_read; @@ -196,7 +196,7 @@ static int rk3588_otp_read(void *context, unsigned int offset, addr_start = round_down(offset, RK3588_NBYTES) / RK3588_NBYTES; addr_end = round_up(offset + bytes, RK3588_NBYTES) / RK3588_NBYTES; addr_len = addr_end - addr_start; - addr_start += RK3588_NO_SECURE_OFFSET; + addr_start += otp->data->read_offset / RK3588_NBYTES; buf = kzalloc(array_size(addr_len, RK3588_NBYTES), GFP_KERNEL); if (!buf) @@ -273,12 +273,21 @@ static const struct rockchip_data px30_data = { .reg_read = px30_otp_read, }; +static const struct rockchip_data rk3576_data = { + .size = 0x100, + .read_offset = 0x700, + .clks = px30_otp_clocks, + .num_clks = ARRAY_SIZE(px30_otp_clocks), + .reg_read = rk3588_otp_read, +}; + static const char * const rk3588_otp_clocks[] = { "otp", "apb_pclk", "phy", "arb", }; static const struct rockchip_data rk3588_data = { .size = 0x400, + .read_offset = 0xc00, .clks = rk3588_otp_clocks, .num_clks = ARRAY_SIZE(rk3588_otp_clocks), .reg_read = rk3588_otp_read, @@ -293,6 +302,10 @@ static const struct of_device_id rockchip_otp_match[] = { .compatible = "rockchip,rk3308-otp", .data = &px30_data, }, + { + .compatible = "rockchip,rk3576-otp", + .data = &rk3576_data, + }, { .compatible = "rockchip,rk3588-otp", .data = &rk3588_data, diff --git a/drivers/nvmem/u-boot-env.c b/drivers/nvmem/u-boot-env.c index adabbfdad6fb6..8712d3709a268 100644 --- a/drivers/nvmem/u-boot-env.c +++ b/drivers/nvmem/u-boot-env.c @@ -132,7 +132,7 @@ static int u_boot_env_parse(struct u_boot_env *priv) size_t crc32_data_offset; size_t crc32_data_len; size_t crc32_offset; - __le32 *crc32_addr; + uint32_t *crc32_addr; size_t data_offset; size_t data_len; size_t dev_size; @@ -183,8 +183,8 @@ static int u_boot_env_parse(struct u_boot_env *priv) goto err_kfree; } - crc32_addr = (__le32 *)(buf + crc32_offset); - crc32 = le32_to_cpu(*crc32_addr); + crc32_addr = (uint32_t *)(buf + crc32_offset); + crc32 = *crc32_addr; crc32_data_len = dev_size - crc32_data_offset; data_len = dev_size - data_offset; diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index 4d57a4e341054..18393800546c1 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -306,15 +306,20 @@ int of_detach_node(struct device_node *np) } EXPORT_SYMBOL_GPL(of_detach_node); +void __of_prop_free(struct property *prop) +{ + kfree(prop->name); + kfree(prop->value); + kfree(prop); +} + static void property_list_free(struct property *prop_list) { struct property *prop, *next; for (prop = prop_list; prop != NULL; prop = next) { next = prop->next; - kfree(prop->name); - kfree(prop->value); - kfree(prop); + __of_prop_free(prop); } } @@ -427,9 +432,7 @@ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags) return new; err_free: - kfree(new->name); - kfree(new->value); - kfree(new); + __of_prop_free(new); return NULL; } @@ -471,9 +474,7 @@ struct device_node *__of_node_dup(const struct device_node *np, if (!new_pp) goto err_prop; if (__of_add_property(node, new_pp)) { - kfree(new_pp->name); - kfree(new_pp->value); - kfree(new_pp); + __of_prop_free(new_pp); goto err_prop; } } @@ -934,12 +935,14 @@ static int of_changeset_add_prop_helper(struct of_changeset *ocs, ret = of_changeset_add_property(ocs, np, new_pp); if (ret) { - kfree(new_pp->name); - kfree(new_pp->value); - kfree(new_pp); + __of_prop_free(new_pp); + return ret; } - return ret; + new_pp->next = np->deadprops; + np->deadprops = new_pp; + + return 0; } /** diff --git a/drivers/of/of_private.h b/drivers/of/of_private.h index 21f8f5e80917d..73b55f4f84a3c 100644 --- a/drivers/of/of_private.h +++ b/drivers/of/of_private.h @@ -123,6 +123,7 @@ extern void *__unflatten_device_tree(const void *blob, * own the devtree lock or work on detached trees only. */ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags); +void __of_prop_free(struct property *prop); struct device_node *__of_node_dup(const struct device_node *np, const char *full_name); diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index a9a292d6d59b2..dc13299586414 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -262,9 +262,7 @@ static struct property *dup_and_fixup_symbol_prop( return new_prop; err_free_new_prop: - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); + __of_prop_free(new_prop); err_free_target_path: kfree(target_path); @@ -361,11 +359,8 @@ static int add_changeset_property(struct overlay_changeset *ovcs, pr_err("WARNING: memory leak will occur if overlay removed, property: %pOF/%s\n", target->np, new_prop->name); - if (ret) { - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); - } + if (ret) + __of_prop_free(new_prop); return ret; } diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 3b22c36bfb0b7..aae4e8ef9e365 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -800,15 +800,11 @@ static void __init of_unittest_property_copy(void) new = __of_prop_dup(&p1, GFP_KERNEL); unittest(new && propcmp(&p1, new), "empty property didn't copy correctly\n"); - kfree(new->value); - kfree(new->name); - kfree(new); + __of_prop_free(new); new = __of_prop_dup(&p2, GFP_KERNEL); unittest(new && propcmp(&p2, new), "non-empty property didn't copy correctly\n"); - kfree(new->value); - kfree(new->name); - kfree(new); + __of_prop_free(new); #endif } @@ -3665,9 +3661,7 @@ static __init void of_unittest_overlay_high_level(void) goto err_unlock; } if (__of_add_property(of_symbols, new_prop)) { - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); + __of_prop_free(new_prop); /* "name" auto-generated by unflatten */ if (!strcmp(prop->name, "name")) continue; @@ -3963,6 +3957,7 @@ static int of_unittest_pci_node_verify(struct pci_dev *pdev, bool add) unittest(!np, "Child device tree node is not removed\n"); child_dev = device_find_any_child(&pdev->dev); unittest(!child_dev, "Child device is not removed\n"); + put_device(child_dev); } failed: diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index e9ae66cc4189b..a3927daebeb02 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -180,6 +180,12 @@ config PCI_P2PDMA P2P DMA transactions must be between devices behind the same root port. + Enabling this option will reduce the entropy of x86 KASLR memory + regions. For example - on a 46 bit system, the entropy goes down + from 16 bits to 15 bits. The actual reduction in entropy depends + on the physical address bits, on processor features, kernel config + (5 level page table) and physical memory present on the system. + If unsure, say N. config PCI_LABEL diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index f76a358e2b5b6..753a4c615781f 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -48,6 +48,7 @@ enum link_status { #define J721E_MODE_RC BIT(7) #define LANE_COUNT(n) ((n) << 8) +#define ACSPCIE_PAD_DISABLE_MASK GENMASK(1, 0) #define GENERATION_SEL_MASK GENMASK(1, 0) struct j721e_pcie { @@ -225,6 +226,36 @@ static int j721e_pcie_set_lane_count(struct j721e_pcie *pcie, return ret; } +static int j721e_enable_acspcie_refclk(struct j721e_pcie *pcie, + struct regmap *syscon) +{ + struct device *dev = pcie->cdns_pcie->dev; + struct device_node *node = dev->of_node; + u32 mask = ACSPCIE_PAD_DISABLE_MASK; + struct of_phandle_args args; + u32 val; + int ret; + + ret = of_parse_phandle_with_fixed_args(node, + "ti,syscon-acspcie-proxy-ctrl", + 1, 0, &args); + if (ret) { + dev_err(dev, + "ti,syscon-acspcie-proxy-ctrl has invalid arguments\n"); + return ret; + } + + /* Clear PAD IO disable bits to enable refclk output */ + val = ~(args.args[0]); + ret = regmap_update_bits(syscon, 0, mask, val); + if (ret) { + dev_err(dev, "failed to enable ACSPCIE refclk: %d\n", ret); + return ret; + } + + return 0; +} + static int j721e_pcie_ctrl_init(struct j721e_pcie *pcie) { struct device *dev = pcie->cdns_pcie->dev; @@ -246,6 +277,25 @@ static int j721e_pcie_ctrl_init(struct j721e_pcie *pcie) if (!ret) offset = args.args[0]; + /* + * The PCIe Controller's registers have different "reset-values" + * depending on the "strap" settings programmed into the PCIEn_CTRL + * register within the CTRL_MMR memory-mapped register space. + * The registers latch onto a "reset-value" based on the "strap" + * settings sampled after the PCIe Controller is powered on. + * To ensure that the "reset-values" are sampled accurately, power + * off the PCIe Controller before programming the "strap" settings + * and power it on after that. The runtime PM APIs namely + * pm_runtime_put_sync() and pm_runtime_get_sync() will decrement and + * increment the usage counter respectively, causing GENPD to power off + * and power on the PCIe Controller. + */ + ret = pm_runtime_put_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to power off PCIe Controller\n"); + return ret; + } + ret = j721e_pcie_set_mode(pcie, syscon, offset); if (ret < 0) { dev_err(dev, "Failed to set pci mode\n"); @@ -264,7 +314,19 @@ static int j721e_pcie_ctrl_init(struct j721e_pcie *pcie) return ret; } - return 0; + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to power on PCIe Controller\n"); + return ret; + } + + /* Enable ACSPCIE refclk output if the optional property exists */ + syscon = syscon_regmap_lookup_by_phandle_optional(node, + "ti,syscon-acspcie-proxy-ctrl"); + if (!syscon) + return 0; + + return j721e_enable_acspcie_refclk(pcie, syscon); } static int cdns_ti_pcie_config_read(struct pci_bus *bus, unsigned int devfn, diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c index a87dab9abba26..08fb16918043f 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -294,13 +294,14 @@ static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u8 vfn, struct cdns_pcie *pcie = &ep->pcie; u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET; u32 val, reg; + u16 actual_interrupts = interrupts + 1; fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); reg = cap + PCI_MSIX_FLAGS; val = cdns_pcie_ep_fn_readw(pcie, fn, reg); val &= ~PCI_MSIX_FLAGS_QSIZE; - val |= interrupts; + val |= interrupts; /* 0's based value */ cdns_pcie_ep_fn_writew(pcie, fn, reg, val); /* Set MSIX BAR and offset */ @@ -310,7 +311,7 @@ static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u8 vfn, /* Set PBA BAR and offset. BAR must match MSIX BAR */ reg = cap + PCI_MSIX_PBA; - val = (offset + (interrupts * PCI_MSIX_ENTRY_SIZE)) | bir; + val = (offset + (actual_interrupts * PCI_MSIX_ENTRY_SIZE)) | bir; cdns_pcie_ep_fn_writel(pcie, fn, reg, val); return 0; diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index 8af95e9da7cec..675b7ea6ff784 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -452,7 +452,7 @@ static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1); cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1); - if (pcie->ops->cpu_addr_fixup) + if (pcie->ops && pcie->ops->cpu_addr_fixup) cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) | @@ -570,14 +570,5 @@ int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) if (!bridge->ops) bridge->ops = &cdns_pcie_host_ops; - ret = pci_host_probe(bridge); - if (ret < 0) - goto err_init; - - return 0; - - err_init: - pm_runtime_put_sync(dev); - - return ret; + return pci_host_probe(bridge); } diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c index 4251fac5e3106..a1b66dbfc10f8 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.c +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -90,7 +90,7 @@ void cdns_pcie_set_outbound_region(struct cdns_pcie *pcie, u8 busnr, u8 fn, cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(r), desc1); /* Set the CPU address */ - if (pcie->ops->cpu_addr_fixup) + if (pcie->ops && pcie->ops->cpu_addr_fixup) cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(nbits) | @@ -120,7 +120,7 @@ void cdns_pcie_set_outbound_region_for_normal_msg(struct cdns_pcie *pcie, } /* Set the CPU address */ - if (pcie->ops->cpu_addr_fixup) + if (pcie->ops && pcie->ops->cpu_addr_fixup) cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(17) | diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 9efb71cbe6996..d2c310b7fc99b 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -494,7 +494,7 @@ static inline u32 cdns_pcie_ep_fn_readl(struct cdns_pcie *pcie, u8 fn, u32 reg) static inline int cdns_pcie_start_link(struct cdns_pcie *pcie) { - if (pcie->ops->start_link) + if (pcie->ops && pcie->ops->start_link) return pcie->ops->start_link(pcie); return 0; @@ -502,13 +502,13 @@ static inline int cdns_pcie_start_link(struct cdns_pcie *pcie) static inline void cdns_pcie_stop_link(struct cdns_pcie *pcie) { - if (pcie->ops->stop_link) + if (pcie->ops && pcie->ops->stop_link) pcie->ops->stop_link(pcie); } static inline bool cdns_pcie_link_up(struct cdns_pcie *pcie) { - if (pcie->ops->link_up) + if (pcie->ops && pcie->ops->link_up) return pcie->ops->link_up(pcie); return true; diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 822a750b064b2..23b8dda70cb2f 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -1043,7 +1043,10 @@ static const struct pci_epc_features imx8m_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = false, - .reserved_bar = 1 << BAR_1 | 1 << BAR_3, + .reserved_bar = 1 << BAR_1 | 1 << BAR_3 | 1 << BAR_5, + .bar_fixed_size = { + [BAR_4] = SZ_256, + }, .align = SZ_64K, }; @@ -1098,8 +1101,6 @@ static int imx6_add_pcie_ep(struct imx6_pcie *imx6_pcie, dev_err(dev, "failed to initialize endpoint\n"); return ret; } - /* Start LTSSM. */ - imx6_pcie_ltssm_enable(dev); return 0; } @@ -1283,7 +1284,8 @@ static int imx6_pcie_probe(struct platform_device *pdev) case IMX8MQ_EP: if (dbi_base->start == IMX8MQ_PCIE2_BASE_ADDR) imx6_pcie->controller_id = 1; - + fallthrough; + case IMX7D: imx6_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev, "pciephy"); if (IS_ERR(imx6_pcie->pciephy_reset)) { diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index bf9a961c9f276..9055ce34c636b 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -1213,8 +1213,8 @@ static int ks_pcie_probe(struct platform_device *pdev) if (irq < 0) return irq; - ret = request_irq(irq, ks_pcie_err_irq_handler, IRQF_SHARED, - "ks-pcie-error-irq", ks_pcie); + ret = devm_request_irq(dev, irq, ks_pcie_err_irq_handler, IRQF_SHARED, + "ks-pcie-error-irq", ks_pcie); if (ret < 0) { dev_err(dev, "failed to request error IRQ %d\n", irq); diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index f2e5feba55267..26ad643fb4248 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -281,7 +281,7 @@ static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr, u32 index; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - for (index = 0; index < pci->num_ob_windows; index++) { + for_each_set_bit(index, ep->ob_window_map, pci->num_ob_windows) { if (ep->outbound_addr[index] != addr) continue; *atu_index = index; diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 2b60d20dfdf59..46b12e157bebe 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -748,22 +748,19 @@ static void dw_pcie_link_set_max_link_width(struct dw_pcie *pci, u32 num_lanes) /* Set link width speed control register */ lwsc = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); lwsc &= ~PORT_LOGIC_LINK_WIDTH_MASK; + lwsc |= PORT_LOGIC_LINK_WIDTH_1_LANES; switch (num_lanes) { case 1: plc |= PORT_LINK_MODE_1_LANES; - lwsc |= PORT_LOGIC_LINK_WIDTH_1_LANES; break; case 2: plc |= PORT_LINK_MODE_2_LANES; - lwsc |= PORT_LOGIC_LINK_WIDTH_2_LANES; break; case 4: plc |= PORT_LINK_MODE_4_LANES; - lwsc |= PORT_LOGIC_LINK_WIDTH_4_LANES; break; case 8: plc |= PORT_LINK_MODE_8_LANES; - lwsc |= PORT_LOGIC_LINK_WIDTH_8_LANES; break; default: dev_err(pci->dev, "num-lanes %u: invalid value\n", num_lanes); @@ -930,9 +927,7 @@ static int dw_pcie_edma_irq_verify(struct dw_pcie *pci) char name[6]; int ret; - if (pci->edma.nr_irqs == 1) - return 0; - else if (pci->edma.nr_irqs > 1) + if (pci->edma.nr_irqs > 1) return pci->edma.nr_irqs != ch_cnt ? -EINVAL : 0; ret = platform_get_irq_byname_optional(pdev, "dma"); diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index 9b1256da096cb..8af7a837a0612 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -275,8 +275,8 @@ static int rockchip_pcie_phy_init(struct rockchip_pcie *rockchip) static void rockchip_pcie_phy_deinit(struct rockchip_pcie *rockchip) { - phy_exit(rockchip->phy); phy_power_off(rockchip->phy); + phy_exit(rockchip->phy); } static const struct dw_pcie_ops dw_pcie_ops = { diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 416d6b45d1fe8..ce2a7a6dab90c 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -1218,6 +1218,7 @@ static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, struct mrq_uphy_response resp; struct tegra_bpmp_message msg; struct mrq_uphy_request req; + int err; /* * Controller-5 doesn't need to have its state set by BPMP-FW in @@ -1240,7 +1241,13 @@ static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, msg.rx.data = &resp; msg.rx.size = sizeof(resp); - return tegra_bpmp_transfer(pcie->bpmp, &msg); + err = tegra_bpmp_transfer(pcie->bpmp, &msg); + if (err) + return err; + if (msg.rx.ret) + return -EINVAL; + + return 0; } static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, @@ -1249,6 +1256,7 @@ static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, struct mrq_uphy_response resp; struct tegra_bpmp_message msg; struct mrq_uphy_request req; + int err; memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); @@ -1268,7 +1276,13 @@ static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, msg.rx.data = &resp; msg.rx.size = sizeof(resp); - return tegra_bpmp_transfer(pcie->bpmp, &msg); + err = tegra_bpmp_transfer(pcie->bpmp, &msg); + if (err) + return err; + if (msg.rx.ret) + return -EINVAL; + + return 0; } static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) @@ -1734,9 +1748,9 @@ static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) ret); } - ret = tegra_pcie_bpmp_set_pll_state(pcie, false); + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); if (ret) - dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret); + dev_err(pcie->dev, "Failed to disable controller: %d\n", ret); pcie->ep_state = EP_STATE_DISABLED; dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); @@ -1949,6 +1963,15 @@ static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) return IRQ_HANDLED; } +static void tegra_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar; + + for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) + dw_pcie_ep_reset_bar(pci, bar); +}; + static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) { /* Tegra194 supports only INTA */ @@ -1963,10 +1986,10 @@ static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) { - if (unlikely(irq > 31)) + if (unlikely(irq > 32)) return -EINVAL; - appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1); + appl_writel(pcie, BIT(irq - 1), APPL_MSI_CTRL_1); return 0; } @@ -2022,6 +2045,7 @@ tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) } static const struct dw_pcie_ep_ops pcie_ep_ops = { + .ep_init = tegra_pcie_ep_init, .raise_irq = tegra_pcie_ep_raise_irq, .get_features = tegra_pcie_ep_get_features, }; diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c index 6be3266cd7b5b..e2602e38ae452 100644 --- a/drivers/pci/controller/pci-host-common.c +++ b/drivers/pci/controller/pci-host-common.c @@ -73,10 +73,6 @@ int pci_host_common_probe(struct platform_device *pdev) if (IS_ERR(cfg)) return PTR_ERR(cfg); - /* Do not reassign resources if probe only */ - if (!pci_has_flag(PCI_PROBE_ONLY)) - pci_add_flags(PCI_REASSIGN_ALL_BUS); - bridge->sysdata = cfg; bridge->ops = (struct pci_ops *)&ops->pci_ops; bridge->msi_domain = true; diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index 038d974a318ea..da37569460b1a 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -14,6 +14,7 @@ */ #include +#include #include #include #include @@ -269,7 +270,7 @@ struct tegra_msi { DECLARE_BITMAP(used, INT_PCI_MSI_NR); struct irq_domain *domain; struct mutex map_lock; - spinlock_t mask_lock; + raw_spinlock_t mask_lock; void *virt; dma_addr_t phys; int irq; @@ -1343,7 +1344,7 @@ static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port) unsigned int i; int err; - port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL); + port->phys = devm_kcalloc(dev, port->lanes, sizeof(phy), GFP_KERNEL); if (!port->phys) return -ENOMEM; @@ -1604,14 +1605,13 @@ static void tegra_msi_irq_mask(struct irq_data *d) struct tegra_msi *msi = irq_data_get_irq_chip_data(d); struct tegra_pcie *pcie = msi_to_pcie(msi); unsigned int index = d->hwirq / 32; - unsigned long flags; u32 value; - spin_lock_irqsave(&msi->mask_lock, flags); - value = afi_readl(pcie, AFI_MSI_EN_VEC(index)); - value &= ~BIT(d->hwirq % 32); - afi_writel(pcie, value, AFI_MSI_EN_VEC(index)); - spin_unlock_irqrestore(&msi->mask_lock, flags); + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = afi_readl(pcie, AFI_MSI_EN_VEC(index)); + value &= ~BIT(d->hwirq % 32); + afi_writel(pcie, value, AFI_MSI_EN_VEC(index)); + } } static void tegra_msi_irq_unmask(struct irq_data *d) @@ -1619,14 +1619,13 @@ static void tegra_msi_irq_unmask(struct irq_data *d) struct tegra_msi *msi = irq_data_get_irq_chip_data(d); struct tegra_pcie *pcie = msi_to_pcie(msi); unsigned int index = d->hwirq / 32; - unsigned long flags; u32 value; - spin_lock_irqsave(&msi->mask_lock, flags); - value = afi_readl(pcie, AFI_MSI_EN_VEC(index)); - value |= BIT(d->hwirq % 32); - afi_writel(pcie, value, AFI_MSI_EN_VEC(index)); - spin_unlock_irqrestore(&msi->mask_lock, flags); + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = afi_readl(pcie, AFI_MSI_EN_VEC(index)); + value |= BIT(d->hwirq % 32); + afi_writel(pcie, value, AFI_MSI_EN_VEC(index)); + } } static int tegra_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) @@ -1742,7 +1741,7 @@ static int tegra_pcie_msi_setup(struct tegra_pcie *pcie) int err; mutex_init(&msi->map_lock); - spin_lock_init(&msi->mask_lock); + raw_spin_lock_init(&msi->mask_lock); if (IS_ENABLED(CONFIG_PCI_MSI)) { err = tegra_allocate_domains(msi); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index f7a248393a8f1..8dfea64a51e0f 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -541,7 +541,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); /* Assert PERST# before setting up the clock */ - gpiod_set_value(reset, 1); + gpiod_set_value_cansleep(reset, 1); ret = apple_pcie_setup_refclk(pcie, port); if (ret < 0) @@ -552,7 +552,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, /* Deassert PERST# */ rmw_set(PORT_PERST_OFF, port->base + PORT_PERST); - gpiod_set_value(reset, 0); + gpiod_set_value_cansleep(reset, 0); /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ msleep(100); @@ -585,6 +585,9 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, list_add_tail(&port->entry, &pcie->ports); init_completion(&pcie->event); + /* In the success path, we keep a reference to np around */ + of_node_get(np); + ret = apple_pcie_port_register_irqs(port); WARN_ON(ret); @@ -764,7 +767,6 @@ static int apple_pcie_init(struct pci_config_window *cfg) { struct device *dev = cfg->parent; struct platform_device *platform = to_platform_device(dev); - struct device_node *of_port; struct apple_pcie *pcie; int ret; @@ -787,11 +789,10 @@ static int apple_pcie_init(struct pci_config_window *cfg) if (ret) return ret; - for_each_child_of_node(dev->of_node, of_port) { + for_each_available_child_of_node_scoped(dev->of_node, of_port) { ret = apple_pcie_setup_port(pcie, of_port); if (ret) { dev_err(pcie->dev, "Port %pOF setup fail: %d\n", of_port, ret); - of_node_put(of_port); return ret; } } diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 940af934ce1bb..9bcf4c68058eb 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -284,8 +284,8 @@ static int brcm_pcie_encode_ibar_size(u64 size) if (log2_in >= 12 && log2_in <= 15) /* Covers 4KB to 32KB (inclusive) */ return (log2_in - 12) + 0x1c; - else if (log2_in >= 16 && log2_in <= 35) - /* Covers 64KB to 32GB, (inclusive) */ + else if (log2_in >= 16 && log2_in <= 36) + /* Covers 64KB to 64GB, (inclusive) */ return log2_in - 15; /* Something is awry so disable */ return 0; @@ -1632,3 +1632,4 @@ module_platform_driver(brcm_pcie_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Broadcom STB PCIe RC driver"); MODULE_AUTHOR("Broadcom"); +MODULE_SOFTDEP("pre: irq_bcm2712_mip"); diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c index 704ab5d723a95..77f6366e7db10 100644 --- a/drivers/pci/controller/pcie-rcar-host.c +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -12,6 +12,7 @@ */ #include +#include #include #include #include @@ -36,7 +37,7 @@ struct rcar_msi { DECLARE_BITMAP(used, INT_PCI_MSI_NR); struct irq_domain *domain; struct mutex map_lock; - spinlock_t mask_lock; + raw_spinlock_t mask_lock; int irq1; int irq2; }; @@ -50,20 +51,13 @@ struct rcar_pcie_host { int (*phy_init_fn)(struct rcar_pcie_host *host); }; -static DEFINE_SPINLOCK(pmsr_lock); - static int rcar_pcie_wakeup(struct device *pcie_dev, void __iomem *pcie_base) { - unsigned long flags; u32 pmsr, val; int ret = 0; - spin_lock_irqsave(&pmsr_lock, flags); - - if (!pcie_base || pm_runtime_suspended(pcie_dev)) { - ret = -EINVAL; - goto unlock_exit; - } + if (!pcie_base || pm_runtime_suspended(pcie_dev)) + return -EINVAL; pmsr = readl(pcie_base + PMSR); @@ -85,8 +79,6 @@ static int rcar_pcie_wakeup(struct device *pcie_dev, void __iomem *pcie_base) writel(L1FAEG | PMEL1RX, pcie_base + PMSR); } -unlock_exit: - spin_unlock_irqrestore(&pmsr_lock, flags); return ret; } @@ -633,28 +625,26 @@ static void rcar_msi_irq_mask(struct irq_data *d) { struct rcar_msi *msi = irq_data_get_irq_chip_data(d); struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; - unsigned long flags; u32 value; - spin_lock_irqsave(&msi->mask_lock, flags); - value = rcar_pci_read_reg(pcie, PCIEMSIIER); - value &= ~BIT(d->hwirq); - rcar_pci_write_reg(pcie, value, PCIEMSIIER); - spin_unlock_irqrestore(&msi->mask_lock, flags); + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = rcar_pci_read_reg(pcie, PCIEMSIIER); + value &= ~BIT(d->hwirq); + rcar_pci_write_reg(pcie, value, PCIEMSIIER); + } } static void rcar_msi_irq_unmask(struct irq_data *d) { struct rcar_msi *msi = irq_data_get_irq_chip_data(d); struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; - unsigned long flags; u32 value; - spin_lock_irqsave(&msi->mask_lock, flags); - value = rcar_pci_read_reg(pcie, PCIEMSIIER); - value |= BIT(d->hwirq); - rcar_pci_write_reg(pcie, value, PCIEMSIIER); - spin_unlock_irqrestore(&msi->mask_lock, flags); + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = rcar_pci_read_reg(pcie, PCIEMSIIER); + value |= BIT(d->hwirq); + rcar_pci_write_reg(pcie, value, PCIEMSIIER); + } } static int rcar_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) @@ -770,7 +760,7 @@ static int rcar_pcie_enable_msi(struct rcar_pcie_host *host) int err; mutex_init(&msi->map_lock); - spin_lock_init(&msi->mask_lock); + raw_spin_lock_init(&msi->mask_lock); err = of_address_to_resource(dev->of_node, 0, &res); if (err) diff --git a/drivers/pci/controller/pcie-rockchip-host.c b/drivers/pci/controller/pcie-rockchip-host.c index afbbdccd195d9..c25f32abbdf64 100644 --- a/drivers/pci/controller/pcie-rockchip-host.c +++ b/drivers/pci/controller/pcie-rockchip-host.c @@ -11,6 +11,7 @@ * ARM PCI Host generic driver. */ +#include #include #include #include @@ -40,18 +41,18 @@ static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) { u32 status; - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); } static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) { u32 status; - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); } static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) @@ -269,7 +270,7 @@ static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) scale = 3; /* 0.001x */ curr = curr / 1000; /* convert to mA */ power = (curr * 3300) / 1000; /* milliwatt */ - while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { + while (power > FIELD_MAX(PCI_EXP_DEVCAP_PWR_VAL)) { if (!scale) { dev_warn(rockchip->dev, "invalid power supply\n"); return; @@ -278,10 +279,10 @@ static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) power = power / 10; } - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); - status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | - (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP); + status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_VAL, power); + status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_SCL, scale); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP); } /** @@ -309,14 +310,14 @@ static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) rockchip_pcie_set_power_limit(rockchip); /* Set RC's clock architecture as common clock */ - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); status |= PCI_EXP_LNKSTA_SLC << 16; - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); /* Set RC's RCB to 128 */ - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); status |= PCI_EXP_LNKCTL_RCB; - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); /* Enable Gen1 training */ rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, @@ -338,9 +339,13 @@ static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) * Enable retrain for gen2. This should be configured only after * gen1 finished. */ - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2); + status &= ~PCI_EXP_LNKCTL2_TLS; + status |= PCI_EXP_LNKCTL2_TLS_5_0GT; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); status |= PCI_EXP_LNKCTL_RL; - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, status, PCIE_LINK_IS_GEN2(status), 20, @@ -377,15 +382,15 @@ static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) /* Clear L0s from RC's link cap */ if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); - status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP); + status &= ~PCI_EXP_LNKCAP_ASPM_L0S; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP); } - status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); - status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; - status |= PCIE_RC_CONFIG_DCSR_MPS_256; - rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL); + status &= ~PCI_EXP_DEVCTL_PAYLOAD; + status |= PCI_EXP_DEVCTL_PAYLOAD_256B; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL); return 0; err_power_off_phy: @@ -436,7 +441,7 @@ static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) dev_dbg(dev, "malformed TLP received from the link\n"); if (sub_reg & PCIE_CORE_INT_UCR) - dev_dbg(dev, "malformed TLP received from the link\n"); + dev_dbg(dev, "Unexpected Completion received from the link\n"); if (sub_reg & PCIE_CORE_INT_FCE) dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); diff --git a/drivers/pci/controller/pcie-rockchip.h b/drivers/pci/controller/pcie-rockchip.h index 15ee949f2485e..049ad984a4164 100644 --- a/drivers/pci/controller/pcie-rockchip.h +++ b/drivers/pci/controller/pcie-rockchip.h @@ -144,16 +144,7 @@ #define PCIE_EP_CONFIG_BASE 0xa00000 #define PCIE_EP_CONFIG_DID_VID (PCIE_EP_CONFIG_BASE + 0x00) #define PCIE_RC_CONFIG_RID_CCR (PCIE_RC_CONFIG_BASE + 0x08) -#define PCIE_RC_CONFIG_DCR (PCIE_RC_CONFIG_BASE + 0xc4) -#define PCIE_RC_CONFIG_DCR_CSPL_SHIFT 18 -#define PCIE_RC_CONFIG_DCR_CSPL_LIMIT 0xff -#define PCIE_RC_CONFIG_DCR_CPLS_SHIFT 26 -#define PCIE_RC_CONFIG_DCSR (PCIE_RC_CONFIG_BASE + 0xc8) -#define PCIE_RC_CONFIG_DCSR_MPS_MASK GENMASK(7, 5) -#define PCIE_RC_CONFIG_DCSR_MPS_256 (0x1 << 5) -#define PCIE_RC_CONFIG_LINK_CAP (PCIE_RC_CONFIG_BASE + 0xcc) -#define PCIE_RC_CONFIG_LINK_CAP_L0S BIT(10) -#define PCIE_RC_CONFIG_LCS (PCIE_RC_CONFIG_BASE + 0xd0) +#define PCIE_RC_CONFIG_CR (PCIE_RC_CONFIG_BASE + 0xc0) #define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c) #define PCIE_RC_CONFIG_THP_CAP (PCIE_RC_CONFIG_BASE + 0x274) #define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK GENMASK(31, 20) diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index dfa222e02c4da..ad82feff0405e 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -17,6 +17,8 @@ #include #include +#include + #include #define VMD_CFGBAR 0 @@ -981,6 +983,24 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) struct vmd_dev *vmd; int err; + if (xen_domain()) { + /* + * Xen doesn't have knowledge about devices in the VMD bus + * because the config space of devices behind the VMD bridge is + * not known to Xen, and hence Xen cannot discover or configure + * them in any way. + * + * Bypass of MSI remapping won't work in that case as direct + * write by Linux to the MSI entries won't result in functional + * interrupts, as Xen is the entity that manages the host + * interrupt controller and must configure interrupts. However + * multiplexing of interrupts by the VMD bridge will work under + * Xen, so force the usage of that mode which must always be + * supported by VMD bridges. + */ + features &= ~VMD_FEAT_CAN_BYPASS_MSI_REMAP; + } + if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20)) return -ENOMEM; diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index ac1dae113f2d9..4cf20ca25ea74 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -282,17 +282,20 @@ static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test) if (!epf_test->dma_supported) return; - dma_release_channel(epf_test->dma_chan_tx); - if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) { + if (epf_test->dma_chan_tx) { + dma_release_channel(epf_test->dma_chan_tx); + if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) { + epf_test->dma_chan_tx = NULL; + epf_test->dma_chan_rx = NULL; + return; + } epf_test->dma_chan_tx = NULL; - epf_test->dma_chan_rx = NULL; - return; } - dma_release_channel(epf_test->dma_chan_rx); - epf_test->dma_chan_rx = NULL; - - return; + if (epf_test->dma_chan_rx) { + dma_release_channel(epf_test->dma_chan_rx); + epf_test->dma_chan_rx = NULL; + } } static void pci_epf_test_print_rate(struct pci_epf_test *epf_test, diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 3368f483f818d..33c3f9b980e68 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -531,7 +531,7 @@ static int epf_ntb_db_bar_init(struct epf_ntb *ntb) struct device *dev = &ntb->epf->dev; int ret; struct pci_epf_bar *epf_bar; - void __iomem *mw_addr; + void *mw_addr; enum pci_barno barno; size_t size = sizeof(u32) * ntb->db_count; @@ -711,7 +711,7 @@ static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) barno = pci_epc_get_next_free_bar(epc_features, barno); if (barno < 0) { dev_err(dev, "Fail to get NTB function BAR\n"); - return barno; + return -ENOENT; } ntb->epf_ntb_bar[bar] = barno; } diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index 0ea64e24ed616..c17dff4bd19b4 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -683,6 +683,7 @@ void pci_ep_cfs_remove_epf_group(struct config_group *group) if (IS_ERR_OR_NULL(group)) return; + list_del(&group->group_entry); configfs_unregister_default_group(group); } EXPORT_SYMBOL(pci_ep_cfs_remove_epf_group); diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index 059f8639f21e9..03673ade6ce4d 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -311,7 +311,7 @@ static void pci_epf_remove_cfs(struct pci_epf_driver *driver) mutex_lock(&pci_epf_mutex); list_for_each_entry_safe(group, tmp, &driver->epf_group, group_entry) pci_ep_cfs_remove_epf_group(group); - list_del(&driver->epf_group); + WARN_ON(!list_empty(&driver->epf_group)); mutex_unlock(&pci_epf_mutex); } diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c index 092c9ac0d26d2..ec7828ad66617 100644 --- a/drivers/pci/hotplug/pnv_php.c +++ b/drivers/pci/hotplug/pnv_php.c @@ -3,11 +3,14 @@ * PCI Hotplug Driver for PowerPC PowerNV platform. * * Copyright Gavin Shan, IBM Corporation 2016. + * Copyright (C) 2025 Raptor Engineering, LLC + * Copyright (C) 2025 Raptor Computing Systems, LLC */ #include #include #include +#include #include #include @@ -35,8 +38,10 @@ static void pnv_php_register(struct device_node *dn); static void pnv_php_unregister_one(struct device_node *dn); static void pnv_php_unregister(struct device_node *dn); +static void pnv_php_enable_irq(struct pnv_php_slot *php_slot); + static void pnv_php_disable_irq(struct pnv_php_slot *php_slot, - bool disable_device) + bool disable_device, bool disable_msi) { struct pci_dev *pdev = php_slot->pdev; u16 ctrl; @@ -52,19 +57,15 @@ static void pnv_php_disable_irq(struct pnv_php_slot *php_slot, php_slot->irq = 0; } - if (php_slot->wq) { - destroy_workqueue(php_slot->wq); - php_slot->wq = NULL; - } - - if (disable_device) { + if (disable_device || disable_msi) { if (pdev->msix_enabled) pci_disable_msix(pdev); else if (pdev->msi_enabled) pci_disable_msi(pdev); + } + if (disable_device) pci_disable_device(pdev); - } } static void pnv_php_free_slot(struct kref *kref) @@ -73,7 +74,8 @@ static void pnv_php_free_slot(struct kref *kref) struct pnv_php_slot, kref); WARN_ON(!list_empty(&php_slot->children)); - pnv_php_disable_irq(php_slot, false); + pnv_php_disable_irq(php_slot, false, false); + destroy_workqueue(php_slot->wq); kfree(php_slot->name); kfree(php_slot); } @@ -390,6 +392,20 @@ static int pnv_php_get_power_state(struct hotplug_slot *slot, u8 *state) return 0; } +static int pcie_check_link_active(struct pci_dev *pdev) +{ + u16 lnk_status; + int ret; + + ret = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status); + if (ret == PCIBIOS_DEVICE_NOT_FOUND || PCI_POSSIBLE_ERROR(lnk_status)) + return -ENODEV; + + ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA); + + return ret; +} + static int pnv_php_get_adapter_state(struct hotplug_slot *slot, u8 *state) { struct pnv_php_slot *php_slot = to_pnv_php_slot(slot); @@ -402,6 +418,19 @@ static int pnv_php_get_adapter_state(struct hotplug_slot *slot, u8 *state) */ ret = pnv_pci_get_presence_state(php_slot->id, &presence); if (ret >= 0) { + if (pci_pcie_type(php_slot->pdev) == PCI_EXP_TYPE_DOWNSTREAM && + presence == OPAL_PCI_SLOT_EMPTY) { + /* + * Similar to pciehp_hpc, check whether the Link Active + * bit is set to account for broken downstream bridges + * that don't properly assert Presence Detect State, as + * was observed on the Microsemi Switchtec PM8533 PFX + * [11f8:8533]. + */ + if (pcie_check_link_active(php_slot->pdev) > 0) + presence = OPAL_PCI_SLOT_PRESENT; + } + *state = presence; ret = 0; } else { @@ -441,6 +470,61 @@ static int pnv_php_set_attention_state(struct hotplug_slot *slot, u8 state) return 0; } +static int pnv_php_activate_slot(struct pnv_php_slot *php_slot, + struct hotplug_slot *slot) +{ + int ret, i; + + /* + * Issue initial slot activation command to firmware + * + * Firmware will power slot on, attempt to train the link, and + * discover any downstream devices. If this process fails, firmware + * will return an error code and an invalid device tree. Failure + * can be caused for multiple reasons, including a faulty + * downstream device, poor connection to the downstream device, or + * a previously latched PHB fence. On failure, issue fundamental + * reset up to three times before aborting. + */ + ret = pnv_php_set_slot_power_state(slot, OPAL_PCI_SLOT_POWER_ON); + if (ret) { + SLOT_WARN( + php_slot, + "PCI slot activation failed with error code %d, possible frozen PHB", + ret); + SLOT_WARN( + php_slot, + "Attempting complete PHB reset before retrying slot activation\n"); + for (i = 0; i < 3; i++) { + /* + * Slot activation failed, PHB may be fenced from a + * prior device failure. + * + * Use the OPAL fundamental reset call to both try a + * device reset and clear any potentially active PHB + * fence / freeze. + */ + SLOT_WARN(php_slot, "Try %d...\n", i + 1); + pci_set_pcie_reset_state(php_slot->pdev, + pcie_warm_reset); + msleep(250); + pci_set_pcie_reset_state(php_slot->pdev, + pcie_deassert_reset); + + ret = pnv_php_set_slot_power_state( + slot, OPAL_PCI_SLOT_POWER_ON); + if (!ret) + break; + } + + if (i >= 3) + SLOT_WARN(php_slot, + "Failed to bring slot online, aborting!\n"); + } + + return ret; +} + static int pnv_php_enable(struct pnv_php_slot *php_slot, bool rescan) { struct hotplug_slot *slot = &php_slot->slot; @@ -503,7 +587,7 @@ static int pnv_php_enable(struct pnv_php_slot *php_slot, bool rescan) goto scan; /* Power is off, turn it on and then scan the slot */ - ret = pnv_php_set_slot_power_state(slot, OPAL_PCI_SLOT_POWER_ON); + ret = pnv_php_activate_slot(php_slot, slot); if (ret) return ret; @@ -560,8 +644,58 @@ static int pnv_php_reset_slot(struct hotplug_slot *slot, bool probe) static int pnv_php_enable_slot(struct hotplug_slot *slot) { struct pnv_php_slot *php_slot = to_pnv_php_slot(slot); + u32 prop32; + int ret; + + ret = pnv_php_enable(php_slot, true); + if (ret) + return ret; + + /* (Re-)enable interrupt if the slot supports surprise hotplug */ + ret = of_property_read_u32(php_slot->dn, "ibm,slot-surprise-pluggable", + &prop32); + if (!ret && prop32) + pnv_php_enable_irq(php_slot); + + return 0; +} + +/* + * Disable any hotplug interrupts for all slots on the provided bus, as well as + * all downstream slots in preparation for a hot unplug. + */ +static int pnv_php_disable_all_irqs(struct pci_bus *bus) +{ + struct pci_bus *child_bus; + struct pci_slot *slot; + + /* First go down child buses */ + list_for_each_entry(child_bus, &bus->children, node) + pnv_php_disable_all_irqs(child_bus); + + /* Disable IRQs for all pnv_php slots on this bus */ + list_for_each_entry(slot, &bus->slots, list) { + struct pnv_php_slot *php_slot = to_pnv_php_slot(slot->hotplug); - return pnv_php_enable(php_slot, true); + pnv_php_disable_irq(php_slot, false, true); + } + + return 0; +} + +/* + * Disable any hotplug interrupts for all downstream slots on the provided + * bus in preparation for a hot unplug. + */ +static int pnv_php_disable_all_downstream_irqs(struct pci_bus *bus) +{ + struct pci_bus *child_bus; + + /* Go down child buses, recursively deactivating their IRQs */ + list_for_each_entry(child_bus, &bus->children, node) + pnv_php_disable_all_irqs(child_bus); + + return 0; } static int pnv_php_disable_slot(struct hotplug_slot *slot) @@ -578,6 +712,13 @@ static int pnv_php_disable_slot(struct hotplug_slot *slot) php_slot->state != PNV_PHP_STATE_REGISTERED) return 0; + /* + * Free all IRQ resources from all child slots before remove. + * Note that we do not disable the root slot IRQ here as that + * would also deactivate the slot hot (re)plug interrupt! + */ + pnv_php_disable_all_downstream_irqs(php_slot->bus); + /* Remove all devices behind the slot */ pci_lock_rescan_remove(); pci_hp_remove_devices(php_slot->bus); @@ -646,6 +787,15 @@ static struct pnv_php_slot *pnv_php_alloc_slot(struct device_node *dn) return NULL; } + /* Allocate workqueue for this slot's interrupt handling */ + php_slot->wq = alloc_workqueue("pciehp-%s", 0, 0, php_slot->name); + if (!php_slot->wq) { + SLOT_WARN(php_slot, "Cannot alloc workqueue\n"); + kfree(php_slot->name); + kfree(php_slot); + return NULL; + } + if (dn->child && PCI_DN(dn->child)) php_slot->slot_no = PCI_SLOT(PCI_DN(dn->child)->devfn); else @@ -744,16 +894,63 @@ static int pnv_php_enable_msix(struct pnv_php_slot *php_slot) return entry.vector; } +static void +pnv_php_detect_clear_suprise_removal_freeze(struct pnv_php_slot *php_slot) +{ + struct pci_dev *pdev = php_slot->pdev; + struct eeh_dev *edev; + struct eeh_pe *pe; + int i, rc; + + /* + * When a device is surprise removed from a downstream bridge slot, + * the upstream bridge port can still end up frozen due to related EEH + * events, which will in turn block the MSI interrupts for slot hotplug + * detection. + * + * Detect and thaw any frozen upstream PE after slot deactivation. + */ + edev = pci_dev_to_eeh_dev(pdev); + pe = edev ? edev->pe : NULL; + rc = eeh_pe_get_state(pe); + if ((rc == -ENODEV) || (rc == -ENOENT)) { + SLOT_WARN( + php_slot, + "Upstream bridge PE state unknown, hotplug detect may fail\n"); + } else { + if (pe->state & EEH_PE_ISOLATED) { + SLOT_WARN( + php_slot, + "Upstream bridge PE %02x frozen, thawing...\n", + pe->addr); + for (i = 0; i < 3; i++) + if (!eeh_unfreeze_pe(pe)) + break; + if (i >= 3) + SLOT_WARN( + php_slot, + "Unable to thaw PE %02x, hotplug detect will fail!\n", + pe->addr); + else + SLOT_WARN(php_slot, + "PE %02x thawed successfully\n", + pe->addr); + } + } +} + static void pnv_php_event_handler(struct work_struct *work) { struct pnv_php_event *event = container_of(work, struct pnv_php_event, work); struct pnv_php_slot *php_slot = event->php_slot; - if (event->added) + if (event->added) { pnv_php_enable_slot(&php_slot->slot); - else + } else { pnv_php_disable_slot(&php_slot->slot); + pnv_php_detect_clear_suprise_removal_freeze(php_slot); + } kfree(event); } @@ -842,14 +1039,6 @@ static void pnv_php_init_irq(struct pnv_php_slot *php_slot, int irq) u16 sts, ctrl; int ret; - /* Allocate workqueue */ - php_slot->wq = alloc_workqueue("pciehp-%s", 0, 0, php_slot->name); - if (!php_slot->wq) { - SLOT_WARN(php_slot, "Cannot alloc workqueue\n"); - pnv_php_disable_irq(php_slot, true); - return; - } - /* Check PDC (Presence Detection Change) is broken or not */ ret = of_property_read_u32(php_slot->dn, "ibm,slot-broken-pdc", &broken_pdc); @@ -868,7 +1057,7 @@ static void pnv_php_init_irq(struct pnv_php_slot *php_slot, int irq) ret = request_irq(irq, pnv_php_interrupt, IRQF_SHARED, php_slot->name, php_slot); if (ret) { - pnv_php_disable_irq(php_slot, true); + pnv_php_disable_irq(php_slot, true, true); SLOT_WARN(php_slot, "Error %d enabling IRQ %d\n", ret, irq); return; } diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 25dbe85c42175..f97c4f0e1c7a3 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -581,15 +581,18 @@ static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) if (dev->no_vf_scan) return 0; + pci_lock_rescan_remove(); for (i = 0; i < num_vfs; i++) { rc = pci_iov_add_virtfn(dev, i); if (rc) goto failed; } + pci_unlock_rescan_remove(); return 0; failed: while (i--) pci_iov_remove_virtfn(dev, i); + pci_unlock_rescan_remove(); return rc; } @@ -709,8 +712,10 @@ static void sriov_del_vfs(struct pci_dev *dev) struct pci_sriov *iov = dev->sriov; int i; + pci_lock_rescan_remove(); for (i = 0; i < iov->num_VFs; i++) pci_iov_remove_virtfn(dev, i); + pci_unlock_rescan_remove(); } static void sriov_disable(struct pci_dev *dev) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 053bb9fac6e3e..b638731aa5ff2 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -610,6 +610,9 @@ void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) if (desc->pci.msi_attrib.can_mask) { void __iomem *addr = pci_msix_desc_addr(desc); + /* Workaround for SUN NIU insanity, which requires write before read */ + if (dev->dev_flags & PCI_DEV_FLAGS_MSIX_TOUCH_ENTRY_DATA_FIRST) + writel(0, addr + PCI_MSIX_ENTRY_DATA); desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } } diff --git a/drivers/pci/of.c b/drivers/pci/of.c index 51e3dd0ea5abe..48ba95e4ab053 100644 --- a/drivers/pci/of.c +++ b/drivers/pci/of.c @@ -234,27 +234,61 @@ int of_get_pci_domain_nr(struct device_node *node) EXPORT_SYMBOL_GPL(of_get_pci_domain_nr); /** - * of_pci_check_probe_only - Setup probe only mode if linux,pci-probe-only - * is present and valid + * of_pci_preserve_config - Return true if the boot configuration needs to + * be preserved + * @node: Device tree node. + * + * Look for "linux,pci-probe-only" property for a given PCI controller's + * node and return true if found. Also look in the chosen node if the + * property is not found in the given controller's node. Having this + * property ensures that the kernel doesn't reconfigure the BARs and bridge + * windows that are already done by the platform firmware. + * + * Return: true if the property exists; false otherwise. */ -void of_pci_check_probe_only(void) +bool of_pci_preserve_config(struct device_node *node) { - u32 val; + u32 val = 0; int ret; - ret = of_property_read_u32(of_chosen, "linux,pci-probe-only", &val); + if (!node) { + pr_warn("device node is NULL, trying with of_chosen\n"); + node = of_chosen; + } + +retry: + ret = of_property_read_u32(node, "linux,pci-probe-only", &val); if (ret) { - if (ret == -ENODATA || ret == -EOVERFLOW) - pr_warn("linux,pci-probe-only without valid value, ignoring\n"); - return; + if (ret == -ENODATA || ret == -EOVERFLOW) { + pr_warn("Incorrect value for linux,pci-probe-only in %pOF, ignoring\n", + node); + return false; + } + if (ret == -EINVAL) { + if (node == of_chosen) + return false; + + node = of_chosen; + goto retry; + } } if (val) + return true; + else + return false; +} + +/** + * of_pci_check_probe_only - Setup probe only mode if linux,pci-probe-only + * is present and valid + */ +void of_pci_check_probe_only(void) +{ + if (of_pci_preserve_config(of_chosen)) pci_add_flags(PCI_PROBE_ONLY); else pci_clear_flags(PCI_PROBE_ONLY); - - pr_info("PROBE_ONLY %s\n", val ? "enabled" : "disabled"); } EXPORT_SYMBOL_GPL(of_pci_check_probe_only); diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index b8becc7257cda..0f1e431bbfc20 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -351,7 +351,7 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, pages_free: devm_memunmap_pages(&pdev->dev, pgmap); pgmap_free: - devm_kfree(&pdev->dev, pgmap); + devm_kfree(&pdev->dev, p2p_pgmap); return error; } EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource); diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index 05b7357bd2586..5ff4007100292 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -119,6 +119,28 @@ phys_addr_t acpi_pci_root_get_mcfg_addr(acpi_handle handle) return (phys_addr_t)mcfg_addr; } +bool pci_acpi_preserve_config(struct pci_host_bridge *host_bridge) +{ + if (ACPI_HANDLE(&host_bridge->dev)) { + union acpi_object *obj; + + /* + * Evaluate the "PCI Boot Configuration" _DSM Function. If it + * exists and returns 0, we must preserve any PCI resource + * assignments made by firmware for this host bridge. + */ + obj = acpi_evaluate_dsm_typed(ACPI_HANDLE(&host_bridge->dev), + &pci_acpi_dsm_guid, + 1, DSM_PCI_PRESERVE_BOOT_CONFIG, + NULL, ACPI_TYPE_INTEGER); + if (obj && obj->integer.value == 0) + return true; + ACPI_FREE(obj); + } + + return false; +} + /* _HPX PCI Setting Record (Type 0); same as _HPP */ struct hpx_type0 { u32 revision; /* Not present in _HPP */ diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 9c59bf03d6579..11a90b55c1873 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -1598,7 +1598,7 @@ static int pci_uevent(const struct device *dev, struct kobj_uevent_env *env) return 0; } -#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH) +#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH) || defined(CONFIG_S390) /** * pci_uevent_ers - emit a uevent during recovery path of PCI device * @pdev: PCI device undergoing error recovery @@ -1612,6 +1612,7 @@ void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type) switch (err_type) { case PCI_ERS_RESULT_NONE: case PCI_ERS_RESULT_CAN_RECOVER: + case PCI_ERS_RESULT_NEED_RESET: envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY"; envp[idx++] = "DEVICE_ONLINE=0"; break; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 92e8ac85f58e8..300caafcfa100 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -186,9 +186,15 @@ static ssize_t max_link_speed_show(struct device *dev, struct device_attribute *attr, char *buf) { struct pci_dev *pdev = to_pci_dev(dev); + ssize_t ret; - return sysfs_emit(buf, "%s\n", - pci_speed_string(pcie_get_speed_cap(pdev))); + /* We read PCI_EXP_LNKCAP, so we need the device to be accessible. */ + pci_config_pm_runtime_get(pdev); + ret = sysfs_emit(buf, "%s\n", + pci_speed_string(pcie_get_speed_cap(pdev))); + pci_config_pm_runtime_put(pdev); + + return ret; } static DEVICE_ATTR_RO(max_link_speed); @@ -196,8 +202,14 @@ static ssize_t max_link_width_show(struct device *dev, struct device_attribute *attr, char *buf) { struct pci_dev *pdev = to_pci_dev(dev); + ssize_t ret; + + /* We read PCI_EXP_LNKCAP, so we need the device to be accessible. */ + pci_config_pm_runtime_get(pdev); + ret = sysfs_emit(buf, "%u\n", pcie_get_width_cap(pdev)); + pci_config_pm_runtime_put(pdev); - return sysfs_emit(buf, "%u\n", pcie_get_width_cap(pdev)); + return ret; } static DEVICE_ATTR_RO(max_link_width); @@ -209,7 +221,10 @@ static ssize_t current_link_speed_show(struct device *dev, int err; enum pci_bus_speed speed; + pci_config_pm_runtime_get(pci_dev); err = pcie_capability_read_word(pci_dev, PCI_EXP_LNKSTA, &linkstat); + pci_config_pm_runtime_put(pci_dev); + if (err) return -EINVAL; @@ -226,7 +241,10 @@ static ssize_t current_link_width_show(struct device *dev, u16 linkstat; int err; + pci_config_pm_runtime_get(pci_dev); err = pcie_capability_read_word(pci_dev, PCI_EXP_LNKSTA, &linkstat); + pci_config_pm_runtime_put(pci_dev); + if (err) return -EINVAL; @@ -242,7 +260,10 @@ static ssize_t secondary_bus_number_show(struct device *dev, u8 sec_bus; int err; + pci_config_pm_runtime_get(pci_dev); err = pci_read_config_byte(pci_dev, PCI_SECONDARY_BUS, &sec_bus); + pci_config_pm_runtime_put(pci_dev); + if (err) return -EINVAL; @@ -258,7 +279,10 @@ static ssize_t subordinate_bus_number_show(struct device *dev, u8 sub_bus; int err; + pci_config_pm_runtime_get(pci_dev); err = pci_read_config_byte(pci_dev, PCI_SUBORDINATE_BUS, &sub_bus); + pci_config_pm_runtime_put(pci_dev); + if (err) return -EINVAL; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index c09ec5c6d1399..c295c56c94bae 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1257,6 +1257,11 @@ int pci_power_up(struct pci_dev *dev) return -EIO; } + if (pci_dev_is_disconnected(dev)) { + dev->current_state = PCI_D3cold; + return -EIO; + } + pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); if (PCI_POSSIBLE_ERROR(pmcsr)) { pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n", @@ -5176,7 +5181,7 @@ int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type) delay); if (!pcie_wait_for_link_delay(dev, true, delay)) { /* Did not train, no need to wait any further */ - pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n"); + pci_info(dev, "Data Link Layer Link Active not set in %d msec\n", delay); return -ENOTTY; } @@ -5778,7 +5783,8 @@ static void pci_slot_unlock(struct pci_slot *slot) continue; if (dev->subordinate) pci_bus_unlock(dev->subordinate); - pci_dev_unlock(dev); + else + pci_dev_unlock(dev); } } diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 6fe041ba74163..4254960031b53 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -646,6 +646,7 @@ int of_pci_get_max_link_speed(struct device_node *node); u32 of_pci_get_slot_power_limit(struct device_node *node, u8 *slot_power_limit_value, u8 *slot_power_limit_scale); +bool of_pci_preserve_config(struct device_node *node); int pci_set_of_node(struct pci_dev *dev); void pci_release_of_node(struct pci_dev *dev); void pci_set_bus_of_node(struct pci_bus *bus); @@ -684,6 +685,11 @@ of_pci_get_slot_power_limit(struct device_node *node, return 0; } +static inline bool of_pci_preserve_config(struct device_node *node) +{ + return false; +} + static inline int pci_set_of_node(struct pci_dev *dev) { return 0; } static inline void pci_release_of_node(struct pci_dev *dev) { } static inline void pci_set_bus_of_node(struct pci_bus *bus) { } @@ -730,6 +736,7 @@ static inline void pci_restore_aer_state(struct pci_dev *dev) { } #endif #ifdef CONFIG_ACPI +bool pci_acpi_preserve_config(struct pci_host_bridge *bridge); int pci_acpi_program_hp_params(struct pci_dev *dev); extern const struct attribute_group pci_dev_acpi_attr_group; void pci_set_acpi_fwnode(struct pci_dev *dev); @@ -743,6 +750,10 @@ int acpi_pci_wakeup(struct pci_dev *dev, bool enable); bool acpi_pci_need_resume(struct pci_dev *dev); pci_power_t acpi_pci_choose_state(struct pci_dev *pdev); #else +static inline bool pci_acpi_preserve_config(struct pci_host_bridge *bridge) +{ + return false; +} static inline int pci_dev_acpi_reset(struct pci_dev *dev, bool probe) { return -ENOTTY; diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index eeb9ea9044b43..8e700020ee0bc 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -38,7 +38,7 @@ #define AER_ERROR_SOURCES_MAX 128 #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */ -#define AER_MAX_TYPEOF_UNCOR_ERRS 27 /* as per PCI_ERR_UNCOR_STATUS*/ +#define AER_MAX_TYPEOF_UNCOR_ERRS 32 /* as per PCI_ERR_UNCOR_STATUS*/ struct aer_err_source { unsigned int status; @@ -510,11 +510,11 @@ static const char *aer_uncorrectable_error_string[] = { "AtomicOpBlocked", /* Bit Position 24 */ "TLPBlockedErr", /* Bit Position 25 */ "PoisonTLPBlocked", /* Bit Position 26 */ - NULL, /* Bit Position 27 */ - NULL, /* Bit Position 28 */ - NULL, /* Bit Position 29 */ - NULL, /* Bit Position 30 */ - NULL, /* Bit Position 31 */ + "DMWrReqBlocked", /* Bit Position 27 */ + "IDECheck", /* Bit Position 28 */ + "MisIDETLP", /* Bit Position 29 */ + "PCRC_CHECK", /* Bit Position 30 */ + "TLPXlatBlocked", /* Bit Position 31 */ }; static const char *aer_agent_string[] = { diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c index a5cec2a4e057d..3c3ecb9cf57af 100644 --- a/drivers/pci/pcie/dpc.c +++ b/drivers/pci/pcie/dpc.c @@ -263,7 +263,7 @@ static int dpc_get_aer_uncorrect_severity(struct pci_dev *dev, void dpc_process_error(struct pci_dev *pdev) { u16 cap = pdev->dpc_cap, status, source, reason, ext_reason; - struct aer_err_info info; + struct aer_err_info info = {}; pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status); pci_read_config_word(pdev, cap + PCI_EXP_DPC_SOURCE_ID, &source); diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index 705893b5f7b09..197191178cddc 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -108,6 +108,12 @@ static int report_normal_detected(struct pci_dev *dev, void *data) return report_error_detected(dev, pci_channel_io_normal, data); } +static int report_perm_failure_detected(struct pci_dev *dev, void *data) +{ + pci_uevent_ers(dev, PCI_ERS_RESULT_DISCONNECT); + return 0; +} + static int report_mmio_enabled(struct pci_dev *dev, void *data) { struct pci_driver *pdrv; @@ -275,7 +281,7 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, failed: pci_walk_bridge(bridge, pci_pm_runtime_put, NULL); - pci_uevent_ers(bridge, PCI_ERS_RESULT_DISCONNECT); + pci_walk_bridge(bridge, report_perm_failure_detected, NULL); /* TODO: Should kernel panic here? */ pci_info(bridge, "device recovery failed\n"); diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index f22b0f3e278b4..42af31ec43c2d 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -877,6 +877,17 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus) dev_set_msi_domain(&bus->dev, d); } +static bool pci_preserve_config(struct pci_host_bridge *host_bridge) +{ + if (pci_acpi_preserve_config(host_bridge)) + return true; + + if (host_bridge->dev.parent && host_bridge->dev.parent->of_node) + return of_pci_preserve_config(host_bridge->dev.parent->of_node); + + return false; +} + static int pci_register_host_bridge(struct pci_host_bridge *bridge) { struct device *parent = bridge->dev.parent; @@ -972,6 +983,9 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE) dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n"); + /* Check if the boot configuration by FW needs to be preserved */ + bridge->preserve_config = pci_preserve_config(bridge); + /* Coalesce contiguous windows */ resource_list_for_each_entry_safe(window, n, &resources) { if (list_is_last(&window->node, &resources)) @@ -3043,20 +3057,18 @@ int pci_host_probe(struct pci_host_bridge *bridge) bus = bridge->bus; + /* If we must preserve the resource configuration, claim now */ + if (bridge->preserve_config) + pci_bus_claim_resources(bus); + /* - * We insert PCI resources into the iomem_resource and - * ioport_resource trees in either pci_bus_claim_resources() - * or pci_bus_assign_resources(). + * Assign whatever was left unassigned. If we didn't claim above, + * this will reassign everything. */ - if (pci_has_flag(PCI_PROBE_ONLY)) { - pci_bus_claim_resources(bus); - } else { - pci_bus_size_bridges(bus); - pci_bus_assign_resources(bus); + pci_assign_unassigned_root_bus_resources(bus); - list_for_each_entry(child, &bus->children, node) - pcie_bus_configure_settings(child); - } + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); pci_bus_add_devices(bus); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 70f484b811dea..aae702063ba3d 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -2712,6 +2712,7 @@ static void quirk_disable_msi(struct pci_dev *dev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8131_BRIDGE, quirk_disable_msi); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, 0xa238, quirk_disable_msi); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x5a3f, quirk_disable_msi); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_RDC, 0x1031, quirk_disable_msi); /* * The APC bridge device in AMD 780 family northbridges has some random @@ -4988,6 +4989,18 @@ static int pci_quirk_brcm_acs(struct pci_dev *dev, u16 acs_flags) PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF); } +static int pci_quirk_loongson_acs(struct pci_dev *dev, u16 acs_flags) +{ + /* + * Loongson PCIe Root Ports don't advertise an ACS capability, but + * they do not allow peer-to-peer transactions between Root Ports. + * Allow each Root Port to be in a separate IOMMU group by masking + * SV/RR/CR/UF bits. + */ + return pci_acs_ctrl_enabled(acs_flags, + PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF); +} + /* * Wangxun 40G/25G/10G/1G NICs have no ACS capability, but on * multi-function devices, the hardware isolates the functions by @@ -5121,6 +5134,17 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_BROADCOM, 0x1762, pci_quirk_mf_endpoint_acs }, { PCI_VENDOR_ID_BROADCOM, 0x1763, pci_quirk_mf_endpoint_acs }, { PCI_VENDOR_ID_BROADCOM, 0xD714, pci_quirk_brcm_acs }, + /* Loongson PCIe Root Ports */ + { PCI_VENDOR_ID_LOONGSON, 0x3C09, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x3C19, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x3C29, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x7A09, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x7A19, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x7A29, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x7A39, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x7A49, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x7A59, pci_quirk_loongson_acs }, + { PCI_VENDOR_ID_LOONGSON, 0x7A69, pci_quirk_loongson_acs }, /* Amazon Annapurna Labs */ { PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS, 0x0031, pci_quirk_al_acs }, /* Zhaoxin multi-function devices */ diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index fba402f4f6330..3f40be417856e 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -802,11 +802,9 @@ static resource_size_t calculate_iosize(resource_size_t size, size = (size & 0xff) + ((size & ~0xffUL) << 2); #endif size = size + size1; - if (size < old_size) - size = old_size; - size = ALIGN(max(size, add_size) + children_add_size, align); - return size; + size = max(size, add_size) + children_add_size; + return ALIGN(max(size, old_size), align); } static resource_size_t calculate_memsize(resource_size_t size, diff --git a/drivers/pcmcia/omap_cf.c b/drivers/pcmcia/omap_cf.c index e613818dc0bc9..a8e0dd5d30c43 100644 --- a/drivers/pcmcia/omap_cf.c +++ b/drivers/pcmcia/omap_cf.c @@ -215,6 +215,8 @@ static int __init omap_cf_probe(struct platform_device *pdev) return -EINVAL; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; cf = kzalloc(sizeof *cf, GFP_KERNEL); if (!cf) @@ -303,7 +305,13 @@ static int __exit omap_cf_remove(struct platform_device *pdev) return 0; } -static struct platform_driver omap_cf_driver = { +/* + * omap_cf_remove() lives in .exit.text. For drivers registered via + * platform_driver_probe() this is ok because they cannot get unbound at + * runtime. So mark the driver struct with __refdata to prevent modpost + * triggering a section mismatch warning. + */ +static struct platform_driver omap_cf_driver __refdata = { .driver = { .name = driver_name, }, diff --git a/drivers/pcmcia/rsrc_iodyn.c b/drivers/pcmcia/rsrc_iodyn.c index b04b16496b0c4..2677b577c1f85 100644 --- a/drivers/pcmcia/rsrc_iodyn.c +++ b/drivers/pcmcia/rsrc_iodyn.c @@ -62,6 +62,9 @@ static struct resource *__iodyn_find_io_region(struct pcmcia_socket *s, unsigned long min = base; int ret; + if (!res) + return NULL; + data.mask = align - 1; data.offset = base & data.mask; diff --git a/drivers/pcmcia/rsrc_nonstatic.c b/drivers/pcmcia/rsrc_nonstatic.c index bf9d070a44966..da494fe451baf 100644 --- a/drivers/pcmcia/rsrc_nonstatic.c +++ b/drivers/pcmcia/rsrc_nonstatic.c @@ -375,7 +375,9 @@ static int do_validate_mem(struct pcmcia_socket *s, if (validate && !s->fake_cis) { /* move it to the validated data set */ - add_interval(&s_data->mem_db_valid, base, size); + ret = add_interval(&s_data->mem_db_valid, base, size); + if (ret) + return ret; sub_interval(&s_data->mem_db, base, size); } diff --git a/drivers/perf/amlogic/meson_ddr_pmu_core.c b/drivers/perf/amlogic/meson_ddr_pmu_core.c index bbc7285fd934a..5f8699612a9ad 100644 --- a/drivers/perf/amlogic/meson_ddr_pmu_core.c +++ b/drivers/perf/amlogic/meson_ddr_pmu_core.c @@ -510,7 +510,7 @@ int meson_ddr_pmu_create(struct platform_device *pdev) fmt_attr_fill(pmu->info.hw_info->fmt_attr); - pmu->cpu = smp_processor_id(); + pmu->cpu = raw_smp_processor_id(); name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME); if (!name) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 7bd1733d79770..77aa37de59880 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -684,8 +684,8 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, if ((chan == 5 && cmn->rsp_vc_num < 2) || (chan == 6 && cmn->dat_vc_num < 2) || - (chan == 7 && cmn->snp_vc_num < 2) || - (chan == 8 && cmn->req_vc_num < 2)) + (chan == 7 && cmn->req_vc_num < 2) || + (chan == 8 && cmn->snp_vc_num < 2)) return 0; } @@ -841,8 +841,8 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, _CMN_EVENT_XP(pub_##_name, (_event) | (4 << 5)), \ _CMN_EVENT_XP(rsp2_##_name, (_event) | (5 << 5)), \ _CMN_EVENT_XP(dat2_##_name, (_event) | (6 << 5)), \ - _CMN_EVENT_XP(snp2_##_name, (_event) | (7 << 5)), \ - _CMN_EVENT_XP(req2_##_name, (_event) | (8 << 5)) + _CMN_EVENT_XP(req2_##_name, (_event) | (7 << 5)), \ + _CMN_EVENT_XP(snp2_##_name, (_event) | (8 << 5)) #define CMN_EVENT_XP_DAT(_name, _event) \ _CMN_EVENT_XP_PORT(dat_##_name, (_event) | (3 << 5)), \ @@ -2443,6 +2443,7 @@ static int arm_cmn_probe(struct platform_device *pdev) cmn->dev = &pdev->dev; cmn->part = (unsigned long)device_get_match_data(cmn->dev); + cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); platform_set_drvdata(pdev, cmn); if (cmn->part == PART_CMN600 && has_acpi_companion(cmn->dev)) { @@ -2470,7 +2471,6 @@ static int arm_cmn_probe(struct platform_device *pdev) if (err) return err; - cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); cmn->pmu = (struct pmu) { .module = THIS_MODULE, .attr_groups = arm_cmn_attr_groups, diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 291eb1f2d932a..bbc579b3efae3 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -751,10 +751,10 @@ static void armv8pmu_start(struct arm_pmu *cpu_pmu) else armv8pmu_disable_user_access(); + kvm_vcpu_pmu_resync_el0(); + /* Enable all counters */ armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E); - - kvm_vcpu_pmu_resync_el0(); } static void armv8pmu_stop(struct arm_pmu *cpu_pmu) diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index 2bec2e3af0bd6..affa78376b6a8 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -96,7 +96,8 @@ struct arm_spe_pmu { #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu)) /* Convert a free-running index from perf into an SPE buffer offset */ -#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) +#define PERF_IDX2OFF(idx, buf) \ + ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) /* Keep track of our dynamic hotplug state */ static enum cpuhp_state arm_spe_pmu_online; diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 308c9969642e1..c03df0f528898 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -881,7 +881,7 @@ static int cxl_pmu_probe(struct device *dev) return rc; irq = rc; - irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow\n", dev_name); + irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow", dev_name); if (!irq_name) return -ENOMEM; diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c index 04031450d5fec..c3013059cca82 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c @@ -212,7 +212,7 @@ int hisi_uncore_pmu_event_init(struct perf_event *event) return -EINVAL; hisi_pmu = to_hisi_pmu(event->pmu); - if (event->attr.config > hisi_pmu->check_event) + if ((event->attr.config & HISI_EVENTID_MASK) > hisi_pmu->check_event) return -EINVAL; if (hisi_pmu->on_cpu == -1) diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.h b/drivers/perf/hisilicon/hisi_uncore_pmu.h index 92402aa69d70f..67d1c3d3a41c0 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pmu.h +++ b/drivers/perf/hisilicon/hisi_uncore_pmu.h @@ -43,7 +43,8 @@ return FIELD_GET(GENMASK_ULL(hi, lo), event->attr.config); \ } -#define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff) +#define HISI_EVENTID_MASK GENMASK(7, 0) +#define HISI_GET_EVENTID(ev) ((ev)->hw.config_base & HISI_EVENTID_MASK) #define HISI_PMU_EVTYPE_BITS 8 #define HISI_PMU_EVTYPE_SHIFT(idx) ((idx) % 4 * HISI_PMU_EVTYPE_BITS) diff --git a/drivers/phy/broadcom/phy-bcm-ns-usb3.c b/drivers/phy/broadcom/phy-bcm-ns-usb3.c index 69584b685edbb..2c8b1b7dda5bd 100644 --- a/drivers/phy/broadcom/phy-bcm-ns-usb3.c +++ b/drivers/phy/broadcom/phy-bcm-ns-usb3.c @@ -16,10 +16,11 @@ #include #include #include +#include #include -#include #include #include +#include #include #define BCM_NS_USB3_PHY_BASE_ADDR_REG 0x1f @@ -189,7 +190,6 @@ static int bcm_ns_usb3_mdio_phy_write(struct bcm_ns_usb3 *usb3, u16 reg, static int bcm_ns_usb3_mdio_probe(struct mdio_device *mdiodev) { struct device *dev = &mdiodev->dev; - const struct of_device_id *of_id; struct phy_provider *phy_provider; struct device_node *syscon_np; struct bcm_ns_usb3 *usb3; @@ -203,10 +203,7 @@ static int bcm_ns_usb3_mdio_probe(struct mdio_device *mdiodev) usb3->dev = dev; usb3->mdiodev = mdiodev; - of_id = of_match_device(bcm_ns_usb3_id_table, dev); - if (!of_id) - return -EINVAL; - usb3->family = (uintptr_t)of_id->data; + usb3->family = (enum bcm_ns_family)device_get_match_data(dev); syscon_np = of_parse_phandle(dev->of_node, "usb3-dmp-syscon", 0); err = of_address_to_resource(syscon_np, 0, &res); diff --git a/drivers/phy/cadence/cdns-dphy.c b/drivers/phy/cadence/cdns-dphy.c index dddb66de6dba1..a24df36e5c376 100644 --- a/drivers/phy/cadence/cdns-dphy.c +++ b/drivers/phy/cadence/cdns-dphy.c @@ -30,6 +30,7 @@ #define DPHY_CMN_SSM DPHY_PMA_CMN(0x20) #define DPHY_CMN_SSM_EN BIT(0) +#define DPHY_CMN_SSM_CAL_WAIT_TIME GENMASK(8, 1) #define DPHY_CMN_TX_MODE_EN BIT(9) #define DPHY_CMN_PWM DPHY_PMA_CMN(0x40) @@ -79,6 +80,7 @@ struct cdns_dphy_cfg { u8 pll_ipdiv; u8 pll_opdiv; u16 pll_fbdiv; + u32 hs_clk_rate; unsigned int nlanes; }; @@ -99,6 +101,8 @@ struct cdns_dphy_ops { void (*set_pll_cfg)(struct cdns_dphy *dphy, const struct cdns_dphy_cfg *cfg); unsigned long (*get_wakeup_time_ns)(struct cdns_dphy *dphy); + int (*wait_for_pll_lock)(struct cdns_dphy *dphy); + int (*wait_for_cmn_ready)(struct cdns_dphy *dphy); }; struct cdns_dphy { @@ -108,6 +112,8 @@ struct cdns_dphy { struct clk *pll_ref_clk; const struct cdns_dphy_ops *ops; struct phy *phy; + bool is_configured; + bool is_powered; }; /* Order of bands is important since the index is the band number. */ @@ -139,7 +145,7 @@ static int cdns_dsi_get_dphy_pll_cfg(struct cdns_dphy *dphy, dlane_bps = opts->hs_clk_rate; - if (dlane_bps > 2500000000UL || dlane_bps < 160000000UL) + if (dlane_bps > 2500000000UL || dlane_bps < 80000000UL) return -EINVAL; else if (dlane_bps >= 1250000000) cfg->pll_opdiv = 1; @@ -149,11 +155,16 @@ static int cdns_dsi_get_dphy_pll_cfg(struct cdns_dphy *dphy, cfg->pll_opdiv = 4; else if (dlane_bps >= 160000000) cfg->pll_opdiv = 8; + else if (dlane_bps >= 80000000) + cfg->pll_opdiv = 16; cfg->pll_fbdiv = DIV_ROUND_UP_ULL(dlane_bps * 2 * cfg->pll_opdiv * cfg->pll_ipdiv, pll_ref_hz); + cfg->hs_clk_rate = div_u64((u64)pll_ref_hz * cfg->pll_fbdiv, + 2 * cfg->pll_opdiv * cfg->pll_ipdiv); + return 0; } @@ -191,6 +202,16 @@ static unsigned long cdns_dphy_get_wakeup_time_ns(struct cdns_dphy *dphy) return dphy->ops->get_wakeup_time_ns(dphy); } +static int cdns_dphy_wait_for_pll_lock(struct cdns_dphy *dphy) +{ + return dphy->ops->wait_for_pll_lock ? dphy->ops->wait_for_pll_lock(dphy) : 0; +} + +static int cdns_dphy_wait_for_cmn_ready(struct cdns_dphy *dphy) +{ + return dphy->ops->wait_for_cmn_ready ? dphy->ops->wait_for_cmn_ready(dphy) : 0; +} + static unsigned long cdns_dphy_ref_get_wakeup_time_ns(struct cdns_dphy *dphy) { /* Default wakeup time is 800 ns (in a simulated environment). */ @@ -232,7 +253,6 @@ static unsigned long cdns_dphy_j721e_get_wakeup_time_ns(struct cdns_dphy *dphy) static void cdns_dphy_j721e_set_pll_cfg(struct cdns_dphy *dphy, const struct cdns_dphy_cfg *cfg) { - u32 status; /* * set the PWM and PLL Byteclk divider settings to recommended values @@ -249,13 +269,6 @@ static void cdns_dphy_j721e_set_pll_cfg(struct cdns_dphy *dphy, writel(DPHY_TX_J721E_WIZ_LANE_RSTB, dphy->regs + DPHY_TX_J721E_WIZ_RST_CTRL); - - readl_poll_timeout(dphy->regs + DPHY_TX_J721E_WIZ_PLL_CTRL, status, - (status & DPHY_TX_WIZ_PLL_LOCK), 0, POLL_TIMEOUT_US); - - readl_poll_timeout(dphy->regs + DPHY_TX_J721E_WIZ_STATUS, status, - (status & DPHY_TX_WIZ_O_CMN_READY), 0, - POLL_TIMEOUT_US); } static void cdns_dphy_j721e_set_psm_div(struct cdns_dphy *dphy, u8 div) @@ -263,6 +276,23 @@ static void cdns_dphy_j721e_set_psm_div(struct cdns_dphy *dphy, u8 div) writel(div, dphy->regs + DPHY_TX_J721E_WIZ_PSM_FREQ); } +static int cdns_dphy_j721e_wait_for_pll_lock(struct cdns_dphy *dphy) +{ + u32 status; + + return readl_poll_timeout(dphy->regs + DPHY_TX_J721E_WIZ_PLL_CTRL, status, + status & DPHY_TX_WIZ_PLL_LOCK, 0, POLL_TIMEOUT_US); +} + +static int cdns_dphy_j721e_wait_for_cmn_ready(struct cdns_dphy *dphy) +{ + u32 status; + + return readl_poll_timeout(dphy->regs + DPHY_TX_J721E_WIZ_STATUS, status, + status & DPHY_TX_WIZ_O_CMN_READY, 0, + POLL_TIMEOUT_US); +} + /* * This is the reference implementation of DPHY hooks. Specific integration of * this IP may have to re-implement some of them depending on how they decided @@ -278,6 +308,8 @@ static const struct cdns_dphy_ops j721e_dphy_ops = { .get_wakeup_time_ns = cdns_dphy_j721e_get_wakeup_time_ns, .set_pll_cfg = cdns_dphy_j721e_set_pll_cfg, .set_psm_div = cdns_dphy_j721e_set_psm_div, + .wait_for_pll_lock = cdns_dphy_j721e_wait_for_pll_lock, + .wait_for_cmn_ready = cdns_dphy_j721e_wait_for_cmn_ready, }; static int cdns_dphy_config_from_opts(struct phy *phy, @@ -297,6 +329,7 @@ static int cdns_dphy_config_from_opts(struct phy *phy, if (ret) return ret; + opts->hs_clk_rate = cfg->hs_clk_rate; opts->wakeup = cdns_dphy_get_wakeup_time_ns(dphy) / 1000; return 0; @@ -334,21 +367,36 @@ static int cdns_dphy_validate(struct phy *phy, enum phy_mode mode, int submode, static int cdns_dphy_configure(struct phy *phy, union phy_configure_opts *opts) { struct cdns_dphy *dphy = phy_get_drvdata(phy); - struct cdns_dphy_cfg cfg = { 0 }; - int ret, band_ctrl; - unsigned int reg; + int ret; - ret = cdns_dphy_config_from_opts(phy, &opts->mipi_dphy, &cfg); - if (ret) - return ret; + ret = cdns_dphy_config_from_opts(phy, &opts->mipi_dphy, &dphy->cfg); + if (!ret) + dphy->is_configured = true; + + return ret; +} + +static int cdns_dphy_power_on(struct phy *phy) +{ + struct cdns_dphy *dphy = phy_get_drvdata(phy); + int ret; + u32 reg; + + if (!dphy->is_configured || dphy->is_powered) + return -EINVAL; + + clk_prepare_enable(dphy->psm_clk); + clk_prepare_enable(dphy->pll_ref_clk); /* * Configure the internal PSM clk divider so that the DPHY has a * 1MHz clk (or something close). */ ret = cdns_dphy_setup_psm(dphy); - if (ret) - return ret; + if (ret) { + dev_err(&dphy->phy->dev, "Failed to setup PSM with error %d\n", ret); + goto err_power_on; + } /* * Configure attach clk lanes to data lanes: the DPHY has 2 clk lanes @@ -363,40 +411,61 @@ static int cdns_dphy_configure(struct phy *phy, union phy_configure_opts *opts) * Configure the DPHY PLL that will be used to generate the TX byte * clk. */ - cdns_dphy_set_pll_cfg(dphy, &cfg); + cdns_dphy_set_pll_cfg(dphy, &dphy->cfg); - band_ctrl = cdns_dphy_tx_get_band_ctrl(opts->mipi_dphy.hs_clk_rate); - if (band_ctrl < 0) - return band_ctrl; + ret = cdns_dphy_tx_get_band_ctrl(dphy->cfg.hs_clk_rate); + if (ret < 0) { + dev_err(&dphy->phy->dev, "Failed to get band control value with error %d\n", ret); + goto err_power_on; + } - reg = FIELD_PREP(DPHY_BAND_CFG_LEFT_BAND, band_ctrl) | - FIELD_PREP(DPHY_BAND_CFG_RIGHT_BAND, band_ctrl); + reg = FIELD_PREP(DPHY_BAND_CFG_LEFT_BAND, ret) | + FIELD_PREP(DPHY_BAND_CFG_RIGHT_BAND, ret); writel(reg, dphy->regs + DPHY_BAND_CFG); - return 0; -} + /* Start TX state machine. */ + reg = readl(dphy->regs + DPHY_CMN_SSM); + writel((reg & DPHY_CMN_SSM_CAL_WAIT_TIME) | DPHY_CMN_SSM_EN | DPHY_CMN_TX_MODE_EN, + dphy->regs + DPHY_CMN_SSM); -static int cdns_dphy_power_on(struct phy *phy) -{ - struct cdns_dphy *dphy = phy_get_drvdata(phy); + ret = cdns_dphy_wait_for_pll_lock(dphy); + if (ret) { + dev_err(&dphy->phy->dev, "Failed to lock PLL with error %d\n", ret); + goto err_power_on; + } - clk_prepare_enable(dphy->psm_clk); - clk_prepare_enable(dphy->pll_ref_clk); + ret = cdns_dphy_wait_for_cmn_ready(dphy); + if (ret) { + dev_err(&dphy->phy->dev, "O_CMN_READY signal failed to assert with error %d\n", + ret); + goto err_power_on; + } - /* Start TX state machine. */ - writel(DPHY_CMN_SSM_EN | DPHY_CMN_TX_MODE_EN, - dphy->regs + DPHY_CMN_SSM); + dphy->is_powered = true; return 0; + +err_power_on: + clk_disable_unprepare(dphy->pll_ref_clk); + clk_disable_unprepare(dphy->psm_clk); + + return ret; } static int cdns_dphy_power_off(struct phy *phy) { struct cdns_dphy *dphy = phy_get_drvdata(phy); + u32 reg; clk_disable_unprepare(dphy->pll_ref_clk); clk_disable_unprepare(dphy->psm_clk); + /* Stop TX state machine. */ + reg = readl(dphy->regs + DPHY_CMN_SSM); + writel(reg & ~DPHY_CMN_SSM_EN, dphy->regs + DPHY_CMN_SSM); + + dphy->is_powered = false; + return 0; } diff --git a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c index adc6394626ce8..f914f016b3d2c 100644 --- a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c +++ b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c @@ -95,12 +95,12 @@ static u32 phy_tx_preemp_amp_tune_from_property(u32 microamp) static u32 phy_tx_vboost_level_from_property(u32 microvolt) { switch (microvolt) { - case 0 ... 960: - return 0; - case 961 ... 1160: - return 2; - default: + case 1156: + return 5; + case 844: return 3; + default: + return 4; } } diff --git a/drivers/phy/marvell/phy-berlin-usb.c b/drivers/phy/marvell/phy-berlin-usb.c index 78ef6ae72a9a7..f26bf630da2c9 100644 --- a/drivers/phy/marvell/phy-berlin-usb.c +++ b/drivers/phy/marvell/phy-berlin-usb.c @@ -8,9 +8,10 @@ #include #include -#include +#include #include #include +#include #include #define USB_PHY_PLL 0x04 @@ -162,8 +163,6 @@ MODULE_DEVICE_TABLE(of, phy_berlin_usb_of_match); static int phy_berlin_usb_probe(struct platform_device *pdev) { - const struct of_device_id *match = - of_match_device(phy_berlin_usb_of_match, &pdev->dev); struct phy_berlin_usb_priv *priv; struct phy *phy; struct phy_provider *phy_provider; @@ -180,7 +179,7 @@ static int phy_berlin_usb_probe(struct platform_device *pdev) if (IS_ERR(priv->rst_ctrl)) return PTR_ERR(priv->rst_ctrl); - priv->pll_divider = *((u32 *)match->data); + priv->pll_divider = *((u32 *)device_get_match_data(&pdev->dev)); phy = devm_phy_create(&pdev->dev, NULL, &phy_berlin_usb_ops); if (IS_ERR(phy)) { diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c index a892e1d7e2d02..9417372a01554 100644 --- a/drivers/phy/phy-core.c +++ b/drivers/phy/phy-core.c @@ -400,13 +400,14 @@ EXPORT_SYMBOL_GPL(phy_power_off); int phy_set_mode_ext(struct phy *phy, enum phy_mode mode, int submode) { - int ret; + int ret = 0; - if (!phy || !phy->ops->set_mode) + if (!phy) return 0; mutex_lock(&phy->mutex); - ret = phy->ops->set_mode(phy, mode, submode); + if (phy->ops->set_mode) + ret = phy->ops->set_mode(phy, mode, submode); if (!ret) phy->attrs.mode = mode; mutex_unlock(&phy->mutex); diff --git a/drivers/phy/qualcomm/phy-qcom-m31.c b/drivers/phy/qualcomm/phy-qcom-m31.c index 89c9d74e35466..a5e60039a264a 100644 --- a/drivers/phy/qualcomm/phy-qcom-m31.c +++ b/drivers/phy/qualcomm/phy-qcom-m31.c @@ -58,14 +58,16 @@ #define USB2_0_TX_ENABLE BIT(2) #define USB2PHY_USB_PHY_M31_XCFGI_4 0xc8 - #define HSTX_SLEW_RATE_565PS GENMASK(1, 0) + #define HSTX_SLEW_RATE_400PS GENMASK(2, 0) #define PLL_CHARGING_PUMP_CURRENT_35UA GENMASK(4, 3) #define ODT_VALUE_38_02_OHM GENMASK(7, 6) #define USB2PHY_USB_PHY_M31_XCFGI_5 0xcc - #define ODT_VALUE_45_02_OHM BIT(2) #define HSTX_PRE_EMPHASIS_LEVEL_0_55MA BIT(0) +#define USB2PHY_USB_PHY_M31_XCFGI_9 0xdc + #define HSTX_CURRENT_17_1MA_385MV BIT(1) + #define USB2PHY_USB_PHY_M31_XCFGI_11 0xe4 #define XCFG_COARSE_TUNE_NUM BIT(1) #define XCFG_FINE_TUNE_NUM BIT(3) @@ -120,7 +122,7 @@ static struct m31_phy_regs m31_ipq5332_regs[] = { }, { USB2PHY_USB_PHY_M31_XCFGI_4, - HSTX_SLEW_RATE_565PS | PLL_CHARGING_PUMP_CURRENT_35UA | ODT_VALUE_38_02_OHM, + HSTX_SLEW_RATE_400PS | PLL_CHARGING_PUMP_CURRENT_35UA | ODT_VALUE_38_02_OHM, 0 }, { @@ -130,9 +132,13 @@ static struct m31_phy_regs m31_ipq5332_regs[] = { }, { USB2PHY_USB_PHY_M31_XCFGI_5, - ODT_VALUE_45_02_OHM | HSTX_PRE_EMPHASIS_LEVEL_0_55MA, + HSTX_PRE_EMPHASIS_LEVEL_0_55MA, 4 }, + { + USB2PHY_USB_PHY_M31_XCFGI_9, + HSTX_CURRENT_17_1MA_385MV, + }, { USB_PHY_UTMI_CTRL5, 0x0, diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c index c697d01b2a2a1..5072bca0af7c5 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c @@ -2044,12 +2044,16 @@ static void __iomem *qmp_usb_iomap(struct device *dev, struct device_node *np, int index, bool exclusive) { struct resource res; + void __iomem *mem; if (!exclusive) { if (of_address_to_resource(np, index, &res)) return IOMEM_ERR_PTR(-EINVAL); - return devm_ioremap(dev, res.start, resource_size(&res)); + mem = devm_ioremap(dev, res.start, resource_size(&res)); + if (!mem) + return IOMEM_ERR_PTR(-ENOMEM); + return mem; } return devm_of_iomap(dev, np, index, NULL); diff --git a/drivers/phy/ralink/phy-ralink-usb.c b/drivers/phy/ralink/phy-ralink-usb.c index 2bd8ad2e76eda..41bce5290e922 100644 --- a/drivers/phy/ralink/phy-ralink-usb.c +++ b/drivers/phy/ralink/phy-ralink-usb.c @@ -13,9 +13,10 @@ #include #include #include -#include +#include #include #include +#include #include #include @@ -171,18 +172,13 @@ static int ralink_usb_phy_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct phy_provider *phy_provider; - const struct of_device_id *match; struct ralink_usb_phy *phy; - match = of_match_device(ralink_usb_phy_of_match, &pdev->dev); - if (!match) - return -ENODEV; - phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL); if (!phy) return -ENOMEM; - phy->clk = (uintptr_t)match->data; + phy->clk = (uintptr_t)device_get_match_data(&pdev->dev); phy->base = NULL; phy->sysctl = syscon_regmap_lookup_by_phandle(dev->of_node, "ralink,sysctl"); diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c index 6387c0d34c551..9a6391361a0be 100644 --- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c +++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c @@ -9,6 +9,7 @@ * Copyright (C) 2014 Cogent Embedded, Inc. */ +#include #include #include #include @@ -19,12 +20,14 @@ #include #include #include +#include #include #include #include /******* USB2.0 Host registers (original offset is +0x200) *******/ #define USB2_INT_ENABLE 0x000 +#define USB2_AHB_BUS_CTR 0x008 #define USB2_USBCTR 0x00c #define USB2_SPD_RSM_TIMSET 0x10c #define USB2_OC_TIMSET 0x110 @@ -40,6 +43,10 @@ #define USB2_INT_ENABLE_USBH_INTB_EN BIT(2) /* For EHCI */ #define USB2_INT_ENABLE_USBH_INTA_EN BIT(1) /* For OHCI */ +/* AHB_BUS_CTR */ +#define USB2_AHB_BUS_CTR_MBL_MASK GENMASK(1, 0) +#define USB2_AHB_BUS_CTR_MBL_INCR4 2 + /* USBCTR */ #define USB2_USBCTR_DIRPD BIT(2) #define USB2_USBCTR_PLL_RST BIT(1) @@ -101,7 +108,6 @@ struct rcar_gen3_phy { struct rcar_gen3_chan *ch; u32 int_enable_bits; bool initialized; - bool otg_initialized; bool powered; }; @@ -111,10 +117,10 @@ struct rcar_gen3_chan { struct extcon_dev *extcon; struct rcar_gen3_phy rphys[NUM_OF_PHYS]; struct regulator *vbus; + struct reset_control *rstc; struct work_struct work; - struct mutex lock; /* protects rphys[...].powered */ + spinlock_t lock; /* protects access to hardware and driver data structure. */ enum usb_dr_mode dr_mode; - int irq; u32 obint_enable_bits; bool extcon_host; bool is_otg_channel; @@ -125,6 +131,7 @@ struct rcar_gen3_chan { struct rcar_gen3_phy_drv_data { const struct phy_ops *phy_usb2_ops; bool no_adp_ctrl; + bool init_bus; }; /* @@ -309,16 +316,15 @@ static bool rcar_gen3_is_any_rphy_initialized(struct rcar_gen3_chan *ch) return false; } -static bool rcar_gen3_needs_init_otg(struct rcar_gen3_chan *ch) +static bool rcar_gen3_is_any_otg_rphy_initialized(struct rcar_gen3_chan *ch) { - int i; - - for (i = 0; i < NUM_OF_PHYS; i++) { - if (ch->rphys[i].otg_initialized) - return false; + for (enum rcar_gen3_phy_index i = PHY_INDEX_BOTH_HC; i <= PHY_INDEX_EHCI; + i++) { + if (ch->rphys[i].initialized) + return true; } - return true; + return false; } static bool rcar_gen3_are_all_rphys_power_off(struct rcar_gen3_chan *ch) @@ -340,7 +346,9 @@ static ssize_t role_store(struct device *dev, struct device_attribute *attr, bool is_b_device; enum phy_mode cur_mode, new_mode; - if (!ch->is_otg_channel || !rcar_gen3_is_any_rphy_initialized(ch)) + guard(spinlock_irqsave)(&ch->lock); + + if (!ch->is_otg_channel || !rcar_gen3_is_any_otg_rphy_initialized(ch)) return -EIO; if (sysfs_streq(buf, "host")) @@ -378,7 +386,7 @@ static ssize_t role_show(struct device *dev, struct device_attribute *attr, { struct rcar_gen3_chan *ch = dev_get_drvdata(dev); - if (!ch->is_otg_channel || !rcar_gen3_is_any_rphy_initialized(ch)) + if (!ch->is_otg_channel || !rcar_gen3_is_any_otg_rphy_initialized(ch)) return -EIO; return sprintf(buf, "%s\n", rcar_gen3_is_host(ch) ? "host" : @@ -391,6 +399,9 @@ static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch) void __iomem *usb2_base = ch->base; u32 val; + if (!ch->is_otg_channel || rcar_gen3_is_any_otg_rphy_initialized(ch)) + return; + /* Should not use functions of read-modify-write a register */ val = readl(usb2_base + USB2_LINECTRL1); val = (val & ~USB2_LINECTRL1_DP_RPD) | USB2_LINECTRL1_DPRPD_EN | @@ -404,7 +415,7 @@ static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch) val = readl(usb2_base + USB2_ADPCTRL); writel(val | USB2_ADPCTRL_IDPULLUP, usb2_base + USB2_ADPCTRL); } - msleep(20); + mdelay(20); writel(0xffffffff, usb2_base + USB2_OBINTSTA); writel(ch->obint_enable_bits, usb2_base + USB2_OBINTEN); @@ -416,16 +427,27 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch) { struct rcar_gen3_chan *ch = _ch; void __iomem *usb2_base = ch->base; - u32 status = readl(usb2_base + USB2_OBINTSTA); + struct device *dev = ch->dev; irqreturn_t ret = IRQ_NONE; + u32 status; + + pm_runtime_get_noresume(dev); + + if (pm_runtime_suspended(dev)) + goto rpm_put; - if (status & ch->obint_enable_bits) { - dev_vdbg(ch->dev, "%s: %08x\n", __func__, status); - writel(ch->obint_enable_bits, usb2_base + USB2_OBINTSTA); - rcar_gen3_device_recognition(ch); - ret = IRQ_HANDLED; + scoped_guard(spinlock, &ch->lock) { + status = readl(usb2_base + USB2_OBINTSTA); + if (status & ch->obint_enable_bits) { + dev_vdbg(dev, "%s: %08x\n", __func__, status); + writel(ch->obint_enable_bits, usb2_base + USB2_OBINTSTA); + rcar_gen3_device_recognition(ch); + ret = IRQ_HANDLED; + } } +rpm_put: + pm_runtime_put_noidle(dev); return ret; } @@ -435,32 +457,23 @@ static int rcar_gen3_phy_usb2_init(struct phy *p) struct rcar_gen3_chan *channel = rphy->ch; void __iomem *usb2_base = channel->base; u32 val; - int ret; - if (!rcar_gen3_is_any_rphy_initialized(channel) && channel->irq >= 0) { - INIT_WORK(&channel->work, rcar_gen3_phy_usb2_work); - ret = request_irq(channel->irq, rcar_gen3_phy_usb2_irq, - IRQF_SHARED, dev_name(channel->dev), channel); - if (ret < 0) { - dev_err(channel->dev, "No irq handler (%d)\n", channel->irq); - return ret; - } - } + guard(spinlock_irqsave)(&channel->lock); /* Initialize USB2 part */ val = readl(usb2_base + USB2_INT_ENABLE); val |= USB2_INT_ENABLE_UCOM_INTEN | rphy->int_enable_bits; writel(val, usb2_base + USB2_INT_ENABLE); - writel(USB2_SPD_RSM_TIMSET_INIT, usb2_base + USB2_SPD_RSM_TIMSET); - writel(USB2_OC_TIMSET_INIT, usb2_base + USB2_OC_TIMSET); - - /* Initialize otg part */ - if (channel->is_otg_channel) { - if (rcar_gen3_needs_init_otg(channel)) - rcar_gen3_init_otg(channel); - rphy->otg_initialized = true; + + if (!rcar_gen3_is_any_rphy_initialized(channel)) { + writel(USB2_SPD_RSM_TIMSET_INIT, usb2_base + USB2_SPD_RSM_TIMSET); + writel(USB2_OC_TIMSET_INIT, usb2_base + USB2_OC_TIMSET); } + /* Initialize otg part (only if we initialize a PHY with IRQs). */ + if (rphy->int_enable_bits) + rcar_gen3_init_otg(channel); + rphy->initialized = true; return 0; @@ -473,10 +486,9 @@ static int rcar_gen3_phy_usb2_exit(struct phy *p) void __iomem *usb2_base = channel->base; u32 val; - rphy->initialized = false; + guard(spinlock_irqsave)(&channel->lock); - if (channel->is_otg_channel) - rphy->otg_initialized = false; + rphy->initialized = false; val = readl(usb2_base + USB2_INT_ENABLE); val &= ~rphy->int_enable_bits; @@ -484,9 +496,6 @@ static int rcar_gen3_phy_usb2_exit(struct phy *p) val &= ~USB2_INT_ENABLE_UCOM_INTEN; writel(val, usb2_base + USB2_INT_ENABLE); - if (channel->irq >= 0 && !rcar_gen3_is_any_rphy_initialized(channel)) - free_irq(channel->irq, channel); - return 0; } @@ -498,16 +507,17 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p) u32 val; int ret = 0; - mutex_lock(&channel->lock); - if (!rcar_gen3_are_all_rphys_power_off(channel)) - goto out; - if (channel->vbus) { ret = regulator_enable(channel->vbus); if (ret) - goto out; + return ret; } + guard(spinlock_irqsave)(&channel->lock); + + if (!rcar_gen3_are_all_rphys_power_off(channel)) + goto out; + val = readl(usb2_base + USB2_USBCTR); val |= USB2_USBCTR_PLL_RST; writel(val, usb2_base + USB2_USBCTR); @@ -517,7 +527,6 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p) out: /* The powered flag should be set for any other phys anyway */ rphy->powered = true; - mutex_unlock(&channel->lock); return 0; } @@ -528,18 +537,20 @@ static int rcar_gen3_phy_usb2_power_off(struct phy *p) struct rcar_gen3_chan *channel = rphy->ch; int ret = 0; - mutex_lock(&channel->lock); - rphy->powered = false; + scoped_guard(spinlock_irqsave, &channel->lock) { + rphy->powered = false; - if (!rcar_gen3_are_all_rphys_power_off(channel)) - goto out; + if (rcar_gen3_are_all_rphys_power_off(channel)) { + u32 val = readl(channel->base + USB2_USBCTR); + + val |= USB2_USBCTR_PLL_RST; + writel(val, channel->base + USB2_USBCTR); + } + } if (channel->vbus) ret = regulator_disable(channel->vbus); -out: - mutex_unlock(&channel->lock); - return ret; } @@ -647,13 +658,42 @@ static enum usb_dr_mode rcar_gen3_get_dr_mode(struct device_node *np) return candidate; } +static int rcar_gen3_phy_usb2_init_bus(struct rcar_gen3_chan *channel) +{ + struct device *dev = channel->dev; + int ret; + u32 val; + + channel->rstc = devm_reset_control_array_get_shared(dev); + if (IS_ERR(channel->rstc)) + return PTR_ERR(channel->rstc); + + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; + + ret = reset_control_deassert(channel->rstc); + if (ret) + goto rpm_put; + + val = readl(channel->base + USB2_AHB_BUS_CTR); + val &= ~USB2_AHB_BUS_CTR_MBL_MASK; + val |= USB2_AHB_BUS_CTR_MBL_INCR4; + writel(val, channel->base + USB2_AHB_BUS_CTR); + +rpm_put: + pm_runtime_put(dev); + + return ret; +} + static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev) { const struct rcar_gen3_phy_drv_data *phy_data; struct device *dev = &pdev->dev; struct rcar_gen3_chan *channel; struct phy_provider *provider; - int ret = 0, i; + int ret = 0, i, irq; if (!dev->of_node) { dev_err(dev, "This driver needs device tree\n"); @@ -669,8 +709,6 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev) return PTR_ERR(channel->base); channel->obint_enable_bits = USB2_OBINT_BITS; - /* get irq number here and request_irq for OTG in phy_init */ - channel->irq = platform_get_irq_optional(pdev, 0); channel->dr_mode = rcar_gen3_get_dr_mode(dev->of_node); if (channel->dr_mode != USB_DR_MODE_UNKNOWN) { channel->is_otg_channel = true; @@ -700,11 +738,20 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev) goto error; } + platform_set_drvdata(pdev, channel); + channel->dev = dev; + + if (phy_data->init_bus) { + ret = rcar_gen3_phy_usb2_init_bus(channel); + if (ret) + goto error; + } + channel->soc_no_adp_ctrl = phy_data->no_adp_ctrl; if (phy_data->no_adp_ctrl) channel->obint_enable_bits = USB2_OBINT_IDCHG_EN; - mutex_init(&channel->lock); + spin_lock_init(&channel->lock); for (i = 0; i < NUM_OF_PHYS; i++) { channel->rphys[i].phy = devm_phy_create(dev, NULL, phy_data->phy_usb2_ops); @@ -727,8 +774,19 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev) channel->vbus = NULL; } - platform_set_drvdata(pdev, channel); - channel->dev = dev; + irq = platform_get_irq_optional(pdev, 0); + if (irq < 0 && irq != -ENXIO) { + ret = irq; + goto error; + } else if (irq > 0) { + INIT_WORK(&channel->work, rcar_gen3_phy_usb2_work); + ret = devm_request_irq(dev, irq, rcar_gen3_phy_usb2_irq, + IRQF_SHARED, dev_name(dev), channel); + if (ret < 0) { + dev_err(dev, "Failed to request irq (%d)\n", irq); + goto error; + } + } provider = devm_of_phy_provider_register(dev, rcar_gen3_phy_usb2_xlate); if (IS_ERR(provider)) { @@ -756,6 +814,7 @@ static void rcar_gen3_phy_usb2_remove(struct platform_device *pdev) if (channel->is_otg_channel) device_remove_file(&pdev->dev, &dev_attr_role); + reset_control_assert(channel->rstc); pm_runtime_disable(&pdev->dev); }; diff --git a/drivers/phy/renesas/r8a779f0-ether-serdes.c b/drivers/phy/renesas/r8a779f0-ether-serdes.c index 683b19bc411a8..59546301a6b9e 100644 --- a/drivers/phy/renesas/r8a779f0-ether-serdes.c +++ b/drivers/phy/renesas/r8a779f0-ether-serdes.c @@ -49,6 +49,13 @@ static void r8a779f0_eth_serdes_write32(void __iomem *addr, u32 offs, u32 bank, iowrite32(data, addr + offs); } +static u32 r8a779f0_eth_serdes_read32(void __iomem *addr, u32 offs, u32 bank) +{ + iowrite32(bank, addr + R8A779F0_ETH_SERDES_BANK_SELECT); + + return ioread32(addr + offs); +} + static int r8a779f0_eth_serdes_reg_wait(struct r8a779f0_eth_serdes_channel *channel, u32 offs, u32 bank, u32 mask, u32 expected) @@ -261,6 +268,7 @@ static int r8a779f0_eth_serdes_hw_init_late(struct r8a779f0_eth_serdes_channel *channel) { int ret; + u32 val; ret = r8a779f0_eth_serdes_chan_setting(channel); if (ret) @@ -274,6 +282,26 @@ static int r8a779f0_eth_serdes_hw_init_late(struct r8a779f0_eth_serdes_channel r8a779f0_eth_serdes_write32(channel->addr, 0x03d0, 0x380, 0x0000); + val = r8a779f0_eth_serdes_read32(channel->addr, 0x00c0, 0x180); + r8a779f0_eth_serdes_write32(channel->addr, 0x00c0, 0x180, val | BIT(8)); + ret = r8a779f0_eth_serdes_reg_wait(channel, 0x0100, 0x180, BIT(0), 1); + if (ret) + return ret; + r8a779f0_eth_serdes_write32(channel->addr, 0x00c0, 0x180, val & ~BIT(8)); + ret = r8a779f0_eth_serdes_reg_wait(channel, 0x0100, 0x180, BIT(0), 0); + if (ret) + return ret; + + val = r8a779f0_eth_serdes_read32(channel->addr, 0x0144, 0x180); + r8a779f0_eth_serdes_write32(channel->addr, 0x0144, 0x180, val | BIT(4)); + ret = r8a779f0_eth_serdes_reg_wait(channel, 0x0180, 0x180, BIT(0), 1); + if (ret) + return ret; + r8a779f0_eth_serdes_write32(channel->addr, 0x0144, 0x180, val & ~BIT(4)); + ret = r8a779f0_eth_serdes_reg_wait(channel, 0x0180, 0x180, BIT(0), 0); + if (ret) + return ret; + return r8a779f0_eth_serdes_monitor_linkup(channel); } diff --git a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c index 98c92d6c482fe..279e19e7546b6 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c @@ -87,10 +87,11 @@ struct dphy_reg { u32 offset; u32 mask; u32 shift; + u8 valid; }; #define PHY_REG(_offset, _width, _shift) \ - { .offset = _offset, .mask = BIT(_width) - 1, .shift = _shift, } + { .offset = _offset, .mask = BIT(_width) - 1, .shift = _shift, .valid = 1, } static const struct dphy_reg rk1808_grf_dphy_regs[] = { [GRF_DPHY_CSIPHY_FORCERXMODE] = PHY_REG(RK1808_GRF_PD_VI_CON_OFFSET, 4, 0), @@ -145,7 +146,7 @@ static inline void write_grf_reg(struct rockchip_inno_csidphy *priv, const struct dphy_drv_data *drv_data = priv->drv_data; const struct dphy_reg *reg = &drv_data->grf_regs[index]; - if (reg->offset) + if (reg->valid) regmap_write(priv->grf, reg->offset, HIWORD_UPDATE(value, reg->mask, reg->shift)); } diff --git a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c index 2354ce8b21594..9ef496092b7e6 100644 --- a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c +++ b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c @@ -122,6 +122,8 @@ struct rockchip_combphy_grfcfg { struct combphy_reg pipe_xpcs_phy_ready; struct combphy_reg pipe_pcie1l0_sel; struct combphy_reg pipe_pcie1l1_sel; + struct combphy_reg u3otg0_port_en; + struct combphy_reg u3otg1_port_en; }; struct rockchip_combphy_cfg { @@ -431,6 +433,14 @@ static int rk3568_combphy_cfg(struct rockchip_combphy_priv *priv) rockchip_combphy_param_write(priv->phy_grf, &cfg->pipe_txcomp_sel, false); rockchip_combphy_param_write(priv->phy_grf, &cfg->pipe_txelec_sel, false); rockchip_combphy_param_write(priv->phy_grf, &cfg->usb_mode_set, true); + switch (priv->id) { + case 0: + rockchip_combphy_param_write(priv->pipe_grf, &cfg->u3otg0_port_en, true); + break; + case 1: + rockchip_combphy_param_write(priv->pipe_grf, &cfg->u3otg1_port_en, true); + break; + } break; case PHY_TYPE_SATA: @@ -574,6 +584,8 @@ static const struct rockchip_combphy_grfcfg rk3568_combphy_grfcfgs = { /* pipe-grf */ .pipe_con0_for_sata = { 0x0000, 15, 0, 0x00, 0x2220 }, .pipe_xpcs_phy_ready = { 0x0040, 2, 2, 0x00, 0x01 }, + .u3otg0_port_en = { 0x0104, 15, 0, 0x0181, 0x1100 }, + .u3otg1_port_en = { 0x0144, 15, 0, 0x0181, 0x1100 }, }; static const struct rockchip_combphy_cfg rk3568_combphy_cfgs = { diff --git a/drivers/phy/rockchip/phy-rockchip-pcie.c b/drivers/phy/rockchip/phy-rockchip-pcie.c index 8234b83fdd88c..4669e87b32435 100644 --- a/drivers/phy/rockchip/phy-rockchip-pcie.c +++ b/drivers/phy/rockchip/phy-rockchip-pcie.c @@ -12,10 +12,9 @@ #include #include #include -#include -#include #include #include +#include #include #include @@ -31,9 +30,8 @@ #define PHY_CFG_ADDR_SHIFT 1 #define PHY_CFG_DATA_MASK 0xf #define PHY_CFG_ADDR_MASK 0x3f -#define PHY_CFG_RD_MASK 0x3ff #define PHY_CFG_WR_ENABLE 1 -#define PHY_CFG_WR_DISABLE 1 +#define PHY_CFG_WR_DISABLE 0 #define PHY_CFG_WR_SHIFT 0 #define PHY_CFG_WR_MASK 1 #define PHY_CFG_PLL_LOCK 0x10 @@ -63,7 +61,7 @@ struct rockchip_pcie_data { }; struct rockchip_pcie_phy { - struct rockchip_pcie_data *phy_data; + const struct rockchip_pcie_data *phy_data; struct regmap *reg_base; struct phy_pcie_instance { struct phy *phy; @@ -350,7 +348,6 @@ static int rockchip_pcie_phy_probe(struct platform_device *pdev) struct rockchip_pcie_phy *rk_phy; struct phy_provider *phy_provider; struct regmap *grf; - const struct of_device_id *of_id; int i; u32 phy_num; @@ -364,11 +361,10 @@ static int rockchip_pcie_phy_probe(struct platform_device *pdev) if (!rk_phy) return -ENOMEM; - of_id = of_match_device(rockchip_pcie_phy_dt_ids, &pdev->dev); - if (!of_id) + rk_phy->phy_data = device_get_match_data(&pdev->dev); + if (!rk_phy->phy_data) return -EINVAL; - rk_phy->phy_data = (struct rockchip_pcie_data *)of_id->data; rk_phy->reg_base = grf; mutex_init(&rk_phy->pcie_mutex); diff --git a/drivers/phy/rockchip/phy-rockchip-usb.c b/drivers/phy/rockchip/phy-rockchip-usb.c index 8454285977ebc..666a896c8f0a0 100644 --- a/drivers/phy/rockchip/phy-rockchip-usb.c +++ b/drivers/phy/rockchip/phy-rockchip-usb.c @@ -13,10 +13,9 @@ #include #include #include -#include -#include #include #include +#include #include #include #include @@ -458,7 +457,6 @@ static int rockchip_usb_phy_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct rockchip_usb_phy_base *phy_base; struct phy_provider *phy_provider; - const struct of_device_id *match; struct device_node *child; int err; @@ -466,14 +464,12 @@ static int rockchip_usb_phy_probe(struct platform_device *pdev) if (!phy_base) return -ENOMEM; - match = of_match_device(dev->driver->of_match_table, dev); - if (!match || !match->data) { + phy_base->pdata = device_get_match_data(dev); + if (!phy_base->pdata) { dev_err(dev, "missing phy data\n"); return -EINVAL; } - phy_base->pdata = match->data; - phy_base->dev = dev; phy_base->reg_base = ERR_PTR(-ENODEV); if (dev->parent && dev->parent->of_node) diff --git a/drivers/phy/starfive/phy-jh7110-usb.c b/drivers/phy/starfive/phy-jh7110-usb.c index 633912f8a05d0..bf52b41110db8 100644 --- a/drivers/phy/starfive/phy-jh7110-usb.c +++ b/drivers/phy/starfive/phy-jh7110-usb.c @@ -16,6 +16,8 @@ #include #define USB_125M_CLK_RATE 125000000 +#define USB_CLK_MODE_OFF 0x0 +#define USB_CLK_MODE_RX_NORMAL_PWR BIT(1) #define USB_LS_KEEPALIVE_OFF 0x4 #define USB_LS_KEEPALIVE_ENABLE BIT(4) @@ -68,6 +70,7 @@ static int jh7110_usb2_phy_init(struct phy *_phy) { struct jh7110_usb2_phy *phy = phy_get_drvdata(_phy); int ret; + unsigned int val; ret = clk_set_rate(phy->usb_125m_clk, USB_125M_CLK_RATE); if (ret) @@ -77,6 +80,10 @@ static int jh7110_usb2_phy_init(struct phy *_phy) if (ret) return ret; + val = readl(phy->regs + USB_CLK_MODE_OFF); + val |= USB_CLK_MODE_RX_NORMAL_PWR; + writel(val, phy->regs + USB_CLK_MODE_OFF); + return 0; } diff --git a/drivers/phy/tegra/xusb-tegra186.c b/drivers/phy/tegra/xusb-tegra186.c index fae6242aa730e..e818f6c3980e6 100644 --- a/drivers/phy/tegra/xusb-tegra186.c +++ b/drivers/phy/tegra/xusb-tegra186.c @@ -237,6 +237,8 @@ #define DATA0_VAL_PD BIT(1) #define USE_XUSB_AO BIT(4) +#define TEGRA_UTMI_PAD_MAX 4 + #define TEGRA186_LANE(_name, _offset, _shift, _mask, _type) \ { \ .name = _name, \ @@ -269,7 +271,7 @@ struct tegra186_xusb_padctl { /* UTMI bias and tracking */ struct clk *usb2_trk_clk; - unsigned int bias_pad_enable; + DECLARE_BITMAP(utmi_pad_enabled, TEGRA_UTMI_PAD_MAX); /* padctl context */ struct tegra186_xusb_padctl_context context; @@ -603,12 +605,8 @@ static void tegra186_utmi_bias_pad_power_on(struct tegra_xusb_padctl *padctl) u32 value; int err; - mutex_lock(&padctl->lock); - - if (priv->bias_pad_enable++ > 0) { - mutex_unlock(&padctl->lock); + if (!bitmap_empty(priv->utmi_pad_enabled, TEGRA_UTMI_PAD_MAX)) return; - } err = clk_prepare_enable(priv->usb2_trk_clk); if (err < 0) @@ -650,16 +648,15 @@ static void tegra186_utmi_bias_pad_power_on(struct tegra_xusb_padctl *padctl) udelay(100); } - if (padctl->soc->trk_hw_mode) { - value = padctl_readl(padctl, XUSB_PADCTL_USB2_BIAS_PAD_CTL2); - value |= USB2_TRK_HW_MODE; + value = padctl_readl(padctl, XUSB_PADCTL_USB2_BIAS_PAD_CTL2); + if (padctl->soc->trk_update_on_idle) value &= ~CYA_TRK_CODE_UPDATE_ON_IDLE; - padctl_writel(padctl, value, XUSB_PADCTL_USB2_BIAS_PAD_CTL2); - } else { - clk_disable_unprepare(priv->usb2_trk_clk); - } + if (padctl->soc->trk_hw_mode) + value |= USB2_TRK_HW_MODE; + padctl_writel(padctl, value, XUSB_PADCTL_USB2_BIAS_PAD_CTL2); - mutex_unlock(&padctl->lock); + if (!padctl->soc->trk_hw_mode) + clk_disable_unprepare(priv->usb2_trk_clk); } static void tegra186_utmi_bias_pad_power_off(struct tegra_xusb_padctl *padctl) @@ -667,17 +664,8 @@ static void tegra186_utmi_bias_pad_power_off(struct tegra_xusb_padctl *padctl) struct tegra186_xusb_padctl *priv = to_tegra186_xusb_padctl(padctl); u32 value; - mutex_lock(&padctl->lock); - - if (WARN_ON(priv->bias_pad_enable == 0)) { - mutex_unlock(&padctl->lock); - return; - } - - if (--priv->bias_pad_enable > 0) { - mutex_unlock(&padctl->lock); + if (!bitmap_empty(priv->utmi_pad_enabled, TEGRA_UTMI_PAD_MAX)) return; - } value = padctl_readl(padctl, XUSB_PADCTL_USB2_BIAS_PAD_CTL1); value |= USB2_PD_TRK; @@ -690,13 +678,13 @@ static void tegra186_utmi_bias_pad_power_off(struct tegra_xusb_padctl *padctl) clk_disable_unprepare(priv->usb2_trk_clk); } - mutex_unlock(&padctl->lock); } static void tegra186_utmi_pad_power_on(struct phy *phy) { struct tegra_xusb_lane *lane = phy_get_drvdata(phy); struct tegra_xusb_padctl *padctl = lane->pad->padctl; + struct tegra186_xusb_padctl *priv = to_tegra186_xusb_padctl(padctl); struct tegra_xusb_usb2_port *port; struct device *dev = padctl->dev; unsigned int index = lane->index; @@ -705,9 +693,16 @@ static void tegra186_utmi_pad_power_on(struct phy *phy) if (!phy) return; + mutex_lock(&padctl->lock); + if (test_bit(index, priv->utmi_pad_enabled)) { + mutex_unlock(&padctl->lock); + return; + } + port = tegra_xusb_find_usb2_port(padctl, index); if (!port) { dev_err(dev, "no port found for USB2 lane %u\n", index); + mutex_unlock(&padctl->lock); return; } @@ -724,18 +719,28 @@ static void tegra186_utmi_pad_power_on(struct phy *phy) value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index)); value &= ~USB2_OTG_PD_DR; padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index)); + + set_bit(index, priv->utmi_pad_enabled); + mutex_unlock(&padctl->lock); } static void tegra186_utmi_pad_power_down(struct phy *phy) { struct tegra_xusb_lane *lane = phy_get_drvdata(phy); struct tegra_xusb_padctl *padctl = lane->pad->padctl; + struct tegra186_xusb_padctl *priv = to_tegra186_xusb_padctl(padctl); unsigned int index = lane->index; u32 value; if (!phy) return; + mutex_lock(&padctl->lock); + if (!test_bit(index, priv->utmi_pad_enabled)) { + mutex_unlock(&padctl->lock); + return; + } + dev_dbg(padctl->dev, "power down UTMI pad %u\n", index); value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index)); @@ -748,7 +753,11 @@ static void tegra186_utmi_pad_power_down(struct phy *phy) udelay(2); + clear_bit(index, priv->utmi_pad_enabled); + tegra186_utmi_bias_pad_power_off(padctl); + + mutex_unlock(&padctl->lock); } static int tegra186_xusb_padctl_vbus_override(struct tegra_xusb_padctl *padctl, @@ -774,13 +783,15 @@ static int tegra186_xusb_padctl_vbus_override(struct tegra_xusb_padctl *padctl, } static int tegra186_xusb_padctl_id_override(struct tegra_xusb_padctl *padctl, - bool status) + struct tegra_xusb_usb2_port *port, bool status) { - u32 value; + u32 value, id_override; + int err = 0; dev_dbg(padctl->dev, "%s id override\n", status ? "set" : "clear"); value = padctl_readl(padctl, USB2_VBUS_ID); + id_override = value & ID_OVERRIDE(~0); if (status) { if (value & VBUS_OVERRIDE) { @@ -791,15 +802,35 @@ static int tegra186_xusb_padctl_id_override(struct tegra_xusb_padctl *padctl, value = padctl_readl(padctl, USB2_VBUS_ID); } - value &= ~ID_OVERRIDE(~0); - value |= ID_OVERRIDE_GROUNDED; + if (id_override != ID_OVERRIDE_GROUNDED) { + value &= ~ID_OVERRIDE(~0); + value |= ID_OVERRIDE_GROUNDED; + padctl_writel(padctl, value, USB2_VBUS_ID); + + err = regulator_enable(port->supply); + if (err) { + dev_err(padctl->dev, "Failed to enable regulator: %d\n", err); + return err; + } + } } else { - value &= ~ID_OVERRIDE(~0); - value |= ID_OVERRIDE_FLOATING; + if (id_override == ID_OVERRIDE_GROUNDED) { + /* + * The regulator is disabled only when the role transitions + * from USB_ROLE_HOST to USB_ROLE_NONE. + */ + err = regulator_disable(port->supply); + if (err) { + dev_err(padctl->dev, "Failed to disable regulator: %d\n", err); + return err; + } + + value &= ~ID_OVERRIDE(~0); + value |= ID_OVERRIDE_FLOATING; + padctl_writel(padctl, value, USB2_VBUS_ID); + } } - padctl_writel(padctl, value, USB2_VBUS_ID); - return 0; } @@ -818,27 +849,20 @@ static int tegra186_utmi_phy_set_mode(struct phy *phy, enum phy_mode mode, if (mode == PHY_MODE_USB_OTG) { if (submode == USB_ROLE_HOST) { - tegra186_xusb_padctl_id_override(padctl, true); - - err = regulator_enable(port->supply); + err = tegra186_xusb_padctl_id_override(padctl, port, true); + if (err) + goto out; } else if (submode == USB_ROLE_DEVICE) { tegra186_xusb_padctl_vbus_override(padctl, true); } else if (submode == USB_ROLE_NONE) { - /* - * When port is peripheral only or role transitions to - * USB_ROLE_NONE from USB_ROLE_DEVICE, regulator is not - * enabled. - */ - if (regulator_is_enabled(port->supply)) - regulator_disable(port->supply); - - tegra186_xusb_padctl_id_override(padctl, false); + err = tegra186_xusb_padctl_id_override(padctl, port, false); + if (err) + goto out; tegra186_xusb_padctl_vbus_override(padctl, false); } } - +out: mutex_unlock(&padctl->lock); - return err; } @@ -1702,7 +1726,8 @@ const struct tegra_xusb_padctl_soc tegra234_xusb_padctl_soc = { .num_supplies = ARRAY_SIZE(tegra194_xusb_padctl_supply_names), .supports_gen2 = true, .poll_trk_completed = true, - .trk_hw_mode = true, + .trk_hw_mode = false, + .trk_update_on_idle = true, .supports_lp_cfg_en = true, }; EXPORT_SYMBOL_GPL(tegra234_xusb_padctl_soc); diff --git a/drivers/phy/tegra/xusb-tegra210.c b/drivers/phy/tegra/xusb-tegra210.c index ebc8a7e21a318..3409924498e9c 100644 --- a/drivers/phy/tegra/xusb-tegra210.c +++ b/drivers/phy/tegra/xusb-tegra210.c @@ -3164,18 +3164,22 @@ tegra210_xusb_padctl_probe(struct device *dev, } pdev = of_find_device_by_node(np); + of_node_put(np); if (!pdev) { dev_warn(dev, "PMC device is not available\n"); goto out; } - if (!platform_get_drvdata(pdev)) + if (!platform_get_drvdata(pdev)) { + put_device(&pdev->dev); return ERR_PTR(-EPROBE_DEFER); + } padctl->regmap = dev_get_regmap(&pdev->dev, "usb_sleepwalk"); if (!padctl->regmap) dev_info(dev, "failed to find PMC regmap\n"); + put_device(&pdev->dev); out: return &padctl->base; } diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c index 983a6e6173bd2..3a04a56ca52de 100644 --- a/drivers/phy/tegra/xusb.c +++ b/drivers/phy/tegra/xusb.c @@ -548,16 +548,16 @@ static int tegra_xusb_port_init(struct tegra_xusb_port *port, err = dev_set_name(&port->dev, "%s-%u", name, index); if (err < 0) - goto unregister; + goto put_device; err = device_add(&port->dev); if (err < 0) - goto unregister; + goto put_device; return 0; -unregister: - device_unregister(&port->dev); +put_device: + put_device(&port->dev); return err; } diff --git a/drivers/phy/tegra/xusb.h b/drivers/phy/tegra/xusb.h index 6e45d194c6894..d2b5f95651324 100644 --- a/drivers/phy/tegra/xusb.h +++ b/drivers/phy/tegra/xusb.h @@ -434,6 +434,7 @@ struct tegra_xusb_padctl_soc { bool need_fake_usb3_port; bool poll_trk_completed; bool trk_hw_mode; + bool trk_update_on_idle; bool supports_lp_cfg_en; }; diff --git a/drivers/phy/ti/phy-omap-control.c b/drivers/phy/ti/phy-omap-control.c index 76c5595f0859c..2fdb8f4241c74 100644 --- a/drivers/phy/ti/phy-omap-control.c +++ b/drivers/phy/ti/phy-omap-control.c @@ -8,9 +8,9 @@ #include #include +#include #include #include -#include #include #include #include @@ -268,20 +268,15 @@ MODULE_DEVICE_TABLE(of, omap_control_phy_id_table); static int omap_control_phy_probe(struct platform_device *pdev) { - const struct of_device_id *of_id; struct omap_control_phy *control_phy; - of_id = of_match_device(omap_control_phy_id_table, &pdev->dev); - if (!of_id) - return -EINVAL; - control_phy = devm_kzalloc(&pdev->dev, sizeof(*control_phy), GFP_KERNEL); if (!control_phy) return -ENOMEM; control_phy->dev = &pdev->dev; - control_phy->type = *(enum omap_control_phy_type *)of_id->data; + control_phy->type = *(enum omap_control_phy_type *)device_get_match_data(&pdev->dev); if (control_phy->type == OMAP_CTRL_TYPE_OTGHS) { control_phy->otghs_control = diff --git a/drivers/phy/ti/phy-omap-usb2.c b/drivers/phy/ti/phy-omap-usb2.c index 6bd3c74923306..0fea766a98d75 100644 --- a/drivers/phy/ti/phy-omap-usb2.c +++ b/drivers/phy/ti/phy-omap-usb2.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -362,6 +363,13 @@ static void omap_usb2_init_errata(struct omap_usb *phy) phy->flags |= OMAP_USB2_DISABLE_CHRG_DET; } +static void omap_usb2_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + static int omap_usb2_probe(struct platform_device *pdev) { struct omap_usb *phy; @@ -371,16 +379,13 @@ static int omap_usb2_probe(struct platform_device *pdev) struct device_node *node = pdev->dev.of_node; struct device_node *control_node; struct platform_device *control_pdev; - const struct of_device_id *of_id; - struct usb_phy_data *phy_data; - - of_id = of_match_device(omap_usb2_id_table, &pdev->dev); + const struct usb_phy_data *phy_data; + int ret; - if (!of_id) + phy_data = device_get_match_data(&pdev->dev); + if (!phy_data) return -EINVAL; - phy_data = (struct usb_phy_data *)of_id->data; - phy = devm_kzalloc(&pdev->dev, sizeof(*phy), GFP_KERNEL); if (!phy) return -ENOMEM; @@ -426,6 +431,11 @@ static int omap_usb2_probe(struct platform_device *pdev) return -EINVAL; } phy->control_dev = &control_pdev->dev; + + ret = devm_add_action_or_reset(&pdev->dev, omap_usb2_put_device, + phy->control_dev); + if (ret) + return ret; } else { if (of_property_read_u32_index(node, "syscon-phy-power", 1, diff --git a/drivers/phy/ti/phy-ti-pipe3.c b/drivers/phy/ti/phy-ti-pipe3.c index 507e1552db5e8..8e94d2c6e266a 100644 --- a/drivers/phy/ti/phy-ti-pipe3.c +++ b/drivers/phy/ti/phy-ti-pipe3.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -666,12 +667,20 @@ static int ti_pipe3_get_clk(struct ti_pipe3 *phy) return 0; } +static void ti_pipe3_put_device(void *_dev) +{ + struct device *dev = _dev; + + put_device(dev); +} + static int ti_pipe3_get_sysctrl(struct ti_pipe3 *phy) { struct device *dev = phy->dev; struct device_node *node = dev->of_node; struct device_node *control_node; struct platform_device *control_pdev; + int ret; phy->phy_power_syscon = syscon_regmap_lookup_by_phandle(node, "syscon-phy-power"); @@ -703,6 +712,11 @@ static int ti_pipe3_get_sysctrl(struct ti_pipe3 *phy) } phy->control_dev = &control_pdev->dev; + + ret = devm_add_action_or_reset(dev, ti_pipe3_put_device, + phy->control_dev); + if (ret) + return ret; } if (phy->mode == PIPE3_MODE_PCIE) { @@ -778,23 +792,16 @@ static int ti_pipe3_probe(struct platform_device *pdev) struct phy_provider *phy_provider; struct device *dev = &pdev->dev; int ret; - const struct of_device_id *match; - struct pipe3_data *data; + const struct pipe3_data *data; phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL); if (!phy) return -ENOMEM; - match = of_match_device(ti_pipe3_id_table, dev); - if (!match) + data = device_get_match_data(dev); + if (!data) return -EINVAL; - data = (struct pipe3_data *)match->data; - if (!data) { - dev_err(dev, "no driver data\n"); - return -EINVAL; - } - phy->dev = dev; phy->mode = data->mode; phy->dpll_map = data->dpll_map; diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index 7dfb7190580ef..ab3908a923e3a 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -438,6 +438,7 @@ config PINCTRL_STMFX tristate "STMicroelectronics STMFX GPIO expander pinctrl driver" depends on I2C depends on OF_GPIO + depends on HAS_IOMEM select GENERIC_PINCONF select GPIOLIB_IRQCHIP select MFD_STMFX diff --git a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c index cf6efa9c0364a..a039b490cdb8e 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c @@ -72,7 +72,7 @@ static enum bcm281xx_pin_type hdmi_pin = BCM281XX_PIN_TYPE_HDMI; struct bcm281xx_pin_function { const char *name; const char * const *groups; - const unsigned ngroups; + const unsigned int ngroups; }; /* @@ -84,10 +84,10 @@ struct bcm281xx_pinctrl_data { /* List of all pins */ const struct pinctrl_pin_desc *pins; - const unsigned npins; + const unsigned int npins; const struct bcm281xx_pin_function *functions; - const unsigned nfunctions; + const unsigned int nfunctions; struct regmap *regmap; }; @@ -941,7 +941,7 @@ static struct bcm281xx_pinctrl_data bcm281xx_pinctrl = { }; static inline enum bcm281xx_pin_type pin_type_get(struct pinctrl_dev *pctldev, - unsigned pin) + unsigned int pin) { struct bcm281xx_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev); @@ -985,7 +985,7 @@ static int bcm281xx_pinctrl_get_groups_count(struct pinctrl_dev *pctldev) } static const char *bcm281xx_pinctrl_get_group_name(struct pinctrl_dev *pctldev, - unsigned group) + unsigned int group) { struct bcm281xx_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev); @@ -993,9 +993,9 @@ static const char *bcm281xx_pinctrl_get_group_name(struct pinctrl_dev *pctldev, } static int bcm281xx_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, - unsigned group, + unsigned int group, const unsigned **pins, - unsigned *num_pins) + unsigned int *num_pins) { struct bcm281xx_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev); @@ -1007,7 +1007,7 @@ static int bcm281xx_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, static void bcm281xx_pinctrl_pin_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s, - unsigned offset) + unsigned int offset) { seq_printf(s, " %s", dev_name(pctldev->dev)); } @@ -1029,7 +1029,7 @@ static int bcm281xx_pinctrl_get_fcns_count(struct pinctrl_dev *pctldev) } static const char *bcm281xx_pinctrl_get_fcn_name(struct pinctrl_dev *pctldev, - unsigned function) + unsigned int function) { struct bcm281xx_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev); @@ -1037,9 +1037,9 @@ static const char *bcm281xx_pinctrl_get_fcn_name(struct pinctrl_dev *pctldev, } static int bcm281xx_pinctrl_get_fcn_groups(struct pinctrl_dev *pctldev, - unsigned function, + unsigned int function, const char * const **groups, - unsigned * const num_groups) + unsigned int * const num_groups) { struct bcm281xx_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev); @@ -1050,8 +1050,8 @@ static int bcm281xx_pinctrl_get_fcn_groups(struct pinctrl_dev *pctldev, } static int bcm281xx_pinmux_set(struct pinctrl_dev *pctldev, - unsigned function, - unsigned group) + unsigned int function, + unsigned int group) { struct bcm281xx_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev); const struct bcm281xx_pin_function *f = &pdata->functions[function]; @@ -1082,7 +1082,7 @@ static const struct pinmux_ops bcm281xx_pinctrl_pinmux_ops = { }; static int bcm281xx_pinctrl_pin_config_get(struct pinctrl_dev *pctldev, - unsigned pin, + unsigned int pin, unsigned long *config) { return -ENOTSUPP; @@ -1091,9 +1091,9 @@ static int bcm281xx_pinctrl_pin_config_get(struct pinctrl_dev *pctldev, /* Goes through the configs and update register val/mask */ static int bcm281xx_std_pin_update(struct pinctrl_dev *pctldev, - unsigned pin, + unsigned int pin, unsigned long *configs, - unsigned num_configs, + unsigned int num_configs, u32 *val, u32 *mask) { @@ -1207,9 +1207,9 @@ static const u16 bcm281xx_pullup_map[] = { /* Goes through the configs and update register val/mask */ static int bcm281xx_i2c_pin_update(struct pinctrl_dev *pctldev, - unsigned pin, + unsigned int pin, unsigned long *configs, - unsigned num_configs, + unsigned int num_configs, u32 *val, u32 *mask) { @@ -1277,9 +1277,9 @@ static int bcm281xx_i2c_pin_update(struct pinctrl_dev *pctldev, /* Goes through the configs and update register val/mask */ static int bcm281xx_hdmi_pin_update(struct pinctrl_dev *pctldev, - unsigned pin, + unsigned int pin, unsigned long *configs, - unsigned num_configs, + unsigned int num_configs, u32 *val, u32 *mask) { @@ -1321,9 +1321,9 @@ static int bcm281xx_hdmi_pin_update(struct pinctrl_dev *pctldev, } static int bcm281xx_pinctrl_pin_config_set(struct pinctrl_dev *pctldev, - unsigned pin, + unsigned int pin, unsigned long *configs, - unsigned num_configs) + unsigned int num_configs) { struct bcm281xx_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev); enum bcm281xx_pin_type pin_type; diff --git a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c index c096463184195..e1ac89be7c847 100644 --- a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c +++ b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c @@ -532,6 +532,11 @@ static int cs42l43_gpio_add_pin_ranges(struct gpio_chip *chip) return ret; } +static void cs42l43_fwnode_put(void *data) +{ + fwnode_handle_put(data); +} + static int cs42l43_pin_probe(struct platform_device *pdev) { struct cs42l43 *cs42l43 = dev_get_drvdata(pdev->dev.parent); @@ -563,10 +568,20 @@ static int cs42l43_pin_probe(struct platform_device *pdev) priv->gpio_chip.ngpio = CS42L43_NUM_GPIOS; if (is_of_node(fwnode)) { - fwnode = fwnode_get_named_child_node(fwnode, "pinctrl"); - - if (fwnode && !fwnode->dev) - fwnode->dev = priv->dev; + struct fwnode_handle *child; + + child = fwnode_get_named_child_node(fwnode, "pinctrl"); + if (child) { + ret = devm_add_action_or_reset(&pdev->dev, + cs42l43_fwnode_put, child); + if (ret) { + fwnode_handle_put(child); + return ret; + } + if (!child->dev) + child->dev = priv->dev; + fwnode = child; + } } priv->gpio_chip.fwnode = fwnode; diff --git a/drivers/pinctrl/devicetree.c b/drivers/pinctrl/devicetree.c index 5ee746cb81f59..6520b88db1105 100644 --- a/drivers/pinctrl/devicetree.c +++ b/drivers/pinctrl/devicetree.c @@ -143,10 +143,14 @@ static int dt_to_map_one_config(struct pinctrl *p, pctldev = get_pinctrl_dev_from_of_node(np_pctldev); if (pctldev) break; - /* Do not defer probing of hogs (circular loop) */ + /* + * Do not defer probing of hogs (circular loop) + * + * Return 1 to let the caller catch the case. + */ if (np_pctldev == p->dev->of_node) { of_node_put(np_pctldev); - return -ENODEV; + return 1; } } of_node_put(np_pctldev); @@ -265,6 +269,8 @@ int pinctrl_dt_to_map(struct pinctrl *p, struct pinctrl_dev *pctldev) ret = dt_to_map_one_config(p, pctldev, statename, np_config); of_node_put(np_config); + if (ret == 1) + continue; if (ret < 0) goto err; } diff --git a/drivers/pinctrl/meson/pinctrl-meson-gxl.c b/drivers/pinctrl/meson/pinctrl-meson-gxl.c index 51408996255bd..e2601e45935e6 100644 --- a/drivers/pinctrl/meson/pinctrl-meson-gxl.c +++ b/drivers/pinctrl/meson/pinctrl-meson-gxl.c @@ -187,6 +187,9 @@ static const unsigned int i2c_sda_c_pins[] = { GPIODV_28 }; static const unsigned int i2c_sck_c_dv19_pins[] = { GPIODV_19 }; static const unsigned int i2c_sda_c_dv18_pins[] = { GPIODV_18 }; +static const unsigned int i2c_sck_d_pins[] = { GPIOX_11 }; +static const unsigned int i2c_sda_d_pins[] = { GPIOX_10 }; + static const unsigned int eth_mdio_pins[] = { GPIOZ_0 }; static const unsigned int eth_mdc_pins[] = { GPIOZ_1 }; static const unsigned int eth_clk_rx_clk_pins[] = { GPIOZ_2 }; @@ -411,6 +414,8 @@ static struct meson_pmx_group meson_gxl_periphs_groups[] = { GPIO_GROUP(GPIO_TEST_N), /* Bank X */ + GROUP(i2c_sda_d, 5, 5), + GROUP(i2c_sck_d, 5, 4), GROUP(sdio_d0, 5, 31), GROUP(sdio_d1, 5, 30), GROUP(sdio_d2, 5, 29), @@ -651,6 +656,10 @@ static const char * const i2c_c_groups[] = { "i2c_sck_c", "i2c_sda_c", "i2c_sda_c_dv18", "i2c_sck_c_dv19", }; +static const char * const i2c_d_groups[] = { + "i2c_sck_d", "i2c_sda_d", +}; + static const char * const eth_groups[] = { "eth_mdio", "eth_mdc", "eth_clk_rx_clk", "eth_rx_dv", "eth_rxd0", "eth_rxd1", "eth_rxd2", "eth_rxd3", @@ -777,6 +786,7 @@ static struct meson_pmx_func meson_gxl_periphs_functions[] = { FUNCTION(i2c_a), FUNCTION(i2c_b), FUNCTION(i2c_c), + FUNCTION(i2c_d), FUNCTION(eth), FUNCTION(pwm_a), FUNCTION(pwm_b), diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c index 524424ee6c4e7..5cc00fdc48d84 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.c +++ b/drivers/pinctrl/meson/pinctrl-meson.c @@ -486,7 +486,7 @@ static int meson_pinconf_get(struct pinctrl_dev *pcdev, unsigned int pin, case PIN_CONFIG_BIAS_PULL_DOWN: case PIN_CONFIG_BIAS_PULL_UP: if (meson_pinconf_get_pull(pc, pin) == param) - arg = 1; + arg = 60000; else return -EINVAL; break; diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index c34719b7506da..ef87a6045e073 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -358,9 +358,7 @@ static int armada_37xx_pmx_set_by_name(struct pinctrl_dev *pctldev, val = grp->val[func]; - regmap_update_bits(info->regmap, reg, mask, val); - - return 0; + return regmap_update_bits(info->regmap, reg, mask, val); } static int armada_37xx_pmx_set(struct pinctrl_dev *pctldev, @@ -402,10 +400,13 @@ static int armada_37xx_gpio_get_direction(struct gpio_chip *chip, struct armada_37xx_pinctrl *info = gpiochip_get_data(chip); unsigned int reg = OUTPUT_EN; unsigned int val, mask; + int ret; armada_37xx_update_reg(®, &offset); mask = BIT(offset); - regmap_read(info->regmap, reg, &val); + ret = regmap_read(info->regmap, reg, &val); + if (ret) + return ret; if (val & mask) return GPIO_LINE_DIRECTION_OUT; @@ -417,20 +418,22 @@ static int armada_37xx_gpio_direction_output(struct gpio_chip *chip, unsigned int offset, int value) { struct armada_37xx_pinctrl *info = gpiochip_get_data(chip); - unsigned int reg = OUTPUT_EN; + unsigned int en_offset = offset; + unsigned int reg = OUTPUT_VAL; unsigned int mask, val, ret; armada_37xx_update_reg(®, &offset); mask = BIT(offset); + val = value ? mask : 0; - ret = regmap_update_bits(info->regmap, reg, mask, mask); - + ret = regmap_update_bits(info->regmap, reg, mask, val); if (ret) return ret; - reg = OUTPUT_VAL; - val = value ? mask : 0; - regmap_update_bits(info->regmap, reg, mask, val); + reg = OUTPUT_EN; + armada_37xx_update_reg(®, &en_offset); + + regmap_update_bits(info->regmap, reg, mask, mask); return 0; } @@ -440,11 +443,14 @@ static int armada_37xx_gpio_get(struct gpio_chip *chip, unsigned int offset) struct armada_37xx_pinctrl *info = gpiochip_get_data(chip); unsigned int reg = INPUT_VAL; unsigned int val, mask; + int ret; armada_37xx_update_reg(®, &offset); mask = BIT(offset); - regmap_read(info->regmap, reg, &val); + ret = regmap_read(info->regmap, reg, &val); + if (ret) + return ret; return (val & mask) != 0; } @@ -469,16 +475,17 @@ static int armada_37xx_pmx_gpio_set_direction(struct pinctrl_dev *pctldev, { struct armada_37xx_pinctrl *info = pinctrl_dev_get_drvdata(pctldev); struct gpio_chip *chip = range->gc; + int ret; dev_dbg(info->dev, "gpio_direction for pin %u as %s-%d to %s\n", offset, range->name, offset, input ? "input" : "output"); if (input) - armada_37xx_gpio_direction_input(chip, offset); + ret = armada_37xx_gpio_direction_input(chip, offset); else - armada_37xx_gpio_direction_output(chip, offset, 0); + ret = armada_37xx_gpio_direction_output(chip, offset, 0); - return 0; + return ret; } static int armada_37xx_gpio_request_enable(struct pinctrl_dev *pctldev, diff --git a/drivers/pinctrl/nxp/pinctrl-s32cc.c b/drivers/pinctrl/nxp/pinctrl-s32cc.c index f0cad2c501f76..44887960ccc88 100644 --- a/drivers/pinctrl/nxp/pinctrl-s32cc.c +++ b/drivers/pinctrl/nxp/pinctrl-s32cc.c @@ -392,6 +392,7 @@ static int s32_pmx_gpio_request_enable(struct pinctrl_dev *pctldev, gpio_pin->pin_id = offset; gpio_pin->config = config; + INIT_LIST_HEAD(&gpio_pin->list); spin_lock_irqsave(&ipctl->gpio_configs_lock, flags); list_add(&gpio_pin->list, &ipctl->gpio_configs); @@ -945,7 +946,7 @@ int s32_pinctrl_probe(struct platform_device *pdev, spin_lock_init(&ipctl->gpio_configs_lock); s32_pinctrl_desc = - devm_kmalloc(&pdev->dev, sizeof(*s32_pinctrl_desc), GFP_KERNEL); + devm_kzalloc(&pdev->dev, sizeof(*s32_pinctrl_desc), GFP_KERNEL); if (!s32_pinctrl_desc) return -ENOMEM; diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c index 75bff325a4251..ba38173d3ed3c 100644 --- a/drivers/pinctrl/pinctrl-amd.c +++ b/drivers/pinctrl/pinctrl-amd.c @@ -933,6 +933,17 @@ static int amd_gpio_suspend_hibernate_common(struct device *dev, bool is_suspend pin, is_suspend ? "suspend" : "hibernate"); } + /* + * debounce enabled over suspend has shown issues with a GPIO + * being unable to wake the system, as we're only interested in + * the actual wakeup event, clear it. + */ + if (gpio_dev->saved_regs[i] & (DB_CNTRl_MASK << DB_CNTRL_OFF)) { + amd_gpio_set_debounce(gpio_dev, pin, 0); + pm_pr_dbg("Clearing debounce for GPIO #%d during %s.\n", + pin, is_suspend ? "suspend" : "hibernate"); + } + raw_spin_unlock_irqrestore(&gpio_dev->lock, flags); } diff --git a/drivers/pinctrl/pinctrl-at91.c b/drivers/pinctrl/pinctrl-at91.c index d7b66928a4e50..3c09f743b68c2 100644 --- a/drivers/pinctrl/pinctrl-at91.c +++ b/drivers/pinctrl/pinctrl-at91.c @@ -1825,12 +1825,16 @@ static int at91_gpio_probe(struct platform_device *pdev) struct at91_gpio_chip *at91_chip = NULL; struct gpio_chip *chip; struct pinctrl_gpio_range *range; + int alias_idx; int ret = 0; int irq, i; - int alias_idx = of_alias_get_id(np, "gpio"); uint32_t ngpio; char **names; + alias_idx = of_alias_get_id(np, "gpio"); + if (alias_idx < 0) + return alias_idx; + BUG_ON(alias_idx >= ARRAY_SIZE(gpio_chips)); if (gpio_chips[alias_idx]) return dev_err_probe(dev, -EBUSY, "%d slot is occupied.\n", alias_idx); diff --git a/drivers/pinctrl/pinctrl-keembay.c b/drivers/pinctrl/pinctrl-keembay.c index 152c35bce8ecc..94b11a23829f6 100644 --- a/drivers/pinctrl/pinctrl-keembay.c +++ b/drivers/pinctrl/pinctrl-keembay.c @@ -1606,7 +1606,8 @@ static int keembay_build_functions(struct keembay_pinctrl *kpc) * being part of 8 (hw maximum) globally unique muxes. */ kpc->nfuncs = 0; - keembay_funcs = kcalloc(kpc->npins * 8, sizeof(*keembay_funcs), GFP_KERNEL); + keembay_funcs = devm_kcalloc(kpc->dev, kpc->npins * 8, + sizeof(*keembay_funcs), GFP_KERNEL); if (!keembay_funcs) return -ENOMEM; @@ -1637,7 +1638,9 @@ static int keembay_build_functions(struct keembay_pinctrl *kpc) } /* Reallocate memory based on actual number of functions */ - new_funcs = krealloc(keembay_funcs, kpc->nfuncs * sizeof(*new_funcs), GFP_KERNEL); + new_funcs = devm_krealloc_array(kpc->dev, keembay_funcs, + kpc->nfuncs, sizeof(*new_funcs), + GFP_KERNEL); if (!new_funcs) { kfree(keembay_funcs); return -ENOMEM; diff --git a/drivers/pinctrl/pinctrl-mcp23s08.c b/drivers/pinctrl/pinctrl-mcp23s08.c index fd97b6ee2a8d1..ca45c1f36a89b 100644 --- a/drivers/pinctrl/pinctrl-mcp23s08.c +++ b/drivers/pinctrl/pinctrl-mcp23s08.c @@ -612,6 +612,14 @@ int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev, mcp->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + /* + * Reset the chip - we don't really know what state it's in, so reset + * all pins to input first to prevent surprises. + */ + ret = mcp_write(mcp, MCP_IODIR, mcp->chip.ngpio == 16 ? 0xFFFF : 0xFF); + if (ret < 0) + return ret; + /* verify MCP_IOCON.SEQOP = 0, so sequential reads work, * and MCP_IOCON.HAEN = 1, so we work with all chips. */ diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 6c670203b3ac2..7684039be10cb 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -587,8 +587,10 @@ static int pcs_pinconf_set(struct pinctrl_dev *pctldev, break; case PIN_CONFIG_BIAS_PULL_DOWN: case PIN_CONFIG_BIAS_PULL_UP: - if (arg) + if (arg) { pcs_pinconf_clear_bias(pctldev, pin); + data = pcs->read(pcs->base + offset); + } fallthrough; case PIN_CONFIG_INPUT_SCHMITT_ENABLE: data &= ~func->conf[i].mask; diff --git a/drivers/pinctrl/pinmux.c b/drivers/pinctrl/pinmux.c index 97e8af88df851..1c324d216ad77 100644 --- a/drivers/pinctrl/pinmux.c +++ b/drivers/pinctrl/pinmux.c @@ -238,18 +238,7 @@ static const char *pin_free(struct pinctrl_dev *pctldev, int pin, if (desc->mux_usecount) return NULL; } - } - - /* - * If there is no kind of request function for the pin we just assume - * we got it by default and proceed. - */ - if (gpio_range && ops->gpio_disable_free) - ops->gpio_disable_free(pctldev, gpio_range, pin); - else if (ops->free) - ops->free(pctldev, pin); - scoped_guard(mutex, &desc->mux_lock) { if (gpio_range) { owner = desc->gpio_owner; desc->gpio_owner = NULL; @@ -260,6 +249,15 @@ static const char *pin_free(struct pinctrl_dev *pctldev, int pin, } } + /* + * If there is no kind of request function for the pin we just assume + * we got it by default and proceed. + */ + if (gpio_range && ops->gpio_disable_free) + ops->gpio_disable_free(pctldev, gpio_range, pin); + else if (ops->free) + ops->free(pctldev, pin); + module_put(pctldev->owner); return owner; @@ -341,7 +339,7 @@ static int pinmux_func_name_to_selector(struct pinctrl_dev *pctldev, while (selector < nfuncs) { const char *fname = ops->get_function_name(pctldev, selector); - if (!strcmp(function, fname)) + if (fname && !strcmp(function, fname)) return selector; selector++; diff --git a/drivers/pinctrl/qcom/pinctrl-apq8064.c b/drivers/pinctrl/qcom/pinctrl-apq8064.c index 20c3b90250445..a18df41622999 100644 --- a/drivers/pinctrl/qcom/pinctrl-apq8064.c +++ b/drivers/pinctrl/qcom/pinctrl-apq8064.c @@ -629,7 +629,7 @@ static struct platform_driver apq8064_pinctrl_driver = { .of_match_table = apq8064_pinctrl_of_match, }, .probe = apq8064_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init apq8064_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-apq8084.c b/drivers/pinctrl/qcom/pinctrl-apq8084.c index 3fc0a40762b63..afada80e52a23 100644 --- a/drivers/pinctrl/qcom/pinctrl-apq8084.c +++ b/drivers/pinctrl/qcom/pinctrl-apq8084.c @@ -1207,7 +1207,7 @@ static struct platform_driver apq8084_pinctrl_driver = { .of_match_table = apq8084_pinctrl_of_match, }, .probe = apq8084_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init apq8084_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-ipq4019.c b/drivers/pinctrl/qcom/pinctrl-ipq4019.c index 1f7944dd829d1..cb13576ad6cfb 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq4019.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq4019.c @@ -710,7 +710,7 @@ static struct platform_driver ipq4019_pinctrl_driver = { .of_match_table = ipq4019_pinctrl_of_match, }, .probe = ipq4019_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init ipq4019_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-ipq5018.c b/drivers/pinctrl/qcom/pinctrl-ipq5018.c index e2951f81c3eeb..68f65b57003e9 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq5018.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq5018.c @@ -754,7 +754,7 @@ static struct platform_driver ipq5018_pinctrl_driver = { .of_match_table = ipq5018_pinctrl_of_match, }, .probe = ipq5018_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init ipq5018_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-ipq5332.c b/drivers/pinctrl/qcom/pinctrl-ipq5332.c index 625f8014051f6..8821751189708 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq5332.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq5332.c @@ -834,7 +834,7 @@ static struct platform_driver ipq5332_pinctrl_driver = { .of_match_table = ipq5332_pinctrl_of_match, }, .probe = ipq5332_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init ipq5332_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-ipq6018.c b/drivers/pinctrl/qcom/pinctrl-ipq6018.c index 0ad08647dbcdf..ac330d8712b5c 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq6018.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq6018.c @@ -1080,7 +1080,7 @@ static struct platform_driver ipq6018_pinctrl_driver = { .of_match_table = ipq6018_pinctrl_of_match, }, .probe = ipq6018_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init ipq6018_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-ipq8064.c b/drivers/pinctrl/qcom/pinctrl-ipq8064.c index e2bb94e86aef6..e10e1bc4c9113 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq8064.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq8064.c @@ -631,7 +631,7 @@ static struct platform_driver ipq8064_pinctrl_driver = { .of_match_table = ipq8064_pinctrl_of_match, }, .probe = ipq8064_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init ipq8064_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-ipq8074.c b/drivers/pinctrl/qcom/pinctrl-ipq8074.c index 337f3a1c92c19..fee32c1d1d3e9 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq8074.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq8074.c @@ -1041,7 +1041,7 @@ static struct platform_driver ipq8074_pinctrl_driver = { .of_match_table = ipq8074_pinctrl_of_match, }, .probe = ipq8074_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init ipq8074_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-ipq9574.c b/drivers/pinctrl/qcom/pinctrl-ipq9574.c index e2491617b2364..20ab59cb621bc 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq9574.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq9574.c @@ -799,7 +799,7 @@ static struct platform_driver ipq9574_pinctrl_driver = { .of_match_table = ipq9574_pinctrl_of_match, }, .probe = ipq9574_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init ipq9574_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-mdm9607.c b/drivers/pinctrl/qcom/pinctrl-mdm9607.c index e7cd3ef1cf3e8..415d24e16267d 100644 --- a/drivers/pinctrl/qcom/pinctrl-mdm9607.c +++ b/drivers/pinctrl/qcom/pinctrl-mdm9607.c @@ -1059,7 +1059,7 @@ static struct platform_driver mdm9607_pinctrl_driver = { .of_match_table = mdm9607_pinctrl_of_match, }, .probe = mdm9607_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init mdm9607_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-mdm9615.c b/drivers/pinctrl/qcom/pinctrl-mdm9615.c index 0a2ae383d3d57..3f2eafea0b246 100644 --- a/drivers/pinctrl/qcom/pinctrl-mdm9615.c +++ b/drivers/pinctrl/qcom/pinctrl-mdm9615.c @@ -446,7 +446,7 @@ static struct platform_driver mdm9615_pinctrl_driver = { .of_match_table = mdm9615_pinctrl_of_match, }, .probe = mdm9615_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init mdm9615_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c index b252fc22f64e6..13dc8bc1d0cff 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm.c +++ b/drivers/pinctrl/qcom/pinctrl-msm.c @@ -43,7 +43,6 @@ * @pctrl: pinctrl handle. * @chip: gpiochip handle. * @desc: pin controller descriptor - * @restart_nb: restart notifier block. * @irq: parent irq for the TLMM irq_chip. * @intr_target_use_scm: route irq to application cpu using scm calls * @lock: Spinlock to protect register resources as well @@ -63,7 +62,6 @@ struct msm_pinctrl { struct pinctrl_dev *pctrl; struct gpio_chip chip; struct pinctrl_desc desc; - struct notifier_block restart_nb; int irq; @@ -1033,6 +1031,25 @@ static bool msm_gpio_needs_dual_edge_parent_workaround(struct irq_data *d, test_bit(d->hwirq, pctrl->skip_wake_irqs); } +static void msm_gpio_irq_init_valid_mask(struct gpio_chip *gc, + unsigned long *valid_mask, + unsigned int ngpios) +{ + struct msm_pinctrl *pctrl = gpiochip_get_data(gc); + const struct msm_pingroup *g; + int i; + + bitmap_fill(valid_mask, ngpios); + + for (i = 0; i < ngpios; i++) { + g = &pctrl->soc->groups[i]; + + if (g->intr_detection_width != 1 && + g->intr_detection_width != 2) + clear_bit(i, valid_mask); + } +} + static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int type) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); @@ -1394,6 +1411,7 @@ static int msm_gpio_init(struct msm_pinctrl *pctrl) girq->default_type = IRQ_TYPE_NONE; girq->handler = handle_bad_irq; girq->parents[0] = pctrl->irq; + girq->init_valid_mask = msm_gpio_irq_init_valid_mask; ret = gpiochip_add_data(&pctrl->chip, pctrl); if (ret) { @@ -1424,10 +1442,9 @@ static int msm_gpio_init(struct msm_pinctrl *pctrl) return 0; } -static int msm_ps_hold_restart(struct notifier_block *nb, unsigned long action, - void *data) +static int msm_ps_hold_restart(struct sys_off_data *data) { - struct msm_pinctrl *pctrl = container_of(nb, struct msm_pinctrl, restart_nb); + struct msm_pinctrl *pctrl = data->cb_data; writel(0, pctrl->regs[0] + PS_HOLD_OFFSET); mdelay(1000); @@ -1438,7 +1455,11 @@ static struct msm_pinctrl *poweroff_pctrl; static void msm_ps_hold_poweroff(void) { - msm_ps_hold_restart(&poweroff_pctrl->restart_nb, 0, NULL); + struct sys_off_data data = { + .cb_data = poweroff_pctrl, + }; + + msm_ps_hold_restart(&data); } static void msm_pinctrl_setup_pm_reset(struct msm_pinctrl *pctrl) @@ -1448,9 +1469,11 @@ static void msm_pinctrl_setup_pm_reset(struct msm_pinctrl *pctrl) for (i = 0; i < pctrl->soc->nfunctions; i++) if (!strcmp(func[i].name, "ps_hold")) { - pctrl->restart_nb.notifier_call = msm_ps_hold_restart; - pctrl->restart_nb.priority = 128; - if (register_restart_handler(&pctrl->restart_nb)) + if (devm_register_sys_off_handler(pctrl->dev, + SYS_OFF_MODE_RESTART, + 128, + msm_ps_hold_restart, + pctrl)) dev_err(pctrl->dev, "failed to setup restart handler.\n"); poweroff_pctrl = pctrl; @@ -1547,15 +1570,11 @@ int msm_pinctrl_probe(struct platform_device *pdev, } EXPORT_SYMBOL(msm_pinctrl_probe); -int msm_pinctrl_remove(struct platform_device *pdev) +void msm_pinctrl_remove(struct platform_device *pdev) { struct msm_pinctrl *pctrl = platform_get_drvdata(pdev); gpiochip_remove(&pctrl->chip); - - unregister_restart_handler(&pctrl->restart_nb); - - return 0; } EXPORT_SYMBOL(msm_pinctrl_remove); diff --git a/drivers/pinctrl/qcom/pinctrl-msm.h b/drivers/pinctrl/qcom/pinctrl-msm.h index 1d2f2e904da19..4968d08a384da 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm.h +++ b/drivers/pinctrl/qcom/pinctrl-msm.h @@ -166,6 +166,6 @@ extern const struct dev_pm_ops msm_pinctrl_dev_pm_ops; int msm_pinctrl_probe(struct platform_device *pdev, const struct msm_pinctrl_soc_data *soc_data); -int msm_pinctrl_remove(struct platform_device *pdev); +void msm_pinctrl_remove(struct platform_device *pdev); #endif diff --git a/drivers/pinctrl/qcom/pinctrl-msm8226.c b/drivers/pinctrl/qcom/pinctrl-msm8226.c index 994619840a706..90b4004e7faf1 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8226.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8226.c @@ -638,7 +638,7 @@ static struct platform_driver msm8226_pinctrl_driver = { .of_match_table = msm8226_pinctrl_of_match, }, .probe = msm8226_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8226_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8660.c b/drivers/pinctrl/qcom/pinctrl-msm8660.c index 999a5f867eb50..dba6d531b4a14 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8660.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8660.c @@ -981,7 +981,7 @@ static struct platform_driver msm8660_pinctrl_driver = { .of_match_table = msm8660_pinctrl_of_match, }, .probe = msm8660_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8660_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8909.c b/drivers/pinctrl/qcom/pinctrl-msm8909.c index 756856d20d6b5..14b17ba9f9061 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8909.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8909.c @@ -929,7 +929,7 @@ static struct platform_driver msm8909_pinctrl_driver = { .of_match_table = msm8909_pinctrl_of_match, }, .probe = msm8909_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8909_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8916.c b/drivers/pinctrl/qcom/pinctrl-msm8916.c index cea5c54f92fec..184dcf8422735 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8916.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8916.c @@ -969,7 +969,7 @@ static struct platform_driver msm8916_pinctrl_driver = { .of_match_table = msm8916_pinctrl_of_match, }, .probe = msm8916_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8916_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8953.c b/drivers/pinctrl/qcom/pinctrl-msm8953.c index 998351bdfee13..c2253821ae8d3 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8953.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8953.c @@ -1816,7 +1816,7 @@ static struct platform_driver msm8953_pinctrl_driver = { .of_match_table = msm8953_pinctrl_of_match, }, .probe = msm8953_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8953_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8960.c b/drivers/pinctrl/qcom/pinctrl-msm8960.c index ebe230b3b437c..6b9148d226e9b 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8960.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8960.c @@ -1246,7 +1246,7 @@ static struct platform_driver msm8960_pinctrl_driver = { .of_match_table = msm8960_pinctrl_of_match, }, .probe = msm8960_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8960_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8976.c b/drivers/pinctrl/qcom/pinctrl-msm8976.c index c30d80e4e98ca..9a951888e8a1b 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8976.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8976.c @@ -1096,7 +1096,7 @@ static struct platform_driver msm8976_pinctrl_driver = { .of_match_table = msm8976_pinctrl_of_match, }, .probe = msm8976_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8976_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8994.c b/drivers/pinctrl/qcom/pinctrl-msm8994.c index b1a6759ab4a5e..1ed1dd32d6c79 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8994.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8994.c @@ -1343,7 +1343,7 @@ static struct platform_driver msm8994_pinctrl_driver = { .of_match_table = msm8994_pinctrl_of_match, }, .probe = msm8994_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8994_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8996.c b/drivers/pinctrl/qcom/pinctrl-msm8996.c index 46cc0b49dbab5..5f0e7f78fd517 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8996.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8996.c @@ -1906,7 +1906,7 @@ static struct platform_driver msm8996_pinctrl_driver = { .of_match_table = msm8996_pinctrl_of_match, }, .probe = msm8996_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8996_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8998.c b/drivers/pinctrl/qcom/pinctrl-msm8998.c index b7cbf32b3125a..4aaf45e54f3a7 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8998.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8998.c @@ -1535,7 +1535,7 @@ static struct platform_driver msm8998_pinctrl_driver = { .of_match_table = msm8998_pinctrl_of_match, }, .probe = msm8998_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8998_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-msm8x74.c b/drivers/pinctrl/qcom/pinctrl-msm8x74.c index d5fe62992849c..58b4f6f31ae6a 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm8x74.c +++ b/drivers/pinctrl/qcom/pinctrl-msm8x74.c @@ -1071,7 +1071,7 @@ static struct platform_driver msm8x74_pinctrl_driver = { .of_match_table = msm8x74_pinctrl_of_match, }, .probe = msm8x74_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init msm8x74_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-qcm2290.c b/drivers/pinctrl/qcom/pinctrl-qcm2290.c index ba699eac9ee8b..61b7c22e963c2 100644 --- a/drivers/pinctrl/qcom/pinctrl-qcm2290.c +++ b/drivers/pinctrl/qcom/pinctrl-qcm2290.c @@ -165,6 +165,10 @@ static const struct pinctrl_pin_desc qcm2290_pins[] = { PINCTRL_PIN(62, "GPIO_62"), PINCTRL_PIN(63, "GPIO_63"), PINCTRL_PIN(64, "GPIO_64"), + PINCTRL_PIN(65, "GPIO_65"), + PINCTRL_PIN(66, "GPIO_66"), + PINCTRL_PIN(67, "GPIO_67"), + PINCTRL_PIN(68, "GPIO_68"), PINCTRL_PIN(69, "GPIO_69"), PINCTRL_PIN(70, "GPIO_70"), PINCTRL_PIN(71, "GPIO_71"), @@ -179,12 +183,17 @@ static const struct pinctrl_pin_desc qcm2290_pins[] = { PINCTRL_PIN(80, "GPIO_80"), PINCTRL_PIN(81, "GPIO_81"), PINCTRL_PIN(82, "GPIO_82"), + PINCTRL_PIN(83, "GPIO_83"), + PINCTRL_PIN(84, "GPIO_84"), + PINCTRL_PIN(85, "GPIO_85"), PINCTRL_PIN(86, "GPIO_86"), PINCTRL_PIN(87, "GPIO_87"), PINCTRL_PIN(88, "GPIO_88"), PINCTRL_PIN(89, "GPIO_89"), PINCTRL_PIN(90, "GPIO_90"), PINCTRL_PIN(91, "GPIO_91"), + PINCTRL_PIN(92, "GPIO_92"), + PINCTRL_PIN(93, "GPIO_93"), PINCTRL_PIN(94, "GPIO_94"), PINCTRL_PIN(95, "GPIO_95"), PINCTRL_PIN(96, "GPIO_96"), @@ -1113,7 +1122,7 @@ static struct platform_driver qcm2290_pinctrl_driver = { .of_match_table = qcm2290_pinctrl_of_match, }, .probe = qcm2290_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init qcm2290_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-qcs404.c b/drivers/pinctrl/qcom/pinctrl-qcs404.c index ae7224012f8aa..9a875b7dc9989 100644 --- a/drivers/pinctrl/qcom/pinctrl-qcs404.c +++ b/drivers/pinctrl/qcom/pinctrl-qcs404.c @@ -1644,7 +1644,7 @@ static struct platform_driver qcs404_pinctrl_driver = { .of_match_table = qcs404_pinctrl_of_match, }, .probe = qcs404_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init qcs404_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c b/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c index b5808fcfb13cd..4d2f6f495163b 100644 --- a/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c +++ b/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c @@ -145,7 +145,7 @@ static struct platform_driver qdf2xxx_pinctrl_driver = { .acpi_match_table = qdf2xxx_acpi_ids, }, .probe = qdf2xxx_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init qdf2xxx_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-qdu1000.c b/drivers/pinctrl/qcom/pinctrl-qdu1000.c index 47bc529ef550d..da4f940bc8d4e 100644 --- a/drivers/pinctrl/qcom/pinctrl-qdu1000.c +++ b/drivers/pinctrl/qcom/pinctrl-qdu1000.c @@ -1248,7 +1248,7 @@ static struct platform_driver qdu1000_tlmm_driver = { .of_match_table = qdu1000_tlmm_of_match, }, .probe = qdu1000_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init qdu1000_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sa8775p.c b/drivers/pinctrl/qcom/pinctrl-sa8775p.c index 8fdea25d8d67e..5459c0c681a23 100644 --- a/drivers/pinctrl/qcom/pinctrl-sa8775p.c +++ b/drivers/pinctrl/qcom/pinctrl-sa8775p.c @@ -1530,7 +1530,7 @@ static struct platform_driver sa8775p_pinctrl_driver = { .of_match_table = sa8775p_pinctrl_of_match, }, .probe = sa8775p_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sa8775p_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sc7180.c b/drivers/pinctrl/qcom/pinctrl-sc7180.c index 6eb0c73791c0b..c27aaa599b917 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc7180.c +++ b/drivers/pinctrl/qcom/pinctrl-sc7180.c @@ -1159,7 +1159,7 @@ static struct platform_driver sc7180_pinctrl_driver = { .of_match_table = sc7180_pinctrl_of_match, }, .probe = sc7180_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sc7180_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sc7280.c b/drivers/pinctrl/qcom/pinctrl-sc7280.c index 0c10eeb60b55e..c2db663e396eb 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc7280.c +++ b/drivers/pinctrl/qcom/pinctrl-sc7280.c @@ -1505,7 +1505,7 @@ static struct platform_driver sc7280_pinctrl_driver = { .of_match_table = sc7280_pinctrl_of_match, }, .probe = sc7280_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sc7280_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sc8180x.c b/drivers/pinctrl/qcom/pinctrl-sc8180x.c index d6a79ad41a40a..cfa7c8be9770c 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc8180x.c +++ b/drivers/pinctrl/qcom/pinctrl-sc8180x.c @@ -1720,7 +1720,7 @@ static struct platform_driver sc8180x_pinctrl_driver = { .acpi_match_table = sc8180x_pinctrl_acpi_match, }, .probe = sc8180x_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sc8180x_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c index 96f4fb5a5d297..4b1c49697698d 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c +++ b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c @@ -1926,7 +1926,7 @@ static struct platform_driver sc8280xp_pinctrl_driver = { .of_match_table = sc8280xp_pinctrl_of_match, }, .probe = sc8280xp_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sc8280xp_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sdm660.c b/drivers/pinctrl/qcom/pinctrl-sdm660.c index c2e0d5c034acf..b0c29a24b09b5 100644 --- a/drivers/pinctrl/qcom/pinctrl-sdm660.c +++ b/drivers/pinctrl/qcom/pinctrl-sdm660.c @@ -1428,7 +1428,7 @@ static struct platform_driver sdm660_pinctrl_driver = { .of_match_table = sdm660_pinctrl_of_match, }, .probe = sdm660_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sdm660_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sdm670.c b/drivers/pinctrl/qcom/pinctrl-sdm670.c index cc3cce077de4e..1e694a966953a 100644 --- a/drivers/pinctrl/qcom/pinctrl-sdm670.c +++ b/drivers/pinctrl/qcom/pinctrl-sdm670.c @@ -1318,7 +1318,7 @@ static struct platform_driver sdm670_pinctrl_driver = { .of_match_table = sdm670_pinctrl_of_match, }, .probe = sdm670_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sdm670_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sdm845.c b/drivers/pinctrl/qcom/pinctrl-sdm845.c index cc05c415ed155..3f3265e0018d6 100644 --- a/drivers/pinctrl/qcom/pinctrl-sdm845.c +++ b/drivers/pinctrl/qcom/pinctrl-sdm845.c @@ -1351,7 +1351,7 @@ static struct platform_driver sdm845_pinctrl_driver = { .acpi_match_table = ACPI_PTR(sdm845_pinctrl_acpi_match), }, .probe = sdm845_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sdm845_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sdx55.c b/drivers/pinctrl/qcom/pinctrl-sdx55.c index 8826db9d21d04..c88b8bfcacb6a 100644 --- a/drivers/pinctrl/qcom/pinctrl-sdx55.c +++ b/drivers/pinctrl/qcom/pinctrl-sdx55.c @@ -990,7 +990,7 @@ static struct platform_driver sdx55_pinctrl_driver = { .of_match_table = sdx55_pinctrl_of_match, }, .probe = sdx55_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sdx55_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sdx65.c b/drivers/pinctrl/qcom/pinctrl-sdx65.c index f6f319c997fc7..bd44ec0fcab43 100644 --- a/drivers/pinctrl/qcom/pinctrl-sdx65.c +++ b/drivers/pinctrl/qcom/pinctrl-sdx65.c @@ -939,7 +939,7 @@ static struct platform_driver sdx65_pinctrl_driver = { .of_match_table = sdx65_pinctrl_of_match, }, .probe = sdx65_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sdx65_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sdx75.c b/drivers/pinctrl/qcom/pinctrl-sdx75.c index 3cfe8c7f04df8..396f6fc779a2e 100644 --- a/drivers/pinctrl/qcom/pinctrl-sdx75.c +++ b/drivers/pinctrl/qcom/pinctrl-sdx75.c @@ -1124,7 +1124,7 @@ static struct platform_driver sdx75_pinctrl_driver = { .of_match_table = sdx75_pinctrl_of_match, }, .probe = sdx75_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sdx75_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm6115.c b/drivers/pinctrl/qcom/pinctrl-sm6115.c index 2a06025f48858..87057089b2b64 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm6115.c +++ b/drivers/pinctrl/qcom/pinctrl-sm6115.c @@ -895,7 +895,7 @@ static struct platform_driver sm6115_tlmm_driver = { .of_match_table = sm6115_tlmm_of_match, }, .probe = sm6115_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm6115_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm6125.c b/drivers/pinctrl/qcom/pinctrl-sm6125.c index d5e2b896954c2..e07339ba72bca 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm6125.c +++ b/drivers/pinctrl/qcom/pinctrl-sm6125.c @@ -1249,7 +1249,7 @@ static struct platform_driver sm6125_tlmm_driver = { .of_match_table = sm6125_tlmm_of_match, }, .probe = sm6125_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm6125_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm6350.c b/drivers/pinctrl/qcom/pinctrl-sm6350.c index f3828c07b1345..4aeb1ba43ee3d 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm6350.c +++ b/drivers/pinctrl/qcom/pinctrl-sm6350.c @@ -1373,7 +1373,7 @@ static struct platform_driver sm6350_tlmm_driver = { .of_match_table = sm6350_tlmm_of_match, }, .probe = sm6350_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm6350_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm6375.c b/drivers/pinctrl/qcom/pinctrl-sm6375.c index c82c8516932ea..d86630d7125c2 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm6375.c +++ b/drivers/pinctrl/qcom/pinctrl-sm6375.c @@ -1516,7 +1516,7 @@ static struct platform_driver sm6375_tlmm_driver = { .of_match_table = sm6375_tlmm_of_match, }, .probe = sm6375_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm6375_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm7150.c b/drivers/pinctrl/qcom/pinctrl-sm7150.c index edb5984cd3519..b9f067de8ef0e 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm7150.c +++ b/drivers/pinctrl/qcom/pinctrl-sm7150.c @@ -1254,7 +1254,7 @@ static struct platform_driver sm7150_tlmm_driver = { .of_match_table = sm7150_tlmm_of_match, }, .probe = sm7150_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm7150_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm8150.c b/drivers/pinctrl/qcom/pinctrl-sm8150.c index 01aea9c70b7a7..f8f5bee74f1dc 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8150.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8150.c @@ -1542,7 +1542,7 @@ static struct platform_driver sm8150_pinctrl_driver = { .of_match_table = sm8150_pinctrl_of_match, }, .probe = sm8150_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm8150_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm8250.c b/drivers/pinctrl/qcom/pinctrl-sm8250.c index e9961a49ff981..54fda77bf2968 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8250.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8250.c @@ -1351,7 +1351,7 @@ static struct platform_driver sm8250_pinctrl_driver = { .of_match_table = sm8250_pinctrl_of_match, }, .probe = sm8250_pinctrl_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm8250_pinctrl_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm8350.c b/drivers/pinctrl/qcom/pinctrl-sm8350.c index 9c69458bd9109..ac7f2820f2cbf 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8350.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8350.c @@ -1642,7 +1642,7 @@ static struct platform_driver sm8350_tlmm_driver = { .of_match_table = sm8350_tlmm_of_match, }, .probe = sm8350_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm8350_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm8450.c b/drivers/pinctrl/qcom/pinctrl-sm8450.c index d11bb1ee9e3d8..6172867116952 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8450.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8450.c @@ -1677,7 +1677,7 @@ static struct platform_driver sm8450_tlmm_driver = { .of_match_table = sm8450_tlmm_of_match, }, .probe = sm8450_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm8450_tlmm_init(void) diff --git a/drivers/pinctrl/qcom/pinctrl-sm8550.c b/drivers/pinctrl/qcom/pinctrl-sm8550.c index 3c847d9cb5d93..9184e0183755d 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8550.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8550.c @@ -1762,7 +1762,7 @@ static struct platform_driver sm8550_tlmm_driver = { .of_match_table = sm8550_tlmm_of_match, }, .probe = sm8550_tlmm_probe, - .remove = msm_pinctrl_remove, + .remove_new = msm_pinctrl_remove, }; static int __init sm8550_tlmm_init(void) diff --git a/drivers/pinctrl/renesas/pinctrl.c b/drivers/pinctrl/renesas/pinctrl.c index 4d9d58fc1356f..852e179f9e383 100644 --- a/drivers/pinctrl/renesas/pinctrl.c +++ b/drivers/pinctrl/renesas/pinctrl.c @@ -730,7 +730,8 @@ static int sh_pfc_pinconf_group_set(struct pinctrl_dev *pctldev, unsigned group, struct sh_pfc_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); const unsigned int *pins; unsigned int num_pins; - unsigned int i, ret; + unsigned int i; + int ret; pins = pmx->pfc->info->groups[group].pins; num_pins = pmx->pfc->info->groups[group].nr_pins; diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.h b/drivers/pinctrl/samsung/pinctrl-samsung.h index 9af93e3d8d9ff..1efbae6c99876 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.h +++ b/drivers/pinctrl/samsung/pinctrl-samsung.h @@ -352,10 +352,6 @@ extern const struct samsung_pinctrl_of_match_data exynos850_of_data; extern const struct samsung_pinctrl_of_match_data exynosautov9_of_data; extern const struct samsung_pinctrl_of_match_data fsd_of_data; extern const struct samsung_pinctrl_of_match_data s3c64xx_of_data; -extern const struct samsung_pinctrl_of_match_data s3c2412_of_data; -extern const struct samsung_pinctrl_of_match_data s3c2416_of_data; -extern const struct samsung_pinctrl_of_match_data s3c2440_of_data; -extern const struct samsung_pinctrl_of_match_data s3c2450_of_data; extern const struct samsung_pinctrl_of_match_data s5pv210_of_data; #endif /* __PINCTRL_SAMSUNG_H */ diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c index 84121b125d90e..67c2791ee246f 100644 --- a/drivers/pinctrl/stm32/pinctrl-stm32.c +++ b/drivers/pinctrl/stm32/pinctrl-stm32.c @@ -418,6 +418,7 @@ static struct irq_chip stm32_gpio_irq_chip = { .irq_set_wake = irq_chip_set_wake_parent, .irq_request_resources = stm32_gpio_irq_request_resources, .irq_release_resources = stm32_gpio_irq_release_resources, + .irq_set_affinity = IS_ENABLED(CONFIG_SMP) ? irq_chip_set_affinity_parent : NULL, }; static int stm32_gpio_domain_translate(struct irq_domain *d, diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c index 73bcf806af0ec..fc11c3d55fa8c 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c +++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c @@ -395,6 +395,7 @@ static int sunxi_pctrl_dt_node_to_map(struct pinctrl_dev *pctldev, const char *function, *pin_prop; const char *group; int ret, npins, nmaps, configlen = 0, i = 0; + struct pinctrl_map *new_map; *map = NULL; *num_maps = 0; @@ -469,9 +470,13 @@ static int sunxi_pctrl_dt_node_to_map(struct pinctrl_dev *pctldev, * We know have the number of maps we need, we can resize our * map array */ - *map = krealloc(*map, i * sizeof(struct pinctrl_map), GFP_KERNEL); - if (!*map) - return -ENOMEM; + new_map = krealloc(*map, i * sizeof(struct pinctrl_map), GFP_KERNEL); + if (!new_map) { + ret = -ENOMEM; + goto err_free_map; + } + + *map = new_map; return 0; diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.c b/drivers/pinctrl/tegra/pinctrl-tegra.c index 7c12a3470642c..637b89ebe0e45 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra.c @@ -280,8 +280,8 @@ static int tegra_pinctrl_set_mux(struct pinctrl_dev *pctldev, return 0; } -static const struct tegra_pingroup *tegra_pinctrl_get_group(struct pinctrl_dev *pctldev, - unsigned int offset) +static int tegra_pinctrl_get_group_index(struct pinctrl_dev *pctldev, + unsigned int offset) { struct tegra_pmx *pmx = pinctrl_dev_get_drvdata(pctldev); unsigned int group, num_pins, j; @@ -294,12 +294,35 @@ static const struct tegra_pingroup *tegra_pinctrl_get_group(struct pinctrl_dev * continue; for (j = 0; j < num_pins; j++) { if (offset == pins[j]) - return &pmx->soc->groups[group]; + return group; } } - dev_err(pctldev->dev, "Pingroup not found for pin %u\n", offset); - return NULL; + return -EINVAL; +} + +static const struct tegra_pingroup *tegra_pinctrl_get_group(struct pinctrl_dev *pctldev, + unsigned int offset, + int group_index) +{ + struct tegra_pmx *pmx = pinctrl_dev_get_drvdata(pctldev); + + if (group_index < 0 || group_index >= pmx->soc->ngroups) + return NULL; + + return &pmx->soc->groups[group_index]; +} + +static struct tegra_pingroup_config *tegra_pinctrl_get_group_config(struct pinctrl_dev *pctldev, + unsigned int offset, + int group_index) +{ + struct tegra_pmx *pmx = pinctrl_dev_get_drvdata(pctldev); + + if (group_index < 0) + return NULL; + + return &pmx->pingroup_configs[group_index]; } static int tegra_pinctrl_gpio_request_enable(struct pinctrl_dev *pctldev, @@ -308,12 +331,15 @@ static int tegra_pinctrl_gpio_request_enable(struct pinctrl_dev *pctldev, { struct tegra_pmx *pmx = pinctrl_dev_get_drvdata(pctldev); const struct tegra_pingroup *group; + struct tegra_pingroup_config *config; + int group_index; u32 value; if (!pmx->soc->sfsel_in_mux) return 0; - group = tegra_pinctrl_get_group(pctldev, offset); + group_index = tegra_pinctrl_get_group_index(pctldev, offset); + group = tegra_pinctrl_get_group(pctldev, offset, group_index); if (!group) return -EINVAL; @@ -321,7 +347,11 @@ static int tegra_pinctrl_gpio_request_enable(struct pinctrl_dev *pctldev, if (group->mux_reg < 0 || group->sfsel_bit < 0) return -EINVAL; + config = tegra_pinctrl_get_group_config(pctldev, offset, group_index); + if (!config) + return -EINVAL; value = pmx_readl(pmx, group->mux_bank, group->mux_reg); + config->is_sfsel = (value & BIT(group->sfsel_bit)) != 0; value &= ~BIT(group->sfsel_bit); pmx_writel(pmx, value, group->mux_bank, group->mux_reg); @@ -334,12 +364,15 @@ static void tegra_pinctrl_gpio_disable_free(struct pinctrl_dev *pctldev, { struct tegra_pmx *pmx = pinctrl_dev_get_drvdata(pctldev); const struct tegra_pingroup *group; + struct tegra_pingroup_config *config; + int group_index; u32 value; if (!pmx->soc->sfsel_in_mux) return; - group = tegra_pinctrl_get_group(pctldev, offset); + group_index = tegra_pinctrl_get_group_index(pctldev, offset); + group = tegra_pinctrl_get_group(pctldev, offset, group_index); if (!group) return; @@ -347,8 +380,12 @@ static void tegra_pinctrl_gpio_disable_free(struct pinctrl_dev *pctldev, if (group->mux_reg < 0 || group->sfsel_bit < 0) return; + config = tegra_pinctrl_get_group_config(pctldev, offset, group_index); + if (!config) + return; value = pmx_readl(pmx, group->mux_bank, group->mux_reg); - value |= BIT(group->sfsel_bit); + if (config->is_sfsel) + value |= BIT(group->sfsel_bit); pmx_writel(pmx, value, group->mux_bank, group->mux_reg); } @@ -785,6 +822,12 @@ int tegra_pinctrl_probe(struct platform_device *pdev, pmx->dev = &pdev->dev; pmx->soc = soc_data; + pmx->pingroup_configs = devm_kcalloc(&pdev->dev, + pmx->soc->ngroups, sizeof(*pmx->pingroup_configs), + GFP_KERNEL); + if (!pmx->pingroup_configs) + return -ENOMEM; + /* * Each mux group will appear in 4 functions' list of groups. * This over-allocates slightly, since not all groups are mux groups. diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.h b/drivers/pinctrl/tegra/pinctrl-tegra.h index b3289bdf727d8..b97136685f7a8 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra.h +++ b/drivers/pinctrl/tegra/pinctrl-tegra.h @@ -8,6 +8,10 @@ #ifndef __PINMUX_TEGRA_H__ #define __PINMUX_TEGRA_H__ +struct tegra_pingroup_config { + bool is_sfsel; +}; + struct tegra_pmx { struct device *dev; struct pinctrl_dev *pctl; @@ -21,6 +25,8 @@ struct tegra_pmx { int nbanks; void __iomem **regs; u32 *backup_regs; + /* Array of size soc->ngroups */ + struct tegra_pingroup_config *pingroup_configs; }; enum tegra_pinconf_param { diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 47d19f7e295a7..e82f433e542ef 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -313,6 +313,9 @@ EXPORT_SYMBOL(cros_ec_register); */ void cros_ec_unregister(struct cros_ec_device *ec_dev) { + if (ec_dev->mkbp_event_supported) + blocking_notifier_chain_unregister(&ec_dev->event_notifier, + &ec_dev->notifier_ready); platform_device_unregister(ec_dev->pd); platform_device_unregister(ec_dev->ec); mutex_destroy(&ec_dev->lock); diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 66fdc6fa73ec5..76807ceb313a9 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -1179,8 +1179,8 @@ static int cros_typec_probe(struct platform_device *pdev) typec->ec = dev_get_drvdata(pdev->dev.parent); if (!typec->ec) { - dev_err(dev, "couldn't find parent EC device\n"); - return -ENODEV; + dev_warn(dev, "couldn't find parent EC device\n"); + return -EPROBE_DEFER; } platform_set_drvdata(pdev, typec); diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c index 99203584949da..5fcfa3a7970b7 100644 --- a/drivers/platform/loongarch/loongson-laptop.c +++ b/drivers/platform/loongarch/loongson-laptop.c @@ -56,8 +56,7 @@ static struct input_dev *generic_inputdev; static acpi_handle hotkey_handle; static struct key_entry hotkey_keycode_map[GENERIC_HOTKEY_MAP_MAX]; -int loongson_laptop_turn_on_backlight(void); -int loongson_laptop_turn_off_backlight(void); +static bool bl_powered; static int loongson_laptop_backlight_update(struct backlight_device *bd); /* 2. ACPI Helpers and device model */ @@ -354,16 +353,42 @@ static int ec_backlight_level(u8 level) return level; } +static int ec_backlight_set_power(bool state) +{ + int status; + union acpi_object arg0 = { ACPI_TYPE_INTEGER }; + struct acpi_object_list args = { 1, &arg0 }; + + arg0.integer.value = state; + status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL); + if (ACPI_FAILURE(status)) { + pr_info("Loongson lvds error: 0x%x\n", status); + return -EIO; + } + + return 0; +} + static int loongson_laptop_backlight_update(struct backlight_device *bd) { - int lvl = ec_backlight_level(bd->props.brightness); + bool target_powered = !backlight_is_blank(bd); + int ret = 0, lvl = ec_backlight_level(bd->props.brightness); if (lvl < 0) return -EIO; + if (ec_set_brightness(lvl)) return -EIO; - return 0; + if (target_powered != bl_powered) { + ret = ec_backlight_set_power(target_powered); + if (ret < 0) + return ret; + + bl_powered = target_powered; + } + + return ret; } static int loongson_laptop_get_brightness(struct backlight_device *bd) @@ -384,7 +409,7 @@ static const struct backlight_ops backlight_laptop_ops = { static int laptop_backlight_register(void) { - int status = 0; + int status = 0, ret; struct backlight_properties props; memset(&props, 0, sizeof(props)); @@ -392,44 +417,20 @@ static int laptop_backlight_register(void) if (!acpi_evalf(hotkey_handle, &status, "ECLL", "d")) return -EIO; - props.brightness = 1; + ret = ec_backlight_set_power(true); + if (ret) + return ret; + + bl_powered = true; + props.max_brightness = status; + props.brightness = ec_get_brightness(); + props.power = FB_BLANK_UNBLANK; props.type = BACKLIGHT_PLATFORM; backlight_device_register("loongson_laptop", NULL, NULL, &backlight_laptop_ops, &props); - return 0; -} - -int loongson_laptop_turn_on_backlight(void) -{ - int status; - union acpi_object arg0 = { ACPI_TYPE_INTEGER }; - struct acpi_object_list args = { 1, &arg0 }; - - arg0.integer.value = 1; - status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL); - if (ACPI_FAILURE(status)) { - pr_info("Loongson lvds error: 0x%x\n", status); - return -ENODEV; - } - - return 0; -} - -int loongson_laptop_turn_off_backlight(void) -{ - int status; - union acpi_object arg0 = { ACPI_TYPE_INTEGER }; - struct acpi_object_list args = { 1, &arg0 }; - - arg0.integer.value = 0; - status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL); - if (ACPI_FAILURE(status)) { - pr_info("Loongson lvds error: 0x%x\n", status); - return -ENODEV; - } return 0; } @@ -611,11 +612,17 @@ static int __init generic_acpi_laptop_init(void) static void __exit generic_acpi_laptop_exit(void) { + int i; + if (generic_inputdev) { - if (input_device_registered) - input_unregister_device(generic_inputdev); - else + if (!input_device_registered) { input_free_device(generic_inputdev); + } else { + input_unregister_device(generic_inputdev); + + for (i = 0; i < ARRAY_SIZE(generic_sub_drivers); i++) + generic_subdriver_exit(&generic_sub_drivers[i]); + } } } diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 39828eb84e0ba..1015948ef43eb 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -281,7 +281,8 @@ static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo, vring->align = SMP_CACHE_BYTES; vring->index = i; vring->vdev_id = tm_vdev->vdev.id.device; - vring->drop_desc.len = VRING_DROP_DESC_MAX_LEN; + vring->drop_desc.len = cpu_to_virtio32(&tm_vdev->vdev, + VRING_DROP_DESC_MAX_LEN); dev = &tm_vdev->vdev.dev; size = vring_size(vring->num, vring->align); diff --git a/drivers/platform/mellanox/mlxreg-lc.c b/drivers/platform/mellanox/mlxreg-lc.c index 8d833836a6d32..74e9d78ff01ef 100644 --- a/drivers/platform/mellanox/mlxreg-lc.c +++ b/drivers/platform/mellanox/mlxreg-lc.c @@ -688,7 +688,7 @@ static int mlxreg_lc_completion_notify(void *handle, struct i2c_adapter *parent, if (regval & mlxreg_lc->data->mask) { mlxreg_lc->state |= MLXREG_LC_SYNCED; mlxreg_lc_state_update_locked(mlxreg_lc, MLXREG_LC_SYNCED, 1); - if (mlxreg_lc->state & ~MLXREG_LC_POWERED) { + if (!(mlxreg_lc->state & MLXREG_LC_POWERED)) { err = mlxreg_lc_power_on_off(mlxreg_lc, 1); if (err) goto mlxreg_lc_regmap_power_on_off_fail; diff --git a/drivers/platform/mellanox/nvsw-sn2201.c b/drivers/platform/mellanox/nvsw-sn2201.c index 1a7c45aa41bbf..6b4d3c44d7bd9 100644 --- a/drivers/platform/mellanox/nvsw-sn2201.c +++ b/drivers/platform/mellanox/nvsw-sn2201.c @@ -1088,7 +1088,7 @@ static int nvsw_sn2201_i2c_completion_notify(void *handle, int id) if (!nvsw_sn2201->main_mux_devs->adapter) { err = -ENODEV; dev_err(nvsw_sn2201->dev, "Failed to get adapter for bus %d\n", - nvsw_sn2201->cpld_devs->nr); + nvsw_sn2201->main_mux_devs->nr); goto i2c_get_adapter_main_fail; } diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 07eea525091b0..1fd57eb867662 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -466,6 +466,7 @@ config LENOVO_YMC tristate "Lenovo Yoga Tablet Mode Control" depends on ACPI_WMI depends on INPUT + depends on IDEAPAD_LAPTOP select INPUT_SPARSEKMAP help This driver maps the Tablet Mode Control switch to SW_TABLET_MODE input diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index b457de5abf7d7..8012de299174f 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -58,6 +58,8 @@ obj-$(CONFIG_X86_PLATFORM_DRIVERS_HP) += hp/ # Hewlett Packard Enterprise obj-$(CONFIG_UV_SYSFS) += uv_sysfs.o +obj-$(CONFIG_FW_ATTR_CLASS) += firmware_attributes_class.o + # IBM Thinkpad and Lenovo obj-$(CONFIG_IBM_RTL) += ibm_rtl.o obj-$(CONFIG_IDEAPAD_LAPTOP) += ideapad-laptop.o @@ -108,7 +110,6 @@ obj-$(CONFIG_SYSTEM76_ACPI) += system76_acpi.o obj-$(CONFIG_TOPSTAR_LAPTOP) += topstar-laptop.o # Platform drivers -obj-$(CONFIG_FW_ATTR_CLASS) += firmware_attributes_class.o obj-$(CONFIG_SERIAL_MULTI_INSTANTIATE) += serial-multi-instantiate.o obj-$(CONFIG_MLX_PLATFORM) += mlx-platform.o obj-$(CONFIG_TOUCHSCREEN_DMI) += touchscreen_dmi.o diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c index b4f49720c87f6..9fd2829ee2ab4 100644 --- a/drivers/platform/x86/amd/pmc/pmc-quirks.c +++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c @@ -189,6 +189,15 @@ static const struct dmi_system_id fwbug_list[] = { DMI_MATCH(DMI_PRODUCT_NAME, "82XQ"), } }, + /* https://gitlab.freedesktop.org/drm/amd/-/issues/4434 */ + { + .ident = "Lenovo Yoga 6 13ALC6", + .driver_data = &quirk_s2idle_bug, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "82ND"), + } + }, /* https://gitlab.freedesktop.org/drm/amd/-/issues/2684 */ { .ident = "HP Laptop 15s-eq2xxx", @@ -217,6 +226,51 @@ static const struct dmi_system_id fwbug_list[] = { DMI_MATCH(DMI_BIOS_VERSION, "03.05"), } }, + { + .ident = "MECHREVO Wujie 14X (GX4HRXL)", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "WUJIE14-GX4HRXL"), + } + }, + { + .ident = "MECHREVO Yilong15Pro Series GM5HG7A", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "MECHREVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "Yilong15Pro Series GM5HG7A"), + } + }, + /* https://bugzilla.kernel.org/show_bug.cgi?id=220116 */ + { + .ident = "PCSpecialist Lafite Pro V 14M", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "PCSpecialist"), + DMI_MATCH(DMI_PRODUCT_NAME, "Lafite Pro V 14M"), + } + }, + { + .ident = "TUXEDO Stellaris Slim 15 AMD Gen6", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GMxHGxx"), + } + }, + { + .ident = "TUXEDO InfinityBook Pro 14/15 AMD Gen10", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "XxHP4NAx"), + } + }, + { + .ident = "TUXEDO InfinityBook Pro 14/15 AMD Gen10", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "XxKK4NAx_XxSP4NAx"), + } + }, {} }; diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c index 70907e8f3ea96..af5cc8aa7988c 100644 --- a/drivers/platform/x86/amd/pmc/pmc.c +++ b/drivers/platform/x86/amd/pmc/pmc.c @@ -332,6 +332,8 @@ static int amd_pmc_setup_smu_logging(struct amd_pmc_dev *dev) return -ENOMEM; } + memset_io(dev->smu_virt_addr, 0, sizeof(struct smu_metrics)); + /* Start the logging */ amd_pmc_send_cmd(dev, 0, NULL, SMU_MSG_LOG_RESET, false); amd_pmc_send_cmd(dev, 0, NULL, SMU_MSG_LOG_START, false); @@ -823,10 +825,9 @@ static void amd_pmc_s2idle_check(void) struct smu_metrics table; int rc; - /* CZN: Ensure that future s0i3 entry attempts at least 10ms passed */ - if (pdev->cpu_id == AMD_CPU_ID_CZN && !get_metrics_table(pdev, &table) && - table.s0i3_last_entry_status) - usleep_range(10000, 20000); + /* Avoid triggering OVP */ + if (!get_metrics_table(pdev, &table) && table.s0i3_last_entry_status) + msleep(2500); /* Dump the IdleMask before we add to the STB */ amd_pmc_idlemask_read(pdev, pdev->dev, NULL); diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index 222e429931ef9..2c894ea8aa817 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -4404,7 +4404,8 @@ static int asus_wmi_add(struct platform_device *pdev) goto fail_leds; asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_WLAN, &result); - if (result & (ASUS_WMI_DSTS_PRESENCE_BIT | ASUS_WMI_DSTS_USER_BIT)) + if ((result & (ASUS_WMI_DSTS_PRESENCE_BIT | ASUS_WMI_DSTS_USER_BIT)) == + (ASUS_WMI_DSTS_PRESENCE_BIT | ASUS_WMI_DSTS_USER_BIT)) asus->driver->wlan_ctrl_by_user = 1; if (!(asus->driver->wlan_ctrl_by_user && ashs_present())) { diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h index 3ad33a094588c..817ee7ba07ca0 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h +++ b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h @@ -89,6 +89,11 @@ extern struct wmi_sysman_priv wmi_priv; enum { ENUM, INT, STR, PO }; +#define ENUM_MIN_ELEMENTS 8 +#define INT_MIN_ELEMENTS 9 +#define STR_MIN_ELEMENTS 8 +#define PO_MIN_ELEMENTS 4 + enum { ATTR_NAME, DISPL_NAME_LANG_CODE, diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c b/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c index 8cc212c852668..fc2f58b4cbc6e 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c @@ -23,9 +23,10 @@ static ssize_t current_value_show(struct kobject *kobj, struct kobj_attribute *a obj = get_wmiobj_pointer(instance_id, DELL_WMI_BIOS_ENUMERATION_ATTRIBUTE_GUID); if (!obj) return -EIO; - if (obj->package.elements[CURRENT_VAL].type != ACPI_TYPE_STRING) { + if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count < ENUM_MIN_ELEMENTS || + obj->package.elements[CURRENT_VAL].type != ACPI_TYPE_STRING) { kfree(obj); - return -EINVAL; + return -EIO; } ret = snprintf(buf, PAGE_SIZE, "%s\n", obj->package.elements[CURRENT_VAL].string.pointer); kfree(obj); diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/int-attributes.c b/drivers/platform/x86/dell/dell-wmi-sysman/int-attributes.c index 951e75b538fad..7352480642391 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/int-attributes.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/int-attributes.c @@ -25,9 +25,10 @@ static ssize_t current_value_show(struct kobject *kobj, struct kobj_attribute *a obj = get_wmiobj_pointer(instance_id, DELL_WMI_BIOS_INTEGER_ATTRIBUTE_GUID); if (!obj) return -EIO; - if (obj->package.elements[CURRENT_VAL].type != ACPI_TYPE_INTEGER) { + if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count < INT_MIN_ELEMENTS || + obj->package.elements[CURRENT_VAL].type != ACPI_TYPE_INTEGER) { kfree(obj); - return -EINVAL; + return -EIO; } ret = snprintf(buf, PAGE_SIZE, "%lld\n", obj->package.elements[CURRENT_VAL].integer.value); kfree(obj); diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/passobj-attributes.c b/drivers/platform/x86/dell/dell-wmi-sysman/passobj-attributes.c index 230e6ee966366..3167e06d416ed 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/passobj-attributes.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/passobj-attributes.c @@ -26,9 +26,10 @@ static ssize_t is_enabled_show(struct kobject *kobj, struct kobj_attribute *attr obj = get_wmiobj_pointer(instance_id, DELL_WMI_BIOS_PASSOBJ_ATTRIBUTE_GUID); if (!obj) return -EIO; - if (obj->package.elements[IS_PASS_SET].type != ACPI_TYPE_INTEGER) { + if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count < PO_MIN_ELEMENTS || + obj->package.elements[IS_PASS_SET].type != ACPI_TYPE_INTEGER) { kfree(obj); - return -EINVAL; + return -EIO; } ret = snprintf(buf, PAGE_SIZE, "%lld\n", obj->package.elements[IS_PASS_SET].integer.value); kfree(obj); @@ -45,7 +46,7 @@ static ssize_t current_password_store(struct kobject *kobj, int length; length = strlen(buf); - if (buf[length-1] == '\n') + if (length && buf[length - 1] == '\n') length--; /* firmware does verifiation of min/max password length, diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/string-attributes.c b/drivers/platform/x86/dell/dell-wmi-sysman/string-attributes.c index c392f0ecf8b55..0d2c74f8d1aad 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/string-attributes.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/string-attributes.c @@ -25,9 +25,10 @@ static ssize_t current_value_show(struct kobject *kobj, struct kobj_attribute *a obj = get_wmiobj_pointer(instance_id, DELL_WMI_BIOS_STRING_ATTRIBUTE_GUID); if (!obj) return -EIO; - if (obj->package.elements[CURRENT_VAL].type != ACPI_TYPE_STRING) { + if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count < STR_MIN_ELEMENTS || + obj->package.elements[CURRENT_VAL].type != ACPI_TYPE_STRING) { kfree(obj); - return -EINVAL; + return -EIO; } ret = snprintf(buf, PAGE_SIZE, "%s\n", obj->package.elements[CURRENT_VAL].string.pointer); kfree(obj); diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c index af49dd6b31ade..f5402b7146572 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c @@ -25,7 +25,6 @@ struct wmi_sysman_priv wmi_priv = { /* reset bios to defaults */ static const char * const reset_types[] = {"builtinsafe", "lastknowngood", "factory", "custom"}; static int reset_option = -1; -static struct class *fw_attr_class; /** @@ -408,10 +407,10 @@ static int init_bios_attributes(int attr_type, const char *guid) return retval; switch (attr_type) { - case ENUM: min_elements = 8; break; - case INT: min_elements = 9; break; - case STR: min_elements = 8; break; - case PO: min_elements = 4; break; + case ENUM: min_elements = ENUM_MIN_ELEMENTS; break; + case INT: min_elements = INT_MIN_ELEMENTS; break; + case STR: min_elements = STR_MIN_ELEMENTS; break; + case PO: min_elements = PO_MIN_ELEMENTS; break; default: pr_err("Error: Unknown attr_type: %d\n", attr_type); return -EINVAL; @@ -541,15 +540,11 @@ static int __init sysman_init(void) goto err_exit_bios_attr_pass_interface; } - ret = fw_attributes_class_get(&fw_attr_class); - if (ret) - goto err_exit_bios_attr_pass_interface; - - wmi_priv.class_dev = device_create(fw_attr_class, NULL, MKDEV(0, 0), + wmi_priv.class_dev = device_create(&firmware_attributes_class, NULL, MKDEV(0, 0), NULL, "%s", DRIVER_NAME); if (IS_ERR(wmi_priv.class_dev)) { ret = PTR_ERR(wmi_priv.class_dev); - goto err_unregister_class; + goto err_exit_bios_attr_pass_interface; } wmi_priv.main_dir_kset = kset_create_and_add("attributes", NULL, @@ -602,10 +597,7 @@ static int __init sysman_init(void) release_attributes_data(); err_destroy_classdev: - device_destroy(fw_attr_class, MKDEV(0, 0)); - -err_unregister_class: - fw_attributes_class_put(); + device_unregister(wmi_priv.class_dev); err_exit_bios_attr_pass_interface: exit_bios_attr_pass_interface(); @@ -619,8 +611,7 @@ static int __init sysman_init(void) static void __exit sysman_exit(void) { release_attributes_data(); - device_destroy(fw_attr_class, MKDEV(0, 0)); - fw_attributes_class_put(); + device_unregister(wmi_priv.class_dev); exit_bios_attr_set_interface(); exit_bios_attr_pass_interface(); } diff --git a/drivers/platform/x86/dell/dell_rbu.c b/drivers/platform/x86/dell/dell_rbu.c index 9f51e0fcab04e..fee20866b41e4 100644 --- a/drivers/platform/x86/dell/dell_rbu.c +++ b/drivers/platform/x86/dell/dell_rbu.c @@ -292,7 +292,7 @@ static int packet_read_list(char *data, size_t * pread_length) remaining_bytes = *pread_length; bytes_read = rbu_data.packet_read_count; - list_for_each_entry(newpacket, (&packet_data_head.list)->next, list) { + list_for_each_entry(newpacket, &packet_data_head.list, list) { bytes_copied = do_packet_read(pdest, newpacket, remaining_bytes, bytes_read, &temp_count); remaining_bytes -= bytes_copied; @@ -315,14 +315,14 @@ static void packet_empty_list(void) { struct packet_data *newpacket, *tmp; - list_for_each_entry_safe(newpacket, tmp, (&packet_data_head.list)->next, list) { + list_for_each_entry_safe(newpacket, tmp, &packet_data_head.list, list) { list_del(&newpacket->list); /* * zero out the RBU packet memory before freeing * to make sure there are no stale RBU packets left in memory */ - memset(newpacket->data, 0, rbu_data.packetsize); + memset(newpacket->data, 0, newpacket->length); set_memory_wb((unsigned long)newpacket->data, 1 << newpacket->ordernum); free_pages((unsigned long) newpacket->data, diff --git a/drivers/platform/x86/firmware_attributes_class.c b/drivers/platform/x86/firmware_attributes_class.c index fafe8eaf6e3e4..e214efc97311e 100644 --- a/drivers/platform/x86/firmware_attributes_class.c +++ b/drivers/platform/x86/firmware_attributes_class.c @@ -2,48 +2,35 @@ /* Firmware attributes class helper module */ -#include -#include #include #include "firmware_attributes_class.h" -static DEFINE_MUTEX(fw_attr_lock); -static int fw_attr_inuse; - -static struct class firmware_attributes_class = { +const struct class firmware_attributes_class = { .name = "firmware-attributes", }; +EXPORT_SYMBOL_GPL(firmware_attributes_class); + +static __init int fw_attributes_class_init(void) +{ + return class_register(&firmware_attributes_class); +} +module_init(fw_attributes_class_init); + +static __exit void fw_attributes_class_exit(void) +{ + class_unregister(&firmware_attributes_class); +} +module_exit(fw_attributes_class_exit); -int fw_attributes_class_get(struct class **fw_attr_class) +int fw_attributes_class_get(const struct class **fw_attr_class) { - int err; - - mutex_lock(&fw_attr_lock); - if (!fw_attr_inuse) { /*first time class is being used*/ - err = class_register(&firmware_attributes_class); - if (err) { - mutex_unlock(&fw_attr_lock); - return err; - } - } - fw_attr_inuse++; *fw_attr_class = &firmware_attributes_class; - mutex_unlock(&fw_attr_lock); return 0; } EXPORT_SYMBOL_GPL(fw_attributes_class_get); int fw_attributes_class_put(void) { - mutex_lock(&fw_attr_lock); - if (!fw_attr_inuse) { - mutex_unlock(&fw_attr_lock); - return -EINVAL; - } - fw_attr_inuse--; - if (!fw_attr_inuse) /* No more consumers */ - class_unregister(&firmware_attributes_class); - mutex_unlock(&fw_attr_lock); return 0; } EXPORT_SYMBOL_GPL(fw_attributes_class_put); diff --git a/drivers/platform/x86/firmware_attributes_class.h b/drivers/platform/x86/firmware_attributes_class.h index 486485cb1f54e..ef6c3764a8349 100644 --- a/drivers/platform/x86/firmware_attributes_class.h +++ b/drivers/platform/x86/firmware_attributes_class.h @@ -5,7 +5,10 @@ #ifndef FW_ATTR_CLASS_H #define FW_ATTR_CLASS_H -int fw_attributes_class_get(struct class **fw_attr_class); +#include + +extern const struct class firmware_attributes_class; +int fw_attributes_class_get(const struct class **fw_attr_class); int fw_attributes_class_put(void); #endif /* FW_ATTR_CLASS_H */ diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c index 085e044e888e4..d7ef4f046d99b 100644 --- a/drivers/platform/x86/fujitsu-laptop.c +++ b/drivers/platform/x86/fujitsu-laptop.c @@ -17,13 +17,13 @@ /* * fujitsu-laptop.c - Fujitsu laptop support, providing access to additional * features made available on a range of Fujitsu laptops including the - * P2xxx/P5xxx/S6xxx/S7xxx series. + * P2xxx/P5xxx/S2xxx/S6xxx/S7xxx series. * * This driver implements a vendor-specific backlight control interface for * Fujitsu laptops and provides support for hotkeys present on certain Fujitsu * laptops. * - * This driver has been tested on a Fujitsu Lifebook S6410, S7020 and + * This driver has been tested on a Fujitsu Lifebook S2110, S6410, S7020 and * P8010. It should work on most P-series and S-series Lifebooks, but * YMMV. * @@ -102,7 +102,11 @@ #define KEY2_CODE 0x411 #define KEY3_CODE 0x412 #define KEY4_CODE 0x413 -#define KEY5_CODE 0x420 +#define KEY5_CODE 0x414 +#define KEY6_CODE 0x415 +#define KEY7_CODE 0x416 +#define KEY8_CODE 0x417 +#define KEY9_CODE 0x420 /* Hotkey ringbuffer limits */ #define MAX_HOTKEY_RINGBUFFER_SIZE 100 @@ -450,7 +454,7 @@ static const struct key_entry keymap_default[] = { { KE_KEY, KEY2_CODE, { KEY_PROG2 } }, { KE_KEY, KEY3_CODE, { KEY_PROG3 } }, { KE_KEY, KEY4_CODE, { KEY_PROG4 } }, - { KE_KEY, KEY5_CODE, { KEY_RFKILL } }, + { KE_KEY, KEY9_CODE, { KEY_RFKILL } }, /* Soft keys read from status flags */ { KE_KEY, FLAG_RFKILL, { KEY_RFKILL } }, { KE_KEY, FLAG_TOUCHPAD_TOGGLE, { KEY_TOUCHPAD_TOGGLE } }, @@ -474,6 +478,18 @@ static const struct key_entry keymap_p8010[] = { { KE_END, 0 } }; +static const struct key_entry keymap_s2110[] = { + { KE_KEY, KEY1_CODE, { KEY_PROG1 } }, /* "A" */ + { KE_KEY, KEY2_CODE, { KEY_PROG2 } }, /* "B" */ + { KE_KEY, KEY3_CODE, { KEY_WWW } }, /* "Internet" */ + { KE_KEY, KEY4_CODE, { KEY_EMAIL } }, /* "E-mail" */ + { KE_KEY, KEY5_CODE, { KEY_STOPCD } }, + { KE_KEY, KEY6_CODE, { KEY_PLAYPAUSE } }, + { KE_KEY, KEY7_CODE, { KEY_PREVIOUSSONG } }, + { KE_KEY, KEY8_CODE, { KEY_NEXTSONG } }, + { KE_END, 0 } +}; + static const struct key_entry *keymap = keymap_default; static int fujitsu_laptop_dmi_keymap_override(const struct dmi_system_id *id) @@ -511,6 +527,15 @@ static const struct dmi_system_id fujitsu_laptop_dmi_table[] = { }, .driver_data = (void *)keymap_p8010 }, + { + .callback = fujitsu_laptop_dmi_keymap_override, + .ident = "Fujitsu LifeBook S2110", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU SIEMENS"), + DMI_MATCH(DMI_PRODUCT_NAME, "LIFEBOOK S2110"), + }, + .driver_data = (void *)keymap_s2110 + }, {} }; diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c index 6ddca857cc4d1..b62b158cffd85 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c +++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c @@ -24,8 +24,6 @@ struct bioscfg_priv bioscfg_drv = { .mutex = __MUTEX_INITIALIZER(bioscfg_drv.mutex), }; -static struct class *fw_attr_class; - ssize_t display_name_language_code_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -974,11 +972,7 @@ static int __init hp_init(void) if (ret) return ret; - ret = fw_attributes_class_get(&fw_attr_class); - if (ret) - goto err_unregister_class; - - bioscfg_drv.class_dev = device_create(fw_attr_class, NULL, MKDEV(0, 0), + bioscfg_drv.class_dev = device_create(&firmware_attributes_class, NULL, MKDEV(0, 0), NULL, "%s", DRIVER_NAME); if (IS_ERR(bioscfg_drv.class_dev)) { ret = PTR_ERR(bioscfg_drv.class_dev); @@ -1045,10 +1039,9 @@ static int __init hp_init(void) release_attributes_data(); err_destroy_classdev: - device_destroy(fw_attr_class, MKDEV(0, 0)); + device_unregister(bioscfg_drv.class_dev); err_unregister_class: - fw_attributes_class_put(); hp_exit_attr_set_interface(); return ret; @@ -1057,9 +1050,8 @@ static int __init hp_init(void) static void __exit hp_exit(void) { release_attributes_data(); - device_destroy(fw_attr_class, MKDEV(0, 0)); + device_unregister(bioscfg_drv.class_dev); - fw_attributes_class_put(); hp_exit_attr_set_interface(); } diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c index 88eefccb6ed27..17a9e4bd638dc 100644 --- a/drivers/platform/x86/ideapad-laptop.c +++ b/drivers/platform/x86/ideapad-laptop.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -86,6 +88,34 @@ enum { SALS_FNLOCK_OFF = 0xf, }; +enum { + VPCCMD_R_VPC1 = 0x10, + VPCCMD_R_BL_MAX, + VPCCMD_R_BL, + VPCCMD_W_BL, + VPCCMD_R_WIFI, + VPCCMD_W_WIFI, + VPCCMD_R_BT, + VPCCMD_W_BT, + VPCCMD_R_BL_POWER, + VPCCMD_R_NOVO, + VPCCMD_R_VPC2, + VPCCMD_R_TOUCHPAD, + VPCCMD_W_TOUCHPAD, + VPCCMD_R_CAMERA, + VPCCMD_W_CAMERA, + VPCCMD_R_3G, + VPCCMD_W_3G, + VPCCMD_R_ODD, /* 0x21 */ + VPCCMD_W_FAN, + VPCCMD_R_RF, + VPCCMD_W_RF, + VPCCMD_W_YMC = 0x2A, + VPCCMD_R_FAN = 0x2B, + VPCCMD_R_SPECIAL_BUTTONS = 0x31, + VPCCMD_W_BL_POWER = 0x33, +}; + /* * These correspond to the number of supported states - 1 * Future keyboard types may need a new system, if there's a collision @@ -145,6 +175,7 @@ struct ideapad_private { bool touchpad_ctrl_via_ec : 1; bool ctrl_ps2_aux_port : 1; bool usb_charging : 1; + bool ymc_ec_trigger : 1; } features; struct { bool initialized; @@ -188,6 +219,12 @@ MODULE_PARM_DESC(touchpad_ctrl_via_ec, "Enable registering a 'touchpad' sysfs-attribute which can be used to manually " "tell the EC to enable/disable the touchpad. This may not work on all models."); +static bool ymc_ec_trigger __read_mostly; +module_param(ymc_ec_trigger, bool, 0444); +MODULE_PARM_DESC(ymc_ec_trigger, + "Enable EC triggering work-around to force emitting tablet mode events. " + "If you need this please report this to: platform-driver-x86@vger.kernel.org"); + /* * shared data */ @@ -227,6 +264,21 @@ static void ideapad_shared_exit(struct ideapad_private *priv) /* * ACPI Helpers */ +#define IDEAPAD_EC_TIMEOUT 200 /* in ms */ + +/* + * Some models (e.g., ThinkBook since 2024) have a low tolerance for being + * polled too frequently. Doing so may break the state machine in the EC, + * resulting in a hard shutdown. + * + * It is also observed that frequent polls may disturb the ongoing operation + * and notably delay the availability of EC response. + * + * These values are used as the delay before the first poll and the interval + * between subsequent polls to solve the above issues. + */ +#define IDEAPAD_EC_POLL_MIN_US 150 +#define IDEAPAD_EC_POLL_MAX_US 300 static int eval_int(acpi_handle handle, const char *name, unsigned long *res) { @@ -242,6 +294,29 @@ static int eval_int(acpi_handle handle, const char *name, unsigned long *res) return 0; } +static int eval_int_with_arg(acpi_handle handle, const char *name, unsigned long arg, + unsigned long *res) +{ + struct acpi_object_list params; + unsigned long long result; + union acpi_object in_obj; + acpi_status status; + + params.count = 1; + params.pointer = &in_obj; + in_obj.type = ACPI_TYPE_INTEGER; + in_obj.integer.value = arg; + + status = acpi_evaluate_integer(handle, (char *)name, ¶ms, &result); + if (ACPI_FAILURE(status)) + return -EIO; + + if (res) + *res = result; + + return 0; +} + static int exec_simple_method(acpi_handle handle, const char *name, unsigned long arg) { acpi_status status = acpi_execute_simple_method(handle, (char *)name, arg); @@ -284,6 +359,89 @@ static int eval_dytc(acpi_handle handle, unsigned long cmd, unsigned long *res) return eval_int_with_arg(handle, "DYTC", cmd, res); } +static int eval_vpcr(acpi_handle handle, unsigned long cmd, unsigned long *res) +{ + return eval_int_with_arg(handle, "VPCR", cmd, res); +} + +static int eval_vpcw(acpi_handle handle, unsigned long cmd, unsigned long data) +{ + struct acpi_object_list params; + union acpi_object in_obj[2]; + acpi_status status; + + params.count = 2; + params.pointer = in_obj; + in_obj[0].type = ACPI_TYPE_INTEGER; + in_obj[0].integer.value = cmd; + in_obj[1].type = ACPI_TYPE_INTEGER; + in_obj[1].integer.value = data; + + status = acpi_evaluate_object(handle, "VPCW", ¶ms, NULL); + if (ACPI_FAILURE(status)) + return -EIO; + + return 0; +} + +static int read_ec_data(acpi_handle handle, unsigned long cmd, unsigned long *data) +{ + unsigned long end_jiffies, val; + int err; + + err = eval_vpcw(handle, 1, cmd); + if (err) + return err; + + end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1; + + while (time_before(jiffies, end_jiffies)) { + usleep_range(IDEAPAD_EC_POLL_MIN_US, IDEAPAD_EC_POLL_MAX_US); + + err = eval_vpcr(handle, 1, &val); + if (err) + return err; + + if (val == 0) + return eval_vpcr(handle, 0, data); + } + + acpi_handle_err(handle, "timeout in %s\n", __func__); + + return -ETIMEDOUT; +} + +static int write_ec_cmd(acpi_handle handle, unsigned long cmd, unsigned long data) +{ + unsigned long end_jiffies, val; + int err; + + err = eval_vpcw(handle, 0, data); + if (err) + return err; + + err = eval_vpcw(handle, 1, cmd); + if (err) + return err; + + end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1; + + while (time_before(jiffies, end_jiffies)) { + usleep_range(IDEAPAD_EC_POLL_MIN_US, IDEAPAD_EC_POLL_MAX_US); + + err = eval_vpcr(handle, 1, &val); + if (err) + return err; + + if (val == 0) + return 0; + } + + acpi_handle_err(handle, "timeout in %s\n", __func__); + + return -ETIMEDOUT; +} + /* * debugfs */ @@ -1101,6 +1259,9 @@ static const struct key_entry ideapad_keymap[] = { { KE_KEY, 0x27 | IDEAPAD_WMI_KEY, { KEY_HELP } }, /* Refresh Rate Toggle */ { KE_KEY, 0x0a | IDEAPAD_WMI_KEY, { KEY_DISPLAYTOGGLE } }, + /* Specific to some newer models */ + { KE_KEY, 0x3e | IDEAPAD_WMI_KEY, { KEY_MICMUTE } }, + { KE_KEY, 0x3f | IDEAPAD_WMI_KEY, { KEY_RFKILL } }, { KE_END }, }; @@ -1439,7 +1600,7 @@ static int ideapad_kbd_bl_init(struct ideapad_private *priv) priv->kbd_bl.led.name = "platform::" LED_FUNCTION_KBD_BACKLIGHT; priv->kbd_bl.led.brightness_get = ideapad_kbd_bl_led_cdev_brightness_get; priv->kbd_bl.led.brightness_set_blocking = ideapad_kbd_bl_led_cdev_brightness_set; - priv->kbd_bl.led.flags = LED_BRIGHT_HW_CHANGED; + priv->kbd_bl.led.flags = LED_BRIGHT_HW_CHANGED | LED_RETAIN_AT_SHUTDOWN; err = led_classdev_register(&priv->platform_device->dev, &priv->kbd_bl.led); if (err) @@ -1498,6 +1659,79 @@ static void ideapad_sync_touchpad_state(struct ideapad_private *priv, bool send_ priv->r_touchpad_val = value; } +static const struct dmi_system_id ymc_ec_trigger_quirk_dmi_table[] = { + { + /* Lenovo Yoga 7 14ARB7 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "82QF"), + }, + }, + { + /* Lenovo Yoga 7 14ACN6 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "82N7"), + }, + }, + { } +}; + +static void ideapad_laptop_trigger_ec(void) +{ + struct ideapad_private *priv; + int ret; + + guard(mutex)(&ideapad_shared_mutex); + + priv = ideapad_shared; + if (!priv) + return; + + if (!priv->features.ymc_ec_trigger) + return; + + ret = write_ec_cmd(priv->adev->handle, VPCCMD_W_YMC, 1); + if (ret) + dev_warn(&priv->platform_device->dev, "Could not write YMC: %d\n", ret); +} + +static int ideapad_laptop_nb_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + switch (action) { + case IDEAPAD_LAPTOP_YMC_EVENT: + ideapad_laptop_trigger_ec(); + break; + } + + return 0; +} + +static struct notifier_block ideapad_laptop_notifier = { + .notifier_call = ideapad_laptop_nb_notify, +}; + +static BLOCKING_NOTIFIER_HEAD(ideapad_laptop_chain_head); + +int ideapad_laptop_register_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&ideapad_laptop_chain_head, nb); +} +EXPORT_SYMBOL_NS_GPL(ideapad_laptop_register_notifier, IDEAPAD_LAPTOP); + +int ideapad_laptop_unregister_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&ideapad_laptop_chain_head, nb); +} +EXPORT_SYMBOL_NS_GPL(ideapad_laptop_unregister_notifier, IDEAPAD_LAPTOP); + +void ideapad_laptop_call_notifier(unsigned long action, void *data) +{ + blocking_notifier_call_chain(&ideapad_laptop_chain_head, action, data); +} +EXPORT_SYMBOL_NS_GPL(ideapad_laptop_call_notifier, IDEAPAD_LAPTOP); + static void ideapad_acpi_notify(acpi_handle handle, u32 event, void *data) { struct ideapad_private *priv = data; @@ -1634,6 +1868,8 @@ static void ideapad_check_features(struct ideapad_private *priv) priv->features.ctrl_ps2_aux_port = ctrl_ps2_aux_port || dmi_check_system(ctrl_ps2_aux_port_list); priv->features.touchpad_ctrl_via_ec = touchpad_ctrl_via_ec; + priv->features.ymc_ec_trigger = + ymc_ec_trigger || dmi_check_system(ymc_ec_trigger_quirk_dmi_table); if (!read_ec_data(handle, VPCCMD_R_FAN, &val)) priv->features.fan_mode = true; @@ -1869,6 +2105,8 @@ static int ideapad_acpi_add(struct platform_device *pdev) if (err) goto shared_init_failed; + ideapad_laptop_register_notifier(&ideapad_laptop_notifier); + return 0; shared_init_failed: @@ -1900,6 +2138,8 @@ static void ideapad_acpi_remove(struct platform_device *pdev) struct ideapad_private *priv = dev_get_drvdata(&pdev->dev); int i; + ideapad_laptop_unregister_notifier(&ideapad_laptop_notifier); + ideapad_shared_exit(priv); acpi_remove_notify_handler(priv->adev->handle, diff --git a/drivers/platform/x86/ideapad-laptop.h b/drivers/platform/x86/ideapad-laptop.h index 4498a96de5976..1e52f2aa0aac3 100644 --- a/drivers/platform/x86/ideapad-laptop.h +++ b/drivers/platform/x86/ideapad-laptop.h @@ -9,144 +9,14 @@ #ifndef _IDEAPAD_LAPTOP_H_ #define _IDEAPAD_LAPTOP_H_ -#include -#include -#include +#include -enum { - VPCCMD_R_VPC1 = 0x10, - VPCCMD_R_BL_MAX, - VPCCMD_R_BL, - VPCCMD_W_BL, - VPCCMD_R_WIFI, - VPCCMD_W_WIFI, - VPCCMD_R_BT, - VPCCMD_W_BT, - VPCCMD_R_BL_POWER, - VPCCMD_R_NOVO, - VPCCMD_R_VPC2, - VPCCMD_R_TOUCHPAD, - VPCCMD_W_TOUCHPAD, - VPCCMD_R_CAMERA, - VPCCMD_W_CAMERA, - VPCCMD_R_3G, - VPCCMD_W_3G, - VPCCMD_R_ODD, /* 0x21 */ - VPCCMD_W_FAN, - VPCCMD_R_RF, - VPCCMD_W_RF, - VPCCMD_W_YMC = 0x2A, - VPCCMD_R_FAN = 0x2B, - VPCCMD_R_SPECIAL_BUTTONS = 0x31, - VPCCMD_W_BL_POWER = 0x33, +enum ideapad_laptop_notifier_actions { + IDEAPAD_LAPTOP_YMC_EVENT, }; -static inline int eval_int_with_arg(acpi_handle handle, const char *name, unsigned long arg, unsigned long *res) -{ - struct acpi_object_list params; - unsigned long long result; - union acpi_object in_obj; - acpi_status status; +int ideapad_laptop_register_notifier(struct notifier_block *nb); +int ideapad_laptop_unregister_notifier(struct notifier_block *nb); +void ideapad_laptop_call_notifier(unsigned long action, void *data); - params.count = 1; - params.pointer = &in_obj; - in_obj.type = ACPI_TYPE_INTEGER; - in_obj.integer.value = arg; - - status = acpi_evaluate_integer(handle, (char *)name, ¶ms, &result); - if (ACPI_FAILURE(status)) - return -EIO; - - if (res) - *res = result; - - return 0; -} - -static inline int eval_vpcr(acpi_handle handle, unsigned long cmd, unsigned long *res) -{ - return eval_int_with_arg(handle, "VPCR", cmd, res); -} - -static inline int eval_vpcw(acpi_handle handle, unsigned long cmd, unsigned long data) -{ - struct acpi_object_list params; - union acpi_object in_obj[2]; - acpi_status status; - - params.count = 2; - params.pointer = in_obj; - in_obj[0].type = ACPI_TYPE_INTEGER; - in_obj[0].integer.value = cmd; - in_obj[1].type = ACPI_TYPE_INTEGER; - in_obj[1].integer.value = data; - - status = acpi_evaluate_object(handle, "VPCW", ¶ms, NULL); - if (ACPI_FAILURE(status)) - return -EIO; - - return 0; -} - -#define IDEAPAD_EC_TIMEOUT 200 /* in ms */ - -static inline int read_ec_data(acpi_handle handle, unsigned long cmd, unsigned long *data) -{ - unsigned long end_jiffies, val; - int err; - - err = eval_vpcw(handle, 1, cmd); - if (err) - return err; - - end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1; - - while (time_before(jiffies, end_jiffies)) { - schedule(); - - err = eval_vpcr(handle, 1, &val); - if (err) - return err; - - if (val == 0) - return eval_vpcr(handle, 0, data); - } - - acpi_handle_err(handle, "timeout in %s\n", __func__); - - return -ETIMEDOUT; -} - -static inline int write_ec_cmd(acpi_handle handle, unsigned long cmd, unsigned long data) -{ - unsigned long end_jiffies, val; - int err; - - err = eval_vpcw(handle, 0, data); - if (err) - return err; - - err = eval_vpcw(handle, 1, cmd); - if (err) - return err; - - end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1; - - while (time_before(jiffies, end_jiffies)) { - schedule(); - - err = eval_vpcr(handle, 1, &val); - if (err) - return err; - - if (val == 0) - return 0; - } - - acpi_handle_err(handle, "timeout in %s\n", __func__); - - return -ETIMEDOUT; -} - -#undef IDEAPAD_EC_TIMEOUT #endif /* !_IDEAPAD_LAPTOP_H_ */ diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_mmio.c b/drivers/platform/x86/intel/speed_select_if/isst_if_mmio.c index ff49025ec0856..bb38e5f021a80 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_mmio.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_mmio.c @@ -106,11 +106,11 @@ static int isst_if_probe(struct pci_dev *pdev, const struct pci_device_id *ent) ret = pci_read_config_dword(pdev, 0xD0, &mmio_base); if (ret) - return ret; + return pcibios_err_to_errno(ret); ret = pci_read_config_dword(pdev, 0xFC, &pcu_base); if (ret) - return ret; + return pcibios_err_to_errno(ret); pcu_base &= GENMASK(10, 0); base_addr = (u64)mmio_base << 23 | (u64) pcu_base << 12; diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c index 1050221645482..5be3b26876151 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c @@ -269,10 +269,13 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_ /* Get the package ID from the TPMI core */ plat_info = tpmi_get_platform_data(auxdev); - if (plat_info) - pkg = plat_info->package_id; - else + if (unlikely(!plat_info)) { dev_info(&auxdev->dev, "Platform information is NULL\n"); + ret = -ENODEV; + goto err_rem_common; + } + + pkg = plat_info->package_id; for (i = 0; i < num_resources; ++i) { struct tpmi_uncore_power_domain_info *pd_info; diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c index a3b25253b6fde..2c9c5cc7d854e 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c @@ -121,15 +121,13 @@ static int uncore_event_cpu_online(unsigned int cpu) { struct uncore_data *data; int target; + int ret; /* Check if there is an online cpu in the package for uncore MSR */ target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); if (target < nr_cpu_ids) return 0; - /* Use this CPU on this die as a control CPU */ - cpumask_set_cpu(cpu, &uncore_cpu_mask); - data = uncore_get_instance(cpu); if (!data) return 0; @@ -138,7 +136,14 @@ static int uncore_event_cpu_online(unsigned int cpu) data->die_id = topology_die_id(cpu); data->domain_id = UNCORE_DOMAIN_ID_INVALID; - return uncore_freq_add_entry(data, cpu); + ret = uncore_freq_add_entry(data, cpu); + if (ret) + return ret; + + /* Use this CPU on this die as a control CPU */ + cpumask_set_cpu(cpu, &uncore_cpu_mask); + + return 0; } static int uncore_event_cpu_offline(unsigned int cpu) diff --git a/drivers/platform/x86/lenovo-ymc.c b/drivers/platform/x86/lenovo-ymc.c index ef2c267ab485c..bd9f95404c7cb 100644 --- a/drivers/platform/x86/lenovo-ymc.c +++ b/drivers/platform/x86/lenovo-ymc.c @@ -20,32 +20,10 @@ #define LENOVO_YMC_QUERY_INSTANCE 0 #define LENOVO_YMC_QUERY_METHOD 0x01 -static bool ec_trigger __read_mostly; -module_param(ec_trigger, bool, 0444); -MODULE_PARM_DESC(ec_trigger, "Enable EC triggering work-around to force emitting tablet mode events"); - static bool force; module_param(force, bool, 0444); MODULE_PARM_DESC(force, "Force loading on boards without a convertible DMI chassis-type"); -static const struct dmi_system_id ec_trigger_quirk_dmi_table[] = { - { - /* Lenovo Yoga 7 14ARB7 */ - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), - DMI_MATCH(DMI_PRODUCT_NAME, "82QF"), - }, - }, - { - /* Lenovo Yoga 7 14ACN6 */ - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), - DMI_MATCH(DMI_PRODUCT_NAME, "82N7"), - }, - }, - { } -}; - static const struct dmi_system_id allowed_chasis_types_dmi_table[] = { { .matches = { @@ -62,21 +40,8 @@ static const struct dmi_system_id allowed_chasis_types_dmi_table[] = { struct lenovo_ymc_private { struct input_dev *input_dev; - struct acpi_device *ec_acpi_dev; }; -static void lenovo_ymc_trigger_ec(struct wmi_device *wdev, struct lenovo_ymc_private *priv) -{ - int err; - - if (!priv->ec_acpi_dev) - return; - - err = write_ec_cmd(priv->ec_acpi_dev->handle, VPCCMD_W_YMC, 1); - if (err) - dev_warn(&wdev->dev, "Could not write YMC: %d\n", err); -} - static const struct key_entry lenovo_ymc_keymap[] = { /* Ignore the uninitialized state */ { KE_IGNORE, 0x00 }, @@ -127,11 +92,9 @@ static void lenovo_ymc_notify(struct wmi_device *wdev, union acpi_object *data) free_obj: kfree(obj); - lenovo_ymc_trigger_ec(wdev, priv); + ideapad_laptop_call_notifier(IDEAPAD_LAPTOP_YMC_EVENT, &code); } -static void acpi_dev_put_helper(void *p) { acpi_dev_put(p); } - static int lenovo_ymc_probe(struct wmi_device *wdev, const void *ctx) { struct lenovo_ymc_private *priv; @@ -145,29 +108,10 @@ static int lenovo_ymc_probe(struct wmi_device *wdev, const void *ctx) return -ENODEV; } - ec_trigger |= dmi_check_system(ec_trigger_quirk_dmi_table); - priv = devm_kzalloc(&wdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - if (ec_trigger) { - pr_debug("Lenovo YMC enable EC triggering.\n"); - priv->ec_acpi_dev = acpi_dev_get_first_match_dev("VPC2004", NULL, -1); - - if (!priv->ec_acpi_dev) { - dev_err(&wdev->dev, "Could not find EC ACPI device.\n"); - return -ENODEV; - } - err = devm_add_action_or_reset(&wdev->dev, - acpi_dev_put_helper, priv->ec_acpi_dev); - if (err) { - dev_err(&wdev->dev, - "Could not clean up EC ACPI device: %d\n", err); - return err; - } - } - input_dev = devm_input_allocate_device(&wdev->dev); if (!input_dev) return -ENOMEM; @@ -194,7 +138,6 @@ static int lenovo_ymc_probe(struct wmi_device *wdev, const void *ctx) dev_set_drvdata(&wdev->dev, priv); /* Report the state for the first time on probe */ - lenovo_ymc_trigger_ec(wdev, priv); lenovo_ymc_notify(wdev, NULL); return 0; } @@ -219,3 +162,4 @@ module_wmi_driver(lenovo_ymc_driver); MODULE_AUTHOR("Gergo Koteles "); MODULE_DESCRIPTION("Lenovo Yoga Mode Control driver"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(IDEAPAD_LAPTOP); diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c index 9569f11dec8c5..bd7323cf9a97f 100644 --- a/drivers/platform/x86/sony-laptop.c +++ b/drivers/platform/x86/sony-laptop.c @@ -757,7 +757,6 @@ static union acpi_object *__call_snc_method(acpi_handle handle, char *method, return result; } -#define MIN(a, b) (a > b ? b : a) static int sony_nc_buffer_call(acpi_handle handle, char *name, u64 *value, void *buffer, size_t buflen) { diff --git a/drivers/platform/x86/think-lmi.c b/drivers/platform/x86/think-lmi.c index 2396decdb3cb3..d5319b4637e18 100644 --- a/drivers/platform/x86/think-lmi.c +++ b/drivers/platform/x86/think-lmi.c @@ -195,7 +195,6 @@ static const char * const level_options[] = { [TLMI_LEVEL_MASTER] = "master", }; static struct think_lmi tlmi_priv; -static struct class *fw_attr_class; static DEFINE_MUTEX(tlmi_mutex); /* ------ Utility functions ------------*/ @@ -917,6 +916,7 @@ static const struct attribute_group auth_attr_group = { .is_visible = auth_attr_is_visible, .attrs = auth_attrs, }; +__ATTRIBUTE_GROUPS(auth_attr); /* ---- Attributes sysfs --------------------------------------------------------- */ static ssize_t display_name_show(struct kobject *kobj, struct kobj_attribute *attr, @@ -1120,6 +1120,7 @@ static const struct attribute_group tlmi_attr_group = { .is_visible = attr_is_visible, .attrs = tlmi_attrs, }; +__ATTRIBUTE_GROUPS(tlmi_attr); static void tlmi_attr_setting_release(struct kobject *kobj) { @@ -1139,11 +1140,13 @@ static void tlmi_pwd_setting_release(struct kobject *kobj) static const struct kobj_type tlmi_attr_setting_ktype = { .release = &tlmi_attr_setting_release, .sysfs_ops = &kobj_sysfs_ops, + .default_groups = tlmi_attr_groups, }; static const struct kobj_type tlmi_pwd_setting_ktype = { .release = &tlmi_pwd_setting_release, .sysfs_ops = &kobj_sysfs_ops, + .default_groups = auth_attr_groups, }; static ssize_t pending_reboot_show(struct kobject *kobj, struct kobj_attribute *attr, @@ -1213,19 +1216,16 @@ static struct kobj_attribute debug_cmd = __ATTR_WO(debug_cmd); /* ---- Initialisation --------------------------------------------------------- */ static void tlmi_release_attr(void) { - int i; + struct kobject *pos, *n; /* Attribute structures */ - for (i = 0; i < TLMI_SETTINGS_COUNT; i++) { - if (tlmi_priv.setting[i]) { - sysfs_remove_group(&tlmi_priv.setting[i]->kobj, &tlmi_attr_group); - kobject_put(&tlmi_priv.setting[i]->kobj); - } - } sysfs_remove_file(&tlmi_priv.attribute_kset->kobj, &pending_reboot.attr); if (tlmi_priv.can_debug_cmd && debug_support) sysfs_remove_file(&tlmi_priv.attribute_kset->kobj, &debug_cmd.attr); + list_for_each_entry_safe(pos, n, &tlmi_priv.attribute_kset->list, entry) + kobject_put(pos); + kset_unregister(tlmi_priv.attribute_kset); /* Free up any saved signatures */ @@ -1233,19 +1233,8 @@ static void tlmi_release_attr(void) kfree(tlmi_priv.pwd_admin->save_signature); /* Authentication structures */ - sysfs_remove_group(&tlmi_priv.pwd_admin->kobj, &auth_attr_group); - kobject_put(&tlmi_priv.pwd_admin->kobj); - sysfs_remove_group(&tlmi_priv.pwd_power->kobj, &auth_attr_group); - kobject_put(&tlmi_priv.pwd_power->kobj); - - if (tlmi_priv.opcode_support) { - sysfs_remove_group(&tlmi_priv.pwd_system->kobj, &auth_attr_group); - kobject_put(&tlmi_priv.pwd_system->kobj); - sysfs_remove_group(&tlmi_priv.pwd_hdd->kobj, &auth_attr_group); - kobject_put(&tlmi_priv.pwd_hdd->kobj); - sysfs_remove_group(&tlmi_priv.pwd_nvme->kobj, &auth_attr_group); - kobject_put(&tlmi_priv.pwd_nvme->kobj); - } + list_for_each_entry_safe(pos, n, &tlmi_priv.authentication_kset->list, entry) + kobject_put(pos); kset_unregister(tlmi_priv.authentication_kset); } @@ -1272,11 +1261,7 @@ static int tlmi_sysfs_init(void) { int i, ret; - ret = fw_attributes_class_get(&fw_attr_class); - if (ret) - return ret; - - tlmi_priv.class_dev = device_create(fw_attr_class, NULL, MKDEV(0, 0), + tlmi_priv.class_dev = device_create(&firmware_attributes_class, NULL, MKDEV(0, 0), NULL, "%s", "thinklmi"); if (IS_ERR(tlmi_priv.class_dev)) { ret = PTR_ERR(tlmi_priv.class_dev); @@ -1290,6 +1275,14 @@ static int tlmi_sysfs_init(void) goto fail_device_created; } + tlmi_priv.authentication_kset = kset_create_and_add("authentication", NULL, + &tlmi_priv.class_dev->kobj); + if (!tlmi_priv.authentication_kset) { + kset_unregister(tlmi_priv.attribute_kset); + ret = -ENOMEM; + goto fail_device_created; + } + for (i = 0; i < TLMI_SETTINGS_COUNT; i++) { /* Check if index is a valid setting - skip if it isn't */ if (!tlmi_priv.setting[i]) @@ -1306,12 +1299,8 @@ static int tlmi_sysfs_init(void) /* Build attribute */ tlmi_priv.setting[i]->kobj.kset = tlmi_priv.attribute_kset; - ret = kobject_add(&tlmi_priv.setting[i]->kobj, NULL, - "%s", tlmi_priv.setting[i]->display_name); - if (ret) - goto fail_create_attr; - - ret = sysfs_create_group(&tlmi_priv.setting[i]->kobj, &tlmi_attr_group); + ret = kobject_init_and_add(&tlmi_priv.setting[i]->kobj, &tlmi_attr_setting_ktype, + NULL, "%s", tlmi_priv.setting[i]->display_name); if (ret) goto fail_create_attr; } @@ -1327,55 +1316,34 @@ static int tlmi_sysfs_init(void) } /* Create authentication entries */ - tlmi_priv.authentication_kset = kset_create_and_add("authentication", NULL, - &tlmi_priv.class_dev->kobj); - if (!tlmi_priv.authentication_kset) { - ret = -ENOMEM; - goto fail_create_attr; - } tlmi_priv.pwd_admin->kobj.kset = tlmi_priv.authentication_kset; - ret = kobject_add(&tlmi_priv.pwd_admin->kobj, NULL, "%s", "Admin"); - if (ret) - goto fail_create_attr; - - ret = sysfs_create_group(&tlmi_priv.pwd_admin->kobj, &auth_attr_group); + ret = kobject_init_and_add(&tlmi_priv.pwd_admin->kobj, &tlmi_pwd_setting_ktype, + NULL, "%s", "Admin"); if (ret) goto fail_create_attr; tlmi_priv.pwd_power->kobj.kset = tlmi_priv.authentication_kset; - ret = kobject_add(&tlmi_priv.pwd_power->kobj, NULL, "%s", "Power-on"); - if (ret) - goto fail_create_attr; - - ret = sysfs_create_group(&tlmi_priv.pwd_power->kobj, &auth_attr_group); + ret = kobject_init_and_add(&tlmi_priv.pwd_power->kobj, &tlmi_pwd_setting_ktype, + NULL, "%s", "Power-on"); if (ret) goto fail_create_attr; if (tlmi_priv.opcode_support) { tlmi_priv.pwd_system->kobj.kset = tlmi_priv.authentication_kset; - ret = kobject_add(&tlmi_priv.pwd_system->kobj, NULL, "%s", "System"); - if (ret) - goto fail_create_attr; - - ret = sysfs_create_group(&tlmi_priv.pwd_system->kobj, &auth_attr_group); + ret = kobject_init_and_add(&tlmi_priv.pwd_system->kobj, &tlmi_pwd_setting_ktype, + NULL, "%s", "System"); if (ret) goto fail_create_attr; tlmi_priv.pwd_hdd->kobj.kset = tlmi_priv.authentication_kset; - ret = kobject_add(&tlmi_priv.pwd_hdd->kobj, NULL, "%s", "HDD"); - if (ret) - goto fail_create_attr; - - ret = sysfs_create_group(&tlmi_priv.pwd_hdd->kobj, &auth_attr_group); + ret = kobject_init_and_add(&tlmi_priv.pwd_hdd->kobj, &tlmi_pwd_setting_ktype, + NULL, "%s", "HDD"); if (ret) goto fail_create_attr; tlmi_priv.pwd_nvme->kobj.kset = tlmi_priv.authentication_kset; - ret = kobject_add(&tlmi_priv.pwd_nvme->kobj, NULL, "%s", "NVMe"); - if (ret) - goto fail_create_attr; - - ret = sysfs_create_group(&tlmi_priv.pwd_nvme->kobj, &auth_attr_group); + ret = kobject_init_and_add(&tlmi_priv.pwd_nvme->kobj, &tlmi_pwd_setting_ktype, + NULL, "%s", "NVMe"); if (ret) goto fail_create_attr; } @@ -1385,9 +1353,8 @@ static int tlmi_sysfs_init(void) fail_create_attr: tlmi_release_attr(); fail_device_created: - device_destroy(fw_attr_class, MKDEV(0, 0)); + device_unregister(tlmi_priv.class_dev); fail_class_created: - fw_attributes_class_put(); return ret; } @@ -1409,8 +1376,6 @@ static struct tlmi_pwd_setting *tlmi_create_auth(const char *pwd_type, new_pwd->maxlen = tlmi_priv.pwdcfg.core.max_length; new_pwd->index = 0; - kobject_init(&new_pwd->kobj, &tlmi_pwd_setting_ktype); - return new_pwd; } @@ -1514,7 +1479,6 @@ static int tlmi_analyze(void) if (setting->possible_values) strreplace(setting->possible_values, ',', ';'); - kobject_init(&setting->kobj, &tlmi_attr_setting_ktype); tlmi_priv.setting[i] = setting; kfree(item); } @@ -1610,8 +1574,7 @@ static int tlmi_analyze(void) static void tlmi_remove(struct wmi_device *wdev) { tlmi_release_attr(); - device_destroy(fw_attr_class, MKDEV(0, 0)); - fw_attributes_class_put(); + device_unregister(tlmi_priv.class_dev); } static int tlmi_probe(struct wmi_device *wdev, const void *context) diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c index cde5f845cf255..88364a5502e69 100644 --- a/drivers/platform/x86/thinkpad_acpi.c +++ b/drivers/platform/x86/thinkpad_acpi.c @@ -212,6 +212,7 @@ enum tpacpi_hkey_event_t { /* Thermal events */ TP_HKEY_EV_ALARM_BAT_HOT = 0x6011, /* battery too hot */ TP_HKEY_EV_ALARM_BAT_XHOT = 0x6012, /* battery critically hot */ + TP_HKEY_EV_ALARM_BAT_LIM_CHANGE = 0x6013, /* battery charge limit changed*/ TP_HKEY_EV_ALARM_SENSOR_HOT = 0x6021, /* sensor too hot */ TP_HKEY_EV_ALARM_SENSOR_XHOT = 0x6022, /* sensor critically hot */ TP_HKEY_EV_THM_TABLE_CHANGED = 0x6030, /* windows; thermal table changed */ @@ -536,12 +537,12 @@ static unsigned long __init tpacpi_check_quirks( return 0; } -static inline bool __pure __init tpacpi_is_lenovo(void) +static __always_inline bool __pure __init tpacpi_is_lenovo(void) { return thinkpad_id.vendor == PCI_VENDOR_ID_LENOVO; } -static inline bool __pure __init tpacpi_is_ibm(void) +static __always_inline bool __pure __init tpacpi_is_ibm(void) { return thinkpad_id.vendor == PCI_VENDOR_ID_IBM; } @@ -3942,6 +3943,10 @@ static bool hotkey_notify_6xxx(const u32 hkey, pr_alert("THERMAL EMERGENCY: battery is extremely hot!\n"); /* recommended action: immediate sleep/hibernate */ break; + case TP_HKEY_EV_ALARM_BAT_LIM_CHANGE: + pr_debug("Battery Info: battery charge threshold changed\n"); + /* User changed charging threshold. No action needed */ + return true; case TP_HKEY_EV_ALARM_SENSOR_HOT: pr_crit("THERMAL ALARM: a sensor reports something is too hot!\n"); /* recommended action: warn user through gui, that */ @@ -11315,6 +11320,8 @@ static int __must_check __init get_thinkpad_model_data( tp->vendor = PCI_VENDOR_ID_IBM; else if (dmi_name_in_vendors("LENOVO")) tp->vendor = PCI_VENDOR_ID_LENOVO; + else if (dmi_name_in_vendors("NEC")) + tp->vendor = PCI_VENDOR_ID_LENOVO; else return 0; diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index d62a776c89a12..e592f819c8fa0 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -306,6 +306,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } static const struct of_device_id apple_pmgr_ps_of_match[] = { + { .compatible = "apple,t8103-pmgr-pwrstate" }, { .compatible = "apple,pmgr-pwrstate" }, {} }; diff --git a/drivers/pmdomain/imx/gpc.c b/drivers/pmdomain/imx/gpc.c index 419ed15cc10c4..d7c0301a7121b 100644 --- a/drivers/pmdomain/imx/gpc.c +++ b/drivers/pmdomain/imx/gpc.c @@ -512,7 +512,7 @@ static int imx_gpc_probe(struct platform_device *pdev) return 0; } -static int imx_gpc_remove(struct platform_device *pdev) +static void imx_gpc_remove(struct platform_device *pdev) { struct device_node *pgc_node; int ret; @@ -522,7 +522,7 @@ static int imx_gpc_remove(struct platform_device *pdev) /* bail out if DT too old and doesn't provide the necessary info */ if (!of_property_read_bool(pdev->dev.of_node, "#power-domain-cells") && !pgc_node) - return 0; + return; /* * If the old DT binding is used the toplevel driver needs to @@ -532,16 +532,22 @@ static int imx_gpc_remove(struct platform_device *pdev) of_genpd_del_provider(pdev->dev.of_node); ret = pm_genpd_remove(&imx_gpc_domains[GPC_PGC_DOMAIN_PU].base); - if (ret) - return ret; + if (ret) { + dev_err(&pdev->dev, "Failed to remove PU power domain (%pe)\n", + ERR_PTR(ret)); + return; + } imx_pgc_put_clocks(&imx_gpc_domains[GPC_PGC_DOMAIN_PU]); ret = pm_genpd_remove(&imx_gpc_domains[GPC_PGC_DOMAIN_ARM].base); - if (ret) - return ret; + if (ret) { + dev_err(&pdev->dev, "Failed to remove ARM power domain (%pe)\n", + ERR_PTR(ret)); + return; + } } - return 0; + of_node_put(pgc_node); } static struct platform_driver imx_gpc_driver = { @@ -550,6 +556,6 @@ static struct platform_driver imx_gpc_driver = { .of_match_table = imx_gpc_dt_ids, }, .probe = imx_gpc_probe, - .remove = imx_gpc_remove, + .remove_new = imx_gpc_remove, }; builtin_platform_driver(imx_gpc_driver) diff --git a/drivers/pmdomain/imx/gpcv2.c b/drivers/pmdomain/imx/gpcv2.c index 13fce2b134f60..84d68c805cac8 100644 --- a/drivers/pmdomain/imx/gpcv2.c +++ b/drivers/pmdomain/imx/gpcv2.c @@ -1350,7 +1350,7 @@ static int imx_pgc_domain_probe(struct platform_device *pdev) } if (IS_ENABLED(CONFIG_LOCKDEP) && - of_property_read_bool(domain->dev->of_node, "power-domains")) + of_property_present(domain->dev->of_node, "power-domains")) lockdep_set_subclass(&domain->genpd.mlock, 1); ret = of_genpd_add_provider_simple(domain->dev->of_node, diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c index cc5ef6e2f0a8c..0dfaf1d14035d 100644 --- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c @@ -664,6 +664,11 @@ static const struct imx8m_blk_ctrl_data imx8mn_disp_blk_ctl_dev_data = { #define LCDIF_1_RD_HURRY GENMASK(15, 13) #define LCDIF_0_RD_HURRY GENMASK(12, 10) +#define ISI_CACHE_CTRL 0x50 +#define ISI_V_WR_HURRY GENMASK(28, 26) +#define ISI_U_WR_HURRY GENMASK(25, 23) +#define ISI_Y_WR_HURRY GENMASK(22, 20) + static int imx8mp_media_power_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -693,6 +698,11 @@ static int imx8mp_media_power_notifier(struct notifier_block *nb, regmap_set_bits(bc->regmap, LCDIF_ARCACHE_CTRL, FIELD_PREP(LCDIF_1_RD_HURRY, 7) | FIELD_PREP(LCDIF_0_RD_HURRY, 7)); + /* Same here for ISI */ + regmap_set_bits(bc->regmap, ISI_CACHE_CTRL, + FIELD_PREP(ISI_V_WR_HURRY, 7) | + FIELD_PREP(ISI_U_WR_HURRY, 7) | + FIELD_PREP(ISI_Y_WR_HURRY, 7)); } return NOTIFY_OK; diff --git a/drivers/pmdomain/samsung/exynos-pm-domains.c b/drivers/pmdomain/samsung/exynos-pm-domains.c index 9b502e8751d18..0f065748f9ec0 100644 --- a/drivers/pmdomain/samsung/exynos-pm-domains.c +++ b/drivers/pmdomain/samsung/exynos-pm-domains.c @@ -92,13 +92,14 @@ static const struct of_device_id exynos_pm_domain_of_match[] = { { }, }; -static const char *exynos_get_domain_name(struct device_node *node) +static const char *exynos_get_domain_name(struct device *dev, + struct device_node *node) { const char *name; if (of_property_read_string(node, "label", &name) < 0) name = kbasename(node->full_name); - return kstrdup_const(name, GFP_KERNEL); + return devm_kstrdup_const(dev, name, GFP_KERNEL); } static int exynos_pd_probe(struct platform_device *pdev) @@ -115,15 +116,13 @@ static int exynos_pd_probe(struct platform_device *pdev) if (!pd) return -ENOMEM; - pd->pd.name = exynos_get_domain_name(np); + pd->pd.name = exynos_get_domain_name(dev, np); if (!pd->pd.name) return -ENOMEM; pd->base = of_iomap(np, 0); - if (!pd->base) { - kfree_const(pd->pd.name); + if (!pd->base) return -ENODEV; - } pd->pd.power_off = exynos_pd_power_off; pd->pd.power_on = exynos_pd_power_on; diff --git a/drivers/power/reset/at91-reset.c b/drivers/power/reset/at91-reset.c index aa9b012d3d00b..bafe4cc6fafdc 100644 --- a/drivers/power/reset/at91-reset.c +++ b/drivers/power/reset/at91-reset.c @@ -129,12 +129,11 @@ static int at91_reset(struct notifier_block *this, unsigned long mode, " str %4, [%0, %6]\n\t" /* Disable SDRAM1 accesses */ "1: tst %1, #0\n\t" - " beq 2f\n\t" " strne %3, [%1, #" __stringify(AT91_DDRSDRC_RTR) "]\n\t" /* Power down SDRAM1 */ " strne %4, [%1, %6]\n\t" /* Reset CPU */ - "2: str %5, [%2, #" __stringify(AT91_RSTC_CR) "]\n\t" + " str %5, [%2, #" __stringify(AT91_RSTC_CR) "]\n\t" " b .\n\t" : @@ -145,7 +144,7 @@ static int at91_reset(struct notifier_block *this, unsigned long mode, "r" cpu_to_le32(AT91_DDRSDRC_LPCB_POWER_DOWN), "r" (reset->data->reset_args), "r" (reset->ramc_lpr) - : "r4"); + ); return NOTIFY_DONE; } diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 23c8736567574..1f06dee4b8b4e 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1872,8 +1872,8 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) bool has_singe_flag = di->opts & BQ27XXX_O_ZERO; cache.flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, has_singe_flag); - if ((cache.flags & 0xff) == 0xff) - cache.flags = -1; /* read error */ + if (di->chip == BQ27000 && (cache.flags & 0xff) == 0xff) + cache.flags = -ENODEV; /* bq27000 hdq read error */ if (cache.flags >= 0) { cache.temperature = bq27xxx_battery_read_temperature(di); if (di->regs[BQ27XXX_REG_TTE] != INVALID_REG_ADDR) @@ -2044,7 +2044,7 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, mutex_unlock(&di->lock); if (psp != POWER_SUPPLY_PROP_PRESENT && di->cache.flags < 0) - return -ENODEV; + return di->cache.flags; switch (psp) { case POWER_SUPPLY_PROP_STATUS: diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c index 886e0a8e2abd1..8877fa333cd02 100644 --- a/drivers/power/supply/bq27xxx_battery_i2c.c +++ b/drivers/power/supply/bq27xxx_battery_i2c.c @@ -6,6 +6,7 @@ * Andrew F. Davis */ +#include #include #include #include @@ -32,6 +33,7 @@ static int bq27xxx_battery_i2c_read(struct bq27xxx_device_info *di, u8 reg, struct i2c_msg msg[2]; u8 data[2]; int ret; + int retry = 0; if (!client->adapter) return -ENODEV; @@ -48,7 +50,16 @@ static int bq27xxx_battery_i2c_read(struct bq27xxx_device_info *di, u8 reg, else msg[1].len = 2; - ret = i2c_transfer(client->adapter, msg, ARRAY_SIZE(msg)); + do { + ret = i2c_transfer(client->adapter, msg, ARRAY_SIZE(msg)); + if (ret == -EBUSY && ++retry < 3) { + /* sleep 10 milliseconds when busy */ + usleep_range(10000, 11000); + continue; + } + break; + } while (1); + if (ret < 0) return ret; diff --git a/drivers/power/supply/collie_battery.c b/drivers/power/supply/collie_battery.c index 68390bd1004f0..3daf7befc0bf6 100644 --- a/drivers/power/supply/collie_battery.c +++ b/drivers/power/supply/collie_battery.c @@ -440,6 +440,7 @@ static int collie_bat_probe(struct ucb1x00_dev *dev) static void collie_bat_remove(struct ucb1x00_dev *dev) { + device_init_wakeup(&ucb->dev, 0); free_irq(gpiod_to_irq(collie_bat_main.gpio_full), &collie_bat_main); power_supply_unregister(collie_bat_bu.psy); power_supply_unregister(collie_bat_main.psy); diff --git a/drivers/power/supply/cpcap-charger.c b/drivers/power/supply/cpcap-charger.c index be9764541d52a..05a3f4b208a42 100644 --- a/drivers/power/supply/cpcap-charger.c +++ b/drivers/power/supply/cpcap-charger.c @@ -689,9 +689,8 @@ static void cpcap_usb_detect(struct work_struct *work) struct power_supply *battery; battery = power_supply_get_by_name("battery"); - if (IS_ERR_OR_NULL(battery)) { - dev_err(ddata->dev, "battery power_supply not available %li\n", - PTR_ERR(battery)); + if (!battery) { + dev_err(ddata->dev, "battery power_supply not available\n"); return; } diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c index 99f3ccdc30a6a..434e3233c9f8c 100644 --- a/drivers/power/supply/cw2015_battery.c +++ b/drivers/power/supply/cw2015_battery.c @@ -702,8 +702,7 @@ static int cw_bat_probe(struct i2c_client *client) if (!cw_bat->battery_workqueue) return -ENOMEM; - devm_delayed_work_autocancel(&client->dev, - &cw_bat->battery_delay_work, cw_bat_work); + devm_delayed_work_autocancel(&client->dev, &cw_bat->battery_delay_work, cw_bat_work); queue_delayed_work(cw_bat->battery_workqueue, &cw_bat->battery_delay_work, msecs_to_jiffies(10)); return 0; diff --git a/drivers/power/supply/max14577_charger.c b/drivers/power/supply/max14577_charger.c index 96f9de775043a..daa5a6d41bb48 100644 --- a/drivers/power/supply/max14577_charger.c +++ b/drivers/power/supply/max14577_charger.c @@ -501,7 +501,7 @@ static struct max14577_charger_platform_data *max14577_charger_dt_init( static struct max14577_charger_platform_data *max14577_charger_dt_init( struct platform_device *pdev) { - return NULL; + return ERR_PTR(-ENODATA); } #endif /* CONFIG_OF */ @@ -572,7 +572,7 @@ static int max14577_charger_probe(struct platform_device *pdev) chg->max14577 = max14577; chg->pdata = max14577_charger_dt_init(pdev); - if (IS_ERR_OR_NULL(chg->pdata)) + if (IS_ERR(chg->pdata)) return PTR_ERR(chg->pdata); ret = max14577_charger_reg_init(chg); diff --git a/drivers/power/supply/max77976_charger.c b/drivers/power/supply/max77976_charger.c index 99659dc8f5a6d..56ab502e1a80b 100644 --- a/drivers/power/supply/max77976_charger.c +++ b/drivers/power/supply/max77976_charger.c @@ -292,10 +292,10 @@ static int max77976_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_ONLINE: err = max77976_get_online(chg, &val->intval); break; - case POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT_MAX: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: val->intval = MAX77976_CHG_CC_MAX; break; - case POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: err = max77976_get_integer(chg, CHG_CC, MAX77976_CHG_CC_MIN, MAX77976_CHG_CC_MAX, @@ -330,7 +330,7 @@ static int max77976_set_property(struct power_supply *psy, int err = 0; switch (psp) { - case POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: err = max77976_set_integer(chg, CHG_CC, MAX77976_CHG_CC_MIN, MAX77976_CHG_CC_MAX, @@ -355,7 +355,7 @@ static int max77976_property_is_writeable(struct power_supply *psy, enum power_supply_property psp) { switch (psp) { - case POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: return true; default: @@ -368,8 +368,8 @@ static enum power_supply_property max77976_psy_props[] = { POWER_SUPPLY_PROP_CHARGE_TYPE, POWER_SUPPLY_PROP_HEALTH, POWER_SUPPLY_PROP_ONLINE, - POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT, - POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, POWER_SUPPLY_PROP_MODEL_NAME, POWER_SUPPLY_PROP_MANUFACTURER, diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c index 5b3681b9100c1..0c993780d3ef2 100644 --- a/drivers/power/supply/qcom_battmgr.c +++ b/drivers/power/supply/qcom_battmgr.c @@ -29,8 +29,9 @@ enum qcom_battmgr_variant { #define NOTIF_BAT_PROPERTY 0x30 #define NOTIF_USB_PROPERTY 0x32 #define NOTIF_WLS_PROPERTY 0x34 -#define NOTIF_BAT_INFO 0x81 #define NOTIF_BAT_STATUS 0x80 +#define NOTIF_BAT_INFO 0x81 +#define NOTIF_BAT_CHARGING_STATE 0x83 #define BATTMGR_BAT_INFO 0x9 @@ -940,12 +941,14 @@ static void qcom_battmgr_notification(struct qcom_battmgr *battmgr, } notification = le32_to_cpu(msg->notification); + notification &= 0xff; switch (notification) { case NOTIF_BAT_INFO: battmgr->info.valid = false; fallthrough; case NOTIF_BAT_STATUS: case NOTIF_BAT_PROPERTY: + case NOTIF_BAT_CHARGING_STATE: power_supply_changed(battmgr->bat_psy); break; case NOTIF_USB_PROPERTY: @@ -975,8 +978,11 @@ static void qcom_battmgr_sc8280xp_strcpy(char *dest, const char *src) static unsigned int qcom_battmgr_sc8280xp_parse_technology(const char *chemistry) { - if (!strncmp(chemistry, "LIO", BATTMGR_CHEMISTRY_LEN)) + if ((!strncmp(chemistry, "LIO", BATTMGR_CHEMISTRY_LEN)) || + (!strncmp(chemistry, "OOI", BATTMGR_CHEMISTRY_LEN))) return POWER_SUPPLY_TECHNOLOGY_LION; + if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN)) + return POWER_SUPPLY_TECHNOLOGY_LIPO; pr_err("Unknown battery technology '%s'\n", chemistry); return POWER_SUPPLY_TECHNOLOGY_UNKNOWN; diff --git a/drivers/power/supply/sbs-charger.c b/drivers/power/supply/sbs-charger.c index f4adde4492707..d405c3657ce93 100644 --- a/drivers/power/supply/sbs-charger.c +++ b/drivers/power/supply/sbs-charger.c @@ -154,8 +154,7 @@ static const struct regmap_config sbs_regmap = { .val_format_endian = REGMAP_ENDIAN_LITTLE, /* since based on SMBus */ }; -static const struct power_supply_desc sbs_desc = { - .name = "sbs-charger", +static const struct power_supply_desc sbs_default_desc = { .type = POWER_SUPPLY_TYPE_MAINS, .properties = sbs_properties, .num_properties = ARRAY_SIZE(sbs_properties), @@ -165,9 +164,20 @@ static const struct power_supply_desc sbs_desc = { static int sbs_probe(struct i2c_client *client) { struct power_supply_config psy_cfg = {}; + struct power_supply_desc *sbs_desc; struct sbs_info *chip; int ret, val; + sbs_desc = devm_kmemdup(&client->dev, &sbs_default_desc, + sizeof(*sbs_desc), GFP_KERNEL); + if (!sbs_desc) + return -ENOMEM; + + sbs_desc->name = devm_kasprintf(&client->dev, GFP_KERNEL, "sbs-%s", + dev_name(&client->dev)); + if (!sbs_desc->name) + return -ENOMEM; + chip = devm_kzalloc(&client->dev, sizeof(struct sbs_info), GFP_KERNEL); if (!chip) return -ENOMEM; @@ -191,7 +201,7 @@ static int sbs_probe(struct i2c_client *client) return dev_err_probe(&client->dev, ret, "Failed to get device status\n"); chip->last_state = val; - chip->power_supply = devm_power_supply_register(&client->dev, &sbs_desc, &psy_cfg); + chip->power_supply = devm_power_supply_register(&client->dev, sbs_desc, &psy_cfg); if (IS_ERR(chip->power_supply)) return dev_err_probe(&client->dev, PTR_ERR(chip->power_supply), "Failed to register power supply\n"); diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c index 9662309e1d7ef..11e283fb76619 100644 --- a/drivers/powercap/dtpm_cpu.c +++ b/drivers/powercap/dtpm_cpu.c @@ -98,6 +98,8 @@ static u64 get_pd_power_uw(struct dtpm *dtpm) int i; pd = em_cpu_get(dtpm_cpu->cpu); + if (!pd) + return 0; pd_mask = em_span_cpus(pd); diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index f1de4111e98d9..5a09a56698f4a 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -338,12 +338,28 @@ static int set_domain_enable(struct powercap_zone *power_zone, bool mode) { struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); struct rapl_defaults *defaults = get_defaults(rd->rp); + u64 val; int ret; cpus_read_lock(); ret = rapl_write_pl_data(rd, POWER_LIMIT1, PL_ENABLE, mode); - if (!ret && defaults->set_floor_freq) + if (ret) + goto end; + + ret = rapl_read_pl_data(rd, POWER_LIMIT1, PL_ENABLE, false, &val); + if (ret) + goto end; + + if (mode != val) { + pr_debug("%s cannot be %s\n", power_zone->name, + str_enabled_disabled(mode)); + goto end; + } + + if (defaults->set_floor_freq) defaults->set_floor_freq(rd, mode); + +end: cpus_read_unlock(); return ret; diff --git a/drivers/pps/clients/pps-gpio.c b/drivers/pps/clients/pps-gpio.c index 2f4b11b4dfcd9..198cffe7c8a94 100644 --- a/drivers/pps/clients/pps-gpio.c +++ b/drivers/pps/clients/pps-gpio.c @@ -206,8 +206,8 @@ static int pps_gpio_probe(struct platform_device *pdev) } /* register IRQ interrupt handler */ - ret = devm_request_irq(dev, data->irq, pps_gpio_irq_handler, - get_irqf_trigger_flags(data), data->info.name, data); + ret = request_irq(data->irq, pps_gpio_irq_handler, + get_irqf_trigger_flags(data), data->info.name, data); if (ret) { pps_unregister_source(data->pps); dev_err(dev, "failed to acquire IRQ %d\n", data->irq); @@ -224,6 +224,7 @@ static int pps_gpio_remove(struct platform_device *pdev) { struct pps_gpio_device_data *data = platform_get_drvdata(pdev); + free_irq(data->irq, data); pps_unregister_source(data->pps); del_timer_sync(&data->echo_timer); /* reset echo pin in any case */ diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c index 6b7e8b7ebcef5..d252d018bfe77 100644 --- a/drivers/ptp/ptp_clock.c +++ b/drivers/ptp/ptp_clock.c @@ -79,10 +79,13 @@ static int ptp_clock_settime(struct posix_clock *pc, const struct timespec64 *tp struct ptp_clock *ptp = container_of(pc, struct ptp_clock, clock); if (ptp_clock_freerun(ptp)) { - pr_err("ptp: physical clock is free running\n"); + pr_err_ratelimited("ptp: physical clock is free running\n"); return -EBUSY; } + if (!timespec64_valid_settod(tp)) + return -EINVAL; + return ptp->info->settime64(ptp->info, tp); } @@ -104,7 +107,8 @@ static int ptp_clock_adjtime(struct posix_clock *pc, struct __kernel_timex *tx) struct ptp_clock_info *ops; int err = -EOPNOTSUPP; - if (ptp_clock_freerun(ptp)) { + if (tx->modes & (ADJ_SETOFFSET | ADJ_FREQUENCY | ADJ_OFFSET) && + ptp_clock_freerun(ptp)) { pr_err("ptp: physical clock is free running\n"); return -EBUSY; } @@ -112,7 +116,7 @@ static int ptp_clock_adjtime(struct posix_clock *pc, struct __kernel_timex *tx) ops = ptp->info; if (tx->modes & ADJ_SETOFFSET) { - struct timespec64 ts; + struct timespec64 ts, ts2; ktime_t kt; s64 delta; @@ -125,6 +129,14 @@ static int ptp_clock_adjtime(struct posix_clock *pc, struct __kernel_timex *tx) if ((unsigned long) ts.tv_nsec >= NSEC_PER_SEC) return -EINVAL; + /* Make sure the offset is valid */ + err = ptp_clock_gettime(pc, &ts2); + if (err) + return err; + ts2 = timespec64_add(ts2, ts); + if (!timespec64_valid_settod(&ts2)) + return -EINVAL; + kt = timespec64_to_ktime(ts); delta = ktime_to_ns(kt); err = ops->adjtime(ops, delta); diff --git a/drivers/ptp/ptp_private.h b/drivers/ptp/ptp_private.h index b8d4f61f14be4..3fbd1d68a9bcb 100644 --- a/drivers/ptp/ptp_private.h +++ b/drivers/ptp/ptp_private.h @@ -20,6 +20,11 @@ #define PTP_BUF_TIMESTAMPS 30 #define PTP_DEFAULT_MAX_VCLOCKS 20 +enum { + PTP_LOCK_PHYSICAL = 0, + PTP_LOCK_VIRTUAL, +}; + struct timestamp_event_queue { struct ptp_extts_event buf[PTP_MAX_TIMESTAMPS]; int head; @@ -91,10 +96,20 @@ static inline bool ptp_vclock_in_use(struct ptp_clock *ptp) { bool in_use = false; + /* Virtual clocks can't be stacked on top of virtual clocks. + * Avoid acquiring the n_vclocks_mux on virtual clocks, to allow this + * function to be called from code paths where the n_vclocks_mux of the + * parent physical clock is already held. Functionally that's not an + * issue, but lockdep would complain, because they have the same lock + * class. + */ + if (ptp->is_virtual_clock) + return false; + if (mutex_lock_interruptible(&ptp->n_vclocks_mux)) return true; - if (!ptp->is_virtual_clock && ptp->n_vclocks) + if (ptp->n_vclocks) in_use = true; mutex_unlock(&ptp->n_vclocks_mux); diff --git a/drivers/ptp/ptp_vclock.c b/drivers/ptp/ptp_vclock.c index dcf752c9e0450..7d08ff3b30fc2 100644 --- a/drivers/ptp/ptp_vclock.c +++ b/drivers/ptp/ptp_vclock.c @@ -154,6 +154,11 @@ static long ptp_vclock_refresh(struct ptp_clock_info *ptp) return PTP_VCLOCK_REFRESH_INTERVAL; } +static void ptp_vclock_set_subclass(struct ptp_clock *ptp) +{ + lockdep_set_subclass(&ptp->clock.rwsem, PTP_LOCK_VIRTUAL); +} + static const struct ptp_clock_info ptp_vclock_info = { .owner = THIS_MODULE, .name = "ptp virtual clock", @@ -213,6 +218,8 @@ struct ptp_vclock *ptp_vclock_register(struct ptp_clock *pclock) return NULL; } + ptp_vclock_set_subclass(vclock->clock); + timecounter_init(&vclock->tc, &vclock->cc, 0); ptp_schedule_worker(vclock->clock, PTP_VCLOCK_REFRESH_INTERVAL); diff --git a/drivers/pwm/pwm-berlin.c b/drivers/pwm/pwm-berlin.c index 0971c666afd13..14729a6ae09a9 100644 --- a/drivers/pwm/pwm-berlin.c +++ b/drivers/pwm/pwm-berlin.c @@ -273,7 +273,7 @@ static int berlin_pwm_suspend(struct device *dev) if (!channel) continue; - channel->enable = berlin_pwm_readl(bpc, i, BERLIN_PWM_ENABLE); + channel->enable = berlin_pwm_readl(bpc, i, BERLIN_PWM_EN); channel->ctrl = berlin_pwm_readl(bpc, i, BERLIN_PWM_CONTROL); channel->duty = berlin_pwm_readl(bpc, i, BERLIN_PWM_DUTY); channel->tcnt = berlin_pwm_readl(bpc, i, BERLIN_PWM_TCNT); @@ -304,7 +304,7 @@ static int berlin_pwm_resume(struct device *dev) berlin_pwm_writel(bpc, i, channel->ctrl, BERLIN_PWM_CONTROL); berlin_pwm_writel(bpc, i, channel->duty, BERLIN_PWM_DUTY); berlin_pwm_writel(bpc, i, channel->tcnt, BERLIN_PWM_TCNT); - berlin_pwm_writel(bpc, i, channel->enable, BERLIN_PWM_ENABLE); + berlin_pwm_writel(bpc, i, channel->enable, BERLIN_PWM_EN); } return 0; diff --git a/drivers/pwm/pwm-imx-tpm.c b/drivers/pwm/pwm-imx-tpm.c index 2fc6163eace31..6591f8f84ce8e 100644 --- a/drivers/pwm/pwm-imx-tpm.c +++ b/drivers/pwm/pwm-imx-tpm.c @@ -204,6 +204,15 @@ static int pwm_imx_tpm_apply_hw(struct pwm_chip *chip, val |= FIELD_PREP(PWM_IMX_TPM_SC_PS, p->prescale); writel(val, tpm->base + PWM_IMX_TPM_SC); + /* + * if the counter is disabled (CMOD == 0), programming the new + * period length (MOD) will not reset the counter (CNT). If + * CNT.COUNT happens to be bigger than the new MOD value then + * the counter will end up being reset way too late. Therefore, + * manually reset it to 0. + */ + if (!cmod) + writel(0x0, tpm->base + PWM_IMX_TPM_CNT); /* * set period count: * if the PWM is disabled (CMOD[1:0] = 2b00), then MOD register diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c index 6b1a75b6bd12f..34413607b86f4 100644 --- a/drivers/pwm/pwm-mediatek.c +++ b/drivers/pwm/pwm-mediatek.c @@ -118,6 +118,26 @@ static inline void pwm_mediatek_writel(struct pwm_mediatek_chip *chip, writel(value, chip->regs + chip->soc->reg_offset[num] + offset); } +static void pwm_mediatek_enable(struct pwm_chip *chip, struct pwm_device *pwm) +{ + struct pwm_mediatek_chip *pc = to_pwm_mediatek_chip(chip); + u32 value; + + value = readl(pc->regs); + value |= BIT(pwm->hwpwm); + writel(value, pc->regs); +} + +static void pwm_mediatek_disable(struct pwm_chip *chip, struct pwm_device *pwm) +{ + struct pwm_mediatek_chip *pc = to_pwm_mediatek_chip(chip); + u32 value; + + value = readl(pc->regs); + value &= ~BIT(pwm->hwpwm); + writel(value, pc->regs); +} + static int pwm_mediatek_config(struct pwm_chip *chip, struct pwm_device *pwm, int duty_ns, int period_ns) { @@ -133,8 +153,10 @@ static int pwm_mediatek_config(struct pwm_chip *chip, struct pwm_device *pwm, return ret; clk_rate = clk_get_rate(pc->clk_pwms[pwm->hwpwm]); - if (!clk_rate) - return -EINVAL; + if (!clk_rate) { + ret = -EINVAL; + goto out; + } /* Make sure we use the bus clock and not the 26MHz clock */ if (pc->soc->has_ck_26m_sel) @@ -145,7 +167,10 @@ static int pwm_mediatek_config(struct pwm_chip *chip, struct pwm_device *pwm, do_div(resolution, clk_rate); cnt_period = DIV_ROUND_CLOSEST_ULL((u64)period_ns * 1000, resolution); - while (cnt_period > 8191) { + if (!cnt_period) + return -EINVAL; + + while (cnt_period > 8192) { resolution *= 2; clkdiv++; cnt_period = DIV_ROUND_CLOSEST_ULL((u64)period_ns * 1000, @@ -153,9 +178,9 @@ static int pwm_mediatek_config(struct pwm_chip *chip, struct pwm_device *pwm, } if (clkdiv > PWM_CLK_DIV_MAX) { - pwm_mediatek_clk_disable(chip, pwm); dev_err(chip->dev, "period of %d ns not supported\n", period_ns); - return -EINVAL; + ret = -EINVAL; + goto out; } if (pc->soc->pwm45_fixup && pwm->hwpwm > 2) { @@ -168,42 +193,21 @@ static int pwm_mediatek_config(struct pwm_chip *chip, struct pwm_device *pwm, } cnt_duty = DIV_ROUND_CLOSEST_ULL((u64)duty_ns * 1000, resolution); - pwm_mediatek_writel(pc, pwm->hwpwm, PWMCON, BIT(15) | clkdiv); - pwm_mediatek_writel(pc, pwm->hwpwm, reg_width, cnt_period); - pwm_mediatek_writel(pc, pwm->hwpwm, reg_thres, cnt_duty); - pwm_mediatek_clk_disable(chip, pwm); - - return 0; -} - -static int pwm_mediatek_enable(struct pwm_chip *chip, struct pwm_device *pwm) -{ - struct pwm_mediatek_chip *pc = to_pwm_mediatek_chip(chip); - u32 value; - int ret; - - ret = pwm_mediatek_clk_enable(chip, pwm); - if (ret < 0) - return ret; - - value = readl(pc->regs); - value |= BIT(pwm->hwpwm); - writel(value, pc->regs); - - return 0; -} - -static void pwm_mediatek_disable(struct pwm_chip *chip, struct pwm_device *pwm) -{ - struct pwm_mediatek_chip *pc = to_pwm_mediatek_chip(chip); - u32 value; + pwm_mediatek_writel(pc, pwm->hwpwm, PWMCON, BIT(15) | clkdiv); + pwm_mediatek_writel(pc, pwm->hwpwm, reg_width, cnt_period - 1); - value = readl(pc->regs); - value &= ~BIT(pwm->hwpwm); - writel(value, pc->regs); + if (cnt_duty) { + pwm_mediatek_writel(pc, pwm->hwpwm, reg_thres, cnt_duty - 1); + pwm_mediatek_enable(chip, pwm); + } else { + pwm_mediatek_disable(chip, pwm); + } +out: pwm_mediatek_clk_disable(chip, pwm); + + return ret; } static int pwm_mediatek_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -215,8 +219,10 @@ static int pwm_mediatek_apply(struct pwm_chip *chip, struct pwm_device *pwm, return -EINVAL; if (!state->enabled) { - if (pwm->state.enabled) + if (pwm->state.enabled) { pwm_mediatek_disable(chip, pwm); + pwm_mediatek_clk_disable(chip, pwm); + } return 0; } @@ -226,7 +232,7 @@ static int pwm_mediatek_apply(struct pwm_chip *chip, struct pwm_device *pwm, return err; if (!pwm->state.enabled) - err = pwm_mediatek_enable(chip, pwm); + err = pwm_mediatek_clk_enable(chip, pwm); return err; } diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index ecbfd7e954ecb..d693b2d65cccf 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -167,7 +167,7 @@ static int set_prescale_div(unsigned long rqst_prescaler, u16 *prescale_div, *prescale_div = (1 << clkdiv) * (hspclkdiv ? (hspclkdiv * 2) : 1); - if (*prescale_div > rqst_prescaler) { + if (*prescale_div >= rqst_prescaler) { *tb_clk_div = (clkdiv << TBCTL_CLKDIV_SHIFT) | (hspclkdiv << TBCTL_HSPCLKDIV_SHIFT); return 0; @@ -266,7 +266,7 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, pc->period_cycles[pwm->hwpwm] = period_cycles; /* Configure clock prescaler to support Low frequency PWM wave */ - if (set_prescale_div(period_cycles/PERIOD_MAX, &ps_divval, + if (set_prescale_div(DIV_ROUND_UP(period_cycles, PERIOD_MAX), &ps_divval, &tb_divval)) { dev_err(chip->dev, "Unsupported values\n"); return -EINVAL; diff --git a/drivers/rapidio/rio_cm.c b/drivers/rapidio/rio_cm.c index 49f8d111e5462..897642e62068a 100644 --- a/drivers/rapidio/rio_cm.c +++ b/drivers/rapidio/rio_cm.c @@ -787,6 +787,9 @@ static int riocm_ch_send(u16 ch_id, void *buf, int len) if (buf == NULL || ch_id == 0 || len == 0 || len > RIO_MAX_MSG_SIZE) return -EINVAL; + if (len < sizeof(struct rio_ch_chan_hdr)) + return -EINVAL; /* insufficient data from user */ + ch = riocm_get_channel(ch_id); if (!ch) { riocm_error("%s(%d) ch_%d not found", current->comm, diff --git a/drivers/regulator/ad5398.c b/drivers/regulator/ad5398.c index 40f7dba42b5ad..404cbe32711e7 100644 --- a/drivers/regulator/ad5398.c +++ b/drivers/regulator/ad5398.c @@ -14,6 +14,7 @@ #include #include #include +#include #define AD5398_CURRENT_EN_MASK 0x8000 @@ -221,15 +222,20 @@ static int ad5398_probe(struct i2c_client *client) const struct ad5398_current_data_format *df = (struct ad5398_current_data_format *)id->driver_data; - if (!init_data) - return -EINVAL; - chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL); if (!chip) return -ENOMEM; config.dev = &client->dev; + if (client->dev.of_node) + init_data = of_get_regulator_init_data(&client->dev, + client->dev.of_node, + &ad5398_reg); + if (!init_data) + return -EINVAL; + config.init_data = init_data; + config.of_node = client->dev.of_node; config.driver_data = chip; chip->client = client; diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index d2e21dc61dd7d..1d49612eeb7e5 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -5422,6 +5422,7 @@ static void regulator_remove_coupling(struct regulator_dev *rdev) ERR_PTR(err)); } + rdev->coupling_desc.n_coupled = 0; kfree(rdev->coupling_desc.coupled_rdevs); rdev->coupling_desc.coupled_rdevs = NULL; } diff --git a/drivers/regulator/fan53555.c b/drivers/regulator/fan53555.c index 48f312167e535..8912f5be72707 100644 --- a/drivers/regulator/fan53555.c +++ b/drivers/regulator/fan53555.c @@ -147,6 +147,7 @@ struct fan53555_device_info { unsigned int slew_mask; const unsigned int *ramp_delay_table; unsigned int n_ramp_values; + unsigned int enable_time; unsigned int slew_rate; }; @@ -282,6 +283,7 @@ static int fan53526_voltages_setup_fairchild(struct fan53555_device_info *di) di->slew_mask = CTL_SLEW_MASK; di->ramp_delay_table = slew_rates; di->n_ramp_values = ARRAY_SIZE(slew_rates); + di->enable_time = 250; di->vsel_count = FAN53526_NVOLTAGES; return 0; @@ -296,10 +298,12 @@ static int fan53555_voltages_setup_fairchild(struct fan53555_device_info *di) case FAN53555_CHIP_REV_00: di->vsel_min = 600000; di->vsel_step = 10000; + di->enable_time = 400; break; case FAN53555_CHIP_REV_13: di->vsel_min = 800000; di->vsel_step = 10000; + di->enable_time = 400; break; default: dev_err(di->dev, @@ -311,13 +315,19 @@ static int fan53555_voltages_setup_fairchild(struct fan53555_device_info *di) case FAN53555_CHIP_ID_01: case FAN53555_CHIP_ID_03: case FAN53555_CHIP_ID_05: + di->vsel_min = 600000; + di->vsel_step = 10000; + di->enable_time = 400; + break; case FAN53555_CHIP_ID_08: di->vsel_min = 600000; di->vsel_step = 10000; + di->enable_time = 175; break; case FAN53555_CHIP_ID_04: di->vsel_min = 603000; di->vsel_step = 12826; + di->enable_time = 400; break; default: dev_err(di->dev, @@ -350,6 +360,7 @@ static int fan53555_voltages_setup_rockchip(struct fan53555_device_info *di) di->slew_mask = CTL_SLEW_MASK; di->ramp_delay_table = slew_rates; di->n_ramp_values = ARRAY_SIZE(slew_rates); + di->enable_time = 360; di->vsel_count = FAN53555_NVOLTAGES; return 0; @@ -372,6 +383,7 @@ static int rk8602_voltages_setup_rockchip(struct fan53555_device_info *di) di->slew_mask = CTL_SLEW_MASK; di->ramp_delay_table = slew_rates; di->n_ramp_values = ARRAY_SIZE(slew_rates); + di->enable_time = 360; di->vsel_count = RK8602_NVOLTAGES; return 0; @@ -395,6 +407,7 @@ static int fan53555_voltages_setup_silergy(struct fan53555_device_info *di) di->slew_mask = CTL_SLEW_MASK; di->ramp_delay_table = slew_rates; di->n_ramp_values = ARRAY_SIZE(slew_rates); + di->enable_time = 400; di->vsel_count = FAN53555_NVOLTAGES; return 0; @@ -594,6 +607,7 @@ static int fan53555_regulator_register(struct fan53555_device_info *di, rdesc->ramp_mask = di->slew_mask; rdesc->ramp_delay_table = di->ramp_delay_table; rdesc->n_ramp_values = di->n_ramp_values; + rdesc->enable_time = di->enable_time; rdesc->owner = THIS_MODULE; rdev = devm_regulator_register(di->dev, &di->desc, config); diff --git a/drivers/regulator/fixed.c b/drivers/regulator/fixed.c index 55130efae9b8b..4477bdeb9e60f 100644 --- a/drivers/regulator/fixed.c +++ b/drivers/regulator/fixed.c @@ -286,6 +286,7 @@ static int reg_fixed_voltage_probe(struct platform_device *pdev) ret = dev_err_probe(&pdev->dev, PTR_ERR(drvdata->dev), "Failed to register regulator: %ld\n", PTR_ERR(drvdata->dev)); + gpiod_put(cfg.ena_gpiod); return ret; } diff --git a/drivers/regulator/gpio-regulator.c b/drivers/regulator/gpio-regulator.c index 65927fa2ef161..1bdd494cf8821 100644 --- a/drivers/regulator/gpio-regulator.c +++ b/drivers/regulator/gpio-regulator.c @@ -260,8 +260,10 @@ static int gpio_regulator_probe(struct platform_device *pdev) return -ENOMEM; } - drvdata->gpiods = devm_kzalloc(dev, sizeof(struct gpio_desc *), - GFP_KERNEL); + drvdata->gpiods = devm_kcalloc(dev, config->ngpios, + sizeof(struct gpio_desc *), GFP_KERNEL); + if (!drvdata->gpiods) + return -ENOMEM; if (config->input_supply) { drvdata->desc.supply_name = devm_kstrdup(&pdev->dev, @@ -274,8 +276,6 @@ static int gpio_regulator_probe(struct platform_device *pdev) } } - if (!drvdata->gpiods) - return -ENOMEM; for (i = 0; i < config->ngpios; i++) { drvdata->gpiods[i] = devm_gpiod_get_index(dev, NULL, diff --git a/drivers/regulator/max14577-regulator.c b/drivers/regulator/max14577-regulator.c index 5e7171b9065ae..41fd15adfd1fd 100644 --- a/drivers/regulator/max14577-regulator.c +++ b/drivers/regulator/max14577-regulator.c @@ -40,11 +40,14 @@ static int max14577_reg_get_current_limit(struct regulator_dev *rdev) struct max14577 *max14577 = rdev_get_drvdata(rdev); const struct maxim_charger_current *limits = &maxim_charger_currents[max14577->dev_type]; + int ret; if (rdev_get_id(rdev) != MAX14577_CHARGER) return -EINVAL; - max14577_read_reg(rmap, MAX14577_CHG_REG_CHG_CTRL4, ®_data); + ret = max14577_read_reg(rmap, MAX14577_CHG_REG_CHG_CTRL4, ®_data); + if (ret < 0) + return ret; if ((reg_data & CHGCTRL4_MBCICHWRCL_MASK) == 0) return limits->min; diff --git a/drivers/regulator/max20086-regulator.c b/drivers/regulator/max20086-regulator.c index 32f47b896fd1e..8fcca52026bf1 100644 --- a/drivers/regulator/max20086-regulator.c +++ b/drivers/regulator/max20086-regulator.c @@ -5,6 +5,7 @@ // Copyright (C) 2022 Laurent Pinchart // Copyright (C) 2018 Avnet, Inc. +#include #include #include #include @@ -28,7 +29,7 @@ #define MAX20086_REG_ADC4 0x09 /* DEVICE IDs */ -#define MAX20086_DEVICE_ID_MAX20086 0x40 +#define MAX20086_DEVICE_ID_MAX20086 0x30 #define MAX20086_DEVICE_ID_MAX20087 0x20 #define MAX20086_DEVICE_ID_MAX20088 0x10 #define MAX20086_DEVICE_ID_MAX20089 0x00 @@ -132,23 +133,27 @@ static int max20086_regulators_register(struct max20086 *chip) static int max20086_parse_regulators_dt(struct max20086 *chip, bool *boot_on) { - struct of_regulator_match matches[MAX20086_MAX_REGULATORS] = { }; - struct device_node *node; + struct of_regulator_match *matches; unsigned int i; int ret; - node = of_get_child_by_name(chip->dev->of_node, "regulators"); + struct device_node *node __free(device_node) = + of_get_child_by_name(chip->dev->of_node, "regulators"); if (!node) { dev_err(chip->dev, "regulators node not found\n"); return -ENODEV; } + matches = devm_kcalloc(chip->dev, chip->info->num_outputs, + sizeof(*matches), GFP_KERNEL); + if (!matches) + return -ENOMEM; + for (i = 0; i < chip->info->num_outputs; ++i) matches[i].name = max20086_output_names[i]; ret = of_regulator_match(chip->dev, node, matches, chip->info->num_outputs); - of_node_put(node); if (ret < 0) { dev_err(chip->dev, "Failed to match regulators\n"); return -EINVAL; @@ -259,7 +264,7 @@ static int max20086_i2c_probe(struct i2c_client *i2c) * shutdown. */ flags = boot_on ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; - chip->ena_gpiod = devm_gpiod_get(chip->dev, "enable", flags); + chip->ena_gpiod = devm_gpiod_get_optional(chip->dev, "enable", flags); if (IS_ERR(chip->ena_gpiod)) { ret = PTR_ERR(chip->ena_gpiod); dev_err(chip->dev, "Failed to get enable GPIO: %d\n", ret); diff --git a/drivers/regulator/pwm-regulator.c b/drivers/regulator/pwm-regulator.c index 226ca4c62673f..60cfcd741c2af 100644 --- a/drivers/regulator/pwm-regulator.c +++ b/drivers/regulator/pwm-regulator.c @@ -157,6 +157,13 @@ static int pwm_regulator_get_voltage(struct regulator_dev *rdev) pwm_get_state(drvdata->pwm, &pstate); + if (!pstate.enabled) { + if (pstate.polarity == PWM_POLARITY_INVERSED) + pstate.duty_cycle = pstate.period; + else + pstate.duty_cycle = 0; + } + voltage = pwm_get_relative_duty_cycle(&pstate, duty_unit); if (voltage < min(max_uV_duty, min_uV_duty) || voltage > max(max_uV_duty, min_uV_duty)) @@ -316,6 +323,32 @@ static int pwm_regulator_init_continuous(struct platform_device *pdev, return 0; } +static int pwm_regulator_init_boot_on(struct platform_device *pdev, + struct pwm_regulator_data *drvdata, + const struct regulator_init_data *init_data) +{ + struct pwm_state pstate; + + if (!init_data->constraints.boot_on || drvdata->enb_gpio) + return 0; + + pwm_get_state(drvdata->pwm, &pstate); + if (pstate.enabled) + return 0; + + /* + * Update the duty cycle so the output does not change + * when the regulator core enables the regulator (and + * thus the PWM channel). + */ + if (pstate.polarity == PWM_POLARITY_INVERSED) + pstate.duty_cycle = pstate.period; + else + pstate.duty_cycle = 0; + + return pwm_apply_might_sleep(drvdata->pwm, &pstate); +} + static int pwm_regulator_probe(struct platform_device *pdev) { const struct regulator_init_data *init_data; @@ -375,6 +408,13 @@ static int pwm_regulator_probe(struct platform_device *pdev) if (ret) return ret; + ret = pwm_regulator_init_boot_on(pdev, drvdata, init_data); + if (ret) { + dev_err(&pdev->dev, "Failed to apply boot_on settings: %d\n", + ret); + return ret; + } + regulator = devm_regulator_register(&pdev->dev, &drvdata->desc, &config); if (IS_ERR(regulator)) { diff --git a/drivers/regulator/scmi-regulator.c b/drivers/regulator/scmi-regulator.c index 29ab217297d6d..432654fbd7a12 100644 --- a/drivers/regulator/scmi-regulator.c +++ b/drivers/regulator/scmi-regulator.c @@ -257,7 +257,8 @@ static int process_scmi_regulator_of_node(struct scmi_device *sdev, struct device_node *np, struct scmi_regulator_info *rinfo) { - u32 dom, ret; + u32 dom; + int ret; ret = of_property_read_u32(np, "reg", &dom); if (ret) diff --git a/drivers/regulator/sy7636a-regulator.c b/drivers/regulator/sy7636a-regulator.c index d1e7ba1fb3e1a..27e3d939b7bb9 100644 --- a/drivers/regulator/sy7636a-regulator.c +++ b/drivers/regulator/sy7636a-regulator.c @@ -83,9 +83,11 @@ static int sy7636a_regulator_probe(struct platform_device *pdev) if (!regmap) return -EPROBE_DEFER; - gdp = devm_gpiod_get(pdev->dev.parent, "epd-pwr-good", GPIOD_IN); + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); + + gdp = devm_gpiod_get(&pdev->dev, "epd-pwr-good", GPIOD_IN); if (IS_ERR(gdp)) { - dev_err(pdev->dev.parent, "Power good GPIO fault %ld\n", PTR_ERR(gdp)); + dev_err(&pdev->dev, "Power good GPIO fault %ld\n", PTR_ERR(gdp)); return PTR_ERR(gdp); } @@ -105,7 +107,6 @@ static int sy7636a_regulator_probe(struct platform_device *pdev) } config.dev = &pdev->dev; - config.dev->of_node = pdev->dev.parent->of_node; config.regmap = regmap; rdev = devm_regulator_register(&pdev->dev, &desc, &config); diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 610a69928dff2..251f9840d85bd 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -1088,8 +1088,8 @@ static int imx_rproc_clk_enable(struct imx_rproc *priv) struct device *dev = priv->dev; int ret; - /* Remote core is not under control of Linux */ - if (dcfg->method == IMX_RPROC_NONE) + /* Remote core is not under control of Linux or it is managed by SCU API */ + if (dcfg->method == IMX_RPROC_NONE || dcfg->method == IMX_RPROC_SCU_API) return 0; priv->clk = devm_clk_get(dev, NULL); diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c index 327f0c7ee3d6b..7ffbae209a31f 100644 --- a/drivers/remoteproc/pru_rproc.c +++ b/drivers/remoteproc/pru_rproc.c @@ -340,7 +340,7 @@ EXPORT_SYMBOL_GPL(pru_rproc_put); */ int pru_rproc_set_ctable(struct rproc *rproc, enum pru_ctable_idx c, u32 addr) { - struct pru_rproc *pru = rproc->priv; + struct pru_rproc *pru; unsigned int reg; u32 mask, set; u16 idx; @@ -352,6 +352,7 @@ int pru_rproc_set_ctable(struct rproc *rproc, enum pru_ctable_idx c, u32 addr) if (!rproc->dev.parent || !is_pru_rproc(rproc->dev.parent)) return -ENODEV; + pru = rproc->priv; /* pointer is 16 bit and index is 8-bit so mask out the rest */ idx_mask = (c >= PRU_C28) ? 0xFFFF : 0xFF; diff --git a/drivers/remoteproc/qcom_q6v5.c b/drivers/remoteproc/qcom_q6v5.c index 4ee5e67a9f03f..58d5b85e58cda 100644 --- a/drivers/remoteproc/qcom_q6v5.c +++ b/drivers/remoteproc/qcom_q6v5.c @@ -156,9 +156,6 @@ int qcom_q6v5_wait_for_start(struct qcom_q6v5 *q6v5, int timeout) int ret; ret = wait_for_completion_timeout(&q6v5->start_done, timeout); - if (!ret) - disable_irq(q6v5->handover_irq); - return !ret ? -ETIMEDOUT : 0; } EXPORT_SYMBOL_GPL(qcom_q6v5_wait_for_start); @@ -167,6 +164,11 @@ static irqreturn_t q6v5_handover_interrupt(int irq, void *data) { struct qcom_q6v5 *q6v5 = data; + if (q6v5->handover_issued) { + dev_err(q6v5->dev, "Handover signaled, but it already happened\n"); + return IRQ_HANDLED; + } + if (q6v5->handover) q6v5->handover(q6v5); diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index 90de22c81da97..37174b544113f 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -117,10 +117,10 @@ static const struct wcnss_data pronto_v1_data = { .pmu_offset = 0x1004, .spare_offset = 0x1088, - .pd_names = { "mx", "cx" }, + .pd_names = { "cx", "mx" }, .vregs = (struct wcnss_vreg_info[]) { - { "vddmx", 950000, 1150000, 0 }, { "vddcx", .super_turbo = true}, + { "vddmx", 950000, 1150000, 0 }, { "vddpx", 1800000, 1800000, 0 }, }, .num_pd_vregs = 2, @@ -131,10 +131,10 @@ static const struct wcnss_data pronto_v2_data = { .pmu_offset = 0x1004, .spare_offset = 0x1088, - .pd_names = { "mx", "cx" }, + .pd_names = { "cx", "mx" }, .vregs = (struct wcnss_vreg_info[]) { - { "vddmx", 1287500, 1287500, 0 }, { "vddcx", .super_turbo = true }, + { "vddmx", 1287500, 1287500, 0 }, { "vddpx", 1800000, 1800000, 0 }, }, .num_pd_vregs = 2, @@ -397,8 +397,17 @@ static irqreturn_t wcnss_stop_ack_interrupt(int irq, void *dev) static int wcnss_init_pds(struct qcom_wcnss *wcnss, const char * const pd_names[WCNSS_MAX_PDS]) { + struct device *dev = wcnss->dev; int i, ret; + /* Handle single power domain */ + if (dev->pm_domain) { + wcnss->pds[0] = dev; + wcnss->num_pds = 1; + pm_runtime_enable(dev); + return 0; + } + for (i = 0; i < WCNSS_MAX_PDS; i++) { if (!pd_names[i]) break; @@ -418,8 +427,15 @@ static int wcnss_init_pds(struct qcom_wcnss *wcnss, static void wcnss_release_pds(struct qcom_wcnss *wcnss) { + struct device *dev = wcnss->dev; int i; + /* Handle single power domain */ + if (wcnss->num_pds == 1 && dev->pm_domain) { + pm_runtime_disable(dev); + return; + } + for (i = 0; i < wcnss->num_pds; i++) dev_pm_domain_detach(wcnss->pds[i], false); } @@ -437,10 +453,14 @@ static int wcnss_init_regulators(struct qcom_wcnss *wcnss, * the regulators for the power domains. For old device trees we need to * reserve extra space to manage them through the regulator interface. */ - if (wcnss->num_pds) - info += num_pd_vregs; - else + if (wcnss->num_pds) { + info += wcnss->num_pds; + /* Handle single power domain case */ + if (wcnss->num_pds < num_pd_vregs) + num_vregs += num_pd_vregs - wcnss->num_pds; + } else { num_vregs += num_pd_vregs; + } bulk = devm_kcalloc(wcnss->dev, num_vregs, sizeof(struct regulator_bulk_data), diff --git a/drivers/remoteproc/qcom_wcnss_iris.c b/drivers/remoteproc/qcom_wcnss_iris.c index dd36fd077911a..1e197f7734742 100644 --- a/drivers/remoteproc/qcom_wcnss_iris.c +++ b/drivers/remoteproc/qcom_wcnss_iris.c @@ -197,6 +197,7 @@ struct qcom_iris *qcom_iris_probe(struct device *parent, bool *use_48mhz_xo) err_device_del: device_del(&iris->dev); + put_device(&iris->dev); return ERR_PTR(ret); } @@ -204,4 +205,5 @@ struct qcom_iris *qcom_iris_probe(struct device *parent, bool *use_48mhz_xo) void qcom_iris_remove(struct qcom_iris *iris) { device_del(&iris->dev); + put_device(&iris->dev); } diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 78d27bff0e8ab..e232500ac1c47 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -1617,7 +1617,7 @@ static int rproc_attach(struct rproc *rproc) ret = rproc_set_rsc_table(rproc); if (ret) { dev_err(dev, "can't load resource table: %d\n", ret); - goto unprepare_device; + goto clean_up_resources; } /* reset max_notifyid */ @@ -1634,7 +1634,7 @@ static int rproc_attach(struct rproc *rproc) ret = rproc_handle_resources(rproc, rproc_loading_handlers); if (ret) { dev_err(dev, "Failed to process resources: %d\n", ret); - goto unprepare_device; + goto clean_up_resources; } /* Allocate carveout resources associated to rproc */ @@ -1653,9 +1653,9 @@ static int rproc_attach(struct rproc *rproc) clean_up_resources: rproc_resource_cleanup(rproc); -unprepare_device: /* release HW resources if needed */ rproc_unprepare_device(rproc); + kfree(rproc->clean_table); disable_iommu: rproc_disable_iommu(rproc); return ret; diff --git a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c index 5491b1b17ca36..37005640c784c 100644 --- a/drivers/remoteproc/ti_k3_r5_remoteproc.c +++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c @@ -194,10 +194,6 @@ static void k3_r5_rproc_mbox_callback(struct mbox_client *client, void *data) const char *name = kproc->rproc->name; u32 msg = omap_mbox_message(data); - /* Do not forward message from a detached core */ - if (kproc->rproc->state == RPROC_DETACHED) - return; - dev_dbg(dev, "mbox msg: 0x%x\n", msg); switch (msg) { @@ -233,10 +229,6 @@ static void k3_r5_rproc_kick(struct rproc *rproc, int vqid) mbox_msg_t msg = (mbox_msg_t)vqid; int ret; - /* Do not forward message to a detached core */ - if (kproc->rproc->state == RPROC_DETACHED) - return; - /* send the index of the triggered virtqueue in the mailbox payload */ ret = mbox_send_message(kproc->mbox, (void *)msg); if (ret < 0) diff --git a/drivers/remoteproc/wkup_m3_rproc.c b/drivers/remoteproc/wkup_m3_rproc.c index 36a55f7ffa64d..c39bd2bf2c1e7 100644 --- a/drivers/remoteproc/wkup_m3_rproc.c +++ b/drivers/remoteproc/wkup_m3_rproc.c @@ -148,7 +148,9 @@ static int wkup_m3_rproc_probe(struct platform_device *pdev) return -ENODEV; } - pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(dev); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to enable runtime PM\n"); ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "pm_runtime_get_sync() failed\n"); @@ -219,7 +221,6 @@ static int wkup_m3_rproc_probe(struct platform_device *pdev) rproc_free(rproc); err: pm_runtime_put_noidle(dev); - pm_runtime_disable(dev); return ret; } @@ -230,7 +231,6 @@ static void wkup_m3_rproc_remove(struct platform_device *pdev) rproc_del(rproc); rproc_free(rproc); pm_runtime_put_sync(&pdev->dev); - pm_runtime_disable(&pdev->dev); } #ifdef CONFIG_PM diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index ccd59ddd76100..9f25eb3aec256 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -51,8 +51,8 @@ config RESET_BERLIN config RESET_BRCMSTB tristate "Broadcom STB reset controller" - depends on ARCH_BRCMSTB || COMPILE_TEST - default ARCH_BRCMSTB + depends on ARCH_BRCMSTB || ARCH_BCM2835 || COMPILE_TEST + default ARCH_BRCMSTB || ARCH_BCM2835 help This enables the reset controller driver for Broadcom STB SoCs using a SUN_TOP_CTRL_SW_INIT style controller. @@ -60,11 +60,11 @@ config RESET_BRCMSTB config RESET_BRCMSTB_RESCAL tristate "Broadcom STB RESCAL reset controller" depends on HAS_IOMEM - depends on ARCH_BRCMSTB || COMPILE_TEST - default ARCH_BRCMSTB + depends on ARCH_BRCMSTB || ARCH_BCM2835 || COMPILE_TEST + default ARCH_BRCMSTB || ARCH_BCM2835 help This enables the RESCAL reset controller for SATA, PCIe0, or PCIe1 on - BCM7216. + BCM7216 or the BCM2712. config RESET_HSDK bool "Synopsys HSDK Reset Driver" diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c index 43f601c84b4fc..79d35ab43729e 100644 --- a/drivers/rpmsg/qcom_smd.c +++ b/drivers/rpmsg/qcom_smd.c @@ -746,7 +746,7 @@ static int __qcom_smd_send(struct qcom_smd_channel *channel, const void *data, __le32 hdr[5] = { cpu_to_le32(len), }; int tlen = sizeof(hdr) + len; unsigned long flags; - int ret; + int ret = 0; /* Word aligned channels only accept word size aligned data */ if (channel->info_word && len % 4) diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c index edfd942f8c549..af10850fbac2e 100644 --- a/drivers/rtc/class.c +++ b/drivers/rtc/class.c @@ -323,7 +323,7 @@ static void rtc_device_get_offset(struct rtc_device *rtc) * * Otherwise the offset seconds should be 0. */ - if (rtc->start_secs > rtc->range_max || + if ((rtc->start_secs >= 0 && rtc->start_secs > rtc->range_max) || rtc->start_secs + range_secs - 1 < rtc->range_min) rtc->offset_secs = rtc->start_secs - rtc->range_min; else if (rtc->start_secs > rtc->range_min) diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c index 4a7c41a6c21e7..93baffe110c00 100644 --- a/drivers/rtc/interface.c +++ b/drivers/rtc/interface.c @@ -443,6 +443,29 @@ static int __rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) else err = rtc->ops->set_alarm(rtc->dev.parent, alarm); + /* + * Check for potential race described above. If the waiting for next + * second, and the second just ticked since the check above, either + * + * 1) It ticked after the alarm was set, and an alarm irq should be + * generated. + * + * 2) It ticked before the alarm was set, and alarm irq most likely will + * not be generated. + * + * While we cannot easily check for which of these two scenarios we + * are in, we can return -ETIME to signal that the timer has already + * expired, which is true in both cases. + */ + if ((scheduled - now) <= 1) { + err = __rtc_read_time(rtc, &tm); + if (err) + return err; + now = rtc_tm_to_time64(&tm); + if (scheduled <= now) + return -ETIME; + } + trace_rtc_set_alarm(rtc_tm_to_time64(&alarm->time), err); return err; } @@ -594,6 +617,10 @@ int rtc_update_irq_enable(struct rtc_device *rtc, unsigned int enabled) rtc->uie_rtctimer.node.expires = ktime_add(now, onesec); rtc->uie_rtctimer.period = ktime_set(1, 0); err = rtc_timer_enqueue(rtc, &rtc->uie_rtctimer); + if (!err && rtc->ops && rtc->ops->alarm_irq_enable) + err = rtc->ops->alarm_irq_enable(rtc->dev.parent, 1); + if (err) + goto out; } else { rtc_timer_remove(rtc, &rtc->uie_rtctimer); } diff --git a/drivers/rtc/lib.c b/drivers/rtc/lib.c index fe361652727a3..13b5b1f204651 100644 --- a/drivers/rtc/lib.c +++ b/drivers/rtc/lib.c @@ -46,24 +46,38 @@ EXPORT_SYMBOL(rtc_year_days); * rtc_time64_to_tm - converts time64_t to rtc_time. * * @time: The number of seconds since 01-01-1970 00:00:00. - * (Must be positive.) + * Works for values since at least 1900 * @tm: Pointer to the struct rtc_time. */ void rtc_time64_to_tm(time64_t time, struct rtc_time *tm) { - unsigned int secs; - int days; + int days, secs; u64 u64tmp; u32 u32tmp, udays, century, day_of_century, year_of_century, year, day_of_year, month, day; bool is_Jan_or_Feb, is_leap_year; - /* time must be positive */ + /* + * Get days and seconds while preserving the sign to + * handle negative time values (dates before 1970-01-01) + */ days = div_s64_rem(time, 86400, &secs); + /* + * We need 0 <= secs < 86400 which isn't given for negative + * values of time. Fixup accordingly. + */ + if (secs < 0) { + days -= 1; + secs += 86400; + } + /* day of the week, 1970-01-01 was a Thursday */ tm->tm_wday = (days + 4) % 7; + /* Ensure tm_wday is always positive */ + if (tm->tm_wday < 0) + tm->tm_wday += 7; /* * The following algorithm is, basically, Proposition 6.3 of Neri @@ -93,7 +107,7 @@ void rtc_time64_to_tm(time64_t time, struct rtc_time *tm) * thus, is slightly different from [1]. */ - udays = ((u32) days) + 719468; + udays = days + 719468; u32tmp = 4 * udays + 3; century = u32tmp / 146097; diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index 5849d2970bba4..095de4e0e4f38 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -697,8 +697,12 @@ static irqreturn_t cmos_interrupt(int irq, void *p) { u8 irqstat; u8 rtc_control; + unsigned long flags; - spin_lock(&rtc_lock); + /* We cannot use spin_lock() here, as cmos_interrupt() is also called + * in a non-irq context. + */ + spin_lock_irqsave(&rtc_lock, flags); /* When the HPET interrupt handler calls us, the interrupt * status is passed as arg1 instead of the irq number. But @@ -732,7 +736,7 @@ static irqreturn_t cmos_interrupt(int irq, void *p) hpet_mask_rtc_irq_bit(RTC_AIE); CMOS_READ(RTC_INTR_FLAGS); } - spin_unlock(&rtc_lock); + spin_unlock_irqrestore(&rtc_lock, flags); if (is_intr(irqstat)) { rtc_update_irq(p, 1, irqstat); @@ -1300,9 +1304,7 @@ static void cmos_check_wkalrm(struct device *dev) * ACK the rtc irq here */ if (t_now >= cmos->alarm_expires && cmos_use_acpi_alarm()) { - local_irq_disable(); cmos_interrupt(0, (void *)cmos->rtc); - local_irq_enable(); return; } diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c index 506b7d1c23970..74aad2b12460c 100644 --- a/drivers/rtc/rtc-ds1307.c +++ b/drivers/rtc/rtc-ds1307.c @@ -274,6 +274,13 @@ static int ds1307_get_time(struct device *dev, struct rtc_time *t) if (tmp & DS1340_BIT_OSF) return -EINVAL; break; + case ds_1341: + ret = regmap_read(ds1307->regmap, DS1337_REG_STATUS, &tmp); + if (ret) + return ret; + if (tmp & DS1337_BIT_OSF) + return -EINVAL; + break; case ds_1388: ret = regmap_read(ds1307->regmap, DS1388_REG_FLAG, &tmp); if (ret) @@ -372,6 +379,10 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t) regmap_update_bits(ds1307->regmap, DS1340_REG_FLAG, DS1340_BIT_OSF, 0); break; + case ds_1341: + regmap_update_bits(ds1307->regmap, DS1337_REG_STATUS, + DS1337_BIT_OSF, 0); + break; case ds_1388: regmap_update_bits(ds1307->regmap, DS1388_REG_FLAG, DS1388_BIT_OSF, 0); @@ -1461,7 +1472,7 @@ static long ds3231_clk_sqw_round_rate(struct clk_hw *hw, unsigned long rate, return ds3231_clk_sqw_rates[i]; } - return 0; + return ds3231_clk_sqw_rates[ARRAY_SIZE(ds3231_clk_sqw_rates) - 1]; } static int ds3231_clk_sqw_set_rate(struct clk_hw *hw, unsigned long rate, @@ -1802,18 +1813,14 @@ static int ds1307_probe(struct i2c_client *client) * For some variants, be sure alarms can trigger when we're * running on Vbackup (BBSQI/BBSQW) */ - if (want_irq || ds1307_can_wakeup_device) { + if (want_irq || ds1307_can_wakeup_device) regs[0] |= DS1337_BIT_INTCN | chip->bbsqi_bit; - regs[0] &= ~(DS1337_BIT_A2IE | DS1337_BIT_A1IE); - } regmap_write(ds1307->regmap, DS1337_REG_CONTROL, regs[0]); - /* oscillator fault? clear flag, and warn */ + /* oscillator fault? warn */ if (regs[1] & DS1337_BIT_OSF) { - regmap_write(ds1307->regmap, DS1337_REG_STATUS, - regs[1] & ~DS1337_BIT_OSF); dev_warn(ds1307->dev, "SET TIME!\n"); } break; diff --git a/drivers/rtc/rtc-hym8563.c b/drivers/rtc/rtc-hym8563.c index b018535c842b3..b873918042e33 100644 --- a/drivers/rtc/rtc-hym8563.c +++ b/drivers/rtc/rtc-hym8563.c @@ -294,7 +294,7 @@ static long hym8563_clkout_round_rate(struct clk_hw *hw, unsigned long rate, if (clkout_rates[i] <= rate) return clkout_rates[i]; - return 0; + return clkout_rates[0]; } static int hym8563_clkout_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/rtc/rtc-loongson.c b/drivers/rtc/rtc-loongson.c index 90e9d97a86b48..c9d5b91a6544d 100644 --- a/drivers/rtc/rtc-loongson.c +++ b/drivers/rtc/rtc-loongson.c @@ -129,6 +129,14 @@ static u32 loongson_rtc_handler(void *id) { struct loongson_rtc_priv *priv = (struct loongson_rtc_priv *)id; + rtc_update_irq(priv->rtcdev, 1, RTC_AF | RTC_IRQF); + + /* + * The TOY_MATCH0_REG should be cleared 0 here, + * otherwise the interrupt cannot be cleared. + */ + regmap_write(priv->regmap, TOY_MATCH0_REG, 0); + spin_lock(&priv->lock); /* Disable RTC alarm wakeup and interrupt */ writel(readl(priv->pm_base + PM1_EN_REG) & ~RTC_EN, diff --git a/drivers/rtc/rtc-nct3018y.c b/drivers/rtc/rtc-nct3018y.c index c4533c0f53896..5e7a8c7ad58c5 100644 --- a/drivers/rtc/rtc-nct3018y.c +++ b/drivers/rtc/rtc-nct3018y.c @@ -342,7 +342,7 @@ static long nct3018y_clkout_round_rate(struct clk_hw *hw, unsigned long rate, if (clkout_rates[i] <= rate) return clkout_rates[i]; - return 0; + return clkout_rates[0]; } static int nct3018y_clkout_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/rtc/rtc-optee.c b/drivers/rtc/rtc-optee.c index 9f8b5d4a8f6b6..6b77c122fdc10 100644 --- a/drivers/rtc/rtc-optee.c +++ b/drivers/rtc/rtc-optee.c @@ -320,6 +320,7 @@ static int optee_rtc_remove(struct device *dev) { struct optee_rtc *priv = dev_get_drvdata(dev); + tee_shm_free(priv->shm); tee_client_close_session(priv->ctx, priv->session_id); tee_client_close_context(priv->ctx); diff --git a/drivers/rtc/rtc-pcf2127.c b/drivers/rtc/rtc-pcf2127.c index 9c04c4e1a49c3..05a54f4d4d9a6 100644 --- a/drivers/rtc/rtc-pcf2127.c +++ b/drivers/rtc/rtc-pcf2127.c @@ -41,6 +41,7 @@ #define PCF2127_BIT_CTRL2_AF BIT(4) #define PCF2127_BIT_CTRL2_TSF2 BIT(5) #define PCF2127_BIT_CTRL2_WDTF BIT(6) +#define PCF2127_BIT_CTRL2_MSF BIT(7) /* Control register 3 */ #define PCF2127_REG_CTRL3 0x02 #define PCF2127_BIT_CTRL3_BLIE BIT(0) @@ -94,7 +95,8 @@ #define PCF2127_CTRL2_IRQ_MASK ( \ PCF2127_BIT_CTRL2_AF | \ PCF2127_BIT_CTRL2_WDTF | \ - PCF2127_BIT_CTRL2_TSF2) + PCF2127_BIT_CTRL2_TSF2 | \ + PCF2127_BIT_CTRL2_MSF) #define PCF2127_MAX_TS_SUPPORTED 4 @@ -526,6 +528,21 @@ static int pcf2127_watchdog_init(struct device *dev, struct pcf2127 *pcf2127) set_bit(WDOG_HW_RUNNING, &pcf2127->wdd.status); } + /* + * When using interrupt pin (INT A) as watchdog output, only allow + * watchdog interrupt (PCF2131_BIT_INT_WD_CD) and disable (mask) all + * other interrupts. + */ + if (pcf2127->cfg->type == PCF2131) { + ret = regmap_write(pcf2127->regmap, + PCF2131_REG_INT_A_MASK1, + PCF2131_BIT_INT_BLIE | + PCF2131_BIT_INT_BIE | + PCF2131_BIT_INT_AIE | + PCF2131_BIT_INT_SI | + PCF2131_BIT_INT_MI); + } + return devm_watchdog_register_device(dev, &pcf2127->wdd); } @@ -1456,7 +1473,12 @@ static int pcf2127_spi_probe(struct spi_device *spi) variant = &pcf21xx_cfg[type]; } - config.max_register = variant->max_register, + if (variant->type == PCF2131) { + config.read_flag_mask = 0x0; + config.write_flag_mask = 0x0; + } + + config.max_register = variant->max_register; regmap = devm_regmap_init_spi(spi, &config); if (IS_ERR(regmap)) { diff --git a/drivers/rtc/rtc-pcf85063.c b/drivers/rtc/rtc-pcf85063.c index 73848f764559b..2b4921c23467d 100644 --- a/drivers/rtc/rtc-pcf85063.c +++ b/drivers/rtc/rtc-pcf85063.c @@ -410,7 +410,7 @@ static long pcf85063_clkout_round_rate(struct clk_hw *hw, unsigned long rate, if (clkout_rates[i] <= rate) return clkout_rates[i]; - return 0; + return clkout_rates[0]; } static int pcf85063_clkout_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index ea82b89d8929f..61be91c25cf18 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -386,7 +386,7 @@ static long pcf8563_clkout_round_rate(struct clk_hw *hw, unsigned long rate, if (clkout_rates[i] <= rate) return clkout_rates[i]; - return 0; + return clkout_rates[0]; } static int pcf8563_clkout_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/rtc/rtc-rv3028.c b/drivers/rtc/rtc-rv3028.c index 2f001c59c61d5..86b7f821e937b 100644 --- a/drivers/rtc/rtc-rv3028.c +++ b/drivers/rtc/rtc-rv3028.c @@ -738,7 +738,7 @@ static long rv3028_clkout_round_rate(struct clk_hw *hw, unsigned long rate, if (clkout_rates[i] <= rate) return clkout_rates[i]; - return 0; + return clkout_rates[0]; } static int rv3028_clkout_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/rtc/rtc-rv3032.c b/drivers/rtc/rtc-rv3032.c index 35b2e36b426a0..cb01038a2e27f 100644 --- a/drivers/rtc/rtc-rv3032.c +++ b/drivers/rtc/rtc-rv3032.c @@ -69,7 +69,7 @@ #define RV3032_CLKOUT2_FD_MSK GENMASK(6, 5) #define RV3032_CLKOUT2_OS BIT(7) -#define RV3032_CTRL1_EERD BIT(3) +#define RV3032_CTRL1_EERD BIT(2) #define RV3032_CTRL1_WADA BIT(5) #define RV3032_CTRL2_STOP BIT(0) diff --git a/drivers/rtc/rtc-rx8025.c b/drivers/rtc/rtc-rx8025.c index aabe62c283a15..7e9f7cb90c288 100644 --- a/drivers/rtc/rtc-rx8025.c +++ b/drivers/rtc/rtc-rx8025.c @@ -316,7 +316,7 @@ static int rx8025_init_client(struct i2c_client *client) return hour_reg; rx8025->is_24 = (hour_reg & RX8035_BIT_HOUR_1224); } else { - rx8025->is_24 = (ctrl[1] & RX8025_BIT_CTRL1_1224); + rx8025->is_24 = (ctrl[0] & RX8025_BIT_CTRL1_1224); } out: return err; diff --git a/drivers/rtc/rtc-sh.c b/drivers/rtc/rtc-sh.c index cd146b5741431..341b1b776e1a3 100644 --- a/drivers/rtc/rtc-sh.c +++ b/drivers/rtc/rtc-sh.c @@ -485,9 +485,15 @@ static int __init sh_rtc_probe(struct platform_device *pdev) return -ENOENT; } - rtc->periodic_irq = ret; - rtc->carry_irq = platform_get_irq(pdev, 1); - rtc->alarm_irq = platform_get_irq(pdev, 2); + if (!pdev->dev.of_node) { + rtc->periodic_irq = ret; + rtc->carry_irq = platform_get_irq(pdev, 1); + rtc->alarm_irq = platform_get_irq(pdev, 2); + } else { + rtc->alarm_irq = ret; + rtc->periodic_irq = platform_get_irq(pdev, 1); + rtc->carry_irq = platform_get_irq(pdev, 2); + } res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (!res) diff --git a/drivers/rtc/rtc-x1205.c b/drivers/rtc/rtc-x1205.c index 807f953ae0aed..b7a7ea036f7ad 100644 --- a/drivers/rtc/rtc-x1205.c +++ b/drivers/rtc/rtc-x1205.c @@ -669,7 +669,7 @@ static const struct i2c_device_id x1205_id[] = { MODULE_DEVICE_TABLE(i2c, x1205_id); static const struct of_device_id x1205_dt_ids[] = { - { .compatible = "xircom,x1205", }, + { .compatible = "xicor,x1205", }, {}, }; MODULE_DEVICE_TABLE(of, x1205_dt_ids); diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 6fa0fb35e5210..3409822dfbb41 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -76,6 +76,13 @@ unsigned long sclp_console_full; /* The currently active SCLP command word. */ static sclp_cmdw_t active_cmd; +static inline struct sccb_header *sclpint_to_sccb(u32 sccb_int) +{ + if (sccb_int) + return __va(sccb_int); + return NULL; +} + static inline void sclp_trace(int prio, char *id, u32 a, u64 b, bool err) { struct sclp_trace_entry e; @@ -620,7 +627,7 @@ __sclp_find_req(u32 sccb) static bool ok_response(u32 sccb_int, sclp_cmdw_t cmd) { - struct sccb_header *sccb = (struct sccb_header *)__va(sccb_int); + struct sccb_header *sccb = sclpint_to_sccb(sccb_int); struct evbuf_header *evbuf; u16 response; @@ -659,7 +666,7 @@ static void sclp_interrupt_handler(struct ext_code ext_code, /* INT: Interrupt received (a=intparm, b=cmd) */ sclp_trace_sccb(0, "INT", param32, active_cmd, active_cmd, - (struct sccb_header *)__va(finished_sccb), + sclpint_to_sccb(finished_sccb), !ok_response(finished_sccb, active_cmd)); if (finished_sccb) { diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 6b374026cd4f4..3942aa4921769 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -1318,23 +1318,34 @@ void ccw_device_schedule_recovery(void) spin_unlock_irqrestore(&recovery_lock, flags); } -static int purge_fn(struct device *dev, void *data) +static int purge_fn(struct subchannel *sch, void *data) { - struct ccw_device *cdev = to_ccwdev(dev); - struct ccw_dev_id *id = &cdev->private->dev_id; - struct subchannel *sch = to_subchannel(cdev->dev.parent); + struct ccw_device *cdev; - spin_lock_irq(cdev->ccwlock); - if (is_blacklisted(id->ssid, id->devno) && - (cdev->private->state == DEV_STATE_OFFLINE) && - (atomic_cmpxchg(&cdev->private->onoff, 0, 1) == 0)) { - CIO_MSG_EVENT(3, "ccw: purging 0.%x.%04x\n", id->ssid, - id->devno); + spin_lock_irq(sch->lock); + if (sch->st != SUBCHANNEL_TYPE_IO || !sch->schib.pmcw.dnv) + goto unlock; + + if (!is_blacklisted(sch->schid.ssid, sch->schib.pmcw.dev)) + goto unlock; + + cdev = sch_get_cdev(sch); + if (cdev) { + if (cdev->private->state != DEV_STATE_OFFLINE) + goto unlock; + + if (atomic_cmpxchg(&cdev->private->onoff, 0, 1) != 0) + goto unlock; ccw_device_sched_todo(cdev, CDEV_TODO_UNREG); - css_sched_sch_todo(sch, SCH_TODO_UNREG); atomic_set(&cdev->private->onoff, 0); } - spin_unlock_irq(cdev->ccwlock); + + css_sched_sch_todo(sch, SCH_TODO_UNREG); + CIO_MSG_EVENT(3, "ccw: purging 0.%x.%04x%s\n", sch->schid.ssid, + sch->schib.pmcw.dev, cdev ? "" : " (no cdev)"); + +unlock: + spin_unlock_irq(sch->lock); /* Abort loop in case of pending signal. */ if (signal_pending(current)) return -EINTR; @@ -1350,7 +1361,7 @@ static int purge_fn(struct device *dev, void *data) int ccw_purge_blacklisted(void) { CIO_MSG_EVENT(2, "ccw: purging blacklisted devices\n"); - bus_for_each_dev(&ccw_bus_type, NULL, NULL, purge_fn); + for_each_subchannel_staged(purge_fn, NULL, NULL); return 0; } diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 70fcb5c40cfe3..21fc6151af79f 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -1333,7 +1333,7 @@ static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns) if (!uapqns || nr_apqns == 0) return NULL; - return memdup_user(uapqns, nr_apqns * sizeof(struct pkey_apqn)); + return memdup_array_user(uapqns, nr_apqns, sizeof(struct pkey_apqn)); } static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index d6ea2fd4c2a02..d4151f519e8b2 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -834,48 +834,66 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) vfio_put_device(&matrix_mdev->vdev); } -#define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \ - "already assigned to %s" +#define MDEV_SHARING_ERR "Userspace may not assign queue %02lx.%04lx to mdev: already assigned to %s" -static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev, - unsigned long *apm, - unsigned long *aqm) +#define MDEV_IN_USE_ERR "Can not reserve queue %02lx.%04lx for host driver: in use by mdev" + +static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *assignee, + struct ap_matrix_mdev *assigned_to, + unsigned long *apm, unsigned long *aqm) { unsigned long apid, apqi; - const struct device *dev = mdev_dev(matrix_mdev->mdev); - const char *mdev_name = dev_name(dev); - for_each_set_bit_inv(apid, apm, AP_DEVICES) + for_each_set_bit_inv(apid, apm, AP_DEVICES) { + for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { + dev_warn(mdev_dev(assignee->mdev), MDEV_SHARING_ERR, + apid, apqi, dev_name(mdev_dev(assigned_to->mdev))); + } + } +} + +static void vfio_ap_mdev_log_in_use_err(struct ap_matrix_mdev *assignee, + unsigned long *apm, unsigned long *aqm) +{ + unsigned long apid, apqi; + + for_each_set_bit_inv(apid, apm, AP_DEVICES) { for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) - dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name); + dev_warn(mdev_dev(assignee->mdev), MDEV_IN_USE_ERR, apid, apqi); + } } /** * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs * + * @assignee: the matrix mdev to which @mdev_apm and @mdev_aqm are being + * assigned; or, NULL if this function was called by the AP bus + * driver in_use callback to verify none of the APQNs being reserved + * for the host device driver are in use by a vfio_ap mediated device * @mdev_apm: mask indicating the APIDs of the APQNs to be verified * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified * - * Verifies that each APQN derived from the Cartesian product of a bitmap of - * AP adapter IDs and AP queue indexes is not configured for any matrix - * mediated device. AP queue sharing is not allowed. + * Verifies that each APQN derived from the Cartesian product of APIDs + * represented by the bits set in @mdev_apm and the APQIs of the bits set in + * @mdev_aqm is not assigned to a mediated device other than the mdev to which + * the APQN is being assigned (@assignee). AP queue sharing is not allowed. * * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE. */ -static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, +static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *assignee, + unsigned long *mdev_apm, unsigned long *mdev_aqm) { - struct ap_matrix_mdev *matrix_mdev; + struct ap_matrix_mdev *assigned_to; DECLARE_BITMAP(apm, AP_DEVICES); DECLARE_BITMAP(aqm, AP_DOMAINS); - list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + list_for_each_entry(assigned_to, &matrix_dev->mdev_list, node) { /* - * If the input apm and aqm are fields of the matrix_mdev - * object, then move on to the next matrix_mdev. + * If the mdev to which the mdev_apm and mdev_aqm is being + * assigned is the same as the mdev being verified */ - if (mdev_apm == matrix_mdev->matrix.apm && - mdev_aqm == matrix_mdev->matrix.aqm) + if (assignee == assigned_to) continue; memset(apm, 0, sizeof(apm)); @@ -885,15 +903,16 @@ static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, * We work on full longs, as we can only exclude the leftover * bits in non-inverse order. The leftover is all zeros. */ - if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm, - AP_DEVICES)) + if (!bitmap_and(apm, mdev_apm, assigned_to->matrix.apm, AP_DEVICES)) continue; - if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm, - AP_DOMAINS)) + if (!bitmap_and(aqm, mdev_aqm, assigned_to->matrix.aqm, AP_DOMAINS)) continue; - vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm); + if (assignee) + vfio_ap_mdev_log_sharing_err(assignee, assigned_to, apm, aqm); + else + vfio_ap_mdev_log_in_use_err(assigned_to, apm, aqm); return -EADDRINUSE; } @@ -922,7 +941,8 @@ static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev) matrix_mdev->matrix.aqm)) return -EADDRNOTAVAIL; - return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm, + return vfio_ap_mdev_verify_no_sharing(matrix_mdev, + matrix_mdev->matrix.apm, matrix_mdev->matrix.aqm); } @@ -2271,7 +2291,7 @@ int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); - ret = vfio_ap_mdev_verify_no_sharing(apm, aqm); + ret = vfio_ap_mdev_verify_no_sharing(NULL, apm, aqm); mutex_unlock(&matrix_dev->mdevs_lock); mutex_unlock(&matrix_dev->guests_lock); diff --git a/drivers/s390/net/ctcm_mpc.c b/drivers/s390/net/ctcm_mpc.c index 7a2f34a5e0e09..f8b3e3f7b8ce0 100644 --- a/drivers/s390/net/ctcm_mpc.c +++ b/drivers/s390/net/ctcm_mpc.c @@ -700,7 +700,6 @@ static void mpc_rcvd_sweep_req(struct mpcg_info *mpcginfo) grp->sweep_req_pend_num--; ctcmpc_send_sweep_resp(ch); - kfree(mpcginfo); return; } diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c index af0d90beba638..76ba36c83e522 100644 --- a/drivers/s390/net/ism_drv.c +++ b/drivers/s390/net/ism_drv.c @@ -130,6 +130,7 @@ static int ism_cmd(struct ism_dev *ism, void *cmd) struct ism_req_hdr *req = cmd; struct ism_resp_hdr *resp = cmd; + spin_lock(&ism->cmd_lock); __ism_write_cmd(ism, req + 1, sizeof(*req), req->len - sizeof(*req)); __ism_write_cmd(ism, req, 0, sizeof(*req)); @@ -143,6 +144,7 @@ static int ism_cmd(struct ism_dev *ism, void *cmd) } __ism_read_cmd(ism, resp + 1, sizeof(*resp), resp->len - sizeof(*resp)); out: + spin_unlock(&ism->cmd_lock); return resp->ret; } @@ -630,6 +632,7 @@ static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM; spin_lock_init(&ism->lock); + spin_lock_init(&ism->cmd_lock); dev_set_drvdata(&pdev->dev, ism); ism->pdev = pdev; ism->dev.parent = &pdev->dev; diff --git a/drivers/s390/scsi/zfcp_sysfs.c b/drivers/s390/scsi/zfcp_sysfs.c index cb67fa80fb12c..a95da6768f665 100644 --- a/drivers/s390/scsi/zfcp_sysfs.c +++ b/drivers/s390/scsi/zfcp_sysfs.c @@ -450,6 +450,8 @@ static ssize_t zfcp_sysfs_unit_add_store(struct device *dev, if (kstrtoull(buf, 0, (unsigned long long *) &fcp_lun)) return -EINVAL; + flush_work(&port->rport_work); + retval = zfcp_unit_add(port, fcp_lun); if (retval) return retval; diff --git a/drivers/scsi/aacraid/comminit.c b/drivers/scsi/aacraid/comminit.c index 0f64b02443037..31b95e6c96c5f 100644 --- a/drivers/scsi/aacraid/comminit.c +++ b/drivers/scsi/aacraid/comminit.c @@ -481,8 +481,7 @@ void aac_define_int_mode(struct aac_dev *dev) pci_find_capability(dev->pdev, PCI_CAP_ID_MSIX)) { min_msix = 2; i = pci_alloc_irq_vectors(dev->pdev, - min_msix, msi_count, - PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); + min_msix, msi_count, PCI_IRQ_MSIX); if (i > 0) { dev->msi_enabled = 1; msi_count = i; diff --git a/drivers/scsi/bfa/bfad_im.c b/drivers/scsi/bfa/bfad_im.c index a9d3d8562d3c1..0ec76d1cb6fbe 100644 --- a/drivers/scsi/bfa/bfad_im.c +++ b/drivers/scsi/bfa/bfad_im.c @@ -706,6 +706,7 @@ bfad_im_probe(struct bfad_s *bfad) if (bfad_thread_workq(bfad) != BFA_STATUS_OK) { kfree(im); + bfad->im = NULL; return BFA_STATUS_FAILED; } diff --git a/drivers/scsi/elx/efct/efct_hw.c b/drivers/scsi/elx/efct/efct_hw.c index 5a5525054d71c..5b079b8b7a082 100644 --- a/drivers/scsi/elx/efct/efct_hw.c +++ b/drivers/scsi/elx/efct/efct_hw.c @@ -1120,7 +1120,7 @@ int efct_hw_parse_filter(struct efct_hw *hw, void *value) { int rc = 0; - char *p = NULL; + char *p = NULL, *pp = NULL; char *token; u32 idx = 0; @@ -1132,6 +1132,7 @@ efct_hw_parse_filter(struct efct_hw *hw, void *value) efc_log_err(hw->os, "p is NULL\n"); return -ENOMEM; } + pp = p; idx = 0; while ((token = strsep(&p, ",")) && *token) { @@ -1144,7 +1145,7 @@ efct_hw_parse_filter(struct efct_hw *hw, void *value) if (idx == ARRAY_SIZE(hw->config.filter_def)) break; } - kfree(p); + kfree(pp); return rc; } diff --git a/drivers/scsi/elx/efct/efct_lio.c b/drivers/scsi/elx/efct/efct_lio.c index a982b9cf98701..49d7fe150684b 100644 --- a/drivers/scsi/elx/efct/efct_lio.c +++ b/drivers/scsi/elx/efct/efct_lio.c @@ -382,7 +382,7 @@ efct_lio_sg_unmap(struct efct_io *io) return; dma_unmap_sg(&io->efct->pci->dev, cmd->t_data_sg, - ocp->seg_map_cnt, cmd->data_direction); + cmd->t_data_nents, cmd->data_direction); ocp->seg_map_cnt = 0; } diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index 7e64661d215bd..3ad58250bf6b2 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -1844,33 +1844,14 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device) } hisi_sas_dereg_device(hisi_hba, device); - rc = hisi_sas_debug_I_T_nexus_reset(device); - if (rc == TMF_RESP_FUNC_COMPLETE && dev_is_sata(device)) { - struct sas_phy *local_phy; - + if (dev_is_sata(device)) { rc = hisi_sas_softreset_ata_disk(device); - switch (rc) { - case -ECOMM: - rc = -ENODEV; - break; - case TMF_RESP_FUNC_FAILED: - case -EMSGSIZE: - case -EIO: - local_phy = sas_get_local_phy(device); - rc = sas_phy_enable(local_phy, 0); - if (!rc) { - local_phy->enabled = 0; - dev_err(dev, "Disabled local phy of ATA disk %016llx due to softreset fail (%d)\n", - SAS_ADDR(device->sas_addr), rc); - rc = -ENODEV; - } - sas_put_local_phy(local_phy); - break; - default: - break; - } + if (rc == TMF_RESP_FUNC_FAILED) + dev_err(dev, "ata disk %016llx reset (%d)\n", + SAS_ADDR(device->sas_addr), rc); } + rc = hisi_sas_debug_I_T_nexus_reset(device); if ((rc == TMF_RESP_FUNC_COMPLETE) || (rc == -ENODEV)) hisi_sas_release_task(hisi_hba, device); diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index 445f4a220df3e..f274812aee1e8 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -602,8 +602,9 @@ int scsi_host_busy(struct Scsi_Host *shost) { int cnt = 0; - blk_mq_tagset_busy_iter(&shost->tag_set, - scsi_host_check_in_flight, &cnt); + if (shost->tag_set.ops) + blk_mq_tagset_busy_iter(&shost->tag_set, + scsi_host_check_in_flight, &cnt); return cnt; } EXPORT_SYMBOL(scsi_host_busy); diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index 49c57a9c110b5..42a89ab33e1c7 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -6528,18 +6528,21 @@ static int hpsa_big_passthru_ioctl(struct ctlr_info *h, while (left) { sz = (left > ioc->malloc_size) ? ioc->malloc_size : left; buff_size[sg_used] = sz; - buff[sg_used] = kmalloc(sz, GFP_KERNEL); - if (buff[sg_used] == NULL) { - status = -ENOMEM; - goto cleanup1; - } + if (ioc->Request.Type.Direction & XFER_WRITE) { - if (copy_from_user(buff[sg_used], data_ptr, sz)) { - status = -EFAULT; + buff[sg_used] = memdup_user(data_ptr, sz); + if (IS_ERR(buff[sg_used])) { + status = PTR_ERR(buff[sg_used]); goto cleanup1; } - } else - memset(buff[sg_used], 0, sz); + } else { + buff[sg_used] = kzalloc(sz, GFP_KERNEL); + if (!buff[sg_used]) { + status = -ENOMEM; + goto cleanup1; + } + } + left -= sz; data_ptr += sz; sg_used++; diff --git a/drivers/scsi/ibmvscsi_tgt/libsrp.c b/drivers/scsi/ibmvscsi_tgt/libsrp.c index 8a0e28aec928e..0ecad398ed3db 100644 --- a/drivers/scsi/ibmvscsi_tgt/libsrp.c +++ b/drivers/scsi/ibmvscsi_tgt/libsrp.c @@ -184,7 +184,8 @@ static int srp_direct_data(struct ibmvscsis_cmd *cmd, struct srp_direct_buf *md, err = rdma_io(cmd, sg, nsg, md, 1, dir, len); if (dma_map) - dma_unmap_sg(iue->target->dev, sg, nsg, DMA_BIDIRECTIONAL); + dma_unmap_sg(iue->target->dev, sg, cmd->se_cmd.t_data_nents, + DMA_BIDIRECTIONAL); return err; } @@ -256,7 +257,8 @@ static int srp_indirect_data(struct ibmvscsis_cmd *cmd, struct srp_cmd *srp_cmd, err = rdma_io(cmd, sg, nsg, md, nmd, dir, len); if (dma_map) - dma_unmap_sg(iue->target->dev, sg, nsg, DMA_BIDIRECTIONAL); + dma_unmap_sg(iue->target->dev, sg, cmd->se_cmd.t_data_nents, + DMA_BIDIRECTIONAL); free_mem: if (token && dma_map) { diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c index 6277162a028bb..6eefa7e5f4cf6 100644 --- a/drivers/scsi/isci/init.c +++ b/drivers/scsi/isci/init.c @@ -65,11 +65,7 @@ #include "task.h" #include "probe_roms.h" -#define MAJ 1 -#define MIN 2 -#define BUILD 0 -#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \ - __stringify(BUILD) +#define DRV_VERSION "1.2.0" MODULE_VERSION(DRV_VERSION); diff --git a/drivers/scsi/isci/request.c b/drivers/scsi/isci/request.c index 7162a5029b37a..3841db665fe4b 100644 --- a/drivers/scsi/isci/request.c +++ b/drivers/scsi/isci/request.c @@ -2907,7 +2907,7 @@ static void isci_request_io_request_complete(struct isci_host *ihost, task->total_xfer_len, task->data_dir); else /* unmap the sgl dma addresses */ dma_unmap_sg(&ihost->pdev->dev, task->scatter, - request->num_sg_entries, task->data_dir); + task->num_scatter, task->data_dir); break; case SAS_PROTOCOL_SMP: { struct scatterlist *sg = &task->smp_task.smp_req; diff --git a/drivers/scsi/libfc/fc_encode.h b/drivers/scsi/libfc/fc_encode.h index 7dcac3b6baa7e..992250ca8b9fd 100644 --- a/drivers/scsi/libfc/fc_encode.h +++ b/drivers/scsi/libfc/fc_encode.h @@ -354,7 +354,7 @@ static inline int fc_ct_ms_fill(struct fc_lport *lport, put_unaligned_be16(len, &entry->len); snprintf((char *)&entry->value, FC_FDMI_HBA_ATTR_OSNAMEVERSION_LEN, - "%s v%s", + "%.62s v%.62s", init_utsname()->sysname, init_utsname()->release); diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c index 0fda8905eabd8..916c076484608 100644 --- a/drivers/scsi/libiscsi.c +++ b/drivers/scsi/libiscsi.c @@ -3184,7 +3184,8 @@ iscsi_conn_setup(struct iscsi_cls_session *cls_session, int dd_size, return NULL; conn = cls_conn->dd_data; - conn->dd_data = cls_conn->dd_data + sizeof(*conn); + if (dd_size) + conn->dd_data = cls_conn->dd_data + sizeof(*conn); conn->session = session; conn->cls_conn = cls_conn; conn->c_stage = ISCSI_CONN_INITIAL_STAGE; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 20662b4f339eb..6b6c964e8076e 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -6291,7 +6291,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) } phba->nvmeio_trc_on = 1; phba->nvmeio_trc_output_idx = 0; - phba->nvmeio_trc = NULL; } else { nvmeio_off: phba->nvmeio_trc_size = 0; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index 8d2e8d05bbc05..52b14671eaa94 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -44,6 +44,9 @@ /* hbqinfo output buffer size */ #define LPFC_HBQINFO_SIZE 8192 +/* hdwqinfo output buffer size */ +#define LPFC_HDWQINFO_SIZE 8192 + /* nvmestat output buffer size */ #define LPFC_NVMESTAT_SIZE 8192 #define LPFC_IOKTIME_SIZE 8192 diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index ebe84bb7bb3dd..2e9972a587810 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -12094,7 +12094,11 @@ lpfc_sli4_els_xri_aborted(struct lpfc_hba *phba, sglq_entry->state = SGL_FREED; spin_unlock_irqrestore(&phba->sli4_hba.sgl_list_lock, iflag); - + lpfc_printf_log(phba, KERN_INFO, LOG_ELS | LOG_SLI | + LOG_DISCOVERY | LOG_NODE, + "0732 ELS XRI ABORT on Node: ndlp=x%px " + "xri=x%x\n", + ndlp, xri); if (ndlp) { lpfc_set_rrq_active(phba, ndlp, sglq_entry->sli4_lxritag, diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 0ad8a10002ce3..ff44283338771 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -5101,7 +5101,7 @@ lpfc_check_sli_ndlp(struct lpfc_hba *phba, case CMD_GEN_REQUEST64_CR: if (iocb->ndlp == ndlp) return 1; - fallthrough; + break; case CMD_ELS_REQUEST64_CR: if (remote_id == ndlp->nlp_DID) return 1; @@ -5646,6 +5646,7 @@ static struct lpfc_nodelist * __lpfc_findnode_did(struct lpfc_vport *vport, uint32_t did) { struct lpfc_nodelist *ndlp; + struct lpfc_nodelist *np = NULL; uint32_t data1; list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) { @@ -5660,14 +5661,20 @@ __lpfc_findnode_did(struct lpfc_vport *vport, uint32_t did) ndlp, ndlp->nlp_DID, ndlp->nlp_flag, data1, ndlp->nlp_rpi, ndlp->active_rrqs_xri_bitmap); - return ndlp; + + /* Check for new or potentially stale node */ + if (ndlp->nlp_state != NLP_STE_UNUSED_NODE) + return ndlp; + np = ndlp; } } - /* FIND node did NOT FOUND */ - lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE, - "0932 FIND node did x%x NOT FOUND.\n", did); - return NULL; + if (!np) + /* FIND node did NOT FOUND */ + lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE, + "0932 FIND node did x%x NOT FOUND.\n", did); + + return np; } struct lpfc_nodelist * diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 424b39a8155cb..b0eac09de5ad5 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -3047,13 +3047,6 @@ lpfc_cleanup(struct lpfc_vport *vport) lpfc_vmid_vport_cleanup(vport); list_for_each_entry_safe(ndlp, next_ndlp, &vport->fc_nodes, nlp_listp) { - if (vport->port_type != LPFC_PHYSICAL_PORT && - ndlp->nlp_DID == Fabric_DID) { - /* Just free up ndlp with Fabric_DID for vports */ - lpfc_nlp_put(ndlp); - continue; - } - if (ndlp->nlp_DID == Fabric_Cntl_DID && ndlp->nlp_state == NLP_STE_UNUSED_NODE) { lpfc_nlp_put(ndlp); @@ -13180,6 +13173,7 @@ lpfc_sli4_enable_msi(struct lpfc_hba *phba) eqhdl = lpfc_get_eq_hdl(0); rc = pci_irq_vector(phba->pcidev, 0); if (rc < 0) { + free_irq(phba->pcidev->irq, phba); pci_free_irq_vectors(phba->pcidev); lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, "0496 MSI pci_irq_vec failed (%d)\n", rc); @@ -13260,6 +13254,7 @@ lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode) eqhdl = lpfc_get_eq_hdl(0); retval = pci_irq_vector(phba->pcidev, 0); if (retval < 0) { + free_irq(phba->pcidev->irq, phba); lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, "0502 INTR pci_irq_vec failed (%d)\n", retval); diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index d41fea53e41e9..502be061cc658 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -1243,7 +1243,7 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, struct lpfc_nvmet_tgtport *tgtp; struct lpfc_async_xchg_ctx *ctxp = container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); - struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; + struct rqb_dmabuf *nvmebuf; struct lpfc_hba *phba = ctxp->phba; unsigned long iflag; @@ -1251,13 +1251,18 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n", ctxp->oxid, ctxp->size, raw_smp_processor_id()); + spin_lock_irqsave(&ctxp->ctxlock, iflag); + nvmebuf = ctxp->rqb_buffer; if (!nvmebuf) { + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, "6425 Defer rcv: no buffer oxid x%x: " "flg %x ste %x\n", ctxp->oxid, ctxp->flag, ctxp->state); return; } + ctxp->rqb_buffer = NULL; + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); tgtp = phba->targetport->private; if (tgtp) @@ -1265,9 +1270,6 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, /* Free the nvmebuf since a new buffer already replaced it */ nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); - spin_lock_irqsave(&ctxp->ctxlock, iflag); - ctxp->rqb_buffer = NULL; - spin_unlock_irqrestore(&ctxp->ctxlock, iflag); } /** diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 070654cc92920..4734694f59a4c 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -390,6 +390,10 @@ lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; + /* may be called before queues established if hba_setup fails */ + if (!phba->sli4_hba.hdwq) + return; + spin_lock_irqsave(&phba->hbalock, iflag); for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { qp = &phba->sli4_hba.hdwq[idx]; @@ -5916,7 +5920,7 @@ lpfc_chk_tgt_mapped(struct lpfc_vport *vport, struct fc_rport *rport) /** * lpfc_reset_flush_io_context - * @vport: The virtual port (scsi_host) for the flush context - * @tgt_id: If aborting by Target contect - specifies the target id + * @tgt_id: If aborting by Target context - specifies the target id * @lun_id: If aborting by Lun context - specifies the lun id * @context: specifies the context level to flush at. * @@ -6090,8 +6094,14 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd) pnode->nlp_fcp_info &= ~NLP_FCP_2_DEVICE; spin_unlock_irqrestore(&pnode->lock, flags); } - lpfc_reset_flush_io_context(vport, tgt_id, lun_id, - LPFC_CTX_TGT); + status = lpfc_reset_flush_io_context(vport, tgt_id, lun_id, + LPFC_CTX_TGT); + if (status != SUCCESS) { + lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP, + "0726 Target Reset flush status x%x\n", + status); + return status; + } return FAST_IO_FAIL; } @@ -6187,7 +6197,7 @@ lpfc_host_reset_handler(struct scsi_cmnd *cmnd) int rc, ret = SUCCESS; lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP, - "3172 SCSI layer issued Host Reset Data:\n"); + "3172 SCSI layer issued Host Reset\n"); lpfc_offline_prep(phba, LPFC_MBX_WAIT); lpfc_offline(phba); diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 4a9fa00eeb798..4cf935b7223af 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -6014,9 +6014,9 @@ lpfc_sli4_get_ctl_attr(struct lpfc_hba *phba) phba->sli4_hba.flash_id = bf_get(lpfc_cntl_attr_flash_id, cntl_attr); phba->sli4_hba.asic_rev = bf_get(lpfc_cntl_attr_asic_rev, cntl_attr); - memset(phba->BIOSVersion, 0, sizeof(phba->BIOSVersion)); - strlcat(phba->BIOSVersion, (char *)cntl_attr->bios_ver_str, + memcpy(phba->BIOSVersion, cntl_attr->bios_ver_str, sizeof(phba->BIOSVersion)); + phba->BIOSVersion[sizeof(phba->BIOSVersion) - 1] = '\0'; lpfc_printf_log(phba, KERN_INFO, LOG_SLI, "3086 lnk_type:%d, lnk_numb:%d, bios_ver:%s, " diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index dd3630b09aa24..54f66142e14a8 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -5908,7 +5908,11 @@ megasas_set_high_iops_queue_affinity_and_hint(struct megasas_instance *instance) const struct cpumask *mask; if (instance->perf_mode == MR_BALANCED_PERF_MODE) { - mask = cpumask_of_node(dev_to_node(&instance->pdev->dev)); + int nid = dev_to_node(&instance->pdev->dev); + + if (nid == NUMA_NO_NODE) + nid = 0; + mask = cpumask_of_node(nid); for (i = 0; i < instance->low_latency_index_start; i++) { irq = pci_irq_vector(instance->pdev, i); diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h index ae98d15c30b1d..90da783457fbf 100644 --- a/drivers/scsi/mpi3mr/mpi3mr.h +++ b/drivers/scsi/mpi3mr/mpi3mr.h @@ -1025,6 +1025,8 @@ struct scmd_priv { * @logdata_buf: Circular buffer to store log data entries * @logdata_buf_idx: Index of entry in buffer to store * @logdata_entry_sz: log data entry size + * @adm_req_q_bar_writeq_lock: Admin request queue lock + * @adm_reply_q_bar_writeq_lock: Admin reply queue lock * @pend_large_data_sz: Counter to track pending large data * @io_throttle_data_length: I/O size to track in 512b blocks * @io_throttle_high: I/O size to start throttle in 512b blocks @@ -1055,7 +1057,7 @@ struct mpi3mr_ioc { char name[MPI3MR_NAME_LENGTH]; char driver_name[MPI3MR_NAME_LENGTH]; - volatile struct mpi3_sysif_registers __iomem *sysif_regs; + struct mpi3_sysif_registers __iomem *sysif_regs; resource_size_t sysif_regs_phys; int bars; u64 dma_mask; @@ -1207,6 +1209,8 @@ struct mpi3mr_ioc { u8 *logdata_buf; u16 logdata_buf_idx; u16 logdata_entry_sz; + spinlock_t adm_req_q_bar_writeq_lock; + spinlock_t adm_reply_q_bar_writeq_lock; atomic_t pend_large_data_sz; u32 io_throttle_data_length; diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c index 0d148c39ebcc9..b6ae7ba6de523 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_fw.c +++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c @@ -23,17 +23,22 @@ module_param(poll_queues, int, 0444); MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)"); #if defined(writeq) && defined(CONFIG_64BIT) -static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr) +static inline void mpi3mr_writeq(__u64 b, void __iomem *addr, + spinlock_t *write_queue_lock) { writeq(b, addr); } #else -static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr) +static inline void mpi3mr_writeq(__u64 b, void __iomem *addr, + spinlock_t *write_queue_lock) { __u64 data_out = b; + unsigned long flags; + spin_lock_irqsave(write_queue_lock, flags); writel((u32)(data_out), addr); writel((u32)(data_out >> 32), (addr + 4)); + spin_unlock_irqrestore(write_queue_lock, flags); } #endif @@ -174,6 +179,9 @@ static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc, char *desc = NULL; u16 event; + if (!(mrioc->logging_level & MPI3_DEBUG_EVENT)) + return; + event = event_reply->event; switch (event) { @@ -408,8 +416,8 @@ static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc, MPI3MR_SENSE_BUF_SZ); } if (cmdptr->is_waiting) { - complete(&cmdptr->done); cmdptr->is_waiting = 0; + complete(&cmdptr->done); } else if (cmdptr->callback) cmdptr->callback(mrioc, cmdptr); } @@ -2109,6 +2117,8 @@ static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc) { int retval = 0; u16 num_queues = 0, i = 0, msix_count_op_q = 1; + u32 ioc_status; + enum mpi3mr_iocstate ioc_state; num_queues = min_t(int, mrioc->facts.max_op_reply_q, mrioc->facts.max_op_req_q); @@ -2164,6 +2174,14 @@ static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc) retval = -1; goto out_failed; } + ioc_status = readl(&mrioc->sysif_regs->ioc_status); + ioc_state = mpi3mr_get_iocstate(mrioc); + if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) || + ioc_state != MRIOC_STATE_READY) { + mpi3mr_print_fault_info(mrioc); + retval = -1; + goto out_failed; + } mrioc->num_op_reply_q = mrioc->num_op_req_q = i; ioc_info(mrioc, "successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n", @@ -2663,9 +2681,11 @@ static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc) (mrioc->num_admin_req); writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries); mpi3mr_writeq(mrioc->admin_req_dma, - &mrioc->sysif_regs->admin_request_queue_address); + &mrioc->sysif_regs->admin_request_queue_address, + &mrioc->adm_req_q_bar_writeq_lock); mpi3mr_writeq(mrioc->admin_reply_dma, - &mrioc->sysif_regs->admin_reply_queue_address); + &mrioc->sysif_regs->admin_reply_queue_address, + &mrioc->adm_reply_q_bar_writeq_lock); writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi); writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci); return retval; diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c index 7880675a68dba..1bf3572c7aac4 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_os.c +++ b/drivers/scsi/mpi3mr/mpi3mr_os.c @@ -49,6 +49,13 @@ static void mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event, #define MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH (0xFFFE) +/* + * SAS Log info code for a NCQ collateral abort after an NCQ error: + * IOC_LOGINFO_PREFIX_PL | PL_LOGINFO_CODE_SATA_NCQ_FAIL_ALL_CMDS_AFTR_ERR + * See: drivers/message/fusion/lsi/mpi_log_sas.h + */ +#define IOC_LOGINFO_SATA_NCQ_FAIL_AFTER_ERR 0x31080000 + /** * mpi3mr_host_tag_for_scmd - Get host tag for a scmd * @mrioc: Adapter instance reference @@ -3270,7 +3277,18 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc, scmd->result = DID_NO_CONNECT << 16; break; case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED: - scmd->result = DID_SOFT_ERROR << 16; + if (ioc_loginfo == IOC_LOGINFO_SATA_NCQ_FAIL_AFTER_ERR) { + /* + * This is a ATA NCQ command aborted due to another NCQ + * command failure. We must retry this command + * immediately but without incrementing its retry + * counter. + */ + WARN_ON_ONCE(xfer_count != 0); + scmd->result = DID_IMM_RETRY << 16; + } else { + scmd->result = DID_SOFT_ERROR << 16; + } break; case MPI3_IOCSTATUS_SCSI_TASK_TERMINATED: case MPI3_IOCSTATUS_SCSI_EXT_TERMINATED: @@ -5084,6 +5102,8 @@ mpi3mr_probe(struct pci_dev *pdev, const struct pci_device_id *id) spin_lock_init(&mrioc->tgtdev_lock); spin_lock_init(&mrioc->watchdog_lock); spin_lock_init(&mrioc->chain_buf_lock); + spin_lock_init(&mrioc->adm_req_q_bar_writeq_lock); + spin_lock_init(&mrioc->adm_reply_q_bar_writeq_lock); spin_lock_init(&mrioc->sas_node_lock); INIT_LIST_HEAD(&mrioc->fwevt_list); diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c index e289f18fc7643..daef90ee431f5 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c @@ -679,6 +679,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, size_t data_in_sz = 0; long ret; u16 device_handle = MPT3SAS_INVALID_DEVICE_HANDLE; + int tm_ret; issue_reset = 0; @@ -1120,18 +1121,25 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, if (pcie_device && (!ioc->tm_custom_handling) && (!(mpt3sas_scsih_is_pcie_scsi_device( pcie_device->device_info)))) - mpt3sas_scsih_issue_locked_tm(ioc, + tm_ret = mpt3sas_scsih_issue_locked_tm(ioc, le16_to_cpu(mpi_request->FunctionDependent1), 0, 0, 0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0, pcie_device->reset_timeout, MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE); else - mpt3sas_scsih_issue_locked_tm(ioc, + tm_ret = mpt3sas_scsih_issue_locked_tm(ioc, le16_to_cpu(mpi_request->FunctionDependent1), 0, 0, 0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0, 30, MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET); + + if (tm_ret != SUCCESS) { + ioc_info(ioc, + "target reset failed, issue hard reset: handle (0x%04x)\n", + le16_to_cpu(mpi_request->FunctionDependent1)); + mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); + } } else mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); } diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index f270b0d829f6e..7bef42a2fb576 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -196,6 +196,14 @@ struct sense_info { #define MPT3SAS_PORT_ENABLE_COMPLETE (0xFFFD) #define MPT3SAS_ABRT_TASK_SET (0xFFFE) #define MPT3SAS_REMOVE_UNRESPONDING_DEVICES (0xFFFF) + +/* + * SAS Log info code for a NCQ collateral abort after an NCQ error: + * IOC_LOGINFO_PREFIX_PL | PL_LOGINFO_CODE_SATA_NCQ_FAIL_ALL_CMDS_AFTR_ERR + * See: drivers/message/fusion/lsi/mpi_log_sas.h + */ +#define IOC_LOGINFO_SATA_NCQ_FAIL_AFTER_ERR 0x31080000 + /** * struct fw_event_work - firmware event struct * @list: link list framework @@ -5824,6 +5832,17 @@ _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply) scmd->result = DID_TRANSPORT_DISRUPTED << 16; goto out; } + if (log_info == IOC_LOGINFO_SATA_NCQ_FAIL_AFTER_ERR) { + /* + * This is a ATA NCQ command aborted due to another NCQ + * command failure. We must retry this command + * immediately but without incrementing its retry + * counter. + */ + WARN_ON_ONCE(xfer_cnt != 0); + scmd->result = DID_IMM_RETRY << 16; + break; + } if (log_info == 0x31110630) { if (scmd->retries > 2) { scmd->result = DID_NO_CONNECT << 16; @@ -10818,8 +10837,7 @@ _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event) break; case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST: _scsih_pcie_topology_change_event(ioc, fw_event); - ioc->current_event = NULL; - return; + break; } out: fw_event_work_put(fw_event); diff --git a/drivers/scsi/mpt3sas/mpt3sas_transport.c b/drivers/scsi/mpt3sas/mpt3sas_transport.c index e8a4750f6ec47..02c970575464c 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_transport.c +++ b/drivers/scsi/mpt3sas/mpt3sas_transport.c @@ -166,6 +166,9 @@ _transport_convert_phy_link_rate(u8 link_rate) case MPI25_SAS_NEG_LINK_RATE_12_0: rc = SAS_LINK_RATE_12_0_GBPS; break; + case MPI26_SAS_NEG_LINK_RATE_22_5: + rc = SAS_LINK_RATE_22_5_GBPS; + break; case MPI2_SAS_NEG_LINK_RATE_PHY_DISABLED: rc = SAS_PHY_DISABLED; break; @@ -991,11 +994,9 @@ mpt3sas_transport_port_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address, list_for_each_entry_safe(mpt3sas_phy, next_phy, &mpt3sas_port->phy_list, port_siblings) { if ((ioc->logging_level & MPT_DEBUG_TRANSPORT)) - dev_printk(KERN_INFO, &mpt3sas_port->port->dev, - "remove: sas_addr(0x%016llx), phy(%d)\n", - (unsigned long long) - mpt3sas_port->remote_identify.sas_address, - mpt3sas_phy->phy_id); + ioc_info(ioc, "remove: sas_addr(0x%016llx), phy(%d)\n", + (unsigned long long) mpt3sas_port->remote_identify.sas_address, + mpt3sas_phy->phy_id); mpt3sas_phy->phy_belongs_to_port = 0; if (!ioc->remove_host) sas_port_delete_phy(mpt3sas_port->port, diff --git a/drivers/scsi/mvsas/mv_init.c b/drivers/scsi/mvsas/mv_init.c index 43ebb331e2167..47c4434dc7e79 100644 --- a/drivers/scsi/mvsas/mv_init.c +++ b/drivers/scsi/mvsas/mv_init.c @@ -139,7 +139,7 @@ static void mvs_free(struct mvs_info *mvi) if (mvi->shost) scsi_host_put(mvi->shost); list_for_each_entry(mwq, &mvi->wq_list, entry) - cancel_delayed_work(&mwq->work_q); + cancel_delayed_work_sync(&mwq->work_q); kfree(mvi->rsvd_tags); kfree(mvi); } diff --git a/drivers/scsi/mvsas/mv_sas.c b/drivers/scsi/mvsas/mv_sas.c index 1444b1f1c4c88..d6897432cf0f5 100644 --- a/drivers/scsi/mvsas/mv_sas.c +++ b/drivers/scsi/mvsas/mv_sas.c @@ -828,7 +828,7 @@ static int mvs_task_prep(struct sas_task *task, struct mvs_info *mvi, int is_tmf dev_printk(KERN_ERR, mvi->dev, "mvsas prep failed[%d]!\n", rc); if (!sas_protocol_ata(task->task_proto)) if (n_elem) - dma_unmap_sg(mvi->dev, task->scatter, n_elem, + dma_unmap_sg(mvi->dev, task->scatter, task->num_scatter, task->data_dir); prep_out: return rc; @@ -874,7 +874,7 @@ static void mvs_slot_task_free(struct mvs_info *mvi, struct sas_task *task, if (!sas_protocol_ata(task->task_proto)) if (slot->n_elem) dma_unmap_sg(mvi->dev, task->scatter, - slot->n_elem, task->data_dir); + task->num_scatter, task->data_dir); switch (task->task_proto) { case SAS_PROTOCOL_SMP: diff --git a/drivers/scsi/myrs.c b/drivers/scsi/myrs.c index e824be9d9bbb9..9075836ccc3e2 100644 --- a/drivers/scsi/myrs.c +++ b/drivers/scsi/myrs.c @@ -498,14 +498,14 @@ static bool myrs_enable_mmio_mbox(struct myrs_hba *cs, /* Temporary dma mapping, used only in the scope of this function */ mbox = dma_alloc_coherent(&pdev->dev, sizeof(union myrs_cmd_mbox), &mbox_addr, GFP_KERNEL); - if (dma_mapping_error(&pdev->dev, mbox_addr)) + if (!mbox) return false; /* These are the base addresses for the command memory mailbox array */ cs->cmd_mbox_size = MYRS_MAX_CMD_MBOX * sizeof(union myrs_cmd_mbox); cmd_mbox = dma_alloc_coherent(&pdev->dev, cs->cmd_mbox_size, &cs->cmd_mbox_addr, GFP_KERNEL); - if (dma_mapping_error(&pdev->dev, cs->cmd_mbox_addr)) { + if (!cmd_mbox) { dev_err(&pdev->dev, "Failed to map command mailbox\n"); goto out_free; } @@ -520,7 +520,7 @@ static bool myrs_enable_mmio_mbox(struct myrs_hba *cs, cs->stat_mbox_size = MYRS_MAX_STAT_MBOX * sizeof(struct myrs_stat_mbox); stat_mbox = dma_alloc_coherent(&pdev->dev, cs->stat_mbox_size, &cs->stat_mbox_addr, GFP_KERNEL); - if (dma_mapping_error(&pdev->dev, cs->stat_mbox_addr)) { + if (!stat_mbox) { dev_err(&pdev->dev, "Failed to map status mailbox\n"); goto out_free; } @@ -533,7 +533,7 @@ static bool myrs_enable_mmio_mbox(struct myrs_hba *cs, cs->fwstat_buf = dma_alloc_coherent(&pdev->dev, sizeof(struct myrs_fwstat), &cs->fwstat_addr, GFP_KERNEL); - if (dma_mapping_error(&pdev->dev, cs->fwstat_addr)) { + if (!cs->fwstat_buf) { dev_err(&pdev->dev, "Failed to map firmware health buffer\n"); cs->fwstat_buf = NULL; goto out_free; diff --git a/drivers/scsi/pm8001/pm8001_ctl.c b/drivers/scsi/pm8001/pm8001_ctl.c index 5c26a13ffbd26..d3ff212d28b2e 100644 --- a/drivers/scsi/pm8001/pm8001_ctl.c +++ b/drivers/scsi/pm8001/pm8001_ctl.c @@ -534,23 +534,25 @@ static ssize_t pm8001_ctl_iop_log_show(struct device *cdev, char *str = buf; u32 read_size = pm8001_ha->main_cfg_tbl.pm80xx_tbl.event_log_size / 1024; - static u32 start, end, count; u32 max_read_times = 32; u32 max_count = (read_size * 1024) / (max_read_times * 4); u32 *temp = (u32 *)pm8001_ha->memoryMap.region[IOP].virt_ptr; - if ((count % max_count) == 0) { - start = 0; - end = max_read_times; - count = 0; + mutex_lock(&pm8001_ha->iop_log_lock); + + if ((pm8001_ha->iop_log_count % max_count) == 0) { + pm8001_ha->iop_log_start = 0; + pm8001_ha->iop_log_end = max_read_times; + pm8001_ha->iop_log_count = 0; } else { - start = end; - end = end + max_read_times; + pm8001_ha->iop_log_start = pm8001_ha->iop_log_end; + pm8001_ha->iop_log_end = pm8001_ha->iop_log_end + max_read_times; } - for (; start < end; start++) - str += sprintf(str, "%08x ", *(temp+start)); - count++; + for (; pm8001_ha->iop_log_start < pm8001_ha->iop_log_end; pm8001_ha->iop_log_start++) + str += sprintf(str, "%08x ", *(temp+pm8001_ha->iop_log_start)); + pm8001_ha->iop_log_count++; + mutex_unlock(&pm8001_ha->iop_log_lock); return str - buf; } static DEVICE_ATTR(iop_log, S_IRUGO, pm8001_ctl_iop_log_show, NULL); @@ -680,7 +682,7 @@ static int pm8001_set_nvmd(struct pm8001_hba_info *pm8001_ha) struct pm8001_ioctl_payload *payload; DECLARE_COMPLETION_ONSTACK(completion); u8 *ioctlbuffer; - u32 ret; + int ret; u32 length = 1024 * 5 + sizeof(*payload) - 1; if (pm8001_ha->fw_image->size > 4096) { diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c index c2f6151cbd2d0..00664bd2caab1 100644 --- a/drivers/scsi/pm8001/pm8001_init.c +++ b/drivers/scsi/pm8001/pm8001_init.c @@ -529,6 +529,7 @@ static struct pm8001_hba_info *pm8001_pci_alloc(struct pci_dev *pdev, pm8001_ha->id = pm8001_id++; pm8001_ha->logging_level = logging_level; pm8001_ha->non_fatal_count = 0; + mutex_init(&pm8001_ha->iop_log_lock); if (link_rate >= 1 && link_rate <= 15) pm8001_ha->link_rate = (link_rate << 8); else { diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c index a9d6dac413346..4daab8b6d6752 100644 --- a/drivers/scsi/pm8001/pm8001_sas.c +++ b/drivers/scsi/pm8001/pm8001_sas.c @@ -703,6 +703,7 @@ static void pm8001_dev_gone_notify(struct domain_device *dev) unsigned long flags = 0; struct pm8001_hba_info *pm8001_ha; struct pm8001_device *pm8001_dev = dev->lldd_dev; + struct domain_device *parent_dev = dev->parent; pm8001_ha = pm8001_find_ha_by_dev(dev); spin_lock_irqsave(&pm8001_ha->lock, flags); @@ -719,7 +720,13 @@ static void pm8001_dev_gone_notify(struct domain_device *dev) spin_lock_irqsave(&pm8001_ha->lock, flags); } PM8001_CHIP_DISP->dereg_dev_req(pm8001_ha, device_id); - pm8001_ha->phy[pm8001_dev->attached_phy].phy_attached = 0; + + /* + * The phy array only contains local phys. Thus, we cannot clear + * phy_attached for a device behind an expander. + */ + if (!(parent_dev && dev_is_expander(parent_dev->dev_type))) + pm8001_ha->phy[pm8001_dev->attached_phy].phy_attached = 0; pm8001_free_dev(pm8001_dev); } else { pm8001_dbg(pm8001_ha, DISC, "Found dev has gone.\n"); diff --git a/drivers/scsi/pm8001/pm8001_sas.h b/drivers/scsi/pm8001/pm8001_sas.h index 2fadd353f1c13..72cd1523235ca 100644 --- a/drivers/scsi/pm8001/pm8001_sas.h +++ b/drivers/scsi/pm8001/pm8001_sas.h @@ -543,6 +543,10 @@ struct pm8001_hba_info { u32 ci_offset; u32 pi_offset; u32 max_memcnt; + u32 iop_log_start; + u32 iop_log_end; + u32 iop_log_count; + struct mutex iop_log_lock; }; struct pm8001_work { diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c index 9a81d14aef6b9..17b19b39699a3 100644 --- a/drivers/scsi/qedf/qedf_main.c +++ b/drivers/scsi/qedf/qedf_main.c @@ -699,7 +699,7 @@ static u32 qedf_get_login_failures(void *cookie) } static struct qed_fcoe_cb_ops qedf_cb_ops = { - { + .common = { .link_update = qedf_link_update, .bw_update = qedf_bw_update, .schedule_recovery_handler = qedf_schedule_recovery_handler, diff --git a/drivers/scsi/qla2xxx/qla_edif.c b/drivers/scsi/qla2xxx/qla_edif.c index dcde55c8ee5de..be20e2c457b8e 100644 --- a/drivers/scsi/qla2xxx/qla_edif.c +++ b/drivers/scsi/qla2xxx/qla_edif.c @@ -1797,7 +1797,7 @@ qla24xx_sadb_update(struct bsg_job *bsg_job) switch (rval) { case QLA_SUCCESS: break; - case EAGAIN: + case -EAGAIN: msleep(EDIF_MSLEEP_INTERVAL); cnt++; if (cnt < EDIF_RETRY_COUNT) @@ -3648,7 +3648,7 @@ int qla_edif_process_els(scsi_qla_host_t *vha, struct bsg_job *bsg_job) p->e.extra_rx_xchg_address, p->e.extra_control_flags, sp->handle, sp->remap.req.len, bsg_job); break; - case EAGAIN: + case -EAGAIN: msleep(EDIF_MSLEEP_INTERVAL); cnt++; if (cnt < EDIF_RETRY_COUNT) diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index 5fa2727c1bea7..e881d704b45d1 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -2059,11 +2059,11 @@ static void qla_marker_sp_done(srb_t *sp, int res) int cnt = 5; \ do { \ if (_chip_gen != sp->vha->hw->chip_reset || _login_gen != sp->fcport->login_gen) {\ - _rval = EINVAL; \ + _rval = -EINVAL; \ break; \ } \ _rval = qla2x00_start_sp(_sp); \ - if (_rval == EAGAIN) \ + if (_rval == -EAGAIN) \ msleep(1); \ else \ break; \ diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c index 0cd6f3e148824..13b6cb1b93acd 100644 --- a/drivers/scsi/qla2xxx/qla_mbx.c +++ b/drivers/scsi/qla2xxx/qla_mbx.c @@ -2147,7 +2147,7 @@ qla24xx_get_port_database(scsi_qla_host_t *vha, u16 nport_handle, pdb_dma = dma_map_single(&vha->hw->pdev->dev, pdb, sizeof(*pdb), DMA_FROM_DEVICE); - if (!pdb_dma) { + if (dma_mapping_error(&vha->hw->pdev->dev, pdb_dma)) { ql_log(ql_log_warn, vha, 0x1116, "Failed to map dma buffer.\n"); return QLA_MEMORY_ALLOC_FAILED; } diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index 8f4cc136a9c9c..080670cb2aa51 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -420,7 +420,7 @@ static int qla_nvme_xmt_ls_rsp(struct nvme_fc_local_port *lport, switch (rval) { case QLA_SUCCESS: break; - case EAGAIN: + case -EAGAIN: msleep(PURLS_MSLEEP_INTERVAL); cnt++; if (cnt < PURLS_RETRY_COUNT) diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c index 675332e49a7b0..d91efd7c983ce 100644 --- a/drivers/scsi/qla4xxx/ql4_os.c +++ b/drivers/scsi/qla4xxx/ql4_os.c @@ -3420,6 +3420,8 @@ static int qla4xxx_alloc_pdu(struct iscsi_task *task, uint8_t opcode) task_data->data_dma = dma_map_single(&ha->pdev->dev, task->data, task->data_count, DMA_TO_DEVICE); + if (dma_mapping_error(&ha->pdev->dev, task_data->data_dma)) + return -ENOMEM; } DEBUG2(ql4_printk(KERN_INFO, ha, "%s: MaxRecvLen %u, iscsi hrd %d\n", @@ -6604,6 +6606,8 @@ static struct iscsi_endpoint *qla4xxx_get_ep_fwdb(struct scsi_qla_host *ha, ep = qla4xxx_ep_connect(ha->host, (struct sockaddr *)dst_addr, 0); vfree(dst_addr); + if (IS_ERR(ep)) + return NULL; return ep; } diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index b0f2d2cf4a0dc..a279d0efeb6d8 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -1039,6 +1039,9 @@ void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, struct scsi_eh_save *ses, unsigned char *cmnd, int cmnd_size, unsigned sense_bytes) { struct scsi_device *sdev = scmd->device; +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct request *rq = scsi_cmd_to_rq(scmd); +#endif /* * We need saved copies of a number of fields - this is because @@ -1090,6 +1093,18 @@ void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, struct scsi_eh_save *ses, scmd->cmnd[1] = (scmd->cmnd[1] & 0x1f) | (sdev->lun << 5 & 0xe0); + /* + * Encryption must be disabled for the commands submitted by the error handler. + * Hence, clear the encryption context information. + */ +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + ses->rq_crypt_keyslot = rq->crypt_keyslot; + ses->rq_crypt_ctx = rq->crypt_ctx; + + rq->crypt_keyslot = NULL; + rq->crypt_ctx = NULL; +#endif + /* * Zero the sense buffer. The scsi spec mandates that any * untransferred sense data should be interpreted as being zero. @@ -1107,6 +1122,10 @@ EXPORT_SYMBOL(scsi_eh_prep_cmnd); */ void scsi_eh_restore_cmnd(struct scsi_cmnd* scmd, struct scsi_eh_save *ses) { +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct request *rq = scsi_cmd_to_rq(scmd); +#endif + /* * Restore original data */ @@ -1119,6 +1138,11 @@ void scsi_eh_restore_cmnd(struct scsi_cmnd* scmd, struct scsi_eh_save *ses) scmd->underflow = ses->underflow; scmd->prot_op = ses->prot_op; scmd->eh_eflags = ses->eh_eflags; + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + rq->crypt_keyslot = ses->rq_crypt_keyslot; + rq->crypt_ctx = ses->rq_crypt_ctx; +#endif } EXPORT_SYMBOL(scsi_eh_restore_cmnd); diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index cead0fbbe5dbd..8ee74dddef16e 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -1851,7 +1851,7 @@ int scsi_scan_host_selected(struct Scsi_Host *shost, unsigned int channel, return 0; } - +EXPORT_SYMBOL(scsi_scan_host_selected); static void scsi_sysfs_add_devices(struct Scsi_Host *shost) { struct scsi_device *sdev; diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index 24f6eefb68030..df37ac81620e6 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -265,7 +265,7 @@ show_shost_supported_mode(struct device *dev, struct device_attribute *attr, return show_shost_mode(supported_mode, buf); } -static DEVICE_ATTR(supported_mode, S_IRUGO | S_IWUSR, show_shost_supported_mode, NULL); +static DEVICE_ATTR(supported_mode, S_IRUGO, show_shost_supported_mode, NULL); static ssize_t show_shost_active_mode(struct device *dev, @@ -279,7 +279,7 @@ show_shost_active_mode(struct device *dev, return show_shost_mode(shost->active_mode, buf); } -static DEVICE_ATTR(active_mode, S_IRUGO | S_IWUSR, show_shost_active_mode, NULL); +static DEVICE_ATTR(active_mode, S_IRUGO, show_shost_active_mode, NULL); static int check_reset_type(const char *str) { diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c index 0c30fec555475..22bec2453bac9 100644 --- a/drivers/scsi/scsi_transport_iscsi.c +++ b/drivers/scsi/scsi_transport_iscsi.c @@ -2169,6 +2169,8 @@ static int iscsi_iter_destroy_conn_fn(struct device *dev, void *data) return 0; iscsi_remove_conn(iscsi_dev_to_conn(dev)); + iscsi_put_conn(iscsi_dev_to_conn(dev)); + return 0; } @@ -3525,7 +3527,7 @@ static int iscsi_new_flashnode(struct iscsi_transport *transport, pr_err("%s could not find host no %u\n", __func__, ev->u.new_flashnode.host_no); err = -ENODEV; - goto put_host; + goto exit_new_fnode; } index = transport->new_flashnode(shost, data, len); @@ -3535,7 +3537,6 @@ static int iscsi_new_flashnode(struct iscsi_transport *transport, else err = -EIO; -put_host: scsi_host_put(shost); exit_new_fnode: @@ -3560,7 +3561,7 @@ static int iscsi_del_flashnode(struct iscsi_transport *transport, pr_err("%s could not find host no %u\n", __func__, ev->u.del_flashnode.host_no); err = -ENODEV; - goto put_host; + goto exit_del_fnode; } idx = ev->u.del_flashnode.flashnode_idx; @@ -3602,7 +3603,7 @@ static int iscsi_login_flashnode(struct iscsi_transport *transport, pr_err("%s could not find host no %u\n", __func__, ev->u.login_flashnode.host_no); err = -ENODEV; - goto put_host; + goto exit_login_fnode; } idx = ev->u.login_flashnode.flashnode_idx; @@ -3654,7 +3655,7 @@ static int iscsi_logout_flashnode(struct iscsi_transport *transport, pr_err("%s could not find host no %u\n", __func__, ev->u.logout_flashnode.host_no); err = -ENODEV; - goto put_host; + goto exit_logout_fnode; } idx = ev->u.logout_flashnode.flashnode_idx; @@ -3704,7 +3705,7 @@ static int iscsi_logout_flashnode_sid(struct iscsi_transport *transport, pr_err("%s could not find host no %u\n", __func__, ev->u.logout_flashnode.host_no); err = -ENODEV; - goto put_host; + goto exit_logout_sid; } session = iscsi_session_lookup(ev->u.logout_flashnode_sid.sid); diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index 7fdd2b61fe855..7b4c4752e2160 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c @@ -40,6 +40,8 @@ #include #include "scsi_sas_internal.h" +#include "scsi_priv.h" + struct sas_host_attrs { struct list_head rphy_list; struct mutex lock; @@ -1681,32 +1683,66 @@ int scsi_is_sas_rphy(const struct device *dev) } EXPORT_SYMBOL(scsi_is_sas_rphy); - -/* - * SCSI scan helper - */ - -static int sas_user_scan(struct Scsi_Host *shost, uint channel, - uint id, u64 lun) +static void scan_channel_zero(struct Scsi_Host *shost, uint id, u64 lun) { struct sas_host_attrs *sas_host = to_sas_host_attrs(shost); struct sas_rphy *rphy; - mutex_lock(&sas_host->lock); list_for_each_entry(rphy, &sas_host->rphy_list, list) { if (rphy->identify.device_type != SAS_END_DEVICE || rphy->scsi_target_id == -1) continue; - if ((channel == SCAN_WILD_CARD || channel == 0) && - (id == SCAN_WILD_CARD || id == rphy->scsi_target_id)) { + if (id == SCAN_WILD_CARD || id == rphy->scsi_target_id) { scsi_scan_target(&rphy->dev, 0, rphy->scsi_target_id, lun, SCSI_SCAN_MANUAL); } } - mutex_unlock(&sas_host->lock); +} - return 0; +/* + * SCSI scan helper + */ + +static int sas_user_scan(struct Scsi_Host *shost, uint channel, + uint id, u64 lun) +{ + struct sas_host_attrs *sas_host = to_sas_host_attrs(shost); + int res = 0; + int i; + + switch (channel) { + case 0: + mutex_lock(&sas_host->lock); + scan_channel_zero(shost, id, lun); + mutex_unlock(&sas_host->lock); + break; + + case SCAN_WILD_CARD: + mutex_lock(&sas_host->lock); + scan_channel_zero(shost, id, lun); + mutex_unlock(&sas_host->lock); + + for (i = 1; i <= shost->max_channel; i++) { + res = scsi_scan_host_selected(shost, i, id, lun, + SCSI_SCAN_MANUAL); + if (res) + goto exit_scan; + } + break; + + default: + if (channel < shost->max_channel) { + res = scsi_scan_host_selected(shost, channel, id, lun, + SCSI_SCAN_MANUAL); + } else { + res = -EINVAL; + } + break; + } + +exit_scan: + return res; } diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index cd414f4470970..b0e8dd387bb19 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -3900,7 +3900,9 @@ static void sd_shutdown(struct device *dev) if ((system_state != SYSTEM_RESTART && sdkp->device->manage_system_start_stop) || (system_state == SYSTEM_POWER_OFF && - sdkp->device->manage_shutdown)) { + sdkp->device->manage_shutdown) || + (system_state == SYSTEM_RUNNING && + sdkp->device->manage_runtime_start_stop)) { sd_printk(KERN_NOTICE, sdkp, "Stopping disk\n"); sd_start_stop_device(sdkp, 0); } diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 2144f8781435c..13bd09139824f 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -202,6 +202,7 @@ static void *sd_zbc_alloc_report_buffer(struct scsi_disk *sdkp, unsigned int nr_zones, size_t *buflen) { struct request_queue *q = sdkp->disk->queue; + unsigned int max_segments; size_t bufsize; void *buf; @@ -213,12 +214,15 @@ static void *sd_zbc_alloc_report_buffer(struct scsi_disk *sdkp, * Furthermore, since the report zone command cannot be split, make * sure that the allocated buffer can always be mapped by limiting the * number of pages allocated to the HBA max segments limit. + * Since max segments can be larger than the max inline bio vectors, + * further limit the allocated buffer to BIO_MAX_INLINE_VECS. */ nr_zones = min(nr_zones, sdkp->zone_info.nr_zones); bufsize = roundup((nr_zones + 1) * 64, SECTOR_SIZE); bufsize = min_t(size_t, bufsize, queue_max_hw_sectors(q) << SECTOR_SHIFT); - bufsize = min_t(size_t, bufsize, queue_max_segments(q) << PAGE_SHIFT); + max_segments = min(BIO_MAX_INLINE_VECS, queue_max_segments(q)); + bufsize = min_t(size_t, bufsize, max_segments << PAGE_SHIFT); while (bufsize >= SECTOR_SIZE) { buf = kvzalloc(bufsize, GFP_KERNEL | __GFP_NORETRY); diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 62574886a9111..9258a1a8c23c1 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -2212,9 +2212,17 @@ sg_remove_sfp_usercontext(struct work_struct *work) write_lock_irqsave(&sfp->rq_list_lock, iflags); while (!list_empty(&sfp->rq_list)) { srp = list_first_entry(&sfp->rq_list, Sg_request, entry); - sg_finish_rem_req(srp); list_del(&srp->entry); + write_unlock_irqrestore(&sfp->rq_list_lock, iflags); + + sg_finish_rem_req(srp); + /* + * sg_rq_end_io() uses srp->parentfp. Hence, only clear + * srp->parentfp after blk_mq_free_request() has been called. + */ srp->parentfp = NULL; + + write_lock_irqsave(&sfp->rq_list_lock, iflags); } write_unlock_irqrestore(&sfp->rq_list_lock, iflags); diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c index 900322bad4f3b..f9ab45c4bb40d 100644 --- a/drivers/scsi/st.c +++ b/drivers/scsi/st.c @@ -953,7 +953,6 @@ static void reset_state(struct scsi_tape *STp) STp->partition = find_partition(STp); if (STp->partition < 0) STp->partition = 0; - STp->new_partition = STp->partition; } } @@ -2895,7 +2894,6 @@ static int st_int_ioctl(struct scsi_tape *STp, unsigned int cmd_in, unsigned lon timeout = STp->long_timeout * 8; DEBC_printk(STp, "Erasing tape.\n"); - fileno = blkno = at_sm = 0; break; case MTSETBLK: /* Set block length */ case MTSETDENSITY: /* Set tape density */ @@ -2928,14 +2926,17 @@ static int st_int_ioctl(struct scsi_tape *STp, unsigned int cmd_in, unsigned lon if (cmd_in == MTSETDENSITY) { (STp->buffer)->b_data[4] = arg; STp->density_changed = 1; /* At least we tried ;-) */ + STp->changed_density = arg; } else if (cmd_in == SET_DENS_AND_BLK) (STp->buffer)->b_data[4] = arg >> 24; else (STp->buffer)->b_data[4] = STp->density; if (cmd_in == MTSETBLK || cmd_in == SET_DENS_AND_BLK) { ltmp = arg & MT_ST_BLKSIZE_MASK; - if (cmd_in == MTSETBLK) + if (cmd_in == MTSETBLK) { STp->blksize_changed = 1; /* At least we tried ;-) */ + STp->changed_blksize = arg; + } } else ltmp = STp->block_size; (STp->buffer)->b_data[9] = (ltmp >> 16); @@ -3082,7 +3083,9 @@ static int st_int_ioctl(struct scsi_tape *STp, unsigned int cmd_in, unsigned lon cmd_in == MTSETDRVBUFFER || cmd_in == SET_DENS_AND_BLK) { if (cmdstatp->sense_hdr.sense_key == ILLEGAL_REQUEST && - !(STp->use_pf & PF_TESTED)) { + cmdstatp->sense_hdr.asc == 0x24 && + (STp->device)->scsi_level <= SCSI_2 && + !(STp->use_pf & PF_TESTED)) { /* Try the other possible state of Page Format if not already tried */ STp->use_pf = (STp->use_pf ^ USE_PF) | PF_TESTED; @@ -3634,9 +3637,25 @@ static long st_ioctl(struct file *file, unsigned int cmd_in, unsigned long arg) retval = (-EIO); goto out; } - reset_state(STp); + reset_state(STp); /* Clears pos_unknown */ /* remove this when the midlevel properly clears was_reset */ STp->device->was_reset = 0; + + /* Fix the device settings after reset, ignore errors */ + if (mtc.mt_op == MTREW || mtc.mt_op == MTSEEK || + mtc.mt_op == MTEOM) { + if (STp->can_partitions) { + /* STp->new_partition contains the + * latest partition set + */ + STp->partition = 0; + switch_partition(STp); + } + if (STp->density_changed) + st_int_ioctl(STp, MTSETDENSITY, STp->changed_density); + if (STp->blksize_changed) + st_int_ioctl(STp, MTSETBLK, STp->changed_blksize); + } } if (mtc.mt_op != MTNOP && mtc.mt_op != MTSETBLK && diff --git a/drivers/scsi/st.h b/drivers/scsi/st.h index 1aaaf5369a40f..6d31b894ee84c 100644 --- a/drivers/scsi/st.h +++ b/drivers/scsi/st.h @@ -165,6 +165,7 @@ struct scsi_tape { unsigned char compression_changed; unsigned char drv_buffer; unsigned char density; + unsigned char changed_density; unsigned char door_locked; unsigned char autorew_dev; /* auto-rewind device */ unsigned char rew_at_close; /* rewind necessary at close */ @@ -172,6 +173,7 @@ struct scsi_tape { unsigned char cleaning_req; /* cleaning requested? */ unsigned char first_tur; /* first TEST UNIT READY */ int block_size; + int changed_blksize; int min_block; int max_block; int recover_count; /* From tape opening */ diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index b8186feccdf5a..954a1cc50ba74 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -362,7 +362,7 @@ MODULE_PARM_DESC(ring_avail_percent_lowater, /* * Timeout in seconds for all devices managed by this driver. */ -static int storvsc_timeout = 180; +static const int storvsc_timeout = 180; #if IS_ENABLED(CONFIG_SCSI_FC_ATTRS) static struct scsi_transport_template *fc_transport_template; @@ -768,7 +768,7 @@ static void handle_multichannel_storage(struct hv_device *device, int max_chns) return; } - t = wait_for_completion_timeout(&request->wait_event, 10*HZ); + t = wait_for_completion_timeout(&request->wait_event, storvsc_timeout * HZ); if (t == 0) { dev_err(dev, "Failed to create sub-channel: timed out\n"); return; @@ -833,7 +833,7 @@ static int storvsc_execute_vstor_op(struct hv_device *device, if (ret != 0) return ret; - t = wait_for_completion_timeout(&request->wait_event, 5*HZ); + t = wait_for_completion_timeout(&request->wait_event, storvsc_timeout * HZ); if (t == 0) return -ETIMEDOUT; @@ -1351,6 +1351,8 @@ static int storvsc_connect_to_vsp(struct hv_device *device, u32 ring_size, return ret; ret = storvsc_channel_init(device, is_fc); + if (ret) + vmbus_close(device->channel); return ret; } @@ -1668,7 +1670,7 @@ static int storvsc_host_reset_handler(struct scsi_cmnd *scmnd) if (ret != 0) return FAILED; - t = wait_for_completion_timeout(&request->wait_event, 5*HZ); + t = wait_for_completion_timeout(&request->wait_event, storvsc_timeout * HZ); if (t == 0) return TIMEOUT_ERROR; @@ -1819,6 +1821,7 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) return SCSI_MLQUEUE_DEVICE_BUSY; } + payload->rangecount = 1; payload->range.len = length; payload->range.offset = offset_in_hvpg; diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 24bd619ec5e48..1da1dfd9cb199 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -48,6 +48,7 @@ struct apple_rtkit { struct apple_rtkit_shmem ioreport_buffer; struct apple_rtkit_shmem crashlog_buffer; + struct apple_rtkit_shmem oslog_buffer; struct apple_rtkit_shmem syslog_buffer; char *syslog_msg_buffer; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index d9f19dc99da5e..2c37216f423d2 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -66,8 +66,9 @@ enum { #define APPLE_RTKIT_SYSLOG_MSG_SIZE GENMASK_ULL(31, 24) #define APPLE_RTKIT_OSLOG_TYPE GENMASK_ULL(63, 56) -#define APPLE_RTKIT_OSLOG_INIT 1 -#define APPLE_RTKIT_OSLOG_ACK 3 +#define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1 +#define APPLE_RTKIT_OSLOG_SIZE GENMASK_ULL(55, 36) +#define APPLE_RTKIT_OSLOG_IOVA GENMASK_ULL(35, 0) #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12 @@ -256,15 +257,21 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, struct apple_rtkit_shmem *buffer, u8 ep, u64 msg) { - size_t n_4kpages = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg); u64 reply; int err; + /* The different size vs. IOVA shifts look odd but are indeed correct this way */ + if (ep == APPLE_RTKIT_EP_OSLOG) { + buffer->size = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg); + buffer->iova = FIELD_GET(APPLE_RTKIT_OSLOG_IOVA, msg) << 12; + } else { + buffer->size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12; + buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); + } + buffer->buffer = NULL; buffer->iomem = NULL; buffer->is_mapped = false; - buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); - buffer->size = n_4kpages << 12; dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n", buffer->size, &buffer->iova); @@ -289,11 +296,21 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, } if (!buffer->is_mapped) { - reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE, - APPLE_RTKIT_BUFFER_REQUEST); - reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, n_4kpages); - reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, - buffer->iova); + /* oslog uses different fields and needs a shifted IOVA instead of size */ + if (ep == APPLE_RTKIT_EP_OSLOG) { + reply = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE, + APPLE_RTKIT_OSLOG_BUFFER_REQUEST); + reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_SIZE, buffer->size); + reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_IOVA, + buffer->iova >> 12); + } else { + reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE, + APPLE_RTKIT_BUFFER_REQUEST); + reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, + buffer->size >> 12); + reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, + buffer->iova); + } apple_rtkit_send_message(rtk, ep, reply, NULL, false); } @@ -487,25 +504,18 @@ static void apple_rtkit_syslog_rx(struct apple_rtkit *rtk, u64 msg) } } -static void apple_rtkit_oslog_rx_init(struct apple_rtkit *rtk, u64 msg) -{ - u64 ack; - - dev_dbg(rtk->dev, "RTKit: oslog init: msg: 0x%llx\n", msg); - ack = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE, APPLE_RTKIT_OSLOG_ACK); - apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_OSLOG, ack, NULL, false); -} - static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg); switch (type) { - case APPLE_RTKIT_OSLOG_INIT: - apple_rtkit_oslog_rx_init(rtk, msg); + case APPLE_RTKIT_OSLOG_BUFFER_REQUEST: + apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer, + APPLE_RTKIT_EP_OSLOG, msg); break; default: - dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", msg); + dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", + msg); } } @@ -744,7 +754,7 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, rtk->mbox_cl.rx_callback = &apple_rtkit_rx; rtk->mbox_cl.tx_done = &apple_rtkit_tx_done; - rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_MEM_RECLAIM, + rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); if (!rtk->wq) { ret = -ENOMEM; @@ -787,6 +797,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); + apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); kfree(rtk->syslog_msg_buffer); @@ -967,6 +978,7 @@ void apple_rtkit_free(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); + apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); kfree(rtk->syslog_msg_buffer); diff --git a/drivers/soc/aspeed/aspeed-lpc-snoop.c b/drivers/soc/aspeed/aspeed-lpc-snoop.c index 773dbcbc03a6c..3dd2ab8336aa3 100644 --- a/drivers/soc/aspeed/aspeed-lpc-snoop.c +++ b/drivers/soc/aspeed/aspeed-lpc-snoop.c @@ -58,6 +58,7 @@ struct aspeed_lpc_snoop_model_data { }; struct aspeed_lpc_snoop_channel { + bool enabled; struct kfifo fifo; wait_queue_head_t wq; struct miscdevice miscdev; @@ -166,7 +167,7 @@ static int aspeed_lpc_snoop_config_irq(struct aspeed_lpc_snoop *lpc_snoop, int rc; lpc_snoop->irq = platform_get_irq(pdev, 0); - if (!lpc_snoop->irq) + if (lpc_snoop->irq < 0) return -ENODEV; rc = devm_request_irq(dev, lpc_snoop->irq, @@ -190,6 +191,9 @@ static int aspeed_lpc_enable_snoop(struct aspeed_lpc_snoop *lpc_snoop, const struct aspeed_lpc_snoop_model_data *model_data = of_device_get_match_data(dev); + if (WARN_ON(lpc_snoop->chan[channel].enabled)) + return -EBUSY; + init_waitqueue_head(&lpc_snoop->chan[channel].wq); /* Create FIFO datastructure */ rc = kfifo_alloc(&lpc_snoop->chan[channel].fifo, @@ -200,11 +204,15 @@ static int aspeed_lpc_enable_snoop(struct aspeed_lpc_snoop *lpc_snoop, lpc_snoop->chan[channel].miscdev.minor = MISC_DYNAMIC_MINOR; lpc_snoop->chan[channel].miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "%s%d", DEVICE_NAME, channel); + if (!lpc_snoop->chan[channel].miscdev.name) { + rc = -ENOMEM; + goto err_free_fifo; + } lpc_snoop->chan[channel].miscdev.fops = &snoop_fops; lpc_snoop->chan[channel].miscdev.parent = dev; rc = misc_register(&lpc_snoop->chan[channel].miscdev); if (rc) - return rc; + goto err_free_fifo; /* Enable LPC snoop channel at requested port */ switch (channel) { @@ -221,7 +229,8 @@ static int aspeed_lpc_enable_snoop(struct aspeed_lpc_snoop *lpc_snoop, hicrb_en = HICRB_ENSNP1D; break; default: - return -EINVAL; + rc = -EINVAL; + goto err_misc_deregister; } regmap_update_bits(lpc_snoop->regmap, HICR5, hicr5_en, hicr5_en); @@ -231,12 +240,23 @@ static int aspeed_lpc_enable_snoop(struct aspeed_lpc_snoop *lpc_snoop, regmap_update_bits(lpc_snoop->regmap, HICRB, hicrb_en, hicrb_en); + lpc_snoop->chan[channel].enabled = true; + + return 0; + +err_misc_deregister: + misc_deregister(&lpc_snoop->chan[channel].miscdev); +err_free_fifo: + kfifo_free(&lpc_snoop->chan[channel].fifo); return rc; } static void aspeed_lpc_disable_snoop(struct aspeed_lpc_snoop *lpc_snoop, int channel) { + if (!lpc_snoop->chan[channel].enabled) + return; + switch (channel) { case 0: regmap_update_bits(lpc_snoop->regmap, HICR5, @@ -252,8 +272,10 @@ static void aspeed_lpc_disable_snoop(struct aspeed_lpc_snoop *lpc_snoop, return; } - kfifo_free(&lpc_snoop->chan[channel].fifo); + lpc_snoop->chan[channel].enabled = false; + /* Consider improving safety wrt concurrent reader(s) */ misc_deregister(&lpc_snoop->chan[channel].miscdev); + kfifo_free(&lpc_snoop->chan[channel].fifo); } static int aspeed_lpc_snoop_probe(struct platform_device *pdev) diff --git a/drivers/soc/aspeed/aspeed-socinfo.c b/drivers/soc/aspeed/aspeed-socinfo.c index 3f759121dc00a..67e9ac3d08ecc 100644 --- a/drivers/soc/aspeed/aspeed-socinfo.c +++ b/drivers/soc/aspeed/aspeed-socinfo.c @@ -27,6 +27,10 @@ static struct { { "AST2620", 0x05010203 }, { "AST2605", 0x05030103 }, { "AST2625", 0x05030403 }, + /* AST2700 */ + { "AST2750", 0x06000003 }, + { "AST2700", 0x06000103 }, + { "AST2720", 0x06000203 }, }; static const char *siliconid_to_name(u32 siliconid) diff --git a/drivers/soc/oplus/bluetooth b/drivers/soc/oplus/bluetooth new file mode 120000 index 0000000000000..0f3dcc2314bd4 --- /dev/null +++ b/drivers/soc/oplus/bluetooth @@ -0,0 +1 @@ +../../../../../vendor/oplus/kernel/bluetooth \ No newline at end of file diff --git a/drivers/soc/oplus/storage b/drivers/soc/oplus/storage new file mode 120000 index 0000000000000..5836f496ad7dc --- /dev/null +++ b/drivers/soc/oplus/storage @@ -0,0 +1 @@ +../../../../../vendor/oplus/kernel/storage \ No newline at end of file diff --git a/drivers/soc/qcom/mdt_loader.c b/drivers/soc/qcom/mdt_loader.c index 6f177e46fa0f8..0afecda0bfaa3 100644 --- a/drivers/soc/qcom/mdt_loader.c +++ b/drivers/soc/qcom/mdt_loader.c @@ -17,6 +17,39 @@ #include #include +static bool mdt_header_valid(const struct firmware *fw) +{ + const struct elf32_hdr *ehdr; + size_t phend; + size_t shend; + + if (fw->size < sizeof(*ehdr)) + return false; + + ehdr = (struct elf32_hdr *)fw->data; + + if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG)) + return false; + + if (ehdr->e_phentsize != sizeof(struct elf32_phdr)) + return false; + + phend = size_add(size_mul(sizeof(struct elf32_phdr), ehdr->e_phnum), ehdr->e_phoff); + if (phend > fw->size) + return false; + + if (ehdr->e_shentsize || ehdr->e_shnum) { + if (ehdr->e_shentsize != sizeof(struct elf32_shdr)) + return false; + + shend = size_add(size_mul(sizeof(struct elf32_shdr), ehdr->e_shnum), ehdr->e_shoff); + if (shend > fw->size) + return false; + } + + return true; +} + static bool mdt_phdr_valid(const struct elf32_phdr *phdr) { if (phdr->p_type != PT_LOAD) @@ -84,8 +117,11 @@ ssize_t qcom_mdt_get_size(const struct firmware *fw) phys_addr_t max_addr = 0; int i; + if (!mdt_header_valid(fw)) + return -EINVAL; + ehdr = (struct elf32_hdr *)fw->data; - phdrs = (struct elf32_phdr *)(ehdr + 1); + phdrs = (struct elf32_phdr *)(fw->data + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { phdr = &phdrs[i]; @@ -136,8 +172,11 @@ void *qcom_mdt_read_metadata(const struct firmware *fw, size_t *data_len, ssize_t ret; void *data; + if (!mdt_header_valid(fw)) + return ERR_PTR(-EINVAL); + ehdr = (struct elf32_hdr *)fw->data; - phdrs = (struct elf32_phdr *)(ehdr + 1); + phdrs = (struct elf32_phdr *)(fw->data + ehdr->e_phoff); if (ehdr->e_phnum < 2) return ERR_PTR(-EINVAL); @@ -216,8 +255,11 @@ int qcom_mdt_pas_init(struct device *dev, const struct firmware *fw, int ret; int i; + if (!mdt_header_valid(fw)) + return -EINVAL; + ehdr = (struct elf32_hdr *)fw->data; - phdrs = (struct elf32_phdr *)(ehdr + 1); + phdrs = (struct elf32_phdr *)(fw->data + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { phdr = &phdrs[i]; @@ -272,7 +314,7 @@ static bool qcom_mdt_bins_are_split(const struct firmware *fw, const char *fw_na int i; ehdr = (struct elf32_hdr *)fw->data; - phdrs = (struct elf32_phdr *)(ehdr + 1); + phdrs = (struct elf32_phdr *)(fw->data + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { /* @@ -312,9 +354,12 @@ static int __qcom_mdt_load(struct device *dev, const struct firmware *fw, if (!fw || !mem_region || !mem_phys || !mem_size) return -EINVAL; + if (!mdt_header_valid(fw)) + return -EINVAL; + is_split = qcom_mdt_bins_are_split(fw, fw_name); ehdr = (struct elf32_hdr *)fw->data; - phdrs = (struct elf32_phdr *)(ehdr + 1); + phdrs = (struct elf32_phdr *)(fw->data + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { phdr = &phdrs[i]; diff --git a/drivers/soc/qcom/pmic_glink.c b/drivers/soc/qcom/pmic_glink.c index 7e50eeb4cc6f2..2222ca4fa6e21 100644 --- a/drivers/soc/qcom/pmic_glink.c +++ b/drivers/soc/qcom/pmic_glink.c @@ -145,7 +145,10 @@ static int pmic_glink_rpmsg_callback(struct rpmsg_device *rpdev, void *data, return 0; } -static void pmic_glink_aux_release(struct device *dev) {} +static void pmic_glink_aux_release(struct device *dev) +{ + of_node_put(dev->of_node); +} static int pmic_glink_add_aux_device(struct pmic_glink *pg, struct auxiliary_device *aux, @@ -159,8 +162,10 @@ static int pmic_glink_add_aux_device(struct pmic_glink *pg, aux->dev.release = pmic_glink_aux_release; device_set_of_node_from_dev(&aux->dev, parent); ret = auxiliary_device_init(aux); - if (ret) + if (ret) { + of_node_put(aux->dev.of_node); return ret; + } ret = auxiliary_device_add(aux); if (ret) diff --git a/drivers/soc/qcom/qmi_encdec.c b/drivers/soc/qcom/qmi_encdec.c index 5c7161b18b724..645c4ee24f5b4 100644 --- a/drivers/soc/qcom/qmi_encdec.c +++ b/drivers/soc/qcom/qmi_encdec.c @@ -304,6 +304,8 @@ static int qmi_encode(const struct qmi_elem_info *ei_array, void *out_buf, const void *buf_src; int encode_tlv = 0; int rc; + u8 val8; + u16 val16; if (!ei_array) return 0; @@ -338,7 +340,6 @@ static int qmi_encode(const struct qmi_elem_info *ei_array, void *out_buf, break; case QMI_DATA_LEN: - memcpy(&data_len_value, buf_src, temp_ei->elem_size); data_len_sz = temp_ei->elem_size == sizeof(u8) ? sizeof(u8) : sizeof(u16); /* Check to avoid out of range buffer access */ @@ -348,8 +349,17 @@ static int qmi_encode(const struct qmi_elem_info *ei_array, void *out_buf, __func__); return -ETOOSMALL; } - rc = qmi_encode_basic_elem(buf_dst, &data_len_value, - 1, data_len_sz); + if (data_len_sz == sizeof(u8)) { + val8 = *(u8 *)buf_src; + data_len_value = (u32)val8; + rc = qmi_encode_basic_elem(buf_dst, &val8, + 1, data_len_sz); + } else { + val16 = *(u16 *)buf_src; + data_len_value = (u32)le16_to_cpu(val16); + rc = qmi_encode_basic_elem(buf_dst, &val16, + 1, data_len_sz); + } UPDATE_ENCODE_VARIABLES(temp_ei, buf_dst, encoded_bytes, tlv_len, encode_tlv, rc); @@ -523,14 +533,23 @@ static int qmi_decode_string_elem(const struct qmi_elem_info *ei_array, u32 string_len = 0; u32 string_len_sz = 0; const struct qmi_elem_info *temp_ei = ei_array; + u8 val8; + u16 val16; if (dec_level == 1) { string_len = tlv_len; } else { string_len_sz = temp_ei->elem_len <= U8_MAX ? sizeof(u8) : sizeof(u16); - rc = qmi_decode_basic_elem(&string_len, buf_src, - 1, string_len_sz); + if (string_len_sz == sizeof(u8)) { + rc = qmi_decode_basic_elem(&val8, buf_src, + 1, string_len_sz); + string_len = (u32)val8; + } else { + rc = qmi_decode_basic_elem(&val16, buf_src, + 1, string_len_sz); + string_len = (u32)val16; + } decoded_bytes += rc; } @@ -604,6 +623,9 @@ static int qmi_decode(const struct qmi_elem_info *ei_array, void *out_c_struct, u32 decoded_bytes = 0; const void *buf_src = in_buf; int rc; + u8 val8; + u16 val16; + u32 val32; while (decoded_bytes < in_buf_len) { if (dec_level >= 2 && temp_ei->data_type == QMI_EOTI) @@ -642,9 +664,17 @@ static int qmi_decode(const struct qmi_elem_info *ei_array, void *out_c_struct, if (temp_ei->data_type == QMI_DATA_LEN) { data_len_sz = temp_ei->elem_size == sizeof(u8) ? sizeof(u8) : sizeof(u16); - rc = qmi_decode_basic_elem(&data_len_value, buf_src, - 1, data_len_sz); - memcpy(buf_dst, &data_len_value, sizeof(u32)); + if (data_len_sz == sizeof(u8)) { + rc = qmi_decode_basic_elem(&val8, buf_src, + 1, data_len_sz); + data_len_value = (u32)val8; + } else { + rc = qmi_decode_basic_elem(&val16, buf_src, + 1, data_len_sz); + data_len_value = (u32)val16; + } + val32 = cpu_to_le32(data_len_value); + memcpy(buf_dst, &val32, sizeof(u32)); temp_ei = temp_ei + 1; buf_dst = out_c_struct + temp_ei->offset; tlv_len -= data_len_sz; diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index dfc2d4e38fa9b..d0ec42ec0d041 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -453,13 +453,10 @@ static irqreturn_t tcs_tx_done(int irq, void *p) trace_rpmh_tx_done(drv, i, req); - /* - * If wake tcs was re-purposed for sending active - * votes, clear AMC trigger & enable modes and + /* Clear AMC trigger & enable modes and * disable interrupt for this TCS */ - if (!drv->tcs[ACTIVE_TCS].num_tcs) - __tcs_set_trigger(drv, i, false); + __tcs_set_trigger(drv, i, false); skip: /* Reclaim the TCS */ write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0); @@ -1075,7 +1072,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); drv->ver.minor >>= MINOR_VER_SHIFT; - if (drv->ver.major == 3) + if (drv->ver.major >= 3) drv->regs = rpmh_rsc_reg_offset_ver_3_0; else drv->regs = rpmh_rsc_reg_offset_ver_2_7; diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c index 2e8568d6cde94..aead7dd482ea3 100644 --- a/drivers/soc/qcom/smem.c +++ b/drivers/soc/qcom/smem.c @@ -870,7 +870,7 @@ static u32 qcom_smem_get_item_count(struct qcom_smem *smem) if (IS_ERR_OR_NULL(ptable)) return SMEM_ITEM_COUNT; - info = (struct smem_info *)&ptable->entry[ptable->num_entries]; + info = (struct smem_info *)&ptable->entry[le32_to_cpu(ptable->num_entries)]; if (memcmp(info->magic, SMEM_INFO_MAGIC, sizeof(info->magic))) return SMEM_ITEM_COUNT; diff --git a/drivers/soc/tegra/cbb/tegra234-cbb.c b/drivers/soc/tegra/cbb/tegra234-cbb.c index 5cf0e8c341644..e8cc46874c729 100644 --- a/drivers/soc/tegra/cbb/tegra234-cbb.c +++ b/drivers/soc/tegra/cbb/tegra234-cbb.c @@ -185,6 +185,8 @@ static void tegra234_cbb_error_clear(struct tegra_cbb *cbb) { struct tegra234_cbb *priv = to_tegra234_cbb(cbb); + writel(0, priv->mon + FABRIC_MN_MASTER_ERR_FORCE_0); + writel(0x3f, priv->mon + FABRIC_MN_MASTER_ERR_STATUS_0); dsb(sy); } diff --git a/drivers/soc/tegra/fuse/fuse-tegra30.c b/drivers/soc/tegra/fuse/fuse-tegra30.c index 402cf939c0326..a6b3217c3ccef 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra30.c +++ b/drivers/soc/tegra/fuse/fuse-tegra30.c @@ -116,6 +116,124 @@ const struct tegra_fuse_soc tegra30_fuse_soc = { #endif #ifdef CONFIG_ARCH_TEGRA_114_SOC +static const struct nvmem_cell_info tegra114_fuse_cells[] = { + { + .name = "tsensor-cpu1", + .offset = 0x084, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-cpu2", + .offset = 0x088, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-common", + .offset = 0x08c, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-cpu0", + .offset = 0x098, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "xusb-pad-calibration", + .offset = 0x0f0, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-cpu3", + .offset = 0x12c, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-gpu", + .offset = 0x154, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-mem0", + .offset = 0x158, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-mem1", + .offset = 0x15c, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, { + .name = "tsensor-pllx", + .offset = 0x160, + .bytes = 4, + .bit_offset = 0, + .nbits = 32, + }, +}; + +static const struct nvmem_cell_lookup tegra114_fuse_lookups[] = { + { + .nvmem_name = "fuse", + .cell_name = "xusb-pad-calibration", + .dev_id = "7009f000.padctl", + .con_id = "calibration", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-common", + .dev_id = "700e2000.thermal-sensor", + .con_id = "common", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-cpu0", + .dev_id = "700e2000.thermal-sensor", + .con_id = "cpu0", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-cpu1", + .dev_id = "700e2000.thermal-sensor", + .con_id = "cpu1", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-cpu2", + .dev_id = "700e2000.thermal-sensor", + .con_id = "cpu2", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-cpu3", + .dev_id = "700e2000.thermal-sensor", + .con_id = "cpu3", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-mem0", + .dev_id = "700e2000.thermal-sensor", + .con_id = "mem0", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-mem1", + .dev_id = "700e2000.thermal-sensor", + .con_id = "mem1", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-gpu", + .dev_id = "700e2000.thermal-sensor", + .con_id = "gpu", + }, { + .nvmem_name = "fuse", + .cell_name = "tsensor-pllx", + .dev_id = "700e2000.thermal-sensor", + .con_id = "pllx", + }, +}; + static const struct tegra_fuse_info tegra114_fuse_info = { .read = tegra30_fuse_read, .size = 0x2a0, @@ -126,6 +244,10 @@ const struct tegra_fuse_soc tegra114_fuse_soc = { .init = tegra30_fuse_init, .speedo_init = tegra114_init_speedo_data, .info = &tegra114_fuse_info, + .lookups = tegra114_fuse_lookups, + .num_lookups = ARRAY_SIZE(tegra114_fuse_lookups), + .cells = tegra114_fuse_cells, + .num_cells = ARRAY_SIZE(tegra114_fuse_cells), .soc_attr_group = &tegra_soc_attr_group, .clk_suspend_on = false, }; diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index 162f52456f654..1306e3b8b5c04 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -1232,7 +1232,7 @@ static int tegra_powergate_of_get_clks(struct tegra_powergate *pg, } static int tegra_powergate_of_get_resets(struct tegra_powergate *pg, - struct device_node *np, bool off) + struct device_node *np) { struct device *dev = pg->pmc->dev; int err; @@ -1247,22 +1247,6 @@ static int tegra_powergate_of_get_resets(struct tegra_powergate *pg, err = reset_control_acquire(pg->reset); if (err < 0) { pr_err("failed to acquire resets: %d\n", err); - goto out; - } - - if (off) { - err = reset_control_assert(pg->reset); - } else { - err = reset_control_deassert(pg->reset); - if (err < 0) - goto out; - - reset_control_release(pg->reset); - } - -out: - if (err) { - reset_control_release(pg->reset); reset_control_put(pg->reset); } @@ -1307,20 +1291,43 @@ static int tegra_powergate_add(struct tegra_pmc *pmc, struct device_node *np) goto set_available; } - err = tegra_powergate_of_get_resets(pg, np, off); + err = tegra_powergate_of_get_resets(pg, np); if (err < 0) { dev_err(dev, "failed to get resets for %pOFn: %d\n", np, err); goto remove_clks; } - if (!IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS)) { - if (off) - WARN_ON(tegra_powergate_power_up(pg, true)); + /* + * If the power-domain is off, then ensure the resets are asserted. + * If the power-domain is on, then power down to ensure that when is + * it turned on the power-domain, clocks and resets are all in the + * expected state. + */ + if (off) { + err = reset_control_assert(pg->reset); + if (err) { + pr_err("failed to assert resets: %d\n", err); + goto remove_resets; + } + } else { + err = tegra_powergate_power_down(pg); + if (err) { + dev_err(dev, "failed to turn off PM domain %s: %d\n", + pg->genpd.name, err); + goto remove_resets; + } + } + /* + * If PM_GENERIC_DOMAINS is not enabled, power-on + * the domain and skip the genpd registration. + */ + if (!IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS)) { + WARN_ON(tegra_powergate_power_up(pg, true)); goto remove_resets; } - err = pm_genpd_init(&pg->genpd, NULL, off); + err = pm_genpd_init(&pg->genpd, NULL, true); if (err < 0) { dev_err(dev, "failed to initialise PM domain %pOFn: %d\n", np, err); diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c index 6ea9b8c7d335c..7a3bdef5a7c0d 100644 --- a/drivers/soc/ti/k3-socinfo.c +++ b/drivers/soc/ti/k3-socinfo.c @@ -63,6 +63,12 @@ k3_chipinfo_partno_to_names(unsigned int partno, return -EINVAL; } +static const struct regmap_config k3_chipinfo_regmap_cfg = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + static int k3_chipinfo_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; @@ -70,13 +76,18 @@ static int k3_chipinfo_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct soc_device *soc_dev; struct regmap *regmap; + void __iomem *base; u32 partno_id; u32 variant; u32 jtag_id; u32 mfg; int ret; - regmap = device_node_to_regmap(node); + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + regmap = regmap_init_mmio(dev, base, &k3_chipinfo_regmap_cfg); if (IS_ERR(regmap)) return PTR_ERR(regmap); diff --git a/drivers/soc/ti/knav_dma.c b/drivers/soc/ti/knav_dma.c index 0fbc37cd51231..f599eaab3c562 100644 --- a/drivers/soc/ti/knav_dma.c +++ b/drivers/soc/ti/knav_dma.c @@ -402,7 +402,7 @@ static int of_channel_match_helper(struct device_node *np, const char *name, * @name: slave channel name * @config: dma configuration parameters * - * Returns pointer to appropriate DMA channel on success or error. + * Return: Pointer to appropriate DMA channel on success or NULL on error. */ void *knav_dma_open_channel(struct device *dev, const char *name, struct knav_dma_cfg *config) @@ -414,13 +414,13 @@ void *knav_dma_open_channel(struct device *dev, const char *name, if (!kdev) { pr_err("keystone-navigator-dma driver not registered\n"); - return (void *)-EINVAL; + return NULL; } chan_num = of_channel_match_helper(dev->of_node, name, &instance); if (chan_num < 0) { dev_err(kdev->dev, "No DMA instance with name %s\n", name); - return (void *)-EINVAL; + return NULL; } dev_dbg(kdev->dev, "initializing %s channel %d from DMA %s\n", @@ -431,7 +431,7 @@ void *knav_dma_open_channel(struct device *dev, const char *name, if (config->direction != DMA_MEM_TO_DEV && config->direction != DMA_DEV_TO_MEM) { dev_err(kdev->dev, "bad direction\n"); - return (void *)-EINVAL; + return NULL; } /* Look for correct dma instance */ @@ -443,7 +443,7 @@ void *knav_dma_open_channel(struct device *dev, const char *name, } if (!dma) { dev_err(kdev->dev, "No DMA instance with name %s\n", instance); - return (void *)-EINVAL; + return NULL; } /* Look for correct dma channel from dma instance */ @@ -463,14 +463,14 @@ void *knav_dma_open_channel(struct device *dev, const char *name, if (!chan) { dev_err(kdev->dev, "channel %d is not in DMA %s\n", chan_num, instance); - return (void *)-EINVAL; + return NULL; } if (atomic_read(&chan->ref_count) >= 1) { if (!check_config(chan, config)) { dev_err(kdev->dev, "channel %d config miss-match\n", chan_num); - return (void *)-EINVAL; + return NULL; } } diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index 79173ab540a6b..3d8937245c180 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -205,7 +205,7 @@ static u64 amd_sdw_send_cmd_get_resp(struct amd_sdw_manager *amd_manager, u32 lo if (sts & AMD_SDW_IMM_RES_VALID) { dev_err(amd_manager->dev, "SDW%x manager is in bad state\n", amd_manager->instance); - writel(0x00, amd_manager->mmio + ACP_SW_IMM_CMD_STS); + writel(AMD_SDW_IMM_RES_VALID, amd_manager->mmio + ACP_SW_IMM_CMD_STS); } writel(upper_data, amd_manager->mmio + ACP_SW_IMM_CMD_UPPER_WORD); writel(lower_data, amd_manager->mmio + ACP_SW_IMM_CMD_LOWER_QWORD); @@ -1104,10 +1104,10 @@ static int __maybe_unused amd_pm_prepare(struct device *dev) * device is not in runtime suspend state, observed that device alerts are missing * without pm_prepare on AMD platforms in clockstop mode0. */ - if (amd_manager->power_mode_mask & AMD_SDW_CLK_STOP_MODE) { - ret = pm_request_resume(dev); + if (amd_manager->power_mode_mask) { + ret = pm_runtime_resume(dev); if (ret < 0) { - dev_err(bus->dev, "pm_request_resume failed: %d\n", ret); + dev_err(bus->dev, "pm_runtime_resume failed: %d\n", ret); return 0; } } @@ -1135,9 +1135,12 @@ static int __maybe_unused amd_suspend(struct device *dev) } if (amd_manager->power_mode_mask & AMD_SDW_CLK_STOP_MODE) { + cancel_work_sync(&amd_manager->amd_sdw_work); amd_sdw_wake_enable(amd_manager, false); return amd_sdw_clock_stop(amd_manager); } else if (amd_manager->power_mode_mask & AMD_SDW_POWER_OFF_MODE) { + cancel_work_sync(&amd_manager->amd_sdw_work); + amd_sdw_wake_enable(amd_manager, false); /* * As per hardware programming sequence on AMD platforms, * clock stop should be invoked first before powering-off @@ -1165,6 +1168,7 @@ static int __maybe_unused amd_suspend_runtime(struct device *dev) amd_sdw_wake_enable(amd_manager, true); return amd_sdw_clock_stop(amd_manager); } else if (amd_manager->power_mode_mask & AMD_SDW_POWER_OFF_MODE) { + amd_sdw_wake_enable(amd_manager, true); ret = amd_sdw_clock_stop(amd_manager); if (ret) return ret; diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index e7553c38be59d..e7397fd8e9ad9 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -121,6 +121,10 @@ int sdw_bus_master_add(struct sdw_bus *bus, struct device *parent, set_bit(SDW_GROUP13_DEV_NUM, bus->assigned); set_bit(SDW_MASTER_DEV_NUM, bus->assigned); + ret = sdw_irq_create(bus, fwnode); + if (ret) + return ret; + /* * SDW is an enumerable bus, but devices can be powered off. So, * they won't be able to report as present. @@ -137,6 +141,7 @@ int sdw_bus_master_add(struct sdw_bus *bus, struct device *parent, if (ret < 0) { dev_err(bus->dev, "Finding slaves failed:%d\n", ret); + sdw_irq_delete(bus); return ret; } @@ -155,10 +160,6 @@ int sdw_bus_master_add(struct sdw_bus *bus, struct device *parent, bus->params.curr_bank = SDW_BANK0; bus->params.next_bank = SDW_BANK1; - ret = sdw_irq_create(bus, fwnode); - if (ret) - return ret; - return 0; } EXPORT_SYMBOL(sdw_bus_master_add); @@ -1729,15 +1730,15 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) /* Update the Slave driver */ if (slave_notify) { + if (slave->prop.use_domain_irq && slave->irq) + handle_nested_irq(slave->irq); + mutex_lock(&slave->sdw_dev_lock); if (slave->probed) { struct device *dev = &slave->dev; struct sdw_driver *drv = drv_to_sdw_driver(dev->driver); - if (slave->prop.use_domain_irq && slave->irq) - handle_nested_irq(slave->irq); - if (drv->ops && drv->ops->interrupt_callback) { slave_intr.sdca_cascade = sdca_cascade; slave_intr.control_port = clear; diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c index 68d54887992d9..8ebfa44078e85 100644 --- a/drivers/soundwire/stream.c +++ b/drivers/soundwire/stream.c @@ -1409,7 +1409,7 @@ static int _sdw_prepare_stream(struct sdw_stream_runtime *stream, if (ret < 0) { dev_err(bus->dev, "Prepare port(s) failed ret = %d\n", ret); - return ret; + goto restore_params; } } diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c index 1ca857c2a4aa3..8df12efeea21c 100644 --- a/drivers/spi/spi-bcm63xx-hsspi.c +++ b/drivers/spi/spi-bcm63xx-hsspi.c @@ -745,7 +745,7 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev) if (IS_ERR(clk)) return PTR_ERR(clk); - reset = devm_reset_control_get_optional_exclusive(dev, NULL); + reset = devm_reset_control_get_optional_shared(dev, NULL); if (IS_ERR(reset)) return PTR_ERR(reset); diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c index ef3a7226db125..a95badb7b7114 100644 --- a/drivers/spi/spi-bcm63xx.c +++ b/drivers/spi/spi-bcm63xx.c @@ -523,7 +523,7 @@ static int bcm63xx_spi_probe(struct platform_device *pdev) return PTR_ERR(clk); } - reset = devm_reset_control_get_optional_exclusive(dev, NULL); + reset = devm_reset_control_get_optional_shared(dev, NULL); if (IS_ERR(reset)) return PTR_ERR(reset); diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index bf9b816637d02..7b809644436eb 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -687,6 +687,7 @@ static int cqspi_read_setup(struct cqspi_flash_pdata *f_pdata, reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK; reg |= (op->addr.nbytes - 1); writel(reg, reg_base + CQSPI_REG_SIZE); + readl(reg_base + CQSPI_REG_SIZE); /* Flush posted write. */ return 0; } @@ -726,6 +727,7 @@ static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata, reinit_completion(&cqspi->transfer_complete); writel(CQSPI_REG_INDIRECTRD_START_MASK, reg_base + CQSPI_REG_INDIRECTRD); + readl(reg_base + CQSPI_REG_INDIRECTRD); /* Flush posted write. */ while (remaining > 0) { if (!wait_for_completion_timeout(&cqspi->transfer_complete, @@ -1004,6 +1006,7 @@ static int cqspi_write_setup(struct cqspi_flash_pdata *f_pdata, reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK; reg |= (op->addr.nbytes - 1); writel(reg, reg_base + CQSPI_REG_SIZE); + readl(reg_base + CQSPI_REG_SIZE); /* Flush posted write. */ return 0; } @@ -1029,6 +1032,8 @@ static int cqspi_indirect_write_execute(struct cqspi_flash_pdata *f_pdata, reinit_completion(&cqspi->transfer_complete); writel(CQSPI_REG_INDIRECTWR_START_MASK, reg_base + CQSPI_REG_INDIRECTWR); + readl(reg_base + CQSPI_REG_INDIRECTWR); /* Flush posted write. */ + /* * As per 66AK2G02 TRM SPRUHY8F section 11.15.5.3 Indirect Access * Controller programming sequence, couple of cycles of diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c index bcb0de864d34d..3206c84c6f22f 100644 --- a/drivers/spi/spi-fsl-dspi.c +++ b/drivers/spi/spi-fsl-dspi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ // // Copyright 2013 Freescale Semiconductor, Inc. -// Copyright 2020 NXP +// Copyright 2020-2025 NXP // // Freescale DSPI driver // This file contains a driver for the Freescale DSPI @@ -62,6 +62,7 @@ #define SPI_SR_TFIWF BIT(18) #define SPI_SR_RFDF BIT(17) #define SPI_SR_CMDFFF BIT(16) +#define SPI_SR_TXRXS BIT(30) #define SPI_SR_CLEAR (SPI_SR_TCFQF | \ SPI_SR_TFUF | SPI_SR_TFFF | \ SPI_SR_CMDTCF | SPI_SR_SPEF | \ @@ -926,9 +927,20 @@ static int dspi_transfer_one_message(struct spi_controller *ctlr, struct spi_transfer *transfer; bool cs = false; int status = 0; + u32 val = 0; + bool cs_change = false; message->actual_length = 0; + /* Put DSPI in running mode if halted. */ + regmap_read(dspi->regmap, SPI_MCR, &val); + if (val & SPI_MCR_HALT) { + regmap_update_bits(dspi->regmap, SPI_MCR, SPI_MCR_HALT, 0); + while (regmap_read(dspi->regmap, SPI_SR, &val) >= 0 && + !(val & SPI_SR_TXRXS)) + ; + } + list_for_each_entry(transfer, &message->transfers, transfer_list) { dspi->cur_transfer = transfer; dspi->cur_msg = message; @@ -958,6 +970,7 @@ static int dspi_transfer_one_message(struct spi_controller *ctlr, dspi->tx_cmd |= SPI_PUSHR_CMD_CONT; } + cs_change = transfer->cs_change; dspi->tx = transfer->tx_buf; dspi->rx = transfer->rx_buf; dspi->len = transfer->len; @@ -967,17 +980,28 @@ static int dspi_transfer_one_message(struct spi_controller *ctlr, SPI_MCR_CLR_TXF | SPI_MCR_CLR_RXF, SPI_MCR_CLR_TXF | SPI_MCR_CLR_RXF); + regmap_write(dspi->regmap, SPI_SR, SPI_SR_CLEAR); + spi_take_timestamp_pre(dspi->ctlr, dspi->cur_transfer, dspi->progress, !dspi->irq); if (dspi->devtype_data->trans_mode == DSPI_DMA_MODE) { status = dspi_dma_xfer(dspi); } else { + /* + * Reinitialize the completion before transferring data + * to avoid the case where it might remain in the done + * state due to a spurious interrupt from a previous + * transfer. This could falsely signal that the current + * transfer has completed. + */ + if (dspi->irq) + reinit_completion(&dspi->xfer_done); + dspi_fifo_write(dspi); if (dspi->irq) { wait_for_completion(&dspi->xfer_done); - reinit_completion(&dspi->xfer_done); } else { do { status = dspi_poll(dspi); @@ -993,6 +1017,15 @@ static int dspi_transfer_one_message(struct spi_controller *ctlr, dspi_deassert_cs(spi, &cs); } + if (status || !cs_change) { + /* Put DSPI in stop mode */ + regmap_update_bits(dspi->regmap, SPI_MCR, + SPI_MCR_HALT, SPI_MCR_HALT); + while (regmap_read(dspi->regmap, SPI_SR, &val) >= 0 && + val & SPI_SR_TXRXS) + ; + } + message->status = status; spi_finalize_current_message(ctlr); @@ -1163,6 +1196,20 @@ static int dspi_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(dspi_pm, dspi_suspend, dspi_resume); +static const struct regmap_range dspi_yes_ranges[] = { + regmap_reg_range(SPI_MCR, SPI_MCR), + regmap_reg_range(SPI_TCR, SPI_CTAR(3)), + regmap_reg_range(SPI_SR, SPI_TXFR3), + regmap_reg_range(SPI_RXFR0, SPI_RXFR3), + regmap_reg_range(SPI_CTARE(0), SPI_CTARE(3)), + regmap_reg_range(SPI_SREX, SPI_SREX), +}; + +static const struct regmap_access_table dspi_access_table = { + .yes_ranges = dspi_yes_ranges, + .n_yes_ranges = ARRAY_SIZE(dspi_yes_ranges), +}; + static const struct regmap_range dspi_volatile_ranges[] = { regmap_reg_range(SPI_MCR, SPI_TCR), regmap_reg_range(SPI_SR, SPI_SR), @@ -1180,6 +1227,8 @@ static const struct regmap_config dspi_regmap_config = { .reg_stride = 4, .max_register = 0x88, .volatile_table = &dspi_volatile_table, + .rd_table = &dspi_access_table, + .wr_table = &dspi_access_table, }; static const struct regmap_range dspi_xspi_volatile_ranges[] = { @@ -1201,6 +1250,8 @@ static const struct regmap_config dspi_xspi_regmap_config[] = { .reg_stride = 4, .max_register = 0x13c, .volatile_table = &dspi_xspi_volatile_table, + .rd_table = &dspi_access_table, + .wr_table = &dspi_access_table, }, { .name = "pushr", @@ -1223,6 +1274,8 @@ static int dspi_init(struct fsl_dspi *dspi) if (!spi_controller_is_target(dspi->ctlr)) mcr |= SPI_MCR_HOST; + mcr |= SPI_MCR_HALT; + regmap_write(dspi->regmap, SPI_MCR, mcr); regmap_write(dspi->regmap, SPI_SR, SPI_SR_CLEAR); diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index 9e2541dee56e5..8ef82a11ebb0f 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -3,8 +3,9 @@ // Freescale i.MX7ULP LPSPI driver // // Copyright 2016 Freescale Semiconductor, Inc. -// Copyright 2018 NXP Semiconductors +// Copyright 2018, 2023, 2025 NXP +#include #include #include #include @@ -70,7 +71,7 @@ #define DER_TDDE BIT(0) #define CFGR1_PCSCFG BIT(27) #define CFGR1_PINCFG (BIT(24)|BIT(25)) -#define CFGR1_PCSPOL BIT(8) +#define CFGR1_PCSPOL_MASK GENMASK(11, 8) #define CFGR1_NOSTALL BIT(3) #define CFGR1_HOST BIT(0) #define FSR_TXCOUNT (0xFF) @@ -82,6 +83,8 @@ #define TCR_RXMSK BIT(19) #define TCR_TXMSK BIT(18) +#define SR_CLEAR_MASK GENMASK(13, 8) + struct fsl_lpspi_devtype_data { u8 prescale_max; }; @@ -330,13 +333,11 @@ static int fsl_lpspi_set_bitrate(struct fsl_lpspi_data *fsl_lpspi) } if (config.speed_hz > perclk_rate / 2) { - dev_err(fsl_lpspi->dev, - "per-clk should be at least two times of transfer speed"); - return -EINVAL; + div = 2; + } else { + div = DIV_ROUND_UP(perclk_rate, config.speed_hz); } - div = DIV_ROUND_UP(perclk_rate, config.speed_hz); - for (prescale = 0; prescale <= prescale_max; prescale++) { scldiv = div / (1 << prescale) - 2; if (scldiv >= 0 && scldiv < 256) { @@ -422,7 +423,9 @@ static int fsl_lpspi_config(struct fsl_lpspi_data *fsl_lpspi) else temp = CFGR1_PINCFG; if (fsl_lpspi->config.mode & SPI_CS_HIGH) - temp |= CFGR1_PCSPOL; + temp |= FIELD_PREP(CFGR1_PCSPOL_MASK, + BIT(fsl_lpspi->config.chip_select)); + writel(temp, fsl_lpspi->base + IMX7ULP_CFGR1); temp = readl(fsl_lpspi->base + IMX7ULP_CR); @@ -531,14 +534,13 @@ static int fsl_lpspi_reset(struct fsl_lpspi_data *fsl_lpspi) fsl_lpspi_intctrl(fsl_lpspi, 0); } - /* W1C for all flags in SR */ - temp = 0x3F << 8; - writel(temp, fsl_lpspi->base + IMX7ULP_SR); - /* Clear FIFO and disable module */ temp = CR_RRF | CR_RTF; writel(temp, fsl_lpspi->base + IMX7ULP_CR); + /* W1C for all flags in SR */ + writel(SR_CLEAR_MASK, fsl_lpspi->base + IMX7ULP_SR); + return 0; } @@ -729,12 +731,10 @@ static int fsl_lpspi_pio_transfer(struct spi_controller *controller, fsl_lpspi_write_tx_fifo(fsl_lpspi); ret = fsl_lpspi_wait_for_completion(controller); - if (ret) - return ret; fsl_lpspi_reset(fsl_lpspi); - return 0; + return ret; } static int fsl_lpspi_transfer_one(struct spi_controller *controller, @@ -782,7 +782,7 @@ static irqreturn_t fsl_lpspi_isr(int irq, void *dev_id) if (temp_SR & SR_MBF || readl(fsl_lpspi->base + IMX7ULP_FSR) & FSR_TXCOUNT) { writel(SR_FCF, fsl_lpspi->base + IMX7ULP_SR); - fsl_lpspi_intctrl(fsl_lpspi, IER_FCIE); + fsl_lpspi_intctrl(fsl_lpspi, IER_FCIE | (temp_IER & IER_TDIE)); return IRQ_HANDLED; } diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c index 79bac30e79af6..21e357966d2a2 100644 --- a/drivers/spi/spi-fsl-qspi.c +++ b/drivers/spi/spi-fsl-qspi.c @@ -839,6 +839,19 @@ static const struct spi_controller_mem_ops fsl_qspi_mem_ops = { .get_name = fsl_qspi_get_name, }; +static void fsl_qspi_cleanup(void *data) +{ + struct fsl_qspi *q = data; + + /* disable the hardware */ + qspi_writel(q, QUADSPI_MCR_MDIS_MASK, q->iobase + QUADSPI_MCR); + qspi_writel(q, 0x0, q->iobase + QUADSPI_RSER); + + fsl_qspi_clk_disable_unprep(q); + + mutex_destroy(&q->lock); +} + static int fsl_qspi_probe(struct platform_device *pdev) { struct spi_controller *ctlr; @@ -928,15 +941,16 @@ static int fsl_qspi_probe(struct platform_device *pdev) ctlr->dev.of_node = np; + ret = devm_add_action_or_reset(dev, fsl_qspi_cleanup, q); + if (ret) + goto err_put_ctrl; + ret = devm_spi_register_controller(dev, ctlr); if (ret) - goto err_destroy_mutex; + goto err_put_ctrl; return 0; -err_destroy_mutex: - mutex_destroy(&q->lock); - err_disable_clk: fsl_qspi_clk_disable_unprep(q); @@ -947,19 +961,6 @@ static int fsl_qspi_probe(struct platform_device *pdev) return ret; } -static void fsl_qspi_remove(struct platform_device *pdev) -{ - struct fsl_qspi *q = platform_get_drvdata(pdev); - - /* disable the hardware */ - qspi_writel(q, QUADSPI_MCR_MDIS_MASK, q->iobase + QUADSPI_MCR); - qspi_writel(q, 0x0, q->iobase + QUADSPI_RSER); - - fsl_qspi_clk_disable_unprep(q); - - mutex_destroy(&q->lock); -} - static int fsl_qspi_suspend(struct device *dev) { return 0; @@ -997,7 +998,6 @@ static struct platform_driver fsl_qspi_driver = { .pm = &fsl_qspi_pm_ops, }, .probe = fsl_qspi_probe, - .remove_new = fsl_qspi_remove, }; module_platform_driver(fsl_qspi_driver); diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c index bbf2015d8e5cc..41e1fb27a9f34 100644 --- a/drivers/spi/spi-loopback-test.c +++ b/drivers/spi/spi-loopback-test.c @@ -421,7 +421,7 @@ MODULE_LICENSE("GPL"); static void spi_test_print_hex_dump(char *pre, const void *ptr, size_t len) { /* limit the hex_dump */ - if (len < 1024) { + if (len <= 1024) { print_hex_dump(KERN_INFO, pre, DUMP_PREFIX_OFFSET, 16, 1, ptr, len, 0); @@ -447,7 +447,7 @@ static void spi_test_dump_message(struct spi_device *spi, int i; u8 b; - dev_info(&spi->dev, " spi_msg@%pK\n", msg); + dev_info(&spi->dev, " spi_msg@%p\n", msg); if (msg->status) dev_info(&spi->dev, " status: %i\n", msg->status); @@ -457,15 +457,15 @@ static void spi_test_dump_message(struct spi_device *spi, msg->actual_length); list_for_each_entry(xfer, &msg->transfers, transfer_list) { - dev_info(&spi->dev, " spi_transfer@%pK\n", xfer); + dev_info(&spi->dev, " spi_transfer@%p\n", xfer); dev_info(&spi->dev, " len: %i\n", xfer->len); - dev_info(&spi->dev, " tx_buf: %pK\n", xfer->tx_buf); + dev_info(&spi->dev, " tx_buf: %p\n", xfer->tx_buf); if (dump_data && xfer->tx_buf) spi_test_print_hex_dump(" TX: ", xfer->tx_buf, xfer->len); - dev_info(&spi->dev, " rx_buf: %pK\n", xfer->rx_buf); + dev_info(&spi->dev, " rx_buf: %p\n", xfer->rx_buf); if (dump_data && xfer->rx_buf) spi_test_print_hex_dump(" RX: ", xfer->rx_buf, @@ -559,7 +559,7 @@ static int spi_check_rx_ranges(struct spi_device *spi, /* if still not found then something has modified too much */ /* we could list the "closest" transfer here... */ dev_err(&spi->dev, - "loopback strangeness - rx changed outside of allowed range at: %pK\n", + "loopback strangeness - rx changed outside of allowed range at: %p\n", addr); /* do not return, only set ret, * so that we list all addresses @@ -697,7 +697,7 @@ static int spi_test_translate(struct spi_device *spi, } dev_err(&spi->dev, - "PointerRange [%pK:%pK[ not in range [%pK:%pK[ or [%pK:%pK[\n", + "PointerRange [%p:%p[ not in range [%p:%p[ or [%p:%p[\n", *ptr, *ptr + len, RX(0), RX(SPI_TEST_MAX_SIZE), TX(0), TX(SPI_TEST_MAX_SIZE)); diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index bc6c086ddd43f..731504ec7ef8b 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -665,6 +665,12 @@ static void nxp_fspi_dll_calibration(struct nxp_fspi *f) 0, POLL_TOUT, true); if (ret) dev_warn(f->dev, "DLL lock failed, please fix it!\n"); + + /* + * For ERR050272, DLL lock status bit is not accurate, + * wait for 4us more as a workaround. + */ + udelay(4); } /* diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c index 1f374cf4d6f65..1615f935c8f03 100644 --- a/drivers/spi/spi-rockchip.c +++ b/drivers/spi/spi-rockchip.c @@ -542,7 +542,7 @@ static int rockchip_spi_config(struct rockchip_spi *rs, cr0 |= (spi->mode & 0x3U) << CR0_SCPH_OFFSET; if (spi->mode & SPI_LSB_FIRST) cr0 |= CR0_FBM_LSB << CR0_FBM_OFFSET; - if (spi->mode & SPI_CS_HIGH) + if ((spi->mode & SPI_CS_HIGH) && !(spi_get_csgpiod(spi, 0))) cr0 |= BIT(spi_get_chipselect(spi, 0)) << CR0_SOI_OFFSET; if (xfer->rx_buf && xfer->tx_buf) diff --git a/drivers/spi/spi-rpc-if.c b/drivers/spi/spi-rpc-if.c index 7cce2d2ab9ca6..a1696672d12fe 100644 --- a/drivers/spi/spi-rpc-if.c +++ b/drivers/spi/spi-rpc-if.c @@ -193,6 +193,8 @@ static int __maybe_unused rpcif_spi_resume(struct device *dev) { struct spi_controller *ctlr = dev_get_drvdata(dev); + rpcif_hw_init(dev, false); + return spi_controller_resume(ctlr); } diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c index 6f12e4fb2e2e1..65c11909659c6 100644 --- a/drivers/spi/spi-sh-msiof.c +++ b/drivers/spi/spi-sh-msiof.c @@ -918,6 +918,7 @@ static int sh_msiof_transfer_one(struct spi_controller *ctlr, void *rx_buf = t->rx_buf; unsigned int len = t->len; unsigned int bits = t->bits_per_word; + unsigned int max_wdlen = 256; unsigned int bytes_per_word; unsigned int words; int n; @@ -931,17 +932,17 @@ static int sh_msiof_transfer_one(struct spi_controller *ctlr, if (!spi_controller_is_target(p->ctlr)) sh_msiof_spi_set_clk_regs(p, t); + if (tx_buf) + max_wdlen = min(max_wdlen, p->tx_fifo_size); + if (rx_buf) + max_wdlen = min(max_wdlen, p->rx_fifo_size); + while (ctlr->dma_tx && len > 15) { /* * DMA supports 32-bit words only, hence pack 8-bit and 16-bit * words, with byte resp. word swapping. */ - unsigned int l = 0; - - if (tx_buf) - l = min(round_down(len, 4), p->tx_fifo_size * 4); - if (rx_buf) - l = min(round_down(len, 4), p->rx_fifo_size * 4); + unsigned int l = min(round_down(len, 4), max_wdlen * 4); if (bits <= 8) { copy32 = copy_bswap32; diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index 40680b5fffc9a..211d9c76665bc 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1808,9 +1808,15 @@ static int stm32_spi_probe(struct platform_device *pdev) struct resource *res; struct reset_control *rst; struct device_node *np = pdev->dev.of_node; + const struct stm32_spi_cfg *cfg; bool device_mode; int ret; - const struct stm32_spi_cfg *cfg = of_device_get_match_data(&pdev->dev); + + cfg = of_device_get_match_data(&pdev->dev); + if (!cfg) { + dev_err(&pdev->dev, "Failed to get match data for platform\n"); + return -ENODEV; + } device_mode = of_property_read_bool(np, "spi-slave"); if (!cfg->has_device_mode && device_mode) { diff --git a/drivers/spi/spi-sun4i.c b/drivers/spi/spi-sun4i.c index b8947265d329e..5b2cb225a4198 100644 --- a/drivers/spi/spi-sun4i.c +++ b/drivers/spi/spi-sun4i.c @@ -263,6 +263,9 @@ static int sun4i_spi_transfer_one(struct spi_master *master, else reg |= SUN4I_CTL_DHB; + /* Now that the settings are correct, enable the interface */ + reg |= SUN4I_CTL_ENABLE; + sun4i_spi_write(sspi, SUN4I_CTL_REG, reg); /* Ensure that we have a parent clock fast enough */ @@ -403,7 +406,7 @@ static int sun4i_spi_runtime_resume(struct device *dev) } sun4i_spi_write(sspi, SUN4I_CTL_REG, - SUN4I_CTL_ENABLE | SUN4I_CTL_MASTER | SUN4I_CTL_TP); + SUN4I_CTL_MASTER | SUN4I_CTL_TP); return 0; diff --git a/drivers/spi/spi-tegra114.c b/drivers/spi/spi-tegra114.c index 460f232dad508..8d7ce4c556aa1 100644 --- a/drivers/spi/spi-tegra114.c +++ b/drivers/spi/spi-tegra114.c @@ -728,9 +728,9 @@ static int tegra_spi_set_hw_cs_timing(struct spi_device *spi) u32 inactive_cycles; u8 cs_state; - if (setup->unit != SPI_DELAY_UNIT_SCK || - hold->unit != SPI_DELAY_UNIT_SCK || - inactive->unit != SPI_DELAY_UNIT_SCK) { + if ((setup->value && setup->unit != SPI_DELAY_UNIT_SCK) || + (hold->value && hold->unit != SPI_DELAY_UNIT_SCK) || + (inactive->value && inactive->unit != SPI_DELAY_UNIT_SCK)) { dev_err(&spi->dev, "Invalid delay unit %d, should be SPI_DELAY_UNIT_SCK\n", SPI_DELAY_UNIT_SCK); diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c index e3c236025a7b3..e9afebd724237 100644 --- a/drivers/spi/spi-tegra210-quad.c +++ b/drivers/spi/spi-tegra210-quad.c @@ -134,7 +134,7 @@ #define QSPI_COMMAND_VALUE_SET(X) (((x) & 0xFF) << 0) #define QSPI_CMB_SEQ_CMD_CFG 0x1a0 -#define QSPI_COMMAND_X1_X2_X4(x) (((x) & 0x3) << 13) +#define QSPI_COMMAND_X1_X2_X4(x) ((((x) >> 1) & 0x3) << 13) #define QSPI_COMMAND_X1_X2_X4_MASK (0x03 << 13) #define QSPI_COMMAND_SDR_DDR BIT(12) #define QSPI_COMMAND_SIZE_SET(x) (((x) & 0xFF) << 0) @@ -147,7 +147,7 @@ #define QSPI_ADDRESS_VALUE_SET(X) (((x) & 0xFFFF) << 0) #define QSPI_CMB_SEQ_ADDR_CFG 0x1ac -#define QSPI_ADDRESS_X1_X2_X4(x) (((x) & 0x3) << 13) +#define QSPI_ADDRESS_X1_X2_X4(x) ((((x) >> 1) & 0x3) << 13) #define QSPI_ADDRESS_X1_X2_X4_MASK (0x03 << 13) #define QSPI_ADDRESS_SDR_DDR BIT(12) #define QSPI_ADDRESS_SIZE_SET(x) (((x) & 0xFF) << 0) @@ -1036,10 +1036,6 @@ static u32 tegra_qspi_addr_config(bool is_ddr, u8 bus_width, u8 len) { u32 addr_config = 0; - /* Extract Address configuration and value */ - is_ddr = 0; //Only SDR mode supported - bus_width = 0; //X1 mode - if (is_ddr) addr_config |= QSPI_ADDRESS_SDR_DDR; else @@ -1079,13 +1075,13 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi, switch (transfer_phase) { case CMD_TRANSFER: /* X1 SDR mode */ - cmd_config = tegra_qspi_cmd_config(false, 0, + cmd_config = tegra_qspi_cmd_config(false, xfer->tx_nbits, xfer->len); cmd_value = *((const u8 *)(xfer->tx_buf)); break; case ADDR_TRANSFER: /* X1 SDR mode */ - addr_config = tegra_qspi_addr_config(false, 0, + addr_config = tegra_qspi_addr_config(false, xfer->tx_nbits, xfer->len); address_value = *((const u32 *)(xfer->tx_buf)); break; @@ -1163,26 +1159,22 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi, ret = -EIO; goto exit; } - if (!xfer->cs_change) { - tegra_qspi_transfer_end(spi); - spi_transfer_delay_exec(xfer); - } break; default: ret = -EINVAL; goto exit; } msg->actual_length += xfer->len; + if (!xfer->cs_change && transfer_phase == DATA_TRANSFER) { + tegra_qspi_transfer_end(spi); + spi_transfer_delay_exec(xfer); + } transfer_phase++; } ret = 0; exit: msg->status = ret; - if (ret < 0) { - tegra_qspi_transfer_end(spi); - spi_transfer_delay_exec(xfer); - } return ret; } diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c index 3503e6c0a5c98..b5deb4fe3b832 100644 --- a/drivers/spi/spi-zynqmp-gqspi.c +++ b/drivers/spi/spi-zynqmp-gqspi.c @@ -799,7 +799,6 @@ static void zynqmp_process_dma_irq(struct zynqmp_qspi *xqspi) static irqreturn_t zynqmp_qspi_irq(int irq, void *dev_id) { struct zynqmp_qspi *xqspi = (struct zynqmp_qspi *)dev_id; - irqreturn_t ret = IRQ_NONE; u32 status, mask, dma_status = 0; status = zynqmp_gqspi_read(xqspi, GQSPI_ISR_OFST); @@ -814,27 +813,24 @@ static irqreturn_t zynqmp_qspi_irq(int irq, void *dev_id) dma_status); } - if (mask & GQSPI_ISR_TXNOT_FULL_MASK) { + if (!mask && !dma_status) + return IRQ_NONE; + + if (mask & GQSPI_ISR_TXNOT_FULL_MASK) zynqmp_qspi_filltxfifo(xqspi, GQSPI_TX_FIFO_FILL); - ret = IRQ_HANDLED; - } - if (dma_status & GQSPI_QSPIDMA_DST_I_STS_DONE_MASK) { + if (dma_status & GQSPI_QSPIDMA_DST_I_STS_DONE_MASK) zynqmp_process_dma_irq(xqspi); - ret = IRQ_HANDLED; - } else if (!(mask & GQSPI_IER_RXEMPTY_MASK) && - (mask & GQSPI_IER_GENFIFOEMPTY_MASK)) { + else if (!(mask & GQSPI_IER_RXEMPTY_MASK) && + (mask & GQSPI_IER_GENFIFOEMPTY_MASK)) zynqmp_qspi_readrxfifo(xqspi, GQSPI_RX_FIFO_FILL); - ret = IRQ_HANDLED; - } if (xqspi->bytes_to_receive == 0 && xqspi->bytes_to_transfer == 0 && ((status & GQSPI_IRQ_MASK) == GQSPI_IRQ_MASK)) { zynqmp_gqspi_write(xqspi, GQSPI_IDR_OFST, GQSPI_ISR_IDR_MASK); complete(&xqspi->data_completion); - ret = IRQ_HANDLED; } - return ret; + return IRQ_HANDLED; } /** diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 2f2e908fb8505..e58c3453cdeef 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -2716,6 +2716,16 @@ static acpi_status acpi_register_spi_device(struct spi_controller *ctlr, acpi_set_modalias(adev, acpi_device_hid(adev), spi->modalias, sizeof(spi->modalias)); + /* + * This gets re-tried in spi_probe() for -EPROBE_DEFER handling in case + * the GPIO controller does not have a driver yet. This needs to be done + * here too, because this call sets the GPIO direction and/or bias. + * Setting these needs to be done even if there is no driver, in which + * case spi_probe() will never get called. + */ + if (spi->irq < 0) + spi->irq = acpi_dev_gpio_irq_get(adev, 0); + acpi_device_set_enumerated(adev); adev->power.flags.ignore_parent = true; diff --git a/drivers/staging/axis-fifo/axis-fifo.c b/drivers/staging/axis-fifo/axis-fifo.c index 727b956aa2317..4c38eeda53c4e 100644 --- a/drivers/staging/axis-fifo/axis-fifo.c +++ b/drivers/staging/axis-fifo/axis-fifo.c @@ -42,7 +42,6 @@ #define DRIVER_NAME "axis_fifo" #define READ_BUF_SIZE 128U /* read buffer length in words */ -#define WRITE_BUF_SIZE 128U /* write buffer length in words */ /* ---------------------------- * IP register offsets @@ -397,33 +396,29 @@ static ssize_t axis_fifo_read(struct file *f, char __user *buf, } bytes_available = ioread32(fifo->base_addr + XLLF_RLR_OFFSET); + words_available = bytes_available / sizeof(u32); if (!bytes_available) { - dev_err(fifo->dt_device, "received a packet of length 0 - fifo core will be reset\n"); - reset_ip_core(fifo); + dev_err(fifo->dt_device, "received a packet of length 0\n"); ret = -EIO; goto end_unlock; } if (bytes_available > len) { - dev_err(fifo->dt_device, "user read buffer too small (available bytes=%zu user buffer bytes=%zu) - fifo core will be reset\n", + dev_err(fifo->dt_device, "user read buffer too small (available bytes=%zu user buffer bytes=%zu)\n", bytes_available, len); - reset_ip_core(fifo); ret = -EINVAL; - goto end_unlock; + goto err_flush_rx; } if (bytes_available % sizeof(u32)) { /* this probably can't happen unless IP * registers were previously mishandled */ - dev_err(fifo->dt_device, "received a packet that isn't word-aligned - fifo core will be reset\n"); - reset_ip_core(fifo); + dev_err(fifo->dt_device, "received a packet that isn't word-aligned\n"); ret = -EIO; - goto end_unlock; + goto err_flush_rx; } - words_available = bytes_available / sizeof(u32); - /* read data into an intermediate buffer, copying the contents * to userspace when the buffer is full */ @@ -435,19 +430,23 @@ static ssize_t axis_fifo_read(struct file *f, char __user *buf, tmp_buf[i] = ioread32(fifo->base_addr + XLLF_RDFD_OFFSET); } + words_available -= copy; if (copy_to_user(buf + copied * sizeof(u32), tmp_buf, copy * sizeof(u32))) { - reset_ip_core(fifo); ret = -EFAULT; - goto end_unlock; + goto err_flush_rx; } copied += copy; - words_available -= copy; } + mutex_unlock(&fifo->read_lock); + + return bytes_available; - ret = bytes_available; +err_flush_rx: + while (words_available--) + ioread32(fifo->base_addr + XLLF_RDFD_OFFSET); end_unlock: mutex_unlock(&fifo->read_lock); @@ -475,11 +474,8 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf, { struct axis_fifo *fifo = (struct axis_fifo *)f->private_data; unsigned int words_to_write; - unsigned int copied; - unsigned int copy; - unsigned int i; + u32 *txbuf; int ret; - u32 tmp_buf[WRITE_BUF_SIZE]; if (len % sizeof(u32)) { dev_err(fifo->dt_device, @@ -495,11 +491,17 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf, return -EINVAL; } - if (words_to_write > fifo->tx_fifo_depth) { - dev_err(fifo->dt_device, "tried to write more words [%u] than slots in the fifo buffer [%u]\n", - words_to_write, fifo->tx_fifo_depth); + /* + * In 'Store-and-Forward' mode, the maximum packet that can be + * transmitted is limited by the size of the FIFO, which is + * (C_TX_FIFO_DEPTH–4)*(data interface width/8) bytes. + * + * Do not attempt to send a packet larger than 'tx_fifo_depth - 4', + * otherwise a 'Transmit Packet Overrun Error' interrupt will be + * raised, which requires a reset of the TX circuit to recover. + */ + if (words_to_write > (fifo->tx_fifo_depth - 4)) return -EINVAL; - } if (fifo->write_flags & O_NONBLOCK) { /* @@ -538,33 +540,20 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf, } } - /* write data from an intermediate buffer into the fifo IP, refilling - * the buffer with userspace data as needed - */ - copied = 0; - while (words_to_write > 0) { - copy = min(words_to_write, WRITE_BUF_SIZE); - - if (copy_from_user(tmp_buf, buf + copied * sizeof(u32), - copy * sizeof(u32))) { - reset_ip_core(fifo); - ret = -EFAULT; - goto end_unlock; - } - - for (i = 0; i < copy; i++) - iowrite32(tmp_buf[i], fifo->base_addr + - XLLF_TDFD_OFFSET); - - copied += copy; - words_to_write -= copy; + txbuf = vmemdup_user(buf, len); + if (IS_ERR(txbuf)) { + ret = PTR_ERR(txbuf); + goto end_unlock; } - ret = copied * sizeof(u32); + for (int i = 0; i < words_to_write; ++i) + iowrite32(txbuf[i], fifo->base_addr + XLLF_TDFD_OFFSET); /* write packet size to fifo */ - iowrite32(ret, fifo->base_addr + XLLF_TLR_OFFSET); + iowrite32(len, fifo->base_addr + XLLF_TLR_OFFSET); + ret = len; + kvfree(txbuf); end_unlock: mutex_unlock(&fifo->write_lock); @@ -780,9 +769,6 @@ static int axis_fifo_parse_dt(struct axis_fifo *fifo) goto end; } - /* IP sets TDFV to fifo depth - 4 so we will do the same */ - fifo->tx_fifo_depth -= 4; - ret = get_dts_property(fifo, "xlnx,use-rx-data", &fifo->has_rx_fifo); if (ret) { dev_err(fifo->dt_device, "missing xlnx,use-rx-data property\n"); diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c index eac1d570f4372..dce721f440c5e 100644 --- a/drivers/staging/fbtft/fbtft-core.c +++ b/drivers/staging/fbtft/fbtft-core.c @@ -744,6 +744,7 @@ struct fb_info *fbtft_framebuffer_alloc(struct fbtft_display *display, return info; release_framebuf: + fb_deferred_io_cleanup(info); framebuffer_release(info); alloc_fail: diff --git a/drivers/staging/iio/adc/ad7816.c b/drivers/staging/iio/adc/ad7816.c index 6c14d7bcdd675..081b17f498638 100644 --- a/drivers/staging/iio/adc/ad7816.c +++ b/drivers/staging/iio/adc/ad7816.c @@ -136,7 +136,7 @@ static ssize_t ad7816_store_mode(struct device *dev, struct iio_dev *indio_dev = dev_to_iio_dev(dev); struct ad7816_chip_info *chip = iio_priv(indio_dev); - if (strcmp(buf, "full")) { + if (strcmp(buf, "full") == 0) { gpiod_set_value(chip->rdwr_pin, 1); chip->mode = AD7816_FULL; } else { diff --git a/drivers/staging/iio/impedance-analyzer/ad5933.c b/drivers/staging/iio/impedance-analyzer/ad5933.c index 2d0883a640827..31d82a042dad2 100644 --- a/drivers/staging/iio/impedance-analyzer/ad5933.c +++ b/drivers/staging/iio/impedance-analyzer/ad5933.c @@ -412,7 +412,7 @@ static ssize_t ad5933_store(struct device *dev, ret = ad5933_cmd(st, 0); break; case AD5933_OUT_SETTLING_CYCLES: - val = clamp(val, (u16)0, (u16)0x7FF); + val = clamp(val, (u16)0, (u16)0x7FC); st->settling_cycles = val; /* 2x, 4x handling, see datasheet */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h index a444ec14ff9d5..1c17a87a85720 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h @@ -31,11 +31,6 @@ /* A => B */ #define IMPLIES(a, b) (!(a) || (b)) -/* for preprocessor and array sizing use MIN and MAX - otherwise use min and max */ -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) - #define ROUND_DIV(a, b) (((b) != 0) ? ((a) + ((b) >> 1)) / (b) : 0) #define CEIL_DIV(a, b) (((b) != 0) ? ((a) + (b) - 1) / (b) : 0) #define CEIL_MUL(a, b) (CEIL_DIV(a, b) * (b)) diff --git a/drivers/staging/media/imx/imx-media-csc-scaler.c b/drivers/staging/media/imx/imx-media-csc-scaler.c index 95cca281e8a37..07104e7f5a5f9 100644 --- a/drivers/staging/media/imx/imx-media-csc-scaler.c +++ b/drivers/staging/media/imx/imx-media-csc-scaler.c @@ -914,7 +914,7 @@ imx_media_csc_scaler_device_init(struct imx_media_dev *md) return &priv->vdev; err_m2m: - video_set_drvdata(vfd, NULL); + video_device_release(vfd); err_vfd: kfree(priv); return ERR_PTR(ret); diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c index ac398b5a97360..a1d941b0be00b 100644 --- a/drivers/staging/media/rkvdec/rkvdec.c +++ b/drivers/staging/media/rkvdec/rkvdec.c @@ -213,8 +213,14 @@ static int rkvdec_enum_framesizes(struct file *file, void *priv, if (!fmt) return -EINVAL; - fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE; - fsize->stepwise = fmt->frmsize; + fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS; + fsize->stepwise.min_width = 1; + fsize->stepwise.max_width = fmt->frmsize.max_width; + fsize->stepwise.step_width = 1; + fsize->stepwise.min_height = 1; + fsize->stepwise.max_height = fmt->frmsize.max_height; + fsize->stepwise.step_height = 1; + return 0; } diff --git a/drivers/staging/nvec/nvec_power.c b/drivers/staging/nvec/nvec_power.c index 9943b1fff1905..573521e1703bb 100644 --- a/drivers/staging/nvec/nvec_power.c +++ b/drivers/staging/nvec/nvec_power.c @@ -194,7 +194,7 @@ static int nvec_power_bat_notifier(struct notifier_block *nb, break; case MANUFACTURER: memcpy(power->bat_manu, &res->plc, res->length - 2); - power->bat_model[res->length - 2] = '\0'; + power->bat_manu[res->length - 2] = '\0'; break; case MODEL: memcpy(power->bat_model, &res->plc, res->length - 2); diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c index 7ecdaa2eeaf33..83fdee444c1c0 100644 --- a/drivers/staging/rtl8723bs/core/rtw_security.c +++ b/drivers/staging/rtl8723bs/core/rtw_security.c @@ -869,29 +869,21 @@ static signed int aes_cipher(u8 *key, uint hdrlen, num_blocks, payload_index; u8 pn_vector[6]; - u8 mic_iv[16]; - u8 mic_header1[16]; - u8 mic_header2[16]; - u8 ctr_preload[16]; + u8 mic_iv[16] = {}; + u8 mic_header1[16] = {}; + u8 mic_header2[16] = {}; + u8 ctr_preload[16] = {}; /* Intermediate Buffers */ - u8 chain_buffer[16]; - u8 aes_out[16]; - u8 padded_buffer[16]; + u8 chain_buffer[16] = {}; + u8 aes_out[16] = {}; + u8 padded_buffer[16] = {}; u8 mic[8]; uint frtype = GetFrameType(pframe); uint frsubtype = GetFrameSubType(pframe); frsubtype = frsubtype>>4; - memset((void *)mic_iv, 0, 16); - memset((void *)mic_header1, 0, 16); - memset((void *)mic_header2, 0, 16); - memset((void *)ctr_preload, 0, 16); - memset((void *)chain_buffer, 0, 16); - memset((void *)aes_out, 0, 16); - memset((void *)padded_buffer, 0, 16); - if ((hdrlen == WLAN_HDR_A3_LEN) || (hdrlen == WLAN_HDR_A3_QOS_LEN)) a4_exists = 0; else @@ -1081,15 +1073,15 @@ static signed int aes_decipher(u8 *key, uint hdrlen, num_blocks, payload_index; signed int res = _SUCCESS; u8 pn_vector[6]; - u8 mic_iv[16]; - u8 mic_header1[16]; - u8 mic_header2[16]; - u8 ctr_preload[16]; + u8 mic_iv[16] = {}; + u8 mic_header1[16] = {}; + u8 mic_header2[16] = {}; + u8 ctr_preload[16] = {}; /* Intermediate Buffers */ - u8 chain_buffer[16]; - u8 aes_out[16]; - u8 padded_buffer[16]; + u8 chain_buffer[16] = {}; + u8 aes_out[16] = {}; + u8 padded_buffer[16] = {}; u8 mic[8]; uint frtype = GetFrameType(pframe); @@ -1097,14 +1089,6 @@ static signed int aes_decipher(u8 *key, uint hdrlen, frsubtype = frsubtype>>4; - memset((void *)mic_iv, 0, 16); - memset((void *)mic_header1, 0, 16); - memset((void *)mic_header2, 0, 16); - memset((void *)ctr_preload, 0, 16); - memset((void *)chain_buffer, 0, 16); - memset((void *)aes_out, 0, 16); - memset((void *)padded_buffer, 0, 16); - /* start to decrypt the payload */ num_blocks = (plen-8) / 16; /* plen including LLC, payload_length and mic) */ diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c index 92aa98bbdc662..6028558f96111 100644 --- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c +++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c @@ -720,8 +720,7 @@ int vchiq_shutdown(struct vchiq_instance *instance) int status = 0; struct vchiq_state *state = instance->state; - if (mutex_lock_killable(&state->mutex)) - return -EAGAIN; + mutex_lock(&state->mutex); /* Remove all services */ vchiq_shutdown_internal(state, instance); diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index b516c2893420b..b756d4cfecfe9 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -4323,8 +4323,8 @@ int iscsit_close_connection( spin_unlock(&iscsit_global->ts_bitmap_lock); iscsit_stop_timers_for_cmds(conn); - iscsit_stop_nopin_response_timer(conn); iscsit_stop_nopin_timer(conn); + iscsit_stop_nopin_response_timer(conn); if (conn->conn_transport->iscsit_wait_conn) conn->conn_transport->iscsit_wait_conn(conn); diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c index 4ec99a55ac305..db13c6a379d87 100644 --- a/drivers/target/loopback/tcm_loop.c +++ b/drivers/target/loopback/tcm_loop.c @@ -893,6 +893,9 @@ static ssize_t tcm_loop_tpg_address_show(struct config_item *item, struct tcm_loop_tpg, tl_se_tpg); struct tcm_loop_hba *tl_hba = tl_tpg->tl_hba; + if (!tl_hba->sh) + return -ENODEV; + return snprintf(page, PAGE_SIZE, "%d:0:%d\n", tl_hba->sh->host_no, tl_tpg->tl_tpgt); } diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c index 52411b0a523ae..53ca2b317bcd3 100644 --- a/drivers/target/target_core_configfs.c +++ b/drivers/target/target_core_configfs.c @@ -2738,7 +2738,7 @@ static ssize_t target_lu_gp_members_show(struct config_item *item, char *page) config_item_name(&dev->dev_group.cg_item)); cur_len++; /* Extra byte for NULL terminator */ - if ((cur_len + len) > PAGE_SIZE) { + if ((cur_len + len) > PAGE_SIZE || cur_len > LU_GROUP_NAME_BUF) { pr_warn("Ran out of lu_gp_show_attr" "_members buffer\n"); break; diff --git a/drivers/target/target_core_fabric_lib.c b/drivers/target/target_core_fabric_lib.c index 6600ae44f29d9..d3ab251ba0497 100644 --- a/drivers/target/target_core_fabric_lib.c +++ b/drivers/target/target_core_fabric_lib.c @@ -257,11 +257,41 @@ static int iscsi_get_pr_transport_id_len( return len; } -static char *iscsi_parse_pr_out_transport_id( +static void sas_parse_pr_out_transport_id(char *buf, char *i_str) +{ + char hex[17] = {}; + + bin2hex(hex, buf + 4, 8); + snprintf(i_str, TRANSPORT_IQN_LEN, "naa.%s", hex); +} + +static void srp_parse_pr_out_transport_id(char *buf, char *i_str) +{ + char hex[33] = {}; + + bin2hex(hex, buf + 8, 16); + snprintf(i_str, TRANSPORT_IQN_LEN, "0x%s", hex); +} + +static void fcp_parse_pr_out_transport_id(char *buf, char *i_str) +{ + snprintf(i_str, TRANSPORT_IQN_LEN, "%8phC", buf + 8); +} + +static void sbp_parse_pr_out_transport_id(char *buf, char *i_str) +{ + char hex[17] = {}; + + bin2hex(hex, buf + 8, 8); + snprintf(i_str, TRANSPORT_IQN_LEN, "%s", hex); +} + +static bool iscsi_parse_pr_out_transport_id( struct se_portal_group *se_tpg, char *buf, u32 *out_tid_len, - char **port_nexus_ptr) + char **port_nexus_ptr, + char *i_str) { char *p; int i; @@ -282,7 +312,7 @@ static char *iscsi_parse_pr_out_transport_id( if ((format_code != 0x00) && (format_code != 0x40)) { pr_err("Illegal format code: 0x%02x for iSCSI" " Initiator Transport ID\n", format_code); - return NULL; + return false; } /* * If the caller wants the TransportID Length, we set that value for the @@ -306,7 +336,7 @@ static char *iscsi_parse_pr_out_transport_id( pr_err("Unable to locate \",i,0x\" separator" " for Initiator port identifier: %s\n", &buf[4]); - return NULL; + return false; } *p = '\0'; /* Terminate iSCSI Name */ p += 5; /* Skip over ",i,0x" separator */ @@ -339,7 +369,8 @@ static char *iscsi_parse_pr_out_transport_id( } else *port_nexus_ptr = NULL; - return &buf[4]; + strscpy(i_str, &buf[4], TRANSPORT_IQN_LEN); + return true; } int target_get_pr_transport_id_len(struct se_node_acl *nacl, @@ -387,33 +418,35 @@ int target_get_pr_transport_id(struct se_node_acl *nacl, } } -const char *target_parse_pr_out_transport_id(struct se_portal_group *tpg, - char *buf, u32 *out_tid_len, char **port_nexus_ptr) +bool target_parse_pr_out_transport_id(struct se_portal_group *tpg, + char *buf, u32 *out_tid_len, char **port_nexus_ptr, char *i_str) { - u32 offset; - switch (tpg->proto_id) { case SCSI_PROTOCOL_SAS: /* * Assume the FORMAT CODE 00b from spc4r17, 7.5.4.7 TransportID * for initiator ports using SCSI over SAS Serial SCSI Protocol. */ - offset = 4; + sas_parse_pr_out_transport_id(buf, i_str); break; - case SCSI_PROTOCOL_SBP: case SCSI_PROTOCOL_SRP: + srp_parse_pr_out_transport_id(buf, i_str); + break; case SCSI_PROTOCOL_FCP: - offset = 8; + fcp_parse_pr_out_transport_id(buf, i_str); + break; + case SCSI_PROTOCOL_SBP: + sbp_parse_pr_out_transport_id(buf, i_str); break; case SCSI_PROTOCOL_ISCSI: return iscsi_parse_pr_out_transport_id(tpg, buf, out_tid_len, - port_nexus_ptr); + port_nexus_ptr, i_str); default: pr_err("Unknown proto_id: 0x%02x\n", tpg->proto_id); - return NULL; + return false; } *port_nexus_ptr = NULL; *out_tid_len = 24; - return buf + offset; + return true; } diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h index 408be26d2e9b4..20aab1f505655 100644 --- a/drivers/target/target_core_internal.h +++ b/drivers/target/target_core_internal.h @@ -103,8 +103,8 @@ int target_get_pr_transport_id_len(struct se_node_acl *nacl, int target_get_pr_transport_id(struct se_node_acl *nacl, struct t10_pr_registration *pr_reg, int *format_code, unsigned char *buf); -const char *target_parse_pr_out_transport_id(struct se_portal_group *tpg, - char *buf, u32 *out_tid_len, char **port_nexus_ptr); +bool target_parse_pr_out_transport_id(struct se_portal_group *tpg, + char *buf, u32 *out_tid_len, char **port_nexus_ptr, char *i_str); /* target_core_hba.c */ struct se_hba *core_alloc_hba(const char *, u32, u32); diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index 49d9167bb263b..624d2f68bf385 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -1477,11 +1477,12 @@ core_scsi3_decode_spec_i_port( LIST_HEAD(tid_dest_list); struct pr_transport_id_holder *tidh_new, *tidh, *tidh_tmp; unsigned char *buf, *ptr, proto_ident; - const unsigned char *i_str = NULL; + unsigned char i_str[TRANSPORT_IQN_LEN]; char *iport_ptr = NULL, i_buf[PR_REG_ISID_ID_LEN]; sense_reason_t ret; u32 tpdl, tid_len = 0; u32 dest_rtpi = 0; + bool tid_found; /* * Allocate a struct pr_transport_id_holder and setup the @@ -1570,9 +1571,9 @@ core_scsi3_decode_spec_i_port( dest_rtpi = tmp_lun->lun_tpg->tpg_rtpi; iport_ptr = NULL; - i_str = target_parse_pr_out_transport_id(tmp_tpg, - ptr, &tid_len, &iport_ptr); - if (!i_str) + tid_found = target_parse_pr_out_transport_id(tmp_tpg, + ptr, &tid_len, &iport_ptr, i_str); + if (!tid_found) continue; /* * Determine if this SCSI device server requires that @@ -1841,7 +1842,9 @@ core_scsi3_decode_spec_i_port( } kmem_cache_free(t10_pr_reg_cache, dest_pr_reg); - core_scsi3_lunacl_undepend_item(dest_se_deve); + + if (dest_se_deve) + core_scsi3_lunacl_undepend_item(dest_se_deve); if (is_local) continue; @@ -3150,13 +3153,14 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key, struct t10_pr_registration *pr_reg, *pr_res_holder, *dest_pr_reg; struct t10_reservation *pr_tmpl = &dev->t10_pr; unsigned char *buf; - const unsigned char *initiator_str; + unsigned char initiator_str[TRANSPORT_IQN_LEN]; char *iport_ptr = NULL, i_buf[PR_REG_ISID_ID_LEN] = { }; u32 tid_len, tmp_tid_len; int new_reg = 0, type, scope, matching_iname; sense_reason_t ret; unsigned short rtpi; unsigned char proto_ident; + bool tid_found; if (!se_sess || !se_lun) { pr_err("SPC-3 PR: se_sess || struct se_lun is NULL!\n"); @@ -3275,9 +3279,9 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key, ret = TCM_INVALID_PARAMETER_LIST; goto out; } - initiator_str = target_parse_pr_out_transport_id(dest_se_tpg, - &buf[24], &tmp_tid_len, &iport_ptr); - if (!initiator_str) { + tid_found = target_parse_pr_out_transport_id(dest_se_tpg, + &buf[24], &tmp_tid_len, &iport_ptr, initiator_str); + if (!tid_found) { pr_err("SPC-3 PR REGISTER_AND_MOVE: Unable to locate" " initiator_str from Transport ID\n"); ret = TCM_INVALID_PARAMETER_LIST; diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c index f110f932ba054..675f774be1d30 100644 --- a/drivers/target/target_core_spc.c +++ b/drivers/target/target_core_spc.c @@ -2151,8 +2151,10 @@ spc_rsoc_get_descr(struct se_cmd *cmd, struct target_opcode_descriptor **opcode) if (descr->serv_action_valid) return TCM_INVALID_CDB_FIELD; - if (!descr->enabled || descr->enabled(descr, cmd)) + if (!descr->enabled || descr->enabled(descr, cmd)) { *opcode = descr; + return TCM_NO_SENSE; + } break; case 0x2: /* @@ -2166,8 +2168,10 @@ spc_rsoc_get_descr(struct se_cmd *cmd, struct target_opcode_descriptor **opcode) if (descr->serv_action_valid && descr->service_action == requested_sa) { if (!descr->enabled || descr->enabled(descr, - cmd)) + cmd)) { *opcode = descr; + return TCM_NO_SENSE; + } } else if (!descr->serv_action_valid) return TCM_INVALID_CDB_FIELD; break; @@ -2180,13 +2184,15 @@ spc_rsoc_get_descr(struct se_cmd *cmd, struct target_opcode_descriptor **opcode) */ if (descr->service_action == requested_sa) if (!descr->enabled || descr->enabled(descr, - cmd)) + cmd)) { *opcode = descr; + return TCM_NO_SENSE; + } break; } } - return 0; + return TCM_NO_SENSE; } static sense_reason_t diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c index b8ba360e863ed..927c3d7947f9c 100644 --- a/drivers/tee/optee/ffa_abi.c +++ b/drivers/tee/optee/ffa_abi.c @@ -653,7 +653,7 @@ static int optee_ffa_do_call_with_arg(struct tee_context *ctx, * with a matching configuration. */ -static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev, +static bool optee_ffa_api_is_compatible(struct ffa_device *ffa_dev, const struct ffa_ops *ops) { const struct ffa_msg_ops *msg_ops = ops->msg_ops; @@ -804,7 +804,7 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev) ffa_ops = ffa_dev->ops; - if (!optee_ffa_api_is_compatbile(ffa_dev, ffa_ops)) + if (!optee_ffa_api_is_compatible(ffa_dev, ffa_ops)) return -EINVAL; if (!optee_ffa_exchange_caps(ffa_dev, ffa_ops, &sec_caps, diff --git a/drivers/tee/optee/supp.c b/drivers/tee/optee/supp.c index d0f397c902420..322a543b8c278 100644 --- a/drivers/tee/optee/supp.c +++ b/drivers/tee/optee/supp.c @@ -80,6 +80,7 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params, struct optee *optee = tee_get_drvdata(ctx->teedev); struct optee_supp *supp = &optee->supp; struct optee_supp_req *req; + bool interruptable; u32 ret; /* @@ -110,18 +111,36 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params, /* * Wait for supplicant to process and return result, once we've * returned from wait_for_completion(&req->c) successfully we have - * exclusive access again. Allow the wait to be killable such that - * the wait doesn't turn into an indefinite state if the supplicant - * gets hung for some reason. + * exclusive access again. */ - if (wait_for_completion_killable(&req->c)) { + while (wait_for_completion_interruptible(&req->c)) { mutex_lock(&supp->mutex); - if (req->in_queue) { - list_del(&req->link); - req->in_queue = false; + interruptable = !supp->ctx; + if (interruptable) { + /* + * There's no supplicant available and since the + * supp->mutex currently is held none can + * become available until the mutex released + * again. + * + * Interrupting an RPC to supplicant is only + * allowed as a way of slightly improving the user + * experience in case the supplicant hasn't been + * started yet. During normal operation the supplicant + * will serve all requests in a timely manner and + * interrupting then wouldn't make sense. + */ + if (req->in_queue) { + list_del(&req->link); + req->in_queue = false; + } } mutex_unlock(&supp->mutex); - req->ret = TEEC_ERROR_COMMUNICATION; + + if (interruptable) { + req->ret = TEEC_ERROR_COMMUNICATION; + break; + } } ret = req->ret; diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 0eb342de0b001..976912f3bb5b4 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,7 +20,7 @@ #define TEE_NUM_DEVICES 32 -#define TEE_IOCTL_PARAM_SIZE(x) (sizeof(struct tee_param) * (x)) +#define TEE_IOCTL_PARAM_SIZE(x) (size_mul(sizeof(struct tee_param), (x))) #define TEE_UUID_NS_NAME_SIZE 128 @@ -487,7 +488,7 @@ static int tee_ioctl_open_session(struct tee_context *ctx, if (copy_from_user(&arg, uarg, sizeof(arg))) return -EFAULT; - if (sizeof(arg) + TEE_IOCTL_PARAM_SIZE(arg.num_params) != buf.buf_len) + if (size_add(sizeof(arg), TEE_IOCTL_PARAM_SIZE(arg.num_params)) != buf.buf_len) return -EINVAL; if (arg.num_params) { @@ -565,7 +566,7 @@ static int tee_ioctl_invoke(struct tee_context *ctx, if (copy_from_user(&arg, uarg, sizeof(arg))) return -EFAULT; - if (sizeof(arg) + TEE_IOCTL_PARAM_SIZE(arg.num_params) != buf.buf_len) + if (size_add(sizeof(arg), TEE_IOCTL_PARAM_SIZE(arg.num_params)) != buf.buf_len) return -EINVAL; if (arg.num_params) { @@ -699,7 +700,7 @@ static int tee_ioctl_supp_recv(struct tee_context *ctx, if (get_user(num_params, &uarg->num_params)) return -EFAULT; - if (sizeof(*uarg) + TEE_IOCTL_PARAM_SIZE(num_params) != buf.buf_len) + if (size_add(sizeof(*uarg), TEE_IOCTL_PARAM_SIZE(num_params)) != buf.buf_len) return -EINVAL; params = kcalloc(num_params, sizeof(struct tee_param), GFP_KERNEL); @@ -798,7 +799,7 @@ static int tee_ioctl_supp_send(struct tee_context *ctx, get_user(num_params, &uarg->num_params)) return -EFAULT; - if (sizeof(*uarg) + TEE_IOCTL_PARAM_SIZE(num_params) > buf.buf_len) + if (size_add(sizeof(*uarg), TEE_IOCTL_PARAM_SIZE(num_params)) > buf.buf_len) return -EINVAL; params = kcalloc(num_params, sizeof(struct tee_param), GFP_KERNEL); @@ -888,7 +889,7 @@ struct tee_device *tee_device_alloc(const struct tee_desc *teedesc, if (!teedesc || !teedesc->name || !teedesc->ops || !teedesc->ops->get_version || !teedesc->ops->open || - !teedesc->ops->release || !pool) + !teedesc->ops->release) return ERR_PTR(-EINVAL); teedev = kzalloc(sizeof(*teedev), GFP_KERNEL); diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c index 673cf03594948..426b818f2dd79 100644 --- a/drivers/tee/tee_shm.c +++ b/drivers/tee/tee_shm.c @@ -489,9 +489,13 @@ EXPORT_SYMBOL_GPL(tee_shm_get_from_id); */ void tee_shm_put(struct tee_shm *shm) { - struct tee_device *teedev = shm->ctx->teedev; + struct tee_device *teedev; bool do_release = false; + if (!shm || !shm->ctx || !shm->ctx->teedev) + return; + + teedev = shm->ctx->teedev; mutex_lock(&teedev->mutex); if (refcount_dec_and_test(&shm->refcount)) { /* diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 61c3d450ee605..2e06b26be4ef6 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -331,6 +331,7 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) tj_max = intel_tcc_get_tjmax(cpu); if (tj_max < 0) return tj_max; + tj_max *= 1000; zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL); if (!zonedev) diff --git a/drivers/thermal/mediatek/lvts_thermal.c b/drivers/thermal/mediatek/lvts_thermal.c index 8d0ccf494ba22..603b37ce1eb8e 100644 --- a/drivers/thermal/mediatek/lvts_thermal.c +++ b/drivers/thermal/mediatek/lvts_thermal.c @@ -67,10 +67,14 @@ #define LVTS_CALSCALE_CONF 0x300 #define LVTS_MONINT_CONF 0x8300318C -#define LVTS_MONINT_OFFSET_SENSOR0 0xC -#define LVTS_MONINT_OFFSET_SENSOR1 0x180 -#define LVTS_MONINT_OFFSET_SENSOR2 0x3000 -#define LVTS_MONINT_OFFSET_SENSOR3 0x3000000 +#define LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR0 BIT(3) +#define LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR1 BIT(8) +#define LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR2 BIT(13) +#define LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR3 BIT(25) +#define LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR0 BIT(2) +#define LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR1 BIT(7) +#define LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR2 BIT(12) +#define LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR3 BIT(24) #define LVTS_INT_SENSOR0 0x0009001F #define LVTS_INT_SENSOR1 0x001203E0 @@ -308,23 +312,41 @@ static int lvts_get_temp(struct thermal_zone_device *tz, int *temp) static void lvts_update_irq_mask(struct lvts_ctrl *lvts_ctrl) { - u32 masks[] = { - LVTS_MONINT_OFFSET_SENSOR0, - LVTS_MONINT_OFFSET_SENSOR1, - LVTS_MONINT_OFFSET_SENSOR2, - LVTS_MONINT_OFFSET_SENSOR3, + static const u32 high_offset_inten_masks[] = { + LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR0, + LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR1, + LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR2, + LVTS_MONINT_OFFSET_HIGH_INTEN_SENSOR3, + }; + static const u32 low_offset_inten_masks[] = { + LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR0, + LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR1, + LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR2, + LVTS_MONINT_OFFSET_LOW_INTEN_SENSOR3, }; u32 value = 0; int i; value = readl(LVTS_MONINT(lvts_ctrl->base)); - for (i = 0; i < ARRAY_SIZE(masks); i++) { + for (i = 0; i < ARRAY_SIZE(high_offset_inten_masks); i++) { if (lvts_ctrl->sensors[i].high_thresh == lvts_ctrl->high_thresh - && lvts_ctrl->sensors[i].low_thresh == lvts_ctrl->low_thresh) - value |= masks[i]; - else - value &= ~masks[i]; + && lvts_ctrl->sensors[i].low_thresh == lvts_ctrl->low_thresh) { + /* + * The minimum threshold needs to be configured in the + * OFFSETL register to get working interrupts, but we + * don't actually want to generate interrupts when + * crossing it. + */ + if (lvts_ctrl->low_thresh == -INT_MAX) { + value &= ~low_offset_inten_masks[i]; + value |= high_offset_inten_masks[i]; + } else { + value |= low_offset_inten_masks[i] | high_offset_inten_masks[i]; + } + } else { + value &= ~(low_offset_inten_masks[i] | high_offset_inten_masks[i]); + } } writel(value, LVTS_MONINT(lvts_ctrl->base)); diff --git a/drivers/thermal/qcom/Kconfig b/drivers/thermal/qcom/Kconfig index 2c7f3f9a26ebb..a6bb01082ec69 100644 --- a/drivers/thermal/qcom/Kconfig +++ b/drivers/thermal/qcom/Kconfig @@ -34,7 +34,8 @@ config QCOM_SPMI_TEMP_ALARM config QCOM_LMH tristate "Qualcomm Limits Management Hardware" - depends on ARCH_QCOM && QCOM_SCM + depends on ARCH_QCOM || COMPILE_TEST + select QCOM_SCM help This enables initialization of Qualcomm limits management hardware(LMh). LMh allows for hardware-enforced mitigation for cpus based on diff --git a/drivers/thermal/qcom/lmh.c b/drivers/thermal/qcom/lmh.c index d2d49264cf83a..7c299184c59b1 100644 --- a/drivers/thermal/qcom/lmh.c +++ b/drivers/thermal/qcom/lmh.c @@ -5,6 +5,8 @@ */ #include #include +#include +#include #include #include #include diff --git a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c index 78c5cfe6a0c0f..eeccf905f83ed 100644 --- a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c +++ b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2011-2015, 2017, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -16,6 +17,7 @@ #include "../thermal_hwmon.h" +#define QPNP_TM_REG_DIG_MINOR 0x00 #define QPNP_TM_REG_DIG_MAJOR 0x01 #define QPNP_TM_REG_TYPE 0x04 #define QPNP_TM_REG_SUBTYPE 0x05 @@ -31,7 +33,7 @@ #define STATUS_GEN2_STATE_MASK GENMASK(6, 4) #define STATUS_GEN2_STATE_SHIFT 4 -#define SHUTDOWN_CTRL1_OVERRIDE_S2 BIT(6) +#define SHUTDOWN_CTRL1_OVERRIDE_STAGE2 BIT(6) #define SHUTDOWN_CTRL1_THRESHOLD_MASK GENMASK(1, 0) #define SHUTDOWN_CTRL1_RATE_25HZ BIT(3) @@ -79,6 +81,7 @@ struct qpnp_tm_chip { /* protects .thresh, .stage and chip registers */ struct mutex lock; bool initialized; + bool require_stage2_shutdown; struct iio_channel *adc; const long (*temp_map)[THRESH_COUNT][STAGE_COUNT]; @@ -221,13 +224,13 @@ static int qpnp_tm_update_critical_trip_temp(struct qpnp_tm_chip *chip, { long stage2_threshold_min = (*chip->temp_map)[THRESH_MIN][1]; long stage2_threshold_max = (*chip->temp_map)[THRESH_MAX][1]; - bool disable_s2_shutdown = false; + bool disable_stage2_shutdown = false; u8 reg; WARN_ON(!mutex_is_locked(&chip->lock)); /* - * Default: S2 and S3 shutdown enabled, thresholds at + * Default: Stage 2 and Stage 3 shutdown enabled, thresholds at * lowest threshold set, monitoring at 25Hz */ reg = SHUTDOWN_CTRL1_RATE_25HZ; @@ -242,12 +245,12 @@ static int qpnp_tm_update_critical_trip_temp(struct qpnp_tm_chip *chip, chip->thresh = THRESH_MAX - ((stage2_threshold_max - temp) / TEMP_THRESH_STEP); - disable_s2_shutdown = true; + disable_stage2_shutdown = true; } else { chip->thresh = THRESH_MAX; if (chip->adc) - disable_s2_shutdown = true; + disable_stage2_shutdown = true; else dev_warn(chip->dev, "No ADC is configured and critical temperature %d mC is above the maximum stage 2 threshold of %ld mC! Configuring stage 2 shutdown at %ld mC.\n", @@ -256,8 +259,8 @@ static int qpnp_tm_update_critical_trip_temp(struct qpnp_tm_chip *chip, skip: reg |= chip->thresh; - if (disable_s2_shutdown) - reg |= SHUTDOWN_CTRL1_OVERRIDE_S2; + if (disable_stage2_shutdown && !chip->require_stage2_shutdown) + reg |= SHUTDOWN_CTRL1_OVERRIDE_STAGE2; return qpnp_tm_write(chip, QPNP_TM_REG_SHUTDOWN_CTRL1, reg); } @@ -371,8 +374,8 @@ static int qpnp_tm_probe(struct platform_device *pdev) { struct qpnp_tm_chip *chip; struct device_node *node; - u8 type, subtype, dig_major; - u32 res; + u8 type, subtype, dig_major, dig_minor; + u32 res, dig_revision; int ret, irq; node = pdev->dev.of_node; @@ -424,6 +427,11 @@ static int qpnp_tm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "could not read dig_major\n"); + ret = qpnp_tm_read(chip, QPNP_TM_REG_DIG_MINOR, &dig_minor); + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "could not read dig_minor\n"); + if (type != QPNP_TM_TYPE || (subtype != QPNP_TM_SUBTYPE_GEN1 && subtype != QPNP_TM_SUBTYPE_GEN2)) { dev_err(&pdev->dev, "invalid type 0x%02x or subtype 0x%02x\n", @@ -437,6 +445,23 @@ static int qpnp_tm_probe(struct platform_device *pdev) else chip->temp_map = &temp_map_gen1; + if (chip->subtype == QPNP_TM_SUBTYPE_GEN2) { + dig_revision = (dig_major << 8) | dig_minor; + /* + * Check if stage 2 automatic partial shutdown must remain + * enabled to avoid potential repeated faults upon reaching + * over-temperature stage 3. + */ + switch (dig_revision) { + case 0x0001: + case 0x0002: + case 0x0100: + case 0x0101: + chip->require_stage2_shutdown = true; + break; + } + } + /* * Register the sensor before initializing the hardware to be able to * read the trip points. get_temp() returns the default temperature diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c index 404f01cca4dab..ff8657afb31d3 100644 --- a/drivers/thermal/qoriq_thermal.c +++ b/drivers/thermal/qoriq_thermal.c @@ -18,6 +18,7 @@ #define SITES_MAX 16 #define TMR_DISABLE 0x0 #define TMR_ME 0x80000000 +#define TMR_CMD BIT(29) #define TMR_ALPF 0x0c000000 #define TMR_ALPF_V2 0x03000000 #define TMTMIR_DEFAULT 0x0000000f @@ -356,6 +357,12 @@ static int __maybe_unused qoriq_tmu_suspend(struct device *dev) if (ret) return ret; + if (data->ver > TMU_VER1) { + ret = regmap_set_bits(data->regmap, REGS_TMR, TMR_CMD); + if (ret) + return ret; + } + clk_disable_unprepare(data->clk); return 0; @@ -370,6 +377,12 @@ static int __maybe_unused qoriq_tmu_resume(struct device *dev) if (ret) return ret; + if (data->ver > TMU_VER1) { + ret = regmap_clear_bits(data->regmap, REGS_TMR, TMR_CMD); + if (ret) + return ret; + } + /* Enable monitoring */ return regmap_update_bits(data->regmap, REGS_TMR, TMR_ME, TMR_ME); } diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 2adcd316b517c..17579839fd06e 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -40,10 +40,13 @@ temp_show(struct device *dev, struct device_attribute *attr, char *buf) ret = thermal_zone_get_temp(tz, &temperature); - if (ret) - return ret; + if (!ret) + return sprintf(buf, "%d\n", temperature); - return sprintf(buf, "%d\n", temperature); + if (ret == -EAGAIN) + return -ENODATA; + + return ret; } static ssize_t diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c index d997a4c545f79..a60d7afc041f4 100644 --- a/drivers/thunderbolt/ctl.c +++ b/drivers/thunderbolt/ctl.c @@ -143,6 +143,11 @@ static void tb_cfg_request_dequeue(struct tb_cfg_request *req) struct tb_ctl *ctl = req->ctl; mutex_lock(&ctl->request_queue_lock); + if (!test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags)) { + mutex_unlock(&ctl->request_queue_lock); + return; + } + list_del(&req->list); clear_bit(TB_CFG_REQUEST_ACTIVE, &req->flags); if (test_bit(TB_CFG_REQUEST_CANCELED, &req->flags)) diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 31f3da4e6a08d..73be797acd509 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -36,7 +36,7 @@ static bool match_service_id(const struct tb_service_id *id, return false; } - if (id->match_flags & TBSVC_MATCH_PROTOCOL_VERSION) { + if (id->match_flags & TBSVC_MATCH_PROTOCOL_REVISION) { if (id->protocol_revision != svc->prtcrevs) return false; } diff --git a/drivers/thunderbolt/retimer.c b/drivers/thunderbolt/retimer.c index 2ee8c5ebca7c3..43146c0685dfa 100644 --- a/drivers/thunderbolt/retimer.c +++ b/drivers/thunderbolt/retimer.c @@ -89,9 +89,11 @@ static int tb_retimer_nvm_add(struct tb_retimer *rt) if (ret) goto err_nvm; - ret = tb_nvm_add_non_active(nvm, nvm_write); - if (ret) - goto err_nvm; + if (!rt->no_nvm_upgrade) { + ret = tb_nvm_add_non_active(nvm, nvm_write); + if (ret) + goto err_nvm; + } rt->nvm = nvm; return 0; diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 52cb1a3bb8c78..df83383800a8e 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -1465,7 +1465,7 @@ int tb_dp_port_set_hops(struct tb_port *port, unsigned int video, return ret; data[0] &= ~ADP_DP_CS_0_VIDEO_HOPID_MASK; - data[1] &= ~ADP_DP_CS_1_AUX_RX_HOPID_MASK; + data[1] &= ~ADP_DP_CS_1_AUX_TX_HOPID_MASK; data[1] &= ~ADP_DP_CS_1_AUX_RX_HOPID_MASK; data[0] |= (video << ADP_DP_CS_0_VIDEO_HOPID_SHIFT) & @@ -3439,7 +3439,7 @@ void tb_sw_set_unplugged(struct tb_switch *sw) } } -static int tb_switch_set_wake(struct tb_switch *sw, unsigned int flags) +static int tb_switch_set_wake(struct tb_switch *sw, unsigned int flags, bool runtime) { if (flags) tb_sw_dbg(sw, "enabling wakeup: %#x\n", flags); @@ -3447,7 +3447,7 @@ static int tb_switch_set_wake(struct tb_switch *sw, unsigned int flags) tb_sw_dbg(sw, "disabling wakeup\n"); if (tb_switch_is_usb4(sw)) - return usb4_switch_set_wake(sw, flags); + return usb4_switch_set_wake(sw, flags, runtime); return tb_lc_set_wake(sw, flags); } @@ -3523,7 +3523,7 @@ int tb_switch_resume(struct tb_switch *sw, bool runtime) tb_switch_check_wakes(sw); /* Disable wakes */ - tb_switch_set_wake(sw, 0); + tb_switch_set_wake(sw, 0, true); err = tb_switch_tmu_init(sw); if (err) @@ -3604,7 +3604,7 @@ void tb_switch_suspend(struct tb_switch *sw, bool runtime) flags |= TB_WAKE_ON_USB4 | TB_WAKE_ON_USB3 | TB_WAKE_ON_PCIE; } - tb_switch_set_wake(sw, flags); + tb_switch_set_wake(sw, flags, runtime); if (tb_switch_is_usb4(sw)) usb4_switch_set_sleep(sw); diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 920dac8a63e1d..d67a25f33fd1c 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1266,7 +1266,7 @@ int usb4_switch_read_uid(struct tb_switch *sw, u64 *uid); int usb4_switch_drom_read(struct tb_switch *sw, unsigned int address, void *buf, size_t size); bool usb4_switch_lane_bonding_possible(struct tb_switch *sw); -int usb4_switch_set_wake(struct tb_switch *sw, unsigned int flags); +int usb4_switch_set_wake(struct tb_switch *sw, unsigned int flags, bool runtime); int usb4_switch_set_sleep(struct tb_switch *sw); int usb4_switch_nvm_sector_size(struct tb_switch *sw); int usb4_switch_nvm_read(struct tb_switch *sw, unsigned int address, void *buf, diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c index 8db9bd32f4738..d4b4f58e717c1 100644 --- a/drivers/thunderbolt/usb4.c +++ b/drivers/thunderbolt/usb4.c @@ -405,12 +405,12 @@ bool usb4_switch_lane_bonding_possible(struct tb_switch *sw) * usb4_switch_set_wake() - Enabled/disable wake * @sw: USB4 router * @flags: Wakeup flags (%0 to disable) + * @runtime: Wake is being programmed during system runtime * * Enables/disables router to wake up from sleep. */ -int usb4_switch_set_wake(struct tb_switch *sw, unsigned int flags) +int usb4_switch_set_wake(struct tb_switch *sw, unsigned int flags, bool runtime) { - struct usb4_port *usb4; struct tb_port *port; u64 route = tb_route(sw); u32 val; @@ -440,13 +440,11 @@ int usb4_switch_set_wake(struct tb_switch *sw, unsigned int flags) val |= PORT_CS_19_WOU4; } else { bool configured = val & PORT_CS_19_PC; - usb4 = port->usb4; + bool wakeup = runtime || device_may_wakeup(&port->usb4->dev); - if (((flags & TB_WAKE_ON_CONNECT) | - device_may_wakeup(&usb4->dev)) && !configured) + if ((flags & TB_WAKE_ON_CONNECT) && wakeup && !configured) val |= PORT_CS_19_WOC; - if (((flags & TB_WAKE_ON_DISCONNECT) | - device_may_wakeup(&usb4->dev)) && configured) + if ((flags & TB_WAKE_ON_DISCONNECT) && wakeup && configured) val |= PORT_CS_19_WOD; if ((flags & TB_WAKE_ON_USB4) && configured) val |= PORT_CS_19_WOU4; diff --git a/drivers/tty/hvc/hvc_console.c b/drivers/tty/hvc/hvc_console.c index 959fae54ca394..4c252bbbf8e27 100644 --- a/drivers/tty/hvc/hvc_console.c +++ b/drivers/tty/hvc/hvc_console.c @@ -543,10 +543,10 @@ static ssize_t hvc_write(struct tty_struct *tty, const u8 *buf, size_t count) } /* - * Racy, but harmless, kick thread if there is still pending data. + * Kick thread to flush if there's still pending data + * or to wakeup the write queue. */ - if (hp->n_outbuf) - hvc_kick(); + hvc_kick(); return written; } diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index 8559ba1361c64..901f5c8311fbf 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -460,6 +460,7 @@ static int gsm_send_packet(struct gsm_mux *gsm, struct gsm_msg *msg); static struct gsm_dlci *gsm_dlci_alloc(struct gsm_mux *gsm, int addr); static void gsmld_write_trigger(struct gsm_mux *gsm); static void gsmld_write_task(struct work_struct *work); +static int gsm_modem_send_initial_msc(struct gsm_dlci *dlci); /** * gsm_fcs_add - update FCS @@ -2173,7 +2174,7 @@ static void gsm_dlci_open(struct gsm_dlci *dlci) pr_debug("DLCI %d goes open.\n", dlci->addr); /* Send current modem state */ if (dlci->addr) { - gsm_modem_update(dlci, 0); + gsm_modem_send_initial_msc(dlci); } else { /* Start keep-alive control */ gsm->ka_num = 0; @@ -4156,6 +4157,28 @@ static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk) return gsm_control_wait(dlci->gsm, ctrl); } +/** + * gsm_modem_send_initial_msc - Send initial modem status message + * + * @dlci channel + * + * Send an initial MSC message after DLCI open to set the initial + * modem status lines. This is only done for basic mode. + * Does not wait for a response as we cannot block the input queue + * processing. + */ +static int gsm_modem_send_initial_msc(struct gsm_dlci *dlci) +{ + u8 modembits[2]; + + if (dlci->adaption != 1 || dlci->gsm->encoding != GSM_BASIC_OPT) + return 0; + + modembits[0] = (dlci->addr << 2) | 2 | EA; /* DLCI, Valid, EA */ + modembits[1] = (gsm_encode_modem(dlci) << 1) | EA; + return gsm_control_command(dlci->gsm, CMD_MSC, (const u8 *)&modembits, 2); +} + /** * gsm_modem_update - send modem status line state * @dlci: channel diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index eaf4a907380aa..57b27f9ea1f03 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -653,7 +653,9 @@ static int dw8250_probe(struct platform_device *pdev) if (IS_ERR(data->rst)) return PTR_ERR(data->rst); - reset_control_deassert(data->rst); + err = reset_control_deassert(data->rst); + if (err) + return dev_err_probe(dev, err, "failed to deassert resets\n"); err = devm_add_action_or_reset(dev, dw8250_reset_control_assert, data->rst); if (err) diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c index 27430fdd9e761..e79ee33d432c9 100644 --- a/drivers/tty/serial/8250/8250_exar.c +++ b/drivers/tty/serial/8250/8250_exar.c @@ -33,6 +33,8 @@ #define PCI_DEVICE_ID_ACCESSIO_COM_4SM 0x10db #define PCI_DEVICE_ID_ACCESSIO_COM_8SM 0x10ea +#define PCI_DEVICE_ID_ADVANTECH_XR17V352 0x0018 + #define PCI_DEVICE_ID_COMMTECH_4224PCI335 0x0002 #define PCI_DEVICE_ID_COMMTECH_4222PCI335 0x0004 #define PCI_DEVICE_ID_COMMTECH_2324PCI335 0x000a @@ -845,6 +847,12 @@ static const struct exar8250_board pbn_fastcom35x_8 = { .exit = pci_xr17v35x_exit, }; +static const struct exar8250_board pbn_adv_XR17V352 = { + .num_ports = 2, + .setup = pci_xr17v35x_setup, + .exit = pci_xr17v35x_exit, +}; + static const struct exar8250_board pbn_exar_XR17V4358 = { .num_ports = 12, .setup = pci_xr17v35x_setup, @@ -914,6 +922,9 @@ static const struct pci_device_id exar_pci_tbl[] = { USR_DEVICE(XR17C152, 2980, pbn_exar_XR17C15x), USR_DEVICE(XR17C152, 2981, pbn_exar_XR17C15x), + /* ADVANTECH devices */ + EXAR_DEVICE(ADVANTECH, XR17V352, pbn_adv_XR17V352), + /* Exar Corp. XR17C15[248] Dual/Quad/Octal UART */ EXAR_DEVICE(EXAR, XR17C152, pbn_exar_XR17C15x), EXAR_DEVICE(EXAR, XR17C154, pbn_exar_XR17C15x), diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c index 28f9a2679a20e..c4a2a3e289c14 100644 --- a/drivers/tty/serial/8250/8250_mtk.c +++ b/drivers/tty/serial/8250/8250_mtk.c @@ -435,6 +435,7 @@ static int __maybe_unused mtk8250_runtime_suspend(struct device *dev) while (serial_in(up, MTK_UART_DEBUG0)); + clk_disable_unprepare(data->uart_clk); clk_disable_unprepare(data->bus_clk); return 0; @@ -445,6 +446,7 @@ static int __maybe_unused mtk8250_runtime_resume(struct device *dev) struct mtk8250_data *data = dev_get_drvdata(dev); clk_prepare_enable(data->bus_clk); + clk_prepare_enable(data->uart_clk); return 0; } @@ -475,13 +477,13 @@ static int mtk8250_probe_of(struct platform_device *pdev, struct uart_port *p, int dmacnt; #endif - data->uart_clk = devm_clk_get(&pdev->dev, "baud"); + data->uart_clk = devm_clk_get_enabled(&pdev->dev, "baud"); if (IS_ERR(data->uart_clk)) { /* * For compatibility with older device trees try unnamed * clk when no baud clk can be found. */ - data->uart_clk = devm_clk_get(&pdev->dev, NULL); + data->uart_clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(data->uart_clk)) { dev_warn(&pdev->dev, "Can't get uart clock\n"); return PTR_ERR(data->uart_clk); diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index c2778300e1510..895da5e725439 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -2375,9 +2375,8 @@ int serial8250_do_startup(struct uart_port *port) /* * Now, initialize the UART */ - serial_port_out(port, UART_LCR, UART_LCR_WLEN8); - spin_lock_irqsave(&port->lock, flags); + serial_port_out(port, UART_LCR, UART_LCR_WLEN8); if (up->port.flags & UPF_FOURPORT) { if (!up->port.irq) up->port.mctrl |= TIOCM_OUT1; diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 87eb2835fd34a..d9b165140152a 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -1404,7 +1404,7 @@ config SERIAL_STM32 config SERIAL_STM32_CONSOLE bool "Support for console on STM32" - depends on SERIAL_STM32=y + depends on SERIAL_STM32 select SERIAL_CORE_CONSOLE select SERIAL_EARLYCON diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 349d4849ba5e3..6e39ec5533337 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -234,6 +234,7 @@ struct imx_port { enum imx_tx_state tx_state; struct hrtimer trigger_start_tx; struct hrtimer trigger_stop_tx; + unsigned int rxtl; }; struct imx_port_ucrs { @@ -1337,6 +1338,7 @@ static void imx_uart_clear_rx_errors(struct imx_port *sport) #define TXTL_DEFAULT 8 #define RXTL_DEFAULT 8 /* 8 characters or aging timer */ +#define RXTL_CONSOLE_DEFAULT 1 #define TXTL_DMA 8 /* DMA burst setting */ #define RXTL_DMA 9 /* DMA burst setting */ @@ -1449,7 +1451,7 @@ static void imx_uart_disable_dma(struct imx_port *sport) ucr1 &= ~(UCR1_RXDMAEN | UCR1_TXDMAEN | UCR1_ATDMAEN); imx_uart_writel(sport, ucr1, UCR1); - imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT); + imx_uart_setup_ufcr(sport, TXTL_DEFAULT, sport->rxtl); sport->dma_is_enabled = 0; } @@ -1474,7 +1476,12 @@ static int imx_uart_startup(struct uart_port *port) return retval; } - imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT); + if (uart_console(&sport->port)) + sport->rxtl = RXTL_CONSOLE_DEFAULT; + else + sport->rxtl = RXTL_DEFAULT; + + imx_uart_setup_ufcr(sport, TXTL_DEFAULT, sport->rxtl); /* disable the DREN bit (Data Ready interrupt enable) before * requesting IRQs @@ -1887,7 +1894,7 @@ static int imx_uart_poll_init(struct uart_port *port) if (retval) clk_disable_unprepare(sport->clk_ipg); - imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT); + imx_uart_setup_ufcr(sport, TXTL_DEFAULT, sport->rxtl); spin_lock_irqsave(&sport->port.lock, flags); @@ -1979,7 +1986,7 @@ static int imx_uart_rs485_config(struct uart_port *port, struct ktermios *termio /* If the receiver trigger is 0, set it to a default value */ ufcr = imx_uart_readl(sport, UFCR); if ((ufcr & UFCR_RXTL_MASK) == 0) - imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT); + imx_uart_setup_ufcr(sport, TXTL_DEFAULT, sport->rxtl); imx_uart_start_rx(port); } @@ -2164,7 +2171,7 @@ imx_uart_console_setup(struct console *co, char *options) else imx_uart_console_get_options(sport, &baud, &parity, &bits); - imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT); + imx_uart_setup_ufcr(sport, TXTL_DEFAULT, sport->rxtl); retval = uart_set_options(&sport->port, co, baud, parity, bits, flow); diff --git a/drivers/tty/serial/jsm/jsm_tty.c b/drivers/tty/serial/jsm/jsm_tty.c index 222afc270c88d..1bee624bd484c 100644 --- a/drivers/tty/serial/jsm/jsm_tty.c +++ b/drivers/tty/serial/jsm/jsm_tty.c @@ -451,6 +451,7 @@ int jsm_uart_port_init(struct jsm_board *brd) if (!brd->channels[i]) continue; + brd->channels[i]->uart_port.dev = &brd->pci_dev->dev; brd->channels[i]->uart_port.irq = brd->irq; brd->channels[i]->uart_port.uartclk = 14745600; brd->channels[i]->uart_port.type = PORT_JSM; diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c index e339abff926d3..e10dcdeeda2d3 100644 --- a/drivers/tty/serial/max310x.c +++ b/drivers/tty/serial/max310x.c @@ -1659,6 +1659,8 @@ static int max310x_i2c_probe(struct i2c_client *client) port_client = devm_i2c_new_dummy_device(&client->dev, client->adapter, port_addr); + if (IS_ERR(port_client)) + return PTR_ERR(port_client); regmaps[i] = devm_regmap_init_i2c(port_client, ®cfg_i2c); } diff --git a/drivers/tty/serial/milbeaut_usio.c b/drivers/tty/serial/milbeaut_usio.c index 70a910085e937..9de3883a4e0b0 100644 --- a/drivers/tty/serial/milbeaut_usio.c +++ b/drivers/tty/serial/milbeaut_usio.c @@ -522,7 +522,10 @@ static int mlb_usio_probe(struct platform_device *pdev) } port->membase = devm_ioremap(&pdev->dev, res->start, resource_size(res)); - + if (!port->membase) { + ret = -ENOMEM; + goto failed; + } ret = platform_get_irq_byname(pdev, "rx"); mlb_usio_irq[index][RX] = ret; diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c index cc83b772b7ca9..7b868ea48ad59 100644 --- a/drivers/tty/serial/pch_uart.c +++ b/drivers/tty/serial/pch_uart.c @@ -967,7 +967,7 @@ static unsigned int dma_handle_tx(struct eg20t_port *priv) __func__); return 0; } - dma_sync_sg_for_device(port->dev, priv->sg_tx_p, nent, DMA_TO_DEVICE); + dma_sync_sg_for_device(port->dev, priv->sg_tx_p, num, DMA_TO_DEVICE); priv->desc_tx = desc; desc->callback = pch_dma_tx_complete; desc->callback_param = priv; diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index 8a2ce2ca6b394..b5f9a40f4a819 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -329,8 +329,9 @@ struct sc16is7xx_one { struct kthread_work reg_work; struct kthread_delayed_work ms_work; struct sc16is7xx_one_config config; - bool irda_mode; unsigned int old_mctrl; + u8 old_lcr; /* Value before EFR access. */ + bool irda_mode; }; struct sc16is7xx_port { @@ -355,10 +356,6 @@ static struct uart_driver sc16is7xx_uart = { .nr = SC16IS7XX_MAX_DEVS, }; -static void sc16is7xx_ier_set(struct uart_port *port, u8 bit); -static void sc16is7xx_stop_tx(struct uart_port *port); - -#define to_sc16is7xx_port(p,e) ((container_of((p), struct sc16is7xx_port, e))) #define to_sc16is7xx_one(p,e) ((container_of((p), struct sc16is7xx_one, e))) static u8 sc16is7xx_port_read(struct uart_port *port, u8 reg) @@ -416,6 +413,85 @@ static void sc16is7xx_power(struct uart_port *port, int on) on ? 0 : SC16IS7XX_IER_SLEEP_BIT); } +/* + * In an amazing feat of design, the Enhanced Features Register (EFR) + * shares the address of the Interrupt Identification Register (IIR). + * Access to EFR is switched on by writing a magic value (0xbf) to the + * Line Control Register (LCR). Any interrupt firing during this time will + * see the EFR where it expects the IIR to be, leading to + * "Unexpected interrupt" messages. + * + * Prevent this possibility by claiming a mutex while accessing the EFR, + * and claiming the same mutex from within the interrupt handler. This is + * similar to disabling the interrupt, but that doesn't work because the + * bulk of the interrupt processing is run as a workqueue job in thread + * context. + */ +static void sc16is7xx_efr_lock(struct uart_port *port) +{ + struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); + + mutex_lock(&one->efr_lock); + + /* Backup content of LCR. */ + one->old_lcr = sc16is7xx_port_read(port, SC16IS7XX_LCR_REG); + + /* Enable access to Enhanced register set */ + sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, SC16IS7XX_LCR_CONF_MODE_B); + + /* Disable cache updates when writing to EFR registers */ + regcache_cache_bypass(one->regmap, true); +} + +static void sc16is7xx_efr_unlock(struct uart_port *port) +{ + struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); + + /* Re-enable cache updates when writing to normal registers */ + regcache_cache_bypass(one->regmap, false); + + /* Restore original content of LCR */ + sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, one->old_lcr); + + mutex_unlock(&one->efr_lock); +} + +static void sc16is7xx_ier_clear(struct uart_port *port, u8 bit) +{ + struct sc16is7xx_port *s = dev_get_drvdata(port->dev); + struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); + + lockdep_assert_held_once(&port->lock); + + one->config.flags |= SC16IS7XX_RECONF_IER; + one->config.ier_mask |= bit; + one->config.ier_val &= ~bit; + kthread_queue_work(&s->kworker, &one->reg_work); +} + +static void sc16is7xx_ier_set(struct uart_port *port, u8 bit) +{ + struct sc16is7xx_port *s = dev_get_drvdata(port->dev); + struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); + + lockdep_assert_held_once(&port->lock); + + one->config.flags |= SC16IS7XX_RECONF_IER; + one->config.ier_mask |= bit; + one->config.ier_val |= bit; + kthread_queue_work(&s->kworker, &one->reg_work); +} + +static void sc16is7xx_stop_tx(struct uart_port *port) +{ + sc16is7xx_ier_clear(port, SC16IS7XX_IER_THRI_BIT); +} + +static void sc16is7xx_stop_rx(struct uart_port *port) +{ + sc16is7xx_ier_clear(port, SC16IS7XX_IER_RDI_BIT); +} + static const struct sc16is7xx_devtype sc16is74x_devtype = { .name = "SC16IS74X", .nr_gpio = 0, @@ -506,40 +582,6 @@ static int sc16is7xx_set_baud(struct uart_port *port, int baud) div /= prescaler; } - /* In an amazing feat of design, the Enhanced Features Register shares - * the address of the Interrupt Identification Register, and is - * switched in by writing a magic value (0xbf) to the Line Control - * Register. Any interrupt firing during this time will see the EFR - * where it expects the IIR to be, leading to "Unexpected interrupt" - * messages. - * - * Prevent this possibility by claiming a mutex while accessing the - * EFR, and claiming the same mutex from within the interrupt handler. - * This is similar to disabling the interrupt, but that doesn't work - * because the bulk of the interrupt processing is run as a workqueue - * job in thread context. - */ - mutex_lock(&one->efr_lock); - - lcr = sc16is7xx_port_read(port, SC16IS7XX_LCR_REG); - - /* Open the LCR divisors for configuration */ - sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, - SC16IS7XX_LCR_CONF_MODE_B); - - /* Enable enhanced features */ - regcache_cache_bypass(one->regmap, true); - sc16is7xx_port_update(port, SC16IS7XX_EFR_REG, - SC16IS7XX_EFR_ENABLE_BIT, - SC16IS7XX_EFR_ENABLE_BIT); - - regcache_cache_bypass(one->regmap, false); - - /* Put LCR back to the normal mode */ - sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, lcr); - - mutex_unlock(&one->efr_lock); - /* If bit MCR_CLKSEL is set, the divide by 4 prescaler is activated. */ sc16is7xx_port_update(port, SC16IS7XX_MCR_REG, SC16IS7XX_MCR_CLKSEL_BIT, @@ -547,7 +589,8 @@ static int sc16is7xx_set_baud(struct uart_port *port, int baud) mutex_lock(&one->efr_lock); - /* Open the LCR divisors for configuration */ + /* Backup LCR and access special register set (DLL/DLH) */ + lcr = sc16is7xx_port_read(port, SC16IS7XX_LCR_REG); sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, SC16IS7XX_LCR_CONF_MODE_A); @@ -557,7 +600,7 @@ static int sc16is7xx_set_baud(struct uart_port *port, int baud) sc16is7xx_port_write(port, SC16IS7XX_DLL_REG, div % 256); regcache_cache_bypass(one->regmap, false); - /* Put LCR back to the normal mode */ + /* Restore LCR and access to general register set */ sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, lcr); mutex_unlock(&one->efr_lock); @@ -889,42 +932,6 @@ static void sc16is7xx_reg_proc(struct kthread_work *ws) sc16is7xx_reconf_rs485(&one->port); } -static void sc16is7xx_ier_clear(struct uart_port *port, u8 bit) -{ - struct sc16is7xx_port *s = dev_get_drvdata(port->dev); - struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); - - lockdep_assert_held_once(&port->lock); - - one->config.flags |= SC16IS7XX_RECONF_IER; - one->config.ier_mask |= bit; - one->config.ier_val &= ~bit; - kthread_queue_work(&s->kworker, &one->reg_work); -} - -static void sc16is7xx_ier_set(struct uart_port *port, u8 bit) -{ - struct sc16is7xx_port *s = dev_get_drvdata(port->dev); - struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); - - lockdep_assert_held_once(&port->lock); - - one->config.flags |= SC16IS7XX_RECONF_IER; - one->config.ier_mask |= bit; - one->config.ier_val |= bit; - kthread_queue_work(&s->kworker, &one->reg_work); -} - -static void sc16is7xx_stop_tx(struct uart_port *port) -{ - sc16is7xx_ier_clear(port, SC16IS7XX_IER_THRI_BIT); -} - -static void sc16is7xx_stop_rx(struct uart_port *port) -{ - sc16is7xx_ier_clear(port, SC16IS7XX_IER_RDI_BIT); -} - static void sc16is7xx_ms_proc(struct kthread_work *ws) { struct sc16is7xx_one *one = to_sc16is7xx_one(ws, ms_work.work); @@ -1074,17 +1081,7 @@ static void sc16is7xx_set_termios(struct uart_port *port, if (!(termios->c_cflag & CREAD)) port->ignore_status_mask |= SC16IS7XX_LSR_BRK_ERROR_MASK; - /* As above, claim the mutex while accessing the EFR. */ - mutex_lock(&one->efr_lock); - - sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, - SC16IS7XX_LCR_CONF_MODE_B); - /* Configure flow control */ - regcache_cache_bypass(one->regmap, true); - sc16is7xx_port_write(port, SC16IS7XX_XON1_REG, termios->c_cc[VSTART]); - sc16is7xx_port_write(port, SC16IS7XX_XOFF1_REG, termios->c_cc[VSTOP]); - port->status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS); if (termios->c_cflag & CRTSCTS) { flow |= SC16IS7XX_EFR_AUTOCTS_BIT | @@ -1096,16 +1093,16 @@ static void sc16is7xx_set_termios(struct uart_port *port, if (termios->c_iflag & IXOFF) flow |= SC16IS7XX_EFR_SWFLOW1_BIT; - sc16is7xx_port_update(port, - SC16IS7XX_EFR_REG, - SC16IS7XX_EFR_FLOWCTRL_BITS, - flow); - regcache_cache_bypass(one->regmap, false); - /* Update LCR register */ sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, lcr); - mutex_unlock(&one->efr_lock); + /* Update EFR registers */ + sc16is7xx_efr_lock(port); + sc16is7xx_port_write(port, SC16IS7XX_XON1_REG, termios->c_cc[VSTART]); + sc16is7xx_port_write(port, SC16IS7XX_XOFF1_REG, termios->c_cc[VSTOP]); + sc16is7xx_port_update(port, SC16IS7XX_EFR_REG, + SC16IS7XX_EFR_FLOWCTRL_BITS, flow); + sc16is7xx_efr_unlock(port); /* Get baud rate generator configuration */ baud = uart_get_baud_rate(port, termios, old, @@ -1163,17 +1160,6 @@ static int sc16is7xx_startup(struct uart_port *port) sc16is7xx_port_write(port, SC16IS7XX_FCR_REG, SC16IS7XX_FCR_FIFO_BIT); - /* Enable EFR */ - sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, - SC16IS7XX_LCR_CONF_MODE_B); - - regcache_cache_bypass(one->regmap, true); - - /* Enable write access to enhanced features and internal clock div */ - sc16is7xx_port_update(port, SC16IS7XX_EFR_REG, - SC16IS7XX_EFR_ENABLE_BIT, - SC16IS7XX_EFR_ENABLE_BIT); - /* Enable TCR/TLR */ sc16is7xx_port_update(port, SC16IS7XX_MCR_REG, SC16IS7XX_MCR_TCRTLR_BIT, @@ -1185,7 +1171,8 @@ static int sc16is7xx_startup(struct uart_port *port) SC16IS7XX_TCR_RX_RESUME(24) | SC16IS7XX_TCR_RX_HALT(48)); - regcache_cache_bypass(one->regmap, false); + /* Disable TCR/TLR access */ + sc16is7xx_port_update(port, SC16IS7XX_MCR_REG, SC16IS7XX_MCR_TCRTLR_BIT, 0); /* Now, initialize the UART */ sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, SC16IS7XX_LCR_WORD_LEN_8); diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c index 4350a69d97d7a..247ef6bd66a01 100644 --- a/drivers/tty/serial/sh-sci.c +++ b/drivers/tty/serial/sh-sci.c @@ -104,6 +104,20 @@ struct plat_sci_reg { u8 offset, size; }; +struct sci_suspend_regs { + u16 scdl; + u16 sccks; + u16 scsmr; + u16 scscr; + u16 scfcr; + u16 scsptr; + u16 hssrr; + u16 scpcr; + u16 scpdr; + u8 scbrr; + u8 semr; +}; + struct sci_port_params { const struct plat_sci_reg regs[SCIx_NR_REGS]; unsigned int fifosize; @@ -134,6 +148,8 @@ struct sci_port { struct dma_chan *chan_tx; struct dma_chan *chan_rx; + struct reset_control *rstc; + #ifdef CONFIG_SERIAL_SH_SCI_DMA struct dma_chan *chan_tx_saved; struct dma_chan *chan_rx_saved; @@ -153,10 +169,12 @@ struct sci_port { int rx_trigger; struct timer_list rx_fifo_timer; int rx_fifo_timeout; + struct sci_suspend_regs suspend_regs; u16 hscif_tot; bool has_rtscts; bool autorts; + bool tx_occurred; }; #define SCI_NPORTS CONFIG_SERIAL_SH_SCI_NR_UARTS @@ -165,6 +183,7 @@ static struct sci_port sci_ports[SCI_NPORTS]; static unsigned long sci_ports_in_use; static struct uart_driver sci_uart_driver; static bool sci_uart_earlycon; +static bool sci_uart_earlycon_dev_probing; static inline struct sci_port * to_sci_port(struct uart_port *uart) @@ -821,6 +840,7 @@ static void sci_transmit_chars(struct uart_port *port) { struct circ_buf *xmit = &port->state->xmit; unsigned int stopped = uart_tx_stopped(port); + struct sci_port *s = to_sci_port(port); unsigned short status; unsigned short ctrl; int count; @@ -857,6 +877,7 @@ static void sci_transmit_chars(struct uart_port *port) } serial_port_out(port, SCxTDR, c); + s->tx_occurred = true; port->icount.tx++; } while (--count > 0); @@ -1213,6 +1234,8 @@ static void sci_dma_tx_complete(void *arg) if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) uart_write_wakeup(port); + s->tx_occurred = true; + if (!uart_circ_empty(xmit)) { s->cookie_tx = 0; schedule_work(&s->work_tx); @@ -1702,6 +1725,19 @@ static void sci_flush_buffer(struct uart_port *port) s->cookie_tx = -EINVAL; } } + +static void sci_dma_check_tx_occurred(struct sci_port *s) +{ + struct dma_tx_state state; + enum dma_status status; + + if (!s->chan_tx) + return; + + status = dmaengine_tx_status(s->chan_tx, s->cookie_tx, &state); + if (status == DMA_COMPLETE || status == DMA_IN_PROGRESS) + s->tx_occurred = true; +} #else /* !CONFIG_SERIAL_SH_SCI_DMA */ static inline void sci_request_dma(struct uart_port *port) { @@ -1711,6 +1747,10 @@ static inline void sci_free_dma(struct uart_port *port) { } +static void sci_dma_check_tx_occurred(struct sci_port *s) +{ +} + #define sci_flush_buffer NULL #endif /* !CONFIG_SERIAL_SH_SCI_DMA */ @@ -2047,6 +2087,12 @@ static unsigned int sci_tx_empty(struct uart_port *port) { unsigned short status = serial_port_in(port, SCxSR); unsigned short in_tx_fifo = sci_txfill(port); + struct sci_port *s = to_sci_port(port); + + sci_dma_check_tx_occurred(s); + + if (!s->tx_occurred) + return TIOCSER_TEMT; return (status & SCxSR_TEND(port)) && !in_tx_fifo ? TIOCSER_TEMT : 0; } @@ -2217,6 +2263,7 @@ static int sci_startup(struct uart_port *port) dev_dbg(port->dev, "%s(%d)\n", __func__, port->line); + s->tx_occurred = false; sci_request_dma(port); ret = sci_request_irq(s); @@ -2997,10 +3044,6 @@ static int sci_init_single(struct platform_device *dev, ret = sci_init_clocks(sci_port, &dev->dev); if (ret < 0) return ret; - - port->dev = &dev->dev; - - pm_runtime_enable(&dev->dev); } port->type = p->type; @@ -3030,11 +3073,6 @@ static int sci_init_single(struct platform_device *dev, return 0; } -static void sci_cleanup_single(struct sci_port *port) -{ - pm_runtime_disable(port->port.dev); -} - #if defined(CONFIG_SERIAL_SH_SCI_CONSOLE) || \ defined(CONFIG_SERIAL_SH_SCI_EARLYCON) static void serial_console_putchar(struct uart_port *port, unsigned char ch) @@ -3204,8 +3242,6 @@ static int sci_remove(struct platform_device *dev) sci_ports_in_use &= ~BIT(port->port.line); uart_remove_one_port(&sci_uart_driver, &port->port); - sci_cleanup_single(port); - if (port->port.fifosize > 1) device_remove_file(&dev->dev, &dev_attr_rx_fifo_trigger); if (type == PORT_SCIFA || type == PORT_SCIFB || type == PORT_HSCIF) @@ -3325,6 +3361,7 @@ static struct plat_sci_port *sci_parse_dt(struct platform_device *pdev, } sp = &sci_ports[id]; + sp->rstc = rstc; *dev_id = id; p->type = SCI_OF_TYPE(data); @@ -3338,7 +3375,8 @@ static struct plat_sci_port *sci_parse_dt(struct platform_device *pdev, static int sci_probe_single(struct platform_device *dev, unsigned int index, struct plat_sci_port *p, - struct sci_port *sciport) + struct sci_port *sciport, + struct resource *sci_res) { int ret; @@ -3367,6 +3405,11 @@ static int sci_probe_single(struct platform_device *dev, if (ret) return ret; + sciport->port.dev = &dev->dev; + ret = devm_pm_runtime_enable(&dev->dev); + if (ret) + return ret; + sciport->gpios = mctrl_gpio_init(&sciport->port, 0); if (IS_ERR(sciport->gpios)) return PTR_ERR(sciport->gpios); @@ -3380,13 +3423,31 @@ static int sci_probe_single(struct platform_device *dev, sciport->port.flags |= UPF_HARD_FLOW; } - ret = uart_add_one_port(&sci_uart_driver, &sciport->port); - if (ret) { - sci_cleanup_single(sciport); - return ret; + if (sci_uart_earlycon && sci_ports[0].port.mapbase == sci_res->start) { + /* + * In case: + * - this is the earlycon port (mapped on index 0 in sci_ports[]) and + * - it now maps to an alias other than zero and + * - the earlycon is still alive (e.g., "earlycon keep_bootcon" is + * available in bootargs) + * + * we need to avoid disabling clocks and PM domains through the runtime + * PM APIs called in __device_attach(). For this, increment the runtime + * PM reference counter (the clocks and PM domains were already enabled + * by the bootloader). Otherwise the earlycon may access the HW when it + * has no clocks enabled leading to failures (infinite loop in + * sci_poll_put_char()). + */ + pm_runtime_get_noresume(&dev->dev); + + /* + * Skip cleanup the sci_port[0] in early_console_exit(), this + * port is the same as the earlycon one. + */ + sci_uart_earlycon_dev_probing = true; } - return 0; + return uart_add_one_port(&sci_uart_driver, &sciport->port); } static int sci_probe(struct platform_device *dev) @@ -3444,7 +3505,7 @@ static int sci_probe(struct platform_device *dev) platform_set_drvdata(dev, sp); - ret = sci_probe_single(dev, dev_id, p, sp); + ret = sci_probe_single(dev, dev_id, p, sp, res); if (ret) return ret; @@ -3473,13 +3534,77 @@ static int sci_probe(struct platform_device *dev) return 0; } +static void sci_console_save(struct sci_port *s) +{ + struct sci_suspend_regs *regs = &s->suspend_regs; + struct uart_port *port = &s->port; + + if (sci_getreg(port, SCDL)->size) + regs->scdl = sci_serial_in(port, SCDL); + if (sci_getreg(port, SCCKS)->size) + regs->sccks = sci_serial_in(port, SCCKS); + if (sci_getreg(port, SCSMR)->size) + regs->scsmr = sci_serial_in(port, SCSMR); + if (sci_getreg(port, SCSCR)->size) + regs->scscr = sci_serial_in(port, SCSCR); + if (sci_getreg(port, SCFCR)->size) + regs->scfcr = sci_serial_in(port, SCFCR); + if (sci_getreg(port, SCSPTR)->size) + regs->scsptr = sci_serial_in(port, SCSPTR); + if (sci_getreg(port, SCBRR)->size) + regs->scbrr = sci_serial_in(port, SCBRR); + if (sci_getreg(port, HSSRR)->size) + regs->hssrr = sci_serial_in(port, HSSRR); + if (sci_getreg(port, SCPCR)->size) + regs->scpcr = sci_serial_in(port, SCPCR); + if (sci_getreg(port, SCPDR)->size) + regs->scpdr = sci_serial_in(port, SCPDR); + if (sci_getreg(port, SEMR)->size) + regs->semr = sci_serial_in(port, SEMR); +} + +static void sci_console_restore(struct sci_port *s) +{ + struct sci_suspend_regs *regs = &s->suspend_regs; + struct uart_port *port = &s->port; + + if (sci_getreg(port, SCDL)->size) + sci_serial_out(port, SCDL, regs->scdl); + if (sci_getreg(port, SCCKS)->size) + sci_serial_out(port, SCCKS, regs->sccks); + if (sci_getreg(port, SCSMR)->size) + sci_serial_out(port, SCSMR, regs->scsmr); + if (sci_getreg(port, SCSCR)->size) + sci_serial_out(port, SCSCR, regs->scscr); + if (sci_getreg(port, SCFCR)->size) + sci_serial_out(port, SCFCR, regs->scfcr); + if (sci_getreg(port, SCSPTR)->size) + sci_serial_out(port, SCSPTR, regs->scsptr); + if (sci_getreg(port, SCBRR)->size) + sci_serial_out(port, SCBRR, regs->scbrr); + if (sci_getreg(port, HSSRR)->size) + sci_serial_out(port, HSSRR, regs->hssrr); + if (sci_getreg(port, SCPCR)->size) + sci_serial_out(port, SCPCR, regs->scpcr); + if (sci_getreg(port, SCPDR)->size) + sci_serial_out(port, SCPDR, regs->scpdr); + if (sci_getreg(port, SEMR)->size) + sci_serial_out(port, SEMR, regs->semr); +} + static __maybe_unused int sci_suspend(struct device *dev) { struct sci_port *sport = dev_get_drvdata(dev); - if (sport) + if (sport) { uart_suspend_port(&sci_uart_driver, &sport->port); + if (!console_suspend_enabled && uart_console(&sport->port)) + sci_console_save(sport); + else + return reset_control_assert(sport->rstc); + } + return 0; } @@ -3487,8 +3612,18 @@ static __maybe_unused int sci_resume(struct device *dev) { struct sci_port *sport = dev_get_drvdata(dev); - if (sport) + if (sport) { + if (!console_suspend_enabled && uart_console(&sport->port)) { + sci_console_restore(sport); + } else { + int ret = reset_control_deassert(sport->rstc); + + if (ret) + return ret; + } + uart_resume_port(&sci_uart_driver, &sport->port); + } return 0; } @@ -3527,6 +3662,22 @@ sh_early_platform_init_buffer("earlyprintk", &sci_driver, #ifdef CONFIG_SERIAL_SH_SCI_EARLYCON static struct plat_sci_port port_cfg; +static int early_console_exit(struct console *co) +{ + struct sci_port *sci_port = &sci_ports[0]; + + /* + * Clean the slot used by earlycon. A new SCI device might + * map to this slot. + */ + if (!sci_uart_earlycon_dev_probing) { + memset(sci_port, 0, sizeof(*sci_port)); + sci_uart_earlycon = false; + } + + return 0; +} + static int __init early_console_setup(struct earlycon_device *device, int type) { @@ -3546,6 +3697,8 @@ static int __init early_console_setup(struct earlycon_device *device, SCSCR_RE | SCSCR_TE | port_cfg.scscr); device->con->write = serial_console_write; + device->con->exit = early_console_exit; + return 0; } static int __init sci_early_console_setup(struct earlycon_device *device, diff --git a/drivers/tty/serial/uartlite.c b/drivers/tty/serial/uartlite.c index b225a78f6175f..9f39bafa7fa96 100644 --- a/drivers/tty/serial/uartlite.c +++ b/drivers/tty/serial/uartlite.c @@ -872,16 +872,6 @@ static int ulite_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - if (!ulite_uart_driver.state) { - dev_dbg(&pdev->dev, "uartlite: calling uart_register_driver()\n"); - ret = uart_register_driver(&ulite_uart_driver); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to register driver\n"); - clk_disable_unprepare(pdata->clk); - return ret; - } - } - ret = ulite_assign(&pdev->dev, id, res->start, irq, pdata); pm_runtime_mark_last_busy(&pdev->dev); @@ -922,16 +912,25 @@ static struct platform_driver ulite_platform_driver = { static int __init ulite_init(void) { + int ret; + + pr_debug("uartlite: calling uart_register_driver()\n"); + ret = uart_register_driver(&ulite_uart_driver); + if (ret) + return ret; pr_debug("uartlite: calling platform_driver_register()\n"); - return platform_driver_register(&ulite_platform_driver); + ret = platform_driver_register(&ulite_platform_driver); + if (ret) + uart_unregister_driver(&ulite_uart_driver); + + return ret; } static void __exit ulite_exit(void) { platform_driver_unregister(&ulite_platform_driver); - if (ulite_uart_driver.state) - uart_unregister_driver(&ulite_uart_driver); + uart_unregister_driver(&ulite_uart_driver); } module_init(ulite_init); diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c index dcf2e1ebf9c57..bd001445815e8 100644 --- a/drivers/tty/sysrq.c +++ b/drivers/tty/sysrq.c @@ -524,6 +524,7 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { NULL, /* P */ NULL, /* Q */ NULL, /* R */ + /* S: May be registered by sched_ext for resetting */ NULL, /* S */ NULL, /* T */ NULL, /* U */ diff --git a/drivers/tty/vt/defkeymap.c_shipped b/drivers/tty/vt/defkeymap.c_shipped index 0c043e4f292e8..6af7bf8d5460c 100644 --- a/drivers/tty/vt/defkeymap.c_shipped +++ b/drivers/tty/vt/defkeymap.c_shipped @@ -23,6 +23,22 @@ unsigned short plain_map[NR_KEYS] = { 0xf118, 0xf601, 0xf602, 0xf117, 0xf600, 0xf119, 0xf115, 0xf116, 0xf11a, 0xf10c, 0xf10d, 0xf11b, 0xf11c, 0xf110, 0xf311, 0xf11d, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, }; static unsigned short shift_map[NR_KEYS] = { @@ -42,6 +58,22 @@ static unsigned short shift_map[NR_KEYS] = { 0xf20b, 0xf601, 0xf602, 0xf117, 0xf600, 0xf20a, 0xf115, 0xf116, 0xf11a, 0xf10c, 0xf10d, 0xf11b, 0xf11c, 0xf110, 0xf311, 0xf11d, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, }; static unsigned short altgr_map[NR_KEYS] = { @@ -61,6 +93,22 @@ static unsigned short altgr_map[NR_KEYS] = { 0xf118, 0xf601, 0xf602, 0xf117, 0xf600, 0xf119, 0xf115, 0xf116, 0xf11a, 0xf10c, 0xf10d, 0xf11b, 0xf11c, 0xf110, 0xf311, 0xf11d, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, }; static unsigned short ctrl_map[NR_KEYS] = { @@ -80,6 +128,22 @@ static unsigned short ctrl_map[NR_KEYS] = { 0xf118, 0xf601, 0xf602, 0xf117, 0xf600, 0xf119, 0xf115, 0xf116, 0xf11a, 0xf10c, 0xf10d, 0xf11b, 0xf11c, 0xf110, 0xf311, 0xf11d, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, }; static unsigned short shift_ctrl_map[NR_KEYS] = { @@ -99,6 +163,22 @@ static unsigned short shift_ctrl_map[NR_KEYS] = { 0xf118, 0xf601, 0xf602, 0xf117, 0xf600, 0xf119, 0xf115, 0xf116, 0xf11a, 0xf10c, 0xf10d, 0xf11b, 0xf11c, 0xf110, 0xf311, 0xf11d, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, }; static unsigned short alt_map[NR_KEYS] = { @@ -118,6 +198,22 @@ static unsigned short alt_map[NR_KEYS] = { 0xf118, 0xf210, 0xf211, 0xf117, 0xf600, 0xf119, 0xf115, 0xf116, 0xf11a, 0xf10c, 0xf10d, 0xf11b, 0xf11c, 0xf110, 0xf311, 0xf11d, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, }; static unsigned short ctrl_alt_map[NR_KEYS] = { @@ -137,6 +233,22 @@ static unsigned short ctrl_alt_map[NR_KEYS] = { 0xf118, 0xf601, 0xf602, 0xf117, 0xf600, 0xf119, 0xf115, 0xf20c, 0xf11a, 0xf10c, 0xf10d, 0xf11b, 0xf11c, 0xf110, 0xf311, 0xf11d, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, + 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, }; unsigned short *key_maps[MAX_NR_KEYMAPS] = { diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c index 358f216c6cd6e..18b3c197c1349 100644 --- a/drivers/tty/vt/keyboard.c +++ b/drivers/tty/vt/keyboard.c @@ -1496,7 +1496,7 @@ static void kbd_keycode(unsigned int keycode, int down, bool hw_raw) rc = atomic_notifier_call_chain(&keyboard_notifier_list, KBD_UNICODE, ¶m); if (rc != NOTIFY_STOP) - if (down && !raw_mode) + if (down && !(raw_mode || kbd->kbdmode == VC_OFF)) k_unicode(vc, keysym, !down); return; } diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 6bd1a7785e888..60c878ea95f92 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -962,7 +962,7 @@ void redraw_screen(struct vc_data *vc, int is_switch) } if (redraw) { - int update; + bool update; int old_was_color = vc->vc_can_do_color; set_origin(vc); @@ -999,7 +999,7 @@ int vc_cons_allocated(unsigned int i) return (i < MAX_NR_CONSOLES && vc_cons[i].d); } -static void visual_init(struct vc_data *vc, int num, int init) +static void visual_init(struct vc_data *vc, int num, bool init) { /* ++Geert: vc->vc_sw->con_init determines console size */ if (vc->vc_sw) @@ -1083,7 +1083,7 @@ int vc_allocate(unsigned int currcons) /* return 0 on success */ vc->port.ops = &vc_port_ops; INIT_WORK(&vc_cons[currcons].SAK_work, vc_SAK); - visual_init(vc, currcons, 1); + visual_init(vc, currcons, true); if (!*vc->uni_pagedict_loc) con_set_default_unimap(vc); @@ -1582,7 +1582,7 @@ static void csi_X(struct vc_data *vc, unsigned int vpar) vc_uniscr_clear_line(vc, vc->state.x, count); scr_memsetw((unsigned short *)vc->vc_pos, vc->vc_video_erase_char, 2 * count); if (con_should_update(vc)) - vc->vc_sw->con_clear(vc, vc->state.y, vc->state.x, 1, count); + vc->vc_sw->con_clear(vc, vc->state.y, vc->state.x, count); vc->vc_need_wrap = 0; } @@ -3474,7 +3474,7 @@ static int __init con_init(void) vc_cons[currcons].d = vc = kzalloc(sizeof(struct vc_data), GFP_NOWAIT); INIT_WORK(&vc_cons[currcons].SAK_work, vc_SAK); tty_port_init(&vc->port); - visual_init(vc, currcons, 1); + visual_init(vc, currcons, true); /* Assuming vc->vc_{cols,rows,screenbuf_size} are sane here. */ vc->vc_screenbuf = kzalloc(vc->vc_screenbuf_size, GFP_NOWAIT); vc_init(vc, currcons || !vc->vc_sw->con_save_screen); @@ -3642,7 +3642,7 @@ static int do_bind_con_driver(const struct consw *csw, int first, int last, old_was_color = vc->vc_can_do_color; vc->vc_sw->con_deinit(vc); vc->vc_origin = (unsigned long)vc->vc_screenbuf; - visual_init(vc, i, 0); + visual_init(vc, i, false); set_origin(vc); update_attr(vc); @@ -4392,6 +4392,7 @@ void do_unblank_screen(int leaving_gfx) set_palette(vc); set_cursor(vc); vt_event_post(VT_EVENT_UNBLANK, vc->vc_num, vc->vc_num); + notify_update(vc); } EXPORT_SYMBOL(do_unblank_screen); diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 8c685b5014044..5b21b60547da1 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -1105,8 +1105,6 @@ long vt_compat_ioctl(struct tty_struct *tty, case VT_WAITACTIVE: case VT_RELDISP: case VT_DISALLOCATE: - case VT_RESIZE: - case VT_RESIZEX: return vt_ioctl(tty, cmd, arg); /* diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index c673118cc8c8b..2ecf35e603df7 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -209,7 +209,7 @@ int ufshcd_mcq_memory_alloc(struct ufs_hba *hba) hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size, &hwq->sqe_dma_addr, GFP_KERNEL); - if (!hwq->sqe_dma_addr) { + if (!hwq->sqe_base_addr) { dev_err(hba->dev, "SQE allocation failed\n"); return -ENOMEM; } @@ -218,7 +218,7 @@ int ufshcd_mcq_memory_alloc(struct ufs_hba *hba) hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size, &hwq->cqe_dma_addr, GFP_KERNEL); - if (!hwq->cqe_dma_addr) { + if (!hwq->cqe_base_addr) { dev_err(hba->dev, "CQE allocation failed\n"); return -ENOMEM; } @@ -629,7 +629,6 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd) int tag = scsi_cmd_to_rq(cmd)->tag; struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct ufs_hw_queue *hwq; - unsigned long flags; int err; /* Skip task abort in case previous aborts failed and report failure */ @@ -668,10 +667,5 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd) return FAILED; } - spin_lock_irqsave(&hwq->cq_lock, flags); - if (ufshcd_cmd_inflight(lrbp->cmd)) - ufshcd_release_scsi_cmd(hba, lrbp); - spin_unlock_irqrestore(&hwq->cq_lock, flags); - return SUCCESS; } diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c index 06917153df7ba..5ac645f2948fc 100644 --- a/drivers/ufs/core/ufs-sysfs.c +++ b/drivers/ufs/core/ufs-sysfs.c @@ -1348,7 +1348,7 @@ UFS_UNIT_DESC_PARAM(logical_block_size, _LOGICAL_BLK_SIZE, 1); UFS_UNIT_DESC_PARAM(logical_block_count, _LOGICAL_BLK_COUNT, 8); UFS_UNIT_DESC_PARAM(erase_block_size, _ERASE_BLK_SIZE, 4); UFS_UNIT_DESC_PARAM(provisioning_type, _PROVISIONING_TYPE, 1); -UFS_UNIT_DESC_PARAM(physical_memory_resourse_count, _PHY_MEM_RSRC_CNT, 8); +UFS_UNIT_DESC_PARAM(physical_memory_resource_count, _PHY_MEM_RSRC_CNT, 8); UFS_UNIT_DESC_PARAM(context_capabilities, _CTX_CAPABILITIES, 2); UFS_UNIT_DESC_PARAM(large_unit_granularity, _LARGE_UNIT_SIZE_M1, 1); UFS_UNIT_DESC_PARAM(wb_buf_alloc_units, _WB_BUF_ALLOC_UNITS, 4); @@ -1365,7 +1365,7 @@ static struct attribute *ufs_sysfs_unit_descriptor[] = { &dev_attr_logical_block_count.attr, &dev_attr_erase_block_size.attr, &dev_attr_provisioning_type.attr, - &dev_attr_physical_memory_resourse_count.attr, + &dev_attr_physical_memory_resource_count.attr, &dev_attr_context_capabilities.attr, &dev_attr_large_unit_granularity.attr, &dev_attr_wb_buf_alloc_units.attr, diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 5f43bdbf2a565..6e1cdc8249f48 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -291,6 +291,7 @@ static const struct ufs_dev_quirk ufs_fixups[] = { .model = UFS_ANY_MODEL, .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | + UFS_DEVICE_QUIRK_PA_HIBER8TIME | UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, { .wmanufacturerid = UFS_VENDOR_SKHYNIX, .model = UFS_ANY_MODEL, @@ -307,6 +308,9 @@ static const struct ufs_dev_quirk ufs_fixups[] = { { .wmanufacturerid = UFS_VENDOR_TOSHIBA, .model = "THGLF2G9D8KBADG", .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, + { .wmanufacturerid = UFS_VENDOR_TOSHIBA, + .model = "THGJFJT1E45BATP", + .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, {} }; @@ -322,6 +326,7 @@ static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); static void ufshcd_resume_clkscaling(struct ufs_hba *hba); static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); +static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba); static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up); static irqreturn_t ufshcd_intr(int irq, void *__hba); static int ufshcd_change_power_mode(struct ufs_hba *hba, @@ -1318,6 +1323,7 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) * make sure that there are no outstanding requests when * clock scaling is in progress */ + mutex_lock(&hba->host->scan_mutex); blk_mq_quiesce_tagset(&hba->host->tag_set); mutex_lock(&hba->wb_mutex); down_write(&hba->clk_scaling_lock); @@ -1328,6 +1334,7 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) up_write(&hba->clk_scaling_lock); mutex_unlock(&hba->wb_mutex); blk_mq_unquiesce_tagset(&hba->host->tag_set); + mutex_unlock(&hba->host->scan_mutex); goto out; } @@ -1349,6 +1356,7 @@ static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool sc mutex_unlock(&hba->wb_mutex); blk_mq_unquiesce_tagset(&hba->host->tag_set); + mutex_unlock(&hba->host->scan_mutex); ufshcd_release(hba); } @@ -1408,10 +1416,9 @@ static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) return; } hba->clk_scaling.is_suspended = true; - hba->clk_scaling.window_start_t = 0; spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - devfreq_suspend_device(hba->devfreq); + __ufshcd_suspend_clkscaling(hba); } static void ufshcd_clk_scaling_resume_work(struct work_struct *work) @@ -1596,6 +1603,16 @@ static void ufshcd_devfreq_remove(struct ufs_hba *hba) dev_pm_opp_remove(hba->dev, clki->max_freq); } +static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba) +{ + unsigned long flags; + + devfreq_suspend_device(hba->devfreq); + spin_lock_irqsave(hba->host->host_lock, flags); + hba->clk_scaling.window_start_t = 0; + spin_unlock_irqrestore(hba->host->host_lock, flags); +} + static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) { unsigned long flags; @@ -1608,12 +1625,11 @@ static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) if (!hba->clk_scaling.is_suspended) { suspend = true; hba->clk_scaling.is_suspended = true; - hba->clk_scaling.window_start_t = 0; } spin_unlock_irqrestore(hba->host->host_lock, flags); if (suspend) - devfreq_suspend_device(hba->devfreq); + __ufshcd_suspend_clkscaling(hba); } static void ufshcd_resume_clkscaling(struct ufs_hba *hba) @@ -4153,8 +4169,8 @@ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, get, UIC_GET_ATTR_ID(attr_sel), UFS_UIC_COMMAND_RETRIES - retries); - if (mib_val && !ret) - *mib_val = uic_cmd.argument3; + if (mib_val) + *mib_val = ret == 0 ? uic_cmd.argument3 : 0; if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) && pwr_mode_change) @@ -4251,7 +4267,7 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) hba->uic_async_done = NULL; if (reenable_intr) ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); - if (ret) { + if (ret && !hba->pm_op_in_progress) { ufshcd_set_link_broken(hba); ufshcd_schedule_eh_work(hba); } @@ -4259,6 +4275,14 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) spin_unlock_irqrestore(hba->host->host_lock, flags); mutex_unlock(&hba->uic_cmd_mutex); + /* + * If the h8 exit fails during the runtime resume process, it becomes + * stuck and cannot be recovered through the error handler. To fix + * this, use link recovery instead of the error handler. + */ + if (ret && hba->pm_op_in_progress) + ret = ufshcd_link_recovery(hba); + return ret; } @@ -5619,7 +5643,6 @@ static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, struct ufshcd_lrb *lrbp; struct scsi_cmnd *cmd; unsigned long flags; - u32 hwq_num, utag; int tag; for (tag = 0; tag < hba->nutrs; tag++) { @@ -5629,9 +5652,9 @@ static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, test_bit(SCMD_STATE_COMPLETE, &cmd->state)) continue; - utag = blk_mq_unique_tag(scsi_cmd_to_rq(cmd)); - hwq_num = blk_mq_unique_tag_to_hwq(utag); - hwq = &hba->uhq[hwq_num]; + hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); + if (!hwq) + continue; if (force_compl) { ufshcd_mcq_compl_all_cqes_lock(hba, hwq); @@ -6323,13 +6346,14 @@ void ufshcd_schedule_eh_work(struct ufs_hba *hba) } } -static void ufshcd_force_error_recovery(struct ufs_hba *hba) +void ufshcd_force_error_recovery(struct ufs_hba *hba) { spin_lock_irq(hba->host->host_lock); hba->force_reset = true; ufshcd_schedule_eh_work(hba); spin_unlock_irq(hba->host->host_lock); } +EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery); static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) { @@ -6355,6 +6379,11 @@ static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend) void ufshcd_err_handling_prepare(struct ufs_hba *hba) { + /* + * A WLUN resume failure could potentially lead to the HBA being + * runtime suspended, so take an extra reference on hba->dev. + */ + pm_runtime_get_sync(hba->dev); ufshcd_rpm_get_sync(hba); if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) || hba->is_sys_suspended) { @@ -6396,6 +6425,7 @@ void ufshcd_err_handling_unprepare(struct ufs_hba *hba) if (ufshcd_is_clkscaling_supported(hba)) ufshcd_clk_scaling_suspend(hba, false); ufshcd_rpm_put(hba); + pm_runtime_put(hba->dev); } EXPORT_SYMBOL_GPL(ufshcd_err_handling_unprepare); @@ -6411,28 +6441,42 @@ static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) #ifdef CONFIG_PM static void ufshcd_recover_pm_error(struct ufs_hba *hba) { + struct scsi_target *starget = hba->ufs_device_wlun->sdev_target; struct Scsi_Host *shost = hba->host; struct scsi_device *sdev; struct request_queue *q; - int ret; + bool resume_sdev_queues = false; hba->is_sys_suspended = false; + /* - * Set RPM status of wlun device to RPM_ACTIVE, - * this also clears its runtime error. + * Ensure the parent's error status is cleared before proceeding + * to the child, as the parent must be active to activate the child. */ - ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); + if (hba->dev->power.runtime_error) { + /* hba->dev has no functional parent thus simplily set RPM_ACTIVE */ + pm_runtime_set_active(hba->dev); + resume_sdev_queues = true; + } + + if (hba->ufs_device_wlun->sdev_gendev.power.runtime_error) { + /* + * starget, parent of wlun, might be suspended if wlun resume failed. + * Make sure parent is resumed before set child (wlun) active. + */ + pm_runtime_get_sync(&starget->dev); + pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); + pm_runtime_put_sync(&starget->dev); + resume_sdev_queues = true; + } - /* hba device might have a runtime error otherwise */ - if (ret) - ret = pm_runtime_set_active(hba->dev); /* * If wlun device had runtime error, we also need to resume those * consumer scsi devices in case any of them has failed to be * resumed due to supplier runtime resume failure. This is to unblock * blk_queue_enter in case there are bios waiting inside it. */ - if (!ret) { + if (resume_sdev_queues) { shost_for_each_device(sdev, shost) { q = sdev->request_queue; if (q->dev && (q->rpm_status == RPM_SUSPENDED || @@ -6544,9 +6588,14 @@ static void ufshcd_err_handler(struct work_struct *work) up(&hba->host_sem); return; } - ufshcd_set_eh_in_progress(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_err_handling_prepare(hba); + + spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_set_eh_in_progress(hba); + spin_unlock_irqrestore(hba->host->host_lock, flags); + /* Complete requests that have door-bell cleared by h/w */ ufshcd_complete_requests(hba, false); spin_lock_irqsave(hba->host->host_lock, flags); @@ -8491,6 +8540,31 @@ static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) return ret; } +/** + * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. + * @hba: per-adapter instance + * + * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter + * to ensure proper hibernation timing. This function retrieves the current + * PA_HIBERN8TIME value and increments it by 100us. + */ +static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) +{ + u32 pa_h8time; + int ret; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); + if (ret) { + dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); + return; + } + + /* Increment by 1 to increase hibernation time by 100 µs */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); + if (ret) + dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); +} + static void ufshcd_tune_unipro_params(struct ufs_hba *hba) { if (ufshcd_is_unipro_pa_params_tuning_req(hba)) { @@ -8506,6 +8580,9 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) ufshcd_quirk_tune_host_pa_tactivate(hba); + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) + ufshcd_quirk_override_pa_h8time(hba); } static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) @@ -8660,7 +8737,8 @@ static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) struct ufs_dev_info *dev_info = &hba->dev_info; struct utp_upiu_query_v4_0 *upiu_data; - if (dev_info->wspecversion < 0x400) + if (dev_info->wspecversion < 0x400 || + hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT) return; ufshcd_hold(hba); diff --git a/drivers/ufs/host/ufs-exynos.c b/drivers/ufs/host/ufs-exynos.c index b876a307a160d..32399e3f38711 100644 --- a/drivers/ufs/host/ufs-exynos.c +++ b/drivers/ufs/host/ufs-exynos.c @@ -1028,8 +1028,8 @@ static int exynos_ufs_post_link(struct ufs_hba *hba) hci_writel(ufs, 0xa, HCI_DATA_REORDER); hci_writel(ufs, PRDT_SET_SIZE(12), HCI_TXPRDT_ENTRY_SIZE); hci_writel(ufs, PRDT_SET_SIZE(12), HCI_RXPRDT_ENTRY_SIZE); - hci_writel(ufs, (1 << hba->nutrs) - 1, HCI_UTRL_NEXUS_TYPE); - hci_writel(ufs, (1 << hba->nutmrs) - 1, HCI_UTMRL_NEXUS_TYPE); + hci_writel(ufs, BIT(hba->nutrs) - 1, HCI_UTRL_NEXUS_TYPE); + hci_writel(ufs, BIT(hba->nutmrs) - 1, HCI_UTMRL_NEXUS_TYPE); hci_writel(ufs, 0xf, HCI_AXIDMA_RWDATA_BURST_LEN); if (ufs->opts & EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB) diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index b07e010bdd571..c2450253085a2 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -847,6 +847,69 @@ static void ufs_mtk_vreg_fix_vccqx(struct ufs_hba *hba) } } +static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba) +{ + unsigned long flags; + u32 ah_ms = 10; + u32 ah_scale, ah_timer; + u32 scale_us[] = {1, 10, 100, 1000, 10000, 100000}; + + if (ufshcd_is_clkgating_allowed(hba)) { + if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit) { + ah_scale = FIELD_GET(UFSHCI_AHIBERN8_SCALE_MASK, + hba->ahit); + ah_timer = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, + hba->ahit); + if (ah_scale <= 5) + ah_ms = ah_timer * scale_us[ah_scale] / 1000; + } + + spin_lock_irqsave(hba->host->host_lock, flags); + hba->clk_gating.delay_ms = max(ah_ms, 10U); + spin_unlock_irqrestore(hba->host->host_lock, flags); + } +} + +/* Convert microseconds to Auto-Hibernate Idle Timer register value */ +static u32 ufs_mtk_us_to_ahit(unsigned int timer) +{ + unsigned int scale; + + for (scale = 0; timer > UFSHCI_AHIBERN8_TIMER_MASK; ++scale) + timer /= UFSHCI_AHIBERN8_SCALE_FACTOR; + + return FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, timer) | + FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, scale); +} + +static void ufs_mtk_fix_ahit(struct ufs_hba *hba) +{ + unsigned int us; + + if (ufshcd_is_auto_hibern8_supported(hba)) { + switch (hba->dev_info.wmanufacturerid) { + case UFS_VENDOR_SAMSUNG: + /* configure auto-hibern8 timer to 3.5 ms */ + us = 3500; + break; + + case UFS_VENDOR_MICRON: + /* configure auto-hibern8 timer to 2 ms */ + us = 2000; + break; + + default: + /* configure auto-hibern8 timer to 1 ms */ + us = 1000; + break; + } + + hba->ahit = ufs_mtk_us_to_ahit(us); + } + + ufs_mtk_setup_clk_gating(hba); +} + static void ufs_mtk_init_mcq_irq(struct ufs_hba *hba) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); @@ -1032,6 +1095,28 @@ static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba, ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_NO_ADAPT); + if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) { + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), + DL_FC0ProtectionTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), + DL_TC0ReplayTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), + DL_AFC0ReqTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), + DL_FC1ProtectionTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), + DL_TC1ReplayTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), + DL_AFC1ReqTimeOutVal_Default); + + ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal), + DL_FC0ProtectionTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal), + DL_TC0ReplayTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal), + DL_AFC0ReqTimeOutVal_Default); + } + ret = ufshcd_uic_change_pwr_mode(hba, FASTAUTO_MODE << 4 | FASTAUTO_MODE); @@ -1050,19 +1135,49 @@ static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba, return ret; } +static int ufs_mtk_auto_hibern8_disable(struct ufs_hba *hba) +{ + int ret; + + /* disable auto-hibern8 */ + ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER); + + /* wait host return to idle state when auto-hibern8 off */ + ufs_mtk_wait_idle_state(hba, 5); + + ret = ufs_mtk_wait_link_state(hba, VS_LINK_UP, 100); + if (ret) { + dev_warn(hba->dev, "exit h8 state fail, ret=%d\n", ret); + + ufshcd_force_error_recovery(hba); + + /* trigger error handler and break suspend */ + ret = -EBUSY; + } + + return ret; +} + static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status stage, struct ufs_pa_layer_attr *dev_max_params, struct ufs_pa_layer_attr *dev_req_params) { int ret = 0; + static u32 reg; switch (stage) { case PRE_CHANGE: + if (ufshcd_is_auto_hibern8_supported(hba)) { + reg = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER); + ufs_mtk_auto_hibern8_disable(hba); + } ret = ufs_mtk_pre_pwr_change(hba, dev_max_params, dev_req_params); break; case POST_CHANGE: + if (ufshcd_is_auto_hibern8_supported(hba)) + ufshcd_writel(hba, reg, REG_AUTO_HIBERNATE_IDLE_TIMER); break; default: ret = -EINVAL; @@ -1123,32 +1238,10 @@ static int ufs_mtk_pre_link(struct ufs_hba *hba) return ret; } - -static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba) -{ - u32 ah_ms; - - if (ufshcd_is_clkgating_allowed(hba)) { - if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit) - ah_ms = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, - hba->ahit); - else - ah_ms = 10; - ufshcd_clkgate_delay_set(hba->dev, ah_ms + 5); - } -} - static void ufs_mtk_post_link(struct ufs_hba *hba) { /* enable unipro clock gating feature */ ufs_mtk_cfg_unipro_cg(hba, true); - - /* will be configured during probe hba */ - if (ufshcd_is_auto_hibern8_supported(hba)) - hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 10) | - FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); - - ufs_mtk_setup_clk_gating(hba); } static int ufs_mtk_link_startup_notify(struct ufs_hba *hba, @@ -1175,11 +1268,11 @@ static int ufs_mtk_device_reset(struct ufs_hba *hba) { struct arm_smccc_res res; - /* disable hba before device reset */ - ufshcd_hba_stop(hba); - ufs_mtk_device_reset_ctrl(0, res); + /* disable hba in middle of device reset */ + ufshcd_hba_stop(hba); + /* * The reset signal is active low. UFS devices shall detect * more than or equal to 1us of positive or negative RST_n @@ -1257,6 +1350,9 @@ static void ufs_mtk_vccqx_set_lpm(struct ufs_hba *hba, bool lpm) { struct ufs_vreg *vccqx = NULL; + if (!hba->vreg_info.vccq && !hba->vreg_info.vccq2) + return; + if (hba->vreg_info.vccq) vccqx = hba->vreg_info.vccq; else @@ -1301,21 +1397,6 @@ static void ufs_mtk_dev_vreg_set_lpm(struct ufs_hba *hba, bool lpm) } } -static void ufs_mtk_auto_hibern8_disable(struct ufs_hba *hba) -{ - int ret; - - /* disable auto-hibern8 */ - ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER); - - /* wait host return to idle state when auto-hibern8 off */ - ufs_mtk_wait_idle_state(hba, 5); - - ret = ufs_mtk_wait_link_state(hba, VS_LINK_UP, 100); - if (ret) - dev_warn(hba->dev, "exit h8 state fail, ret=%d\n", ret); -} - static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, enum ufs_notify_change_status status) { @@ -1324,7 +1405,7 @@ static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, if (status == PRE_CHANGE) { if (ufshcd_is_auto_hibern8_supported(hba)) - ufs_mtk_auto_hibern8_disable(hba); + return ufs_mtk_auto_hibern8_disable(hba); return 0; } @@ -1382,8 +1463,21 @@ static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) } return 0; + fail: - return ufshcd_link_recovery(hba); + /* + * Check if the platform (parent) device has resumed, and ensure that + * power, clock, and MTCMOS are all turned on. + */ + err = ufshcd_link_recovery(hba); + if (err) { + dev_err(hba->dev, "Device PM: req=%d, status:%d, err:%d\n", + hba->dev->power.request, + hba->dev->power.runtime_status, + hba->dev->power.runtime_error); + } + + return 0; /* Cannot return a failure, otherwise, the I/O will hang. */ } static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba) @@ -1448,6 +1542,7 @@ static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba) ufs_mtk_vreg_fix_vcc(hba); ufs_mtk_vreg_fix_vccqx(hba); + ufs_mtk_fix_ahit(hba); } static void ufs_mtk_event_notify(struct ufs_hba *hba, diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c index 248a49e5e7f35..469b123eef699 100644 --- a/drivers/ufs/host/ufshcd-pci.c +++ b/drivers/ufs/host/ufshcd-pci.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ struct intel_host { u32 dsm_fns; u32 active_ltr; u32 idle_ltr; + int saved_spm_lvl; struct dentry *debugfs_root; struct gpio_desc *reset_gpio; }; @@ -213,6 +215,32 @@ static int ufs_intel_lkf_apply_dev_quirks(struct ufs_hba *hba) return ret; } +static void ufs_intel_ctrl_uic_compl(struct ufs_hba *hba, bool enable) +{ + u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + + if (enable) + set |= UIC_COMMAND_COMPL; + else + set &= ~UIC_COMMAND_COMPL; + ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); +} + +static void ufs_intel_mtl_h8_notify(struct ufs_hba *hba, + enum uic_cmd_dme cmd, + enum ufs_notify_change_status status) +{ + /* + * Disable UIC COMPL INTR to prevent access to UFSHCI after + * checking HCS.UPMCRS + */ + if (status == PRE_CHANGE && cmd == UIC_CMD_DME_HIBER_ENTER) + ufs_intel_ctrl_uic_compl(hba, false); + + if (status == POST_CHANGE && cmd == UIC_CMD_DME_HIBER_EXIT) + ufs_intel_ctrl_uic_compl(hba, true); +} + #define INTEL_ACTIVELTR 0x804 #define INTEL_IDLELTR 0x808 @@ -349,6 +377,7 @@ static int ufs_intel_common_init(struct ufs_hba *hba) host = devm_kzalloc(hba->dev, sizeof(*host), GFP_KERNEL); if (!host) return -ENOMEM; + host->saved_spm_lvl = -1; ufshcd_set_variant(hba, host); intel_dsm_init(host, hba->dev); if (INTEL_DSM_SUPPORTED(host, RESET)) { @@ -439,10 +468,23 @@ static int ufs_intel_adl_init(struct ufs_hba *hba) return ufs_intel_common_init(hba); } +static void ufs_intel_mtl_late_init(struct ufs_hba *hba) +{ + hba->rpm_lvl = UFS_PM_LVL_2; + hba->spm_lvl = UFS_PM_LVL_2; +} + static int ufs_intel_mtl_init(struct ufs_hba *hba) { + struct ufs_host *ufs_host; + int err; + hba->caps |= UFSHCD_CAP_CRYPTO | UFSHCD_CAP_WB_EN; - return ufs_intel_common_init(hba); + err = ufs_intel_common_init(hba); + /* Get variant after it is set in ufs_intel_common_init() */ + ufs_host = ufshcd_get_variant(hba); + ufs_host->late_init = ufs_intel_mtl_late_init; + return err; } static struct ufs_hba_variant_ops ufs_intel_cnl_hba_vops = { @@ -487,6 +529,7 @@ static struct ufs_hba_variant_ops ufs_intel_mtl_hba_vops = { .init = ufs_intel_mtl_init, .exit = ufs_intel_common_exit, .hce_enable_notify = ufs_intel_hce_enable_notify, + .hibern8_notify = ufs_intel_mtl_h8_notify, .link_startup_notify = ufs_intel_link_startup_notify, .resume = ufs_intel_resume, .device_reset = ufs_intel_device_reset, @@ -502,6 +545,66 @@ static int ufshcd_pci_restore(struct device *dev) return ufshcd_system_resume(dev); } + +static int ufs_intel_suspend_prepare(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct intel_host *host = ufshcd_get_variant(hba); + int err; + + /* + * Only s2idle (S0ix) retains link state. Force power-off + * (UFS_PM_LVL_5) for any other case. + */ + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && hba->spm_lvl < UFS_PM_LVL_5) { + host->saved_spm_lvl = hba->spm_lvl; + hba->spm_lvl = UFS_PM_LVL_5; + } + + err = ufshcd_suspend_prepare(dev); + + if (err < 0 && host->saved_spm_lvl != -1) { + hba->spm_lvl = host->saved_spm_lvl; + host->saved_spm_lvl = -1; + } + + return err; +} + +static void ufs_intel_resume_complete(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct intel_host *host = ufshcd_get_variant(hba); + + ufshcd_resume_complete(dev); + + if (host->saved_spm_lvl != -1) { + hba->spm_lvl = host->saved_spm_lvl; + host->saved_spm_lvl = -1; + } +} + +static int ufshcd_pci_suspend_prepare(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!strcmp(hba->vops->name, "intel-pci")) + return ufs_intel_suspend_prepare(dev); + + return ufshcd_suspend_prepare(dev); +} + +static void ufshcd_pci_resume_complete(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!strcmp(hba->vops->name, "intel-pci")) { + ufs_intel_resume_complete(dev); + return; + } + + ufshcd_resume_complete(dev); +} #endif /** @@ -584,8 +687,8 @@ static const struct dev_pm_ops ufshcd_pci_pm_ops = { .thaw = ufshcd_system_resume, .poweroff = ufshcd_system_suspend, .restore = ufshcd_pci_restore, - .prepare = ufshcd_suspend_prepare, - .complete = ufshcd_resume_complete, + .prepare = ufshcd_pci_suspend_prepare, + .complete = ufshcd_pci_resume_complete, #endif }; diff --git a/drivers/uio/uio_hv_generic.c b/drivers/uio/uio_hv_generic.c index a2c7abf8c289e..69e5016ebd463 100644 --- a/drivers/uio/uio_hv_generic.c +++ b/drivers/uio/uio_hv_generic.c @@ -80,9 +80,15 @@ hv_uio_irqcontrol(struct uio_info *info, s32 irq_state) { struct hv_uio_private_data *pdata = info->priv; struct hv_device *dev = pdata->device; + struct vmbus_channel *primary, *sc; - dev->channel->inbound.ring_buffer->interrupt_mask = !irq_state; - virt_mb(); + primary = dev->channel; + primary->inbound.ring_buffer->interrupt_mask = !irq_state; + + mutex_lock(&vmbus_connection.channel_mutex); + list_for_each_entry(sc, &primary->sc_list, sc_list) + sc->inbound.ring_buffer->interrupt_mask = !irq_state; + mutex_unlock(&vmbus_connection.channel_mutex); return 0; } @@ -93,12 +99,18 @@ hv_uio_irqcontrol(struct uio_info *info, s32 irq_state) static void hv_uio_channel_cb(void *context) { struct vmbus_channel *chan = context; - struct hv_device *hv_dev = chan->device_obj; - struct hv_uio_private_data *pdata = hv_get_drvdata(hv_dev); + struct hv_device *hv_dev; + struct hv_uio_private_data *pdata; - chan->inbound.ring_buffer->interrupt_mask = 1; virt_mb(); + /* + * The callback may come from a subchannel, in which case look + * for the hv device in the primary channel + */ + hv_dev = chan->primary_channel ? + chan->primary_channel->device_obj : chan->device_obj; + pdata = hv_get_drvdata(hv_dev); uio_event_notify(&pdata->info); } @@ -173,8 +185,6 @@ hv_uio_new_channel(struct vmbus_channel *new_sc) return; } - /* Disable interrupts on sub channel */ - new_sc->inbound.ring_buffer->interrupt_mask = 1; set_channel_read_mode(new_sc, HV_CALL_ISR); ret = sysfs_create_bin_file(&new_sc->kobj, &ring_buffer_bin_attr); @@ -218,9 +228,7 @@ hv_uio_open(struct uio_info *info, struct inode *inode) ret = vmbus_connect_ring(dev->channel, hv_uio_channel_cb, dev->channel); - if (ret == 0) - dev->channel->inbound.ring_buffer->interrupt_mask = 1; - else + if (ret) atomic_dec(&pdata->refcnt); return ret; @@ -249,6 +257,7 @@ hv_uio_probe(struct hv_device *dev, struct hv_uio_private_data *pdata; void *ring_buffer; int ret; + size_t ring_size = hv_dev_ring_size(channel); /* Communicating with host has to be via shared memory not hypercall */ if (!channel->offermsg.monitor_allocated) { @@ -256,12 +265,17 @@ hv_uio_probe(struct hv_device *dev, return -ENOTSUPP; } + if (!ring_size) + ring_size = HV_RING_SIZE * PAGE_SIZE; + + /* Adjust ring size if necessary to have it page aligned */ + ring_size = VMBUS_RING_SIZE(ring_size); + pdata = devm_kzalloc(&dev->device, sizeof(*pdata), GFP_KERNEL); if (!pdata) return -ENOMEM; - ret = vmbus_alloc_ring(channel, HV_RING_SIZE * PAGE_SIZE, - HV_RING_SIZE * PAGE_SIZE); + ret = vmbus_alloc_ring(channel, ring_size, ring_size); if (ret) return ret; @@ -288,13 +302,13 @@ hv_uio_probe(struct hv_device *dev, pdata->info.mem[INT_PAGE_MAP].name = "int_page"; pdata->info.mem[INT_PAGE_MAP].addr = (uintptr_t)vmbus_connection.int_page; - pdata->info.mem[INT_PAGE_MAP].size = PAGE_SIZE; + pdata->info.mem[INT_PAGE_MAP].size = HV_HYP_PAGE_SIZE; pdata->info.mem[INT_PAGE_MAP].memtype = UIO_MEM_LOGICAL; pdata->info.mem[MON_PAGE_MAP].name = "monitor_page"; pdata->info.mem[MON_PAGE_MAP].addr = (uintptr_t)vmbus_connection.monitor_pages[1]; - pdata->info.mem[MON_PAGE_MAP].size = PAGE_SIZE; + pdata->info.mem[MON_PAGE_MAP].size = HV_HYP_PAGE_SIZE; pdata->info.mem[MON_PAGE_MAP].memtype = UIO_MEM_LOGICAL; pdata->recv_buf = vzalloc(RECV_BUFFER_SIZE); diff --git a/drivers/usb/atm/cxacru.c b/drivers/usb/atm/cxacru.c index 1443e9cf631a6..db9a8f2731f1a 100644 --- a/drivers/usb/atm/cxacru.c +++ b/drivers/usb/atm/cxacru.c @@ -980,25 +980,60 @@ static int cxacru_fw(struct usb_device *usb_dev, enum cxacru_fw_request fw, return ret; } -static void cxacru_upload_firmware(struct cxacru_data *instance, - const struct firmware *fw, - const struct firmware *bp) + +static int cxacru_find_firmware(struct cxacru_data *instance, + char *phase, const struct firmware **fw_p) { - int ret; + struct usbatm_data *usbatm = instance->usbatm; + struct device *dev = &usbatm->usb_intf->dev; + char buf[16]; + + sprintf(buf, "cxacru-%s.bin", phase); + usb_dbg(usbatm, "cxacru_find_firmware: looking for %s\n", buf); + + if (request_firmware(fw_p, buf, dev)) { + usb_dbg(usbatm, "no stage %s firmware found\n", phase); + return -ENOENT; + } + + usb_info(usbatm, "found firmware %s\n", buf); + + return 0; +} + +static int cxacru_heavy_init(struct usbatm_data *usbatm_instance, + struct usb_interface *usb_intf) +{ + const struct firmware *fw, *bp; + struct cxacru_data *instance = usbatm_instance->driver_data; struct usbatm_data *usbatm = instance->usbatm; struct usb_device *usb_dev = usbatm->usb_dev; __le16 signature[] = { usb_dev->descriptor.idVendor, usb_dev->descriptor.idProduct }; __le32 val; + int ret; - usb_dbg(usbatm, "%s\n", __func__); + ret = cxacru_find_firmware(instance, "fw", &fw); + if (ret) { + usb_warn(usbatm_instance, "firmware (cxacru-fw.bin) unavailable (system misconfigured?)\n"); + return ret; + } + + if (instance->modem_type->boot_rom_patch) { + ret = cxacru_find_firmware(instance, "bp", &bp); + if (ret) { + usb_warn(usbatm_instance, "boot ROM patch (cxacru-bp.bin) unavailable (system misconfigured?)\n"); + release_firmware(fw); + return ret; + } + } /* FirmwarePllFClkValue */ val = cpu_to_le32(instance->modem_type->pll_f_clk); ret = cxacru_fw(usb_dev, FW_WRITE_MEM, 0x2, 0x0, PLLFCLK_ADDR, (u8 *) &val, 4); if (ret) { usb_err(usbatm, "FirmwarePllFClkValue failed: %d\n", ret); - return; + goto done; } /* FirmwarePllBClkValue */ @@ -1006,7 +1041,7 @@ static void cxacru_upload_firmware(struct cxacru_data *instance, ret = cxacru_fw(usb_dev, FW_WRITE_MEM, 0x2, 0x0, PLLBCLK_ADDR, (u8 *) &val, 4); if (ret) { usb_err(usbatm, "FirmwarePllBClkValue failed: %d\n", ret); - return; + goto done; } /* Enable SDRAM */ @@ -1014,7 +1049,7 @@ static void cxacru_upload_firmware(struct cxacru_data *instance, ret = cxacru_fw(usb_dev, FW_WRITE_MEM, 0x2, 0x0, SDRAMEN_ADDR, (u8 *) &val, 4); if (ret) { usb_err(usbatm, "Enable SDRAM failed: %d\n", ret); - return; + goto done; } /* Firmware */ @@ -1022,7 +1057,7 @@ static void cxacru_upload_firmware(struct cxacru_data *instance, ret = cxacru_fw(usb_dev, FW_WRITE_MEM, 0x2, 0x0, FW_ADDR, fw->data, fw->size); if (ret) { usb_err(usbatm, "Firmware upload failed: %d\n", ret); - return; + goto done; } /* Boot ROM patch */ @@ -1031,7 +1066,7 @@ static void cxacru_upload_firmware(struct cxacru_data *instance, ret = cxacru_fw(usb_dev, FW_WRITE_MEM, 0x2, 0x0, BR_ADDR, bp->data, bp->size); if (ret) { usb_err(usbatm, "Boot ROM patching failed: %d\n", ret); - return; + goto done; } } @@ -1039,7 +1074,7 @@ static void cxacru_upload_firmware(struct cxacru_data *instance, ret = cxacru_fw(usb_dev, FW_WRITE_MEM, 0x2, 0x0, SIG_ADDR, (u8 *) signature, 4); if (ret) { usb_err(usbatm, "Signature storing failed: %d\n", ret); - return; + goto done; } usb_info(usbatm, "starting device\n"); @@ -1051,7 +1086,7 @@ static void cxacru_upload_firmware(struct cxacru_data *instance, } if (ret) { usb_err(usbatm, "Passing control to firmware failed: %d\n", ret); - return; + goto done; } /* Delay to allow firmware to start up. */ @@ -1065,53 +1100,10 @@ static void cxacru_upload_firmware(struct cxacru_data *instance, ret = cxacru_cm(instance, CM_REQUEST_CARD_GET_STATUS, NULL, 0, NULL, 0); if (ret < 0) { usb_err(usbatm, "modem failed to initialize: %d\n", ret); - return; - } -} - -static int cxacru_find_firmware(struct cxacru_data *instance, - char *phase, const struct firmware **fw_p) -{ - struct usbatm_data *usbatm = instance->usbatm; - struct device *dev = &usbatm->usb_intf->dev; - char buf[16]; - - sprintf(buf, "cxacru-%s.bin", phase); - usb_dbg(usbatm, "cxacru_find_firmware: looking for %s\n", buf); - - if (request_firmware(fw_p, buf, dev)) { - usb_dbg(usbatm, "no stage %s firmware found\n", phase); - return -ENOENT; - } - - usb_info(usbatm, "found firmware %s\n", buf); - - return 0; -} - -static int cxacru_heavy_init(struct usbatm_data *usbatm_instance, - struct usb_interface *usb_intf) -{ - const struct firmware *fw, *bp; - struct cxacru_data *instance = usbatm_instance->driver_data; - int ret = cxacru_find_firmware(instance, "fw", &fw); - - if (ret) { - usb_warn(usbatm_instance, "firmware (cxacru-fw.bin) unavailable (system misconfigured?)\n"); - return ret; + goto done; } - if (instance->modem_type->boot_rom_patch) { - ret = cxacru_find_firmware(instance, "bp", &bp); - if (ret) { - usb_warn(usbatm_instance, "boot ROM patch (cxacru-bp.bin) unavailable (system misconfigured?)\n"); - release_firmware(fw); - return ret; - } - } - - cxacru_upload_firmware(instance, fw, bp); - +done: if (instance->modem_type->boot_rom_patch) release_firmware(bp); release_firmware(fw); diff --git a/drivers/usb/cdns3/cdnsp-debug.h b/drivers/usb/cdns3/cdnsp-debug.h index f0ca865cce2a0..86860686d8363 100644 --- a/drivers/usb/cdns3/cdnsp-debug.h +++ b/drivers/usb/cdns3/cdnsp-debug.h @@ -131,8 +131,6 @@ static inline const char *cdnsp_trb_type_string(u8 type) return "Endpoint Not ready"; case TRB_HALT_ENDPOINT: return "Halt Endpoint"; - case TRB_FLUSH_ENDPOINT: - return "FLush Endpoint"; default: return "UNKNOWN"; } @@ -189,203 +187,203 @@ static inline const char *cdnsp_decode_trb(char *str, size_t size, u32 field0, switch (type) { case TRB_LINK: - ret = snprintf(str, size, - "LINK %08x%08x intr %ld type '%s' flags %c:%c:%c:%c", - field1, field0, GET_INTR_TARGET(field2), - cdnsp_trb_type_string(type), - field3 & TRB_IOC ? 'I' : 'i', - field3 & TRB_CHAIN ? 'C' : 'c', - field3 & TRB_TC ? 'T' : 't', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "LINK %08x%08x intr %ld type '%s' flags %c:%c:%c:%c", + field1, field0, GET_INTR_TARGET(field2), + cdnsp_trb_type_string(type), + field3 & TRB_IOC ? 'I' : 'i', + field3 & TRB_CHAIN ? 'C' : 'c', + field3 & TRB_TC ? 'T' : 't', + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_TRANSFER: case TRB_COMPLETION: case TRB_PORT_STATUS: case TRB_HC_EVENT: - ret = snprintf(str, size, - "ep%d%s(%d) type '%s' TRB %08x%08x status '%s'" - " len %ld slot %ld flags %c:%c", - ep_num, ep_id % 2 ? "out" : "in", - TRB_TO_EP_INDEX(field3), - cdnsp_trb_type_string(type), field1, field0, - cdnsp_trb_comp_code_string(GET_COMP_CODE(field2)), - EVENT_TRB_LEN(field2), TRB_TO_SLOT_ID(field3), - field3 & EVENT_DATA ? 'E' : 'e', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "ep%d%s(%d) type '%s' TRB %08x%08x status '%s'" + " len %ld slot %ld flags %c:%c", + ep_num, ep_id % 2 ? "out" : "in", + TRB_TO_EP_INDEX(field3), + cdnsp_trb_type_string(type), field1, field0, + cdnsp_trb_comp_code_string(GET_COMP_CODE(field2)), + EVENT_TRB_LEN(field2), TRB_TO_SLOT_ID(field3), + field3 & EVENT_DATA ? 'E' : 'e', + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_MFINDEX_WRAP: - ret = snprintf(str, size, "%s: flags %c", - cdnsp_trb_type_string(type), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, "%s: flags %c", + cdnsp_trb_type_string(type), + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_SETUP: - ret = snprintf(str, size, - "type '%s' bRequestType %02x bRequest %02x " - "wValue %02x%02x wIndex %02x%02x wLength %d " - "length %ld TD size %ld intr %ld Setup ID %ld " - "flags %c:%c:%c", - cdnsp_trb_type_string(type), - field0 & 0xff, - (field0 & 0xff00) >> 8, - (field0 & 0xff000000) >> 24, - (field0 & 0xff0000) >> 16, - (field1 & 0xff00) >> 8, - field1 & 0xff, - (field1 & 0xff000000) >> 16 | - (field1 & 0xff0000) >> 16, - TRB_LEN(field2), GET_TD_SIZE(field2), - GET_INTR_TARGET(field2), - TRB_SETUPID_TO_TYPE(field3), - field3 & TRB_IDT ? 'D' : 'd', - field3 & TRB_IOC ? 'I' : 'i', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "type '%s' bRequestType %02x bRequest %02x " + "wValue %02x%02x wIndex %02x%02x wLength %d " + "length %ld TD size %ld intr %ld Setup ID %ld " + "flags %c:%c:%c", + cdnsp_trb_type_string(type), + field0 & 0xff, + (field0 & 0xff00) >> 8, + (field0 & 0xff000000) >> 24, + (field0 & 0xff0000) >> 16, + (field1 & 0xff00) >> 8, + field1 & 0xff, + (field1 & 0xff000000) >> 16 | + (field1 & 0xff0000) >> 16, + TRB_LEN(field2), GET_TD_SIZE(field2), + GET_INTR_TARGET(field2), + TRB_SETUPID_TO_TYPE(field3), + field3 & TRB_IDT ? 'D' : 'd', + field3 & TRB_IOC ? 'I' : 'i', + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_DATA: - ret = snprintf(str, size, - "type '%s' Buffer %08x%08x length %ld TD size %ld " - "intr %ld flags %c:%c:%c:%c:%c:%c:%c", - cdnsp_trb_type_string(type), - field1, field0, TRB_LEN(field2), - GET_TD_SIZE(field2), - GET_INTR_TARGET(field2), - field3 & TRB_IDT ? 'D' : 'i', - field3 & TRB_IOC ? 'I' : 'i', - field3 & TRB_CHAIN ? 'C' : 'c', - field3 & TRB_NO_SNOOP ? 'S' : 's', - field3 & TRB_ISP ? 'I' : 'i', - field3 & TRB_ENT ? 'E' : 'e', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "type '%s' Buffer %08x%08x length %ld TD size %ld " + "intr %ld flags %c:%c:%c:%c:%c:%c:%c", + cdnsp_trb_type_string(type), + field1, field0, TRB_LEN(field2), + GET_TD_SIZE(field2), + GET_INTR_TARGET(field2), + field3 & TRB_IDT ? 'D' : 'i', + field3 & TRB_IOC ? 'I' : 'i', + field3 & TRB_CHAIN ? 'C' : 'c', + field3 & TRB_NO_SNOOP ? 'S' : 's', + field3 & TRB_ISP ? 'I' : 'i', + field3 & TRB_ENT ? 'E' : 'e', + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_STATUS: - ret = snprintf(str, size, - "Buffer %08x%08x length %ld TD size %ld intr" - "%ld type '%s' flags %c:%c:%c:%c", - field1, field0, TRB_LEN(field2), - GET_TD_SIZE(field2), - GET_INTR_TARGET(field2), - cdnsp_trb_type_string(type), - field3 & TRB_IOC ? 'I' : 'i', - field3 & TRB_CHAIN ? 'C' : 'c', - field3 & TRB_ENT ? 'E' : 'e', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "Buffer %08x%08x length %ld TD size %ld intr" + "%ld type '%s' flags %c:%c:%c:%c", + field1, field0, TRB_LEN(field2), + GET_TD_SIZE(field2), + GET_INTR_TARGET(field2), + cdnsp_trb_type_string(type), + field3 & TRB_IOC ? 'I' : 'i', + field3 & TRB_CHAIN ? 'C' : 'c', + field3 & TRB_ENT ? 'E' : 'e', + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_NORMAL: case TRB_ISOC: case TRB_EVENT_DATA: case TRB_TR_NOOP: - ret = snprintf(str, size, - "type '%s' Buffer %08x%08x length %ld " - "TD size %ld intr %ld " - "flags %c:%c:%c:%c:%c:%c:%c:%c:%c", - cdnsp_trb_type_string(type), - field1, field0, TRB_LEN(field2), - GET_TD_SIZE(field2), - GET_INTR_TARGET(field2), - field3 & TRB_BEI ? 'B' : 'b', - field3 & TRB_IDT ? 'T' : 't', - field3 & TRB_IOC ? 'I' : 'i', - field3 & TRB_CHAIN ? 'C' : 'c', - field3 & TRB_NO_SNOOP ? 'S' : 's', - field3 & TRB_ISP ? 'I' : 'i', - field3 & TRB_ENT ? 'E' : 'e', - field3 & TRB_CYCLE ? 'C' : 'c', - !(field3 & TRB_EVENT_INVALIDATE) ? 'V' : 'v'); + ret = scnprintf(str, size, + "type '%s' Buffer %08x%08x length %ld " + "TD size %ld intr %ld " + "flags %c:%c:%c:%c:%c:%c:%c:%c:%c", + cdnsp_trb_type_string(type), + field1, field0, TRB_LEN(field2), + GET_TD_SIZE(field2), + GET_INTR_TARGET(field2), + field3 & TRB_BEI ? 'B' : 'b', + field3 & TRB_IDT ? 'T' : 't', + field3 & TRB_IOC ? 'I' : 'i', + field3 & TRB_CHAIN ? 'C' : 'c', + field3 & TRB_NO_SNOOP ? 'S' : 's', + field3 & TRB_ISP ? 'I' : 'i', + field3 & TRB_ENT ? 'E' : 'e', + field3 & TRB_CYCLE ? 'C' : 'c', + !(field3 & TRB_EVENT_INVALIDATE) ? 'V' : 'v'); break; case TRB_CMD_NOOP: case TRB_ENABLE_SLOT: - ret = snprintf(str, size, "%s: flags %c", - cdnsp_trb_type_string(type), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, "%s: flags %c", + cdnsp_trb_type_string(type), + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_DISABLE_SLOT: - ret = snprintf(str, size, "%s: slot %ld flags %c", - cdnsp_trb_type_string(type), - TRB_TO_SLOT_ID(field3), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, "%s: slot %ld flags %c", + cdnsp_trb_type_string(type), + TRB_TO_SLOT_ID(field3), + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_ADDR_DEV: - ret = snprintf(str, size, - "%s: ctx %08x%08x slot %ld flags %c:%c", - cdnsp_trb_type_string(type), field1, field0, - TRB_TO_SLOT_ID(field3), - field3 & TRB_BSR ? 'B' : 'b', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "%s: ctx %08x%08x slot %ld flags %c:%c", + cdnsp_trb_type_string(type), field1, field0, + TRB_TO_SLOT_ID(field3), + field3 & TRB_BSR ? 'B' : 'b', + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_CONFIG_EP: - ret = snprintf(str, size, - "%s: ctx %08x%08x slot %ld flags %c:%c", - cdnsp_trb_type_string(type), field1, field0, - TRB_TO_SLOT_ID(field3), - field3 & TRB_DC ? 'D' : 'd', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "%s: ctx %08x%08x slot %ld flags %c:%c", + cdnsp_trb_type_string(type), field1, field0, + TRB_TO_SLOT_ID(field3), + field3 & TRB_DC ? 'D' : 'd', + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_EVAL_CONTEXT: - ret = snprintf(str, size, - "%s: ctx %08x%08x slot %ld flags %c", - cdnsp_trb_type_string(type), field1, field0, - TRB_TO_SLOT_ID(field3), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "%s: ctx %08x%08x slot %ld flags %c", + cdnsp_trb_type_string(type), field1, field0, + TRB_TO_SLOT_ID(field3), + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_RESET_EP: case TRB_HALT_ENDPOINT: - case TRB_FLUSH_ENDPOINT: - ret = snprintf(str, size, - "%s: ep%d%s(%d) ctx %08x%08x slot %ld flags %c", - cdnsp_trb_type_string(type), - ep_num, ep_id % 2 ? "out" : "in", - TRB_TO_EP_INDEX(field3), field1, field0, - TRB_TO_SLOT_ID(field3), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "%s: ep%d%s(%d) ctx %08x%08x slot %ld flags %c %c", + cdnsp_trb_type_string(type), + ep_num, ep_id % 2 ? "out" : "in", + TRB_TO_EP_INDEX(field3), field1, field0, + TRB_TO_SLOT_ID(field3), + field3 & TRB_CYCLE ? 'C' : 'c', + field3 & TRB_ESP ? 'P' : 'p'); break; case TRB_STOP_RING: - ret = snprintf(str, size, - "%s: ep%d%s(%d) slot %ld sp %d flags %c", - cdnsp_trb_type_string(type), - ep_num, ep_id % 2 ? "out" : "in", - TRB_TO_EP_INDEX(field3), - TRB_TO_SLOT_ID(field3), - TRB_TO_SUSPEND_PORT(field3), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "%s: ep%d%s(%d) slot %ld sp %d flags %c", + cdnsp_trb_type_string(type), + ep_num, ep_id % 2 ? "out" : "in", + TRB_TO_EP_INDEX(field3), + TRB_TO_SLOT_ID(field3), + TRB_TO_SUSPEND_PORT(field3), + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_SET_DEQ: - ret = snprintf(str, size, - "%s: ep%d%s(%d) deq %08x%08x stream %ld slot %ld flags %c", - cdnsp_trb_type_string(type), - ep_num, ep_id % 2 ? "out" : "in", - TRB_TO_EP_INDEX(field3), field1, field0, - TRB_TO_STREAM_ID(field2), - TRB_TO_SLOT_ID(field3), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "%s: ep%d%s(%d) deq %08x%08x stream %ld slot %ld flags %c", + cdnsp_trb_type_string(type), + ep_num, ep_id % 2 ? "out" : "in", + TRB_TO_EP_INDEX(field3), field1, field0, + TRB_TO_STREAM_ID(field2), + TRB_TO_SLOT_ID(field3), + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_RESET_DEV: - ret = snprintf(str, size, "%s: slot %ld flags %c", - cdnsp_trb_type_string(type), - TRB_TO_SLOT_ID(field3), - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, "%s: slot %ld flags %c", + cdnsp_trb_type_string(type), + TRB_TO_SLOT_ID(field3), + field3 & TRB_CYCLE ? 'C' : 'c'); break; case TRB_ENDPOINT_NRDY: temp = TRB_TO_HOST_STREAM(field2); - ret = snprintf(str, size, - "%s: ep%d%s(%d) H_SID %x%s%s D_SID %lx flags %c:%c", - cdnsp_trb_type_string(type), - ep_num, ep_id % 2 ? "out" : "in", - TRB_TO_EP_INDEX(field3), temp, - temp == STREAM_PRIME_ACK ? "(PRIME)" : "", - temp == STREAM_REJECTED ? "(REJECTED)" : "", - TRB_TO_DEV_STREAM(field0), - field3 & TRB_STAT ? 'S' : 's', - field3 & TRB_CYCLE ? 'C' : 'c'); + ret = scnprintf(str, size, + "%s: ep%d%s(%d) H_SID %x%s%s D_SID %lx flags %c:%c", + cdnsp_trb_type_string(type), + ep_num, ep_id % 2 ? "out" : "in", + TRB_TO_EP_INDEX(field3), temp, + temp == STREAM_PRIME_ACK ? "(PRIME)" : "", + temp == STREAM_REJECTED ? "(REJECTED)" : "", + TRB_TO_DEV_STREAM(field0), + field3 & TRB_STAT ? 'S' : 's', + field3 & TRB_CYCLE ? 'C' : 'c'); break; default: - ret = snprintf(str, size, - "type '%s' -> raw %08x %08x %08x %08x", - cdnsp_trb_type_string(type), - field0, field1, field2, field3); + ret = scnprintf(str, size, + "type '%s' -> raw %08x %08x %08x %08x", + cdnsp_trb_type_string(type), + field0, field1, field2, field3); } - if (ret >= size) - pr_info("CDNSP: buffer overflowed.\n"); + if (ret == size - 1) + pr_info("CDNSP: buffer may be truncated.\n"); return str; } @@ -468,32 +466,32 @@ static inline const char *cdnsp_decode_portsc(char *str, size_t size, { int ret; - ret = snprintf(str, size, "%s %s %s Link:%s PortSpeed:%d ", - portsc & PORT_POWER ? "Powered" : "Powered-off", - portsc & PORT_CONNECT ? "Connected" : "Not-connected", - portsc & PORT_PED ? "Enabled" : "Disabled", - cdnsp_portsc_link_state_string(portsc), - DEV_PORT_SPEED(portsc)); + ret = scnprintf(str, size, "%s %s %s Link:%s PortSpeed:%d ", + portsc & PORT_POWER ? "Powered" : "Powered-off", + portsc & PORT_CONNECT ? "Connected" : "Not-connected", + portsc & PORT_PED ? "Enabled" : "Disabled", + cdnsp_portsc_link_state_string(portsc), + DEV_PORT_SPEED(portsc)); if (portsc & PORT_RESET) - ret += snprintf(str + ret, size - ret, "In-Reset "); + ret += scnprintf(str + ret, size - ret, "In-Reset "); - ret += snprintf(str + ret, size - ret, "Change: "); + ret += scnprintf(str + ret, size - ret, "Change: "); if (portsc & PORT_CSC) - ret += snprintf(str + ret, size - ret, "CSC "); + ret += scnprintf(str + ret, size - ret, "CSC "); if (portsc & PORT_WRC) - ret += snprintf(str + ret, size - ret, "WRC "); + ret += scnprintf(str + ret, size - ret, "WRC "); if (portsc & PORT_RC) - ret += snprintf(str + ret, size - ret, "PRC "); + ret += scnprintf(str + ret, size - ret, "PRC "); if (portsc & PORT_PLC) - ret += snprintf(str + ret, size - ret, "PLC "); + ret += scnprintf(str + ret, size - ret, "PLC "); if (portsc & PORT_CEC) - ret += snprintf(str + ret, size - ret, "CEC "); - ret += snprintf(str + ret, size - ret, "Wake: "); + ret += scnprintf(str + ret, size - ret, "CEC "); + ret += scnprintf(str + ret, size - ret, "Wake: "); if (portsc & PORT_WKCONN_E) - ret += snprintf(str + ret, size - ret, "WCE "); + ret += scnprintf(str + ret, size - ret, "WCE "); if (portsc & PORT_WKDISC_E) - ret += snprintf(str + ret, size - ret, "WDE "); + ret += scnprintf(str + ret, size - ret, "WDE "); return str; } @@ -565,20 +563,20 @@ static inline const char *cdnsp_decode_ep_context(char *str, size_t size, avg = EP_AVG_TRB_LENGTH(tx_info); - ret = snprintf(str, size, "State %s mult %d max P. Streams %d %s", - cdnsp_ep_state_string(ep_state), mult, - max_pstr, lsa ? "LSA " : ""); + ret = scnprintf(str, size, "State %s mult %d max P. Streams %d %s", + cdnsp_ep_state_string(ep_state), mult, + max_pstr, lsa ? "LSA " : ""); - ret += snprintf(str + ret, size - ret, - "interval %d us max ESIT payload %d CErr %d ", - (1 << interval) * 125, esit, cerr); + ret += scnprintf(str + ret, size - ret, + "interval %d us max ESIT payload %d CErr %d ", + (1 << interval) * 125, esit, cerr); - ret += snprintf(str + ret, size - ret, - "Type %s %sburst %d maxp %d deq %016llx ", - cdnsp_ep_type_string(ep_type), hid ? "HID" : "", - burst, maxp, deq); + ret += scnprintf(str + ret, size - ret, + "Type %s %sburst %d maxp %d deq %016llx ", + cdnsp_ep_type_string(ep_type), hid ? "HID" : "", + burst, maxp, deq); - ret += snprintf(str + ret, size - ret, "avg trb len %d", avg); + ret += scnprintf(str + ret, size - ret, "avg trb len %d", avg); return str; } diff --git a/drivers/usb/cdns3/cdnsp-ep0.c b/drivers/usb/cdns3/cdnsp-ep0.c index f317d3c847810..5cd9b898ce971 100644 --- a/drivers/usb/cdns3/cdnsp-ep0.c +++ b/drivers/usb/cdns3/cdnsp-ep0.c @@ -414,6 +414,7 @@ static int cdnsp_ep0_std_request(struct cdnsp_device *pdev, void cdnsp_setup_analyze(struct cdnsp_device *pdev) { struct usb_ctrlrequest *ctrl = &pdev->setup; + struct cdnsp_ep *pep; int ret = -EINVAL; u16 len; @@ -427,10 +428,21 @@ void cdnsp_setup_analyze(struct cdnsp_device *pdev) goto out; } + pep = &pdev->eps[0]; + /* Restore the ep0 to Stopped/Running state. */ - if (pdev->eps[0].ep_state & EP_HALTED) { - trace_cdnsp_ep0_halted("Restore to normal state"); - cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0); + if (pep->ep_state & EP_HALTED) { + if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_HALTED) + cdnsp_halt_endpoint(pdev, pep, 0); + + /* + * Halt Endpoint Command for SSP2 for ep0 preserve current + * endpoint state and driver has to synchronize the + * software endpoint state with endpoint output context + * state. + */ + pep->ep_state &= ~EP_HALTED; + pep->ep_state |= EP_STOPPED; } /* diff --git a/drivers/usb/cdns3/cdnsp-gadget.c b/drivers/usb/cdns3/cdnsp-gadget.c index 4b67749edb997..fb192b120d77f 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.c +++ b/drivers/usb/cdns3/cdnsp-gadget.c @@ -28,7 +28,8 @@ unsigned int cdnsp_port_speed(unsigned int port_status) { /*Detect gadget speed based on PORTSC register*/ - if (DEV_SUPERSPEEDPLUS(port_status)) + if (DEV_SUPERSPEEDPLUS(port_status) || + DEV_SSP_GEN1x2(port_status) || DEV_SSP_GEN2x2(port_status)) return USB_SPEED_SUPER_PLUS; else if (DEV_SUPERSPEED(port_status)) return USB_SPEED_SUPER; @@ -138,6 +139,26 @@ static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev, (portsc & PORT_CHANGE_BITS), port_regs); } +static void cdnsp_set_apb_timeout_value(struct cdnsp_device *pdev) +{ + struct cdns *cdns = dev_get_drvdata(pdev->dev); + __le32 __iomem *reg; + void __iomem *base; + u32 offset = 0; + u32 val; + + if (!cdns->override_apb_timeout) + return; + + base = &pdev->cap_regs->hc_capbase; + offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP); + reg = base + offset + REG_CHICKEN_BITS_3_OFFSET; + + val = le32_to_cpu(readl(reg)); + val = CHICKEN_APB_TIMEOUT_SET(val, cdns->override_apb_timeout); + writel(cpu_to_le32(val), reg); +} + static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit) { __le32 __iomem *reg; @@ -526,6 +547,7 @@ int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev) dma_addr_t cmd_deq_dma; union cdnsp_trb *event; u32 cycle_state; + u32 retry = 10; int ret, val; u64 cmd_dma; u32 flags; @@ -557,8 +579,23 @@ int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev) flags = le32_to_cpu(event->event_cmd.flags); /* Check the owner of the TRB. */ - if ((flags & TRB_CYCLE) != cycle_state) + if ((flags & TRB_CYCLE) != cycle_state) { + /* + * Give some extra time to get chance controller + * to finish command before returning error code. + * Checking CMD_RING_BUSY is not sufficient because + * this bit is cleared to '0' when the Command + * Descriptor has been executed by controller + * and not when command completion event has + * be added to event ring. + */ + if (retry--) { + udelay(20); + continue; + } + return -EINVAL; + } cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb); @@ -1024,10 +1061,8 @@ static int cdnsp_gadget_ep_disable(struct usb_ep *ep) pep->ep_state |= EP_DIS_IN_RROGRESS; /* Endpoint was unconfigured by Reset Device command. */ - if (!(pep->ep_state & EP_UNCONFIGURED)) { + if (!(pep->ep_state & EP_UNCONFIGURED)) cdnsp_cmd_stop_ep(pdev, pep); - cdnsp_cmd_flush_ep(pdev, pep); - } /* Remove all queued USB requests. */ while (!list_empty(&pep->pending_list)) { @@ -1424,8 +1459,6 @@ static void cdnsp_stop(struct cdnsp_device *pdev) { u32 temp; - cdnsp_cmd_flush_ep(pdev, &pdev->eps[0]); - /* Remove internally queued request for ep0. */ if (!list_empty(&pdev->eps[0].pending_list)) { struct cdnsp_request *req; @@ -1776,6 +1809,8 @@ static void cdnsp_get_rev_cap(struct cdnsp_device *pdev) reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP); pdev->rev_cap = reg; + pdev->rtl_revision = readl(&pdev->rev_cap->rtl_revision); + dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n", readl(&pdev->rev_cap->ctrl_revision), readl(&pdev->rev_cap->rtl_revision), @@ -1801,6 +1836,15 @@ static int cdnsp_gen_setup(struct cdnsp_device *pdev) pdev->hci_version = HC_VERSION(pdev->hcc_params); pdev->hcc_params = readl(&pdev->cap_regs->hcc_params); + /* + * Override the APB timeout value to give the controller more time for + * enabling UTMI clock and synchronizing APB and UTMI clock domains. + * This fix is platform specific and is required to fixes issue with + * reading incorrect value from PORTSC register after resuming + * from L1 state. + */ + cdnsp_set_apb_timeout_value(pdev); + cdnsp_get_rev_cap(pdev); /* Make sure the Device Controller is halted. */ @@ -1931,7 +1975,10 @@ static int __cdnsp_gadget_init(struct cdns *cdns) return 0; del_gadget: - usb_del_gadget_udc(&pdev->gadget); + usb_del_gadget(&pdev->gadget); + cdnsp_gadget_free_endpoints(pdev); + usb_put_gadget(&pdev->gadget); + goto halt_pdev; free_endpoints: cdnsp_gadget_free_endpoints(pdev); halt_pdev: @@ -1953,8 +2000,9 @@ static void cdnsp_gadget_exit(struct cdns *cdns) devm_free_irq(pdev->dev, cdns->dev_irq, pdev); pm_runtime_mark_last_busy(cdns->dev); pm_runtime_put_autosuspend(cdns->dev); - usb_del_gadget_udc(&pdev->gadget); + usb_del_gadget(&pdev->gadget); cdnsp_gadget_free_endpoints(pdev); + usb_put_gadget(&pdev->gadget); cdnsp_mem_cleanup(pdev); kfree(pdev); cdns->gadget_dev = NULL; diff --git a/drivers/usb/cdns3/cdnsp-gadget.h b/drivers/usb/cdns3/cdnsp-gadget.h index 9a5577a772af6..a91cca509db08 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.h +++ b/drivers/usb/cdns3/cdnsp-gadget.h @@ -285,11 +285,15 @@ struct cdnsp_port_regs { #define XDEV_HS (0x3 << 10) #define XDEV_SS (0x4 << 10) #define XDEV_SSP (0x5 << 10) +#define XDEV_SSP1x2 (0x6 << 10) +#define XDEV_SSP2x2 (0x7 << 10) #define DEV_UNDEFSPEED(p) (((p) & DEV_SPEED_MASK) == (0x0 << 10)) #define DEV_FULLSPEED(p) (((p) & DEV_SPEED_MASK) == XDEV_FS) #define DEV_HIGHSPEED(p) (((p) & DEV_SPEED_MASK) == XDEV_HS) #define DEV_SUPERSPEED(p) (((p) & DEV_SPEED_MASK) == XDEV_SS) #define DEV_SUPERSPEEDPLUS(p) (((p) & DEV_SPEED_MASK) == XDEV_SSP) +#define DEV_SSP_GEN1x2(p) (((p) & DEV_SPEED_MASK) == XDEV_SSP1x2) +#define DEV_SSP_GEN2x2(p) (((p) & DEV_SPEED_MASK) == XDEV_SSP2x2) #define DEV_SUPERSPEED_ANY(p) (((p) & DEV_SPEED_MASK) >= XDEV_SS) #define DEV_PORT_SPEED(p) (((p) >> 10) & 0x0f) /* Port Link State Write Strobe - set this when changing link state */ @@ -520,6 +524,9 @@ struct cdnsp_rev_cap { #define REG_CHICKEN_BITS_2_OFFSET 0x48 #define CHICKEN_XDMA_2_TP_CACHE_DIS BIT(28) +#define REG_CHICKEN_BITS_3_OFFSET 0x4C +#define CHICKEN_APB_TIMEOUT_SET(p, val) (((p) & ~GENMASK(21, 0)) | (val)) + /* XBUF Extended Capability ID. */ #define XBUF_CAP_ID 0xCB #define XBUF_RX_TAG_MASK_0_OFFSET 0x1C @@ -980,6 +987,12 @@ enum cdnsp_setup_dev { #define STREAM_ID_FOR_TRB(p) ((((p)) << 16) & GENMASK(31, 16)) #define SCT_FOR_TRB(p) (((p) << 1) & 0x7) +/* + * Halt Endpoint Command TRB field. + * The ESP bit only exists in the SSP2 controller. + */ +#define TRB_ESP BIT(9) + /* Link TRB specific fields. */ #define TRB_TC BIT(1) @@ -1131,8 +1144,6 @@ union cdnsp_trb { #define TRB_HALT_ENDPOINT 54 /* Doorbell Overflow Event. */ #define TRB_DRB_OVERFLOW 57 -/* Flush Endpoint Command. */ -#define TRB_FLUSH_ENDPOINT 58 #define TRB_TYPE_LINK(x) (((x) & TRB_TYPE_BITMASK) == TRB_TYPE(TRB_LINK)) #define TRB_TYPE_LINK_LE32(x) (((x) & cpu_to_le32(TRB_TYPE_BITMASK)) == \ @@ -1359,6 +1370,7 @@ struct cdnsp_port { * @rev_cap: Controller Capabilities Registers. * @hcs_params1: Cached register copies of read-only HCSPARAMS1 * @hcc_params: Cached register copies of read-only HCCPARAMS1 + * @rtl_revision: Cached controller rtl revision. * @setup: Temporary buffer for setup packet. * @ep0_preq: Internal allocated request used during enumeration. * @ep0_stage: ep0 stage during enumeration process. @@ -1413,6 +1425,8 @@ struct cdnsp_device { __u32 hcs_params1; __u32 hcs_params3; __u32 hcc_params; + #define RTL_REVISION_NEW_LPM 0x2700 + __u32 rtl_revision; /* Lock used in interrupt thread context. */ spinlock_t lock; struct usb_ctrlrequest setup; @@ -1542,8 +1556,6 @@ void cdnsp_queue_configure_endpoint(struct cdnsp_device *pdev, void cdnsp_queue_reset_ep(struct cdnsp_device *pdev, unsigned int ep_index); void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index); -void cdnsp_queue_flush_endpoint(struct cdnsp_device *pdev, - unsigned int ep_index); void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num); void cdnsp_queue_reset_device(struct cdnsp_device *pdev); void cdnsp_queue_new_dequeue_state(struct cdnsp_device *pdev, @@ -1577,7 +1589,6 @@ void cdnsp_irq_reset(struct cdnsp_device *pdev); int cdnsp_halt_endpoint(struct cdnsp_device *pdev, struct cdnsp_ep *pep, int value); int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep); -int cdnsp_cmd_flush_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep); void cdnsp_setup_analyze(struct cdnsp_device *pdev); int cdnsp_status_stage(struct cdnsp_device *pdev); int cdnsp_reset_device(struct cdnsp_device *pdev); diff --git a/drivers/usb/cdns3/cdnsp-pci.c b/drivers/usb/cdns3/cdnsp-pci.c index 0725668ffea4c..b9c74c1baccb2 100644 --- a/drivers/usb/cdns3/cdnsp-pci.c +++ b/drivers/usb/cdns3/cdnsp-pci.c @@ -33,6 +33,8 @@ #define CDNS_DRD_ID 0x0100 #define CDNS_DRD_IF (PCI_CLASS_SERIAL_USB << 8 | 0x80) +#define CHICKEN_APB_TIMEOUT_VALUE 0x1C20 + static struct pci_dev *cdnsp_get_second_fun(struct pci_dev *pdev) { /* @@ -88,7 +90,7 @@ static int cdnsp_pci_probe(struct pci_dev *pdev, cdnsp = kzalloc(sizeof(*cdnsp), GFP_KERNEL); if (!cdnsp) { ret = -ENOMEM; - goto disable_pci; + goto put_pci; } } @@ -144,6 +146,14 @@ static int cdnsp_pci_probe(struct pci_dev *pdev, cdnsp->otg_irq = pdev->irq; } + /* + * Cadence PCI based platform require some longer timeout for APB + * to fixes domain clock synchronization issue after resuming + * controller from L1 state. + */ + cdnsp->override_apb_timeout = CHICKEN_APB_TIMEOUT_VALUE; + pci_set_drvdata(pdev, cdnsp); + if (pci_is_enabled(func)) { cdnsp->dev = dev; cdnsp->gadget_init = cdnsp_gadget_init; @@ -153,8 +163,6 @@ static int cdnsp_pci_probe(struct pci_dev *pdev, goto free_cdnsp; } - pci_set_drvdata(pdev, cdnsp); - device_wakeup_enable(&pdev->dev); if (pci_dev_run_wake(pdev)) pm_runtime_put_noidle(&pdev->dev); @@ -165,9 +173,6 @@ static int cdnsp_pci_probe(struct pci_dev *pdev, if (!pci_is_enabled(func)) kfree(cdnsp); -disable_pci: - pci_disable_device(pdev); - put_pci: pci_dev_put(func); diff --git a/drivers/usb/cdns3/cdnsp-ring.c b/drivers/usb/cdns3/cdnsp-ring.c index 1d18d5002ef01..0758f171f73ec 100644 --- a/drivers/usb/cdns3/cdnsp-ring.c +++ b/drivers/usb/cdns3/cdnsp-ring.c @@ -308,7 +308,8 @@ static bool cdnsp_ring_ep_doorbell(struct cdnsp_device *pdev, writel(db_value, reg_addr); - cdnsp_force_l0_go(pdev); + if (pdev->rtl_revision < RTL_REVISION_NEW_LPM) + cdnsp_force_l0_go(pdev); /* Doorbell was set. */ return true; @@ -771,7 +772,9 @@ static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id) } if (port_id != old_port) { - cdnsp_disable_slot(pdev); + if (pdev->slot_id) + cdnsp_disable_slot(pdev); + pdev->active_port = port; cdnsp_enable_slot(pdev); } @@ -2156,19 +2159,6 @@ int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) return ret; } -int cdnsp_cmd_flush_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) -{ - int ret; - - cdnsp_queue_flush_endpoint(pdev, pep->idx); - cdnsp_ring_cmd_db(pdev); - ret = cdnsp_wait_for_cmd_compl(pdev); - - trace_cdnsp_handle_cmd_flush_ep(pep->out_ctx); - - return ret; -} - /* * The transfer burst count field of the isochronous TRB defines the number of * bursts that are required to move all packets in this TD. Only SuperSpeed @@ -2495,18 +2485,8 @@ void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index) { cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_HALT_ENDPOINT) | SLOT_ID_FOR_TRB(pdev->slot_id) | - EP_ID_FOR_TRB(ep_index)); -} - -/* - * Queue a flush endpoint request on the command ring. - */ -void cdnsp_queue_flush_endpoint(struct cdnsp_device *pdev, - unsigned int ep_index) -{ - cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_FLUSH_ENDPOINT) | - SLOT_ID_FOR_TRB(pdev->slot_id) | - EP_ID_FOR_TRB(ep_index)); + EP_ID_FOR_TRB(ep_index) | + (!ep_index ? TRB_ESP : 0)); } void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num) diff --git a/drivers/usb/cdns3/core.h b/drivers/usb/cdns3/core.h index 57d47348dc193..ac30ee21309d0 100644 --- a/drivers/usb/cdns3/core.h +++ b/drivers/usb/cdns3/core.h @@ -79,6 +79,8 @@ struct cdns3_platform_data { * @pdata: platform data from glue layer * @lock: spinlock structure * @xhci_plat_data: xhci private data structure pointer + * @override_apb_timeout: hold value of APB timeout. For value 0 the default + * value in CHICKEN_BITS_3 will be preserved. * @gadget_init: pointer to gadget initialization function */ struct cdns { @@ -117,6 +119,7 @@ struct cdns { struct cdns3_platform_data *pdata; spinlock_t lock; struct xhci_plat_priv *xhci_plat_data; + u32 override_apb_timeout; int (*gadget_init)(struct cdns *cdns); }; diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c index f2ae5f4c58283..0bee561420af2 100644 --- a/drivers/usb/chipidea/udc.c +++ b/drivers/usb/chipidea/udc.c @@ -2213,6 +2213,10 @@ static void udc_suspend(struct ci_hdrc *ci) */ if (hw_read(ci, OP_ENDPTLISTADDR, ~0) == 0) hw_write(ci, OP_ENDPTLISTADDR, ~0, ~0); + + if (ci->gadget.connected && + (!ci->suspended || !device_may_wakeup(ci->dev))) + usb_gadget_disconnect(&ci->gadget); } static void udc_resume(struct ci_hdrc *ci, bool power_lost) @@ -2223,6 +2227,9 @@ static void udc_resume(struct ci_hdrc *ci, bool power_lost) OTGSC_BSVIS | OTGSC_BSVIE); if (ci->vbus_active) usb_gadget_vbus_disconnect(&ci->gadget); + } else if (ci->vbus_active && ci->driver && + !ci->gadget.connected) { + usb_gadget_connect(&ci->gadget); } /* Restore value 0 if it was set for power lost check */ diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index c1d7d87b32cc5..f0c87c914149d 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -1520,6 +1520,12 @@ static int acm_probe(struct usb_interface *intf, goto err_remove_files; } + if (quirks & CLEAR_HALT_CONDITIONS) { + /* errors intentionally ignored */ + usb_clear_halt(usb_dev, acm->in); + usb_clear_halt(usb_dev, acm->out); + } + tty_dev = tty_port_register_device(&acm->port, acm_tty_driver, minor, &control_interface->dev); if (IS_ERR(tty_dev)) { @@ -1527,11 +1533,6 @@ static int acm_probe(struct usb_interface *intf, goto err_release_data_interface; } - if (quirks & CLEAR_HALT_CONDITIONS) { - usb_clear_halt(usb_dev, acm->in); - usb_clear_halt(usb_dev, acm->out); - } - dev_info(&intf->dev, "ttyACM%d: USB ACM device\n", minor); return 0; diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c index 559c121f09230..600ad9412c146 100644 --- a/drivers/usb/class/cdc-wdm.c +++ b/drivers/usb/class/cdc-wdm.c @@ -92,7 +92,6 @@ struct wdm_device { u16 wMaxCommand; u16 wMaxPacketSize; __le16 inum; - int reslength; int length; int read; int count; @@ -214,6 +213,11 @@ static void wdm_in_callback(struct urb *urb) if (desc->rerr == 0 && status != -EPIPE) desc->rerr = status; + if (length == 0) { + dev_dbg(&desc->intf->dev, "received ZLP\n"); + goto skip_zlp; + } + if (length + desc->length > desc->wMaxCommand) { /* The buffer would overflow */ set_bit(WDM_OVERFLOW, &desc->flags); @@ -222,18 +226,18 @@ static void wdm_in_callback(struct urb *urb) if (!test_bit(WDM_OVERFLOW, &desc->flags)) { memmove(desc->ubuf + desc->length, desc->inbuf, length); desc->length += length; - desc->reslength = length; } } skip_error: if (desc->rerr) { /* - * Since there was an error, userspace may decide to not read - * any data after poll'ing. + * If there was a ZLP or an error, userspace may decide to not + * read any data after poll'ing. * We should respond to further attempts from the device to send * data, so that we can get unstuck. */ +skip_zlp: schedule_work(&desc->service_outs_intr); } else { set_bit(WDM_READ, &desc->flags); @@ -585,15 +589,6 @@ static ssize_t wdm_read goto retry; } - if (!desc->reslength) { /* zero length read */ - dev_dbg(&desc->intf->dev, "zero length - clearing WDM_READ\n"); - clear_bit(WDM_READ, &desc->flags); - rv = service_outstanding_interrupt(desc); - spin_unlock_irq(&desc->iuspin); - if (rv < 0) - goto err; - goto retry; - } cntr = desc->length; spin_unlock_irq(&desc->iuspin); } @@ -1016,7 +1011,7 @@ static void service_interrupt_work(struct work_struct *work) spin_lock_irq(&desc->iuspin); service_outstanding_interrupt(desc); - if (!desc->resp_count) { + if (!desc->resp_count && (desc->length || desc->rerr)) { set_bit(WDM_READ, &desc->flags); wake_up(&desc->wait); } diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c index c2e666e82857c..ee45f3c74aecb 100644 --- a/drivers/usb/class/usbtmc.c +++ b/drivers/usb/class/usbtmc.c @@ -482,6 +482,8 @@ static int usbtmc_get_stb(struct usbtmc_file_data *file_data, __u8 *stb) u8 *buffer; u8 tag; int rv; + long wait_rv; + unsigned long expire; dev_dbg(dev, "Enter ioctl_read_stb iin_ep_present: %d\n", data->iin_ep_present); @@ -511,16 +513,18 @@ static int usbtmc_get_stb(struct usbtmc_file_data *file_data, __u8 *stb) } if (data->iin_ep_present) { - rv = wait_event_interruptible_timeout( + expire = msecs_to_jiffies(file_data->timeout); + wait_rv = wait_event_interruptible_timeout( data->waitq, atomic_read(&data->iin_data_valid) != 0, - file_data->timeout); - if (rv < 0) { - dev_dbg(dev, "wait interrupted %d\n", rv); + expire); + if (wait_rv < 0) { + dev_dbg(dev, "wait interrupted %ld\n", wait_rv); + rv = wait_rv; goto exit; } - if (rv == 0) { + if (wait_rv == 0) { dev_dbg(dev, "wait timed out\n"); rv = -ETIMEDOUT; goto exit; @@ -539,6 +543,8 @@ static int usbtmc_get_stb(struct usbtmc_file_data *file_data, __u8 *stb) dev_dbg(dev, "stb:0x%02x received %d\n", (unsigned int)*stb, rv); + rv = 0; + exit: /* bump interrupt bTag */ data->iin_bTag += 1; @@ -559,14 +565,15 @@ static int usbtmc488_ioctl_read_stb(struct usbtmc_file_data *file_data, rv = usbtmc_get_stb(file_data, &stb); - if (rv > 0) { - srq_asserted = atomic_xchg(&file_data->srq_asserted, - srq_asserted); - if (srq_asserted) - stb |= 0x40; /* Set RQS bit */ + if (rv < 0) + return rv; + + srq_asserted = atomic_xchg(&file_data->srq_asserted, srq_asserted); + if (srq_asserted) + stb |= 0x40; /* Set RQS bit */ + + rv = put_user(stb, (__u8 __user *)arg); - rv = put_user(stb, (__u8 __user *)arg); - } return rv; } @@ -602,9 +609,9 @@ static int usbtmc488_ioctl_wait_srq(struct usbtmc_file_data *file_data, { struct usbtmc_device_data *data = file_data->data; struct device *dev = &data->intf->dev; - int rv; u32 timeout; unsigned long expire; + long wait_rv; if (!data->iin_ep_present) { dev_dbg(dev, "no interrupt endpoint present\n"); @@ -618,25 +625,24 @@ static int usbtmc488_ioctl_wait_srq(struct usbtmc_file_data *file_data, mutex_unlock(&data->io_mutex); - rv = wait_event_interruptible_timeout( - data->waitq, - atomic_read(&file_data->srq_asserted) != 0 || - atomic_read(&file_data->closing), - expire); + wait_rv = wait_event_interruptible_timeout( + data->waitq, + atomic_read(&file_data->srq_asserted) != 0 || + atomic_read(&file_data->closing), + expire); mutex_lock(&data->io_mutex); /* Note! disconnect or close could be called in the meantime */ if (atomic_read(&file_data->closing) || data->zombie) - rv = -ENODEV; + return -ENODEV; - if (rv < 0) { - /* dev can be invalid now! */ - pr_debug("%s - wait interrupted %d\n", __func__, rv); - return rv; + if (wait_rv < 0) { + dev_dbg(dev, "%s - wait interrupted %ld\n", __func__, wait_rv); + return wait_rv; } - if (rv == 0) { + if (wait_rv == 0) { dev_dbg(dev, "%s - wait timed out\n", __func__); return -ETIMEDOUT; } @@ -830,6 +836,7 @@ static ssize_t usbtmc_generic_read(struct usbtmc_file_data *file_data, unsigned long expire; int bufcount = 1; int again = 0; + long wait_rv; /* mutex already locked */ @@ -942,19 +949,24 @@ static ssize_t usbtmc_generic_read(struct usbtmc_file_data *file_data, if (!(flags & USBTMC_FLAG_ASYNC)) { dev_dbg(dev, "%s: before wait time %lu\n", __func__, expire); - retval = wait_event_interruptible_timeout( + wait_rv = wait_event_interruptible_timeout( file_data->wait_bulk_in, usbtmc_do_transfer(file_data), expire); - dev_dbg(dev, "%s: wait returned %d\n", - __func__, retval); + dev_dbg(dev, "%s: wait returned %ld\n", + __func__, wait_rv); + + if (wait_rv < 0) { + retval = wait_rv; + goto error; + } - if (retval <= 0) { - if (retval == 0) - retval = -ETIMEDOUT; + if (wait_rv == 0) { + retval = -ETIMEDOUT; goto error; } + } urb = usb_get_from_anchor(&file_data->in_anchor); @@ -1380,7 +1392,10 @@ static ssize_t usbtmc_read(struct file *filp, char __user *buf, if (!buffer) return -ENOMEM; - mutex_lock(&data->io_mutex); + retval = mutex_lock_interruptible(&data->io_mutex); + if (retval < 0) + goto exit_nolock; + if (data->zombie) { retval = -ENODEV; goto exit; @@ -1503,6 +1518,7 @@ static ssize_t usbtmc_read(struct file *filp, char __user *buf, exit: mutex_unlock(&data->io_mutex); +exit_nolock: kfree(buffer); return retval; } @@ -2186,7 +2202,7 @@ static long usbtmc_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case USBTMC_IOCTL_GET_STB: retval = usbtmc_get_stb(file_data, &tmp_byte); - if (retval > 0) + if (!retval) retval = put_user(tmp_byte, (__u8 __user *)arg); break; diff --git a/drivers/usb/common/usb-conn-gpio.c b/drivers/usb/common/usb-conn-gpio.c index 501e8bc9738eb..1096a884c8d70 100644 --- a/drivers/usb/common/usb-conn-gpio.c +++ b/drivers/usb/common/usb-conn-gpio.c @@ -20,6 +20,9 @@ #include #include #include +#include + +static DEFINE_IDA(usb_conn_ida); #define USB_GPIO_DEB_MS 20 /* ms */ #define USB_GPIO_DEB_US ((USB_GPIO_DEB_MS) * 1000) /* us */ @@ -29,6 +32,7 @@ struct usb_conn_info { struct device *dev; + int conn_id; /* store the IDA-allocated ID */ struct usb_role_switch *role_sw; enum usb_role last_role; struct regulator *vbus; @@ -160,7 +164,17 @@ static int usb_conn_psy_register(struct usb_conn_info *info) .of_node = dev->of_node, }; - desc->name = "usb-charger"; + info->conn_id = ida_alloc(&usb_conn_ida, GFP_KERNEL); + if (info->conn_id < 0) + return info->conn_id; + + desc->name = devm_kasprintf(dev, GFP_KERNEL, "usb-charger-%d", + info->conn_id); + if (!desc->name) { + ida_free(&usb_conn_ida, info->conn_id); + return -ENOMEM; + } + desc->properties = usb_charger_properties; desc->num_properties = ARRAY_SIZE(usb_charger_properties); desc->get_property = usb_charger_get_property; @@ -168,8 +182,10 @@ static int usb_conn_psy_register(struct usb_conn_info *info) cfg.drv_data = info; info->charger = devm_power_supply_register(dev, desc, &cfg); - if (IS_ERR(info->charger)) - dev_err(dev, "Unable to register charger\n"); + if (IS_ERR(info->charger)) { + dev_err(dev, "Unable to register charger %d\n", info->conn_id); + ida_free(&usb_conn_ida, info->conn_id); + } return PTR_ERR_OR_ZERO(info->charger); } @@ -277,6 +293,9 @@ static void usb_conn_remove(struct platform_device *pdev) cancel_delayed_work_sync(&info->dw_det); + if (info->charger) + ida_free(&usb_conn_ida, info->conn_id); + if (info->last_role == USB_ROLE_HOST && info->vbus) regulator_disable(info->vbus); diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c index 847dd32c0f5e2..3180419424c06 100644 --- a/drivers/usb/core/config.c +++ b/drivers/usb/core/config.c @@ -81,8 +81,14 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno, */ desc = (struct usb_ss_ep_comp_descriptor *) buffer; - if (desc->bDescriptorType != USB_DT_SS_ENDPOINT_COMP || - size < USB_DT_SS_EP_COMP_SIZE) { + if (size < USB_DT_SS_EP_COMP_SIZE) { + dev_notice(ddev, + "invalid SuperSpeed endpoint companion descriptor " + "of length %d, skipping\n", size); + return; + } + + if (desc->bDescriptorType != USB_DT_SS_ENDPOINT_COMP) { dev_notice(ddev, "No SuperSpeed endpoint companion for config %d " " interface %d altsetting %d ep %d: " "using minimum values\n", diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 12b6dfeaf658c..0b2a3f645d2fe 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -2152,7 +2152,7 @@ static struct urb *request_single_step_set_feature_urb( urb->complete = usb_ehset_completion; urb->status = -EINPROGRESS; urb->actual_length = 0; - urb->transfer_flags = URB_DIR_IN; + urb->transfer_flags = URB_DIR_IN | URB_NO_TRANSFER_DMA_MAP; usb_get_urb(urb); atomic_inc(&urb->use_count); atomic_inc(&urb->dev->urbnum); @@ -2216,9 +2216,15 @@ int ehset_single_step_set_feature(struct usb_hcd *hcd, int port) /* Complete remaining DATA and STATUS stages using the same URB */ urb->status = -EINPROGRESS; + urb->transfer_flags &= ~URB_NO_TRANSFER_DMA_MAP; usb_get_urb(urb); atomic_inc(&urb->use_count); atomic_inc(&urb->dev->urbnum); + if (map_urb_for_dma(hcd, urb, GFP_KERNEL)) { + usb_put_urb(urb); + goto out1; + } + retval = hcd->driver->submit_single_step_set_feature(hcd, urb, 0); if (!retval && !wait_for_completion_timeout(&done, msecs_to_jiffies(2000))) { diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 87a1419cba663..a81c3923a70f9 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -67,6 +67,12 @@ */ #define USB_SHORT_SET_ADDRESS_REQ_TIMEOUT 500 /* ms */ +/* + * Give SS hubs 200ms time after wake to train downstream links before + * assuming no port activity and allowing hub to runtime suspend back. + */ +#define USB_SS_PORT_U0_WAKE_TIME 200 /* ms */ + /* Protect struct usb_device->state and ->children members * Note: Both are also protected by ->dev.sem, except that ->state can * change to USB_STATE_NOTATTACHED even when the semaphore isn't held. */ @@ -1066,6 +1072,7 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) goto init2; goto init3; } + hub_get(hub); /* The superspeed hub except for root hub has to use Hub Depth @@ -1314,6 +1321,17 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) device_unlock(&hdev->dev); } + if (type == HUB_RESUME && hub_is_superspeed(hub->hdev)) { + /* give usb3 downstream links training time after hub resume */ + usb_autopm_get_interface_no_resume( + to_usb_interface(hub->intfdev)); + + queue_delayed_work(system_power_efficient_wq, + &hub->post_resume_work, + msecs_to_jiffies(USB_SS_PORT_U0_WAKE_TIME)); + return; + } + hub_put(hub); } @@ -1332,6 +1350,14 @@ static void hub_init_func3(struct work_struct *ws) hub_activate(hub, HUB_INIT3); } +static void hub_post_resume(struct work_struct *ws) +{ + struct usb_hub *hub = container_of(ws, struct usb_hub, post_resume_work.work); + + usb_autopm_put_interface_async(to_usb_interface(hub->intfdev)); + hub_put(hub); +} + enum hub_quiescing_type { HUB_DISCONNECT, HUB_PRE_RESET, HUB_SUSPEND }; @@ -1357,6 +1383,7 @@ static void hub_quiesce(struct usb_hub *hub, enum hub_quiescing_type type) /* Stop hub_wq and related activity */ del_timer_sync(&hub->irq_urb_retry); + flush_delayed_work(&hub->post_resume_work); usb_kill_urb(hub->urb); if (hub->has_indicators) cancel_delayed_work_sync(&hub->leds); @@ -1915,6 +1942,7 @@ static int hub_probe(struct usb_interface *intf, const struct usb_device_id *id) hub->hdev = hdev; INIT_DELAYED_WORK(&hub->leds, led_work); INIT_DELAYED_WORK(&hub->init_work, NULL); + INIT_DELAYED_WORK(&hub->post_resume_work, hub_post_resume); INIT_WORK(&hub->events, hub_event); INIT_LIST_HEAD(&hub->onboard_hub_devs); spin_lock_init(&hub->irq_urb_lock); @@ -5692,6 +5720,7 @@ static void port_event(struct usb_hub *hub, int port1) struct usb_device *hdev = hub->hdev; u16 portstatus, portchange; int i = 0; + int err; connect_change = test_bit(port1, hub->change_bits); clear_bit(port1, hub->event_bits); @@ -5788,8 +5817,11 @@ static void port_event(struct usb_hub *hub, int port1) } else if (!udev || !(portstatus & USB_PORT_STAT_CONNECTION) || udev->state == USB_STATE_NOTATTACHED) { dev_dbg(&port_dev->dev, "do warm reset, port only\n"); - if (hub_port_reset(hub, port1, NULL, - HUB_BH_RESET_TIME, true) < 0) + err = hub_port_reset(hub, port1, NULL, + HUB_BH_RESET_TIME, true); + if (!udev && err == -ENOTCONN) + connect_change = 0; + else if (err < 0) hub_port_disable(hub, port1, 1); } else { dev_dbg(&port_dev->dev, "do warm reset, full device\n"); @@ -6109,6 +6141,7 @@ static int usb_reset_and_verify_device(struct usb_device *udev) struct usb_hub *parent_hub; struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct usb_device_descriptor descriptor; + struct usb_interface *intf; struct usb_host_bos *bos; int i, j, ret = 0; int port1 = udev->portnum; @@ -6166,6 +6199,18 @@ static int usb_reset_and_verify_device(struct usb_device *udev) if (!udev->actconfig) goto done; + /* + * Some devices can't handle setting default altsetting 0 with a + * Set-Interface request. Disable host-side endpoints of those + * interfaces here. Enable and reset them back after host has set + * its internal endpoint structures during usb_hcd_alloc_bandwith() + */ + for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) { + intf = udev->actconfig->interface[i]; + if (intf->cur_altsetting->desc.bAlternateSetting == 0) + usb_disable_interface(udev, intf, true); + } + mutex_lock(hcd->bandwidth_mutex); ret = usb_hcd_alloc_bandwidth(udev, udev->actconfig, NULL, NULL); if (ret < 0) { @@ -6197,12 +6242,11 @@ static int usb_reset_and_verify_device(struct usb_device *udev) */ for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) { struct usb_host_config *config = udev->actconfig; - struct usb_interface *intf = config->interface[i]; struct usb_interface_descriptor *desc; + intf = config->interface[i]; desc = &intf->cur_altsetting->desc; if (desc->bAlternateSetting == 0) { - usb_disable_interface(udev, intf, true); usb_enable_interface(udev, intf, true); ret = 0; } else { diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index 6610cf6131c67..59c426e0e0b9d 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -69,6 +69,7 @@ struct usb_hub { u8 indicator[USB_MAXCHILDREN]; struct delayed_work leds; struct delayed_work init_work; + struct delayed_work post_resume_work; struct work_struct events; spinlock_t irq_urb_lock; struct timer_list irq_urb_retry; diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index 4903c733d37ae..c322d0c1d965a 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -227,7 +227,8 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x046a, 0x0023), .driver_info = USB_QUIRK_RESET_RESUME }, /* Logitech HD Webcam C270 */ - { USB_DEVICE(0x046d, 0x0825), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x046d, 0x0825), .driver_info = USB_QUIRK_RESET_RESUME | + USB_QUIRK_NO_LPM}, /* Logitech HD Pro Webcams C920, C920-C, C922, C925e and C930e */ { USB_DEVICE(0x046d, 0x082d), .driver_info = USB_QUIRK_DELAY_INIT }, @@ -370,8 +371,12 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x0781, 0x5591), .driver_info = USB_QUIRK_NO_LPM }, /* SanDisk Corp. SanDisk 3.2Gen1 */ + { USB_DEVICE(0x0781, 0x5596), .driver_info = USB_QUIRK_DELAY_INIT }, { USB_DEVICE(0x0781, 0x55a3), .driver_info = USB_QUIRK_DELAY_INIT }, + /* SanDisk Extreme 55AE */ + { USB_DEVICE(0x0781, 0x55ae), .driver_info = USB_QUIRK_NO_LPM }, + /* Realforce 87U Keyboard */ { USB_DEVICE(0x0853, 0x011b), .driver_info = USB_QUIRK_NO_LPM }, @@ -459,6 +464,8 @@ static const struct usb_device_id usb_quirk_list[] = { /* Huawei 4G LTE module */ { USB_DEVICE(0x12d1, 0x15bb), .driver_info = USB_QUIRK_DISCONNECT_SUSPEND }, + { USB_DEVICE(0x12d1, 0x15c1), .driver_info = + USB_QUIRK_DISCONNECT_SUSPEND }, { USB_DEVICE(0x12d1, 0x15c3), .driver_info = USB_QUIRK_DISCONNECT_SUSPEND }, @@ -729,7 +736,7 @@ void usb_detect_quirks(struct usb_device *udev) udev->quirks ^= usb_detect_dynamic_quirks(udev); if (udev->quirks) - dev_dbg(&udev->dev, "USB quirks for this device: %x\n", + dev_dbg(&udev->dev, "USB quirks for this device: 0x%x\n", udev->quirks); #ifdef CONFIG_USB_DEFAULT_PERSIST diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c index 7576920e2d5a3..9f202f575cecc 100644 --- a/drivers/usb/core/urb.c +++ b/drivers/usb/core/urb.c @@ -500,7 +500,7 @@ int usb_submit_urb(struct urb *urb, gfp_t mem_flags) /* Check that the pipe's type matches the endpoint's type */ if (usb_pipe_type_check(urb->dev, urb->pipe)) - dev_WARN(&dev->dev, "BOGUS urb xfer, pipe %x != type %x\n", + dev_warn_once(&dev->dev, "BOGUS urb xfer, pipe %x != type %x\n", usb_pipetype(urb->pipe), pipetypes[xfertype]); /* Check against a simple/standard policy */ diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c index 2a938cf47ccd6..da6d5e5f79e7a 100644 --- a/drivers/usb/core/usb.c +++ b/drivers/usb/core/usb.c @@ -695,15 +695,16 @@ struct usb_device *usb_alloc_dev(struct usb_device *parent, device_set_of_node_from_dev(&dev->dev, bus->sysdev); dev_set_name(&dev->dev, "usb%d", bus->busnum); } else { + int n; + /* match any labeling on the hubs; it's one-based */ if (parent->devpath[0] == '0') { - snprintf(dev->devpath, sizeof dev->devpath, - "%d", port1); + n = snprintf(dev->devpath, sizeof(dev->devpath), "%d", port1); /* Root ports are not counted in route string */ dev->route = 0; } else { - snprintf(dev->devpath, sizeof dev->devpath, - "%s.%d", parent->devpath, port1); + n = snprintf(dev->devpath, sizeof(dev->devpath), "%s.%d", + parent->devpath, port1); /* Route string assumes hubs have less than 16 ports */ if (port1 < 15) dev->route = parent->route + @@ -712,6 +713,11 @@ struct usb_device *usb_alloc_dev(struct usb_device *parent, dev->route = parent->route + (15 << ((parent->level - 1)*4)); } + if (n >= sizeof(dev->devpath)) { + usb_put_hcd(bus_to_hcd(bus)); + usb_put_dev(dev); + return NULL; + } dev->dev.parent = &parent->dev; dev_set_name(&dev->dev, "%d-%s", bus->busnum, dev->devpath); diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index ce20c06a90253..c0db3c52831a2 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -4601,6 +4601,12 @@ static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget) if (!hsotg) return -ENODEV; + /* Exit clock gating when driver is stopped. */ + if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_NONE && + hsotg->bus_suspended && !hsotg->params.no_clock_gating) { + dwc2_gadget_exit_clock_gating(hsotg, 0); + } + /* all endpoints should be shutdown */ for (ep = 1; ep < hsotg->num_of_eps; ep++) { if (hsotg->eps_in[ep]) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 27322c6eed32e..5e710fe8dfa6e 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -151,6 +151,7 @@ void dwc3_set_prtcap(struct dwc3 *dwc, u32 mode, bool ignore_susphy) dwc3_writel(dwc->regs, DWC3_GCTL, reg); dwc->current_dr_role = mode; + trace_dwc3_set_prtcap(mode); } static void __dwc3_set_mode(struct work_struct *work) @@ -2177,12 +2178,15 @@ static int dwc3_core_init_for_resume(struct dwc3 *dwc) static int dwc3_suspend_common(struct dwc3 *dwc, pm_message_t msg) { u32 reg; + int ret; switch (dwc->current_dr_role) { case DWC3_GCTL_PRTCAP_DEVICE: if (pm_runtime_suspended(dwc->dev)) break; - dwc3_gadget_suspend(dwc); + ret = dwc3_gadget_suspend(dwc); + if (ret) + return ret; synchronize_irq(dwc->irq_gadget); dwc3_core_exit(dwc); break; @@ -2213,7 +2217,9 @@ static int dwc3_suspend_common(struct dwc3 *dwc, pm_message_t msg) break; if (dwc->current_otg_role == DWC3_OTG_ROLE_DEVICE) { - dwc3_gadget_suspend(dwc); + ret = dwc3_gadget_suspend(dwc); + if (ret) + return ret; synchronize_irq(dwc->irq_gadget); } diff --git a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h index 09d703852a92b..6e1cdcdce7cc3 100644 --- a/drivers/usb/dwc3/debug.h +++ b/drivers/usb/dwc3/debug.h @@ -13,6 +13,24 @@ #include "core.h" +/** + * dwc3_mode_string - returns mode name + * @mode: GCTL.PrtCapDir value + */ +static inline const char *dwc3_mode_string(u32 mode) +{ + switch (mode) { + case DWC3_GCTL_PRTCAP_HOST: + return "host"; + case DWC3_GCTL_PRTCAP_DEVICE: + return "device"; + case DWC3_GCTL_PRTCAP_OTG: + return "otg"; + default: + return "UNKNOWN"; + } +} + /** * dwc3_gadget_ep_cmd_string - returns endpoint command string * @cmd: command code diff --git a/drivers/usb/dwc3/debugfs.c b/drivers/usb/dwc3/debugfs.c index ebf03468fac4a..d18bf5e32cc8c 100644 --- a/drivers/usb/dwc3/debugfs.c +++ b/drivers/usb/dwc3/debugfs.c @@ -402,6 +402,7 @@ static int dwc3_mode_show(struct seq_file *s, void *unused) struct dwc3 *dwc = s->private; unsigned long flags; u32 reg; + u32 mode; int ret; ret = pm_runtime_resume_and_get(dwc->dev); @@ -412,18 +413,15 @@ static int dwc3_mode_show(struct seq_file *s, void *unused) reg = dwc3_readl(dwc->regs, DWC3_GCTL); spin_unlock_irqrestore(&dwc->lock, flags); - switch (DWC3_GCTL_PRTCAP(reg)) { + mode = DWC3_GCTL_PRTCAP(reg); + switch (mode) { case DWC3_GCTL_PRTCAP_HOST: - seq_puts(s, "host\n"); - break; case DWC3_GCTL_PRTCAP_DEVICE: - seq_puts(s, "device\n"); - break; case DWC3_GCTL_PRTCAP_OTG: - seq_puts(s, "otg\n"); + seq_printf(s, "%s\n", dwc3_mode_string(mode)); break; default: - seq_printf(s, "UNKNOWN %08x\n", DWC3_GCTL_PRTCAP(reg)); + seq_printf(s, "UNKNOWN %08x\n", mode); } pm_runtime_put_sync(dwc->dev); diff --git a/drivers/usb/dwc3/dwc3-imx8mp.c b/drivers/usb/dwc3/dwc3-imx8mp.c index a1e15f2fffdbf..b53468c41f677 100644 --- a/drivers/usb/dwc3/dwc3-imx8mp.c +++ b/drivers/usb/dwc3/dwc3-imx8mp.c @@ -244,7 +244,7 @@ static int dwc3_imx8mp_probe(struct platform_device *pdev) IRQF_ONESHOT, dev_name(dev), dwc3_imx); if (err) { dev_err(dev, "failed to request IRQ #%d --> %d\n", irq, err); - goto depopulate; + goto put_dwc3; } device_set_wakeup_capable(dev, true); @@ -252,6 +252,8 @@ static int dwc3_imx8mp_probe(struct platform_device *pdev) return 0; +put_dwc3: + put_device(&dwc3_imx->dwc3->dev); depopulate: of_platform_depopulate(dev); err_node_put: @@ -272,6 +274,8 @@ static void dwc3_imx8mp_remove(struct platform_device *pdev) struct dwc3_imx8mp *dwc3_imx = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; + put_device(&dwc3_imx->dwc3->dev); + pm_runtime_get_sync(dev); of_platform_depopulate(dev); diff --git a/drivers/usb/dwc3/dwc3-meson-g12a.c b/drivers/usb/dwc3/dwc3-meson-g12a.c index 2c07c038b584d..6ea1a876203d9 100644 --- a/drivers/usb/dwc3/dwc3-meson-g12a.c +++ b/drivers/usb/dwc3/dwc3-meson-g12a.c @@ -837,6 +837,9 @@ static void dwc3_meson_g12a_remove(struct platform_device *pdev) usb_role_switch_unregister(priv->role_switch); + put_device(priv->switch_desc.udc); + put_device(priv->switch_desc.usb2_port); + of_platform_depopulate(dev); for (i = 0 ; i < PHY_COUNT ; ++i) { diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index 54a4ee2b90b7f..39c72cb52ce76 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -41,6 +41,7 @@ #define PCI_DEVICE_ID_INTEL_TGPLP 0xa0ee #define PCI_DEVICE_ID_INTEL_TGPH 0x43ee #define PCI_DEVICE_ID_INTEL_JSP 0x4dee +#define PCI_DEVICE_ID_INTEL_WCL 0x4d7e #define PCI_DEVICE_ID_INTEL_ADL 0x460e #define PCI_DEVICE_ID_INTEL_ADL_PCH 0x51ee #define PCI_DEVICE_ID_INTEL_ADLN 0x465e @@ -431,6 +432,7 @@ static const struct pci_device_id dwc3_pci_id_table[] = { { PCI_DEVICE_DATA(INTEL, TGPLP, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, TGPH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, JSP, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, WCL, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, ADL, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, ADL_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, ADLN, &dwc3_pci_intel_swnode) }, diff --git a/drivers/usb/dwc3/dwc3-qcom.c b/drivers/usb/dwc3/dwc3-qcom.c index 82544374110b0..add808efb8716 100644 --- a/drivers/usb/dwc3/dwc3-qcom.c +++ b/drivers/usb/dwc3/dwc3-qcom.c @@ -854,13 +854,13 @@ static int dwc3_qcom_probe(struct platform_device *pdev) ret = reset_control_deassert(qcom->resets); if (ret) { dev_err(&pdev->dev, "failed to deassert resets, err=%d\n", ret); - goto reset_assert; + return ret; } ret = dwc3_qcom_clk_init(qcom, of_clk_get_parent_count(np)); if (ret) { dev_err_probe(dev, ret, "failed to get clocks\n"); - goto reset_assert; + return ret; } res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -964,8 +964,6 @@ static int dwc3_qcom_probe(struct platform_device *pdev) clk_disable_unprepare(qcom->clks[i]); clk_put(qcom->clks[i]); } -reset_assert: - reset_control_assert(qcom->resets); return ret; } @@ -995,8 +993,6 @@ static void dwc3_qcom_remove(struct platform_device *pdev) qcom->num_clocks = 0; dwc3_qcom_interconnect_exit(qcom); - reset_control_assert(qcom->resets); - pm_runtime_allow(dev); pm_runtime_disable(dev); } diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 874497f86499b..876a839f2d1d0 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -288,7 +288,9 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) dwc3_ep0_prepare_one_trb(dep, dwc->ep0_trb_addr, 8, DWC3_TRBCTL_CONTROL_SETUP, false); ret = dwc3_ep0_start_trans(dep); - WARN_ON(ret < 0); + if (ret < 0) + dev_err(dwc->dev, "ep0 out start transfer failed: %d\n", ret); + for (i = 2; i < DWC3_ENDPOINTS_NUM; i++) { struct dwc3_ep *dwc3_ep; @@ -1061,7 +1063,9 @@ static void __dwc3_ep0_do_control_data(struct dwc3 *dwc, ret = dwc3_ep0_start_trans(dep); } - WARN_ON(ret < 0); + if (ret < 0) + dev_err(dwc->dev, + "ep0 data phase start transfer failed: %d\n", ret); } static int dwc3_ep0_start_control_status(struct dwc3_ep *dep) @@ -1078,7 +1082,12 @@ static int dwc3_ep0_start_control_status(struct dwc3_ep *dep) static void __dwc3_ep0_do_control_status(struct dwc3 *dwc, struct dwc3_ep *dep) { - WARN_ON(dwc3_ep0_start_control_status(dep)); + int ret; + + ret = dwc3_ep0_start_control_status(dep); + if (ret) + dev_err(dwc->dev, + "ep0 status phase start transfer failed: %d\n", ret); } static void dwc3_ep0_do_control_status(struct dwc3 *dwc, @@ -1121,7 +1130,10 @@ void dwc3_ep0_end_control_data(struct dwc3 *dwc, struct dwc3_ep *dep) cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); memset(¶ms, 0, sizeof(params)); ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); - WARN_ON_ONCE(ret); + if (ret) + dev_err_ratelimited(dwc->dev, + "ep0 data phase end transfer failed: %d\n", ret); + dep->resource_index = 0; } diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 0cfb975702b73..ce45560adcae4 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1765,7 +1765,11 @@ static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool int dep->flags |= DWC3_EP_DELAY_STOP; return 0; } - WARN_ON_ONCE(ret); + + if (ret) + dev_err_ratelimited(dep->dwc->dev, + "end transfer failed: %d\n", ret); + dep->resource_index = 0; if (!interrupt) @@ -3737,6 +3741,15 @@ static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep, static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, const struct dwc3_event_depevt *event) { + /* + * During a device-initiated disconnect, a late xferNotReady event can + * be generated after the End Transfer command resets the event filter, + * but before the controller is halted. Ignore it to prevent a new + * transfer from starting. + */ + if (!dep->dwc->connected) + return; + dwc3_gadget_endpoint_frame_from_event(dep, event); /* @@ -4038,7 +4051,9 @@ static void dwc3_clear_stall_all_ep(struct dwc3 *dwc) dep->flags &= ~DWC3_EP_STALL; ret = dwc3_send_clear_stall_ep_cmd(dep); - WARN_ON_ONCE(ret); + if (ret) + dev_err_ratelimited(dwc->dev, + "failed to clear STALL on %s\n", dep->name); } } @@ -4787,7 +4802,7 @@ void dwc3_gadget_exit(struct dwc3 *dwc) if (!dwc->gadget) return; - dwc3_enable_susphy(dwc, false); + dwc3_enable_susphy(dwc, true); usb_del_gadget(dwc->gadget); dwc3_gadget_free_endpoints(dwc); usb_put_gadget(dwc->gadget); @@ -4804,26 +4819,22 @@ int dwc3_gadget_suspend(struct dwc3 *dwc) int ret; ret = dwc3_gadget_soft_disconnect(dwc); - if (ret) - goto err; - - spin_lock_irqsave(&dwc->lock, flags); - if (dwc->gadget_driver) - dwc3_disconnect_gadget(dwc); - spin_unlock_irqrestore(&dwc->lock, flags); - - return 0; - -err: /* * Attempt to reset the controller's state. Likely no * communication can be established until the host * performs a port reset. */ - if (dwc->softconnect) + if (ret && dwc->softconnect) { dwc3_gadget_soft_connect(dwc); + return -EAGAIN; + } - return ret; + spin_lock_irqsave(&dwc->lock, flags); + if (dwc->gadget_driver) + dwc3_disconnect_gadget(dwc); + spin_unlock_irqrestore(&dwc->lock, flags); + + return 0; } int dwc3_gadget_resume(struct dwc3 *dwc) diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index cc6449538a7d8..c5be6d649f5a0 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -173,7 +173,7 @@ void dwc3_host_exit(struct dwc3 *dwc) if (dwc->sys_wakeup) device_init_wakeup(&dwc->xhci->dev, false); - dwc3_enable_susphy(dwc, false); + dwc3_enable_susphy(dwc, true); platform_device_unregister(dwc->xhci); dwc->xhci = NULL; } diff --git a/drivers/usb/dwc3/trace.h b/drivers/usb/dwc3/trace.h index d2997d17cfbe0..3dc9b009e2d74 100644 --- a/drivers/usb/dwc3/trace.h +++ b/drivers/usb/dwc3/trace.h @@ -19,6 +19,23 @@ #include "core.h" #include "debug.h" +DECLARE_EVENT_CLASS(dwc3_log_set_prtcap, + TP_PROTO(u32 mode), + TP_ARGS(mode), + TP_STRUCT__entry( + __field(u32, mode) + ), + TP_fast_assign( + __entry->mode = mode; + ), + TP_printk("mode %s", dwc3_mode_string(__entry->mode)) +); + +DEFINE_EVENT(dwc3_log_set_prtcap, dwc3_set_prtcap, + TP_PROTO(u32 mode), + TP_ARGS(mode) +); + DECLARE_EVENT_CLASS(dwc3_log_io, TP_PROTO(void *base, u32 offset, u32 value), TP_ARGS(base, offset, value), diff --git a/drivers/usb/early/xhci-dbc.c b/drivers/usb/early/xhci-dbc.c index 341408410ed93..41118bba91978 100644 --- a/drivers/usb/early/xhci-dbc.c +++ b/drivers/usb/early/xhci-dbc.c @@ -681,6 +681,10 @@ int __init early_xdbc_setup_hardware(void) xdbc.table_base = NULL; xdbc.out_buf = NULL; + + early_iounmap(xdbc.xhci_base, xdbc.xhci_length); + xdbc.xhci_base = NULL; + xdbc.xhci_length = 0; } return ret; diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index 0e10f38da564d..f842fd55bdeed 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -1795,14 +1795,14 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl) android_set_connected(&cdev->android_opts); if (w_length > USB_COMP_EP0_BUFSIZ) { - if (ctrl->bRequestType & USB_DIR_IN) { + if (ctrl->bRequestType == USB_DIR_OUT) { + goto done; + } else { /* Cast away the const, we are going to overwrite on purpose. */ __le16 *temp = (__le16 *)&ctrl->wLength; *temp = cpu_to_le16(USB_COMP_EP0_BUFSIZ); w_length = USB_COMP_EP0_BUFSIZ; - } else { - goto done; } } @@ -2017,15 +2017,13 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl) if (f->get_status) { status = f->get_status(f); + if (status < 0) break; - } else { - /* Set D0 and D1 bits based on func wakeup capability */ - if (f->config->bmAttributes & USB_CONFIG_ATT_WAKEUP) { - status |= USB_INTRF_STAT_FUNC_RW_CAP; - if (f->func_wakeup_armed) - status |= USB_INTRF_STAT_FUNC_RW; - } + + /* if D5 is not set, then device is not wakeup capable */ + if (!(f->config->bmAttributes & USB_CONFIG_ATT_WAKEUP)) + status &= ~(USB_INTRF_STAT_FUNC_RW_CAP | USB_INTRF_STAT_FUNC_RW); } put_unaligned_le16(status & 0x0000ffff, req->buf); @@ -2522,6 +2520,11 @@ int composite_os_desc_req_prepare(struct usb_composite_dev *cdev, if (!cdev->os_desc_req->buf) { ret = -ENOMEM; usb_ep_free_request(ep0, cdev->os_desc_req); + /* + * Set os_desc_req to NULL so that composite_dev_cleanup() + * will not try to free it again. + */ + cdev->os_desc_req = NULL; goto end; } cdev->os_desc_req->context = cdev; diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 23b941db04b4c..291e24359afac 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -1067,6 +1067,8 @@ static ssize_t webusb_landingPage_store(struct config_item *item, const char *pa unsigned int bytes_to_strip = 0; int l = len; + if (!len) + return len; if (page[l - 1] == '\n') { --l; ++bytes_to_strip; @@ -1190,6 +1192,8 @@ static ssize_t os_desc_qw_sign_store(struct config_item *item, const char *page, struct gadget_info *gi = os_desc_item_to_gadget_info(item); int res, l; + if (!len) + return len; l = min((int)len, OS_STRING_QW_SIGN_LEN >> 1); if (page[l - 1] == '\n') --l; @@ -1748,6 +1752,8 @@ static int configfs_composite_bind(struct usb_gadget *gadget, cdev->use_os_string = true; cdev->b_vendor_code = gi->b_vendor_code; memcpy(cdev->qw_sign, gi->qw_sign, OS_STRING_QW_SIGN_LEN); + } else { + cdev->use_os_string = false; } if (gadget_is_otg(gadget) && !otg_desc[0]) { diff --git a/drivers/usb/gadget/function/android_f_accessory.c b/drivers/usb/gadget/function/android_f_accessory.c index 0d096fa0a20b1..a4446f4b551cc 100644 --- a/drivers/usb/gadget/function/android_f_accessory.c +++ b/drivers/usb/gadget/function/android_f_accessory.c @@ -45,6 +45,12 @@ #define TX_REQ_MAX 4 #define RX_REQ_MAX 2 +/* Safe upper bound for a HID report descriptor*/ +#define MAX_HID_DESC_SIZE 4096 + +/* Maximum number of concurrent AOA‑HID devices*/ +#define MAX_HID_DEVICES 32 + struct acc_hid_dev { struct list_head list; struct hid_device *hid; @@ -596,8 +602,12 @@ static int acc_register_hid(struct acc_dev *dev, int id, int desc_length) struct acc_hid_dev *hid; unsigned long flags; - /* report descriptor length must be > 0 */ - if (desc_length <= 0) + /* + * Report descriptor length must be > 0 and not exceed the maximum size. + * Device ID must be > 0 and within the allowed range. + */ + if (desc_length <= 0 || desc_length > MAX_HID_DESC_SIZE || + id <= 0 || id > MAX_HID_DEVICES) return -EINVAL; spin_lock_irqsave(&dev->lock, flags); diff --git a/drivers/usb/gadget/function/f_ecm.c b/drivers/usb/gadget/function/f_ecm.c index f55f60639e425..2afc30de54ce2 100644 --- a/drivers/usb/gadget/function/f_ecm.c +++ b/drivers/usb/gadget/function/f_ecm.c @@ -892,6 +892,12 @@ static void ecm_resume(struct usb_function *f) gether_resume(&ecm->port); } +static int ecm_get_status(struct usb_function *f) +{ + return (f->func_wakeup_armed ? USB_INTRF_STAT_FUNC_RW : 0) | + USB_INTRF_STAT_FUNC_RW_CAP; +} + static void ecm_free(struct usb_function *f) { struct f_ecm *ecm; @@ -960,6 +966,7 @@ static struct usb_function *ecm_alloc(struct usb_function_instance *fi) ecm->port.func.disable = ecm_disable; ecm->port.func.free_func = ecm_free; ecm->port.func.suspend = ecm_suspend; + ecm->port.func.get_status = ecm_get_status; ecm->port.func.resume = ecm_resume; return &ecm->port.func; diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 4a88546b1b157..14424e395f2a9 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -1941,7 +1941,12 @@ static int ffs_func_eps_enable(struct ffs_function *func) ep = func->eps; epfile = ffs->epfiles; count = ffs->eps_count; - while(count--) { + if (!epfile) { + ret = -ENOMEM; + goto done; + } + + while (count--) { ep->ep->driver_data = ep; ret = config_ep_by_speed(func->gadget, &func->function, ep->ep); @@ -1965,6 +1970,7 @@ static int ffs_func_eps_enable(struct ffs_function *func) } wake_up_interruptible(&ffs->wait); +done: spin_unlock_irqrestore(&func->ffs->eps_lock, flags); return ret; diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c index 2db01e03bfbf0..d8dafebeabea4 100644 --- a/drivers/usb/gadget/function/f_hid.c +++ b/drivers/usb/gadget/function/f_hid.c @@ -118,8 +118,8 @@ static struct hid_descriptor hidg_desc = { .bcdHID = cpu_to_le16(0x0101), .bCountryCode = 0x00, .bNumDescriptors = 0x1, - /*.desc[0].bDescriptorType = DYNAMIC */ - /*.desc[0].wDescriptorLenght = DYNAMIC */ + /*.rpt_desc.bDescriptorType = DYNAMIC */ + /*.rpt_desc.wDescriptorLength = DYNAMIC */ }; /* Super-Speed Support */ @@ -494,7 +494,7 @@ static ssize_t f_hidg_write(struct file *file, const char __user *buffer, } req->status = 0; - req->zero = 0; + req->zero = 1; req->length = count; req->complete = f_hidg_req_complete; req->context = hidg; @@ -728,8 +728,8 @@ static int hidg_setup(struct usb_function *f, struct hid_descriptor hidg_desc_copy = hidg_desc; VDBG(cdev, "USB_REQ_GET_DESCRIPTOR: HID\n"); - hidg_desc_copy.desc[0].bDescriptorType = HID_DT_REPORT; - hidg_desc_copy.desc[0].wDescriptorLength = + hidg_desc_copy.rpt_desc.bDescriptorType = HID_DT_REPORT; + hidg_desc_copy.rpt_desc.wDescriptorLength = cpu_to_le16(hidg->report_desc_length); length = min_t(unsigned short, length, @@ -765,7 +765,7 @@ static int hidg_setup(struct usb_function *f, return -EOPNOTSUPP; respond: - req->zero = 0; + req->zero = 1; req->length = length; status = usb_ep_queue(cdev->gadget->ep0, req, GFP_ATOMIC); if (status < 0) @@ -970,8 +970,8 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) * We can use hidg_desc struct here but we should not relay * that its content won't change after returning from this function. */ - hidg_desc.desc[0].bDescriptorType = HID_DT_REPORT; - hidg_desc.desc[0].wDescriptorLength = + hidg_desc.rpt_desc.bDescriptorType = HID_DT_REPORT; + hidg_desc.rpt_desc.wDescriptorLength = cpu_to_le16(hidg->report_desc_length); hidg_hs_in_ep_desc.bEndpointAddress = diff --git a/drivers/usb/gadget/function/f_midi2.c b/drivers/usb/gadget/function/f_midi2.c index b7dada064890b..d7ed50ff380cf 100644 --- a/drivers/usb/gadget/function/f_midi2.c +++ b/drivers/usb/gadget/function/f_midi2.c @@ -475,7 +475,7 @@ static void reply_ump_stream_ep_info(struct f_midi2_ep *ep) /* reply a UMP EP device info */ static void reply_ump_stream_ep_device(struct f_midi2_ep *ep) { - struct snd_ump_stream_msg_devince_info rep = { + struct snd_ump_stream_msg_device_info rep = { .type = UMP_MSG_TYPE_STREAM, .status = UMP_STREAM_MSG_STATUS_DEVICE_INFO, .manufacture_id = ep->info.manufacturer, @@ -1601,6 +1601,7 @@ static int f_midi2_create_card(struct f_midi2 *midi2) strscpy(fb->info.name, ump_fb_name(b), sizeof(fb->info.name)); } + snd_ump_update_group_attrs(ump); } for (i = 0; i < midi2->num_eps; i++) { @@ -1738,9 +1739,12 @@ static int f_midi2_create_usb_configs(struct f_midi2 *midi2, case USB_SPEED_HIGH: midi2_midi1_ep_out_desc.wMaxPacketSize = cpu_to_le16(512); midi2_midi1_ep_in_desc.wMaxPacketSize = cpu_to_le16(512); - for (i = 0; i < midi2->num_eps; i++) + for (i = 0; i < midi2->num_eps; i++) { midi2_midi2_ep_out_desc[i].wMaxPacketSize = cpu_to_le16(512); + midi2_midi2_ep_in_desc[i].wMaxPacketSize = + cpu_to_le16(512); + } fallthrough; case USB_SPEED_FULL: midi1_in_eps = midi2_midi1_ep_in_descs; @@ -1749,9 +1753,12 @@ static int f_midi2_create_usb_configs(struct f_midi2 *midi2, case USB_SPEED_SUPER: midi2_midi1_ep_out_desc.wMaxPacketSize = cpu_to_le16(1024); midi2_midi1_ep_in_desc.wMaxPacketSize = cpu_to_le16(1024); - for (i = 0; i < midi2->num_eps; i++) + for (i = 0; i < midi2->num_eps; i++) { midi2_midi2_ep_out_desc[i].wMaxPacketSize = cpu_to_le16(1024); + midi2_midi2_ep_in_desc[i].wMaxPacketSize = + cpu_to_le16(1024); + } midi1_in_eps = midi2_midi1_ep_in_ss_descs; midi1_out_eps = midi2_midi1_ep_out_ss_descs; break; diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index f5731d465cd7b..25b136d5326ef 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -1418,7 +1418,7 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) struct usb_composite_dev *cdev = c->cdev; struct f_ncm *ncm = func_to_ncm(f); struct usb_string *us; - int status = 0; + int status; struct usb_ep *ep; struct f_ncm_opts *ncm_opts; @@ -1436,16 +1436,24 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) f->os_desc_table[0].os_desc = &ncm_opts->ncm_os_desc; } - mutex_lock(&ncm_opts->lock); - gether_set_gadget(ncm_opts->net, cdev->gadget); - if (!ncm_opts->bound) + /* + * in drivers/usb/gadget/configfs.c:configfs_composite_bind() + * configurations are bound in sequence with list_for_each_entry, + * in each configuration its functions are bound in sequence + * with list_for_each_entry, so we assume no race condition + * with regard to ncm_opts->bound access + */ + if (!ncm_opts->bound) { + mutex_lock(&ncm_opts->lock); + gether_set_gadget(ncm_opts->net, cdev->gadget); status = gether_register_netdev(ncm_opts->net); - mutex_unlock(&ncm_opts->lock); - - if (status) - goto fail; + mutex_unlock(&ncm_opts->lock); + if (status) + goto fail; + ncm_opts->bound = true; + } - ncm_opts->bound = true; + ncm_string_defs[1].s = ncm->ethaddr; us = usb_gstrings_attach(cdev, ncm_strings, ARRAY_SIZE(ncm_string_defs)); @@ -1708,7 +1716,6 @@ static struct usb_function *ncm_alloc(struct usb_function_instance *fi) mutex_unlock(&opts->lock); return ERR_PTR(-EINVAL); } - ncm_string_defs[STRING_MAC_IDX].s = ncm->ethaddr; spin_lock_init(&ncm->lock); ncm_reset_values(ncm); diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index a7cd0a06879e6..5d0d894953953 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -1297,14 +1297,14 @@ static struct se_portal_group *usbg_make_tpg(struct se_wwn *wwn, struct usbg_tport *tport = container_of(wwn, struct usbg_tport, tport_wwn); struct usbg_tpg *tpg; - unsigned long tpgt; + u16 tpgt; int ret; struct f_tcm_opts *opts; unsigned i; if (strstr(name, "tpgt_") != name) return ERR_PTR(-EINVAL); - if (kstrtoul(name + 5, 0, &tpgt) || tpgt > UINT_MAX) + if (kstrtou16(name + 5, 0, &tpgt)) return ERR_PTR(-EINVAL); ret = -ENODEV; mutex_lock(&tpg_instances_lock); diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index 74c00490e4f84..239c08efa5792 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -362,6 +362,10 @@ uvc_function_set_alt(struct usb_function *f, unsigned interface, unsigned alt) return ret; usb_ep_enable(uvc->video.ep); + uvc->video.max_req_size = uvc->video.ep->maxpacket + * max_t(unsigned int, uvc->video.ep->maxburst, 1) + * (uvc->video.ep->mult); + memset(&v4l2_event, 0, sizeof(v4l2_event)); v4l2_event.type = UVC_EVENT_STREAMON; v4l2_event_queue(&uvc->vdev, &v4l2_event); diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c index 729b0472bab09..4925c874316cb 100644 --- a/drivers/usb/gadget/function/u_serial.c +++ b/drivers/usb/gadget/function/u_serial.c @@ -291,8 +291,8 @@ __acquires(&port->port_lock) break; } - if (do_tty_wake && port->port.tty) - tty_wakeup(port->port.tty); + if (do_tty_wake) + tty_port_tty_wakeup(&port->port); return status; } @@ -539,20 +539,16 @@ static int gs_alloc_requests(struct usb_ep *ep, struct list_head *head, static int gs_start_io(struct gs_port *port) { struct list_head *head = &port->read_pool; - struct usb_ep *ep; + struct usb_ep *ep = port->port_usb->out; int status; unsigned started; - if (!port->port_usb || !port->port.tty) - return -EIO; - /* Allocate RX and TX I/O buffers. We can't easily do this much * earlier (with GFP_KERNEL) because the requests are coupled to * endpoints, as are the packet sizes we'll be using. Different * configurations may use different endpoints with a given port; * and high speed vs full speed changes packet sizes too. */ - ep = port->port_usb->out; status = gs_alloc_requests(ep, head, gs_read_complete, &port->read_allocated); if (status) @@ -573,7 +569,7 @@ static int gs_start_io(struct gs_port *port) gs_start_tx(port); /* Unblock any pending writes into our circular buffer, in case * we didn't in gs_start_tx() */ - tty_wakeup(port->port.tty); + tty_port_tty_wakeup(&port->port); } else { /* Free reqs only if we are still connected */ if (port->port_usb) { diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h index 6f44dd7323150..68262b25915e1 100644 --- a/drivers/usb/gadget/function/uvc.h +++ b/drivers/usb/gadget/function/uvc.h @@ -107,7 +107,7 @@ struct uvc_video { unsigned int width; unsigned int height; unsigned int imagesize; - unsigned int interval; + unsigned int interval; /* in 100ns units */ struct mutex mutex; /* protects frame parameters */ unsigned int uvc_num_requests; @@ -117,6 +117,7 @@ struct uvc_video { /* Requests */ bool is_enabled; /* tracks whether video stream is enabled */ unsigned int req_size; + unsigned int max_req_size; struct list_head ureqs; /* all uvc_requests allocated by uvc_video */ /* USB requests that the video pump thread can encode into */ diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c index 8f7786a91d88f..257e45df2cc1a 100644 --- a/drivers/usb/gadget/function/uvc_configfs.c +++ b/drivers/usb/gadget/function/uvc_configfs.c @@ -2916,8 +2916,15 @@ static struct config_group *uvcg_framebased_make(struct config_group *group, 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71 }; + struct uvcg_color_matching *color_match; + struct config_item *streaming; struct uvcg_framebased *h; + streaming = group->cg_item.ci_parent; + color_match = uvcg_format_get_default_color_match(streaming); + if (!color_match) + return ERR_PTR(-EINVAL); + h = kzalloc(sizeof(*h), GFP_KERNEL); if (!h) return ERR_PTR(-ENOMEM); @@ -2936,6 +2943,9 @@ static struct config_group *uvcg_framebased_make(struct config_group *group, INIT_LIST_HEAD(&h->fmt.frames); h->fmt.type = UVCG_FRAMEBASED; + + h->fmt.color_matching = color_match; + color_match->refcnt++; config_group_init_type_name(&h->fmt.group, name, &uvcg_framebased_type); diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index 5eaeae3e2441c..1d9cc5a449f22 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -86,10 +86,17 @@ static int uvc_buffer_prepare(struct vb2_buffer *vb) buf->bytesused = 0; } else { buf->bytesused = vb2_get_plane_payload(vb, 0); - buf->req_payload_size = - DIV_ROUND_UP(buf->bytesused + - (video->reqs_per_frame * UVCG_REQUEST_HEADER_LEN), - video->reqs_per_frame); + + if (video->reqs_per_frame != 0) { + buf->req_payload_size = + DIV_ROUND_UP(buf->bytesused + + (video->reqs_per_frame * UVCG_REQUEST_HEADER_LEN), + video->reqs_per_frame); + if (buf->req_payload_size > video->req_size) + buf->req_payload_size = video->req_size; + } else { + buf->req_payload_size = video->max_req_size; + } } return 0; diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index c37e60c3e3c4b..f5efa533515c4 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -498,13 +498,11 @@ uvc_video_prep_requests(struct uvc_video *video) { struct uvc_device *uvc = container_of(video, struct uvc_device, video); struct usb_composite_dev *cdev = uvc->func.config->cdev; - unsigned int interval_duration = video->ep->desc->bInterval * 1250; + unsigned int interval_duration; unsigned int max_req_size, req_size, header_size; unsigned int nreq; - max_req_size = video->ep->maxpacket - * max_t(unsigned int, video->ep->maxburst, 1) - * (video->ep->mult); + max_req_size = video->max_req_size; if (!usb_endpoint_xfer_isoc(video->ep->desc)) { video->req_size = max_req_size; @@ -514,8 +512,11 @@ uvc_video_prep_requests(struct uvc_video *video) return; } + interval_duration = 2 << (video->ep->desc->bInterval - 1); if (cdev->gadget->speed < USB_SPEED_HIGH) - interval_duration = video->ep->desc->bInterval * 10000; + interval_duration *= 10000; + else + interval_duration *= 1250; nreq = DIV_ROUND_UP(video->interval, interval_duration); @@ -836,7 +837,6 @@ int uvcg_video_init(struct uvc_video *video, struct uvc_device *uvc) video->interval = 666666; /* Initialize the video buffers queue. */ - uvcg_queue_init(&video->queue, uvc->v4l2_dev.dev->parent, + return uvcg_queue_init(&video->queue, uvc->v4l2_dev.dev->parent, V4L2_BUF_TYPE_VIDEO_OUTPUT, &video->mutex); - return 0; } diff --git a/drivers/usb/gadget/legacy/dbgp.c b/drivers/usb/gadget/legacy/dbgp.c index b62e45235e8e1..346d5ebd3499c 100644 --- a/drivers/usb/gadget/legacy/dbgp.c +++ b/drivers/usb/gadget/legacy/dbgp.c @@ -346,18 +346,17 @@ static int dbgp_setup(struct usb_gadget *gadget, u16 len = 0; if (length > DBGP_REQ_LEN) { - if (ctrl->bRequestType & USB_DIR_IN) { + if (ctrl->bRequestType == USB_DIR_OUT) { + return err; + } else { /* Cast away the const, we are going to overwrite on purpose. */ __le16 *temp = (__le16 *)&ctrl->wLength; *temp = cpu_to_le16(DBGP_REQ_LEN); length = DBGP_REQ_LEN; - } else { - return err; } } - if (request == USB_REQ_GET_DESCRIPTOR) { switch (value>>8) { case USB_DT_DEVICE: diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index ce9e31f3d26bc..3619b1a632f1f 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -1336,14 +1336,14 @@ gadgetfs_setup (struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl) u16 w_length = le16_to_cpu(ctrl->wLength); if (w_length > RBUF_SIZE) { - if (ctrl->bRequestType & USB_DIR_IN) { + if (ctrl->bRequestType == USB_DIR_OUT) { + return value; + } else { /* Cast away the const, we are going to overwrite on purpose. */ __le16 *temp = (__le16 *)&ctrl->wLength; *temp = cpu_to_le16(RBUF_SIZE); w_length = RBUF_SIZE; - } else { - return value; } } diff --git a/drivers/usb/gadget/legacy/raw_gadget.c b/drivers/usb/gadget/legacy/raw_gadget.c index ea106ad665a1f..2deab4a6030d7 100644 --- a/drivers/usb/gadget/legacy/raw_gadget.c +++ b/drivers/usb/gadget/legacy/raw_gadget.c @@ -620,8 +620,6 @@ static void *raw_alloc_io_data(struct usb_raw_ep_io *io, void __user *ptr, return ERR_PTR(-EINVAL); if (!usb_raw_io_flags_valid(io->flags)) return ERR_PTR(-EINVAL); - if (io->length > PAGE_SIZE) - return ERR_PTR(-EINVAL); if (get_from_user) data = memdup_user(ptr + sizeof(*io), io->length); else { diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index a4120a25428e5..bac6dd8e87110 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -1120,6 +1120,7 @@ void usb_gadget_set_state(struct usb_gadget *gadget, { gadget->state = state; schedule_work(&gadget->work); + trace_usb_gadget_set_state(gadget, 0); } EXPORT_SYMBOL_GPL(usb_gadget_set_state); diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c index d5d89fadde433..a06f56c08e19d 100644 --- a/drivers/usb/gadget/udc/dummy_hcd.c +++ b/drivers/usb/gadget/udc/dummy_hcd.c @@ -764,8 +764,7 @@ static int dummy_dequeue(struct usb_ep *_ep, struct usb_request *_req) if (!dum->driver) return -ESHUTDOWN; - local_irq_save(flags); - spin_lock(&dum->lock); + spin_lock_irqsave(&dum->lock, flags); list_for_each_entry(iter, &ep->queue, queue) { if (&iter->req != _req) continue; @@ -775,15 +774,16 @@ static int dummy_dequeue(struct usb_ep *_ep, struct usb_request *_req) retval = 0; break; } - spin_unlock(&dum->lock); if (retval == 0) { dev_dbg(udc_dev(dum), "dequeued req %p from %s, len %d buf %p\n", req, _ep->name, _req->length, _req->buf); + spin_unlock(&dum->lock); usb_gadget_giveback_request(_ep, _req); + spin_lock(&dum->lock); } - local_irq_restore(flags); + spin_unlock_irqrestore(&dum->lock, flags); return retval; } diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c index a93ad93390ba1..34685c714473d 100644 --- a/drivers/usb/gadget/udc/renesas_usb3.c +++ b/drivers/usb/gadget/udc/renesas_usb3.c @@ -2658,6 +2658,7 @@ static void renesas_usb3_remove(struct platform_device *pdev) struct renesas_usb3 *usb3 = platform_get_drvdata(pdev); debugfs_remove_recursive(usb3->dentry); + put_device(usb3->host_dev); device_remove_file(&pdev->dev, &dev_attr_role); cancel_work_sync(&usb3->role_work); diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c index 7aa46d426f31b..9bb54da8a6ae1 100644 --- a/drivers/usb/gadget/udc/tegra-xudc.c +++ b/drivers/usb/gadget/udc/tegra-xudc.c @@ -1749,6 +1749,10 @@ static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep) val = xudc_readl(xudc, CTRL); val &= ~CTRL_RUN; xudc_writel(xudc, val, CTRL); + + val = xudc_readl(xudc, ST); + if (val & ST_RC) + xudc_writel(xudc, ST_RC, ST); } dev_info(xudc->dev, "ep %u disabled\n", ep->index); diff --git a/drivers/usb/gadget/udc/trace.h b/drivers/usb/gadget/udc/trace.h index a5ed26fbc2dad..661158d3a39cd 100644 --- a/drivers/usb/gadget/udc/trace.h +++ b/drivers/usb/gadget/udc/trace.h @@ -81,6 +81,11 @@ DECLARE_EVENT_CLASS(udc_log_gadget, __entry->ret) ); +DEFINE_EVENT(udc_log_gadget, usb_gadget_set_state, + TP_PROTO(struct usb_gadget *g, int ret), + TP_ARGS(g, ret) +); + DEFINE_EVENT(udc_log_gadget, usb_gadget_frame_number, TP_PROTO(struct usb_gadget *g, int ret), TP_ARGS(g, ret) diff --git a/drivers/usb/host/max3421-hcd.c b/drivers/usb/host/max3421-hcd.c index cc1f579f02de1..2d792ab271e2a 100644 --- a/drivers/usb/host/max3421-hcd.c +++ b/drivers/usb/host/max3421-hcd.c @@ -1916,7 +1916,7 @@ max3421_probe(struct spi_device *spi) if (hcd) { kfree(max3421_hcd->tx); kfree(max3421_hcd->rx); - if (max3421_hcd->spi_thread) + if (!IS_ERR_OR_NULL(max3421_hcd->spi_thread)) kthread_stop(max3421_hcd->spi_thread); usb_put_hcd(hcd); } diff --git a/drivers/usb/host/uhci-platform.c b/drivers/usb/host/uhci-platform.c index 3dec5dd3a0d5c..712389599d468 100644 --- a/drivers/usb/host/uhci-platform.c +++ b/drivers/usb/host/uhci-platform.c @@ -121,7 +121,7 @@ static int uhci_hcd_platform_probe(struct platform_device *pdev) } /* Get and enable clock if any specified */ - uhci->clk = devm_clk_get(&pdev->dev, NULL); + uhci->clk = devm_clk_get_optional(&pdev->dev, NULL); if (IS_ERR(uhci->clk)) { ret = PTR_ERR(uhci->clk); goto err_rmr; diff --git a/drivers/usb/host/xhci-dbgcap.c b/drivers/usb/host/xhci-dbgcap.c index fab9e6be4e27a..4d975b26a185c 100644 --- a/drivers/usb/host/xhci-dbgcap.c +++ b/drivers/usb/host/xhci-dbgcap.c @@ -86,13 +86,34 @@ static u32 xhci_dbc_populate_strings(struct dbc_str_descs *strings) return string_length; } +static void xhci_dbc_init_ep_contexts(struct xhci_dbc *dbc) +{ + struct xhci_ep_ctx *ep_ctx; + unsigned int max_burst; + dma_addr_t deq; + + max_burst = DBC_CTRL_MAXBURST(readl(&dbc->regs->control)); + + /* Populate bulk out endpoint context: */ + ep_ctx = dbc_bulkout_ctx(dbc); + deq = dbc_bulkout_enq(dbc); + ep_ctx->ep_info = 0; + ep_ctx->ep_info2 = dbc_epctx_info2(BULK_OUT_EP, 1024, max_burst); + ep_ctx->deq = cpu_to_le64(deq | dbc->ring_out->cycle_state); + + /* Populate bulk in endpoint context: */ + ep_ctx = dbc_bulkin_ctx(dbc); + deq = dbc_bulkin_enq(dbc); + ep_ctx->ep_info = 0; + ep_ctx->ep_info2 = dbc_epctx_info2(BULK_IN_EP, 1024, max_burst); + ep_ctx->deq = cpu_to_le64(deq | dbc->ring_in->cycle_state); +} + static void xhci_dbc_init_contexts(struct xhci_dbc *dbc, u32 string_length) { struct dbc_info_context *info; - struct xhci_ep_ctx *ep_ctx; u32 dev_info; - dma_addr_t deq, dma; - unsigned int max_burst; + dma_addr_t dma; if (!dbc) return; @@ -106,20 +127,8 @@ static void xhci_dbc_init_contexts(struct xhci_dbc *dbc, u32 string_length) info->serial = cpu_to_le64(dma + DBC_MAX_STRING_LENGTH * 3); info->length = cpu_to_le32(string_length); - /* Populate bulk out endpoint context: */ - ep_ctx = dbc_bulkout_ctx(dbc); - max_burst = DBC_CTRL_MAXBURST(readl(&dbc->regs->control)); - deq = dbc_bulkout_enq(dbc); - ep_ctx->ep_info = 0; - ep_ctx->ep_info2 = dbc_epctx_info2(BULK_OUT_EP, 1024, max_burst); - ep_ctx->deq = cpu_to_le64(deq | dbc->ring_out->cycle_state); - - /* Populate bulk in endpoint context: */ - ep_ctx = dbc_bulkin_ctx(dbc); - deq = dbc_bulkin_enq(dbc); - ep_ctx->ep_info = 0; - ep_ctx->ep_info2 = dbc_epctx_info2(BULK_IN_EP, 1024, max_burst); - ep_ctx->deq = cpu_to_le64(deq | dbc->ring_in->cycle_state); + /* Populate bulk in and out endpoint contexts: */ + xhci_dbc_init_ep_contexts(dbc); /* Set DbC context and info registers: */ lo_hi_writeq(dbc->ctx->dma, &dbc->regs->dccp); @@ -421,6 +430,42 @@ dbc_alloc_ctx(struct device *dev, gfp_t flags) return ctx; } +static void xhci_dbc_ring_init(struct xhci_ring *ring) +{ + struct xhci_segment *seg = ring->first_seg; + + /* clear all trbs on ring in case of old ring */ + memset(seg->trbs, 0, TRB_SEGMENT_SIZE); + + /* Only event ring does not use link TRB */ + if (ring->type != TYPE_EVENT) { + union xhci_trb *trb = &seg->trbs[TRBS_PER_SEGMENT - 1]; + + trb->link.segment_ptr = cpu_to_le64(ring->first_seg->dma); + trb->link.control = cpu_to_le32(LINK_TOGGLE | TRB_TYPE(TRB_LINK)); + } + xhci_initialize_ring_info(ring, 1); +} + +static int xhci_dbc_reinit_ep_rings(struct xhci_dbc *dbc) +{ + struct xhci_ring *in_ring = dbc->eps[BULK_IN].ring; + struct xhci_ring *out_ring = dbc->eps[BULK_OUT].ring; + + if (!in_ring || !out_ring || !dbc->ctx) { + dev_warn(dbc->dev, "Can't re-init unallocated endpoints\n"); + return -ENODEV; + } + + xhci_dbc_ring_init(in_ring); + xhci_dbc_ring_init(out_ring); + + /* set ep context enqueue, dequeue, and cycle to initial values */ + xhci_dbc_init_ep_contexts(dbc); + + return 0; +} + static struct xhci_ring * xhci_dbc_ring_alloc(struct device *dev, enum xhci_ring_type type, gfp_t flags) { @@ -449,15 +494,10 @@ xhci_dbc_ring_alloc(struct device *dev, enum xhci_ring_type type, gfp_t flags) seg->dma = dma; - /* Only event ring does not use link TRB */ - if (type != TYPE_EVENT) { - union xhci_trb *trb = &seg->trbs[TRBS_PER_SEGMENT - 1]; - - trb->link.segment_ptr = cpu_to_le64(dma); - trb->link.control = cpu_to_le32(LINK_TOGGLE | TRB_TYPE(TRB_LINK)); - } INIT_LIST_HEAD(&ring->td_list); - xhci_initialize_ring_info(ring, 1); + + xhci_dbc_ring_init(ring); + return ring; dma_fail: kfree(seg); @@ -625,7 +665,8 @@ static int xhci_dbc_start(struct xhci_dbc *dbc) return ret; } - return mod_delayed_work(system_wq, &dbc->event_work, 1); + return mod_delayed_work(system_wq, &dbc->event_work, + msecs_to_jiffies(dbc->poll_interval)); } static void xhci_dbc_stop(struct xhci_dbc *dbc) @@ -639,6 +680,10 @@ static void xhci_dbc_stop(struct xhci_dbc *dbc) case DS_DISABLED: return; case DS_CONFIGURED: + spin_lock(&dbc->lock); + xhci_dbc_flush_requests(dbc); + spin_unlock(&dbc->lock); + if (dbc->driver->disconnect) dbc->driver->disconnect(dbc); break; @@ -810,6 +855,7 @@ static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc) { dma_addr_t deq; union xhci_trb *evt; + enum evtreturn ret = EVT_DONE; u32 ctrl, portsc; bool update_erdp = false; @@ -834,7 +880,8 @@ static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc) dev_info(dbc->dev, "DbC configured\n"); portsc = readl(&dbc->regs->portsc); writel(portsc, &dbc->regs->portsc); - return EVT_GSER; + ret = EVT_GSER; + break; } return EVT_DONE; @@ -846,7 +893,7 @@ static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc) dev_info(dbc->dev, "DbC cable unplugged\n"); dbc->state = DS_ENABLED; xhci_dbc_flush_requests(dbc); - + xhci_dbc_reinit_ep_rings(dbc); return EVT_DISC; } @@ -856,7 +903,7 @@ static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc) writel(portsc, &dbc->regs->portsc); dbc->state = DS_ENABLED; xhci_dbc_flush_requests(dbc); - + xhci_dbc_reinit_ep_rings(dbc); return EVT_DISC; } @@ -894,6 +941,8 @@ static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc) break; case TRB_TYPE(TRB_TRANSFER): dbc_handle_xfer_event(dbc, evt); + if (ret != EVT_GSER) + ret = EVT_XFER_DONE; break; default: break; @@ -912,7 +961,7 @@ static enum evtreturn xhci_dbc_do_handle_events(struct xhci_dbc *dbc) lo_hi_writeq(deq, &dbc->regs->erdp); } - return EVT_DONE; + return ret; } static void xhci_dbc_handle_events(struct work_struct *work) @@ -920,8 +969,11 @@ static void xhci_dbc_handle_events(struct work_struct *work) enum evtreturn evtr; struct xhci_dbc *dbc; unsigned long flags; + unsigned int poll_interval; + unsigned long busypoll_timelimit; dbc = container_of(to_delayed_work(work), struct xhci_dbc, event_work); + poll_interval = dbc->poll_interval; spin_lock_irqsave(&dbc->lock, flags); evtr = xhci_dbc_do_handle_events(dbc); @@ -937,13 +989,28 @@ static void xhci_dbc_handle_events(struct work_struct *work) dbc->driver->disconnect(dbc); break; case EVT_DONE: + /* + * Set fast poll rate if there are pending out transfers, or + * a transfer was recently processed + */ + busypoll_timelimit = dbc->xfer_timestamp + + msecs_to_jiffies(DBC_XFER_INACTIVITY_TIMEOUT); + + if (!list_empty(&dbc->eps[BULK_OUT].list_pending) || + time_is_after_jiffies(busypoll_timelimit)) + poll_interval = 0; + break; + case EVT_XFER_DONE: + dbc->xfer_timestamp = jiffies; + poll_interval = 0; break; default: dev_info(dbc->dev, "stop handling dbc events\n"); return; } - mod_delayed_work(system_wq, &dbc->event_work, 1); + mod_delayed_work(system_wq, &dbc->event_work, + msecs_to_jiffies(poll_interval)); } static ssize_t dbc_show(struct device *dev, @@ -1162,11 +1229,48 @@ static ssize_t dbc_bInterfaceProtocol_store(struct device *dev, return size; } +static ssize_t dbc_poll_interval_ms_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct xhci_dbc *dbc; + struct xhci_hcd *xhci; + + xhci = hcd_to_xhci(dev_get_drvdata(dev)); + dbc = xhci->dbc; + + return sysfs_emit(buf, "%u\n", dbc->poll_interval); +} + +static ssize_t dbc_poll_interval_ms_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct xhci_dbc *dbc; + struct xhci_hcd *xhci; + u32 value; + int ret; + + ret = kstrtou32(buf, 0, &value); + if (ret || value > DBC_POLL_INTERVAL_MAX) + return -EINVAL; + + xhci = hcd_to_xhci(dev_get_drvdata(dev)); + dbc = xhci->dbc; + + dbc->poll_interval = value; + + mod_delayed_work(system_wq, &dbc->event_work, 0); + + return size; +} + static DEVICE_ATTR_RW(dbc); static DEVICE_ATTR_RW(dbc_idVendor); static DEVICE_ATTR_RW(dbc_idProduct); static DEVICE_ATTR_RW(dbc_bcdDevice); static DEVICE_ATTR_RW(dbc_bInterfaceProtocol); +static DEVICE_ATTR_RW(dbc_poll_interval_ms); static struct attribute *dbc_dev_attributes[] = { &dev_attr_dbc.attr, @@ -1174,6 +1278,7 @@ static struct attribute *dbc_dev_attributes[] = { &dev_attr_dbc_idProduct.attr, &dev_attr_dbc_bcdDevice.attr, &dev_attr_dbc_bInterfaceProtocol.attr, + &dev_attr_dbc_poll_interval_ms.attr, NULL }; @@ -1198,6 +1303,7 @@ xhci_alloc_dbc(struct device *dev, void __iomem *base, const struct dbc_driver * dbc->idVendor = DBC_VENDOR_ID; dbc->bcdDevice = DBC_DEVICE_REV; dbc->bInterfaceProtocol = DBC_PROTOCOL; + dbc->poll_interval = DBC_POLL_INTERVAL_DEFAULT; if (readl(&dbc->regs->control) & DBC_CTRL_DBC_ENABLE) goto err; @@ -1275,8 +1381,15 @@ int xhci_dbc_suspend(struct xhci_hcd *xhci) if (!dbc) return 0; - if (dbc->state == DS_CONFIGURED) + switch (dbc->state) { + case DS_ENABLED: + case DS_CONNECTED: + case DS_CONFIGURED: dbc->resume_required = 1; + break; + default: + break; + } xhci_dbc_stop(dbc); diff --git a/drivers/usb/host/xhci-dbgcap.h b/drivers/usb/host/xhci-dbgcap.h index 2de0dc49a3e9f..1fab3eb9c831d 100644 --- a/drivers/usb/host/xhci-dbgcap.h +++ b/drivers/usb/host/xhci-dbgcap.h @@ -93,7 +93,9 @@ struct dbc_ep { #define DBC_QUEUE_SIZE 16 #define DBC_WRITE_BUF_SIZE 8192 - +#define DBC_POLL_INTERVAL_DEFAULT 64 /* milliseconds */ +#define DBC_POLL_INTERVAL_MAX 5000 /* milliseconds */ +#define DBC_XFER_INACTIVITY_TIMEOUT 10 /* milliseconds */ /* * Private structure for DbC hardware state: */ @@ -139,6 +141,8 @@ struct xhci_dbc { enum dbc_state state; struct delayed_work event_work; + unsigned int poll_interval; /* ms */ + unsigned long xfer_timestamp; unsigned resume_required:1; struct dbc_ep eps[2]; @@ -184,6 +188,7 @@ struct dbc_request { enum evtreturn { EVT_ERR = -1, EVT_DONE, + EVT_XFER_DONE, EVT_GSER, EVT_DISC, }; diff --git a/drivers/usb/host/xhci-dbgtty.c b/drivers/usb/host/xhci-dbgtty.c index 0266c2f5bc0d8..aa689fbd3dce6 100644 --- a/drivers/usb/host/xhci-dbgtty.c +++ b/drivers/usb/host/xhci-dbgtty.c @@ -585,6 +585,7 @@ int dbc_tty_init(void) dbc_tty_driver->type = TTY_DRIVER_TYPE_SERIAL; dbc_tty_driver->subtype = SERIAL_TYPE_NORMAL; dbc_tty_driver->init_termios = tty_std_termios; + dbc_tty_driver->init_termios.c_lflag &= ~ECHO; dbc_tty_driver->init_termios.c_cflag = B9600 | CS8 | CREAD | HUPCL | CLOCAL; dbc_tty_driver->init_termios.c_ispeed = 9600; diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index 0df5d807a77e8..d51c22ccb5e15 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -735,8 +735,7 @@ static int xhci_enter_test_mode(struct xhci_hcd *xhci, if (!xhci->devs[i]) continue; - retval = xhci_disable_slot(xhci, i); - xhci_free_virt_device(xhci, i); + retval = xhci_disable_and_free_slot(xhci, i); if (retval) xhci_err(xhci, "Failed to disable slot %d, %d. Enter test mode anyway\n", i, retval); diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index 8d23543430c3c..9d68a6d36aae5 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -855,21 +855,20 @@ int xhci_alloc_tt_info(struct xhci_hcd *xhci, * will be manipulated by the configure endpoint, allocate device, or update * hub functions while this function is removing the TT entries from the list. */ -void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) +void xhci_free_virt_device(struct xhci_hcd *xhci, struct xhci_virt_device *dev, + int slot_id) { - struct xhci_virt_device *dev; int i; int old_active_eps = 0; /* Slot ID 0 is reserved */ - if (slot_id == 0 || !xhci->devs[slot_id]) + if (slot_id == 0 || !dev) return; - dev = xhci->devs[slot_id]; - - xhci->dcbaa->dev_context_ptrs[slot_id] = 0; - if (!dev) - return; + /* If device ctx array still points to _this_ device, clear it */ + if (dev->out_ctx && + xhci->dcbaa->dev_context_ptrs[slot_id] == cpu_to_le64(dev->out_ctx->dma)) + xhci->dcbaa->dev_context_ptrs[slot_id] = 0; trace_xhci_free_virt_device(dev); @@ -908,8 +907,9 @@ void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) if (dev->udev && dev->udev->slot_id) dev->udev->slot_id = 0; - kfree(xhci->devs[slot_id]); - xhci->devs[slot_id] = NULL; + if (xhci->devs[slot_id] == dev) + xhci->devs[slot_id] = NULL; + kfree(dev); } /* @@ -951,7 +951,7 @@ static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_i out: /* we are now at a leaf device */ xhci_debugfs_remove_slot(xhci, slot_id); - xhci_free_virt_device(xhci, slot_id); + xhci_free_virt_device(xhci, xhci->devs[slot_id], slot_id); } int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, @@ -1188,6 +1188,8 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma | dev->eps[0].ring->cycle_state); + ep0_ctx->tx_info = cpu_to_le32(EP_AVG_TRB_LENGTH(8)); + trace_xhci_setup_addressable_virt_device(dev); /* Steps 7 and 8 were done in xhci_alloc_virt_device() */ @@ -1442,6 +1444,10 @@ int xhci_endpoint_init(struct xhci_hcd *xhci, /* Periodic endpoint bInterval limit quirk */ if (usb_endpoint_xfer_int(&ep->desc) || usb_endpoint_xfer_isoc(&ep->desc)) { + if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_9) && + interval >= 9) { + interval = 8; + } if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) && udev->speed >= USB_SPEED_HIGH && interval >= 7) { diff --git a/drivers/usb/host/xhci-pci-renesas.c b/drivers/usb/host/xhci-pci-renesas.c index 93f8b355bc706..4ceed19c64f03 100644 --- a/drivers/usb/host/xhci-pci-renesas.c +++ b/drivers/usb/host/xhci-pci-renesas.c @@ -47,8 +47,9 @@ #define RENESAS_ROM_ERASE_MAGIC 0x5A65726F #define RENESAS_ROM_WRITE_MAGIC 0x53524F4D -#define RENESAS_RETRY 10000 -#define RENESAS_DELAY 10 +#define RENESAS_RETRY 50000 /* 50000 * RENESAS_DELAY ~= 500ms */ +#define RENESAS_CHIP_ERASE_RETRY 500000 /* 500000 * RENESAS_DELAY ~= 5s */ +#define RENESAS_DELAY 10 static int renesas_fw_download_image(struct pci_dev *dev, const u32 *fw, size_t step, bool rom) @@ -405,7 +406,7 @@ static void renesas_rom_erase(struct pci_dev *pdev) /* sleep a bit while ROM is erased */ msleep(20); - for (i = 0; i < RENESAS_RETRY; i++) { + for (i = 0; i < RENESAS_CHIP_ERASE_RETRY; i++) { retval = pci_read_config_byte(pdev, RENESAS_ROM_STATUS, &status); status &= RENESAS_ROM_STATUS_ERASE; diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index c1a172b6feae8..5abc48f148dcb 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -65,12 +65,22 @@ #define PCI_DEVICE_ID_INTEL_ALDER_LAKE_PCH_XHCI 0x51ed #define PCI_DEVICE_ID_INTEL_ALDER_LAKE_N_PCH_XHCI 0x54ed +#define PCI_DEVICE_ID_AMD_ARIEL_TYPEC_XHCI 0x13ed +#define PCI_DEVICE_ID_AMD_ARIEL_TYPEA_XHCI 0x13ee +#define PCI_DEVICE_ID_AMD_STARSHIP_XHCI 0x148c +#define PCI_DEVICE_ID_AMD_FIREFLIGHT_15D4_XHCI 0x15d4 +#define PCI_DEVICE_ID_AMD_FIREFLIGHT_15D5_XHCI 0x15d5 +#define PCI_DEVICE_ID_AMD_RAVEN_15E0_XHCI 0x15e0 +#define PCI_DEVICE_ID_AMD_RAVEN_15E1_XHCI 0x15e1 +#define PCI_DEVICE_ID_AMD_RAVEN2_XHCI 0x15e5 #define PCI_DEVICE_ID_AMD_RENOIR_XHCI 0x1639 #define PCI_DEVICE_ID_AMD_PROMONTORYA_4 0x43b9 #define PCI_DEVICE_ID_AMD_PROMONTORYA_3 0x43ba #define PCI_DEVICE_ID_AMD_PROMONTORYA_2 0x43bb #define PCI_DEVICE_ID_AMD_PROMONTORYA_1 0x43bc +#define PCI_DEVICE_ID_ATI_NAVI10_7316_XHCI 0x7316 + #define PCI_DEVICE_ID_ASMEDIA_1042_XHCI 0x1042 #define PCI_DEVICE_ID_ASMEDIA_1042A_XHCI 0x1142 #define PCI_DEVICE_ID_ASMEDIA_1142_XHCI 0x1242 @@ -348,6 +358,21 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) if (pdev->vendor == PCI_VENDOR_ID_NEC) xhci->quirks |= XHCI_NEC_HOST; + if (pdev->vendor == PCI_VENDOR_ID_AMD && + (pdev->device == PCI_DEVICE_ID_AMD_ARIEL_TYPEC_XHCI || + pdev->device == PCI_DEVICE_ID_AMD_ARIEL_TYPEA_XHCI || + pdev->device == PCI_DEVICE_ID_AMD_STARSHIP_XHCI || + pdev->device == PCI_DEVICE_ID_AMD_FIREFLIGHT_15D4_XHCI || + pdev->device == PCI_DEVICE_ID_AMD_FIREFLIGHT_15D5_XHCI || + pdev->device == PCI_DEVICE_ID_AMD_RAVEN_15E0_XHCI || + pdev->device == PCI_DEVICE_ID_AMD_RAVEN_15E1_XHCI || + pdev->device == PCI_DEVICE_ID_AMD_RAVEN2_XHCI)) + xhci->quirks |= XHCI_LIMIT_ENDPOINT_INTERVAL_9; + + if (pdev->vendor == PCI_VENDOR_ID_ATI && + pdev->device == PCI_DEVICE_ID_ATI_NAVI10_7316_XHCI) + xhci->quirks |= XHCI_LIMIT_ENDPOINT_INTERVAL_9; + if (pdev->vendor == PCI_VENDOR_ID_AMD && xhci->hci_version == 0x96) xhci->quirks |= XHCI_AMD_0x96_HOST; diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 1ab8a9dc7c080..2d3bb45565b3a 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -152,7 +152,7 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s int ret; int irq; struct xhci_plat_priv *priv = NULL; - bool of_match; + const struct of_device_id *of_match; if (usb_disabled()) return -ENODEV; @@ -171,6 +171,7 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s return ret; pm_runtime_set_active(&pdev->dev); + pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_enable(&pdev->dev); pm_runtime_get_noresume(&pdev->dev); @@ -321,7 +322,8 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s } usb3_hcd = xhci_get_usb3_hcd(xhci); - if (usb3_hcd && HCC_MAX_PSA(xhci->hcc_params) >= 4) + if (usb3_hcd && HCC_MAX_PSA(xhci->hcc_params) >= 4 && + !(xhci->quirks & XHCI_BROKEN_STREAMS)) usb3_hcd->can_do_streams = 1; if (xhci->shared_hcd) { diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 5ddf5fc703289..c01f55f2717f0 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -464,9 +464,8 @@ static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) * In the future we should distinguish between -ENODEV and -ETIMEDOUT * and try to recover a -ETIMEDOUT with a host controller reset. */ - ret = xhci_handshake_check_state(xhci, &xhci->op_regs->cmd_ring, - CMD_RING_RUNNING, 0, 5 * 1000 * 1000, - XHCI_STATE_REMOVING); + ret = xhci_handshake(&xhci->op_regs->cmd_ring, + CMD_RING_RUNNING, 0, 5 * 1000 * 1000); if (ret < 0) { xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret); xhci_halt(xhci); @@ -1186,7 +1185,14 @@ static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, */ switch (GET_EP_CTX_STATE(ep_ctx)) { case EP_STATE_HALTED: - xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n"); + xhci_dbg(xhci, "Stop ep completion raced with stall\n"); + /* + * If the halt happened before Stop Endpoint failed, its transfer event + * should have already been handled and Reset Endpoint should be pending. + */ + if (ep->ep_state & EP_HALTED) + goto reset_done; + if (ep->ep_state & EP_HAS_STREAMS) { reset_type = EP_SOFT_RESET; } else { @@ -1197,8 +1203,11 @@ static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, } /* reset ep, reset handler cleans up cancelled tds */ err = xhci_handle_halted_endpoint(xhci, ep, td, reset_type); + xhci_dbg(xhci, "Stop ep completion resetting ep, status %d\n", err); if (err) break; +reset_done: + /* Reset EP handler will clean up cancelled TDs */ ep->ep_state &= ~EP_STOP_CMD_PENDING; return; case EP_STATE_RUNNING: @@ -1305,12 +1314,15 @@ static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci, */ void xhci_hc_died(struct xhci_hcd *xhci) { + bool notify; int i, j; if (xhci->xhc_state & XHCI_STATE_DYING) return; - xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); + notify = !(xhci->xhc_state & XHCI_STATE_REMOVING); + if (notify) + xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); xhci->xhc_state |= XHCI_STATE_DYING; xhci_cleanup_command_queue(xhci); @@ -1324,7 +1336,7 @@ void xhci_hc_died(struct xhci_hcd *xhci) } /* inform usb core hc died if PCI remove isn't already handling it */ - if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) + if (notify) usb_hc_died(xhci_to_hcd(xhci)); } @@ -1551,7 +1563,8 @@ static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id, command->slot_id = 0; } -static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) +static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id, + u32 cmd_comp_code) { struct xhci_virt_device *virt_dev; struct xhci_slot_ctx *slot_ctx; @@ -1566,6 +1579,10 @@ static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) /* Delete default control endpoint resources */ xhci_free_device_endpoint_resources(xhci, virt_dev, true); + if (cmd_comp_code == COMP_SUCCESS) { + xhci->dcbaa->dev_context_ptrs[slot_id] = 0; + xhci->devs[slot_id] = NULL; + } } static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id, @@ -1814,7 +1831,7 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code); break; case TRB_DISABLE_SLOT: - xhci_handle_cmd_disable_slot(xhci, slot_id); + xhci_handle_cmd_disable_slot(xhci, slot_id, cmd_comp_code); break; case TRB_CONFIG_EP: if (!cmd->completion) @@ -4448,7 +4465,8 @@ static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, if ((xhci->xhc_state & XHCI_STATE_DYING) || (xhci->xhc_state & XHCI_STATE_HALTED)) { - xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n"); + xhci_dbg(xhci, "xHCI dying or halted, can't queue_command. state: 0x%x\n", + xhci->xhc_state); return -ESHUTDOWN; } diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c index 76f228e7443cb..89b3079194d7b 100644 --- a/drivers/usb/host/xhci-tegra.c +++ b/drivers/usb/host/xhci-tegra.c @@ -1363,6 +1363,7 @@ static void tegra_xhci_id_work(struct work_struct *work) tegra->otg_usb3_port = tegra_xusb_padctl_get_usb3_companion(tegra->padctl, tegra->otg_usb2_port); + pm_runtime_get_sync(tegra->dev); if (tegra->host_mode) { /* switch to host mode */ if (tegra->otg_usb3_port >= 0) { @@ -1392,6 +1393,7 @@ static void tegra_xhci_id_work(struct work_struct *work) } tegra_xhci_set_port_power(tegra, true, true); + pm_runtime_mark_last_busy(tegra->dev); } else { if (tegra->otg_usb3_port >= 0) @@ -1399,6 +1401,7 @@ static void tegra_xhci_id_work(struct work_struct *work) tegra_xhci_set_port_power(tegra, true, false); } + pm_runtime_put_autosuspend(tegra->dev); } #if IS_ENABLED(CONFIG_PM) || IS_ENABLED(CONFIG_PM_SLEEP) diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 7fb04cf6659cf..9e07e7e0a1b2a 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -142,7 +142,8 @@ int xhci_halt(struct xhci_hcd *xhci) ret = xhci_handshake(&xhci->op_regs->status, STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC); if (ret) { - xhci_warn(xhci, "Host halt failed, %d\n", ret); + if (!(xhci->xhc_state & XHCI_STATE_DYING)) + xhci_warn(xhci, "Host halt failed, %d\n", ret); return ret; } @@ -202,7 +203,8 @@ int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us) state = readl(&xhci->op_regs->status); if (state == ~(u32)0) { - xhci_warn(xhci, "Host not accessible, reset failed.\n"); + if (!(xhci->xhc_state & XHCI_STATE_DYING)) + xhci_warn(xhci, "Host not accessible, reset failed.\n"); return -ENODEV; } @@ -3830,8 +3832,7 @@ static int xhci_discover_or_reset_device(struct usb_hcd *hcd, * Obtaining a new device slot to inform the xHCI host that * the USB device has been reset. */ - ret = xhci_disable_slot(xhci, udev->slot_id); - xhci_free_virt_device(xhci, udev->slot_id); + ret = xhci_disable_and_free_slot(xhci, udev->slot_id); if (!ret) { ret = xhci_alloc_dev(hcd, udev); if (ret == 1) @@ -3986,7 +3987,7 @@ static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev) xhci_disable_slot(xhci, udev->slot_id); spin_lock_irqsave(&xhci->lock, flags); - xhci_free_virt_device(xhci, udev->slot_id); + xhci_free_virt_device(xhci, virt_dev, udev->slot_id); spin_unlock_irqrestore(&xhci->lock, flags); } @@ -4035,6 +4036,16 @@ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id) return 0; } +int xhci_disable_and_free_slot(struct xhci_hcd *xhci, u32 slot_id) +{ + struct xhci_virt_device *vdev = xhci->devs[slot_id]; + int ret; + + ret = xhci_disable_slot(xhci, slot_id); + xhci_free_virt_device(xhci, vdev, slot_id); + return ret; +} + /* * Checks if we have enough host controller resources for the default control * endpoint. @@ -4141,8 +4152,7 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev) return 1; disable_slot: - xhci_disable_slot(xhci, udev->slot_id); - xhci_free_virt_device(xhci, udev->slot_id); + xhci_disable_and_free_slot(xhci, udev->slot_id); return 0; } @@ -4278,8 +4288,7 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev, dev_warn(&udev->dev, "Device not responding to setup %s.\n", act); mutex_unlock(&xhci->mutex); - ret = xhci_disable_slot(xhci, udev->slot_id); - xhci_free_virt_device(xhci, udev->slot_id); + ret = xhci_disable_and_free_slot(xhci, udev->slot_id); if (!ret) { if (xhci_alloc_dev(hcd, udev) == 1) xhci_setup_addressable_virt_dev(xhci, udev); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 6adf2e48a6133..bbe300ca4ea30 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1681,6 +1681,7 @@ struct xhci_hcd { #define XHCI_WRITE_64_HI_LO BIT_ULL(47) #define XHCI_CDNS_SCTX_QUIRK BIT_ULL(48) #define XHCI_ETRON_HOST BIT_ULL(49) +#define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) unsigned int num_active_eps; unsigned int limit_active_eps; @@ -1830,7 +1831,7 @@ void xhci_dbg_trace(struct xhci_hcd *xhci, void (*trace)(struct va_format *), /* xHCI memory management */ void xhci_mem_cleanup(struct xhci_hcd *xhci); int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags); -void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id); +void xhci_free_virt_device(struct xhci_hcd *xhci, struct xhci_virt_device *dev, int slot_id); int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, struct usb_device *udev, gfp_t flags); int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev); void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, @@ -1904,8 +1905,6 @@ void xhci_skip_sec_intr_events(struct xhci_hcd *xhci, /* xHCI host controller glue */ typedef void (*xhci_get_quirks_t)(struct device *, struct xhci_hcd *); int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us); -int xhci_handshake_check_state(struct xhci_hcd *xhci, void __iomem *ptr, - u32 mask, u32 done, int usec, unsigned int exit_state); void xhci_quiesce(struct xhci_hcd *xhci); int xhci_halt(struct xhci_hcd *xhci); int xhci_start(struct xhci_hcd *xhci); @@ -1925,6 +1924,7 @@ void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev); int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev, struct usb_tt *tt, gfp_t mem_flags); int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id); +int xhci_disable_and_free_slot(struct xhci_hcd *xhci, u32 slot_id); int xhci_ext_cap_init(struct xhci_hcd *xhci); int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup); diff --git a/drivers/usb/misc/Kconfig b/drivers/usb/misc/Kconfig index 99b15b77dfd57..fe1b5fec1dfca 100644 --- a/drivers/usb/misc/Kconfig +++ b/drivers/usb/misc/Kconfig @@ -147,6 +147,7 @@ config USB_APPLEDISPLAY config USB_QCOM_EUD tristate "QCOM Embedded USB Debugger(EUD) Driver" depends on ARCH_QCOM || COMPILE_TEST + select QCOM_SCM select USB_ROLE_SWITCH help This module enables support for Qualcomm Technologies, Inc. diff --git a/drivers/usb/misc/apple-mfi-fastcharge.c b/drivers/usb/misc/apple-mfi-fastcharge.c index ac8695195c13c..8e852f4b8262e 100644 --- a/drivers/usb/misc/apple-mfi-fastcharge.c +++ b/drivers/usb/misc/apple-mfi-fastcharge.c @@ -44,6 +44,7 @@ MODULE_DEVICE_TABLE(usb, mfi_fc_id_table); struct mfi_device { struct usb_device *udev; struct power_supply *battery; + struct power_supply_desc battery_desc; int charge_type; }; @@ -178,6 +179,7 @@ static int mfi_fc_probe(struct usb_device *udev) { struct power_supply_config battery_cfg = {}; struct mfi_device *mfi = NULL; + char *battery_name; int err; if (!mfi_fc_match(udev)) @@ -187,23 +189,38 @@ static int mfi_fc_probe(struct usb_device *udev) if (!mfi) return -ENOMEM; + battery_name = kasprintf(GFP_KERNEL, "apple_mfi_fastcharge_%d-%d", + udev->bus->busnum, udev->devnum); + if (!battery_name) { + err = -ENOMEM; + goto err_free_mfi; + } + + mfi->battery_desc = apple_mfi_fc_desc; + mfi->battery_desc.name = battery_name; + battery_cfg.drv_data = mfi; mfi->charge_type = POWER_SUPPLY_CHARGE_TYPE_TRICKLE; mfi->battery = power_supply_register(&udev->dev, - &apple_mfi_fc_desc, + &mfi->battery_desc, &battery_cfg); if (IS_ERR(mfi->battery)) { dev_err(&udev->dev, "Can't register battery\n"); err = PTR_ERR(mfi->battery); - kfree(mfi); - return err; + goto err_free_name; } mfi->udev = usb_get_dev(udev); dev_set_drvdata(&udev->dev, mfi); return 0; + +err_free_name: + kfree(battery_name); +err_free_mfi: + kfree(mfi); + return err; } static void mfi_fc_disconnect(struct usb_device *udev) @@ -213,6 +230,7 @@ static void mfi_fc_disconnect(struct usb_device *udev) mfi = dev_get_drvdata(&udev->dev); if (mfi->battery) power_supply_unregister(mfi->battery); + kfree(mfi->battery_desc.name); dev_set_drvdata(&udev->dev, NULL); usb_put_dev(mfi->udev); kfree(mfi); diff --git a/drivers/usb/misc/qcom_eud.c b/drivers/usb/misc/qcom_eud.c index 7f371ea1248c3..cfb195a0f6ce2 100644 --- a/drivers/usb/misc/qcom_eud.c +++ b/drivers/usb/misc/qcom_eud.c @@ -15,6 +15,7 @@ #include #include #include +#include #define EUD_REG_INT1_EN_MASK 0x0024 #define EUD_REG_INT_STATUS_1 0x0044 @@ -34,7 +35,7 @@ struct eud_chip { struct device *dev; struct usb_role_switch *role_sw; void __iomem *base; - void __iomem *mode_mgr; + phys_addr_t mode_mgr; unsigned int int_status; int irq; bool enabled; @@ -43,18 +44,29 @@ struct eud_chip { static int enable_eud(struct eud_chip *priv) { + int ret; + + ret = qcom_scm_io_writel(priv->mode_mgr + EUD_REG_EUD_EN2, 1); + if (ret) + return ret; + writel(EUD_ENABLE, priv->base + EUD_REG_CSR_EUD_EN); writel(EUD_INT_VBUS | EUD_INT_SAFE_MODE, priv->base + EUD_REG_INT1_EN_MASK); - writel(1, priv->mode_mgr + EUD_REG_EUD_EN2); return usb_role_switch_set_role(priv->role_sw, USB_ROLE_DEVICE); } -static void disable_eud(struct eud_chip *priv) +static int disable_eud(struct eud_chip *priv) { + int ret; + + ret = qcom_scm_io_writel(priv->mode_mgr + EUD_REG_EUD_EN2, 0); + if (ret) + return ret; + writel(0, priv->base + EUD_REG_CSR_EUD_EN); - writel(0, priv->mode_mgr + EUD_REG_EUD_EN2); + return 0; } static ssize_t enable_show(struct device *dev, @@ -82,11 +94,12 @@ static ssize_t enable_store(struct device *dev, chip->enabled = enable; else disable_eud(chip); + } else { - disable_eud(chip); + ret = disable_eud(chip); } - return count; + return ret < 0 ? ret : count; } static DEVICE_ATTR_RW(enable); @@ -178,6 +191,7 @@ static void eud_role_switch_release(void *data) static int eud_probe(struct platform_device *pdev) { struct eud_chip *chip; + struct resource *res; int ret; chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL); @@ -200,9 +214,10 @@ static int eud_probe(struct platform_device *pdev) if (IS_ERR(chip->base)) return PTR_ERR(chip->base); - chip->mode_mgr = devm_platform_ioremap_resource(pdev, 1); - if (IS_ERR(chip->mode_mgr)) - return PTR_ERR(chip->mode_mgr); + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res) + return -ENODEV; + chip->mode_mgr = res->start; chip->irq = platform_get_irq(pdev, 0); ret = devm_request_threaded_irq(&pdev->dev, chip->irq, handle_eud_irq, diff --git a/drivers/usb/mon/mon_bin.c b/drivers/usb/mon/mon_bin.c index 4e30de4db1c0a..dabe1a5c6ae56 100644 --- a/drivers/usb/mon/mon_bin.c +++ b/drivers/usb/mon/mon_bin.c @@ -68,18 +68,20 @@ * The magic limit was calculated so that it allows the monitoring * application to pick data once in two ticks. This way, another application, * which presumably drives the bus, gets to hog CPU, yet we collect our data. - * If HZ is 100, a 480 mbit/s bus drives 614 KB every jiffy. USB has an - * enormous overhead built into the bus protocol, so we need about 1000 KB. + * + * Originally, for a 480 Mbit/s bus this required a buffer of about 1 MB. For + * modern 20 Gbps buses, this value increases to over 50 MB. The maximum + * buffer size is set to 64 MiB to accommodate this. * * This is still too much for most cases, where we just snoop a few * descriptor fetches for enumeration. So, the default is a "reasonable" - * amount for systems with HZ=250 and incomplete bus saturation. + * amount for typical, low-throughput use cases. * * XXX What about multi-megabyte URBs which take minutes to transfer? */ -#define BUFF_MAX CHUNK_ALIGN(1200*1024) -#define BUFF_DFL CHUNK_ALIGN(300*1024) -#define BUFF_MIN CHUNK_ALIGN(8*1024) +#define BUFF_MAX CHUNK_ALIGN(64*1024*1024) +#define BUFF_DFL CHUNK_ALIGN(300*1024) +#define BUFF_MIN CHUNK_ALIGN(8*1024) /* * The per-event API header (2 per URB). diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c index f175cb2c3e7bd..cfac979278654 100644 --- a/drivers/usb/musb/musb_gadget.c +++ b/drivers/usb/musb/musb_gadget.c @@ -1925,6 +1925,7 @@ static int musb_gadget_stop(struct usb_gadget *g) * gadget driver here and have everything work; * that currently misbehaves. */ + usb_gadget_set_state(g, USB_STATE_NOTATTACHED); /* Force check of devctl register for PM runtime */ pm_runtime_mark_last_busy(musb->controller); @@ -2031,6 +2032,7 @@ void musb_g_disconnect(struct musb *musb) case OTG_STATE_B_PERIPHERAL: case OTG_STATE_B_IDLE: musb_set_state(musb, OTG_STATE_B_IDLE); + usb_gadget_set_state(&musb->g, USB_STATE_NOTATTACHED); break; case OTG_STATE_B_SRP_INIT: break; diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c index b4a4c1df4e0d9..a4668c6d575dc 100644 --- a/drivers/usb/musb/omap2430.c +++ b/drivers/usb/musb/omap2430.c @@ -400,7 +400,7 @@ static int omap2430_probe(struct platform_device *pdev) ret = platform_device_add_resources(musb, pdev->resource, pdev->num_resources); if (ret) { dev_err(&pdev->dev, "failed to add resources\n"); - goto err2; + goto err_put_control_otghs; } if (populate_irqs) { @@ -413,7 +413,7 @@ static int omap2430_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) { ret = -EINVAL; - goto err2; + goto err_put_control_otghs; } musb_res[i].start = res->start; @@ -441,14 +441,14 @@ static int omap2430_probe(struct platform_device *pdev) ret = platform_device_add_resources(musb, musb_res, i); if (ret) { dev_err(&pdev->dev, "failed to add IRQ resources\n"); - goto err2; + goto err_put_control_otghs; } } ret = platform_device_add_data(musb, pdata, sizeof(*pdata)); if (ret) { dev_err(&pdev->dev, "failed to add platform_data\n"); - goto err2; + goto err_put_control_otghs; } pm_runtime_enable(glue->dev); @@ -463,7 +463,9 @@ static int omap2430_probe(struct platform_device *pdev) err3: pm_runtime_disable(glue->dev); - +err_put_control_otghs: + if (!IS_ERR(glue->control_otghs)) + put_device(glue->control_otghs); err2: platform_device_put(musb); @@ -477,6 +479,8 @@ static void omap2430_remove(struct platform_device *pdev) platform_device_unregister(glue->musb); pm_runtime_disable(glue->dev); + if (!IS_ERR(glue->control_otghs)) + put_device(glue->control_otghs); } #ifdef CONFIG_PM diff --git a/drivers/usb/phy/phy-twl6030-usb.c b/drivers/usb/phy/phy-twl6030-usb.c index c3ce6b1054f1c..292c32ccae050 100644 --- a/drivers/usb/phy/phy-twl6030-usb.c +++ b/drivers/usb/phy/phy-twl6030-usb.c @@ -328,9 +328,8 @@ static int twl6030_set_vbus(struct phy_companion *comparator, bool enabled) static int twl6030_usb_probe(struct platform_device *pdev) { - u32 ret; struct twl6030_usb *twl; - int status, err; + int status, err, ret; struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c index 205820544f6f9..b720899725e53 100644 --- a/drivers/usb/renesas_usbhs/common.c +++ b/drivers/usb/renesas_usbhs/common.c @@ -674,10 +674,29 @@ static int usbhs_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&priv->notify_hotplug_work, usbhsc_notify_hotplug); spin_lock_init(usbhs_priv_to_lock(priv)); + /* + * Acquire clocks and enable power management (PM) early in the + * probe process, as the driver accesses registers during + * initialization. Ensure the device is active before proceeding. + */ + pm_runtime_enable(dev); + + ret = usbhsc_clk_get(dev, priv); + if (ret) + goto probe_pm_disable; + + ret = pm_runtime_resume_and_get(dev); + if (ret) + goto probe_clk_put; + + ret = usbhsc_clk_prepare_enable(priv); + if (ret) + goto probe_pm_put; + /* call pipe and module init */ ret = usbhs_pipe_probe(priv); if (ret < 0) - return ret; + goto probe_clk_dis_unprepare; ret = usbhs_fifo_probe(priv); if (ret < 0) @@ -694,10 +713,6 @@ static int usbhs_probe(struct platform_device *pdev) if (ret) goto probe_fail_rst; - ret = usbhsc_clk_get(dev, priv); - if (ret) - goto probe_fail_clks; - /* * deviece reset here because * USB device might be used in boot loader. @@ -710,7 +725,7 @@ static int usbhs_probe(struct platform_device *pdev) if (ret) { dev_warn(dev, "USB function not selected (GPIO)\n"); ret = -ENOTSUPP; - goto probe_end_mod_exit; + goto probe_assert_rest; } } @@ -724,14 +739,19 @@ static int usbhs_probe(struct platform_device *pdev) ret = usbhs_platform_call(priv, hardware_init, pdev); if (ret < 0) { dev_err(dev, "platform init failed.\n"); - goto probe_end_mod_exit; + goto probe_assert_rest; } /* reset phy for connection */ usbhs_platform_call(priv, phy_reset, pdev); - /* power control */ - pm_runtime_enable(dev); + /* + * Disable the clocks that were enabled earlier in the probe path, + * and let the driver handle the clocks beyond this point. + */ + usbhsc_clk_disable_unprepare(priv); + pm_runtime_put(dev); + if (!usbhs_get_dparam(priv, runtime_pwctrl)) { usbhsc_power_ctrl(priv, 1); usbhs_mod_autonomy_mode(priv); @@ -748,9 +768,7 @@ static int usbhs_probe(struct platform_device *pdev) return ret; -probe_end_mod_exit: - usbhsc_clk_put(priv); -probe_fail_clks: +probe_assert_rest: reset_control_assert(priv->rsts); probe_fail_rst: usbhs_mod_remove(priv); @@ -758,6 +776,14 @@ static int usbhs_probe(struct platform_device *pdev) usbhs_fifo_remove(priv); probe_end_pipe_exit: usbhs_pipe_remove(priv); +probe_clk_dis_unprepare: + usbhsc_clk_disable_unprepare(priv); +probe_pm_put: + pm_runtime_put(dev); +probe_clk_put: + usbhsc_clk_put(priv); +probe_pm_disable: + pm_runtime_disable(dev); dev_info(dev, "probe failed (%d)\n", ret); diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index b583b31ea5e72..8906c9dc348ed 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -803,6 +803,8 @@ static const struct usb_device_id id_table_combined[] = { .driver_info = (kernel_ulong_t)&ftdi_NDI_device_quirk }, { USB_DEVICE(FTDI_VID, FTDI_NDI_AURORA_SCU_PID), .driver_info = (kernel_ulong_t)&ftdi_NDI_device_quirk }, + { USB_DEVICE(FTDI_NDI_VID, FTDI_NDI_EMGUIDE_GEMINI_PID), + .driver_info = (kernel_ulong_t)&ftdi_NDI_device_quirk }, { USB_DEVICE(TELLDUS_VID, TELLDUS_TELLSTICK_PID) }, { USB_DEVICE(NOVITUS_VID, NOVITUS_BONO_E_PID) }, { USB_DEVICE(FTDI_VID, RTSYSTEMS_USB_VX8_PID) }, diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h index 9acb6f8373276..4cc1fae8acb97 100644 --- a/drivers/usb/serial/ftdi_sio_ids.h +++ b/drivers/usb/serial/ftdi_sio_ids.h @@ -204,6 +204,9 @@ #define FTDI_NDI_FUTURE_3_PID 0xDA73 /* NDI future device #3 */ #define FTDI_NDI_AURORA_SCU_PID 0xDA74 /* NDI Aurora SCU */ +#define FTDI_NDI_VID 0x23F2 +#define FTDI_NDI_EMGUIDE_GEMINI_PID 0x0003 /* NDI Emguide Gemini */ + /* * ChamSys Limited (www.chamsys.co.uk) USB wing/interface product IDs */ diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 5d66951160989..e9d0d8591a01f 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -273,6 +273,7 @@ static void option_instat_callback(struct urb *urb); #define QUECTEL_PRODUCT_EM05CN 0x0312 #define QUECTEL_PRODUCT_EM05G_GR 0x0313 #define QUECTEL_PRODUCT_EM05G_RS 0x0314 +#define QUECTEL_PRODUCT_RG255C 0x0316 #define QUECTEL_PRODUCT_EM12 0x0512 #define QUECTEL_PRODUCT_RM500Q 0x0800 #define QUECTEL_PRODUCT_RM520N 0x0801 @@ -617,6 +618,7 @@ static void option_instat_callback(struct urb *urb); #define UNISOC_VENDOR_ID 0x1782 /* TOZED LT70-C based on UNISOC SL8563 uses UNISOC's vendor ID */ #define TOZED_PRODUCT_LT70C 0x4055 +#define UNISOC_PRODUCT_UIS7720 0x4064 /* Luat Air72*U series based on UNISOC UIS8910 uses UNISOC's vendor ID */ #define LUAT_PRODUCT_AIR720U 0x4e00 @@ -1270,6 +1272,9 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500K, 0xff, 0x00, 0x00) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG650V, 0xff, 0xff, 0x30) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG650V, 0xff, 0, 0) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0xff, 0x30) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0, 0) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0xff, 0x40) }, { USB_DEVICE(CMOTECH_VENDOR_ID, CMOTECH_PRODUCT_6001) }, { USB_DEVICE(CMOTECH_VENDOR_ID, CMOTECH_PRODUCT_CMU_300) }, @@ -1322,7 +1327,18 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(0) | RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1033, 0xff), /* Telit LE910C1-EUX (ECM) */ .driver_info = NCTRL(0) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1034, 0xff), /* Telit LE910C4-WWX (rmnet) */ + .driver_info = RSVD(2) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1035, 0xff) }, /* Telit LE910C4-WWX (ECM) */ + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1036, 0xff) }, /* Telit LE910C4-WWX */ + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1037, 0xff), /* Telit LE910C4-WWX (rmnet) */ + .driver_info = NCTRL(0) | NCTRL(1) | RSVD(4) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1038, 0xff), /* Telit LE910C4-WWX (rmnet) */ + .driver_info = NCTRL(0) | RSVD(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x103b, 0xff), /* Telit LE910C4-WWX */ + .driver_info = NCTRL(0) | NCTRL(1) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x103c, 0xff), /* Telit LE910C4-WWX */ + .driver_info = NCTRL(0) }, { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG0), .driver_info = RSVD(0) | RSVD(1) | NCTRL(2) | RSVD(3) }, { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG1), @@ -1369,6 +1385,12 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(0) | RSVD(1) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1075, 0xff), /* Telit FN990A (PCIe) */ .driver_info = RSVD(0) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1077, 0xff), /* Telit FN990A (rmnet + audio) */ + .driver_info = NCTRL(0) | RSVD(1) | RSVD(2) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1078, 0xff), /* Telit FN990A (MBIM + audio) */ + .driver_info = NCTRL(0) | RSVD(1) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1079, 0xff), /* Telit FN990A (RNDIS + audio) */ + .driver_info = NCTRL(2) | RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1080, 0xff), /* Telit FE990A (rmnet) */ .driver_info = NCTRL(0) | RSVD(1) | RSVD(2) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1081, 0xff), /* Telit FE990A (MBIM) */ @@ -1381,10 +1403,14 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(0) | NCTRL(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a2, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a3, 0xff), /* Telit FN920C04 (ECM) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a4, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a7, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a8, 0xff), /* Telit FN920C04 (ECM) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a9, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(2) | RSVD(3) | RSVD(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10aa, 0xff), /* Telit FN920C04 (MBIM) */ @@ -1415,6 +1441,9 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d0, 0xff, 0xff, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d0, 0xff, 0xff, 0x60) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10c7, 0xff, 0xff, 0x30), /* Telit FE910C04 (ECM) */ + .driver_info = NCTRL(4) }, + { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10c7, 0xff, 0xff, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d1, 0xff, 0xff, 0x30), /* Telit FN990B (MBIM) */ .driver_info = NCTRL(6) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10d1, 0xff, 0xff, 0x40) }, @@ -2094,6 +2123,12 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9003, 0xff) }, /* Simcom SIM7500/SIM7600 MBIM mode */ { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9011, 0xff), /* Simcom SIM7500/SIM7600 RNDIS mode */ .driver_info = RSVD(7) }, + { USB_DEVICE(0x1e0e, 0x9071), /* Simcom SIM8230 RMNET mode */ + .driver_info = RSVD(3) | RSVD(4) }, + { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9078, 0xff), /* Simcom SIM8230 ECM mode */ + .driver_info = RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x907b, 0xff), /* Simcom SIM8230 RNDIS mode */ + .driver_info = RSVD(5) }, { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9205, 0xff) }, /* Simcom SIM7070/SIM7080/SIM7090 AT+ECM mode */ { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9206, 0xff) }, /* Simcom SIM7070/SIM7080/SIM7090 AT-only mode */ { USB_DEVICE(ALCATEL_VENDOR_ID, ALCATEL_PRODUCT_X060S_X200), @@ -2343,6 +2378,10 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe145, 0xff), /* Foxconn T99W651 RNDIS */ .driver_info = RSVD(5) | RSVD(6) }, + { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe15f, 0xff), /* Foxconn T99W709 */ + .driver_info = RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe167, 0xff), /* Foxconn T99W640 MBIM */ + .driver_info = RSVD(3) }, { USB_DEVICE(0x1508, 0x1001), /* Fibocom NL668 (IOT version) */ .driver_info = RSVD(4) | RSVD(5) | RSVD(6) }, { USB_DEVICE(0x1782, 0x4d10) }, /* Fibocom L610 (AT mode) */ @@ -2436,6 +2475,7 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(SIERRA_VENDOR_ID, SIERRA_PRODUCT_EM9291, 0xff, 0xff, 0x30) }, { USB_DEVICE_AND_INTERFACE_INFO(SIERRA_VENDOR_ID, SIERRA_PRODUCT_EM9291, 0xff, 0xff, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(UNISOC_VENDOR_ID, TOZED_PRODUCT_LT70C, 0xff, 0, 0) }, + { USB_DEVICE_AND_INTERFACE_INFO(UNISOC_VENDOR_ID, UNISOC_PRODUCT_UIS7720, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(UNISOC_VENDOR_ID, LUAT_PRODUCT_AIR720U, 0xff, 0, 0) }, { USB_DEVICE_INTERFACE_CLASS(0x1bbb, 0x0530, 0xff), /* TCL IK512 MBIM */ .driver_info = NCTRL(1) }, diff --git a/drivers/usb/serial/pl2303.c b/drivers/usb/serial/pl2303.c index 8e327fcb222f7..1ba2287d4d456 100644 --- a/drivers/usb/serial/pl2303.c +++ b/drivers/usb/serial/pl2303.c @@ -457,6 +457,8 @@ static int pl2303_detect_type(struct usb_serial *serial) case 0x605: case 0x700: /* GR */ case 0x705: + case 0x905: /* GT-2AB */ + case 0x1005: /* GC-Q20 */ return TYPE_HXN; } break; diff --git a/drivers/usb/storage/realtek_cr.c b/drivers/usb/storage/realtek_cr.c index 0c423916d7bfa..a026c6cb6e684 100644 --- a/drivers/usb/storage/realtek_cr.c +++ b/drivers/usb/storage/realtek_cr.c @@ -252,7 +252,7 @@ static int rts51x_bulk_transport(struct us_data *us, u8 lun, return USB_STOR_TRANSPORT_ERROR; } - residue = bcs->Residue; + residue = le32_to_cpu(bcs->Residue); if (bcs->Tag != us->tag) return USB_STOR_TRANSPORT_ERROR; diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index 54f0b1c83317c..dfa5276a5a43e 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -934,6 +934,13 @@ UNUSUAL_DEV( 0x05e3, 0x0723, 0x9451, 0x9451, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_SANE_SENSE ), +/* Added by Maël GUERIN */ +UNUSUAL_DEV( 0x0603, 0x8611, 0x0000, 0xffff, + "Novatek", + "NTK96550-based camera", + USB_SC_SCSI, USB_PR_BULK, NULL, + US_FL_BULK_IGNORE_TAG ), + /* * Reported by Hanno Boeck * Taken from the Lycoris Kernel @@ -1494,6 +1501,28 @@ UNUSUAL_DEV( 0x0bc2, 0x3332, 0x0000, 0x9999, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_NO_WP_DETECT ), +/* + * Reported by Zenm Chen + * Ignore driver CD mode, otherwise usb_modeswitch may fail to switch + * the device into Wi-Fi mode. + */ +UNUSUAL_DEV( 0x0bda, 0x1a2b, 0x0000, 0xffff, + "Realtek", + "DISK", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_DEVICE ), + +/* + * Reported by Zenm Chen + * Ignore driver CD mode, otherwise usb_modeswitch may fail to switch + * the device into Wi-Fi mode. + */ +UNUSUAL_DEV( 0x0bda, 0xa192, 0x0000, 0xffff, + "Realtek", + "DISK", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_DEVICE ), + UNUSUAL_DEV( 0x0d49, 0x7310, 0x0000, 0x9999, "Maxtor", "USB to SATA", diff --git a/drivers/usb/storage/unusual_uas.h b/drivers/usb/storage/unusual_uas.h index d460d71b42578..1477e31d77632 100644 --- a/drivers/usb/storage/unusual_uas.h +++ b/drivers/usb/storage/unusual_uas.h @@ -52,6 +52,13 @@ UNUSUAL_DEV(0x059f, 0x1061, 0x0000, 0x9999, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_NO_REPORT_OPCODES | US_FL_NO_SAME), +/* Reported-by: Zhihong Zhou */ +UNUSUAL_DEV(0x0781, 0x55e8, 0x0000, 0x9999, + "SanDisk", + "", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_UAS), + /* Reported-by: Hongling Zeng */ UNUSUAL_DEV(0x090c, 0x2000, 0x0000, 0x9999, "Hiksemi", diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c index ccff838ab89e1..215bd0507ee45 100644 --- a/drivers/usb/typec/altmodes/displayport.c +++ b/drivers/usb/typec/altmodes/displayport.c @@ -61,6 +61,13 @@ struct dp_altmode { enum dp_state state; bool hpd; bool pending_hpd; + u32 irq_hpd_count; + /* + * hpd is mandatory for irq_hpd assertion, so irq_hpd also needs its own pending flag if + * both hpd and irq_hpd are asserted in the first Status Update before the pin assignment + * is configured. + */ + bool pending_irq_hpd; struct mutex lock; /* device lock */ struct work_struct work; @@ -135,6 +142,7 @@ static int dp_altmode_status_update(struct dp_altmode *dp) { bool configured = !!DP_CONF_GET_PIN_ASSIGN(dp->data.conf); bool hpd = !!(dp->data.status & DP_STATUS_HPD_STATE); + bool irq_hpd = !!(dp->data.status & DP_STATUS_IRQ_HPD); u8 con = DP_STATUS_CONNECTION(dp->data.status); int ret = 0; @@ -151,6 +159,8 @@ static int dp_altmode_status_update(struct dp_altmode *dp) dp->hpd = hpd; dp->pending_hpd = true; } + if (dp->hpd && dp->pending_hpd && irq_hpd) + dp->pending_irq_hpd = true; } } else { if (dp->hpd != hpd) { @@ -158,6 +168,10 @@ static int dp_altmode_status_update(struct dp_altmode *dp) dp->hpd = hpd; sysfs_notify(&dp->alt->dev.kobj, "displayport", "hpd"); } + if (hpd && irq_hpd) { + dp->irq_hpd_count++; + sysfs_notify(&dp->alt->dev.kobj, "displayport", "irq_hpd"); + } } return ret; @@ -176,6 +190,11 @@ static int dp_altmode_configured(struct dp_altmode *dp) drm_connector_oob_hotplug_event(dp->connector_fwnode); sysfs_notify(&dp->alt->dev.kobj, "displayport", "hpd"); dp->pending_hpd = false; + if (dp->pending_irq_hpd) { + dp->irq_hpd_count++; + sysfs_notify(&dp->alt->dev.kobj, "displayport", "irq_hpd"); + dp->pending_irq_hpd = false; + } } return dp_altmode_notify(dp); @@ -323,6 +342,9 @@ static int dp_altmode_vdm(struct typec_altmode *alt, break; case CMDT_RSP_NAK: switch (cmd) { + case DP_CMD_STATUS_UPDATE: + dp->state = DP_STATE_EXIT; + break; case DP_CMD_CONFIGURE: dp->data.conf = 0; ret = dp_altmode_configured(dp); @@ -524,7 +546,7 @@ static ssize_t pin_assignment_show(struct device *dev, assignments = get_current_pin_assignments(dp); - for (i = 0; assignments; assignments >>= 1, i++) { + for (i = 0; assignments && i < DP_PIN_ASSIGN_MAX; assignments >>= 1, i++) { if (assignments & 1) { if (i == cur) len += sprintf(buf + len, "[%s] ", @@ -554,10 +576,19 @@ static ssize_t hpd_show(struct device *dev, struct device_attribute *attr, char } static DEVICE_ATTR_RO(hpd); +static ssize_t irq_hpd_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct dp_altmode *dp = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", dp->irq_hpd_count); +} +static DEVICE_ATTR_RO(irq_hpd); + static struct attribute *displayport_attrs[] = { &dev_attr_configuration.attr, &dev_attr_pin_assignment.attr, &dev_attr_hpd.attr, + &dev_attr_irq_hpd.attr, NULL }; diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 80dd91938d960..a5b2a6f9c5795 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -214,7 +214,7 @@ int typec_switch_set(struct typec_switch *sw, sw_dev = sw->sw_devs[i]; ret = sw_dev->set(sw_dev, orientation); - if (ret) + if (ret && ret != -EOPNOTSUPP) return ret; } @@ -378,7 +378,7 @@ int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) mux_dev = mux->mux_devs[i]; ret = mux_dev->set(mux_dev, state); - if (ret) + if (ret && ret != -EOPNOTSUPP) return ret; } diff --git a/drivers/usb/typec/mux/intel_pmc_mux.c b/drivers/usb/typec/mux/intel_pmc_mux.c index 60ed1f809130d..a174ff7a9abd3 100644 --- a/drivers/usb/typec/mux/intel_pmc_mux.c +++ b/drivers/usb/typec/mux/intel_pmc_mux.c @@ -730,7 +730,7 @@ static int pmc_usb_probe(struct platform_device *pdev) pmc->ipc = devm_intel_scu_ipc_dev_get(&pdev->dev); if (!pmc->ipc) - return -ENODEV; + return -EPROBE_DEFER; pmc->dev = &pdev->dev; diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index bc21006e979c6..03749a392fdbd 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -103,6 +103,7 @@ struct fusb302_chip { bool vconn_on; bool vbus_on; bool charge_on; + bool pd_rx_on; bool vbus_present; enum typec_cc_polarity cc_polarity; enum typec_cc_status cc1; @@ -841,6 +842,11 @@ static int tcpm_set_pd_rx(struct tcpc_dev *dev, bool on) int ret = 0; mutex_lock(&chip->lock); + if (chip->pd_rx_on == on) { + fusb302_log(chip, "pd is already %s", str_on_off(on)); + goto done; + } + ret = fusb302_pd_rx_flush(chip); if (ret < 0) { fusb302_log(chip, "cannot flush pd rx buffer, ret=%d", ret); @@ -863,6 +869,8 @@ static int tcpm_set_pd_rx(struct tcpc_dev *dev, bool on) on ? "on" : "off", ret); goto done; } + + chip->pd_rx_on = on; fusb302_log(chip, "pd := %s", on ? "on" : "off"); done: mutex_unlock(&chip->lock); diff --git a/drivers/usb/typec/tcpm/maxim_contaminant.c b/drivers/usb/typec/tcpm/maxim_contaminant.c index 60f90272fed31..e37bf26cca805 100644 --- a/drivers/usb/typec/tcpm/maxim_contaminant.c +++ b/drivers/usb/typec/tcpm/maxim_contaminant.c @@ -5,6 +5,7 @@ * USB-C module to reduce wakeups due to contaminants. */ +#include #include #include #include @@ -189,6 +190,11 @@ static int max_contaminant_read_comparators(struct max_tcpci_chip *chip, u8 *ven if (ret < 0) return ret; + /* Disable low power mode */ + ret = regmap_update_bits(regmap, TCPC_VENDOR_CC_CTRL2, CCLPMODESEL_MASK, + FIELD_PREP(CCLPMODESEL_MASK, + LOW_POWER_MODE_DISABLE)); + /* Sleep to allow comparators settle */ usleep_range(5000, 6000); ret = regmap_update_bits(regmap, TCPC_TCPC_CTRL, TCPC_TCPC_CTRL_ORIENTATION, PLUG_ORNT_CC1); @@ -322,6 +328,34 @@ static int max_contaminant_enable_dry_detection(struct max_tcpci_chip *chip) return 0; } +static int max_contaminant_enable_toggling(struct max_tcpci_chip *chip) +{ + struct regmap *regmap = chip->data.regmap; + int ret; + + /* Disable dry detection if enabled. */ + ret = regmap_update_bits(regmap, TCPC_VENDOR_CC_CTRL2, CCLPMODESEL_MASK, + ULTRA_LOW_POWER_MODE); + if (ret) + return ret; + + ret = regmap_update_bits(regmap, TCPC_VENDOR_CC_CTRL1, CCCONNDRY, 0); + if (ret) + return ret; + + ret = max_tcpci_write8(chip, TCPC_ROLE_CTRL, TCPC_ROLE_CTRL_DRP | 0xA); + if (ret) + return ret; + + ret = regmap_update_bits(regmap, TCPC_TCPC_CTRL, + TCPC_TCPC_CTRL_EN_LK4CONN_ALRT, + TCPC_TCPC_CTRL_EN_LK4CONN_ALRT); + if (ret) + return ret; + + return max_tcpci_write8(chip, TCPC_COMMAND, TCPC_CMD_LOOK4CONNECTION); +} + bool max_contaminant_is_contaminant(struct max_tcpci_chip *chip, bool disconnect_while_debounce) { u8 cc_status, pwr_cntl; @@ -335,6 +369,12 @@ bool max_contaminant_is_contaminant(struct max_tcpci_chip *chip, bool disconnect if (ret < 0) return false; + if (cc_status & TCPC_CC_STATUS_TOGGLING) { + if (chip->contaminant_state == DETECTED) + return true; + return false; + } + if (chip->contaminant_state == NOT_DETECTED || chip->contaminant_state == SINK) { if (!disconnect_while_debounce) msleep(100); @@ -367,6 +407,12 @@ bool max_contaminant_is_contaminant(struct max_tcpci_chip *chip, bool disconnect max_contaminant_enable_dry_detection(chip); return true; } + + ret = max_contaminant_enable_toggling(chip); + if (ret) + dev_err(chip->dev, + "Failed to enable toggling, ret=%d", + ret); } return false; } else if (chip->contaminant_state == DETECTED) { @@ -375,6 +421,14 @@ bool max_contaminant_is_contaminant(struct max_tcpci_chip *chip, bool disconnect if (chip->contaminant_state == DETECTED) { max_contaminant_enable_dry_detection(chip); return true; + } else { + ret = max_contaminant_enable_toggling(chip); + if (ret) { + dev_err(chip->dev, + "Failed to enable toggling, ret=%d", + ret); + return true; + } } } } diff --git a/drivers/usb/typec/tcpm/tcpci_maxim.h b/drivers/usb/typec/tcpm/tcpci_maxim.h index 2c1c4d161b0dc..861801cc456ff 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim.h +++ b/drivers/usb/typec/tcpm/tcpci_maxim.h @@ -21,6 +21,7 @@ #define CCOVPDIS BIT(6) #define SBURPCTRL BIT(5) #define CCLPMODESEL_MASK GENMASK(4, 3) +#define LOW_POWER_MODE_DISABLE 0 #define ULTRA_LOW_POWER_MODE BIT(3) #define CCRPCTRL_MASK GENMASK(2, 0) #define UA_1_SRC 1 diff --git a/drivers/usb/typec/tcpm/tcpci_maxim_core.c b/drivers/usb/typec/tcpm/tcpci_maxim_core.c index 9454b12a073c9..e460b956ab712 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim_core.c +++ b/drivers/usb/typec/tcpm/tcpci_maxim_core.c @@ -145,7 +145,8 @@ static void process_rx(struct max_tcpci_chip *chip, u16 status) return; } - if (count > sizeof(struct pd_message) || count + 1 > TCPC_RECEIVE_BUFFER_LEN) { + if (count > sizeof(struct pd_message) + 1 || + count + 1 > TCPC_RECEIVE_BUFFER_LEN) { dev_err(chip->dev, "Invalid TCPC_RX_BYTE_CNT %d\n", count); return; } diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index ad9e7da9e942b..2e6a1e2013a8f 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1093,7 +1093,7 @@ static int tcpm_set_attached_state(struct tcpm_port *port, bool attached) port->data_role); } -static int tcpm_set_roles(struct tcpm_port *port, bool attached, +static int tcpm_set_roles(struct tcpm_port *port, bool attached, int state, enum typec_role role, enum typec_data_role data) { enum typec_orientation orientation; @@ -1130,7 +1130,7 @@ static int tcpm_set_roles(struct tcpm_port *port, bool attached, } } - ret = tcpm_mux_set(port, TYPEC_STATE_USB, usb_role, orientation); + ret = tcpm_mux_set(port, state, usb_role, orientation); if (ret < 0) return ret; @@ -3765,16 +3765,6 @@ static int tcpm_src_attach(struct tcpm_port *port) tcpm_enable_auto_vbus_discharge(port, true); - ret = tcpm_set_roles(port, true, TYPEC_SOURCE, tcpm_data_role_for_source(port)); - if (ret < 0) - return ret; - - if (port->pd_supported) { - ret = port->tcpc->set_pd_rx(port->tcpc, true); - if (ret < 0) - goto out_disable_mux; - } - /* * USB Type-C specification, version 1.2, * chapter 4.5.2.2.8.1 (Attached.SRC Requirements) @@ -3784,13 +3774,24 @@ static int tcpm_src_attach(struct tcpm_port *port) (polarity == TYPEC_POLARITY_CC2 && port->cc1 == TYPEC_CC_RA)) { ret = tcpm_set_vconn(port, true); if (ret < 0) - goto out_disable_pd; + return ret; } ret = tcpm_set_vbus(port, true); if (ret < 0) goto out_disable_vconn; + ret = tcpm_set_roles(port, true, TYPEC_STATE_USB, TYPEC_SOURCE, + tcpm_data_role_for_source(port)); + if (ret < 0) + goto out_disable_vbus; + + if (port->pd_supported) { + ret = port->tcpc->set_pd_rx(port->tcpc, true); + if (ret < 0) + goto out_disable_mux; + } + port->pd_capable = false; port->partner = NULL; @@ -3800,14 +3801,14 @@ static int tcpm_src_attach(struct tcpm_port *port) return 0; -out_disable_vconn: - tcpm_set_vconn(port, false); -out_disable_pd: - if (port->pd_supported) - port->tcpc->set_pd_rx(port->tcpc, false); out_disable_mux: tcpm_mux_set(port, TYPEC_STATE_SAFE, USB_ROLE_NONE, TYPEC_ORIENTATION_NONE); +out_disable_vbus: + tcpm_set_vbus(port, false); +out_disable_vconn: + tcpm_set_vconn(port, false); + return ret; } @@ -3923,7 +3924,8 @@ static int tcpm_snk_attach(struct tcpm_port *port) tcpm_enable_auto_vbus_discharge(port, true); - ret = tcpm_set_roles(port, true, TYPEC_SINK, tcpm_data_role_for_sink(port)); + ret = tcpm_set_roles(port, true, TYPEC_STATE_USB, + TYPEC_SINK, tcpm_data_role_for_sink(port)); if (ret < 0) return ret; @@ -3945,12 +3947,24 @@ static void tcpm_snk_detach(struct tcpm_port *port) static int tcpm_acc_attach(struct tcpm_port *port) { int ret; + enum typec_role role; + enum typec_data_role data; + int state = TYPEC_STATE_USB; if (port->attached) return 0; - ret = tcpm_set_roles(port, true, TYPEC_SOURCE, - tcpm_data_role_for_source(port)); + role = tcpm_port_is_sink(port) ? TYPEC_SINK : TYPEC_SOURCE; + data = tcpm_port_is_sink(port) ? tcpm_data_role_for_sink(port) + : tcpm_data_role_for_source(port); + + if (tcpm_port_is_audio(port)) + state = TYPEC_MODE_AUDIO; + + if (tcpm_port_is_debug(port)) + state = TYPEC_MODE_DEBUG; + + ret = tcpm_set_roles(port, true, state, role, data); if (ret < 0) return ret; @@ -4664,7 +4678,7 @@ static void run_state_machine(struct tcpm_port *port) */ tcpm_set_vconn(port, false); tcpm_set_vbus(port, false); - tcpm_set_roles(port, port->self_powered, TYPEC_SOURCE, + tcpm_set_roles(port, port->self_powered, TYPEC_STATE_USB, TYPEC_SOURCE, tcpm_data_role_for_source(port)); /* * If tcpc fails to notify vbus off, TCPM will wait for PD_T_SAFE_0V + @@ -4696,7 +4710,7 @@ static void run_state_machine(struct tcpm_port *port) tcpm_set_vconn(port, false); if (port->pd_capable) tcpm_set_charge(port, false); - tcpm_set_roles(port, port->self_powered, TYPEC_SINK, + tcpm_set_roles(port, port->self_powered, TYPEC_STATE_USB, TYPEC_SINK, tcpm_data_role_for_sink(port)); /* * VBUS may or may not toggle, depending on the adapter. @@ -4802,10 +4816,10 @@ static void run_state_machine(struct tcpm_port *port) case DR_SWAP_CHANGE_DR: tcpm_unregister_altmodes(port); if (port->data_role == TYPEC_HOST) - tcpm_set_roles(port, true, port->pwr_role, + tcpm_set_roles(port, true, TYPEC_STATE_USB, port->pwr_role, TYPEC_DEVICE); else - tcpm_set_roles(port, true, port->pwr_role, + tcpm_set_roles(port, true, TYPEC_STATE_USB, port->pwr_role, TYPEC_HOST); tcpm_ams_finish(port); tcpm_set_state(port, ready_state(port), 0); @@ -5258,7 +5272,7 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, case SNK_TRY_WAIT_DEBOUNCE: if (!tcpm_port_is_sink(port)) { port->max_wait = 0; - tcpm_set_state(port, SRC_TRYWAIT, 0); + tcpm_set_state(port, SRC_TRYWAIT, PD_T_PD_DEBOUNCE); } break; case SRC_TRY_WAIT: @@ -7075,9 +7089,9 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc) port->partner_desc.identity = &port->partner_ident; - port->role_sw = usb_role_switch_get(port->dev); + port->role_sw = fwnode_usb_role_switch_get(tcpc->fwnode); if (!port->role_sw) - port->role_sw = fwnode_usb_role_switch_get(tcpc->fwnode); + port->role_sw = usb_role_switch_get(port->dev); if (IS_ERR(port->role_sw)) { err = PTR_ERR(port->role_sw); goto out_destroy_wq; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 01db27cbf1d10..3559781d01a38 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -488,24 +488,23 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) if (!event) goto err_unlock; + tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); + if (!tps6598x_read_status(tps, &status)) - goto err_clear_ints; + goto err_unlock; if (event & APPLE_CD_REG_INT_POWER_STATUS_UPDATE) if (!tps6598x_read_power_status(tps)) - goto err_clear_ints; + goto err_unlock; if (event & APPLE_CD_REG_INT_DATA_STATUS_UPDATE) if (!tps6598x_read_data_status(tps)) - goto err_clear_ints; + goto err_unlock; /* Handle plug insert or removal */ if (event & APPLE_CD_REG_INT_PLUG_EVENT) tps6598x_handle_plug_event(tps, status); -err_clear_ints: - tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); - err_unlock: mutex_unlock(&tps->lock); @@ -552,25 +551,24 @@ static irqreturn_t tps6598x_interrupt(int irq, void *data) if (!(event1[0] | event1[1] | event2[0] | event2[1])) goto err_unlock; + tps6598x_block_write(tps, TPS_REG_INT_CLEAR1, event1, intev_len); + tps6598x_block_write(tps, TPS_REG_INT_CLEAR2, event2, intev_len); + if (!tps6598x_read_status(tps, &status)) - goto err_clear_ints; + goto err_unlock; if ((event1[0] | event2[0]) & TPS_REG_INT_POWER_STATUS_UPDATE) if (!tps6598x_read_power_status(tps)) - goto err_clear_ints; + goto err_unlock; if ((event1[0] | event2[0]) & TPS_REG_INT_DATA_STATUS_UPDATE) if (!tps6598x_read_data_status(tps)) - goto err_clear_ints; + goto err_unlock; /* Handle plug insert or removal */ if ((event1[0] | event2[0]) & TPS_REG_INT_PLUG_EVENT) tps6598x_handle_plug_event(tps, status); -err_clear_ints: - tps6598x_block_write(tps, TPS_REG_INT_CLEAR1, event1, intev_len); - tps6598x_block_write(tps, TPS_REG_INT_CLEAR2, event2, intev_len); - err_unlock: mutex_unlock(&tps->lock); diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c index 2431febc46151..e3b5fa3b5f955 100644 --- a/drivers/usb/typec/ucsi/displayport.c +++ b/drivers/usb/typec/ucsi/displayport.c @@ -54,7 +54,8 @@ static int ucsi_displayport_enter(struct typec_altmode *alt, u32 *vdo) u8 cur = 0; int ret; - mutex_lock(&dp->con->lock); + if (!ucsi_con_mutex_lock(dp->con)) + return -ENOTCONN; if (!dp->override && dp->initialized) { const struct typec_altmode *p = typec_altmode_get_partner(alt); @@ -100,7 +101,7 @@ static int ucsi_displayport_enter(struct typec_altmode *alt, u32 *vdo) schedule_work(&dp->work); ret = 0; err_unlock: - mutex_unlock(&dp->con->lock); + ucsi_con_mutex_unlock(dp->con); return ret; } @@ -112,7 +113,8 @@ static int ucsi_displayport_exit(struct typec_altmode *alt) u64 command; int ret = 0; - mutex_lock(&dp->con->lock); + if (!ucsi_con_mutex_lock(dp->con)) + return -ENOTCONN; if (!dp->override) { const struct typec_altmode *p = typec_altmode_get_partner(alt); @@ -144,7 +146,7 @@ static int ucsi_displayport_exit(struct typec_altmode *alt) schedule_work(&dp->work); out_unlock: - mutex_unlock(&dp->con->lock); + ucsi_con_mutex_unlock(dp->con); return ret; } @@ -202,20 +204,21 @@ static int ucsi_displayport_vdm(struct typec_altmode *alt, int cmd = PD_VDO_CMD(header); int svdm_version; - mutex_lock(&dp->con->lock); + if (!ucsi_con_mutex_lock(dp->con)) + return -ENOTCONN; if (!dp->override && dp->initialized) { const struct typec_altmode *p = typec_altmode_get_partner(alt); dev_warn(&p->dev, "firmware doesn't support alternate mode overriding\n"); - mutex_unlock(&dp->con->lock); + ucsi_con_mutex_unlock(dp->con); return -EOPNOTSUPP; } svdm_version = typec_altmode_get_svdm_version(alt); if (svdm_version < 0) { - mutex_unlock(&dp->con->lock); + ucsi_con_mutex_unlock(dp->con); return svdm_version; } @@ -259,7 +262,7 @@ static int ucsi_displayport_vdm(struct typec_altmode *alt, break; } - mutex_unlock(&dp->con->lock); + ucsi_con_mutex_unlock(dp->con); return 0; } @@ -296,6 +299,8 @@ void ucsi_displayport_remove_partner(struct typec_altmode *alt) if (!dp) return; + cancel_work_sync(&dp->work); + dp->data.conf = 0; dp->data.status = 0; dp->initialized = false; diff --git a/drivers/usb/typec/ucsi/psy.c b/drivers/usb/typec/ucsi/psy.c index b35c6e07911e9..9b0157063df0a 100644 --- a/drivers/usb/typec/ucsi/psy.c +++ b/drivers/usb/typec/ucsi/psy.c @@ -163,7 +163,7 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, case UCSI_CONSTAT_PWR_OPMODE_DEFAULT: /* UCSI can't tell b/w DCP/CDP or USB2/3x1/3x2 SDP chargers */ default: - val->intval = 0; + val->intval = UCSI_TYPEC_DEFAULT_CURRENT * 1000; break; } return 0; diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index cd62c36748305..dbdd60446bce7 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -907,6 +907,7 @@ static void ucsi_handle_connector_change(struct work_struct *work) if (con->status.change & UCSI_CONSTAT_POWER_DIR_CHANGE) { typec_set_pwr_role(con->port, role); + ucsi_port_psy_changed(con); /* Complete pending power role swap */ if (!completion_done(&con->complete)) @@ -1550,6 +1551,40 @@ void ucsi_set_drvdata(struct ucsi *ucsi, void *data) } EXPORT_SYMBOL_GPL(ucsi_set_drvdata); +/** + * ucsi_con_mutex_lock - Acquire the connector mutex + * @con: The connector interface to lock + * + * Returns true on success, false if the connector is disconnected + */ +bool ucsi_con_mutex_lock(struct ucsi_connector *con) +{ + bool mutex_locked = false; + bool connected = true; + + while (connected && !mutex_locked) { + mutex_locked = mutex_trylock(&con->lock) != 0; + connected = con->status.flags & UCSI_CONSTAT_CONNECTED; + if (connected && !mutex_locked) + msleep(20); + } + + connected = connected && con->partner; + if (!connected && mutex_locked) + mutex_unlock(&con->lock); + + return connected; +} + +/** + * ucsi_con_mutex_unlock - Release the connector mutex + * @con: The connector interface to unlock + */ +void ucsi_con_mutex_unlock(struct ucsi_connector *con) +{ + mutex_unlock(&con->lock); +} + /** * ucsi_create - Allocate UCSI instance * @dev: Device interface to the PPM (Platform Policy Manager) diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h index 42c60eba5fb6e..32ef314e09f92 100644 --- a/drivers/usb/typec/ucsi/ucsi.h +++ b/drivers/usb/typec/ucsi/ucsi.h @@ -15,6 +15,7 @@ struct ucsi; struct ucsi_altmode; +struct ucsi_connector; struct dentry; /* UCSI offsets (Bytes) */ @@ -74,6 +75,8 @@ int ucsi_register(struct ucsi *ucsi); void ucsi_unregister(struct ucsi *ucsi); void *ucsi_get_drvdata(struct ucsi *ucsi); void ucsi_set_drvdata(struct ucsi *ucsi, void *data); +bool ucsi_con_mutex_lock(struct ucsi_connector *con); +void ucsi_con_mutex_unlock(struct ucsi_connector *con); void ucsi_connector_change(struct ucsi *ucsi, u8 num); @@ -333,9 +336,10 @@ struct ucsi { #define UCSI_MAX_SVID 5 #define UCSI_MAX_ALTMODES (UCSI_MAX_SVID * 6) -#define UCSI_TYPEC_VSAFE5V 5000 -#define UCSI_TYPEC_1_5_CURRENT 1500 -#define UCSI_TYPEC_3_0_CURRENT 3000 +#define UCSI_TYPEC_VSAFE5V 5000 +#define UCSI_TYPEC_DEFAULT_CURRENT 100 +#define UCSI_TYPEC_1_5_CURRENT 1500 +#define UCSI_TYPEC_3_0_CURRENT 3000 struct ucsi_connector { int num; diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c index 14a5f55f24fc8..b22e0881bfaf3 100644 --- a/drivers/usb/usbip/vhci_hcd.c +++ b/drivers/usb/usbip/vhci_hcd.c @@ -764,6 +764,17 @@ static int vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flag ctrlreq->wValue, vdev->rhport); vdev->udev = usb_get_dev(urb->dev); + /* + * NOTE: A similar operation has been done via + * USB_REQ_GET_DESCRIPTOR handler below, which is + * supposed to always precede USB_REQ_SET_ADDRESS. + * + * It's not entirely clear if operating on a different + * usb_device instance here is a real possibility, + * otherwise this call and vdev->udev assignment above + * should be dropped. + */ + dev_pm_syscore_device(&vdev->udev->dev, true); usb_put_dev(old); spin_lock(&vdev->ud.lock); @@ -784,6 +795,17 @@ static int vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flag "Not yet?:Get_Descriptor to device 0 (get max pipe size)\n"); vdev->udev = usb_get_dev(urb->dev); + /* + * Set syscore PM flag for the virtually attached + * devices to ensure they will not enter suspend on + * the client side. + * + * Note this doesn't have any impact on the physical + * devices attached to the host system on the server + * side, hence there is no need to undo the operation + * on disconnect. + */ + dev_pm_syscore_device(&vdev->udev->dev, true); usb_put_dev(old); goto out; diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index b56aae3f7be37..9b8b70ffde5a0 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -3420,6 +3420,9 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, ndev->mvdev.max_vqs = max_vqs; mvdev = &ndev->mvdev; mvdev->mdev = mdev; + /* cpu_to_mlx5vdpa16() below depends on this flag */ + mvdev->actual_features = + (device_features & BIT_ULL(VIRTIO_F_VERSION_1)); ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index 610a429c61912..54c3079031e16 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -194,11 +194,10 @@ static int vfio_df_group_open(struct vfio_device_file *df) * implies they expected translation to exist */ if (!capable(CAP_SYS_RAWIO) || - vfio_iommufd_device_has_compat_ioas(device, df->iommufd)) + vfio_iommufd_device_has_compat_ioas(device, df->iommufd)) { ret = -EPERM; - else - ret = 0; - goto out_put_kvm; + goto out_put_kvm; + } } ret = vfio_df_open(df); diff --git a/drivers/vfio/iommufd.c b/drivers/vfio/iommufd.c index 82eba6966fa50..02852899c2aee 100644 --- a/drivers/vfio/iommufd.c +++ b/drivers/vfio/iommufd.c @@ -25,6 +25,10 @@ int vfio_df_iommufd_bind(struct vfio_device_file *df) lockdep_assert_held(&vdev->dev_set->lock); + /* Returns 0 to permit device opening under noiommu mode */ + if (vfio_device_is_noiommu(vdev)) + return 0; + return vdev->ops->bind_iommufd(vdev, ictx, &df->devid); } diff --git a/drivers/vfio/iova_bitmap.c b/drivers/vfio/iova_bitmap.c index 38b51613ecca9..3f48125e2b9f0 100644 --- a/drivers/vfio/iova_bitmap.c +++ b/drivers/vfio/iova_bitmap.c @@ -127,9 +127,8 @@ struct iova_bitmap { static unsigned long iova_bitmap_offset_to_index(struct iova_bitmap *bitmap, unsigned long iova) { - unsigned long pgsize = 1UL << bitmap->mapped.pgshift; - - return iova / (BITS_PER_TYPE(*bitmap->bitmap) * pgsize); + return (iova >> bitmap->mapped.pgshift) / + BITS_PER_TYPE(*bitmap->bitmap); } /* diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index 4d27465c8f1a8..712b178c42aae 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -350,6 +350,32 @@ static int vf_qm_func_stop(struct hisi_qm *qm) return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, 0, 0, 0); } +static int vf_qm_version_check(struct acc_vf_data *vf_data, struct device *dev) +{ + switch (vf_data->acc_magic) { + case ACC_DEV_MAGIC_V2: + if (vf_data->major_ver != ACC_DRV_MAJOR_VER) { + dev_info(dev, "migration driver version<%u.%u> not match!\n", + vf_data->major_ver, vf_data->minor_ver); + return -EINVAL; + } + break; + case ACC_DEV_MAGIC_V1: + /* Correct dma address */ + vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH]; + vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET; + vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW]; + vf_data->aeqe_dma = vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH]; + vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET; + vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW]; + break; + default: + return -EINVAL; + } + + return 0; +} + static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, struct hisi_acc_vf_migration_file *migf) { @@ -363,7 +389,8 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done) return 0; - if (vf_data->acc_magic != ACC_DEV_MAGIC) { + ret = vf_qm_version_check(vf_data, dev); + if (ret) { dev_err(dev, "failed to match ACC_DEV_MAGIC\n"); return -EINVAL; } @@ -399,13 +426,6 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, return -EINVAL; } - ret = qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1); - if (ret) { - dev_err(dev, "failed to write QM_VF_STATE\n"); - return ret; - } - - hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; hisi_acc_vdev->match_done = true; return 0; } @@ -418,7 +438,9 @@ static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, int vf_id = hisi_acc_vdev->vf_id; int ret; - vf_data->acc_magic = ACC_DEV_MAGIC; + vf_data->acc_magic = ACC_DEV_MAGIC_V2; + vf_data->major_ver = ACC_DRV_MAJOR_VER; + vf_data->minor_ver = ACC_DRV_MINOR_VER; /* Save device id */ vf_data->dev_id = hisi_acc_vdev->vf_dev->device; @@ -441,6 +463,19 @@ static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, return 0; } +static void vf_qm_xeqc_save(struct hisi_qm *qm, + struct hisi_acc_vf_migration_file *migf) +{ + struct acc_vf_data *vf_data = &migf->vf_data; + u16 eq_head, aeq_head; + + eq_head = vf_data->qm_eqc_dw[0] & 0xFFFF; + qm_db(qm, 0, QM_DOORBELL_CMD_EQ, eq_head, 0); + + aeq_head = vf_data->qm_aeqc_dw[0] & 0xFFFF; + qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, aeq_head, 0); +} + static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, struct hisi_acc_vf_migration_file *migf) { @@ -456,6 +491,20 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, if (migf->total_length < sizeof(struct acc_vf_data)) return -EINVAL; + if (!vf_data->eqe_dma || !vf_data->aeqe_dma || + !vf_data->sqc_dma || !vf_data->cqc_dma) { + dev_info(dev, "resume dma addr is NULL!\n"); + hisi_acc_vdev->vf_qm_state = QM_NOT_READY; + return 0; + } + + ret = qm_write_regs(qm, QM_VF_STATE, &vf_data->vf_qm_state, 1); + if (ret) { + dev_err(dev, "failed to write QM_VF_STATE\n"); + return -EINVAL; + } + hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; + qm->eqe_dma = vf_data->eqe_dma; qm->aeqe_dma = vf_data->aeqe_dma; qm->sqc_dma = vf_data->sqc_dma; @@ -516,12 +565,12 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, return -EINVAL; /* Every reg is 32 bit, the dma address is 64 bit. */ - vf_data->eqe_dma = vf_data->qm_eqc_dw[1]; + vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH]; vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET; - vf_data->eqe_dma |= vf_data->qm_eqc_dw[0]; - vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1]; + vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW]; + vf_data->aeqe_dma = vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH]; vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET; - vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0]; + vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW]; /* Through SQC_BT/CQC_BT to get sqc and cqc address */ ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma); @@ -537,6 +586,9 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, } migf->total_length = sizeof(struct acc_vf_data); + /* Save eqc and aeqc interrupt information */ + vf_qm_xeqc_save(vf_qm, migf); + return 0; } @@ -1352,6 +1404,7 @@ static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev) hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1; hisi_acc_vdev->pf_qm = pf_qm; hisi_acc_vdev->vf_dev = pdev; + hisi_acc_vdev->vf_qm_state = QM_NOT_READY; mutex_init(&hisi_acc_vdev->state_mutex); core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY; diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h index dcabfeec6ca19..f1d8fe86b6eb2 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h @@ -38,6 +38,9 @@ #define QM_REG_ADDR_OFFSET 0x0004 #define QM_XQC_ADDR_OFFSET 32U +#define QM_XQC_ADDR_LOW 0x1 +#define QM_XQC_ADDR_HIGH 0x2 + #define QM_VF_AEQ_INT_MASK 0x0004 #define QM_VF_EQ_INT_MASK 0x000c #define QM_IFC_INT_SOURCE_V 0x0020 @@ -49,10 +52,15 @@ #define QM_EQC_DW0 0X8000 #define QM_AEQC_DW0 0X8020 +#define ACC_DRV_MAJOR_VER 1 +#define ACC_DRV_MINOR_VER 0 + +#define ACC_DEV_MAGIC_V1 0XCDCDCDCDFEEDAACC +#define ACC_DEV_MAGIC_V2 0xAACCFEEDDECADEDE + struct acc_vf_data { #define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state) /* QM match information */ -#define ACC_DEV_MAGIC 0XCDCDCDCDFEEDAACC u64 acc_magic; u32 qp_num; u32 dev_id; @@ -60,7 +68,9 @@ struct acc_vf_data { u32 qp_base; u32 vf_qm_state; /* QM reserved match information */ - u32 qm_rsv_state[3]; + u16 major_ver; + u16 minor_ver; + u32 qm_rsv_state[2]; /* QM RW regs */ u32 aeq_int_mask; diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c index 2d996c913ecd5..82558fa7712e4 100644 --- a/drivers/vfio/pci/mlx5/cmd.c +++ b/drivers/vfio/pci/mlx5/cmd.c @@ -1389,8 +1389,8 @@ int mlx5vf_start_page_tracker(struct vfio_device *vdev, log_max_msg_size = MLX5_CAP_ADV_VIRTUALIZATION(mdev, pg_track_log_max_msg_size); max_msg_size = (1ULL << log_max_msg_size); /* The RQ must hold at least 4 WQEs/messages for successful QP creation */ - if (rq_size < 4 * max_msg_size) - rq_size = 4 * max_msg_size; + if (rq_size < 4ULL * max_msg_size) + rq_size = 4ULL * max_msg_size; memset(tracker, 0, sizeof(*tracker)); tracker->uar = mlx5_get_uars_page(mdev); diff --git a/drivers/vfio/pci/pds/dirty.c b/drivers/vfio/pci/pds/dirty.c index 27607d7b9030a..6adc5f1ae2eb9 100644 --- a/drivers/vfio/pci/pds/dirty.c +++ b/drivers/vfio/pci/pds/dirty.c @@ -81,7 +81,7 @@ static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_dirty *dirty, host_ack_bmp = vzalloc(bytes); if (!host_ack_bmp) { - bitmap_free(host_seq_bmp); + vfree(host_seq_bmp); return -ENOMEM; } diff --git a/drivers/vfio/pci/pds/vfio_dev.c b/drivers/vfio/pci/pds/vfio_dev.c index a286ebcc71126..69d1bcefd7c32 100644 --- a/drivers/vfio/pci/pds/vfio_dev.c +++ b/drivers/vfio/pci/pds/vfio_dev.c @@ -231,6 +231,7 @@ static const struct vfio_device_ops pds_vfio_ops = { .bind_iommufd = vfio_iommufd_physical_bind, .unbind_iommufd = vfio_iommufd_physical_unbind, .attach_ioas = vfio_iommufd_physical_attach_ioas, + .detach_ioas = vfio_iommufd_physical_detach_ioas, }; const struct vfio_device_ops *pds_vfio_ops_info(void) diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index a2ad4f7c716bf..d9eb8733a324b 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -1813,7 +1813,8 @@ int vfio_config_init(struct vfio_pci_core_device *vdev) cpu_to_le16(PCI_COMMAND_MEMORY); } - if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->nointx) + if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->nointx || + vdev->pdev->irq == IRQ_NOTCONNECTED) vconfig[PCI_INTERRUPT_PIN] = 0; ret = vfio_cap_init(vdev); diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index a8f259bc2f4d0..3f139360752e2 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -731,15 +731,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable); static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type) { if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { - u8 pin; - - if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || - vdev->nointx || vdev->pdev->is_virtfn) - return 0; - - pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); - - return pin ? 1 : 0; + return vdev->vconfig[PCI_INTERRUPT_PIN] ? 1 : 0; } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { u8 pos; u16 flags; @@ -2243,7 +2235,7 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) return -EBUSY; } - if (pci_is_root_bus(pdev->bus)) { + if (pci_is_root_bus(pdev->bus) || pdev->is_virtfn) { ret = vfio_assign_device_set(&vdev->vdev, vdev); } else if (!pci_probe_reset_slot(pdev->slot)) { ret = vfio_assign_device_set(&vdev->vdev, pdev->slot); diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index 620134041b488..c4322faca2bd5 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -269,7 +269,7 @@ static int vfio_intx_enable(struct vfio_pci_core_device *vdev, if (!is_irq_none(vdev)) return -EINVAL; - if (!pdev->irq) + if (!pdev->irq || pdev->irq == IRQ_NOTCONNECTED) return -ENODEV; name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-intx(%s)", pci_name(pdev)); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index eacd6ec04de5a..f63f116b9cd0a 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -294,7 +294,7 @@ static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize) struct rb_node *p; for (p = rb_prev(n); p; p = rb_prev(p)) { - struct vfio_dma *dma = rb_entry(n, + struct vfio_dma *dma = rb_entry(p, struct vfio_dma, node); vfio_dma_bitmap_free(dma); @@ -635,6 +635,13 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, while (npage) { if (!batch->size) { + /* + * Large mappings may take a while to repeatedly refill + * the batch, so conditionally relinquish the CPU when + * needed to avoid stalls. + */ + cond_resched(); + /* Empty batch, so refill it. */ long req_pages = min_t(long, npage, batch->capacity); diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 40732e8ed4c6f..6dfb290c339f9 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -537,7 +537,8 @@ void vfio_df_close(struct vfio_device_file *df) lockdep_assert_held(&device->dev_set->lock); - vfio_assert_device_open(device); + if (!vfio_assert_device_open(device)) + return; if (device->open_count == 1) vfio_df_device_last_close(df); device->open_count--; @@ -1194,7 +1195,7 @@ static int vfio_ioctl_device_feature(struct vfio_device *device, feature.argsz - minsz); default: if (unlikely(!device->ops->device_feature)) - return -EINVAL; + return -ENOTTY; return device->ops->device_feature(device, feature.flags, arg->data, feature.argsz - minsz); diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index f2ed7167c8480..5ad237d77a9a9 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -96,6 +96,7 @@ struct vhost_net_ubuf_ref { atomic_t refcount; wait_queue_head_t wait; struct vhost_virtqueue *vq; + struct rcu_head rcu; }; #define VHOST_NET_BATCH 64 @@ -249,9 +250,13 @@ vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy) static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs) { - int r = atomic_sub_return(1, &ubufs->refcount); + int r; + + rcu_read_lock(); + r = atomic_sub_return(1, &ubufs->refcount); if (unlikely(!r)) wake_up(&ubufs->wait); + rcu_read_unlock(); return r; } @@ -264,7 +269,7 @@ static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs) static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs) { vhost_net_ubuf_put_and_wait(ubufs); - kfree(ubufs); + kfree_rcu(ubufs, rcu); } static void vhost_net_clear_ubuf_info(struct vhost_net *n) diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 8d8a22504d71f..90a3c0fc5ab0a 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -560,6 +560,9 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) int ret; llnode = llist_del_all(&svq->completion_list); + + mutex_lock(&svq->vq.mutex); + llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) { se_cmd = &cmd->tvc_se_cmd; @@ -593,6 +596,8 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) vhost_scsi_release_cmd_res(se_cmd); } + mutex_unlock(&svq->vq.mutex); + if (signal) vhost_signal(&svq->vs->dev, &svq->vq); } @@ -746,7 +751,7 @@ vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter, size_t len = iov_iter_count(iter); unsigned int nbytes = 0; struct page *page; - int i; + int i, ret; if (cmd->tvc_data_direction == DMA_FROM_DEVICE) { cmd->saved_iter_addr = dup_iter(&cmd->saved_iter, iter, @@ -759,6 +764,7 @@ vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter, page = alloc_page(GFP_KERNEL); if (!page) { i--; + ret = -ENOMEM; goto err; } @@ -766,8 +772,10 @@ vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter, sg_set_page(&sg[i], page, nbytes, 0); if (cmd->tvc_data_direction == DMA_TO_DEVICE && - copy_page_from_iter(page, 0, nbytes, iter) != nbytes) + copy_page_from_iter(page, 0, nbytes, iter) != nbytes) { + ret = -EFAULT; goto err; + } len -= nbytes; } @@ -782,7 +790,7 @@ vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter, for (; i >= 0; i--) __free_page(sg_page(&sg[i])); kfree(cmd->saved_iter_addr); - return -ENOMEM; + return ret; } static int @@ -1024,10 +1032,8 @@ vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc, /* validated at handler entry */ vs_tpg = vhost_vq_get_backend(vq); tpg = READ_ONCE(vs_tpg[*vc->target]); - if (unlikely(!tpg)) { - vq_err(vq, "Target 0x%x does not exist\n", *vc->target); + if (unlikely(!tpg)) goto out; - } } if (tpgp) @@ -1221,9 +1227,9 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) " %d\n", cmd, exp_data_len, prot_bytes, data_direction); if (data_direction != DMA_NONE) { - if (unlikely(vhost_scsi_mapal(cmd, prot_bytes, - &prot_iter, exp_data_len, - &data_iter))) { + ret = vhost_scsi_mapal(cmd, prot_bytes, &prot_iter, + exp_data_len, &data_iter); + if (unlikely(ret)) { vq_err(vq, "Failed to map iov to sgl\n"); vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd); goto err; @@ -1301,8 +1307,11 @@ static void vhost_scsi_tmf_resp_work(struct vhost_work *work) resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED; } + mutex_lock(&tmf->svq->vq.mutex); vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs, tmf->vq_desc, &tmf->resp_iov, resp_code); + mutex_unlock(&tmf->svq->vq.mutex); + vhost_scsi_release_tmf_res(tmf); } diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 069c8a23bff9e..e9229f9e0a054 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -2724,6 +2724,9 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, } r = __vhost_add_used_n(vq, heads, count); + if (r < 0) + return r; + /* Make sure buffer is written before we update index. */ smp_wmb(); if (vhost_put_used_idx(vq)) { diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index 7b8fd977f71cc..134cc10af776b 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -1191,6 +1191,7 @@ static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, struct iov_iter iter; u64 translated; int ret; + size_t size; ret = iotlb_translate(vrh, (u64)(uintptr_t)src, len - total_translated, &translated, @@ -1208,9 +1209,9 @@ static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, translated); } - ret = copy_from_iter(dst, translated, &iter); - if (ret < 0) - return ret; + size = copy_from_iter(dst, translated, &iter); + if (size != translated) + return -EFAULT; src += translated; dst += translated; @@ -1237,6 +1238,7 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, struct iov_iter iter; u64 translated; int ret; + size_t size; ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, len - total_translated, &translated, @@ -1254,9 +1256,9 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, translated); } - ret = copy_to_iter(src, translated, &iter); - if (ret < 0) - return ret; + size = copy_to_iter(src, translated, &iter); + if (size != translated) + return -EFAULT; src += translated; dst += translated; diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index d94a06008ff64..9d6c6cdac0502 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -114,6 +114,7 @@ vhost_transport_do_send_pkt(struct vhost_vsock *vsock, struct sk_buff *skb; unsigned out, in; size_t nbytes; + u32 offset; int head; skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); @@ -156,7 +157,8 @@ vhost_transport_do_send_pkt(struct vhost_vsock *vsock, } iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len); - payload_len = skb->len; + offset = VIRTIO_VSOCK_SKB_CB(skb)->offset; + payload_len = skb->len - offset; hdr = virtio_vsock_hdr(skb); /* If the packet is greater than the space available in the @@ -197,8 +199,10 @@ vhost_transport_do_send_pkt(struct vhost_vsock *vsock, break; } - nbytes = copy_to_iter(skb->data, payload_len, &iov_iter); - if (nbytes != payload_len) { + if (skb_copy_datagram_iter(skb, + offset, + &iov_iter, + payload_len)) { kfree_skb(skb); vq_err(vq, "Faulted on copying pkt buf\n"); break; @@ -212,13 +216,13 @@ vhost_transport_do_send_pkt(struct vhost_vsock *vsock, vhost_add_used(vq, head, sizeof(*hdr) + payload_len); added = true; - skb_pull(skb, payload_len); + VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len; total_len += payload_len; /* If we didn't send all the payload we can requeue the packet * to send it with the next available buffer. */ - if (skb->len > 0) { + if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) { hdr->flags |= cpu_to_le32(flags_to_restore); /* We are queueing the same skb to handle @@ -340,6 +344,10 @@ vhost_vsock_alloc_skb(struct vhost_virtqueue *vq, len = iov_length(vq->iov, out); + if (len < VIRTIO_VSOCK_SKB_HEADROOM || + len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM) + return NULL; + /* len contains both payload and hdr */ skb = virtio_vsock_alloc_skb(len, GFP_KERNEL); if (!skb) @@ -363,18 +371,15 @@ vhost_vsock_alloc_skb(struct vhost_virtqueue *vq, return skb; /* The pkt is too big or the length in the header is invalid */ - if (payload_len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || - payload_len + sizeof(*hdr) > len) { + if (payload_len + sizeof(*hdr) > len) { kfree_skb(skb); return NULL; } - virtio_vsock_skb_rx_put(skb); + virtio_vsock_skb_put(skb, payload_len); - nbytes = copy_from_iter(skb->data, payload_len, &iov_iter); - if (nbytes != payload_len) { - vq_err(vq, "Expected %zu byte payload, got %zu bytes\n", - payload_len, nbytes); + if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) { + vq_err(vq, "Failed to copy %zu byte payload\n", payload_len); kfree_skb(skb); return NULL; } diff --git a/drivers/video/backlight/lp855x_bl.c b/drivers/video/backlight/lp855x_bl.c index 7075bfab59c4d..d191560ce285f 100644 --- a/drivers/video/backlight/lp855x_bl.c +++ b/drivers/video/backlight/lp855x_bl.c @@ -22,7 +22,7 @@ #define LP855X_DEVICE_CTRL 0x01 #define LP855X_EEPROM_START 0xA0 #define LP855X_EEPROM_END 0xA7 -#define LP8556_EPROM_START 0xA0 +#define LP8556_EPROM_START 0x98 #define LP8556_EPROM_END 0xAF /* LP8555/7 Registers */ diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c index 10129095a4c17..b19e5f73de8bb 100644 --- a/drivers/video/backlight/qcom-wled.c +++ b/drivers/video/backlight/qcom-wled.c @@ -1406,9 +1406,11 @@ static int wled_configure(struct wled *wled) wled->ctrl_addr = be32_to_cpu(*prop_addr); rc = of_property_read_string(dev->of_node, "label", &wled->name); - if (rc) + if (rc) { wled->name = devm_kasprintf(dev, GFP_KERNEL, "%pOFn", dev->of_node); - + if (!wled->name) + return -ENOMEM; + } switch (wled->version) { case 3: u32_opts = wled3_opts; diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index f1711b2f9ff05..d99e1b3e4e5c1 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -82,6 +82,15 @@ static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) /* Redraw, so that we get putc(s) for output done while blanked */ return 1; } + +static bool dummycon_switch(struct vc_data *vc) +{ + /* + * Redraw, so that we get putc(s) for output done while switched + * away. Informs deferred consoles to take over the display. + */ + return true; +} #else static void dummycon_putc(struct vc_data *vc, int c, int ypos, int xpos) { } static void dummycon_putcs(struct vc_data *vc, const unsigned short *s, @@ -90,6 +99,10 @@ static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) { return 0; } +static bool dummycon_switch(struct vc_data *vc) +{ + return false; +} #endif static const char *dummycon_startup(void) @@ -97,7 +110,7 @@ static const char *dummycon_startup(void) return "dummy device"; } -static void dummycon_init(struct vc_data *vc, int init) +static void dummycon_init(struct vc_data *vc, bool init) { vc->vc_can_do_color = 1; if (init) { @@ -108,8 +121,8 @@ static void dummycon_init(struct vc_data *vc, int init) } static void dummycon_deinit(struct vc_data *vc) { } -static void dummycon_clear(struct vc_data *vc, int sy, int sx, int height, - int width) { } +static void dummycon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) { } static void dummycon_cursor(struct vc_data *vc, int mode) { } static bool dummycon_scroll(struct vc_data *vc, unsigned int top, @@ -119,11 +132,6 @@ static bool dummycon_scroll(struct vc_data *vc, unsigned int top, return false; } -static int dummycon_switch(struct vc_data *vc) -{ - return 0; -} - /* * The console `switch' structure for the dummy console * diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index ef29b321967f0..26b41a8f36c87 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -352,7 +352,7 @@ static const char *mdacon_startup(void) return "MDA-2"; } -static void mdacon_init(struct vc_data *c, int init) +static void mdacon_init(struct vc_data *c, bool init) { c->vc_complement_mask = 0x0800; /* reverse video */ c->vc_display_fg = &mda_display_fg; @@ -442,26 +442,21 @@ static void mdacon_putcs(struct vc_data *c, const unsigned short *s, } } -static void mdacon_clear(struct vc_data *c, int y, int x, - int height, int width) +static void mdacon_clear(struct vc_data *c, unsigned int y, unsigned int x, + unsigned int width) { u16 *dest = mda_addr(x, y); u16 eattr = mda_convert_attr(c->vc_video_erase_char); - if (width <= 0 || height <= 0) + if (width <= 0) return; - if (x==0 && width==mda_num_columns) { - scr_memsetw(dest, eattr, height*width*2); - } else { - for (; height > 0; height--, dest+=mda_num_columns) - scr_memsetw(dest, eattr, width*2); - } + scr_memsetw(dest, eattr, width * 2); } - -static int mdacon_switch(struct vc_data *c) + +static bool mdacon_switch(struct vc_data *c) { - return 1; /* redrawing needed */ + return true; /* redrawing needed */ } static int mdacon_blank(struct vc_data *c, int blank, int mode_switch) diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index e8e4f82cd4a1b..63d96c4bbdccd 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -324,7 +324,7 @@ static const char *newport_startup(void) return NULL; } -static void newport_init(struct vc_data *vc, int init) +static void newport_init(struct vc_data *vc, bool init) { int cols, rows; @@ -346,12 +346,12 @@ static void newport_deinit(struct vc_data *c) } } -static void newport_clear(struct vc_data *vc, int sy, int sx, int height, - int width) +static void newport_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) { int xend = ((sx + width) << 3) - 1; int ystart = ((sy << 4) + topscan) & 0x3ff; - int yend = (((sy + height) << 4) + topscan - 1) & 0x3ff; + int yend = (((sy + 1) << 4) + topscan - 1) & 0x3ff; if (logo_active) return; @@ -462,7 +462,7 @@ static void newport_cursor(struct vc_data *vc, int mode) } } -static int newport_switch(struct vc_data *vc) +static bool newport_switch(struct vc_data *vc) { static int logo_drawn = 0; @@ -476,7 +476,7 @@ static int newport_switch(struct vc_data *vc) } } - return 1; + return true; } static int newport_blank(struct vc_data *c, int blank, int mode_switch) diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 992a4fa431aaa..87900600eff11 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -273,7 +273,7 @@ static int sticon_font_set(struct vc_data *vc, struct console_font *font, return sticon_set_font(vc, font, vpitch); } -static void sticon_init(struct vc_data *c, int init) +static void sticon_init(struct vc_data *c, bool init) { struct sti_struct *sti = sticon_sti; int vc_cols, vc_rows; @@ -300,19 +300,19 @@ static void sticon_deinit(struct vc_data *c) sticon_set_def_font(i); } -static void sticon_clear(struct vc_data *conp, int sy, int sx, int height, - int width) +static void sticon_clear(struct vc_data *conp, unsigned int sy, unsigned int sx, + unsigned int width) { - if (!height || !width) + if (!width) return; - sti_clear(sticon_sti, sy, sx, height, width, + sti_clear(sticon_sti, sy, sx, 1, width, conp->vc_video_erase_char, font_data[conp->vc_num]); } -static int sticon_switch(struct vc_data *conp) +static bool sticon_switch(struct vc_data *conp) { - return 1; /* needs refreshing */ + return true; /* needs refreshing */ } static int sticon_blank(struct vc_data *c, int blank, int mode_switch) diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 7ad047bcae171..14e5312a00308 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -332,7 +332,7 @@ static const char *vgacon_startup(void) return display_desc; } -static void vgacon_init(struct vc_data *c, int init) +static void vgacon_init(struct vc_data *c, bool init) { struct uni_pagedict *p; @@ -349,7 +349,7 @@ static void vgacon_init(struct vc_data *c, int init) c->vc_scan_lines = vga_scan_lines; c->vc_font.height = c->vc_cell_height = vga_video_font_height; - /* set dimensions manually if init != 0 since vc_resize() will fail */ + /* set dimensions manually if init is true since vc_resize() will fail */ if (init) { c->vc_cols = vga_video_num_columns; c->vc_rows = vga_video_num_lines; @@ -585,7 +585,7 @@ static void vgacon_doresize(struct vc_data *c, raw_spin_unlock_irqrestore(&vga_lock, flags); } -static int vgacon_switch(struct vc_data *c) +static bool vgacon_switch(struct vc_data *c) { int x = c->vc_cols * VGA_FONTWIDTH; int y = c->vc_rows * c->vc_cell_height; @@ -614,7 +614,7 @@ static int vgacon_switch(struct vc_data *c) vgacon_doresize(c, c->vc_cols, c->vc_rows); } - return 0; /* Redrawing not needed */ + return false; /* Redrawing not needed */ } static void vga_set_palette(struct vc_data *vc, const unsigned char *table) @@ -1156,8 +1156,8 @@ static bool vgacon_scroll(struct vc_data *c, unsigned int t, unsigned int b, * The console `switch' structure for the VGA based console */ -static void vgacon_clear(struct vc_data *vc, int sy, int sx, int height, - int width) { } +static void vgacon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) { } static void vgacon_putc(struct vc_data *vc, int c, int ypos, int xpos) { } static void vgacon_putcs(struct vc_data *vc, const unsigned short *s, int count, int ypos, int xpos) { } diff --git a/drivers/video/fbdev/aty/atyfb_base.c b/drivers/video/fbdev/aty/atyfb_base.c index 3dcf83f5e7b4a..297ba9517b88a 100644 --- a/drivers/video/fbdev/aty/atyfb_base.c +++ b/drivers/video/fbdev/aty/atyfb_base.c @@ -2611,8 +2611,12 @@ static int aty_init(struct fb_info *info) pr_cont("\n"); } #endif - if (par->pll_ops->init_pll) - par->pll_ops->init_pll(info, &par->pll); + if (par->pll_ops->init_pll) { + ret = par->pll_ops->init_pll(info, &par->pll); + if (ret) + return ret; + } + if (par->pll_ops->resume_pll) par->pll_ops->resume_pll(info, &par->pll); diff --git a/drivers/video/fbdev/core/bitblit.c b/drivers/video/fbdev/core/bitblit.c index 8587c9da06700..8563264d11fac 100644 --- a/drivers/video/fbdev/core/bitblit.c +++ b/drivers/video/fbdev/core/bitblit.c @@ -59,12 +59,11 @@ static void bit_bmove(struct vc_data *vc, struct fb_info *info, int sy, } static void bit_clear(struct vc_data *vc, struct fb_info *info, int sy, - int sx, int height, int width) + int sx, int height, int width, int fg, int bg) { - int bgshift = (vc->vc_hi_font_mask) ? 13 : 12; struct fb_fillrect region; - region.color = attr_bgcol_ec(bgshift, vc, info); + region.color = bg; region.dx = sx * vc->vc_font.width; region.dy = sy * vc->vc_font.height; region.width = width * vc->vc_font.width; @@ -80,12 +79,16 @@ static inline void bit_putcs_aligned(struct vc_data *vc, struct fb_info *info, struct fb_image *image, u8 *buf, u8 *dst) { u16 charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff; + unsigned int charcnt = vc->vc_font.charcount; u32 idx = vc->vc_font.width >> 3; u8 *src; while (cnt--) { - src = vc->vc_font.data + (scr_readw(s++)& - charmask)*cellsize; + u16 ch = scr_readw(s++) & charmask; + + if (ch >= charcnt) + ch = 0; + src = vc->vc_font.data + (unsigned int)ch * cellsize; if (attr) { update_attr(buf, src, attr, vc); @@ -113,14 +116,18 @@ static inline void bit_putcs_unaligned(struct vc_data *vc, u8 *dst) { u16 charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff; + unsigned int charcnt = vc->vc_font.charcount; u32 shift_low = 0, mod = vc->vc_font.width % 8; u32 shift_high = 8; u32 idx = vc->vc_font.width >> 3; u8 *src; while (cnt--) { - src = vc->vc_font.data + (scr_readw(s++)& - charmask)*cellsize; + u16 ch = scr_readw(s++) & charmask; + + if (ch >= charcnt) + ch = 0; + src = vc->vc_font.data + (unsigned int)ch * cellsize; if (attr) { update_attr(buf, src, attr, vc); @@ -161,6 +168,11 @@ static void bit_putcs(struct vc_data *vc, struct fb_info *info, image.height = vc->vc_font.height; image.depth = 1; + if (image.dy >= info->var.yres) + return; + + image.height = min(image.height, info->var.yres - image.dy); + if (attribute) { buf = kmalloc(cellsize, GFP_ATOMIC); if (!buf) @@ -174,6 +186,18 @@ static void bit_putcs(struct vc_data *vc, struct fb_info *info, cnt = count; image.width = vc->vc_font.width * cnt; + + if (image.dx >= info->var.xres) + break; + + if (image.dx + image.width > info->var.xres) { + image.width = info->var.xres - image.dx; + cnt = image.width / vc->vc_font.width; + if (cnt == 0) + break; + image.width = cnt * vc->vc_font.width; + } + pitch = DIV_ROUND_UP(image.width, 8) + scan_align; pitch &= ~scan_align; size = pitch * image.height + buf_align; diff --git a/drivers/video/fbdev/core/fb_cmdline.c b/drivers/video/fbdev/core/fb_cmdline.c index 4d1634c492ec4..594b60424d1c6 100644 --- a/drivers/video/fbdev/core/fb_cmdline.c +++ b/drivers/video/fbdev/core/fb_cmdline.c @@ -40,7 +40,7 @@ int fb_get_options(const char *name, char **option) bool enabled; if (name) - is_of = strncmp(name, "offb", 4); + is_of = !strncmp(name, "offb", 4); enabled = __video_get_options(name, &options, is_of); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 405d587450ef8..78a5b22c8d150 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -115,9 +115,14 @@ static signed char con2fb_map_boot[MAX_NR_CONSOLES]; static struct fb_info *fbcon_info_from_console(int console) { + signed char fb; WARN_CONSOLE_UNLOCKED(); - return fbcon_registered_fb[con2fb_map[console]]; + fb = con2fb_map[console]; + if (fb < 0 || fb >= ARRAY_SIZE(fbcon_registered_fb)) + return NULL; + + return fbcon_registered_fb[fb]; } static int logo_lines; @@ -803,7 +808,8 @@ static void con2fb_init_display(struct vc_data *vc, struct fb_info *info, fg_vc->vc_rows); } - update_screen(vc_cons[fg_console].d); + if (fg_console != unit) + update_screen(vc_cons[fg_console].d); } /** @@ -930,13 +936,13 @@ static const char *fbcon_startup(void) int rows, cols; /* - * If num_registered_fb is zero, this is a call for the dummy part. + * If fbcon_num_registered_fb is zero, this is a call for the dummy part. * The frame buffer devices weren't initialized yet. */ if (!fbcon_num_registered_fb || info_idx == -1) return display_desc; /* - * Instead of blindly using registered_fb[0], we use info_idx, set by + * Instead of blindly using fbcon_registered_fb[0], we use info_idx, set by * fbcon_fb_registered(); */ info = fbcon_registered_fb[info_idx]; @@ -988,7 +994,7 @@ static const char *fbcon_startup(void) return display_desc; } -static void fbcon_init(struct vc_data *vc, int init) +static void fbcon_init(struct vc_data *vc, bool init) { struct fb_info *info; struct fbcon_ops *ops; @@ -1235,12 +1241,12 @@ static void fbcon_deinit(struct vc_data *vc) * restriction is simplicity & efficiency at the moment. */ -static void fbcon_clear(struct vc_data *vc, int sy, int sx, int height, - int width) +static void __fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int height, unsigned int width) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; - + int fg, bg; struct fbcon_display *p = &fb_display[vc->vc_num]; u_int y_break; @@ -1261,16 +1267,24 @@ static void fbcon_clear(struct vc_data *vc, int sy, int sx, int height, fbcon_clear_margins(vc, 0); } + fg = get_color(vc, info, vc->vc_video_erase_char, 1); + bg = get_color(vc, info, vc->vc_video_erase_char, 0); /* Split blits that cross physical y_wrap boundary */ y_break = p->vrows - p->yscroll; if (sy < y_break && sy + height - 1 >= y_break) { u_int b = y_break - sy; - ops->clear(vc, info, real_y(p, sy), sx, b, width); + ops->clear(vc, info, real_y(p, sy), sx, b, width, fg, bg); ops->clear(vc, info, real_y(p, sy + b), sx, height - b, - width); + width, fg, bg); } else - ops->clear(vc, info, real_y(p, sy), sx, height, width); + ops->clear(vc, info, real_y(p, sy), sx, height, width, fg, bg); +} + +static void fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) +{ + __fbcon_clear(vc, sy, sx, 1, width); } static void fbcon_putcs(struct vc_data *vc, const unsigned short *s, @@ -1340,6 +1354,7 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var, struct vc_data *svc; struct fbcon_ops *ops = info->fbcon_par; int rows, cols; + unsigned long ret = 0; p = &fb_display[unit]; @@ -1390,11 +1405,10 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var, rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres); cols /= vc->vc_font.width; rows /= vc->vc_font.height; - vc_resize(vc, cols, rows); + ret = vc_resize(vc, cols, rows); - if (con_is_visible(vc)) { + if (con_is_visible(vc) && !ret) update_screen(vc); - } } static __inline__ void ywrap_up(struct vc_data *vc, int count) @@ -1760,7 +1774,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, case SCROLL_MOVE: fbcon_redraw_blit(vc, info, p, t, b - t - count, count); - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * (b - count)), @@ -1783,7 +1797,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_up; - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); break; case SCROLL_PAN_REDRAW: @@ -1801,7 +1815,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, vc->vc_rows - b, b); } else fbcon_redraw_move(vc, p, t + count, b - t - count, t); - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); break; case SCROLL_PAN_MOVE: @@ -1824,14 +1838,14 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_up; - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); break; case SCROLL_REDRAW: redraw_up: fbcon_redraw(vc, t, b - t - count, count * vc->vc_cols); - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * (b - count)), @@ -1848,7 +1862,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, case SCROLL_MOVE: fbcon_redraw_blit(vc, info, p, b - 1, b - t - count, -count); - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * t), @@ -1871,7 +1885,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_down; - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); break; case SCROLL_PAN_MOVE: @@ -1893,7 +1907,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_down; - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); break; case SCROLL_PAN_REDRAW: @@ -1910,14 +1924,14 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, fbcon_redraw_move(vc, p, count, t, 0); } else fbcon_redraw_move(vc, p, t, b - t - count, t + count); - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); break; case SCROLL_REDRAW: redraw_down: fbcon_redraw(vc, b - 1, b - t - count, -count * vc->vc_cols); - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * t), @@ -2059,7 +2073,7 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width, return 0; } -static int fbcon_switch(struct vc_data *vc) +static bool fbcon_switch(struct vc_data *vc) { struct fb_info *info, *old_info = NULL; struct fbcon_ops *ops; @@ -2181,9 +2195,9 @@ static int fbcon_switch(struct vc_data *vc) vc->vc_origin + vc->vc_size_row * vc->vc_top, vc->vc_size_row * (vc->vc_bottom - vc->vc_top) / 2); - return 0; + return false; } - return 1; + return true; } static void fbcon_generic_blank(struct vc_data *vc, struct fb_info *info, @@ -2196,7 +2210,7 @@ static void fbcon_generic_blank(struct vc_data *vc, struct fb_info *info, oldc = vc->vc_video_erase_char; vc->vc_video_erase_char &= charmask; - fbcon_clear(vc, 0, 0, vc->vc_rows, vc->vc_cols); + __fbcon_clear(vc, 0, 0, vc->vc_rows, vc->vc_cols); vc->vc_video_erase_char = oldc; } } @@ -2469,7 +2483,7 @@ static int fbcon_set_font(struct vc_data *vc, struct console_font *font, unsigned charcount = font->charcount; int w = font->width; int h = font->height; - int size; + int size, alloc_size; int i, csum; u8 *new_data, *data = font->data; int pitch = PITCH(font->width); @@ -2496,9 +2510,16 @@ static int fbcon_set_font(struct vc_data *vc, struct console_font *font, if (fbcon_invalid_charcount(info, charcount)) return -EINVAL; - size = CALC_FONTSZ(h, pitch, charcount); + /* Check for integer overflow in font size calculation */ + if (check_mul_overflow(h, pitch, &size) || + check_mul_overflow(size, charcount, &size)) + return -EINVAL; - new_data = kmalloc(FONT_EXTRA_WORDS * sizeof(int) + size, GFP_USER); + /* Check for overflow in allocation size calculation */ + if (check_add_overflow(FONT_EXTRA_WORDS * sizeof(int), size, &alloc_size)) + return -EINVAL; + + new_data = kmalloc(alloc_size, GFP_USER); if (!new_data) return -ENOMEM; @@ -2803,6 +2824,25 @@ int fbcon_mode_deleted(struct fb_info *info, return found; } +static void fbcon_delete_mode(struct fb_videomode *m) +{ + struct fbcon_display *p; + + for (int i = first_fb_vc; i <= last_fb_vc; i++) { + p = &fb_display[i]; + if (p->mode == m) + p->mode = NULL; + } +} + +void fbcon_delete_modelist(struct list_head *head) +{ + struct fb_modelist *modelist; + + list_for_each_entry(modelist, head, list) + fbcon_delete_mode(&modelist->mode); +} + #ifdef CONFIG_VT_HW_CONSOLE_BINDING static void fbcon_unbind(void) { diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h index 0eaf54a211516..25691d4b027bf 100644 --- a/drivers/video/fbdev/core/fbcon.h +++ b/drivers/video/fbdev/core/fbcon.h @@ -55,7 +55,7 @@ struct fbcon_ops { void (*bmove)(struct vc_data *vc, struct fb_info *info, int sy, int sx, int dy, int dx, int height, int width); void (*clear)(struct vc_data *vc, struct fb_info *info, int sy, - int sx, int height, int width); + int sx, int height, int width, int fb, int bg); void (*putcs)(struct vc_data *vc, struct fb_info *info, const unsigned short *s, int count, int yy, int xx, int fg, int bg); @@ -116,42 +116,6 @@ static inline int mono_col(const struct fb_info *info) return (~(0xfff << max_len)) & 0xff; } -static inline int attr_col_ec(int shift, struct vc_data *vc, - struct fb_info *info, int is_fg) -{ - int is_mono01; - int col; - int fg; - int bg; - - if (!vc) - return 0; - - if (vc->vc_can_do_color) - return is_fg ? attr_fgcol(shift,vc->vc_video_erase_char) - : attr_bgcol(shift,vc->vc_video_erase_char); - - if (!info) - return 0; - - col = mono_col(info); - is_mono01 = info->fix.visual == FB_VISUAL_MONO01; - - if (attr_reverse(vc->vc_video_erase_char)) { - fg = is_mono01 ? col : 0; - bg = is_mono01 ? 0 : col; - } - else { - fg = is_mono01 ? 0 : col; - bg = is_mono01 ? col : 0; - } - - return is_fg ? fg : bg; -} - -#define attr_bgcol_ec(bgshift, vc, info) attr_col_ec(bgshift, vc, info, 0) -#define attr_fgcol_ec(fgshift, vc, info) attr_col_ec(fgshift, vc, info, 1) - /* * Scroll Method */ diff --git a/drivers/video/fbdev/core/fbcon_ccw.c b/drivers/video/fbdev/core/fbcon_ccw.c index 2789ace796342..9f4d65478554a 100644 --- a/drivers/video/fbdev/core/fbcon_ccw.c +++ b/drivers/video/fbdev/core/fbcon_ccw.c @@ -78,14 +78,13 @@ static void ccw_bmove(struct vc_data *vc, struct fb_info *info, int sy, } static void ccw_clear(struct vc_data *vc, struct fb_info *info, int sy, - int sx, int height, int width) + int sx, int height, int width, int fg, int bg) { struct fbcon_ops *ops = info->fbcon_par; struct fb_fillrect region; - int bgshift = (vc->vc_hi_font_mask) ? 13 : 12; u32 vyres = GETVYRES(ops->p, info); - region.color = attr_bgcol_ec(bgshift,vc,info); + region.color = bg; region.dx = sy * vc->vc_font.height; region.dy = vyres - ((sx + width) * vc->vc_font.width); region.height = width * vc->vc_font.width; diff --git a/drivers/video/fbdev/core/fbcon_cw.c b/drivers/video/fbdev/core/fbcon_cw.c index 86a254c1b2b7b..b18e31886da10 100644 --- a/drivers/video/fbdev/core/fbcon_cw.c +++ b/drivers/video/fbdev/core/fbcon_cw.c @@ -63,14 +63,13 @@ static void cw_bmove(struct vc_data *vc, struct fb_info *info, int sy, } static void cw_clear(struct vc_data *vc, struct fb_info *info, int sy, - int sx, int height, int width) + int sx, int height, int width, int fg, int bg) { struct fbcon_ops *ops = info->fbcon_par; struct fb_fillrect region; - int bgshift = (vc->vc_hi_font_mask) ? 13 : 12; u32 vxres = GETVXRES(ops->p, info); - region.color = attr_bgcol_ec(bgshift,vc,info); + region.color = bg; region.dx = vxres - ((sy + height) * vc->vc_font.height); region.dy = sx * vc->vc_font.width; region.height = width * vc->vc_font.width; diff --git a/drivers/video/fbdev/core/fbcon_ud.c b/drivers/video/fbdev/core/fbcon_ud.c index 23bc045769d08..b6b074cfd9dc0 100644 --- a/drivers/video/fbdev/core/fbcon_ud.c +++ b/drivers/video/fbdev/core/fbcon_ud.c @@ -64,15 +64,14 @@ static void ud_bmove(struct vc_data *vc, struct fb_info *info, int sy, } static void ud_clear(struct vc_data *vc, struct fb_info *info, int sy, - int sx, int height, int width) + int sx, int height, int width, int fg, int bg) { struct fbcon_ops *ops = info->fbcon_par; struct fb_fillrect region; - int bgshift = (vc->vc_hi_font_mask) ? 13 : 12; u32 vyres = GETVYRES(ops->p, info); u32 vxres = GETVXRES(ops->p, info); - region.color = attr_bgcol_ec(bgshift,vc,info); + region.color = bg; region.dy = vyres - ((sy + height) * vc->vc_font.height); region.dx = vxres - ((sx + width) * vc->vc_font.width); region.width = width * vc->vc_font.width; diff --git a/drivers/video/fbdev/core/fbcvt.c b/drivers/video/fbdev/core/fbcvt.c index 64843464c6613..cd3821bd82e56 100644 --- a/drivers/video/fbdev/core/fbcvt.c +++ b/drivers/video/fbdev/core/fbcvt.c @@ -312,7 +312,7 @@ int fb_find_mode_cvt(struct fb_videomode *mode, int margins, int rb) cvt.f_refresh = cvt.refresh; cvt.interlace = 1; - if (!cvt.xres || !cvt.yres || !cvt.refresh) { + if (!cvt.xres || !cvt.yres || !cvt.refresh || cvt.f_refresh > INT_MAX) { printk(KERN_INFO "fbcvt: Invalid input parameters\n"); return 1; } diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c index ee44a46a66be1..34b8e93f89a43 100644 --- a/drivers/video/fbdev/core/fbmem.c +++ b/drivers/video/fbdev/core/fbmem.c @@ -868,8 +868,10 @@ fb_set_var(struct fb_info *info, struct fb_var_screeninfo *var) !list_empty(&info->modelist)) ret = fb_add_videomode(&mode, &info->modelist); - if (ret) + if (ret) { + info->var = old_var; return ret; + } event.info = info; event.data = &mode; @@ -928,7 +930,7 @@ static int fb_check_foreignness(struct fb_info *fi) static int do_register_framebuffer(struct fb_info *fb_info) { - int i; + int i, err = 0; struct fb_videomode mode; if (fb_check_foreignness(fb_info)) @@ -937,10 +939,21 @@ static int do_register_framebuffer(struct fb_info *fb_info) if (num_registered_fb == FB_MAX) return -ENXIO; - num_registered_fb++; for (i = 0 ; i < FB_MAX; i++) if (!registered_fb[i]) break; + + if (i >= FB_MAX) + return -ENXIO; + + if (!fb_info->modelist.prev || !fb_info->modelist.next) + INIT_LIST_HEAD(&fb_info->modelist); + + fb_var_to_videomode(&mode, &fb_info->var); + err = fb_add_videomode(&mode, &fb_info->modelist); + if (err < 0) + return err; + fb_info->node = i; refcount_set(&fb_info->count, 1); mutex_init(&fb_info->lock); @@ -966,16 +979,12 @@ static int do_register_framebuffer(struct fb_info *fb_info) if (!fb_info->pixmap.blit_y) fb_info->pixmap.blit_y = ~(u32)0; - if (!fb_info->modelist.prev || !fb_info->modelist.next) - INIT_LIST_HEAD(&fb_info->modelist); - if (fb_info->skip_vt_switch) pm_vt_switch_required(fb_info->device, false); else pm_vt_switch_required(fb_info->device, true); - fb_var_to_videomode(&mode, &fb_info->var); - fb_add_videomode(&mode, &fb_info->modelist); + num_registered_fb++; registered_fb[i] = fb_info; #ifdef CONFIG_GUMSTIX_AM200EPD @@ -1021,6 +1030,7 @@ static void do_unregister_framebuffer(struct fb_info *fb_info) fb_info->pixmap.addr = NULL; } + fbcon_delete_modelist(&fb_info->modelist); fb_destroy_modelist(&fb_info->modelist); registered_fb[fb_info->node] = NULL; num_registered_fb--; diff --git a/drivers/video/fbdev/core/tileblit.c b/drivers/video/fbdev/core/tileblit.c index 2768eff247ba4..b3aa0c6620c7d 100644 --- a/drivers/video/fbdev/core/tileblit.c +++ b/drivers/video/fbdev/core/tileblit.c @@ -32,16 +32,14 @@ static void tile_bmove(struct vc_data *vc, struct fb_info *info, int sy, } static void tile_clear(struct vc_data *vc, struct fb_info *info, int sy, - int sx, int height, int width) + int sx, int height, int width, int fg, int bg) { struct fb_tilerect rect; - int bgshift = (vc->vc_hi_font_mask) ? 13 : 12; - int fgshift = (vc->vc_hi_font_mask) ? 9 : 8; rect.index = vc->vc_video_erase_char & ((vc->vc_hi_font_mask) ? 0x1ff : 0xff); - rect.fg = attr_fgcol_ec(fgshift, vc, info); - rect.bg = attr_bgcol_ec(bgshift, vc, info); + rect.fg = fg; + rect.bg = bg; rect.sx = sx; rect.sy = sy; rect.width = width; @@ -76,7 +74,42 @@ static void tile_putcs(struct vc_data *vc, struct fb_info *info, static void tile_clear_margins(struct vc_data *vc, struct fb_info *info, int color, int bottom_only) { - return; + unsigned int cw = vc->vc_font.width; + unsigned int ch = vc->vc_font.height; + unsigned int rw = info->var.xres - (vc->vc_cols*cw); + unsigned int bh = info->var.yres - (vc->vc_rows*ch); + unsigned int rs = info->var.xres - rw; + unsigned int bs = info->var.yres - bh; + unsigned int vwt = info->var.xres_virtual / cw; + unsigned int vht = info->var.yres_virtual / ch; + struct fb_tilerect rect; + + rect.index = vc->vc_video_erase_char & + ((vc->vc_hi_font_mask) ? 0x1ff : 0xff); + rect.fg = color; + rect.bg = color; + + if ((int) rw > 0 && !bottom_only) { + rect.sx = (info->var.xoffset + rs + cw - 1) / cw; + rect.sy = 0; + rect.width = (rw + cw - 1) / cw; + rect.height = vht; + if (rect.width + rect.sx > vwt) + rect.width = vwt - rect.sx; + if (rect.sx < vwt) + info->tileops->fb_tilefill(info, &rect); + } + + if ((int) bh > 0) { + rect.sx = info->var.xoffset / cw; + rect.sy = (info->var.yoffset + bs) / ch; + rect.width = rs / cw; + rect.height = (bh + ch - 1) / ch; + if (rect.height + rect.sy > vht) + rect.height = vht - rect.sy; + if (rect.sy < vht) + info->tileops->fb_tilefill(info, &rect); + } } static void tile_cursor(struct vc_data *vc, struct fb_info *info, int mode, diff --git a/drivers/video/fbdev/fsl-diu-fb.c b/drivers/video/fbdev/fsl-diu-fb.c index 0bced82fa4940..8cf1268a4e554 100644 --- a/drivers/video/fbdev/fsl-diu-fb.c +++ b/drivers/video/fbdev/fsl-diu-fb.c @@ -1827,6 +1827,7 @@ static void fsl_diu_remove(struct platform_device *pdev) int i; data = dev_get_drvdata(&pdev->dev); + device_remove_file(&pdev->dev, &data->dev_attr); disable_lcdc(&data->fsl_diu_info[0]); free_irq(data->irq, data->diu_reg); diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c index 7042a43b81d85..5f9e98423d656 100644 --- a/drivers/video/fbdev/imxfb.c +++ b/drivers/video/fbdev/imxfb.c @@ -1004,8 +1004,13 @@ static int imxfb_probe(struct platform_device *pdev) info->fix.smem_start = fbi->map_dma; INIT_LIST_HEAD(&info->modelist); - for (i = 0; i < fbi->num_modes; i++) - fb_add_videomode(&fbi->mode[i].mode, &info->modelist); + for (i = 0; i < fbi->num_modes; i++) { + ret = fb_add_videomode(&fbi->mode[i].mode, &info->modelist); + if (ret) { + dev_err(&pdev->dev, "Failed to add videomode\n"); + goto failed_cmap; + } + } /* * This makes sure that our colour bitfield diff --git a/drivers/video/fbdev/pvr2fb.c b/drivers/video/fbdev/pvr2fb.c index 6307364e4a49c..7ae37f91b42c7 100644 --- a/drivers/video/fbdev/pvr2fb.c +++ b/drivers/video/fbdev/pvr2fb.c @@ -192,7 +192,7 @@ static unsigned long pvr2fb_map; #ifdef CONFIG_PVR2_DMA static unsigned int shdma = PVR2_CASCADE_CHAN; -static unsigned int pvr2dma = ONCHIP_NR_DMA_CHANNELS; +static unsigned int pvr2dma = CONFIG_NR_ONCHIP_DMA_CHANNELS; #endif static struct fb_videomode pvr2_modedb[] = { diff --git a/drivers/video/fbdev/valkyriefb.c b/drivers/video/fbdev/valkyriefb.c index 91d070ef69897..6ff059ee16941 100644 --- a/drivers/video/fbdev/valkyriefb.c +++ b/drivers/video/fbdev/valkyriefb.c @@ -329,11 +329,13 @@ static int __init valkyriefb_init(void) if (of_address_to_resource(dp, 0, &r)) { printk(KERN_ERR "can't find address for valkyrie\n"); + of_node_put(dp); return 0; } frame_buffer_phys = r.start; cmap_regs_phys = r.start + 0x304000; + of_node_put(dp); } #endif /* ppc (!CONFIG_MAC) */ diff --git a/drivers/video/screen_info_pci.c b/drivers/video/screen_info_pci.c index 6c58335171410..66bfc1d0a6dc8 100644 --- a/drivers/video/screen_info_pci.c +++ b/drivers/video/screen_info_pci.c @@ -7,8 +7,8 @@ static struct pci_dev *screen_info_lfb_pdev; static size_t screen_info_lfb_bar; -static resource_size_t screen_info_lfb_offset; -static struct resource screen_info_lfb_res = DEFINE_RES_MEM(0, 0); +static resource_size_t screen_info_lfb_res_start; // original start of resource +static resource_size_t screen_info_lfb_offset; // framebuffer offset within resource static bool __screen_info_relocation_is_valid(const struct screen_info *si, struct resource *pr) { @@ -31,7 +31,7 @@ void screen_info_apply_fixups(void) if (screen_info_lfb_pdev) { struct resource *pr = &screen_info_lfb_pdev->resource[screen_info_lfb_bar]; - if (pr->start != screen_info_lfb_res.start) { + if (pr->start != screen_info_lfb_res_start) { if (__screen_info_relocation_is_valid(si, pr)) { /* * Only update base if we have an actual @@ -47,46 +47,67 @@ void screen_info_apply_fixups(void) } } +static int __screen_info_lfb_pci_bus_region(const struct screen_info *si, unsigned int type, + struct pci_bus_region *r) +{ + u64 base, size; + + base = __screen_info_lfb_base(si); + if (!base) + return -EINVAL; + + size = __screen_info_lfb_size(si, type); + if (!size) + return -EINVAL; + + r->start = base; + r->end = base + size - 1; + + return 0; +} + static void screen_info_fixup_lfb(struct pci_dev *pdev) { unsigned int type; - struct resource res[SCREEN_INFO_MAX_RESOURCES]; - size_t i, numres; + struct pci_bus_region bus_region; int ret; + struct resource r = { + .flags = IORESOURCE_MEM, + }; + const struct resource *pr; const struct screen_info *si = &screen_info; if (screen_info_lfb_pdev) return; // already found type = screen_info_video_type(si); - if (type != VIDEO_TYPE_EFI) - return; // only applies to EFI + if (!__screen_info_has_lfb(type)) + return; // only applies to EFI; maybe VESA - ret = screen_info_resources(si, res, ARRAY_SIZE(res)); + ret = __screen_info_lfb_pci_bus_region(si, type, &bus_region); if (ret < 0) return; - numres = ret; - for (i = 0; i < numres; ++i) { - struct resource *r = &res[i]; - const struct resource *pr; - - if (!(r->flags & IORESOURCE_MEM)) - continue; - pr = pci_find_resource(pdev, r); - if (!pr) - continue; - - /* - * We've found a PCI device with the framebuffer - * resource. Store away the parameters to track - * relocation of the framebuffer aperture. - */ - screen_info_lfb_pdev = pdev; - screen_info_lfb_bar = pr - pdev->resource; - screen_info_lfb_offset = r->start - pr->start; - memcpy(&screen_info_lfb_res, r, sizeof(screen_info_lfb_res)); - } + /* + * Translate the PCI bus address to resource. Account + * for an offset if the framebuffer is behind a PCI host + * bridge. + */ + pcibios_bus_to_resource(pdev->bus, &r, &bus_region); + + pr = pci_find_resource(pdev, &r); + if (!pr) + return; + + /* + * We've found a PCI device with the framebuffer + * resource. Store away the parameters to track + * relocation of the framebuffer aperture. + */ + screen_info_lfb_pdev = pdev; + screen_info_lfb_bar = pr - pdev->resource; + screen_info_lfb_offset = r.start - pr->start; + screen_info_lfb_res_start = bus_region.start; } DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY, 16, screen_info_fixup_lfb); diff --git a/drivers/virt/coco/efi_secret/efi_secret.c b/drivers/virt/coco/efi_secret/efi_secret.c index e700a5ef70431..d996feb0509a1 100644 --- a/drivers/virt/coco/efi_secret/efi_secret.c +++ b/drivers/virt/coco/efi_secret/efi_secret.c @@ -136,15 +136,7 @@ static int efi_secret_unlink(struct inode *dir, struct dentry *dentry) if (s->fs_files[i] == dentry) s->fs_files[i] = NULL; - /* - * securityfs_remove tries to lock the directory's inode, but we reach - * the unlink callback when it's already locked - */ - inode_unlock(dir); - securityfs_remove(dentry); - inode_lock(dir); - - return 0; + return simple_unlink(inode, dentry); } static const struct inode_operations efi_secret_dir_inode_operations = { diff --git a/drivers/virt/geniezone/gzvm_ioeventfd.c b/drivers/virt/geniezone/gzvm_ioeventfd.c index ea8c638994cce..bbfab158033bd 100644 --- a/drivers/virt/geniezone/gzvm_ioeventfd.c +++ b/drivers/virt/geniezone/gzvm_ioeventfd.c @@ -279,3 +279,13 @@ int gzvm_init_ioeventfd(struct gzvm *gzvm) return 0; } + +void gzvm_vm_ioeventfd_release(struct gzvm *gzvm) +{ + struct gzvm_ioevent *p, *tmp; + + mutex_lock(&gzvm->ioevent_lock); + list_for_each_entry_safe(p, tmp, &gzvm->ioevents, list) + gzvm_ioevent_release(p); + mutex_unlock(&gzvm->ioevent_lock); +} diff --git a/drivers/virt/geniezone/gzvm_irqfd.c b/drivers/virt/geniezone/gzvm_irqfd.c index 8095a5a68fd8c..086ade86ee7c3 100644 --- a/drivers/virt/geniezone/gzvm_irqfd.c +++ b/drivers/virt/geniezone/gzvm_irqfd.c @@ -354,6 +354,8 @@ void gzvm_vm_irqfd_release(struct gzvm *gzvm) * Block until we know all outstanding shutdown jobs have completed. */ flush_workqueue(irqfd_cleanup_wq); + + cleanup_srcu_struct(&gzvm->irq_srcu); } /** diff --git a/drivers/virt/geniezone/gzvm_vcpu.c b/drivers/virt/geniezone/gzvm_vcpu.c index 1fb71a3c32fde..180374e5c409a 100644 --- a/drivers/virt/geniezone/gzvm_vcpu.c +++ b/drivers/virt/geniezone/gzvm_vcpu.c @@ -232,7 +232,16 @@ static long gzvm_vcpu_ioctl(struct file *filp, unsigned int ioctl, return ret; } +static int gzvm_vcpu_release(struct inode *inode, struct file *filp) +{ + struct gzvm_vcpu *vcpu = filp->private_data; + + gzvm_vm_put(vcpu->gzvm); + return 0; +} + static const struct file_operations gzvm_vcpu_fops = { + .release = gzvm_vcpu_release, .unlocked_ioctl = gzvm_vcpu_ioctl, .llseek = noop_llseek, }; @@ -288,6 +297,8 @@ int gzvm_vm_ioctl_create_vcpu(struct gzvm *gzvm, u32 cpuid) struct gzvm_vcpu *vcpu; int ret; + gzvm_vm_get(gzvm); + if (cpuid >= GZVM_MAX_VCPUS) return -EINVAL; diff --git a/drivers/virt/geniezone/gzvm_vm.c b/drivers/virt/geniezone/gzvm_vm.c index 548ec046c3314..efe3a44f0657f 100644 --- a/drivers/virt/geniezone/gzvm_vm.c +++ b/drivers/virt/geniezone/gzvm_vm.c @@ -360,6 +360,7 @@ static void gzvm_destroy_vm(struct gzvm *gzvm) mutex_lock(&gzvm->lock); gzvm_vm_irqfd_release(gzvm); + gzvm_vm_ioeventfd_release(gzvm); gzvm_destroy_vcpus(gzvm); gzvm_arch_destroy_vm(gzvm->vm_id, gzvm->gzvm_drv->destroy_batch_pages); @@ -385,11 +386,29 @@ static void gzvm_destroy_vm(struct gzvm *gzvm) kfree(gzvm); } +static void __gzvm_vm_put(struct kref *kref) +{ + struct gzvm *gzvm = container_of(kref, struct gzvm, kref); + + gzvm_destroy_vm(gzvm); +} + +void gzvm_vm_put(struct gzvm *gzvm) +{ + kref_put(&gzvm->kref, __gzvm_vm_put); +} + +void gzvm_vm_get(struct gzvm *gzvm) +{ + kref_get(&gzvm->kref); +} + static int gzvm_vm_release(struct inode *inode, struct file *filp) { struct gzvm *gzvm = filp->private_data; - gzvm_destroy_vm(gzvm); + gzvm_vm_put(gzvm); + return 0; } @@ -597,6 +616,8 @@ static struct gzvm *gzvm_create_vm(struct gzvm_driver *drv, unsigned long vm_typ mutex_init(&gzvm->mem_lock); gzvm->pinned_pages = RB_ROOT; + kref_init(&gzvm->kref); + ret = gzvm_vm_irqfd_init(gzvm); if (ret) { pr_err("Failed to initialize irqfd\n"); diff --git a/drivers/virt/gunyah/gunyah_qcom.c b/drivers/virt/gunyah/gunyah_qcom.c index f2342d51a0184..622d6a07db02c 100644 --- a/drivers/virt/gunyah/gunyah_qcom.c +++ b/drivers/virt/gunyah/gunyah_qcom.c @@ -187,7 +187,7 @@ static bool gunyah_has_qcom_extensions(void) uuid_t uuid; u32 *up; - arm_smccc_1_1_smc(GUNYAH_QCOM_EXT_CALL_UUID_ID, &res); + arm_smccc_1_1_invoke(GUNYAH_QCOM_EXT_CALL_UUID_ID, &res); up = (u32 *)&uuid.b[0]; up[0] = lower_32_bits(res.a0); diff --git a/drivers/virt/gunyah/gunyah_vcpu.c b/drivers/virt/gunyah/gunyah_vcpu.c index b3509be57d9e7..975bb9fa12524 100644 --- a/drivers/virt/gunyah/gunyah_vcpu.c +++ b/drivers/virt/gunyah/gunyah_vcpu.c @@ -320,6 +320,8 @@ static int gunyah_vcpu_run(struct gunyah_vcpu *vcpu) schedule(); break; case GUNYAH_VCPU_STATE_POWERED_OFF: + fallthrough; + case GUNYAH_VCPU_STATE_SYSTEM_OFF: /** * vcpu might be off because the VM is shut down * If so, it won't ever run again @@ -369,8 +371,8 @@ static int gunyah_vcpu_run(struct gunyah_vcpu *vcpu) pr_warn_ratelimited( "Unknown vCPU state: %llx\n", vcpu_run_resp.sized_state); - schedule(); - break; + ret = -EINVAL; + goto out; } } else if (gunyah_error == GUNYAH_ERROR_RETRY) { schedule(); diff --git a/drivers/virtio/virtio_dma_buf.c b/drivers/virtio/virtio_dma_buf.c index 19a843fad6c6b..2521a75009c3a 100644 --- a/drivers/virtio/virtio_dma_buf.c +++ b/drivers/virtio/virtio_dma_buf.c @@ -25,14 +25,11 @@ struct dma_buf *virtio_dma_buf_export const struct virtio_dma_buf_ops, ops); if (!exp_info->ops || + exp_info->ops->attach != &virtio_dma_buf_attach || !virtio_ops->get_uuid) { return ERR_PTR(-EINVAL); } - if (!(IS_ENABLED(CONFIG_CFI_CLANG) && IS_ENABLED(CONFIG_MODULES)) && - exp_info->ops->attach != &virtio_dma_buf_attach) - return ERR_PTR(-EINVAL); - return dma_buf_export(exp_info); } EXPORT_SYMBOL(virtio_dma_buf_export); @@ -63,9 +60,6 @@ EXPORT_SYMBOL(virtio_dma_buf_attach); */ bool is_virtio_dma_buf(struct dma_buf *dma_buf) { - if (IS_ENABLED(CONFIG_CFI_CLANG) && IS_ENABLED(CONFIG_MODULES)) - return true; - return dma_buf->ops->attach == &virtio_dma_buf_attach; } EXPORT_SYMBOL(is_virtio_dma_buf); diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 594c45131e55e..63c50180f9edc 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2530,7 +2530,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) struct vring_virtqueue *vq = to_vvq(_vq); if (vq->event_triggered) - vq->event_triggered = false; + data_race(vq->event_triggered = false); return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) : virtqueue_enable_cb_delayed_split(_vq); @@ -2739,7 +2739,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, void (*recycle)(struct virtqueue *vq, void *buf)) { struct vring_virtqueue *vq = to_vvq(_vq); - int err; + int err, err_reset; if (num > vq->vq.num_max) return -E2BIG; @@ -2759,7 +2759,11 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, else err = virtqueue_resize_split(_vq, num); - return virtqueue_enable_after_reset(_vq); + err_reset = virtqueue_enable_after_reset(_vq); + if (err_reset) + return err_reset; + + return err; } EXPORT_SYMBOL_GPL(virtqueue_resize); diff --git a/drivers/watchdog/aspeed_wdt.c b/drivers/watchdog/aspeed_wdt.c index b72a858bbac70..001b2c9311254 100644 --- a/drivers/watchdog/aspeed_wdt.c +++ b/drivers/watchdog/aspeed_wdt.c @@ -11,21 +11,30 @@ #include #include #include +#include #include #include #include #include +#include #include static bool nowayout = WATCHDOG_NOWAYOUT; module_param(nowayout, bool, 0); MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); +struct aspeed_wdt_scu { + const char *compatible; + u32 reset_status_reg; + u32 wdt_reset_mask; + u32 wdt_reset_mask_shift; +}; struct aspeed_wdt_config { u32 ext_pulse_width_mask; u32 irq_shift; u32 irq_mask; + struct aspeed_wdt_scu scu; }; struct aspeed_wdt { @@ -39,18 +48,36 @@ static const struct aspeed_wdt_config ast2400_config = { .ext_pulse_width_mask = 0xff, .irq_shift = 0, .irq_mask = 0, + .scu = { + .compatible = "aspeed,ast2400-scu", + .reset_status_reg = 0x3c, + .wdt_reset_mask = 0x1, + .wdt_reset_mask_shift = 1, + }, }; static const struct aspeed_wdt_config ast2500_config = { .ext_pulse_width_mask = 0xfffff, .irq_shift = 12, .irq_mask = GENMASK(31, 12), + .scu = { + .compatible = "aspeed,ast2500-scu", + .reset_status_reg = 0x3c, + .wdt_reset_mask = 0x1, + .wdt_reset_mask_shift = 2, + }, }; static const struct aspeed_wdt_config ast2600_config = { .ext_pulse_width_mask = 0xfffff, .irq_shift = 0, .irq_mask = GENMASK(31, 10), + .scu = { + .compatible = "aspeed,ast2600-scu", + .reset_status_reg = 0x74, + .wdt_reset_mask = 0xf, + .wdt_reset_mask_shift = 16, + }, }; static const struct of_device_id aspeed_wdt_of_table[] = { @@ -211,6 +238,56 @@ static int aspeed_wdt_restart(struct watchdog_device *wdd, return 0; } +static void aspeed_wdt_update_bootstatus(struct platform_device *pdev, + struct aspeed_wdt *wdt) +{ + const struct resource *res; + struct aspeed_wdt_scu scu = wdt->cfg->scu; + struct regmap *scu_base; + u32 reset_mask_width; + u32 reset_mask_shift; + u32 idx = 0; + u32 status; + int ret; + + if (!of_device_is_compatible(pdev->dev.of_node, "aspeed,ast2400-wdt")) { + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + idx = ((intptr_t)wdt->base & 0x00000fff) / (uintptr_t)resource_size(res); + } + + scu_base = syscon_regmap_lookup_by_compatible(scu.compatible); + if (IS_ERR(scu_base)) { + wdt->wdd.bootstatus = WDIOS_UNKNOWN; + return; + } + + ret = regmap_read(scu_base, scu.reset_status_reg, &status); + if (ret) { + wdt->wdd.bootstatus = WDIOS_UNKNOWN; + return; + } + + reset_mask_width = hweight32(scu.wdt_reset_mask); + reset_mask_shift = scu.wdt_reset_mask_shift + + reset_mask_width * idx; + + if (status & (scu.wdt_reset_mask << reset_mask_shift)) + wdt->wdd.bootstatus = WDIOF_CARDRESET; + + /* clear wdt reset event flag */ + if (of_device_is_compatible(pdev->dev.of_node, "aspeed,ast2400-wdt") || + of_device_is_compatible(pdev->dev.of_node, "aspeed,ast2500-wdt")) { + ret = regmap_read(scu_base, scu.reset_status_reg, &status); + if (!ret) { + status &= ~(scu.wdt_reset_mask << reset_mask_shift); + regmap_write(scu_base, scu.reset_status_reg, status); + } + } else { + regmap_write(scu_base, scu.reset_status_reg, + scu.wdt_reset_mask << reset_mask_shift); + } +} + /* access_cs0 shows if cs0 is accessible, hence the reverted bit */ static ssize_t access_cs0_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -447,10 +524,10 @@ static int aspeed_wdt_probe(struct platform_device *pdev) writel(duration - 1, wdt->base + WDT_RESET_WIDTH); } + aspeed_wdt_update_bootstatus(pdev, wdt); + status = readl(wdt->base + WDT_TIMEOUT_STATUS); if (status & WDT_TIMEOUT_STATUS_BOOT_SECONDARY) { - wdt->wdd.bootstatus = WDIOF_CARDRESET; - if (of_device_is_compatible(np, "aspeed,ast2400-wdt") || of_device_is_compatible(np, "aspeed,ast2500-wdt")) wdt->wdd.groups = bswitch_groups; diff --git a/drivers/watchdog/da9052_wdt.c b/drivers/watchdog/da9052_wdt.c index d708c091bf1b1..180526220d8c4 100644 --- a/drivers/watchdog/da9052_wdt.c +++ b/drivers/watchdog/da9052_wdt.c @@ -164,6 +164,7 @@ static int da9052_wdt_probe(struct platform_device *pdev) da9052_wdt = &driver_data->wdt; da9052_wdt->timeout = DA9052_DEF_TIMEOUT; + da9052_wdt->min_hw_heartbeat_ms = DA9052_TWDMIN; da9052_wdt->info = &da9052_wdt_info; da9052_wdt->ops = &da9052_wdt_ops; da9052_wdt->parent = dev; diff --git a/drivers/watchdog/dw_wdt.c b/drivers/watchdog/dw_wdt.c index 84dca3695f862..e5e6d7f159180 100644 --- a/drivers/watchdog/dw_wdt.c +++ b/drivers/watchdog/dw_wdt.c @@ -644,6 +644,8 @@ static int dw_wdt_drv_probe(struct platform_device *pdev) } else { wdd->timeout = DW_WDT_DEFAULT_SECONDS; watchdog_init_timeout(wdd, 0, dev); + /* Limit timeout value to hardware constraints. */ + dw_wdt_set_timeout(wdd, wdd->timeout); } platform_set_drvdata(pdev, dw_wdt); diff --git a/drivers/watchdog/exar_wdt.c b/drivers/watchdog/exar_wdt.c index 7c61ff3432711..c2e3bb08df899 100644 --- a/drivers/watchdog/exar_wdt.c +++ b/drivers/watchdog/exar_wdt.c @@ -221,7 +221,7 @@ static const struct watchdog_info exar_wdt_info = { .options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE, - .identity = "Exar/MaxLinear XR28V38x Watchdog", + .identity = "Exar XR28V38x Watchdog", }; static const struct watchdog_ops exar_wdt_ops = { diff --git a/drivers/watchdog/iTCO_wdt.c b/drivers/watchdog/iTCO_wdt.c index dd297dcd524c9..68973be2ce626 100644 --- a/drivers/watchdog/iTCO_wdt.c +++ b/drivers/watchdog/iTCO_wdt.c @@ -601,7 +601,11 @@ static int iTCO_wdt_probe(struct platform_device *pdev) /* Check that the heartbeat value is within it's range; if not reset to the default */ if (iTCO_wdt_set_timeout(&p->wddev, heartbeat)) { - iTCO_wdt_set_timeout(&p->wddev, WATCHDOG_TIMEOUT); + ret = iTCO_wdt_set_timeout(&p->wddev, WATCHDOG_TIMEOUT); + if (ret != 0) { + dev_err(dev, "Failed to set watchdog timeout (%d)\n", WATCHDOG_TIMEOUT); + return ret; + } dev_info(dev, "timeout value out of range, using %d\n", WATCHDOG_TIMEOUT); } diff --git a/drivers/watchdog/mpc8xxx_wdt.c b/drivers/watchdog/mpc8xxx_wdt.c index 867f9f3113797..a4b497ecfa205 100644 --- a/drivers/watchdog/mpc8xxx_wdt.c +++ b/drivers/watchdog/mpc8xxx_wdt.c @@ -100,6 +100,8 @@ static int mpc8xxx_wdt_start(struct watchdog_device *w) ddata->swtc = tmp >> 16; set_bit(WDOG_HW_RUNNING, &ddata->wdd.status); + mpc8xxx_wdt_keepalive(ddata); + return 0; } diff --git a/drivers/watchdog/s3c2410_wdt.c b/drivers/watchdog/s3c2410_wdt.c index 0b4bd883ff28a..183d77a6c4b31 100644 --- a/drivers/watchdog/s3c2410_wdt.c +++ b/drivers/watchdog/s3c2410_wdt.c @@ -26,6 +26,7 @@ #include #include #include +#include #define S3C2410_WTCON 0x00 #define S3C2410_WTDAT 0x04 @@ -302,9 +303,14 @@ static inline unsigned long s3c2410wdt_get_freq(struct s3c2410_wdt *wdt) static inline unsigned int s3c2410wdt_max_timeout(struct s3c2410_wdt *wdt) { const unsigned long freq = s3c2410wdt_get_freq(wdt); + const u64 n_max = (u64)(S3C2410_WTCON_PRESCALE_MAX + 1) * + S3C2410_WTCON_MAXDIV * S3C2410_WTCNT_MAXCNT; + u64 t_max = div64_ul(n_max, freq); - return S3C2410_WTCNT_MAXCNT / (freq / (S3C2410_WTCON_PRESCALE_MAX + 1) - / S3C2410_WTCON_MAXDIV); + if (t_max > UINT_MAX) + t_max = UINT_MAX; + + return t_max; } static int s3c2410wdt_disable_wdt_reset(struct s3c2410_wdt *wdt, bool mask) diff --git a/drivers/watchdog/sbsa_gwdt.c b/drivers/watchdog/sbsa_gwdt.c index 5f23913ce3b49..6ce1bfb390641 100644 --- a/drivers/watchdog/sbsa_gwdt.c +++ b/drivers/watchdog/sbsa_gwdt.c @@ -75,11 +75,17 @@ #define SBSA_GWDT_VERSION_MASK 0xF #define SBSA_GWDT_VERSION_SHIFT 16 +#define SBSA_GWDT_IMPL_MASK 0x7FF +#define SBSA_GWDT_IMPL_SHIFT 0 +#define SBSA_GWDT_IMPL_MEDIATEK 0x426 + /** * struct sbsa_gwdt - Internal representation of the SBSA GWDT * @wdd: kernel watchdog_device structure * @clk: store the System Counter clock frequency, in Hz. * @version: store the architecture version + * @need_ws0_race_workaround: + * indicate whether to adjust wdd->timeout to avoid a race with WS0 * @refresh_base: Virtual address of the watchdog refresh frame * @control_base: Virtual address of the watchdog control frame */ @@ -87,6 +93,7 @@ struct sbsa_gwdt { struct watchdog_device wdd; u32 clk; int version; + bool need_ws0_race_workaround; void __iomem *refresh_base; void __iomem *control_base; }; @@ -161,6 +168,31 @@ static int sbsa_gwdt_set_timeout(struct watchdog_device *wdd, */ sbsa_gwdt_reg_write(((u64)gwdt->clk / 2) * timeout, gwdt); + /* + * Some watchdog hardware has a race condition where it will ignore + * sbsa_gwdt_keepalive() if it is called at the exact moment that a + * timeout occurs and WS0 is being asserted. Unfortunately, the default + * behavior of the watchdog core is very likely to trigger this race + * when action=0 because it programs WOR to be half of the desired + * timeout, and watchdog_next_keepalive() chooses the exact same time to + * send keepalive pings. + * + * This triggers a race where sbsa_gwdt_keepalive() can be called right + * as WS0 is being asserted, and affected hardware will ignore that + * write and continue to assert WS0. After another (timeout / 2) + * seconds, the same race happens again. If the driver wins then the + * explicit refresh will reset WS0 to false but if the hardware wins, + * then WS1 is asserted and the system resets. + * + * Avoid the problem by scheduling keepalive heartbeats one second later + * than the WOR timeout. + * + * This workaround might not be needed in a future revision of the + * hardware. + */ + if (gwdt->need_ws0_race_workaround) + wdd->min_hw_heartbeat_ms = timeout * 500 + 1000; + return 0; } @@ -202,12 +234,15 @@ static int sbsa_gwdt_keepalive(struct watchdog_device *wdd) static void sbsa_gwdt_get_version(struct watchdog_device *wdd) { struct sbsa_gwdt *gwdt = watchdog_get_drvdata(wdd); - int ver; + int iidr, ver, impl; - ver = readl(gwdt->control_base + SBSA_GWDT_W_IIDR); - ver = (ver >> SBSA_GWDT_VERSION_SHIFT) & SBSA_GWDT_VERSION_MASK; + iidr = readl(gwdt->control_base + SBSA_GWDT_W_IIDR); + ver = (iidr >> SBSA_GWDT_VERSION_SHIFT) & SBSA_GWDT_VERSION_MASK; + impl = (iidr >> SBSA_GWDT_IMPL_SHIFT) & SBSA_GWDT_IMPL_MASK; gwdt->version = ver; + gwdt->need_ws0_race_workaround = + !action && (impl == SBSA_GWDT_IMPL_MEDIATEK); } static int sbsa_gwdt_start(struct watchdog_device *wdd) @@ -299,6 +334,15 @@ static int sbsa_gwdt_probe(struct platform_device *pdev) else wdd->max_hw_heartbeat_ms = GENMASK_ULL(47, 0) / gwdt->clk * 1000; + if (gwdt->need_ws0_race_workaround) { + /* + * A timeout of 3 seconds means that WOR will be set to 1.5 + * seconds and the heartbeat will be scheduled every 2.5 + * seconds. + */ + wdd->min_timeout = 3; + } + status = readl(cf_base + SBSA_GWDT_WCS); if (status & SBSA_GWDT_WCS_WS1) { dev_warn(dev, "System reset by WDT.\n"); diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c index 5ed33df68e9ab..e611d60316c68 100644 --- a/drivers/watchdog/ziirave_wdt.c +++ b/drivers/watchdog/ziirave_wdt.c @@ -302,6 +302,9 @@ static int ziirave_firm_verify(struct watchdog_device *wdd, const u16 len = be16_to_cpu(rec->len); const u32 addr = be32_to_cpu(rec->addr); + if (len > sizeof(data)) + return -EINVAL; + if (ziirave_firm_addr_readonly(addr)) continue; diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index b0b52fa8fba6d..204ec1bcbd526 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -696,15 +696,18 @@ static int __init balloon_add_regions(void) /* * Extra regions are accounted for in the physmap, but need - * decreasing from current_pages to balloon down the initial - * allocation, because they are already accounted for in - * total_pages. + * decreasing from current_pages and target_pages to balloon + * down the initial allocation, because they are already + * accounted for in total_pages. */ - if (extra_pfn_end - start_pfn >= balloon_stats.current_pages) { + pages = extra_pfn_end - start_pfn; + if (pages >= balloon_stats.current_pages || + pages >= balloon_stats.target_pages) { WARN(1, "Extra pages underflow current target"); return -ERANGE; } - balloon_stats.current_pages -= extra_pfn_end - start_pfn; + balloon_stats.current_pages -= pages; + balloon_stats.target_pages -= pages; } return 0; diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index 9e3b5d21d0987..3a8f2e659e008 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -1326,14 +1326,17 @@ int bind_interdomain_evtchn_to_irq_lateeoi(struct xenbus_device *dev, } EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi); -static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn) +static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn, + bool percpu) { struct evtchn_status status; evtchn_port_t port; - int rc = -ENOENT; + bool exists = false; memset(&status, 0, sizeof(status)); for (port = 0; port < xen_evtchn_max_channels(); port++) { + int rc; + status.dom = DOMID_SELF; status.port = port; rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status); @@ -1341,12 +1344,16 @@ static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn) continue; if (status.status != EVTCHNSTAT_virq) continue; - if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) { + if (status.u.virq != virq) + continue; + if (status.vcpu == xen_vcpu_nr(cpu)) { *evtchn = port; - break; + return 0; + } else if (!percpu) { + exists = true; } } - return rc; + return exists ? -EEXIST : -ENOENT; } /** @@ -1393,8 +1400,11 @@ int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu) evtchn = bind_virq.port; else { if (ret == -EEXIST) - ret = find_virq(virq, cpu, &evtchn); - BUG_ON(ret < 0); + ret = find_virq(virq, cpu, &evtchn, percpu); + if (ret) { + __unbind_from_irq(info, info->irq); + goto out; + } } ret = xen_irq_info_virq_setup(info, cpu, evtchn, virq); @@ -1799,9 +1809,20 @@ static int xen_rebind_evtchn_to_cpu(struct irq_info *info, unsigned int tcpu) * virq or IPI channel, which don't actually need to be rebound. Ignore * it, but don't do the xenlinux-level rebind in that case. */ - if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0) + if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0) { + int old_cpu = info->cpu; + bind_evtchn_to_cpu(info, tcpu, false); + if (info->type == IRQT_VIRQ) { + int virq = info->u.virq; + int irq = per_cpu(virq_to_irq, old_cpu)[virq]; + + per_cpu(virq_to_irq, old_cpu)[virq] = -1; + per_cpu(virq_to_irq, tcpu)[virq] = irq; + } + } + do_unmask(info, EVT_MASK_REASON_TEMPORARY); return 0; diff --git a/drivers/xen/gntdev-common.h b/drivers/xen/gntdev-common.h index 9c286b2a19001..ac8ce3179ba2e 100644 --- a/drivers/xen/gntdev-common.h +++ b/drivers/xen/gntdev-common.h @@ -26,6 +26,10 @@ struct gntdev_priv { /* lock protects maps and freeable_maps. */ struct mutex lock; + /* Free instances of struct gntdev_copy_batch. */ + struct gntdev_copy_batch *batch; + struct mutex batch_lock; + #ifdef CONFIG_XEN_GRANT_DMA_ALLOC /* Device for which DMA memory is allocated. */ struct device *dma_dev; diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 61faea1f06630..1f21607656182 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -56,6 +56,18 @@ MODULE_AUTHOR("Derek G. Murray , " "Gerd Hoffmann "); MODULE_DESCRIPTION("User-space granted page access driver"); +#define GNTDEV_COPY_BATCH 16 + +struct gntdev_copy_batch { + struct gnttab_copy ops[GNTDEV_COPY_BATCH]; + struct page *pages[GNTDEV_COPY_BATCH]; + s16 __user *status[GNTDEV_COPY_BATCH]; + unsigned int nr_ops; + unsigned int nr_pages; + bool writeable; + struct gntdev_copy_batch *next; +}; + static unsigned int limit = 64*1024; module_param(limit, uint, 0644); MODULE_PARM_DESC(limit, @@ -584,6 +596,8 @@ static int gntdev_open(struct inode *inode, struct file *flip) INIT_LIST_HEAD(&priv->maps); mutex_init(&priv->lock); + mutex_init(&priv->batch_lock); + #ifdef CONFIG_XEN_GNTDEV_DMABUF priv->dmabuf_priv = gntdev_dmabuf_init(flip); if (IS_ERR(priv->dmabuf_priv)) { @@ -608,6 +622,7 @@ static int gntdev_release(struct inode *inode, struct file *flip) { struct gntdev_priv *priv = flip->private_data; struct gntdev_grant_map *map; + struct gntdev_copy_batch *batch; pr_debug("priv %p\n", priv); @@ -620,6 +635,14 @@ static int gntdev_release(struct inode *inode, struct file *flip) } mutex_unlock(&priv->lock); + mutex_lock(&priv->batch_lock); + while (priv->batch) { + batch = priv->batch; + priv->batch = batch->next; + kfree(batch); + } + mutex_unlock(&priv->batch_lock); + #ifdef CONFIG_XEN_GNTDEV_DMABUF gntdev_dmabuf_fini(priv->dmabuf_priv); #endif @@ -785,17 +808,6 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u) return rc; } -#define GNTDEV_COPY_BATCH 16 - -struct gntdev_copy_batch { - struct gnttab_copy ops[GNTDEV_COPY_BATCH]; - struct page *pages[GNTDEV_COPY_BATCH]; - s16 __user *status[GNTDEV_COPY_BATCH]; - unsigned int nr_ops; - unsigned int nr_pages; - bool writeable; -}; - static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, unsigned long *gfn) { @@ -953,36 +965,53 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, static long gntdev_ioctl_grant_copy(struct gntdev_priv *priv, void __user *u) { struct ioctl_gntdev_grant_copy copy; - struct gntdev_copy_batch batch; + struct gntdev_copy_batch *batch; unsigned int i; int ret = 0; if (copy_from_user(©, u, sizeof(copy))) return -EFAULT; - batch.nr_ops = 0; - batch.nr_pages = 0; + mutex_lock(&priv->batch_lock); + if (!priv->batch) { + batch = kmalloc(sizeof(*batch), GFP_KERNEL); + } else { + batch = priv->batch; + priv->batch = batch->next; + } + mutex_unlock(&priv->batch_lock); + if (!batch) + return -ENOMEM; + + batch->nr_ops = 0; + batch->nr_pages = 0; for (i = 0; i < copy.count; i++) { struct gntdev_grant_copy_segment seg; if (copy_from_user(&seg, ©.segments[i], sizeof(seg))) { ret = -EFAULT; + gntdev_put_pages(batch); goto out; } - ret = gntdev_grant_copy_seg(&batch, &seg, ©.segments[i].status); - if (ret < 0) + ret = gntdev_grant_copy_seg(batch, &seg, ©.segments[i].status); + if (ret < 0) { + gntdev_put_pages(batch); goto out; + } cond_resched(); } - if (batch.nr_ops) - ret = gntdev_copy(&batch); - return ret; + if (batch->nr_ops) + ret = gntdev_copy(batch); + + out: + mutex_lock(&priv->batch_lock); + batch->next = priv->batch; + priv->batch = batch; + mutex_unlock(&priv->batch_lock); - out: - gntdev_put_pages(&batch); return ret; } diff --git a/drivers/xen/manage.c b/drivers/xen/manage.c index c16df629907e1..55537b673990d 100644 --- a/drivers/xen/manage.c +++ b/drivers/xen/manage.c @@ -116,7 +116,7 @@ static void do_suspend(void) err = dpm_suspend_start(PMSG_FREEZE); if (err) { pr_err("%s: dpm_suspend_start %d\n", __func__, err); - goto out_thaw; + goto out_resume_end; } printk(KERN_DEBUG "suspending xenstore...\n"); @@ -156,6 +156,7 @@ static void do_suspend(void) else xs_suspend_cancel(); +out_resume_end: dpm_resume_end(si.cancelled ? PMSG_THAW : PMSG_RESTORE); out_thaw: diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c index 544d3f9010b92..1db82da56db62 100644 --- a/drivers/xen/platform-pci.c +++ b/drivers/xen/platform-pci.c @@ -26,6 +26,8 @@ #define DRV_NAME "xen-platform-pci" +#define PCI_DEVICE_ID_XEN_PLATFORM_XS61 0x0002 + static unsigned long platform_mmio; static unsigned long platform_mmio_alloc; static unsigned long platform_mmiolen; @@ -174,6 +176,8 @@ static int platform_pci_probe(struct pci_dev *pdev, static const struct pci_device_id platform_pci_tbl[] = { {PCI_VENDOR_ID_XEN, PCI_DEVICE_ID_XEN_PLATFORM, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, + {PCI_VENDOR_ID_XEN, PCI_DEVICE_ID_XEN_PLATFORM_XS61, + PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, {0,} }; diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 0b3bd9a7575e5..5770f3b374ece 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -216,6 +216,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, * buffering it. */ if (dma_capable(dev, dev_addr, size, true) && + !dma_kmalloc_needs_bounce(dev, size, dir) && !range_straddles_page_boundary(phys, size) && !xen_arch_need_swiotlb(dev, phys, dev_addr) && !is_swiotlb_force_bounce(dev)) diff --git a/drivers/xen/xenbus/xenbus.h b/drivers/xen/xenbus/xenbus.h index 2754bdfadcb89..4ba73320694a4 100644 --- a/drivers/xen/xenbus/xenbus.h +++ b/drivers/xen/xenbus/xenbus.h @@ -77,6 +77,7 @@ enum xb_req_state { struct xb_req_data { struct list_head list; wait_queue_head_t wq; + struct kref kref; struct xsd_sockmsg msg; uint32_t caller_req_id; enum xsd_sockmsg_type type; @@ -103,6 +104,7 @@ int xb_init_comms(void); void xb_deinit_comms(void); int xs_watch_msg(struct xs_watch_event *event); void xs_request_exit(struct xb_req_data *req); +void xs_free_req(struct kref *kref); int xenbus_match(struct device *_dev, struct device_driver *_drv); int xenbus_dev_probe(struct device *_dev); diff --git a/drivers/xen/xenbus/xenbus_comms.c b/drivers/xen/xenbus/xenbus_comms.c index e5fda0256feb3..82df2da1b880b 100644 --- a/drivers/xen/xenbus/xenbus_comms.c +++ b/drivers/xen/xenbus/xenbus_comms.c @@ -309,8 +309,8 @@ static int process_msg(void) virt_wmb(); req->state = xb_req_state_got_reply; req->cb(req); - } else - kfree(req); + } + kref_put(&req->kref, xs_free_req); } mutex_unlock(&xs_response_mutex); @@ -386,14 +386,13 @@ static int process_writes(void) state.req->msg.type = XS_ERROR; state.req->err = err; list_del(&state.req->list); - if (state.req->state == xb_req_state_aborted) - kfree(state.req); - else { + if (state.req->state != xb_req_state_aborted) { /* write err, then update state */ virt_wmb(); state.req->state = xb_req_state_got_reply; wake_up(&state.req->wq); } + kref_put(&state.req->kref, xs_free_req); mutex_unlock(&xb_write_mutex); diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c index 0792fda49a15f..c495cff3da308 100644 --- a/drivers/xen/xenbus/xenbus_dev_frontend.c +++ b/drivers/xen/xenbus/xenbus_dev_frontend.c @@ -406,7 +406,7 @@ void xenbus_dev_queue_reply(struct xb_req_data *req) mutex_unlock(&u->reply_mutex); kfree(req->body); - kfree(req); + kref_put(&req->kref, xs_free_req); kref_put(&u->kref, xenbus_file_free); diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index 25164d56c9d99..d3b6908110c6f 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -966,9 +966,15 @@ static int __init xenbus_init(void) if (xen_pv_domain()) xen_store_domain_type = XS_PV; if (xen_hvm_domain()) + { xen_store_domain_type = XS_HVM; - if (xen_hvm_domain() && xen_initial_domain()) - xen_store_domain_type = XS_LOCAL; + err = hvm_get_parameter(HVM_PARAM_STORE_EVTCHN, &v); + if (err) + goto out_error; + xen_store_evtchn = (int)v; + if (!v && xen_initial_domain()) + xen_store_domain_type = XS_LOCAL; + } if (xen_pv_domain() && !xen_start_info->store_evtchn) xen_store_domain_type = XS_LOCAL; if (xen_pv_domain() && xen_start_info->store_evtchn) @@ -987,10 +993,6 @@ static int __init xenbus_init(void) xen_store_interface = gfn_to_virt(xen_store_gfn); break; case XS_HVM: - err = hvm_get_parameter(HVM_PARAM_STORE_EVTCHN, &v); - if (err) - goto out_error; - xen_store_evtchn = (int)v; err = hvm_get_parameter(HVM_PARAM_STORE_PFN, &v); if (err) goto out_error; diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index 028a182bcc9e8..f84c87ee28395 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -112,6 +112,12 @@ static void xs_suspend_exit(void) wake_up_all(&xs_state_enter_wq); } +void xs_free_req(struct kref *kref) +{ + struct xb_req_data *req = container_of(kref, struct xb_req_data, kref); + kfree(req); +} + static uint32_t xs_request_enter(struct xb_req_data *req) { uint32_t rq_id; @@ -237,6 +243,12 @@ static void xs_send(struct xb_req_data *req, struct xsd_sockmsg *msg) req->caller_req_id = req->msg.req_id; req->msg.req_id = xs_request_enter(req); + /* + * Take 2nd ref. One for this thread, and the second for the + * xenbus_thread. + */ + kref_get(&req->kref); + mutex_lock(&xb_write_mutex); list_add_tail(&req->list, &xb_write_list); notify = list_is_singular(&xb_write_list); @@ -261,8 +273,8 @@ static void *xs_wait_for_reply(struct xb_req_data *req, struct xsd_sockmsg *msg) if (req->state == xb_req_state_queued || req->state == xb_req_state_wait_reply) req->state = xb_req_state_aborted; - else - kfree(req); + + kref_put(&req->kref, xs_free_req); mutex_unlock(&xb_write_mutex); return ret; @@ -291,6 +303,7 @@ int xenbus_dev_request_and_reply(struct xsd_sockmsg *msg, void *par) req->cb = xenbus_dev_queue_reply; req->par = par; req->user_req = true; + kref_init(&req->kref); xs_send(req, msg); @@ -319,6 +332,7 @@ static void *xs_talkv(struct xenbus_transaction t, req->num_vecs = num_vecs; req->cb = xs_wake_up; req->user_req = false; + kref_init(&req->kref); msg.req_id = 0; msg.tx_id = t.id; diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index 737d9f99c06a0..f13dfeb4ffd9b 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -561,7 +561,7 @@ static ssize_t caches_show(struct kobject *kobj, spin_lock(&v9fs_sessionlist_lock); list_for_each_entry(v9ses, &v9fs_sessionlist, slist) { if (v9ses->cachetag) { - n = snprintf(buf, limit, "%s\n", v9ses->cachetag); + n = snprintf(buf + count, limit, "%s\n", v9ses->cachetag); if (n < 0) { count = n; break; @@ -597,13 +597,16 @@ static const struct attribute_group v9fs_attr_group = { static int __init v9fs_sysfs_init(void) { + int ret; + v9fs_kobj = kobject_create_and_add("9p", fs_kobj); if (!v9fs_kobj) return -ENOMEM; - if (sysfs_create_group(v9fs_kobj, &v9fs_attr_group)) { + ret = sysfs_create_group(v9fs_kobj, &v9fs_attr_group); + if (ret) { kobject_put(v9fs_kobj); - return -ENOMEM; + return ret; } return 0; diff --git a/fs/afs/server.c b/fs/afs/server.c index 87381c2ffe374..f92ce4b7d73a1 100644 --- a/fs/afs/server.c +++ b/fs/afs/server.c @@ -401,13 +401,14 @@ struct afs_server *afs_use_server(struct afs_server *server, enum afs_server_tra void afs_put_server(struct afs_net *net, struct afs_server *server, enum afs_server_trace reason) { - unsigned int a, debug_id = server->debug_id; + unsigned int a, debug_id; bool zero; int r; if (!server) return; + debug_id = server->debug_id; a = atomic_read(&server->active); zero = __refcount_dec_and_test(&server->ref, &r); trace_afs_server(debug_id, r - 1, a, reason); diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index 24192a7667edf..a25766e90f0a6 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -55,15 +55,26 @@ static struct file_system_type anon_inode_fs_type = { .kill_sb = kill_anon_super, }; -static struct inode *anon_inode_make_secure_inode( - const char *name, - const struct inode *context_inode) +/** + * anon_inode_make_secure_inode - allocate an anonymous inode with security context + * @sb: [in] Superblock to allocate from + * @name: [in] Name of the class of the newfile (e.g., "secretmem") + * @context_inode: + * [in] Optional parent inode for security inheritance + * + * The function ensures proper security initialization through the LSM hook + * security_inode_init_security_anon(). + * + * Return: Pointer to new inode on success, ERR_PTR on failure. + */ +struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *name, + const struct inode *context_inode) { struct inode *inode; const struct qstr qname = QSTR_INIT(name, strlen(name)); int error; - inode = alloc_anon_inode(anon_inode_mnt->mnt_sb); + inode = alloc_anon_inode(sb); if (IS_ERR(inode)) return inode; inode->i_flags &= ~S_PRIVATE; @@ -74,6 +85,7 @@ static struct inode *anon_inode_make_secure_inode( } return inode; } +EXPORT_SYMBOL_GPL_FOR_MODULES(anon_inode_make_secure_inode, "kvm"); static struct file *__anon_inode_getfile(const char *name, const struct file_operations *fops, @@ -88,7 +100,8 @@ static struct file *__anon_inode_getfile(const char *name, return ERR_PTR(-ENOENT); if (secure) { - inode = anon_inode_make_secure_inode(name, context_inode); + inode = anon_inode_make_secure_inode(anon_inode_mnt->mnt_sb, + name, context_inode); if (IS_ERR(inode)) { file = ERR_CAST(inode); goto err; diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 56b54a635a78a..12381173f02c1 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -107,25 +107,6 @@ static struct linux_binfmt elf_format = { #define BAD_ADDR(x) (unlikely((unsigned long)(x) >= TASK_SIZE)) -static int set_brk(unsigned long start, unsigned long end, int prot) -{ - start = ELF_PAGEALIGN(start); - end = ELF_PAGEALIGN(end); - if (end > start) { - /* - * Map the last of the bss segment. - * If the header is requesting these pages to be - * executable, honour that (ppc32 needs this). - */ - int error = vm_brk_flags(start, end - start, - prot & PROT_EXEC ? VM_EXEC : 0); - if (error) - return error; - } - current->mm->start_brk = current->mm->brk = end; - return 0; -} - /* We need to explicitly zero any fractional pages after the data section (i.e. bss). This would contain the junk from the file that should not @@ -403,6 +384,51 @@ static unsigned long elf_map(struct file *filep, unsigned long addr, return(map_addr); } +static unsigned long elf_load(struct file *filep, unsigned long addr, + const struct elf_phdr *eppnt, int prot, int type, + unsigned long total_size) +{ + unsigned long zero_start, zero_end; + unsigned long map_addr; + + if (eppnt->p_filesz) { + map_addr = elf_map(filep, addr, eppnt, prot, type, total_size); + if (BAD_ADDR(map_addr)) + return map_addr; + if (eppnt->p_memsz > eppnt->p_filesz) { + zero_start = map_addr + ELF_PAGEOFFSET(eppnt->p_vaddr) + + eppnt->p_filesz; + zero_end = map_addr + ELF_PAGEOFFSET(eppnt->p_vaddr) + + eppnt->p_memsz; + + /* Zero the end of the last mapped page */ + padzero(zero_start); + } + } else { + map_addr = zero_start = ELF_PAGESTART(addr); + zero_end = zero_start + ELF_PAGEOFFSET(eppnt->p_vaddr) + + eppnt->p_memsz; + } + if (eppnt->p_memsz > eppnt->p_filesz) { + /* + * Map the last of the segment. + * If the header is requesting these pages to be + * executable, honour that (ppc32 needs this). + */ + int error; + + zero_start = ELF_PAGEALIGN(zero_start); + zero_end = ELF_PAGEALIGN(zero_end); + + error = vm_brk_flags(zero_start, zero_end - zero_start, + prot & PROT_EXEC ? VM_EXEC : 0); + if (error) + map_addr = error; + } + return map_addr; +} + + static unsigned long total_mapping_size(const struct elf_phdr *phdr, int nr) { elf_addr_t min_addr = -1; @@ -825,8 +851,8 @@ static int load_elf_binary(struct linux_binprm *bprm) unsigned long error; struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL; struct elf_phdr *elf_property_phdata = NULL; - unsigned long elf_bss, elf_brk; - int bss_prot = 0; + unsigned long elf_brk; + bool brk_moved = false; int retval, i; unsigned long elf_entry; unsigned long e_entry; @@ -1018,7 +1044,6 @@ static int load_elf_binary(struct linux_binprm *bprm) if (retval < 0) goto out_free_dentry; - elf_bss = 0; elf_brk = 0; start_code = ~0UL; @@ -1038,33 +1063,6 @@ static int load_elf_binary(struct linux_binprm *bprm) if (elf_ppnt->p_type != PT_LOAD) continue; - if (unlikely (elf_brk > elf_bss)) { - unsigned long nbyte; - - /* There was a PT_LOAD segment with p_memsz > p_filesz - before this one. Map anonymous pages, if needed, - and clear the area. */ - retval = set_brk(elf_bss + load_bias, - elf_brk + load_bias, - bss_prot); - if (retval) - goto out_free_dentry; - nbyte = ELF_PAGEOFFSET(elf_bss); - if (nbyte) { - nbyte = ELF_MIN_ALIGN - nbyte; - if (nbyte > elf_brk - elf_bss) - nbyte = elf_brk - elf_bss; - if (clear_user((void __user *)elf_bss + - load_bias, nbyte)) { - /* - * This bss-zeroing can fail if the ELF - * file specifies odd protections. So - * we don't check the return value - */ - } - } - } - elf_prot = make_prot(elf_ppnt->p_flags, &arch_state, !!interpreter, false); @@ -1092,15 +1090,49 @@ static int load_elf_binary(struct linux_binprm *bprm) * Header for ET_DYN binaries to calculate the * randomization (load_bias) for all the LOAD * Program Headers. + */ + + /* + * Calculate the entire size of the ELF mapping + * (total_size), used for the initial mapping, + * due to load_addr_set which is set to true later + * once the initial mapping is performed. + * + * Note that this is only sensible when the LOAD + * segments are contiguous (or overlapping). If + * used for LOADs that are far apart, this would + * cause the holes between LOADs to be mapped, + * running the risk of having the mapping fail, + * as it would be larger than the ELF file itself. + * + * As a result, only ET_DYN does this, since + * some ET_EXEC (e.g. ia64) may have large virtual + * memory holes between LOADs. + * + */ + total_size = total_mapping_size(elf_phdata, + elf_ex->e_phnum); + if (!total_size) { + retval = -EINVAL; + goto out_free_dentry; + } + + /* Calculate any requested alignment. */ + alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); + + /** + * DOC: PIE handling * - * There are effectively two types of ET_DYN - * binaries: programs (i.e. PIE: ET_DYN with INTERP) - * and loaders (ET_DYN without INTERP, since they - * _are_ the ELF interpreter). The loaders must - * be loaded away from programs since the program - * may otherwise collide with the loader (especially - * for ET_EXEC which does not have a randomized - * position). For example to handle invocations of + * There are effectively two types of ET_DYN ELF + * binaries: programs (i.e. PIE: ET_DYN with + * PT_INTERP) and loaders (i.e. static PIE: ET_DYN + * without PT_INTERP, usually the ELF interpreter + * itself). Loaders must be loaded away from programs + * since the program may otherwise collide with the + * loader (especially for ET_EXEC which does not have + * a randomized position). + * + * For example, to handle invocations of * "./ld.so someprog" to test out a new version of * the loader, the subsequent program that the * loader loads must avoid the loader itself, so @@ -1113,17 +1145,49 @@ static int load_elf_binary(struct linux_binprm *bprm) * ELF_ET_DYN_BASE and loaders are loaded into the * independently randomized mmap region (0 load_bias * without MAP_FIXED nor MAP_FIXED_NOREPLACE). + * + * See below for "brk" handling details, which is + * also affected by program vs loader and ASLR. */ if (interpreter) { + /* On ET_DYN with PT_INTERP, we do the ASLR. */ load_bias = ELF_ET_DYN_BASE; if (current->flags & PF_RANDOMIZE) load_bias += arch_mmap_rnd(); - alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); + /* Adjust alignment as requested. */ if (alignment) load_bias &= ~(alignment - 1); elf_flags |= MAP_FIXED_NOREPLACE; - } else - load_bias = 0; + } else { + /* + * For ET_DYN without PT_INTERP, we rely on + * the architectures's (potentially ASLR) mmap + * base address (via a load_bias of 0). + * + * When a large alignment is requested, we + * must do the allocation at address "0" right + * now to discover where things will load so + * that we can adjust the resulting alignment. + * In this case (load_bias != 0), we can use + * MAP_FIXED_NOREPLACE to make sure the mapping + * doesn't collide with anything. + */ + if (alignment > ELF_MIN_ALIGN) { + load_bias = elf_load(bprm->file, 0, elf_ppnt, + elf_prot, elf_flags, total_size); + if (BAD_ADDR(load_bias)) { + retval = IS_ERR_VALUE(load_bias) ? + PTR_ERR((void*)load_bias) : -EINVAL; + goto out_free_dentry; + } + vm_munmap(load_bias, total_size); + /* Adjust alignment as requested. */ + if (alignment) + load_bias &= ~(alignment - 1); + elf_flags |= MAP_FIXED_NOREPLACE; + } else + load_bias = 0; + } /* * Since load_bias is used for all subsequent loading @@ -1133,34 +1197,9 @@ static int load_elf_binary(struct linux_binprm *bprm) * is then page aligned. */ load_bias = ELF_PAGESTART(load_bias - vaddr); - - /* - * Calculate the entire size of the ELF mapping - * (total_size), used for the initial mapping, - * due to load_addr_set which is set to true later - * once the initial mapping is performed. - * - * Note that this is only sensible when the LOAD - * segments are contiguous (or overlapping). If - * used for LOADs that are far apart, this would - * cause the holes between LOADs to be mapped, - * running the risk of having the mapping fail, - * as it would be larger than the ELF file itself. - * - * As a result, only ET_DYN does this, since - * some ET_EXEC (e.g. ia64) may have large virtual - * memory holes between LOADs. - * - */ - total_size = total_mapping_size(elf_phdata, - elf_ex->e_phnum); - if (!total_size) { - retval = -EINVAL; - goto out_free_dentry; - } } - error = elf_map(bprm->file, load_bias + vaddr, elf_ppnt, + error = elf_load(bprm->file, load_bias + vaddr, elf_ppnt, elf_prot, elf_flags, total_size); if (BAD_ADDR(error)) { retval = IS_ERR_VALUE(error) ? @@ -1208,41 +1247,23 @@ static int load_elf_binary(struct linux_binprm *bprm) k = elf_ppnt->p_vaddr + elf_ppnt->p_filesz; - if (k > elf_bss) - elf_bss = k; if ((elf_ppnt->p_flags & PF_X) && end_code < k) end_code = k; if (end_data < k) end_data = k; k = elf_ppnt->p_vaddr + elf_ppnt->p_memsz; - if (k > elf_brk) { - bss_prot = elf_prot; + if (k > elf_brk) elf_brk = k; - } } e_entry = elf_ex->e_entry + load_bias; phdr_addr += load_bias; - elf_bss += load_bias; elf_brk += load_bias; start_code += load_bias; end_code += load_bias; start_data += load_bias; end_data += load_bias; - /* Calling set_brk effectively mmaps the pages that we need - * for the bss and break sections. We must do this before - * mapping in the interpreter, to make sure it doesn't wind - * up getting placed where the bss needs to go. - */ - retval = set_brk(elf_bss, elf_brk, bss_prot); - if (retval) - goto out_free_dentry; - if (likely(elf_bss != elf_brk) && unlikely(padzero(elf_bss))) { - retval = -EFAULT; /* Nobody gets to see this, but.. */ - goto out_free_dentry; - } - if (interpreter) { elf_entry = load_elf_interp(interp_elf_ex, interpreter, @@ -1298,24 +1319,44 @@ static int load_elf_binary(struct linux_binprm *bprm) mm->end_data = end_data; mm->start_stack = bprm->p; - if ((current->flags & PF_RANDOMIZE) && (snapshot_randomize_va_space > 1)) { + /** + * DOC: "brk" handling + * + * For architectures with ELF randomization, when executing a + * loader directly (i.e. static PIE: ET_DYN without PT_INTERP), + * move the brk area out of the mmap region and into the unused + * ELF_ET_DYN_BASE region. Since "brk" grows up it may collide + * early with the stack growing down or other regions being put + * into the mmap region by the kernel (e.g. vdso). + * + * In the CONFIG_COMPAT_BRK case, though, everything is turned + * off because we're not allowed to move the brk at all. + */ + if (!IS_ENABLED(CONFIG_COMPAT_BRK) && + IS_ENABLED(CONFIG_ARCH_HAS_ELF_RANDOMIZE) && + elf_ex->e_type == ET_DYN && !interpreter) { + elf_brk = ELF_ET_DYN_BASE; + /* This counts as moving the brk, so let brk(2) know. */ + brk_moved = true; + } + mm->start_brk = mm->brk = ELF_PAGEALIGN(elf_brk); + + if ((current->flags & PF_RANDOMIZE) && snapshot_randomize_va_space > 1) { /* - * For architectures with ELF randomization, when executing - * a loader directly (i.e. no interpreter listed in ELF - * headers), move the brk area out of the mmap region - * (since it grows up, and may collide early with the stack - * growing down), and into the unused ELF_ET_DYN_BASE region. + * If we didn't move the brk to ELF_ET_DYN_BASE (above), + * leave a gap between .bss and brk. */ - if (IS_ENABLED(CONFIG_ARCH_HAS_ELF_RANDOMIZE) && - elf_ex->e_type == ET_DYN && !interpreter) { - mm->brk = mm->start_brk = ELF_ET_DYN_BASE; - } + if (!brk_moved) + mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; mm->brk = mm->start_brk = arch_randomize_brk(mm); + brk_moved = true; + } + #ifdef compat_brk_randomized + if (brk_moved) current->brk_randomized = 1; #endif - } if (current->personality & MMAP_PAGE_ZERO) { /* Why this, you ask??? Well SVr4 maps page 0 as read-only, diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index a2ba1c7fc16af..10f803d0534e6 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -222,8 +222,8 @@ static void free_pref(struct prelim_ref *ref) * A -1 return indicates ref1 is a 'lower' block than ref2, while 1 * indicates a 'higher' block. */ -static int prelim_ref_compare(struct prelim_ref *ref1, - struct prelim_ref *ref2) +static int prelim_ref_compare(const struct prelim_ref *ref1, + const struct prelim_ref *ref2) { if (ref1->level < ref2->level) return -1; @@ -254,7 +254,7 @@ static int prelim_ref_compare(struct prelim_ref *ref1, } static void update_share_count(struct share_check *sc, int oldcount, - int newcount, struct prelim_ref *newref) + int newcount, const struct prelim_ref *newref) { if ((!sc) || (oldcount == 0 && newcount < 1)) return; diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 434cf3d5f4cf1..5a3a41c6d509f 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -23,7 +23,7 @@ #include "extent-tree.h" #ifdef CONFIG_BTRFS_DEBUG -int btrfs_should_fragment_free_space(struct btrfs_block_group *block_group) +int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group) { struct btrfs_fs_info *fs_info = block_group->fs_info; @@ -34,15 +34,28 @@ int btrfs_should_fragment_free_space(struct btrfs_block_group *block_group) } #endif +static inline bool has_unwritten_metadata(struct btrfs_block_group *block_group) +{ + /* The meta_write_pointer is available only on the zoned setup. */ + if (!btrfs_is_zoned(block_group->fs_info)) + return false; + + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) + return false; + + return block_group->start + block_group->alloc_offset > + block_group->meta_write_pointer; +} + /* * Return target flags in extended format or 0 if restripe for this chunk_type * is not in progress * * Should be called with balance_lock held */ -static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags) +static u64 get_restripe_target(const struct btrfs_fs_info *fs_info, u64 flags) { - struct btrfs_balance_control *bctl = fs_info->balance_ctl; + const struct btrfs_balance_control *bctl = fs_info->balance_ctl; u64 target = 0; if (!bctl) @@ -1240,6 +1253,15 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, goto out; spin_lock(&block_group->lock); + /* + * Hitting this WARN means we removed a block group with an unwritten + * region. It will cause "unable to find chunk map for logical" errors. + */ + if (WARN_ON(has_unwritten_metadata(block_group))) + btrfs_warn(fs_info, + "block group %llu is removed before metadata write out", + block_group->start); + set_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags); /* @@ -1418,9 +1440,9 @@ static int inc_block_group_ro(struct btrfs_block_group *cache, int force) } static bool clean_pinned_extents(struct btrfs_trans_handle *trans, - struct btrfs_block_group *bg) + const struct btrfs_block_group *bg) { - struct btrfs_fs_info *fs_info = bg->fs_info; + struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_transaction *prev_trans = NULL; const u64 start = bg->start; const u64 end = start + bg->length - 1; @@ -1563,8 +1585,9 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) * needing to allocate extents from the block group. */ used = btrfs_space_info_used(space_info, true); - if (space_info->total_bytes - block_group->length < used && - block_group->zone_unusable < block_group->length) { + if ((space_info->total_bytes - block_group->length < used && + block_group->zone_unusable < block_group->length) || + has_unwritten_metadata(block_group)) { /* * Add a reference for the list, compensate for the ref * drop under the "next" label for the @@ -1752,14 +1775,14 @@ static int reclaim_bgs_cmp(void *unused, const struct list_head *a, return bg1->used > bg2->used; } -static inline bool btrfs_should_reclaim(struct btrfs_fs_info *fs_info) +static inline bool btrfs_should_reclaim(const struct btrfs_fs_info *fs_info) { if (btrfs_is_zoned(fs_info)) return btrfs_zoned_should_reclaim(fs_info); return true; } -static bool should_reclaim_block_group(struct btrfs_block_group *bg, u64 bytes_freed) +static bool should_reclaim_block_group(const struct btrfs_block_group *bg, u64 bytes_freed) { const struct btrfs_space_info *space_info = bg->space_info; const int reclaim_thresh = READ_ONCE(space_info->bg_reclaim_threshold); @@ -1885,6 +1908,17 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) up_write(&space_info->groups_sem); goto next; } + + /* + * Cache the zone_unusable value before turning the block group + * to read only. As soon as the block group is read only it's + * zone_unusable value gets moved to the block group's read-only + * bytes and isn't available for calculations anymore. We also + * cache it before unlocking the block group, to prevent races + * (reports from KCSAN and such tools) with tasks updating it. + */ + zone_unusable = bg->zone_unusable; + spin_unlock(&bg->lock); /* @@ -1900,13 +1934,6 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) goto next; } - /* - * Cache the zone_unusable value before turning the block group - * to read only. As soon as the blog group is read only it's - * zone_unusable value gets moved to the block group's read-only - * bytes and isn't available for calculations anymore. - */ - zone_unusable = bg->zone_unusable; ret = inc_block_group_ro(bg, 0); up_write(&space_info->groups_sem); if (ret < 0) @@ -1987,8 +2014,8 @@ void btrfs_mark_bg_to_reclaim(struct btrfs_block_group *bg) spin_unlock(&fs_info->unused_bgs_lock); } -static int read_bg_from_eb(struct btrfs_fs_info *fs_info, struct btrfs_key *key, - struct btrfs_path *path) +static int read_bg_from_eb(struct btrfs_fs_info *fs_info, const struct btrfs_key *key, + const struct btrfs_path *path) { struct extent_map_tree *em_tree; struct extent_map *em; @@ -2040,7 +2067,7 @@ static int read_bg_from_eb(struct btrfs_fs_info *fs_info, struct btrfs_key *key, static int find_first_block_group(struct btrfs_fs_info *fs_info, struct btrfs_path *path, - struct btrfs_key *key) + const struct btrfs_key *key) { struct btrfs_root *root = btrfs_block_group_root(fs_info); int ret; @@ -2632,8 +2659,8 @@ static int insert_block_group_item(struct btrfs_trans_handle *trans, } static int insert_dev_extent(struct btrfs_trans_handle *trans, - struct btrfs_device *device, u64 chunk_offset, - u64 start, u64 num_bytes) + const struct btrfs_device *device, u64 chunk_offset, + u64 start, u64 num_bytes) { struct btrfs_fs_info *fs_info = device->fs_info; struct btrfs_root *root = fs_info->dev_root; @@ -2783,7 +2810,7 @@ void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans) * For extent tree v2 we use the block_group_item->chunk_offset to point at our * global root id. For v1 it's always set to BTRFS_FIRST_CHUNK_TREE_OBJECTID. */ -static u64 calculate_global_root_id(struct btrfs_fs_info *fs_info, u64 offset) +static u64 calculate_global_root_id(const struct btrfs_fs_info *fs_info, u64 offset) { u64 div = SZ_1G; u64 index; @@ -3819,8 +3846,8 @@ static void force_metadata_allocation(struct btrfs_fs_info *info) } } -static int should_alloc_chunk(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *sinfo, int force) +static int should_alloc_chunk(const struct btrfs_fs_info *fs_info, + const struct btrfs_space_info *sinfo, int force) { u64 bytes_used = btrfs_space_info_used(sinfo, false); u64 thresh; @@ -4195,7 +4222,7 @@ int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, return ret; } -static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type) +static u64 get_profile_num_devs(const struct btrfs_fs_info *fs_info, u64 type) { u64 num_dev; @@ -4602,7 +4629,7 @@ int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, return 0; } -bool btrfs_block_group_should_use_size_class(struct btrfs_block_group *bg) +bool btrfs_block_group_should_use_size_class(const struct btrfs_block_group *bg) { if (btrfs_is_zoned(bg->fs_info)) return false; diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 089979981e4aa..a8a6a21e393d2 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -250,7 +250,7 @@ struct btrfs_block_group { enum btrfs_block_group_size_class size_class; }; -static inline u64 btrfs_block_group_end(struct btrfs_block_group *block_group) +static inline u64 btrfs_block_group_end(const struct btrfs_block_group *block_group) { return (block_group->start + block_group->length); } @@ -262,8 +262,7 @@ static inline bool btrfs_is_block_group_used(const struct btrfs_block_group *bg) return (bg->used > 0 || bg->reserved > 0 || bg->pinned > 0); } -static inline bool btrfs_is_block_group_data_only( - struct btrfs_block_group *block_group) +static inline bool btrfs_is_block_group_data_only(const struct btrfs_block_group *block_group) { /* * In mixed mode the fragmentation is expected to be high, lowering the @@ -274,7 +273,7 @@ static inline bool btrfs_is_block_group_data_only( } #ifdef CONFIG_BTRFS_DEBUG -int btrfs_should_fragment_free_space(struct btrfs_block_group *block_group); +int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group); #endif struct btrfs_block_group *btrfs_lookup_first_block_group( @@ -355,7 +354,7 @@ static inline u64 btrfs_system_alloc_profile(struct btrfs_fs_info *fs_info) return btrfs_get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); } -static inline int btrfs_block_group_done(struct btrfs_block_group *cache) +static inline int btrfs_block_group_done(const struct btrfs_block_group *cache) { smp_mb(); return cache->cached == BTRFS_CACHE_FINISHED || @@ -372,6 +371,6 @@ enum btrfs_block_group_size_class btrfs_calc_block_group_size_class(u64 size); int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, enum btrfs_block_group_size_class size_class, bool force_wrong_size_class); -bool btrfs_block_group_should_use_size_class(struct btrfs_block_group *bg); +bool btrfs_block_group_should_use_size_class(const struct btrfs_block_group *bg); #endif /* BTRFS_BLOCK_GROUP_H */ diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index db8da4e7b2289..97084ea3af0cc 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -547,7 +547,7 @@ struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans, return ERR_PTR(ret); } -int btrfs_check_trunc_cache_free_space(struct btrfs_fs_info *fs_info, +int btrfs_check_trunc_cache_free_space(const struct btrfs_fs_info *fs_info, struct btrfs_block_rsv *rsv) { u64 needed_bytes; diff --git a/fs/btrfs/block-rsv.h b/fs/btrfs/block-rsv.h index 43a9a6b5a79f4..3c9a15f59731f 100644 --- a/fs/btrfs/block-rsv.h +++ b/fs/btrfs/block-rsv.h @@ -82,7 +82,7 @@ void btrfs_release_global_block_rsv(struct btrfs_fs_info *fs_info); struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans, struct btrfs_root *root, u32 blocksize); -int btrfs_check_trunc_cache_free_space(struct btrfs_fs_info *fs_info, +int btrfs_check_trunc_cache_free_space(const struct btrfs_fs_info *fs_info, struct btrfs_block_rsv *rsv); static inline void btrfs_unuse_block_rsv(struct btrfs_fs_info *fs_info, struct btrfs_block_rsv *block_rsv, diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index ec6679a538c1d..a2e471d51a8f0 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -179,7 +179,7 @@ struct btrfs_inode { u64 new_delalloc_bytes; /* * The offset of the last dir index key that was logged. - * This is used only for directories. + * This is used only for directories. Protected by 'log_mutex'. */ u64 last_dir_index_offset; }; @@ -251,7 +251,8 @@ struct btrfs_inode { struct btrfs_delayed_node *delayed_node; /* File creation time. */ - struct timespec64 i_otime; + u64 i_otime_sec; + u32 i_otime_nsec; /* Hook into fs_info->delayed_iputs */ struct list_head delayed_iput; @@ -488,7 +489,7 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, int btrfs_update_inode(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_inode *inode); int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, - struct btrfs_root *root, struct btrfs_inode *inode); + struct btrfs_inode *inode); int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct btrfs_inode *inode); int btrfs_orphan_cleanup(struct btrfs_root *root); int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size); diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 4b21ca49b6665..31b1b448efc2e 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -347,7 +347,14 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans, write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); - WARN_ON(btrfs_header_generation(buf) > trans->transid); + if (unlikely(btrfs_header_generation(buf) > trans->transid)) { + btrfs_tree_unlock(cow); + free_extent_buffer(cow); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) ret = btrfs_inc_ref(trans, root, cow, 1); else @@ -2712,7 +2719,7 @@ int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, * */ static void fixup_low_keys(struct btrfs_trans_handle *trans, - struct btrfs_path *path, + const struct btrfs_path *path, struct btrfs_disk_key *key, int level) { int i; @@ -2742,7 +2749,7 @@ static void fixup_low_keys(struct btrfs_trans_handle *trans, * that the new key won't break the order */ void btrfs_set_item_key_safe(struct btrfs_trans_handle *trans, - struct btrfs_path *path, + const struct btrfs_path *path, const struct btrfs_key *new_key) { struct btrfs_fs_info *fs_info = trans->fs_info; @@ -2808,8 +2815,8 @@ void btrfs_set_item_key_safe(struct btrfs_trans_handle *trans, * is correct, we only need to bother the last key of @left and the first * key of @right. */ -static bool check_sibling_keys(struct extent_buffer *left, - struct extent_buffer *right) +static bool check_sibling_keys(const struct extent_buffer *left, + const struct extent_buffer *right) { struct btrfs_key left_last; struct btrfs_key right_first; @@ -3049,6 +3056,7 @@ static noinline int insert_new_root(struct btrfs_trans_handle *trans, if (ret < 0) { int ret2; + btrfs_clear_buffer_dirty(trans, c); ret2 = btrfs_free_tree_block(trans, btrfs_root_id(root), c, 0, 1); if (ret2 < 0) btrfs_abort_transaction(trans, ret2); @@ -3077,7 +3085,7 @@ static noinline int insert_new_root(struct btrfs_trans_handle *trans, * blocknr is the block the key points to. */ static int insert_ptr(struct btrfs_trans_handle *trans, - struct btrfs_path *path, + const struct btrfs_path *path, struct btrfs_disk_key *key, u64 bytenr, int slot, int level) { @@ -4168,7 +4176,7 @@ int btrfs_split_item(struct btrfs_trans_handle *trans, * the front. */ void btrfs_truncate_item(struct btrfs_trans_handle *trans, - struct btrfs_path *path, u32 new_size, int from_end) + const struct btrfs_path *path, u32 new_size, int from_end) { int slot; struct extent_buffer *leaf; @@ -4260,7 +4268,7 @@ void btrfs_truncate_item(struct btrfs_trans_handle *trans, * make the item pointed to by the path bigger, data_size is the added size. */ void btrfs_extend_item(struct btrfs_trans_handle *trans, - struct btrfs_path *path, u32 data_size) + const struct btrfs_path *path, u32 data_size) { int slot; struct extent_buffer *leaf; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 7df3ed2945b04..834af67fac231 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -521,7 +521,7 @@ int btrfs_previous_item(struct btrfs_root *root, int btrfs_previous_extent_item(struct btrfs_root *root, struct btrfs_path *path, u64 min_objectid); void btrfs_set_item_key_safe(struct btrfs_trans_handle *trans, - struct btrfs_path *path, + const struct btrfs_path *path, const struct btrfs_key *new_key); struct extent_buffer *btrfs_root_node(struct btrfs_root *root); int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, @@ -555,9 +555,9 @@ int btrfs_block_can_be_shared(struct btrfs_trans_handle *trans, int btrfs_del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_path *path, int level, int slot); void btrfs_extend_item(struct btrfs_trans_handle *trans, - struct btrfs_path *path, u32 data_size); + const struct btrfs_path *path, u32 data_size); void btrfs_truncate_item(struct btrfs_trans_handle *trans, - struct btrfs_path *path, u32 new_size, int from_end); + const struct btrfs_path *path, u32 new_size, int from_end); int btrfs_split_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_path *path, diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 32c5f5a8a0e93..c39e39142abf1 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1849,10 +1849,8 @@ static void fill_stack_inode_item(struct btrfs_trans_handle *trans, btrfs_set_stack_timespec_nsec(&inode_item->ctime, inode_get_ctime(inode).tv_nsec); - btrfs_set_stack_timespec_sec(&inode_item->otime, - BTRFS_I(inode)->i_otime.tv_sec); - btrfs_set_stack_timespec_nsec(&inode_item->otime, - BTRFS_I(inode)->i_otime.tv_nsec); + btrfs_set_stack_timespec_sec(&inode_item->otime, BTRFS_I(inode)->i_otime_sec); + btrfs_set_stack_timespec_nsec(&inode_item->otime, BTRFS_I(inode)->i_otime_nsec); } int btrfs_fill_inode(struct inode *inode, u32 *rdev) @@ -1901,10 +1899,8 @@ int btrfs_fill_inode(struct inode *inode, u32 *rdev) inode_set_ctime(inode, btrfs_stack_timespec_sec(&inode_item->ctime), btrfs_stack_timespec_nsec(&inode_item->ctime)); - BTRFS_I(inode)->i_otime.tv_sec = - btrfs_stack_timespec_sec(&inode_item->otime); - BTRFS_I(inode)->i_otime.tv_nsec = - btrfs_stack_timespec_nsec(&inode_item->otime); + BTRFS_I(inode)->i_otime_sec = btrfs_stack_timespec_sec(&inode_item->otime); + BTRFS_I(inode)->i_otime_nsec = btrfs_stack_timespec_nsec(&inode_item->otime); inode->i_generation = BTRFS_I(inode)->generation; BTRFS_I(inode)->index_cnt = (u64)-1; diff --git a/fs/btrfs/discard.c b/fs/btrfs/discard.c index 944a7340f6a44..d6eef4bd9e9d4 100644 --- a/fs/btrfs/discard.c +++ b/fs/btrfs/discard.c @@ -68,7 +68,7 @@ static int discard_minlen[BTRFS_NR_DISCARD_LISTS] = { }; static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl, - struct btrfs_block_group *block_group) + const struct btrfs_block_group *block_group) { return &discard_ctl->discard_list[block_group->discard_index]; } @@ -80,7 +80,7 @@ static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl, * * Check if the file system is writeable and BTRFS_FS_DISCARD_RUNNING is set. */ -static bool btrfs_run_discard_work(struct btrfs_discard_ctl *discard_ctl) +static bool btrfs_run_discard_work(const struct btrfs_discard_ctl *discard_ctl) { struct btrfs_fs_info *fs_info = container_of(discard_ctl, struct btrfs_fs_info, @@ -167,13 +167,7 @@ static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl, block_group->discard_eligible_time = 0; queued = !list_empty(&block_group->discard_list); list_del_init(&block_group->discard_list); - /* - * If the block group is currently running in the discard workfn, we - * don't want to deref it, since it's still being used by the workfn. - * The workfn will notice this case and deref the block group when it is - * finished. - */ - if (queued && !running) + if (queued) btrfs_put_block_group(block_group); spin_unlock(&discard_ctl->lock); @@ -260,9 +254,10 @@ static struct btrfs_block_group *peek_discard_list( block_group->discard_cursor = block_group->start; block_group->discard_state = BTRFS_DISCARD_EXTENTS; } - discard_ctl->block_group = block_group; } if (block_group) { + btrfs_get_block_group(block_group); + discard_ctl->block_group = block_group; *discard_state = block_group->discard_state; *discard_index = block_group->discard_index; } @@ -493,9 +488,20 @@ static void btrfs_discard_workfn(struct work_struct *work) block_group = peek_discard_list(discard_ctl, &discard_state, &discard_index, now); - if (!block_group || !btrfs_run_discard_work(discard_ctl)) + if (!block_group) return; + if (!btrfs_run_discard_work(discard_ctl)) { + spin_lock(&discard_ctl->lock); + btrfs_put_block_group(block_group); + discard_ctl->block_group = NULL; + spin_unlock(&discard_ctl->lock); + return; + } if (now < block_group->discard_eligible_time) { + spin_lock(&discard_ctl->lock); + btrfs_put_block_group(block_group); + discard_ctl->block_group = NULL; + spin_unlock(&discard_ctl->lock); btrfs_discard_schedule_work(discard_ctl, false); return; } @@ -547,15 +553,7 @@ static void btrfs_discard_workfn(struct work_struct *work) spin_lock(&discard_ctl->lock); discard_ctl->prev_discard = trimmed; discard_ctl->prev_discard_time = now; - /* - * If the block group was removed from the discard list while it was - * running in this workfn, then we didn't deref it, since this function - * still owned that reference. But we set the discard_ctl->block_group - * back to NULL, so we can use that condition to know that now we need - * to deref the block_group. - */ - if (discard_ctl->block_group == NULL) - btrfs_put_block_group(block_group); + btrfs_put_block_group(block_group); discard_ctl->block_group = NULL; __btrfs_discard_schedule_work(discard_ctl, now, false); spin_unlock(&discard_ctl->lock); diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2387210231f23..7ad1734cbbfc9 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2080,10 +2080,10 @@ static int btrfs_replay_log(struct btrfs_fs_info *fs_info, /* returns with log_tree_root freed on success */ ret = btrfs_recover_log_trees(log_tree_root); + btrfs_put_root(log_tree_root); if (ret) { btrfs_handle_fs_error(fs_info, ret, "Failed to recover log tree"); - btrfs_put_root(log_tree_root); return ret; } @@ -2148,8 +2148,7 @@ static int load_global_roots_objectid(struct btrfs_root *tree_root, found = true; root = read_tree_root_path(tree_root, path, &key); if (IS_ERR(root)) { - if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) - ret = PTR_ERR(root); + ret = PTR_ERR(root); break; } set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); @@ -4313,6 +4312,14 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) /* clear out the rbtree of defraggable inodes */ btrfs_cleanup_defrag_inodes(fs_info); + /* + * Handle the error fs first, as it will flush and wait for all ordered + * extents. This will generate delayed iputs, thus we want to handle + * it first. + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) + btrfs_error_commit_super(fs_info); + /* * Wait for any fixup workers to complete. * If we don't wait for them here and they are still running by the time @@ -4333,6 +4340,19 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) */ btrfs_flush_workqueue(fs_info->delalloc_workers); + /* + * We can have ordered extents getting their last reference dropped from + * the fs_info->workers queue because for async writes for data bios we + * queue a work for that queue, at btrfs_wq_submit_bio(), that runs + * run_one_async_done() which calls btrfs_bio_end_io() in case the bio + * has an error, and that later function can do the final + * btrfs_put_ordered_extent() on the ordered extent attached to the bio, + * which adds a delayed iput for the inode. So we must flush the queue + * so that we don't have delayed iputs after committing the current + * transaction below and stopping the cleaner and transaction kthreads. + */ + btrfs_flush_workqueue(fs_info->workers); + /* * When finishing a compressed write bio we schedule a work queue item * to finish an ordered extent - btrfs_finish_compressed_write_work() @@ -4402,9 +4422,6 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) btrfs_err(fs_info, "commit super ret %d", ret); } - if (BTRFS_FS_ERROR(fs_info)) - btrfs_error_commit_super(fs_info); - kthread_stop(fs_info->transaction_kthread); kthread_stop(fs_info->cleaner_kthread); @@ -4541,10 +4558,6 @@ static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info) /* cleanup FS via transaction */ btrfs_cleanup_transaction(fs_info); - mutex_lock(&fs_info->cleaner_mutex); - btrfs_run_delayed_iputs(fs_info); - mutex_unlock(&fs_info->cleaner_mutex); - down_write(&fs_info->cleanup_work_sem); up_write(&fs_info->cleanup_work_sem); } diff --git a/fs/btrfs/export.c b/fs/btrfs/export.c index 203e5964c9b0f..5e46f8cd7ea07 100644 --- a/fs/btrfs/export.c +++ b/fs/btrfs/export.c @@ -24,7 +24,11 @@ static int btrfs_encode_fh(struct inode *inode, u32 *fh, int *max_len, int type; if (parent && (len < BTRFS_FID_SIZE_CONNECTABLE)) { - *max_len = BTRFS_FID_SIZE_CONNECTABLE; + if (btrfs_root_id(BTRFS_I(inode)->root) != + btrfs_root_id(BTRFS_I(parent)->root)) + *max_len = BTRFS_FID_SIZE_CONNECTABLE_ROOT; + else + *max_len = BTRFS_FID_SIZE_CONNECTABLE; return FILEID_INVALID; } else if (len < BTRFS_FID_SIZE_NON_CONNECTABLE) { *max_len = BTRFS_FID_SIZE_NON_CONNECTABLE; @@ -46,6 +50,8 @@ static int btrfs_encode_fh(struct inode *inode, u32 *fh, int *max_len, parent_root_id = BTRFS_I(parent)->root->root_key.objectid; if (parent_root_id != fid->root_objectid) { + if (*max_len < BTRFS_FID_SIZE_CONNECTABLE_ROOT) + return FILEID_INVALID; fid->parent_root_objectid = parent_root_id; len = BTRFS_FID_SIZE_CONNECTABLE_ROOT; type = FILEID_BTRFS_WITH_PARENT_ROOT; diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index af03a1c6ba768..5e3d1a87b7e9d 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -164,6 +164,14 @@ int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); num_refs = btrfs_extent_refs(leaf, ei); + if (unlikely(num_refs == 0)) { + ret = -EUCLEAN; + btrfs_err(fs_info, + "unexpected zero reference count for extent item (%llu %u %llu)", + key.objectid, key.type, key.offset); + btrfs_abort_transaction(trans, ret); + goto out_free; + } extent_flags = btrfs_extent_flags(leaf, ei); } else { ret = -EUCLEAN; @@ -177,8 +185,6 @@ int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, goto out_free; } - - BUG_ON(num_refs == 0); } else { num_refs = 0; extent_flags = 0; @@ -208,10 +214,19 @@ int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, goto search_again; } spin_lock(&head->lock); - if (head->extent_op && head->extent_op->update_flags) + if (head->extent_op && head->extent_op->update_flags) { extent_flags |= head->extent_op->flags_to_set; - else - BUG_ON(num_refs == 0); + } else if (unlikely(num_refs == 0)) { + spin_unlock(&head->lock); + mutex_unlock(&head->mutex); + spin_unlock(&delayed_refs->lock); + ret = -EUCLEAN; + btrfs_err(fs_info, + "unexpected zero reference count for extent %llu (%s)", + bytenr, metadata ? "metadata" : "data"); + btrfs_abort_transaction(trans, ret); + goto out_free; + } num_refs += head->ref_mod; spin_unlock(&head->lock); @@ -3515,6 +3530,21 @@ btrfs_release_block_group(struct btrfs_block_group *cache, btrfs_put_block_group(cache); } +static bool find_free_extent_check_size_class(const struct find_free_extent_ctl *ffe_ctl, + const struct btrfs_block_group *bg) +{ + if (ffe_ctl->policy == BTRFS_EXTENT_ALLOC_ZONED) + return true; + if (!btrfs_block_group_should_use_size_class(bg)) + return true; + if (ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS) + return true; + if (ffe_ctl->loop >= LOOP_UNSET_SIZE_CLASS && + bg->size_class == BTRFS_BG_SZ_NONE) + return true; + return ffe_ctl->size_class == bg->size_class; +} + /* * Helper function for find_free_extent(). * @@ -3536,7 +3566,8 @@ static int find_free_extent_clustered(struct btrfs_block_group *bg, if (!cluster_bg) goto refill_cluster; if (cluster_bg != bg && (cluster_bg->ro || - !block_group_bits(cluster_bg, ffe_ctl->flags))) + !block_group_bits(cluster_bg, ffe_ctl->flags) || + !find_free_extent_check_size_class(ffe_ctl, cluster_bg))) goto release_cluster; offset = btrfs_alloc_from_cluster(cluster_bg, last_ptr, @@ -4092,21 +4123,6 @@ static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info, return -ENOSPC; } -static bool find_free_extent_check_size_class(struct find_free_extent_ctl *ffe_ctl, - struct btrfs_block_group *bg) -{ - if (ffe_ctl->policy == BTRFS_EXTENT_ALLOC_ZONED) - return true; - if (!btrfs_block_group_should_use_size_class(bg)) - return true; - if (ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS) - return true; - if (ffe_ctl->loop >= LOOP_UNSET_SIZE_CLASS && - bg->size_class == BTRFS_BG_SZ_NONE) - return true; - return ffe_ctl->size_class == bg->size_class; -} - static int prepare_allocation_clustered(struct btrfs_fs_info *fs_info, struct find_free_extent_ctl *ffe_ctl, struct btrfs_space_info *space_info, @@ -4159,7 +4175,8 @@ static int prepare_allocation_clustered(struct btrfs_fs_info *fs_info, } static int prepare_allocation_zoned(struct btrfs_fs_info *fs_info, - struct find_free_extent_ctl *ffe_ctl) + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_space_info *space_info) { if (ffe_ctl->for_treelog) { spin_lock(&fs_info->treelog_bg_lock); @@ -4183,6 +4200,7 @@ static int prepare_allocation_zoned(struct btrfs_fs_info *fs_info, u64 avail = block_group->zone_capacity - block_group->alloc_offset; if (block_group_bits(block_group, ffe_ctl->flags) && + block_group->space_info == space_info && avail >= ffe_ctl->num_bytes) { ffe_ctl->hint_byte = block_group->start; break; @@ -4204,7 +4222,7 @@ static int prepare_allocation(struct btrfs_fs_info *fs_info, return prepare_allocation_clustered(fs_info, ffe_ctl, space_info, ins); case BTRFS_EXTENT_ALLOC_ZONED: - return prepare_allocation_zoned(fs_info, ffe_ctl); + return prepare_allocation_zoned(fs_info, ffe_ctl, space_info); default: BUG(); } diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 4020ab19c0a02..d7b8c0e88ddaf 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -104,6 +104,25 @@ struct btrfs_bio_ctrl { blk_opf_t opf; btrfs_bio_end_io_t end_io_func; struct writeback_control *wbc; + struct readahead_control *ractl; + + /* + * The start offset of the last used extent map by a read operation. + * + * This is for proper compressed read merge. + * U64_MAX means we are starting the read and have made no progress yet. + * + * The current btrfs_bio_is_contig() only uses disk_bytenr as + * the condition to check if the read can be merged with previous + * bio, which is not correct. E.g. two file extents pointing to the + * same extent but with different offset. + * + * So here we need to do extra checks to only merge reads that are + * covered by the same extent map. + * Just extent_map::start will be enough, as they are unique + * inside the same inode. + */ + u64 last_em_start; }; static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) @@ -346,6 +365,13 @@ noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, /* step one, find a bunch of delalloc bytes starting at start */ delalloc_start = *start; delalloc_end = 0; + + /* + * If @max_bytes is smaller than a block, btrfs_find_delalloc_range() can + * return early without handling any dirty ranges. + */ + ASSERT(max_bytes >= fs_info->sectorsize); + found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end, max_bytes, &cached_state); if (!found || delalloc_end <= *start || delalloc_start > orig_end) { @@ -376,13 +402,14 @@ noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, delalloc_start, delalloc_end); ASSERT(!ret || ret == -EAGAIN); if (ret == -EAGAIN) { - /* some of the pages are gone, lets avoid looping by - * shortening the size of the delalloc range we're searching + /* + * Some of the pages are gone, lets avoid looping by + * shortening the size of the delalloc range we're searching. */ free_extent_state(cached_state); cached_state = NULL; if (!loops) { - max_bytes = PAGE_SIZE; + max_bytes = fs_info->sectorsize; loops = 1; goto again; } else { @@ -953,6 +980,23 @@ __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset, } return em; } + +static void btrfs_readahead_expand(struct readahead_control *ractl, + const struct extent_map *em) +{ + const u64 ra_pos = readahead_pos(ractl); + const u64 ra_end = ra_pos + readahead_length(ractl); + const u64 em_end = em->start + em->len; + + /* No expansion for holes and inline extents. */ + if (em->block_start > EXTENT_MAP_LAST_BYTE) + return; + + ASSERT(em_end >= ra_pos); + if (em_end > ra_end) + readahead_expand(ractl, ra_pos, em_end - ra_pos); +} + /* * basic readpage implementation. Locked extent state structs are inserted * into the tree that are removed when the IO is done (by the end_io @@ -961,7 +1005,7 @@ __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset, * return 0 on success, otherwise return error */ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, - struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start) + struct btrfs_bio_ctrl *bio_ctrl) { struct inode *inode = page->mapping->host; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); @@ -1033,6 +1077,17 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, iosize = min(extent_map_end(em) - cur, end - cur + 1); iosize = ALIGN(iosize, blocksize); + + /* + * Only expand readahead for extents which are already creating + * the pages anyway in add_ra_bio_pages, which is compressed + * extents in the non subpage case. + */ + if (bio_ctrl->ractl && + !btrfs_is_subpage(fs_info, page) && + compress_type != BTRFS_COMPRESS_NONE) + btrfs_readahead_expand(bio_ctrl->ractl, em); + if (compress_type != BTRFS_COMPRESS_NONE) disk_bytenr = em->block_start; else @@ -1076,12 +1131,11 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, * non-optimal behavior (submitting 2 bios for the same extent). */ if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) && - prev_em_start && *prev_em_start != (u64)-1 && - *prev_em_start != em->start) + bio_ctrl->last_em_start != (u64)-1 && + bio_ctrl->last_em_start != em->start) force_bio_submit = true; - if (prev_em_start) - *prev_em_start = em->start; + bio_ctrl->last_em_start = em->start; free_extent_map(em); em = NULL; @@ -1127,12 +1181,15 @@ int btrfs_read_folio(struct file *file, struct folio *folio) struct btrfs_inode *inode = BTRFS_I(page->mapping->host); u64 start = page_offset(page); u64 end = start + PAGE_SIZE - 1; - struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ }; + struct btrfs_bio_ctrl bio_ctrl = { + .opf = REQ_OP_READ, + .last_em_start = (u64)-1, + }; int ret; btrfs_lock_and_flush_ordered_range(inode, start, end, NULL); - ret = btrfs_do_readpage(page, NULL, &bio_ctrl, NULL); + ret = btrfs_do_readpage(page, NULL, &bio_ctrl); /* * If btrfs_do_readpage() failed we will want to submit the assembled * bio to do the cleanup. @@ -1144,8 +1201,7 @@ int btrfs_read_folio(struct file *file, struct folio *folio) static inline void contiguous_readpages(struct page *pages[], int nr_pages, u64 start, u64 end, struct extent_map **em_cached, - struct btrfs_bio_ctrl *bio_ctrl, - u64 *prev_em_start) + struct btrfs_bio_ctrl *bio_ctrl) { struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host); int index; @@ -1153,8 +1209,7 @@ static inline void contiguous_readpages(struct page *pages[], int nr_pages, btrfs_lock_and_flush_ordered_range(inode, start, end, NULL); for (index = 0; index < nr_pages; index++) { - btrfs_do_readpage(pages[index], em_cached, bio_ctrl, - prev_em_start); + btrfs_do_readpage(pages[index], em_cached, bio_ctrl); put_page(pages[index]); } } @@ -1706,6 +1761,14 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, unlock_page(p); } } + /* + * If the fs is already in error status, do not submit any writeback + * but immediately finish it. + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) { + btrfs_bio_end_io(bbio, errno_to_blk_status(BTRFS_FS_ERROR(fs_info))); + return; + } btrfs_submit_bio(bbio, 0); } @@ -1752,7 +1815,7 @@ static int submit_eb_subpage(struct page *page, struct writeback_control *wbc) subpage->bitmaps)) { spin_unlock_irqrestore(&subpage->lock, flags); spin_unlock(&page->mapping->private_lock); - bit_start++; + bit_start += sectors_per_node; continue; } @@ -2234,10 +2297,13 @@ int extent_writepages(struct address_space *mapping, void extent_readahead(struct readahead_control *rac) { - struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD }; + struct btrfs_bio_ctrl bio_ctrl = { + .opf = REQ_OP_READ | REQ_RAHEAD, + .ractl = rac, + .last_em_start = (u64)-1, + }; struct page *pagepool[16]; struct extent_map *em_cached = NULL; - u64 prev_em_start = (u64)-1; int nr; while ((nr = readahead_page_batch(rac, pagepool))) { @@ -2245,7 +2311,7 @@ void extent_readahead(struct readahead_control *rac) u64 contig_end = contig_start + readahead_batch_length(rac) - 1; contiguous_readpages(pagepool, nr, contig_start, contig_end, - &em_cached, &bio_ctrl, &prev_em_start); + &em_cached, &bio_ctrl); } if (em_cached) @@ -3575,10 +3641,10 @@ struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info, return eb; } -#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info, u64 start) { +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS struct extent_buffer *eb, *exists = NULL; int ret; @@ -3614,8 +3680,11 @@ struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info, free_eb: btrfs_release_extent_buffer(eb); return exists; -} +#else + /* Stub to avoid linker error when compiled with optimizations turned off. */ + return NULL; #endif +} static struct extent_buffer *grab_extent_buffer( struct btrfs_fs_info *fs_info, struct page *page) diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 45cae356e89ba..ea5759a689b97 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -153,7 +153,7 @@ static inline u32 max_ordered_sum_bytes(const struct btrfs_fs_info *fs_info) * Calculate the total size needed to allocate for an ordered sum structure * spanning @bytes in the file. */ -static int btrfs_ordered_sum_size(struct btrfs_fs_info *fs_info, unsigned long bytes) +static int btrfs_ordered_sum_size(const struct btrfs_fs_info *fs_info, unsigned long bytes) { return sizeof(struct btrfs_ordered_sum) + bytes_to_csum_size(fs_info, bytes); } @@ -1263,7 +1263,7 @@ int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, const struct btrfs_path *path, - struct btrfs_file_extent_item *fi, + const struct btrfs_file_extent_item *fi, struct extent_map *em) { struct btrfs_fs_info *fs_info = inode->root->fs_info; diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h index 04bd2d34efb14..2b1d08b88b616 100644 --- a/fs/btrfs/file-item.h +++ b/fs/btrfs/file-item.h @@ -62,7 +62,7 @@ int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path, unsigned long *csum_bitmap); void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, const struct btrfs_path *path, - struct btrfs_file_extent_item *fi, + const struct btrfs_file_extent_item *fi, struct extent_map *em); int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start, u64 len); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index e794606e7c780..9ef543db8aab9 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2825,12 +2825,22 @@ static int btrfs_fallocate_update_isize(struct inode *inode, { struct btrfs_trans_handle *trans; struct btrfs_root *root = BTRFS_I(inode)->root; + u64 range_start; + u64 range_end; int ret; int ret2; if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode)) return 0; + range_start = round_down(i_size_read(inode), root->fs_info->sectorsize); + range_end = round_up(end, root->fs_info->sectorsize); + + ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), range_start, + range_end - range_start); + if (ret) + return ret; + trans = btrfs_start_transaction(root, 1); if (IS_ERR(trans)) return PTR_ERR(trans); diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index a0d8160b53757..1fb635a999760 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -1104,11 +1104,22 @@ static int populate_free_space_tree(struct btrfs_trans_handle *trans, ret = btrfs_search_slot_for_read(extent_root, &key, path, 1, 0); if (ret < 0) goto out_locked; - ASSERT(ret == 0); - + /* + * If ret is 1 (no key found), it means this is an empty block group, + * without any extents allocated from it and there's no block group + * item (key BTRFS_BLOCK_GROUP_ITEM_KEY) located in the extent tree + * because we are using the block group tree feature (so block group + * items are stored in the block group tree) or this is a new block + * group created in the current transaction and its block group item + * was not yet inserted in the extent tree (that happens in + * btrfs_create_pending_block_groups() -> insert_block_group_item()). + * It also means there are no extents allocated for block groups with a + * start offset beyond this block group's end offset (this is the last, + * highest, block group). + */ start = block_group->start; end = block_group->start + block_group->length; - while (1) { + while (ret == 0) { btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); if (key.type == BTRFS_EXTENT_ITEM_KEY || @@ -1138,8 +1149,6 @@ static int populate_free_space_tree(struct btrfs_trans_handle *trans, ret = btrfs_next_item(extent_root, path); if (ret < 0) goto out_locked; - if (ret) - break; } if (start < end) { ret = __add_to_free_space_tree(trans, block_group, path2, @@ -1363,12 +1372,17 @@ static int __add_block_group_free_space(struct btrfs_trans_handle *trans, clear_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, &block_group->runtime_flags); ret = add_new_free_space_info(trans, block_group, path); - if (ret) + if (ret) { + btrfs_abort_transaction(trans, ret); return ret; + } - return __add_to_free_space_tree(trans, block_group, path, - block_group->start, - block_group->length); + ret = __add_to_free_space_tree(trans, block_group, path, + block_group->start, block_group->length); + if (ret) + btrfs_abort_transaction(trans, ret); + + return 0; } int add_block_group_free_space(struct btrfs_trans_handle *trans, @@ -1388,16 +1402,14 @@ int add_block_group_free_space(struct btrfs_trans_handle *trans, path = btrfs_alloc_path(); if (!path) { ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); goto out; } ret = __add_block_group_free_space(trans, block_group, path); - out: btrfs_free_path(path); mutex_unlock(&block_group->free_space_lock); - if (ret) - btrfs_abort_transaction(trans, ret); return ret; } diff --git a/fs/btrfs/inode-item.c b/fs/btrfs/inode-item.c index d3ff97374d48a..ab741a5784240 100644 --- a/fs/btrfs/inode-item.c +++ b/fs/btrfs/inode-item.c @@ -15,7 +15,7 @@ #include "extent-tree.h" #include "file-item.h" -struct btrfs_inode_ref *btrfs_find_name_in_backref(struct extent_buffer *leaf, +struct btrfs_inode_ref *btrfs_find_name_in_backref(const struct extent_buffer *leaf, int slot, const struct fscrypt_str *name) { @@ -43,7 +43,7 @@ struct btrfs_inode_ref *btrfs_find_name_in_backref(struct extent_buffer *leaf, } struct btrfs_inode_extref *btrfs_find_name_in_ext_backref( - struct extent_buffer *leaf, int slot, u64 ref_objectid, + const struct extent_buffer *leaf, int slot, u64 ref_objectid, const struct fscrypt_str *name) { struct btrfs_inode_extref *extref; @@ -424,9 +424,9 @@ int btrfs_lookup_inode(struct btrfs_trans_handle *trans, struct btrfs_root return ret; } -static inline void btrfs_trace_truncate(struct btrfs_inode *inode, - struct extent_buffer *leaf, - struct btrfs_file_extent_item *fi, +static inline void btrfs_trace_truncate(const struct btrfs_inode *inode, + const struct extent_buffer *leaf, + const struct btrfs_file_extent_item *fi, u64 offset, int extent_type, int slot) { if (!inode) diff --git a/fs/btrfs/inode-item.h b/fs/btrfs/inode-item.h index ede43b6c65591..d43633d5620f2 100644 --- a/fs/btrfs/inode-item.h +++ b/fs/btrfs/inode-item.h @@ -100,11 +100,11 @@ struct btrfs_inode_extref *btrfs_lookup_inode_extref( u64 inode_objectid, u64 ref_objectid, int ins_len, int cow); -struct btrfs_inode_ref *btrfs_find_name_in_backref(struct extent_buffer *leaf, +struct btrfs_inode_ref *btrfs_find_name_in_backref(const struct extent_buffer *leaf, int slot, const struct fscrypt_str *name); struct btrfs_inode_extref *btrfs_find_name_in_ext_backref( - struct extent_buffer *leaf, int slot, u64 ref_objectid, + const struct extent_buffer *leaf, int slot, u64 ref_objectid, const struct fscrypt_str *name); #endif diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 11c4d69177f0c..b1be3e0fe7282 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2058,12 +2058,13 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode, /* * If the found extent starts after requested offset, then - * adjust extent_end to be right before this extent begins + * adjust cur_offset to be right before this extent begins. */ if (found_key.offset > cur_offset) { - extent_end = found_key.offset; - extent_type = 0; - goto must_cow; + if (cow_start == (u64)-1) + cow_start = cur_offset; + cur_offset = found_key.offset; + goto next_slot; } /* @@ -3050,9 +3051,10 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent) goto out; } - if (btrfs_is_zoned(fs_info)) - btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr, - ordered_extent->disk_num_bytes); + ret = btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr, + ordered_extent->disk_num_bytes); + if (ret) + goto out; if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { truncated = true; @@ -3076,7 +3078,7 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent) goto out; } trans->block_rsv = &inode->block_rsv; - ret = btrfs_update_inode_fallback(trans, root, inode); + ret = btrfs_update_inode_fallback(trans, inode); if (ret) /* -ENOMEM or corruption */ btrfs_abort_transaction(trans, ret); goto out; @@ -3142,7 +3144,7 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent) &cached_state); btrfs_inode_safe_disk_i_size_write(inode, 0); - ret = btrfs_update_inode_fallback(trans, root, inode); + ret = btrfs_update_inode_fallback(trans, inode); if (ret) { /* -ENOMEM or corruption */ btrfs_abort_transaction(trans, ret); goto out; @@ -3784,10 +3786,8 @@ static int btrfs_read_locked_inode(struct inode *inode, inode_set_ctime(inode, btrfs_timespec_sec(leaf, &inode_item->ctime), btrfs_timespec_nsec(leaf, &inode_item->ctime)); - BTRFS_I(inode)->i_otime.tv_sec = - btrfs_timespec_sec(leaf, &inode_item->otime); - BTRFS_I(inode)->i_otime.tv_nsec = - btrfs_timespec_nsec(leaf, &inode_item->otime); + BTRFS_I(inode)->i_otime_sec = btrfs_timespec_sec(leaf, &inode_item->otime); + BTRFS_I(inode)->i_otime_nsec = btrfs_timespec_nsec(leaf, &inode_item->otime); inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); @@ -3957,10 +3957,8 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, btrfs_set_token_timespec_nsec(&token, &item->ctime, inode_get_ctime(inode).tv_nsec); - btrfs_set_token_timespec_sec(&token, &item->otime, - BTRFS_I(inode)->i_otime.tv_sec); - btrfs_set_token_timespec_nsec(&token, &item->otime, - BTRFS_I(inode)->i_otime.tv_nsec); + btrfs_set_token_timespec_sec(&token, &item->otime, BTRFS_I(inode)->i_otime_sec); + btrfs_set_token_timespec_nsec(&token, &item->otime, BTRFS_I(inode)->i_otime_nsec); btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode)); btrfs_set_token_inode_generation(&token, item, @@ -4013,9 +4011,9 @@ static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, /* * copy everything in the in-memory inode into the btree. */ -noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_inode *inode) +int btrfs_update_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_inode *inode) { struct btrfs_fs_info *fs_info = root->fs_info; int ret; @@ -4042,13 +4040,13 @@ noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, } int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, - struct btrfs_root *root, struct btrfs_inode *inode) + struct btrfs_inode *inode) { int ret; - ret = btrfs_update_inode(trans, root, inode); + ret = btrfs_update_inode(trans, inode->root, inode); if (ret == -ENOSPC) - return btrfs_update_inode_item(trans, root, inode); + return btrfs_update_inode_item(trans, inode->root, inode); return ret; } @@ -4326,7 +4324,7 @@ static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, btrfs_i_size_write(dir, dir->vfs_inode.i_size - fname.disk_name.len * 2); inode_inc_iversion(&dir->vfs_inode); dir->vfs_inode.i_mtime = inode_set_ctime_current(&dir->vfs_inode); - ret = btrfs_update_inode_fallback(trans, root, dir); + ret = btrfs_update_inode_fallback(trans, dir); if (ret) btrfs_abort_transaction(trans, ret); out: @@ -4614,9 +4612,8 @@ static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) { struct inode *inode = d_inode(dentry); struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; - int err = 0; + int ret = 0; struct btrfs_trans_handle *trans; - u64 last_unlink_trans; struct fscrypt_name fname; if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) @@ -4630,55 +4627,56 @@ static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) return btrfs_delete_subvolume(BTRFS_I(dir), dentry); } - err = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname); - if (err) - return err; + ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname); + if (ret) + return ret; /* This needs to handle no-key deletions later on */ trans = __unlink_start_trans(BTRFS_I(dir)); if (IS_ERR(trans)) { - err = PTR_ERR(trans); + ret = PTR_ERR(trans); goto out_notrans; } + /* + * Propagate the last_unlink_trans value of the deleted dir to its + * parent directory. This is to prevent an unrecoverable log tree in the + * case we do something like this: + * 1) create dir foo + * 2) create snapshot under dir foo + * 3) delete the snapshot + * 4) rmdir foo + * 5) mkdir foo + * 6) fsync foo or some file inside foo + * + * This is because we can't unlink other roots when replaying the dir + * deletes for directory foo. + */ + if (BTRFS_I(inode)->last_unlink_trans >= trans->transid) + btrfs_record_snapshot_destroy(trans, BTRFS_I(dir)); + if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { - err = btrfs_unlink_subvol(trans, BTRFS_I(dir), dentry); + ret = btrfs_unlink_subvol(trans, BTRFS_I(dir), dentry); goto out; } - err = btrfs_orphan_add(trans, BTRFS_I(inode)); - if (err) + ret = btrfs_orphan_add(trans, BTRFS_I(inode)); + if (ret) goto out; - last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; - /* now the directory is empty */ - err = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), + ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), &fname.disk_name); - if (!err) { + if (!ret) btrfs_i_size_write(BTRFS_I(inode), 0); - /* - * Propagate the last_unlink_trans value of the deleted dir to - * its parent directory. This is to prevent an unrecoverable - * log tree in the case we do something like this: - * 1) create dir foo - * 2) create snapshot under dir foo - * 3) delete the snapshot - * 4) rmdir foo - * 5) mkdir foo - * 6) fsync foo or some file inside foo - */ - if (last_unlink_trans >= trans->transid) - BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; - } out: btrfs_end_transaction(trans); out_notrans: btrfs_btree_balance_dirty(fs_info); fscrypt_free_filename(&fname); - return err; + return ret; } /* @@ -5643,7 +5641,8 @@ static struct inode *new_simple_dir(struct inode *dir, inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; inode->i_mtime = inode_set_ctime_current(inode); inode->i_atime = dir->i_atime; - BTRFS_I(inode)->i_otime = inode->i_mtime; + BTRFS_I(inode)->i_otime_sec = inode->i_mtime.tv_sec; + BTRFS_I(inode)->i_otime_nsec = inode->i_mtime.tv_nsec; inode->i_uid = dir->i_uid; inode->i_gid = dir->i_gid; @@ -6320,7 +6319,8 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, inode->i_mtime = inode_set_ctime_current(inode); inode->i_atime = inode->i_mtime; - BTRFS_I(inode)->i_otime = inode->i_mtime; + BTRFS_I(inode)->i_otime_sec = inode->i_mtime.tv_sec; + BTRFS_I(inode)->i_otime_nsec = inode->i_mtime.tv_nsec; /* * We're going to fill the inode item now, so at this point the inode @@ -8526,6 +8526,7 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) ei->last_sub_trans = 0; ei->logged_trans = 0; ei->delalloc_bytes = 0; + /* new_delalloc_bytes and last_dir_index_offset are in a union. */ ei->new_delalloc_bytes = 0; ei->defrag_bytes = 0; ei->disk_i_size = 0; @@ -8549,8 +8550,8 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) ei->delayed_node = NULL; - ei->i_otime.tv_sec = 0; - ei->i_otime.tv_nsec = 0; + ei->i_otime_sec = 0; + ei->i_otime_nsec = 0; inode = &ei->vfs_inode; extent_map_tree_init(&ei->extent_tree); @@ -8702,8 +8703,8 @@ static int btrfs_getattr(struct mnt_idmap *idmap, u32 bi_ro_flags = BTRFS_I(inode)->ro_flags; stat->result_mask |= STATX_BTIME; - stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec; - stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec; + stat->btime.tv_sec = BTRFS_I(inode)->i_otime_sec; + stat->btime.tv_nsec = BTRFS_I(inode)->i_otime_nsec; if (bi_flags & BTRFS_INODE_APPEND) stat->attributes |= STATX_ATTR_APPEND; if (bi_flags & BTRFS_INODE_COMPRESS) @@ -8753,6 +8754,7 @@ static int btrfs_rename_exchange(struct inode *old_dir, int ret; int ret2; bool need_abort = false; + bool logs_pinned = false; struct fscrypt_name old_fname, new_fname; struct fscrypt_str *old_name, *new_name; @@ -8876,6 +8878,31 @@ static int btrfs_rename_exchange(struct inode *old_dir, inode_inc_iversion(new_inode); simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + if (old_ino != BTRFS_FIRST_FREE_OBJECTID && + new_ino != BTRFS_FIRST_FREE_OBJECTID) { + /* + * If we are renaming in the same directory (and it's not for + * root entries) pin the log early to prevent any concurrent + * task from logging the directory after we removed the old + * entries and before we add the new entries, otherwise that + * task can sync a log without any entry for the inodes we are + * renaming and therefore replaying that log, if a power failure + * happens after syncing the log, would result in deleting the + * inodes. + * + * If the rename affects two different directories, we want to + * make sure the that there's no log commit that contains + * updates for only one of the directories but not for the + * other. + * + * If we are renaming an entry for a root, we don't care about + * log updates since we called btrfs_set_log_full_commit(). + */ + btrfs_pin_log_trans(root); + btrfs_pin_log_trans(dest); + logs_pinned = true; + } + if (old_dentry->d_parent != new_dentry->d_parent) { btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), BTRFS_I(old_inode), true); @@ -8933,30 +8960,23 @@ static int btrfs_rename_exchange(struct inode *old_dir, BTRFS_I(new_inode)->dir_index = new_idx; /* - * Now pin the logs of the roots. We do it to ensure that no other task - * can sync the logs while we are in progress with the rename, because - * that could result in an inconsistency in case any of the inodes that - * are part of this rename operation were logged before. + * Do the log updates for all inodes. + * + * If either entry is for a root we don't need to update the logs since + * we've called btrfs_set_log_full_commit() before. */ - if (old_ino != BTRFS_FIRST_FREE_OBJECTID) - btrfs_pin_log_trans(root); - if (new_ino != BTRFS_FIRST_FREE_OBJECTID) - btrfs_pin_log_trans(dest); - - /* Do the log updates for all inodes. */ - if (old_ino != BTRFS_FIRST_FREE_OBJECTID) + if (logs_pinned) { btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), old_rename_ctx.index, new_dentry->d_parent); - if (new_ino != BTRFS_FIRST_FREE_OBJECTID) btrfs_log_new_name(trans, new_dentry, BTRFS_I(new_dir), new_rename_ctx.index, old_dentry->d_parent); + } - /* Now unpin the logs. */ - if (old_ino != BTRFS_FIRST_FREE_OBJECTID) +out_fail: + if (logs_pinned) { btrfs_end_log_trans(root); - if (new_ino != BTRFS_FIRST_FREE_OBJECTID) btrfs_end_log_trans(dest); -out_fail: + } ret2 = btrfs_end_transaction(trans); ret = ret ? ret : ret2; out_notrans: @@ -9006,6 +9026,7 @@ static int btrfs_rename(struct mnt_idmap *idmap, int ret2; u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); struct fscrypt_name old_fname, new_fname; + bool logs_pinned = false; if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) return -EPERM; @@ -9140,6 +9161,29 @@ static int btrfs_rename(struct mnt_idmap *idmap, inode_inc_iversion(old_inode); simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + if (old_ino != BTRFS_FIRST_FREE_OBJECTID) { + /* + * If we are renaming in the same directory (and it's not a + * root entry) pin the log to prevent any concurrent task from + * logging the directory after we removed the old entry and + * before we add the new entry, otherwise that task can sync + * a log without any entry for the inode we are renaming and + * therefore replaying that log, if a power failure happens + * after syncing the log, would result in deleting the inode. + * + * If the rename affects two different directories, we want to + * make sure the that there's no log commit that contains + * updates for only one of the directories but not for the + * other. + * + * If we are renaming an entry for a root, we don't care about + * log updates since we called btrfs_set_log_full_commit(). + */ + btrfs_pin_log_trans(root); + btrfs_pin_log_trans(dest); + logs_pinned = true; + } + if (old_dentry->d_parent != new_dentry->d_parent) btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), BTRFS_I(old_inode), true); @@ -9188,7 +9232,7 @@ static int btrfs_rename(struct mnt_idmap *idmap, if (old_inode->i_nlink == 1) BTRFS_I(old_inode)->dir_index = index; - if (old_ino != BTRFS_FIRST_FREE_OBJECTID) + if (logs_pinned) btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), rename_ctx.index, new_dentry->d_parent); @@ -9204,6 +9248,10 @@ static int btrfs_rename(struct mnt_idmap *idmap, } } out_fail: + if (logs_pinned) { + btrfs_end_log_trans(root); + btrfs_end_log_trans(dest); + } ret2 = btrfs_end_transaction(trans); ret = ret ? ret : ret2; out_notrans: diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 86d846eb5ed49..c68e9ecbc438c 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -154,9 +154,10 @@ static struct btrfs_ordered_extent *alloc_ordered_extent( struct btrfs_ordered_extent *entry; int ret; u64 qgroup_rsv = 0; + const bool is_nocow = (flags & + ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))); - if (flags & - ((1 << BTRFS_ORDERED_NOCOW) | (1 << BTRFS_ORDERED_PREALLOC))) { + if (is_nocow) { /* For nocow write, we can release the qgroup rsv right now */ ret = btrfs_qgroup_free_data(inode, NULL, file_offset, num_bytes, &qgroup_rsv); if (ret < 0) @@ -171,8 +172,13 @@ static struct btrfs_ordered_extent *alloc_ordered_extent( return ERR_PTR(ret); } entry = kmem_cache_zalloc(btrfs_ordered_extent_cache, GFP_NOFS); - if (!entry) + if (!entry) { + if (!is_nocow) + btrfs_qgroup_free_refroot(inode->root->fs_info, + btrfs_root_id(inode->root), + qgroup_rsv, BTRFS_QGROUP_RSV_DATA); return ERR_PTR(-ENOMEM); + } entry->file_offset = file_offset; entry->num_bytes = num_bytes; diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 1b9f4f16d1240..c46ea2ecf1881 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -579,22 +579,30 @@ bool btrfs_check_quota_leak(struct btrfs_fs_info *fs_info) /* * This is called from close_ctree() or open_ctree() or btrfs_quota_disable(), - * first two are in single-threaded paths.And for the third one, we have set - * quota_root to be null with qgroup_lock held before, so it is safe to clean - * up the in-memory structures without qgroup_lock held. + * first two are in single-threaded paths. */ void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info) { struct rb_node *n; struct btrfs_qgroup *qgroup; + /* + * btrfs_quota_disable() can be called concurrently with + * btrfs_qgroup_rescan() -> qgroup_rescan_zero_tracking(), so take the + * lock. + */ + spin_lock(&fs_info->qgroup_lock); while ((n = rb_first(&fs_info->qgroup_tree))) { qgroup = rb_entry(n, struct btrfs_qgroup, node); rb_erase(n, &fs_info->qgroup_tree); __del_qgroup_rb(fs_info, qgroup); + spin_unlock(&fs_info->qgroup_lock); btrfs_sysfs_del_one_qgroup(fs_info, qgroup); kfree(qgroup); + spin_lock(&fs_info->qgroup_lock); } + spin_unlock(&fs_info->qgroup_lock); + /* * We call btrfs_free_qgroup_config() when unmounting * filesystem and disabling quota, so we set qgroup_ulist @@ -3616,12 +3624,21 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info) qgroup_rescan_zero_tracking(fs_info); mutex_lock(&fs_info->qgroup_rescan_lock); - fs_info->qgroup_rescan_running = true; - btrfs_queue_work(fs_info->qgroup_rescan_workers, - &fs_info->qgroup_rescan_work); + /* + * The rescan worker is only for full accounting qgroups, check if it's + * enabled as it is pointless to queue it otherwise. A concurrent quota + * disable may also have just cleared BTRFS_FS_QUOTA_ENABLED. + */ + if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) { + fs_info->qgroup_rescan_running = true; + btrfs_queue_work(fs_info->qgroup_rescan_workers, + &fs_info->qgroup_rescan_work); + } else { + ret = -ENOTCONN; + } mutex_unlock(&fs_info->qgroup_rescan_lock); - return 0; + return ret; } int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/ref-verify.c b/fs/btrfs/ref-verify.c index 28ac7995716e0..b8122582d7555 100644 --- a/fs/btrfs/ref-verify.c +++ b/fs/btrfs/ref-verify.c @@ -990,11 +990,18 @@ int btrfs_build_ref_tree(struct btrfs_fs_info *fs_info) if (!btrfs_test_opt(fs_info, REF_VERIFY)) return 0; + extent_root = btrfs_extent_root(fs_info, 0); + /* If the extent tree is damaged we cannot ignore it (IGNOREBADROOTS). */ + if (IS_ERR(extent_root)) { + btrfs_warn(fs_info, "ref-verify: extent tree not available, disabling"); + btrfs_clear_opt(fs_info->mount_opt, REF_VERIFY); + return 0; + } + path = btrfs_alloc_path(); if (!path) return -ENOMEM; - extent_root = btrfs_extent_root(fs_info, 0); eb = btrfs_read_lock_root_node(extent_root); level = btrfs_header_level(eb); path->nodes[level] = eb; diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 537e184b4b1df..b21b5e0f89054 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -693,6 +693,25 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, if (root->root_key.objectid == objectid) { u64 commit_root_gen; + /* + * Relocation will wait for cleaner thread, and any half-dropped + * subvolume will be fully cleaned up at mount time. + * So here we shouldn't hit a subvolume with non-zero drop_progress. + * + * If this isn't the case, error out since it can make us attempt to + * drop references for extents that were already dropped before. + */ + if (unlikely(btrfs_disk_key_objectid(&root->root_item.drop_progress))) { + struct btrfs_key cpu_key; + + btrfs_disk_key_to_cpu(&cpu_key, &root->root_item.drop_progress); + btrfs_err(fs_info, + "cannot relocate partially dropped subvolume %llu, drop progress key (%llu %u %llu)", + objectid, cpu_key.objectid, cpu_key.type, cpu_key.offset); + ret = -EUCLEAN; + goto fail; + } + /* called by btrfs_init_reloc_root */ ret = btrfs_copy_root(trans, root, root->commit_root, &eb, BTRFS_TREE_RELOC_OBJECTID); @@ -2931,6 +2950,7 @@ static int relocate_one_page(struct inode *inode, struct file_ra_state *ra, int ret; ASSERT(page_index <= last_index); +again: page = find_lock_page(inode->i_mapping, page_index); if (!page) { page_cache_sync_readahead(inode->i_mapping, ra, NULL, @@ -2952,6 +2972,11 @@ static int relocate_one_page(struct inode *inode, struct file_ra_state *ra, ret = -EIO; goto release_page; } + if (page->mapping != inode->i_mapping) { + unlock_page(page); + put_page(page); + goto again; + } } /* @@ -3852,6 +3877,7 @@ static noinline_for_stack struct inode *create_reloc_inode( /* * Mark start of chunk relocation that is cancellable. Check if the cancellation * has been requested meanwhile and don't start in that case. + * NOTE: if this returns an error, reloc_chunk_end() must not be called. * * Return: * 0 success @@ -3868,10 +3894,8 @@ static int reloc_chunk_start(struct btrfs_fs_info *fs_info) if (atomic_read(&fs_info->reloc_cancel_req) > 0) { btrfs_info(fs_info, "chunk relocation canceled on start"); - /* - * On cancel, clear all requests but let the caller mark - * the end after cleanup operations. - */ + /* On cancel, clear all requests. */ + clear_and_wake_up_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags); atomic_set(&fs_info->reloc_cancel_req, 0); return -ECANCELED; } @@ -3880,9 +3904,11 @@ static int reloc_chunk_start(struct btrfs_fs_info *fs_info) /* * Mark end of chunk relocation that is cancellable and wake any waiters. + * NOTE: call only if a previous call to reloc_chunk_start() succeeded. */ static void reloc_chunk_end(struct btrfs_fs_info *fs_info) { + ASSERT(test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)); /* Requested after start, clear bit first so any waiters can continue */ if (atomic_read(&fs_info->reloc_cancel_req) > 0) btrfs_info(fs_info, "chunk relocation canceled during operation"); @@ -4094,9 +4120,9 @@ int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start) if (err && rw) btrfs_dec_block_group_ro(rc->block_group); iput(rc->data_inode); + reloc_chunk_end(fs_info); out_put_bg: btrfs_put_block_group(bg); - reloc_chunk_end(fs_info); free_reloc_control(rc); return err; } @@ -4286,8 +4312,8 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) err = ret; out_unset: unset_reloc_control(rc); -out_end: reloc_chunk_end(fs_info); +out_end: free_reloc_control(rc); out: free_reloc_roots(&reloc_roots); diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 6be092bb814fd..a69dce114c5ba 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -153,12 +153,14 @@ struct scrub_stripe { unsigned int init_nr_io_errors; unsigned int init_nr_csum_errors; unsigned int init_nr_meta_errors; + unsigned int init_nr_meta_gen_errors; /* * The following error bitmaps are all for the current status. * Every time we submit a new read, these bitmaps may be updated. * - * error_bitmap = io_error_bitmap | csum_error_bitmap | meta_error_bitmap; + * error_bitmap = io_error_bitmap | csum_error_bitmap | + * meta_error_bitmap | meta_generation_bitmap; * * IO and csum errors can happen for both metadata and data. */ @@ -166,6 +168,7 @@ struct scrub_stripe { unsigned long io_error_bitmap; unsigned long csum_error_bitmap; unsigned long meta_error_bitmap; + unsigned long meta_gen_error_bitmap; /* For writeback (repair or replace) error reporting. */ unsigned long write_error_bitmap; @@ -617,7 +620,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr memcpy(on_disk_csum, header->csum, fs_info->csum_size); if (logical != btrfs_stack_header_bytenr(header)) { - bitmap_set(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree); + bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "tree block %llu mirror %u has bad bytenr, has %llu want %llu", @@ -673,7 +676,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr } if (stripe->sectors[sector_nr].generation != btrfs_stack_header_generation(header)) { - bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); + bitmap_set(&stripe->meta_gen_error_bitmap, sector_nr, sectors_per_tree); bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "tree block %llu mirror %u has bad generation, has %llu want %llu", @@ -685,6 +688,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr bitmap_clear(&stripe->error_bitmap, sector_nr, sectors_per_tree); bitmap_clear(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree); bitmap_clear(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); + bitmap_clear(&stripe->meta_gen_error_bitmap, sector_nr, sectors_per_tree); } static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr) @@ -973,8 +977,22 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, if (__ratelimit(&rs) && dev) scrub_print_common_warning("header error", dev, false, stripe->logical, physical); + if (test_bit(sector_nr, &stripe->meta_gen_error_bitmap)) + if (__ratelimit(&rs) && dev) + scrub_print_common_warning("generation error", dev, false, + stripe->logical, physical); } + /* Update the device stats. */ + for (int i = 0; i < stripe->init_nr_io_errors; i++) + btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_READ_ERRS); + for (int i = 0; i < stripe->init_nr_csum_errors; i++) + btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); + /* Generation mismatch error is based on each metadata, not each block. */ + for (int i = 0; i < stripe->init_nr_meta_gen_errors; + i += (fs_info->nodesize >> fs_info->sectorsize_bits)) + btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_GENERATION_ERRS); + spin_lock(&sctx->stat_lock); sctx->stat.data_extents_scrubbed += stripe->nr_data_extents; sctx->stat.tree_extents_scrubbed += stripe->nr_meta_extents; @@ -983,7 +1001,8 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, sctx->stat.no_csum += nr_nodatacsum_sectors; sctx->stat.read_errors += stripe->init_nr_io_errors; sctx->stat.csum_errors += stripe->init_nr_csum_errors; - sctx->stat.verify_errors += stripe->init_nr_meta_errors; + sctx->stat.verify_errors += stripe->init_nr_meta_errors + + stripe->init_nr_meta_gen_errors; sctx->stat.uncorrectable_errors += bitmap_weight(&stripe->error_bitmap, stripe->nr_sectors); sctx->stat.corrected_errors += nr_repaired_sectors; @@ -1029,6 +1048,8 @@ static void scrub_stripe_read_repair_worker(struct work_struct *work) stripe->nr_sectors); stripe->init_nr_meta_errors = bitmap_weight(&stripe->meta_error_bitmap, stripe->nr_sectors); + stripe->init_nr_meta_gen_errors = bitmap_weight(&stripe->meta_gen_error_bitmap, + stripe->nr_sectors); if (bitmap_empty(&stripe->init_error_bitmap, stripe->nr_sectors)) goto out; @@ -1143,6 +1164,9 @@ static void scrub_write_endio(struct btrfs_bio *bbio) bitmap_set(&stripe->write_error_bitmap, sector_nr, bio_size >> fs_info->sectorsize_bits); spin_unlock_irqrestore(&stripe->write_error_lock, flags); + for (int i = 0; i < (bio_size >> fs_info->sectorsize_bits); i++) + btrfs_dev_stat_inc_and_print(stripe->dev, + BTRFS_DEV_STAT_WRITE_ERRS); } bio_put(&bbio->bio); @@ -1247,8 +1271,7 @@ static void scrub_throttle_dev_io(struct scrub_ctx *sctx, struct btrfs_device *d * Slice is divided into intervals when the IO is submitted, adjust by * bwlimit and maximum of 64 intervals. */ - div = max_t(u32, 1, (u32)(bwlimit / (16 * 1024 * 1024))); - div = min_t(u32, 64, div); + div = clamp(bwlimit / (16 * 1024 * 1024), 1, 64); /* Start new epoch, set deadline */ now = ktime_get(); @@ -1505,10 +1528,12 @@ static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe) stripe->init_nr_io_errors = 0; stripe->init_nr_csum_errors = 0; stripe->init_nr_meta_errors = 0; + stripe->init_nr_meta_gen_errors = 0; stripe->error_bitmap = 0; stripe->io_error_bitmap = 0; stripe->csum_error_bitmap = 0; stripe->meta_error_bitmap = 0; + stripe->meta_gen_error_bitmap = 0; } /* @@ -1538,8 +1563,8 @@ static int scrub_find_fill_first_stripe(struct btrfs_block_group *bg, u64 extent_gen; int ret; - if (unlikely(!extent_root)) { - btrfs_err(fs_info, "no valid extent root for scrub"); + if (unlikely(!extent_root || !csum_root)) { + btrfs_err(fs_info, "no valid extent or csum root for scrub"); return -EUCLEAN; } memset(stripe->sectors, 0, sizeof(struct scrub_sector_verification) * @@ -1975,6 +2000,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx, ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, full_stripe_start, &length, &bioc, NULL, NULL, 1); if (ret < 0) { + bio_put(bio); btrfs_put_bioc(bioc); btrfs_bio_counter_dec(fs_info); goto out; @@ -1984,6 +2010,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx, btrfs_put_bioc(bioc); if (!rbio) { ret = -ENOMEM; + bio_put(bio); btrfs_bio_counter_dec(fs_info); goto out; } diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index aa1e6d88a72c7..c25eb4416a671 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -179,6 +180,7 @@ struct send_ctx { u64 cur_inode_rdev; u64 cur_inode_last_extent; u64 cur_inode_next_write_offset; + struct fs_path cur_inode_path; bool cur_inode_new; bool cur_inode_new_gen; bool cur_inode_deleted; @@ -436,6 +438,14 @@ static void fs_path_reset(struct fs_path *p) } } +static void init_path(struct fs_path *p) +{ + p->reversed = 0; + p->buf = p->inline_buf; + p->buf_len = FS_PATH_INLINE_SIZE; + fs_path_reset(p); +} + static struct fs_path *fs_path_alloc(void) { struct fs_path *p; @@ -443,10 +453,7 @@ static struct fs_path *fs_path_alloc(void) p = kmalloc(sizeof(*p), GFP_KERNEL); if (!p) return NULL; - p->reversed = 0; - p->buf = p->inline_buf; - p->buf_len = FS_PATH_INLINE_SIZE; - fs_path_reset(p); + init_path(p); return p; } @@ -471,7 +478,7 @@ static void fs_path_free(struct fs_path *p) kfree(p); } -static int fs_path_len(struct fs_path *p) +static inline int fs_path_len(const struct fs_path *p) { return p->end - p->start; } @@ -487,10 +494,8 @@ static int fs_path_ensure_buf(struct fs_path *p, int len) if (p->buf_len >= len) return 0; - if (len > PATH_MAX) { - WARN_ON(1); - return -ENOMEM; - } + if (WARN_ON(len > PATH_MAX)) + return -ENAMETOOLONG; path_len = p->end - p->start; old_buf_len = p->buf_len; @@ -626,6 +631,14 @@ static void fs_path_unreverse(struct fs_path *p) p->reversed = 0; } +static inline bool is_current_inode_path(const struct send_ctx *sctx, + const struct fs_path *path) +{ + const struct fs_path *cur = &sctx->cur_inode_path; + + return (strncmp(path->start, cur->start, fs_path_len(cur)) == 0); +} + static struct btrfs_path *alloc_path_for_send(void) { struct btrfs_path *path; @@ -2452,6 +2465,14 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, u64 parent_inode = 0; u64 parent_gen = 0; int stop = 0; + const bool is_cur_inode = (ino == sctx->cur_ino && gen == sctx->cur_inode_gen); + + if (is_cur_inode && fs_path_len(&sctx->cur_inode_path) > 0) { + if (dest != &sctx->cur_inode_path) + return fs_path_copy(dest, &sctx->cur_inode_path); + + return 0; + } name = fs_path_alloc(); if (!name) { @@ -2503,8 +2524,12 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, out: fs_path_free(name); - if (!ret) + if (!ret) { fs_path_unreverse(dest); + if (is_cur_inode && dest != &sctx->cur_inode_path) + ret = fs_path_copy(&sctx->cur_inode_path, dest); + } + return ret; } @@ -2599,6 +2624,47 @@ static int send_subvol_begin(struct send_ctx *sctx) return ret; } +static struct fs_path *get_cur_inode_path(struct send_ctx *sctx) +{ + if (fs_path_len(&sctx->cur_inode_path) == 0) { + int ret; + + ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, + &sctx->cur_inode_path); + if (ret < 0) + return ERR_PTR(ret); + } + + return &sctx->cur_inode_path; +} + +static struct fs_path *get_path_for_command(struct send_ctx *sctx, u64 ino, u64 gen) +{ + struct fs_path *path; + int ret; + + if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen) + return get_cur_inode_path(sctx); + + path = fs_path_alloc(); + if (!path) + return ERR_PTR(-ENOMEM); + + ret = get_cur_path(sctx, ino, gen, path); + if (ret < 0) { + fs_path_free(path); + return ERR_PTR(ret); + } + + return path; +} + +static void free_path_for_command(const struct send_ctx *sctx, struct fs_path *path) +{ + if (path != &sctx->cur_inode_path) + fs_path_free(path); +} + static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size) { struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; @@ -2607,17 +2673,14 @@ static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size) btrfs_debug(fs_info, "send_truncate %llu size=%llu", ino, size); - p = fs_path_alloc(); - if (!p) - return -ENOMEM; + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE); if (ret < 0) goto out; - ret = get_cur_path(sctx, ino, gen, p); - if (ret < 0) - goto out; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size); @@ -2625,7 +2688,7 @@ static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size) tlv_put_failure: out: - fs_path_free(p); + free_path_for_command(sctx, p); return ret; } @@ -2637,17 +2700,14 @@ static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode) btrfs_debug(fs_info, "send_chmod %llu mode=%llu", ino, mode); - p = fs_path_alloc(); - if (!p) - return -ENOMEM; + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD); if (ret < 0) goto out; - ret = get_cur_path(sctx, ino, gen, p); - if (ret < 0) - goto out; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777); @@ -2655,7 +2715,7 @@ static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode) tlv_put_failure: out: - fs_path_free(p); + free_path_for_command(sctx, p); return ret; } @@ -2670,17 +2730,14 @@ static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr) btrfs_debug(fs_info, "send_fileattr %llu fileattr=%llu", ino, fileattr); - p = fs_path_alloc(); - if (!p) - return -ENOMEM; + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR); if (ret < 0) goto out; - ret = get_cur_path(sctx, ino, gen, p); - if (ret < 0) - goto out; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr); @@ -2688,7 +2745,7 @@ static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr) tlv_put_failure: out: - fs_path_free(p); + free_path_for_command(sctx, p); return ret; } @@ -2701,17 +2758,14 @@ static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid) btrfs_debug(fs_info, "send_chown %llu uid=%llu, gid=%llu", ino, uid, gid); - p = fs_path_alloc(); - if (!p) - return -ENOMEM; + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN); if (ret < 0) goto out; - ret = get_cur_path(sctx, ino, gen, p); - if (ret < 0) - goto out; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid); TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid); @@ -2720,7 +2774,7 @@ static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid) tlv_put_failure: out: - fs_path_free(p); + free_path_for_command(sctx, p); return ret; } @@ -2737,9 +2791,9 @@ static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen) btrfs_debug(fs_info, "send_utimes %llu", ino); - p = fs_path_alloc(); - if (!p) - return -ENOMEM; + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); path = alloc_path_for_send(); if (!path) { @@ -2764,9 +2818,6 @@ static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen) if (ret < 0) goto out; - ret = get_cur_path(sctx, ino, gen, p); - if (ret < 0) - goto out; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime); TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime); @@ -2778,7 +2829,7 @@ static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen) tlv_put_failure: out: - fs_path_free(p); + free_path_for_command(sctx, p); btrfs_free_path(path); return ret; } @@ -3115,6 +3166,11 @@ static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, goto out; ret = send_rename(sctx, path, orphan); + if (ret < 0) + goto out; + + if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen) + ret = fs_path_copy(&sctx->cur_inode_path, orphan); out: fs_path_free(orphan); @@ -4168,6 +4224,23 @@ static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref) return ret; } +static int rename_current_inode(struct send_ctx *sctx, + struct fs_path *current_path, + struct fs_path *new_path) +{ + int ret; + + ret = send_rename(sctx, current_path, new_path); + if (ret < 0) + return ret; + + ret = fs_path_copy(&sctx->cur_inode_path, new_path); + if (ret < 0) + return ret; + + return fs_path_copy(current_path, new_path); +} + /* * This does all the move/link/unlink/rmdir magic. */ @@ -4182,9 +4255,9 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) u64 ow_inode = 0; u64 ow_gen; u64 ow_mode; - int did_overwrite = 0; - int is_orphan = 0; u64 last_dir_ino_rm = 0; + bool did_overwrite = false; + bool is_orphan = false; bool can_rename = true; bool orphanized_dir = false; bool orphanized_ancestor = false; @@ -4226,14 +4299,14 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) if (ret < 0) goto out; if (ret) - did_overwrite = 1; + did_overwrite = true; } if (sctx->cur_inode_new || did_overwrite) { ret = gen_unique_name(sctx, sctx->cur_ino, sctx->cur_inode_gen, valid_path); if (ret < 0) goto out; - is_orphan = 1; + is_orphan = true; } else { ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, valid_path); @@ -4358,6 +4431,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) if (ret > 0) { orphanized_ancestor = true; fs_path_reset(valid_path); + fs_path_reset(&sctx->cur_inode_path); ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, valid_path); @@ -4453,13 +4527,10 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) * it depending on the inode mode. */ if (is_orphan && can_rename) { - ret = send_rename(sctx, valid_path, cur->full_path); - if (ret < 0) - goto out; - is_orphan = 0; - ret = fs_path_copy(valid_path, cur->full_path); + ret = rename_current_inode(sctx, valid_path, cur->full_path); if (ret < 0) goto out; + is_orphan = false; } else if (can_rename) { if (S_ISDIR(sctx->cur_inode_mode)) { /* @@ -4467,10 +4538,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) * dirs, we always have one new and one deleted * ref. The deleted ref is ignored later. */ - ret = send_rename(sctx, valid_path, - cur->full_path); - if (!ret) - ret = fs_path_copy(valid_path, + ret = rename_current_inode(sctx, valid_path, cur->full_path); if (ret < 0) goto out; @@ -4517,7 +4585,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) sctx->cur_inode_gen, valid_path); if (ret < 0) goto out; - is_orphan = 1; + is_orphan = true; } list_for_each_entry(cur, &sctx->deleted_refs, list) { @@ -4563,6 +4631,8 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) ret = send_unlink(sctx, cur->full_path); if (ret < 0) goto out; + if (is_current_inode_path(sctx, cur->full_path)) + fs_path_reset(&sctx->cur_inode_path); } ret = dup_ref(cur, &check_dirs); if (ret < 0) @@ -4881,11 +4951,15 @@ static int process_all_refs(struct send_ctx *sctx, } static int send_set_xattr(struct send_ctx *sctx, - struct fs_path *path, const char *name, int name_len, const char *data, int data_len) { - int ret = 0; + struct fs_path *path; + int ret; + + path = get_cur_inode_path(sctx); + if (IS_ERR(path)) + return PTR_ERR(path); ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR); if (ret < 0) @@ -4926,19 +5000,13 @@ static int __process_new_xattr(int num, struct btrfs_key *di_key, const char *name, int name_len, const char *data, int data_len, void *ctx) { - int ret; struct send_ctx *sctx = ctx; - struct fs_path *p; struct posix_acl_xattr_header dummy_acl; /* Capabilities are emitted by finish_inode_if_needed */ if (!strncmp(name, XATTR_NAME_CAPS, name_len)) return 0; - p = fs_path_alloc(); - if (!p) - return -ENOMEM; - /* * This hack is needed because empty acls are stored as zero byte * data in xattrs. Problem with that is, that receiving these zero byte @@ -4955,38 +5023,21 @@ static int __process_new_xattr(int num, struct btrfs_key *di_key, } } - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); - if (ret < 0) - goto out; - - ret = send_set_xattr(sctx, p, name, name_len, data, data_len); - -out: - fs_path_free(p); - return ret; + return send_set_xattr(sctx, name, name_len, data, data_len); } static int __process_deleted_xattr(int num, struct btrfs_key *di_key, const char *name, int name_len, const char *data, int data_len, void *ctx) { - int ret; struct send_ctx *sctx = ctx; struct fs_path *p; - p = fs_path_alloc(); - if (!p) - return -ENOMEM; - - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); - if (ret < 0) - goto out; - - ret = send_remove_xattr(sctx, p, name, name_len); + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); -out: - fs_path_free(p); - return ret; + return send_remove_xattr(sctx, p, name, name_len); } static int process_new_xattr(struct send_ctx *sctx) @@ -5220,21 +5271,13 @@ static int process_verity(struct send_ctx *sctx) if (ret < 0) goto iput; - p = fs_path_alloc(); - if (!p) { - ret = -ENOMEM; + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) { + ret = PTR_ERR(p); goto iput; } - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); - if (ret < 0) - goto free_path; ret = send_verity(sctx, p, sctx->verity_descriptor); - if (ret < 0) - goto free_path; - -free_path: - fs_path_free(p); iput: iput(inode); return ret; @@ -5349,31 +5392,25 @@ static int send_write(struct send_ctx *sctx, u64 offset, u32 len) int ret = 0; struct fs_path *p; - p = fs_path_alloc(); - if (!p) - return -ENOMEM; - btrfs_debug(fs_info, "send_write offset=%llu, len=%d", offset, len); - ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); - if (ret < 0) - goto out; + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); + ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); if (ret < 0) - goto out; + return ret; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); ret = put_file_data(sctx, offset, len); if (ret < 0) - goto out; + return ret; ret = send_cmd(sctx); tlv_put_failure: -out: - fs_path_free(p); return ret; } @@ -5386,6 +5423,7 @@ static int send_clone(struct send_ctx *sctx, { int ret = 0; struct fs_path *p; + struct fs_path *cur_inode_path; u64 gen; btrfs_debug(sctx->send_root->fs_info, @@ -5393,6 +5431,10 @@ static int send_clone(struct send_ctx *sctx, offset, len, clone_root->root->root_key.objectid, clone_root->ino, clone_root->offset); + cur_inode_path = get_cur_inode_path(sctx); + if (IS_ERR(cur_inode_path)) + return PTR_ERR(cur_inode_path); + p = fs_path_alloc(); if (!p) return -ENOMEM; @@ -5401,13 +5443,9 @@ static int send_clone(struct send_ctx *sctx, if (ret < 0) goto out; - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); - if (ret < 0) - goto out; - TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len); - TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, cur_inode_path); if (clone_root->root == sctx->send_root) { ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen); @@ -5458,27 +5496,45 @@ static int send_update_extent(struct send_ctx *sctx, int ret = 0; struct fs_path *p; - p = fs_path_alloc(); - if (!p) - return -ENOMEM; + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT); if (ret < 0) - goto out; + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len); + + ret = send_cmd(sctx); - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); +tlv_put_failure: + return ret; +} + +static int send_fallocate(struct send_ctx *sctx, u32 mode, u64 offset, u64 len) +{ + struct fs_path *path; + int ret; + + path = get_cur_inode_path(sctx); + if (IS_ERR(path)) + return PTR_ERR(path); + + ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE); if (ret < 0) - goto out; + return ret; - TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_MODE, mode); TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len); ret = send_cmd(sctx); tlv_put_failure: -out: - fs_path_free(p); return ret; } @@ -5489,6 +5545,14 @@ static int send_hole(struct send_ctx *sctx, u64 end) u64 offset = sctx->cur_inode_last_extent; int ret = 0; + /* + * Starting with send stream v2 we have fallocate and can use it to + * punch holes instead of sending writes full of zeroes. + */ + if (proto_cmd_ok(sctx, BTRFS_SEND_C_FALLOCATE)) + return send_fallocate(sctx, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + offset, end - offset); + /* * A hole that starts at EOF or beyond it. Since we do not yet support * fallocate (for extent preallocation and hole punching), sending a @@ -5507,12 +5571,10 @@ static int send_hole(struct send_ctx *sctx, u64 end) if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) return send_update_extent(sctx, offset, end - offset); - p = fs_path_alloc(); - if (!p) - return -ENOMEM; - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); - if (ret < 0) - goto tlv_put_failure; + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); + while (offset < end) { u64 len = min(end - offset, read_size); @@ -5533,7 +5595,6 @@ static int send_hole(struct send_ctx *sctx, u64 end) } sctx->cur_inode_next_write_offset = offset; tlv_put_failure: - fs_path_free(p); return ret; } @@ -5556,9 +5617,9 @@ static int send_encoded_inline_extent(struct send_ctx *sctx, if (IS_ERR(inode)) return PTR_ERR(inode); - fspath = fs_path_alloc(); - if (!fspath) { - ret = -ENOMEM; + fspath = get_cur_inode_path(sctx); + if (IS_ERR(fspath)) { + ret = PTR_ERR(fspath); goto out; } @@ -5566,10 +5627,6 @@ static int send_encoded_inline_extent(struct send_ctx *sctx, if (ret < 0) goto out; - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); - if (ret < 0) - goto out; - btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei); @@ -5598,7 +5655,6 @@ static int send_encoded_inline_extent(struct send_ctx *sctx, tlv_put_failure: out: - fs_path_free(fspath); iput(inode); return ret; } @@ -5623,9 +5679,9 @@ static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path, if (IS_ERR(inode)) return PTR_ERR(inode); - fspath = fs_path_alloc(); - if (!fspath) { - ret = -ENOMEM; + fspath = get_cur_inode_path(sctx); + if (IS_ERR(fspath)) { + ret = PTR_ERR(fspath); goto out; } @@ -5633,10 +5689,6 @@ static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path, if (ret < 0) goto out; - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); - if (ret < 0) - goto out; - btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); @@ -5703,7 +5755,6 @@ static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path, tlv_put_failure: out: - fs_path_free(fspath); iput(inode); return ret; } @@ -5833,7 +5884,6 @@ static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path, */ static int send_capabilities(struct send_ctx *sctx) { - struct fs_path *fspath = NULL; struct btrfs_path *path; struct btrfs_dir_item *di; struct extent_buffer *leaf; @@ -5859,25 +5909,19 @@ static int send_capabilities(struct send_ctx *sctx) leaf = path->nodes[0]; buf_len = btrfs_dir_data_len(leaf, di); - fspath = fs_path_alloc(); buf = kmalloc(buf_len, GFP_KERNEL); - if (!fspath || !buf) { + if (!buf) { ret = -ENOMEM; goto out; } - ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); - if (ret < 0) - goto out; - data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di); read_extent_buffer(leaf, buf, data_ptr, buf_len); - ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS, + ret = send_set_xattr(sctx, XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), buf, buf_len); out: kfree(buf); - fs_path_free(fspath); btrfs_free_path(path); return ret; } @@ -6906,6 +6950,7 @@ static int changed_inode(struct send_ctx *sctx, sctx->cur_inode_last_extent = (u64)-1; sctx->cur_inode_next_write_offset = 0; sctx->ignore_cur_inode = false; + fs_path_reset(&sctx->cur_inode_path); /* * Set send_progress to current inode. This will tell all get_cur_xxx @@ -8196,6 +8241,7 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) goto out; } + init_path(&sctx->cur_inode_path); INIT_LIST_HEAD(&sctx->new_refs); INIT_LIST_HEAD(&sctx->deleted_refs); @@ -8481,6 +8527,9 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) btrfs_lru_cache_clear(&sctx->dir_created_cache); btrfs_lru_cache_clear(&sctx->dir_utimes_cache); + if (sctx->cur_inode_path.buf != sctx->cur_inode_path.inline_buf) + kfree(sctx->cur_inode_path.buf); + kfree(sctx); } diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 581bdd709ee0d..27690c518f6d7 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -162,7 +162,7 @@ * thing with or without extra unallocated space. */ -u64 __pure btrfs_space_info_used(struct btrfs_space_info *s_info, +u64 __pure btrfs_space_info_used(const struct btrfs_space_info *s_info, bool may_use_included) { ASSERT(s_info); @@ -342,7 +342,7 @@ struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info, } static u64 calc_available_free_space(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, + const struct btrfs_space_info *space_info, enum btrfs_reserve_flush_enum flush) { u64 profile; @@ -378,7 +378,7 @@ static u64 calc_available_free_space(struct btrfs_fs_info *fs_info, } int btrfs_can_overcommit(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, u64 bytes, + const struct btrfs_space_info *space_info, u64 bytes, enum btrfs_reserve_flush_enum flush) { u64 avail; @@ -483,8 +483,8 @@ static void dump_global_block_rsv(struct btrfs_fs_info *fs_info) DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv); } -static void __btrfs_dump_space_info(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *info) +static void __btrfs_dump_space_info(const struct btrfs_fs_info *fs_info, + const struct btrfs_space_info *info) { const char *flag_str = space_info_flag_to_str(info); lockdep_assert_held(&info->lock); @@ -807,9 +807,8 @@ static void flush_space(struct btrfs_fs_info *fs_info, return; } -static inline u64 -btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info) +static u64 btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, + const struct btrfs_space_info *space_info) { u64 used; u64 avail; @@ -834,7 +833,7 @@ btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, } static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info) + const struct btrfs_space_info *space_info) { const u64 global_rsv_size = btrfs_block_rsv_reserved(&fs_info->global_block_rsv); u64 ordered, delalloc; diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 08a3bd10addcf..b0187f25dbb5e 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -165,7 +165,7 @@ struct reserve_ticket { wait_queue_head_t wait; }; -static inline bool btrfs_mixed_space_info(struct btrfs_space_info *space_info) +static inline bool btrfs_mixed_space_info(const struct btrfs_space_info *space_info) { return ((space_info->flags & BTRFS_BLOCK_GROUP_METADATA) && (space_info->flags & BTRFS_BLOCK_GROUP_DATA)); @@ -206,7 +206,7 @@ void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info, u64 chunk_size); struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info, u64 flags); -u64 __pure btrfs_space_info_used(struct btrfs_space_info *s_info, +u64 __pure btrfs_space_info_used(const struct btrfs_space_info *s_info, bool may_use_included); void btrfs_clear_space_info_full(struct btrfs_fs_info *info); void btrfs_dump_space_info(struct btrfs_fs_info *fs_info, @@ -219,7 +219,7 @@ int btrfs_reserve_metadata_bytes(struct btrfs_fs_info *fs_info, void btrfs_try_granting_tickets(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info); int btrfs_can_overcommit(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, u64 bytes, + const struct btrfs_space_info *space_info, u64 bytes, enum btrfs_reserve_flush_enum flush); static inline void btrfs_space_info_free_bytes_may_use( diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index aa03db69a0164..20add63421b3d 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1796,7 +1796,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, } /* see comments in should_cow_block() */ set_bit(BTRFS_ROOT_FORCE_COW, &root->state); - smp_wmb(); + smp_mb__after_atomic(); btrfs_set_root_node(new_root_item, tmp); /* record when the snapshot was created in key.offset */ @@ -1860,7 +1860,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, btrfs_i_size_write(BTRFS_I(parent_inode), parent_inode->i_size + fname.disk_name.len * 2); parent_inode->i_mtime = inode_set_ctime_current(parent_inode); - ret = btrfs_update_inode_fallback(trans, parent_root, BTRFS_I(parent_inode)); + ret = btrfs_update_inode_fallback(trans, BTRFS_I(parent_inode)); if (ret) { btrfs_abort_transaction(trans, ret); goto fail; diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 6d16506bbdc0d..0d93368c1691a 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -614,7 +614,7 @@ static int check_dir_item(struct extent_buffer *leaf, */ if (key->type == BTRFS_DIR_ITEM_KEY || key->type == BTRFS_XATTR_ITEM_KEY) { - char namebuf[max(BTRFS_NAME_LEN, XATTR_NAME_MAX)]; + char namebuf[MAX(BTRFS_NAME_LEN, XATTR_NAME_MAX)]; read_extent_buffer(leaf, namebuf, (unsigned long)(di + 1), name_len); @@ -1717,10 +1717,10 @@ static int check_inode_ref(struct extent_buffer *leaf, while (ptr < end) { u16 namelen; - if (unlikely(ptr + sizeof(iref) > end)) { + if (unlikely(ptr + sizeof(*iref) > end)) { inode_ref_err(leaf, slot, "inode ref overflow, ptr %lu end %lu inode_ref_size %zu", - ptr, end, sizeof(iref)); + ptr, end, sizeof(*iref)); return -EUCLEAN; } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index cc9a2f8a4ae3b..256116a6e07c6 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -140,11 +140,14 @@ static void wait_log_commit(struct btrfs_root *root, int transid); * and once to do all the other items. */ -static struct inode *btrfs_iget_logging(u64 objectid, struct btrfs_root *root) +static struct btrfs_inode *btrfs_iget_logging(u64 objectid, struct btrfs_root *root) { unsigned int nofs_flag; struct inode *inode; + /* Only meant to be called for subvolume roots and not for log roots. */ + ASSERT(is_fstree(btrfs_root_id(root))); + /* * We're holding a transaction handle whether we are logging or * replaying a log tree, so we must make sure NOFS semantics apply @@ -156,7 +159,10 @@ static struct inode *btrfs_iget_logging(u64 objectid, struct btrfs_root *root) inode = btrfs_iget(root->fs_info->sb, objectid, root); memalloc_nofs_restore(nofs_flag); - return inode; + if (IS_ERR(inode)) + return ERR_CAST(inode); + + return BTRFS_I(inode); } /* @@ -320,8 +326,7 @@ struct walk_control { /* * Ignore any items from the inode currently being processed. Needs - * to be set every time we find a BTRFS_INODE_ITEM_KEY and we are in - * the LOG_WALK_REPLAY_INODES stage. + * to be set every time we find a BTRFS_INODE_ITEM_KEY. */ bool ignore_cur_inode; @@ -613,20 +618,6 @@ static int read_alloc_one_name(struct extent_buffer *eb, void *start, int len, return 0; } -/* - * simple helper to read an inode off the disk from a given root - * This can only be called for subvolume roots and not for the log - */ -static noinline struct inode *read_one_inode(struct btrfs_root *root, - u64 objectid) -{ - struct inode *inode; - - inode = btrfs_iget_logging(objectid, root); - if (IS_ERR(inode)) - inode = NULL; - return inode; -} /* replays a single extent in 'eb' at 'slot' with 'key' into the * subvolume 'root'. path is released on entry and should be released @@ -681,10 +672,15 @@ static noinline int replay_one_extent(struct btrfs_trans_handle *trans, goto out; } - inode = read_one_inode(root, key->objectid); - if (!inode) { - ret = -EIO; - goto out; + { + struct btrfs_inode *btrfs_inode; + + btrfs_inode = btrfs_iget_logging(key->objectid, root); + if (IS_ERR(btrfs_inode)) { + ret = PTR_ERR(btrfs_inode); + goto out; + } + inode = &btrfs_inode->vfs_inode; } /* @@ -963,10 +959,16 @@ static noinline int drop_one_dir_item(struct btrfs_trans_handle *trans, btrfs_release_path(path); - inode = read_one_inode(root, location.objectid); - if (!inode) { - ret = -EIO; - goto out; + { + struct btrfs_inode *btrfs_inode; + + btrfs_inode = btrfs_iget_logging(location.objectid, root); + if (IS_ERR(btrfs_inode)) { + ret = PTR_ERR(btrfs_inode); + inode = NULL; + goto out; + } + inode = &btrfs_inode->vfs_inode; } ret = link_to_fixup_dir(trans, root, path, location.objectid); @@ -1087,7 +1089,9 @@ static inline int __add_inode_ref(struct btrfs_trans_handle *trans, search_key.type = BTRFS_INODE_REF_KEY; search_key.offset = parent_objectid; ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0); - if (ret == 0) { + if (ret < 0) { + return ret; + } else if (ret == 0) { struct btrfs_inode_ref *victim_ref; unsigned long ptr; unsigned long ptr_end; @@ -1160,13 +1164,13 @@ static inline int __add_inode_ref(struct btrfs_trans_handle *trans, struct fscrypt_str victim_name; extref = (struct btrfs_inode_extref *)(base + cur_offset); + victim_name.len = btrfs_inode_extref_name_len(leaf, extref); if (btrfs_inode_extref_parent(leaf, extref) != parent_objectid) goto next; ret = read_alloc_one_name(leaf, &extref->name, - btrfs_inode_extref_name_len(leaf, extref), - &victim_name); + victim_name.len, &victim_name); if (ret) return ret; @@ -1181,18 +1185,21 @@ static inline int __add_inode_ref(struct btrfs_trans_handle *trans, kfree(victim_name.name); return ret; } else if (!ret) { - ret = -ENOENT; - victim_parent = read_one_inode(root, - parent_objectid); - if (victim_parent) { + struct btrfs_inode *btrfs_victim; + + btrfs_victim = btrfs_iget_logging(parent_objectid, root); + if (IS_ERR(btrfs_victim)) { + ret = PTR_ERR(btrfs_victim); + } else { + victim_parent = &btrfs_victim->vfs_inode; inc_nlink(&inode->vfs_inode); btrfs_release_path(path); ret = unlink_inode_for_log_replay(trans, BTRFS_I(victim_parent), inode, &victim_name); + iput(victim_parent); } - iput(victim_parent); kfree(victim_name.name); if (ret) return ret; @@ -1329,11 +1336,16 @@ static int unlink_old_inode_refs(struct btrfs_trans_handle *trans, struct inode *dir; btrfs_release_path(path); - dir = read_one_inode(root, parent_id); - if (!dir) { - ret = -ENOENT; - kfree(name.name); - goto out; + { + struct btrfs_inode *btrfs_dir; + + btrfs_dir = btrfs_iget_logging(parent_id, root); + if (IS_ERR(btrfs_dir)) { + ret = PTR_ERR(btrfs_dir); + kfree(name.name); + goto out; + } + dir = &btrfs_dir->vfs_inode; } ret = unlink_inode_for_log_replay(trans, BTRFS_I(dir), inode, &name); @@ -1404,16 +1416,30 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans, * copy the back ref in. The link count fixup code will take * care of the rest */ - dir = read_one_inode(root, parent_objectid); - if (!dir) { - ret = -ENOENT; - goto out; + { + struct btrfs_inode *btrfs_dir; + + btrfs_dir = btrfs_iget_logging(parent_objectid, root); + if (IS_ERR(btrfs_dir)) { + ret = PTR_ERR(btrfs_dir); + if (ret == -ENOENT) + ret = 0; + dir = NULL; + goto out; + } + dir = &btrfs_dir->vfs_inode; } - inode = read_one_inode(root, inode_objectid); - if (!inode) { - ret = -EIO; - goto out; + { + struct btrfs_inode *btrfs_inode; + + btrfs_inode = btrfs_iget_logging(inode_objectid, root); + if (IS_ERR(btrfs_inode)) { + ret = PTR_ERR(btrfs_inode); + inode = NULL; + goto out; + } + inode = &btrfs_inode->vfs_inode; } while (ref_ptr < ref_end) { @@ -1424,11 +1450,25 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans, * parent object can change from one array * item to another. */ - if (!dir) - dir = read_one_inode(root, parent_objectid); if (!dir) { - ret = -ENOENT; - goto out; + struct btrfs_inode *btrfs_dir; + + btrfs_dir = btrfs_iget_logging(parent_objectid, root); + if (IS_ERR(btrfs_dir)) { + ret = PTR_ERR(btrfs_dir); + dir = NULL; + /* + * A new parent dir may have not been + * logged and not exist in the subvolume + * tree, see the comment above before + * the loop when getting the first + * parent dir. + */ + if (ret == -ENOENT) + ret = 0; + goto out; + } + dir = &btrfs_dir->vfs_inode; } } else { ret = ref_get_fields(eb, ref_ptr, &name, &ref_index); @@ -1502,8 +1542,7 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans, return ret; } -static int count_inode_extrefs(struct btrfs_root *root, - struct btrfs_inode *inode, struct btrfs_path *path) +static int count_inode_extrefs(struct btrfs_inode *inode, struct btrfs_path *path) { int ret = 0; int name_len; @@ -1517,8 +1556,8 @@ static int count_inode_extrefs(struct btrfs_root *root, struct extent_buffer *leaf; while (1) { - ret = btrfs_find_one_extref(root, inode_objectid, offset, path, - &extref, &offset); + ret = btrfs_find_one_extref(inode->root, inode_objectid, offset, + path, &extref, &offset); if (ret) break; @@ -1546,8 +1585,7 @@ static int count_inode_extrefs(struct btrfs_root *root, return nlink; } -static int count_inode_refs(struct btrfs_root *root, - struct btrfs_inode *inode, struct btrfs_path *path) +static int count_inode_refs(struct btrfs_inode *inode, struct btrfs_path *path) { int ret; struct btrfs_key key; @@ -1562,7 +1600,7 @@ static int count_inode_refs(struct btrfs_root *root, key.offset = (u64)-1; while (1) { - ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + ret = btrfs_search_slot(NULL, inode->root, &key, path, 0, 0); if (ret < 0) break; if (ret > 0) { @@ -1614,9 +1652,9 @@ static int count_inode_refs(struct btrfs_root *root, * will free the inode. */ static noinline int fixup_inode_link_count(struct btrfs_trans_handle *trans, - struct btrfs_root *root, struct inode *inode) { + struct btrfs_root *root = BTRFS_I(inode)->root; struct btrfs_path *path; int ret; u64 nlink = 0; @@ -1626,13 +1664,13 @@ static noinline int fixup_inode_link_count(struct btrfs_trans_handle *trans, if (!path) return -ENOMEM; - ret = count_inode_refs(root, BTRFS_I(inode), path); + ret = count_inode_refs(BTRFS_I(inode), path); if (ret < 0) goto out; nlink = ret; - ret = count_inode_extrefs(root, BTRFS_I(inode), path); + ret = count_inode_extrefs(BTRFS_I(inode), path); if (ret < 0) goto out; @@ -1698,13 +1736,18 @@ static noinline int fixup_inode_link_counts(struct btrfs_trans_handle *trans, break; btrfs_release_path(path); - inode = read_one_inode(root, key.offset); - if (!inode) { - ret = -EIO; - break; + { + struct btrfs_inode *btrfs_inode; + + btrfs_inode = btrfs_iget_logging(key.offset, root); + if (IS_ERR(btrfs_inode)) { + ret = PTR_ERR(btrfs_inode); + break; + } + inode = &btrfs_inode->vfs_inode; } - ret = fixup_inode_link_count(trans, root, inode); + ret = fixup_inode_link_count(trans, inode); iput(inode); if (ret) break; @@ -1735,9 +1778,14 @@ static noinline int link_to_fixup_dir(struct btrfs_trans_handle *trans, int ret = 0; struct inode *inode; - inode = read_one_inode(root, objectid); - if (!inode) - return -EIO; + { + struct btrfs_inode *btrfs_inode; + + btrfs_inode = btrfs_iget_logging(objectid, root); + if (IS_ERR(btrfs_inode)) + return PTR_ERR(btrfs_inode); + inode = &btrfs_inode->vfs_inode; + } key.objectid = BTRFS_TREE_LOG_FIXUP_OBJECTID; key.type = BTRFS_ORPHAN_ITEM_KEY; @@ -1775,14 +1823,24 @@ static noinline int insert_one_name(struct btrfs_trans_handle *trans, struct inode *dir; int ret; - inode = read_one_inode(root, location->objectid); - if (!inode) - return -ENOENT; + { + struct btrfs_inode *btrfs_inode; - dir = read_one_inode(root, dirid); - if (!dir) { - iput(inode); - return -EIO; + btrfs_inode = btrfs_iget_logging(location->objectid, root); + if (IS_ERR(btrfs_inode)) + return PTR_ERR(btrfs_inode); + inode = &btrfs_inode->vfs_inode; + } + + { + struct btrfs_inode *btrfs_dir; + + btrfs_dir = btrfs_iget_logging(dirid, root); + if (IS_ERR(btrfs_dir)) { + iput(inode); + return PTR_ERR(btrfs_dir); + } + dir = &btrfs_dir->vfs_inode; } ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name, @@ -1860,9 +1918,14 @@ static noinline int replay_one_name(struct btrfs_trans_handle *trans, bool update_size = true; bool name_added = false; - dir = read_one_inode(root, key->objectid); - if (!dir) - return -EIO; + { + struct btrfs_inode *btrfs_dir; + + btrfs_dir = btrfs_iget_logging(key->objectid, root); + if (IS_ERR(btrfs_dir)) + return PTR_ERR(btrfs_dir); + dir = &btrfs_dir->vfs_inode; + } ret = read_alloc_one_name(eb, di + 1, btrfs_dir_name_len(eb, di), &name); if (ret) @@ -1935,7 +1998,7 @@ static noinline int replay_one_name(struct btrfs_trans_handle *trans, search_key.objectid = log_key.objectid; search_key.type = BTRFS_INODE_EXTREF_KEY; - search_key.offset = key->objectid; + search_key.offset = btrfs_extref_hash(key->objectid, name.name, name.len); ret = backref_in_log(root->log_root, &search_key, key->objectid, &name); if (ret < 0) { goto out; @@ -2164,10 +2227,16 @@ static noinline int check_item_in_log(struct btrfs_trans_handle *trans, btrfs_dir_item_key_to_cpu(eb, di, &location); btrfs_release_path(path); btrfs_release_path(log_path); - inode = read_one_inode(root, location.objectid); - if (!inode) { - ret = -EIO; - goto out; + { + struct btrfs_inode *btrfs_inode; + + btrfs_inode = btrfs_iget_logging(location.objectid, root); + if (IS_ERR(btrfs_inode)) { + ret = PTR_ERR(btrfs_inode); + inode = NULL; + goto out; + } + inode = &btrfs_inode->vfs_inode; } ret = link_to_fixup_dir(trans, root, path, location.objectid); @@ -2318,14 +2387,22 @@ static noinline int replay_dir_deletes(struct btrfs_trans_handle *trans, if (!log_path) return -ENOMEM; - dir = read_one_inode(root, dirid); - /* it isn't an error if the inode isn't there, that can happen - * because we replay the deletes before we copy in the inode item - * from the log - */ - if (!dir) { - btrfs_free_path(log_path); - return 0; + { + struct btrfs_inode *btrfs_dir; + + btrfs_dir = btrfs_iget_logging(dirid, root); + /* + * It isn't an error if the inode isn't there, that can happen because + * we replay the deletes before we copy in the inode item from the log. + */ + if (IS_ERR(btrfs_dir)) { + btrfs_free_path(log_path); + ret = PTR_ERR(btrfs_dir); + if (ret == -ENOENT) + ret = 0; + return ret; + } + dir = &btrfs_dir->vfs_inode; } range_start = 0; @@ -2416,38 +2493,43 @@ static int replay_one_buffer(struct btrfs_root *log, struct extent_buffer *eb, int i; int ret; + if (level != 0) + return 0; + ret = btrfs_read_extent_buffer(eb, &check); if (ret) return ret; - level = btrfs_header_level(eb); - - if (level != 0) - return 0; - path = btrfs_alloc_path(); if (!path) return -ENOMEM; nritems = btrfs_header_nritems(eb); for (i = 0; i < nritems; i++) { - btrfs_item_key_to_cpu(eb, &key, i); + struct btrfs_inode_item *inode_item; - /* inode keys are done during the first stage */ - if (key.type == BTRFS_INODE_ITEM_KEY && - wc->stage == LOG_WALK_REPLAY_INODES) { - struct btrfs_inode_item *inode_item; - u32 mode; + btrfs_item_key_to_cpu(eb, &key, i); - inode_item = btrfs_item_ptr(eb, i, - struct btrfs_inode_item); + if (key.type == BTRFS_INODE_ITEM_KEY) { + inode_item = btrfs_item_ptr(eb, i, struct btrfs_inode_item); /* - * If we have a tmpfile (O_TMPFILE) that got fsync'ed - * and never got linked before the fsync, skip it, as - * replaying it is pointless since it would be deleted - * later. We skip logging tmpfiles, but it's always - * possible we are replaying a log created with a kernel - * that used to log tmpfiles. + * An inode with no links is either: + * + * 1) A tmpfile (O_TMPFILE) that got fsync'ed and never + * got linked before the fsync, skip it, as replaying + * it is pointless since it would be deleted later. + * We skip logging tmpfiles, but it's always possible + * we are replaying a log created with a kernel that + * used to log tmpfiles; + * + * 2) A non-tmpfile which got its last link deleted + * while holding an open fd on it and later got + * fsynced through that fd. We always log the + * parent inodes when inode->last_unlink_trans is + * set to the current transaction, so ignore all the + * inode items for this inode. We will delete the + * inode when processing the parent directory with + * replay_dir_deletes(). */ if (btrfs_inode_nlink(eb, inode_item) == 0) { wc->ignore_cur_inode = true; @@ -2455,8 +2537,14 @@ static int replay_one_buffer(struct btrfs_root *log, struct extent_buffer *eb, } else { wc->ignore_cur_inode = false; } - ret = replay_xattr_deletes(wc->trans, root, log, - path, key.objectid); + } + + /* Inode keys are done during the first stage. */ + if (key.type == BTRFS_INODE_ITEM_KEY && + wc->stage == LOG_WALK_REPLAY_INODES) { + u32 mode; + + ret = replay_xattr_deletes(wc->trans, root, log, path, key.objectid); if (ret) break; mode = btrfs_inode_mode(eb, inode_item); @@ -2484,10 +2572,15 @@ static int replay_one_buffer(struct btrfs_root *log, struct extent_buffer *eb, struct inode *inode; u64 from; - inode = read_one_inode(root, key.objectid); - if (!inode) { - ret = -EIO; - break; + { + struct btrfs_inode *btrfs_inode; + + btrfs_inode = btrfs_iget_logging(key.objectid, root); + if (IS_ERR(btrfs_inode)) { + ret = PTR_ERR(btrfs_inode); + break; + } + inode = &btrfs_inode->vfs_inode; } from = ALIGN(i_size_read(inode), root->fs_info->sectorsize); @@ -2539,9 +2632,8 @@ static int replay_one_buffer(struct btrfs_root *log, struct extent_buffer *eb, key.type == BTRFS_INODE_EXTREF_KEY) { ret = add_inode_ref(wc->trans, root, log, path, eb, i, &key); - if (ret && ret != -ENOENT) + if (ret) break; - ret = 0; } else if (key.type == BTRFS_EXTENT_DATA_KEY) { ret = replay_one_extent(wc->trans, root, path, eb, i, &key); @@ -3262,6 +3354,31 @@ int btrfs_free_log_root_tree(struct btrfs_trans_handle *trans, return 0; } +static bool mark_inode_as_not_logged(const struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + bool ret = false; + + /* + * Do this only if ->logged_trans is still 0 to prevent races with + * concurrent logging as we may see the inode not logged when + * inode_logged() is called but it gets logged after inode_logged() did + * not find it in the log tree and we end up setting ->logged_trans to a + * value less than trans->transid after the concurrent logging task has + * set it to trans->transid. As a consequence, subsequent rename, unlink + * and link operations may end up not logging new names and removing old + * names from the log. + */ + spin_lock(&inode->lock); + if (inode->logged_trans == 0) + inode->logged_trans = trans->transid - 1; + else if (inode->logged_trans == trans->transid) + ret = true; + spin_unlock(&inode->lock); + + return ret; +} + /* * Check if an inode was logged in the current transaction. This correctly deals * with the case where the inode was logged but has a logged_trans of 0, which @@ -3279,15 +3396,32 @@ static int inode_logged(const struct btrfs_trans_handle *trans, struct btrfs_key key; int ret; - if (inode->logged_trans == trans->transid) + /* + * Quick lockless call, since once ->logged_trans is set to the current + * transaction, we never set it to a lower value anywhere else. + */ + if (data_race(inode->logged_trans) == trans->transid) return 1; /* - * If logged_trans is not 0, then we know the inode logged was not logged - * in this transaction, so we can return false right away. + * If logged_trans is not 0 and not trans->transid, then we know the + * inode was not logged in this transaction, so we can return false + * right away. We take the lock to avoid a race caused by load/store + * tearing with a concurrent btrfs_log_inode() call or a concurrent task + * in this function further below - an update to trans->transid can be + * teared into two 32 bits updates for example, in which case we could + * see a positive value that is not trans->transid and assume the inode + * was not logged when it was. */ - if (inode->logged_trans > 0) + spin_lock(&inode->lock); + if (inode->logged_trans == trans->transid) { + spin_unlock(&inode->lock); + return 1; + } else if (inode->logged_trans > 0) { + spin_unlock(&inode->lock); return 0; + } + spin_unlock(&inode->lock); /* * If no log tree was created for this root in this transaction, then @@ -3296,10 +3430,8 @@ static int inode_logged(const struct btrfs_trans_handle *trans, * transaction's ID, to avoid the search below in a future call in case * a log tree gets created after this. */ - if (!test_bit(BTRFS_ROOT_HAS_LOG_TREE, &inode->root->state)) { - inode->logged_trans = trans->transid - 1; - return 0; - } + if (!test_bit(BTRFS_ROOT_HAS_LOG_TREE, &inode->root->state)) + return mark_inode_as_not_logged(trans, inode); /* * We have a log tree and the inode's logged_trans is 0. We can't tell @@ -3353,8 +3485,7 @@ static int inode_logged(const struct btrfs_trans_handle *trans, * Set logged_trans to a value greater than 0 and less then the * current transaction to avoid doing the search in future calls. */ - inode->logged_trans = trans->transid - 1; - return 0; + return mark_inode_as_not_logged(trans, inode); } /* @@ -3362,20 +3493,9 @@ static int inode_logged(const struct btrfs_trans_handle *trans, * the current transacion's ID, to avoid future tree searches as long as * the inode is not evicted again. */ + spin_lock(&inode->lock); inode->logged_trans = trans->transid; - - /* - * If it's a directory, then we must set last_dir_index_offset to the - * maximum possible value, so that the next attempt to log the inode does - * not skip checking if dir index keys found in modified subvolume tree - * leaves have been logged before, otherwise it would result in attempts - * to insert duplicate dir index keys in the log tree. This must be done - * because last_dir_index_offset is an in-memory only field, not persisted - * in the inode item or any other on-disk structure, so its value is lost - * once the inode is evicted. - */ - if (S_ISDIR(inode->vfs_inode.i_mode)) - inode->last_dir_index_offset = (u64)-1; + spin_unlock(&inode->lock); return 1; } @@ -3947,7 +4067,7 @@ static noinline int log_dir_items(struct btrfs_trans_handle *trans, /* * If the inode was logged before and it was evicted, then its - * last_dir_index_offset is (u64)-1, so we don't the value of the last index + * last_dir_index_offset is 0, so we don't know the value of the last index * key offset. If that's the case, search for it and update the inode. This * is to avoid lookups in the log tree every time we try to insert a dir index * key from a leaf changed in the current transaction, and to allow us to always @@ -3963,7 +4083,7 @@ static int update_last_dir_index_offset(struct btrfs_inode *inode, lockdep_assert_held(&inode->log_mutex); - if (inode->last_dir_index_offset != (u64)-1) + if (inode->last_dir_index_offset != 0) return 0; if (!ctx->logged_before) { @@ -4171,6 +4291,9 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, btrfs_set_token_timespec_nsec(&token, &item->ctime, inode_get_ctime(inode).tv_nsec); + btrfs_set_timespec_sec(leaf, &item->otime, BTRFS_I(inode)->i_otime_sec); + btrfs_set_timespec_nsec(leaf, &item->otime, BTRFS_I(inode)->i_otime_nsec); + /* * We do not need to set the nbytes field, in fact during a fast fsync * its value may not even be correct, since a fast fsync does not wait @@ -5419,7 +5542,6 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, ihold(&curr_inode->vfs_inode); while (true) { - struct inode *vfs_inode; struct btrfs_key key; struct btrfs_key found_key; u64 next_index; @@ -5435,7 +5557,7 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, struct extent_buffer *leaf = path->nodes[0]; struct btrfs_dir_item *di; struct btrfs_key di_key; - struct inode *di_inode; + struct btrfs_inode *di_inode; int log_mode = LOG_INODE_EXISTS; int type; @@ -5462,17 +5584,16 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, goto out; } - if (!need_log_inode(trans, BTRFS_I(di_inode))) { - btrfs_add_delayed_iput(BTRFS_I(di_inode)); + if (!need_log_inode(trans, di_inode)) { + btrfs_add_delayed_iput(di_inode); break; } ctx->log_new_dentries = false; if (type == BTRFS_FT_DIR) log_mode = LOG_INODE_ALL; - ret = btrfs_log_inode(trans, BTRFS_I(di_inode), - log_mode, ctx); - btrfs_add_delayed_iput(BTRFS_I(di_inode)); + ret = btrfs_log_inode(trans, di_inode, log_mode, ctx); + btrfs_add_delayed_iput(di_inode); if (ret) goto out; if (ctx->log_new_dentries) { @@ -5514,14 +5635,13 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, kfree(dir_elem); btrfs_add_delayed_iput(curr_inode); - curr_inode = NULL; - vfs_inode = btrfs_iget_logging(ino, root); - if (IS_ERR(vfs_inode)) { - ret = PTR_ERR(vfs_inode); + curr_inode = btrfs_iget_logging(ino, root); + if (IS_ERR(curr_inode)) { + ret = PTR_ERR(curr_inode); + curr_inode = NULL; break; } - curr_inode = BTRFS_I(vfs_inode); } out: btrfs_free_path(path); @@ -5599,7 +5719,7 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, struct btrfs_log_ctx *ctx) { struct btrfs_ino_list *ino_elem; - struct inode *inode; + struct btrfs_inode *inode; /* * It's rare to have a lot of conflicting inodes, in practice it is not @@ -5690,12 +5810,12 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, * inode in LOG_INODE_EXISTS mode and rename operations update the log, * so that the log ends up with the new name and without the old name. */ - if (!need_log_inode(trans, BTRFS_I(inode))) { - btrfs_add_delayed_iput(BTRFS_I(inode)); + if (!need_log_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); return 0; } - btrfs_add_delayed_iput(BTRFS_I(inode)); + btrfs_add_delayed_iput(inode); ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); if (!ino_elem) @@ -5731,7 +5851,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, */ while (!list_empty(&ctx->conflict_inodes)) { struct btrfs_ino_list *curr; - struct inode *inode; + struct btrfs_inode *inode; u64 ino; u64 parent; @@ -5767,9 +5887,8 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, * dir index key range logged for the directory. So we * must make sure the deletion is recorded. */ - ret = btrfs_log_inode(trans, BTRFS_I(inode), - LOG_INODE_ALL, ctx); - btrfs_add_delayed_iput(BTRFS_I(inode)); + ret = btrfs_log_inode(trans, inode, LOG_INODE_ALL, ctx); + btrfs_add_delayed_iput(inode); if (ret) break; continue; @@ -5785,8 +5904,8 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, * it again because if some other task logged the inode after * that, we can avoid doing it again. */ - if (!need_log_inode(trans, BTRFS_I(inode))) { - btrfs_add_delayed_iput(BTRFS_I(inode)); + if (!need_log_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); continue; } @@ -5797,8 +5916,8 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, * well because during a rename we pin the log and update the * log with the new name before we unpin it. */ - ret = btrfs_log_inode(trans, BTRFS_I(inode), LOG_INODE_EXISTS, ctx); - btrfs_add_delayed_iput(BTRFS_I(inode)); + ret = btrfs_log_inode(trans, inode, LOG_INODE_EXISTS, ctx); + btrfs_add_delayed_iput(inode); if (ret) break; } @@ -6290,7 +6409,7 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, list_for_each_entry(item, delayed_ins_list, log_list) { struct btrfs_dir_item *dir_item; - struct inode *di_inode; + struct btrfs_inode *di_inode; struct btrfs_key key; int log_mode = LOG_INODE_EXISTS; @@ -6306,8 +6425,8 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, break; } - if (!need_log_inode(trans, BTRFS_I(di_inode))) { - btrfs_add_delayed_iput(BTRFS_I(di_inode)); + if (!need_log_inode(trans, di_inode)) { + btrfs_add_delayed_iput(di_inode); continue; } @@ -6315,12 +6434,12 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, log_mode = LOG_INODE_ALL; ctx->log_new_dentries = false; - ret = btrfs_log_inode(trans, BTRFS_I(di_inode), log_mode, ctx); + ret = btrfs_log_inode(trans, di_inode, log_mode, ctx); if (!ret && ctx->log_new_dentries) - ret = log_new_dir_dentries(trans, BTRFS_I(di_inode), ctx); + ret = log_new_dir_dentries(trans, di_inode, ctx); - btrfs_add_delayed_iput(BTRFS_I(di_inode)); + btrfs_add_delayed_iput(di_inode); if (ret) break; @@ -6646,7 +6765,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * a power failure unless the log was synced as part of an fsync * against any other unrelated inode. */ - if (inode_only != LOG_INODE_EXISTS) + if (!ctx->logging_new_name && inode_only != LOG_INODE_EXISTS) inode->last_log_commit = inode->last_sub_trans; spin_unlock(&inode->lock); @@ -6728,7 +6847,7 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, ptr = btrfs_item_ptr_offset(leaf, slot); while (cur_offset < item_size) { struct btrfs_key inode_key; - struct inode *dir_inode; + struct btrfs_inode *dir_inode; inode_key.type = BTRFS_INODE_ITEM_KEY; inode_key.offset = 0; @@ -6777,18 +6896,16 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, goto out; } - if (!need_log_inode(trans, BTRFS_I(dir_inode))) { - btrfs_add_delayed_iput(BTRFS_I(dir_inode)); + if (!need_log_inode(trans, dir_inode)) { + btrfs_add_delayed_iput(dir_inode); continue; } ctx->log_new_dentries = false; - ret = btrfs_log_inode(trans, BTRFS_I(dir_inode), - LOG_INODE_ALL, ctx); + ret = btrfs_log_inode(trans, dir_inode, LOG_INODE_ALL, ctx); if (!ret && ctx->log_new_dentries) - ret = log_new_dir_dentries(trans, - BTRFS_I(dir_inode), ctx); - btrfs_add_delayed_iput(BTRFS_I(dir_inode)); + ret = log_new_dir_dentries(trans, dir_inode, ctx); + btrfs_add_delayed_iput(dir_inode); if (ret) goto out; } @@ -6813,7 +6930,7 @@ static int log_new_ancestors(struct btrfs_trans_handle *trans, struct extent_buffer *leaf; int slot; struct btrfs_key search_key; - struct inode *inode; + struct btrfs_inode *inode; u64 ino; int ret = 0; @@ -6828,11 +6945,10 @@ static int log_new_ancestors(struct btrfs_trans_handle *trans, if (IS_ERR(inode)) return PTR_ERR(inode); - if (BTRFS_I(inode)->generation >= trans->transid && - need_log_inode(trans, BTRFS_I(inode))) - ret = btrfs_log_inode(trans, BTRFS_I(inode), - LOG_INODE_EXISTS, ctx); - btrfs_add_delayed_iput(BTRFS_I(inode)); + if (inode->generation >= trans->transid && + need_log_inode(trans, inode)) + ret = btrfs_log_inode(trans, inode, LOG_INODE_EXISTS, ctx); + btrfs_add_delayed_iput(inode); if (ret) return ret; @@ -7235,11 +7351,14 @@ int btrfs_recover_log_trees(struct btrfs_root *log_root_tree) wc.replay_dest->log_root = log; ret = btrfs_record_root_in_trans(trans, wc.replay_dest); - if (ret) + if (ret) { /* The loop needs to continue due to the root refs */ btrfs_abort_transaction(trans, ret); - else + } else { ret = walk_log_tree(trans, log, &wc); + if (ret) + btrfs_abort_transaction(trans, ret); + } if (!ret && wc.stage == LOG_WALK_REPLAY_ALL) { ret = fixup_inode_link_counts(trans, wc.replay_dest, @@ -7301,7 +7420,6 @@ int btrfs_recover_log_trees(struct btrfs_root *log_root_tree) log_root_tree->log_root = NULL; clear_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags); - btrfs_put_root(log_root_tree); return 0; error: diff --git a/fs/btrfs/tree-mod-log.c b/fs/btrfs/tree-mod-log.c index 3df6153d5d5a8..febc014a510df 100644 --- a/fs/btrfs/tree-mod-log.c +++ b/fs/btrfs/tree-mod-log.c @@ -171,7 +171,7 @@ static noinline int tree_mod_log_insert(struct btrfs_fs_info *fs_info, * write unlock fs_info::tree_mod_log_lock. */ static inline bool tree_mod_dont_log(struct btrfs_fs_info *fs_info, - struct extent_buffer *eb) + const struct extent_buffer *eb) { if (!test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags)) return true; @@ -189,7 +189,7 @@ static inline bool tree_mod_dont_log(struct btrfs_fs_info *fs_info, /* Similar to tree_mod_dont_log, but doesn't acquire any locks. */ static inline bool tree_mod_need_log(const struct btrfs_fs_info *fs_info, - struct extent_buffer *eb) + const struct extent_buffer *eb) { if (!test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags)) return false; @@ -199,7 +199,7 @@ static inline bool tree_mod_need_log(const struct btrfs_fs_info *fs_info, return true; } -static struct tree_mod_elem *alloc_tree_mod_elem(struct extent_buffer *eb, +static struct tree_mod_elem *alloc_tree_mod_elem(const struct extent_buffer *eb, int slot, enum btrfs_mod_log_op op) { @@ -222,7 +222,7 @@ static struct tree_mod_elem *alloc_tree_mod_elem(struct extent_buffer *eb, return tm; } -int btrfs_tree_mod_log_insert_key(struct extent_buffer *eb, int slot, +int btrfs_tree_mod_log_insert_key(const struct extent_buffer *eb, int slot, enum btrfs_mod_log_op op) { struct tree_mod_elem *tm; @@ -259,7 +259,7 @@ int btrfs_tree_mod_log_insert_key(struct extent_buffer *eb, int slot, return ret; } -static struct tree_mod_elem *tree_mod_log_alloc_move(struct extent_buffer *eb, +static struct tree_mod_elem *tree_mod_log_alloc_move(const struct extent_buffer *eb, int dst_slot, int src_slot, int nr_items) { @@ -279,7 +279,7 @@ static struct tree_mod_elem *tree_mod_log_alloc_move(struct extent_buffer *eb, return tm; } -int btrfs_tree_mod_log_insert_move(struct extent_buffer *eb, +int btrfs_tree_mod_log_insert_move(const struct extent_buffer *eb, int dst_slot, int src_slot, int nr_items) { @@ -536,7 +536,7 @@ static struct tree_mod_elem *tree_mod_log_search(struct btrfs_fs_info *fs_info, } int btrfs_tree_mod_log_eb_copy(struct extent_buffer *dst, - struct extent_buffer *src, + const struct extent_buffer *src, unsigned long dst_offset, unsigned long src_offset, int nr_items) diff --git a/fs/btrfs/tree-mod-log.h b/fs/btrfs/tree-mod-log.h index 94f10afeee972..5f94ab681fa43 100644 --- a/fs/btrfs/tree-mod-log.h +++ b/fs/btrfs/tree-mod-log.h @@ -31,7 +31,7 @@ void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, int btrfs_tree_mod_log_insert_root(struct extent_buffer *old_root, struct extent_buffer *new_root, bool log_removal); -int btrfs_tree_mod_log_insert_key(struct extent_buffer *eb, int slot, +int btrfs_tree_mod_log_insert_key(const struct extent_buffer *eb, int slot, enum btrfs_mod_log_op op); int btrfs_tree_mod_log_free_eb(struct extent_buffer *eb); struct extent_buffer *btrfs_tree_mod_log_rewind(struct btrfs_fs_info *fs_info, @@ -41,11 +41,11 @@ struct extent_buffer *btrfs_tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *btrfs_get_old_root(struct btrfs_root *root, u64 time_seq); int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq); int btrfs_tree_mod_log_eb_copy(struct extent_buffer *dst, - struct extent_buffer *src, + const struct extent_buffer *src, unsigned long dst_offset, unsigned long src_offset, int nr_items); -int btrfs_tree_mod_log_insert_move(struct extent_buffer *eb, +int btrfs_tree_mod_log_insert_move(const struct extent_buffer *eb, int dst_slot, int src_slot, int nr_items); u64 btrfs_tree_mod_log_lowest_seq(struct btrfs_fs_info *fs_info); diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index b9a0b26d08e1c..1eb543602ff12 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -3174,6 +3174,12 @@ int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset) device->bytes_used - dev_extent_len); atomic64_add(dev_extent_len, &fs_info->free_chunk_space); btrfs_clear_space_info_full(fs_info); + + if (list_empty(&device->post_commit_list)) { + list_add_tail(&device->post_commit_list, + &trans->transaction->dev_update_list); + } + mutex_unlock(&fs_info->chunk_mutex); } } diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 197dfafbf4013..6e8b8c46ba18f 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1992,10 +1992,15 @@ bool btrfs_zone_activate(struct btrfs_block_group *block_group) goto out_unlock; } - /* No space left */ - if (btrfs_zoned_bg_is_full(block_group)) { - ret = false; - goto out_unlock; + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) { + /* The caller should check if the block group is full. */ + if (WARN_ON_ONCE(btrfs_zoned_bg_is_full(block_group))) { + ret = false; + goto out_unlock; + } + } else { + /* Since it is already written, it should have been active. */ + WARN_ON_ONCE(block_group->meta_write_pointer != block_group->start); } for (i = 0; i < map->num_stripes; i++) { @@ -2258,16 +2263,17 @@ bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags) return ret; } -void btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 length) +int btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 length) { struct btrfs_block_group *block_group; u64 min_alloc_bytes; if (!btrfs_is_zoned(fs_info)) - return; + return 0; block_group = btrfs_lookup_block_group(fs_info, logical); - ASSERT(block_group); + if (WARN_ON_ONCE(!block_group)) + return -ENOENT; /* No MIXED_BG on zoned btrfs. */ if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) @@ -2284,6 +2290,7 @@ void btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 len out: btrfs_put_block_group(block_group); + return 0; } static void btrfs_zone_finish_endio_workfn(struct work_struct *work) @@ -2346,12 +2353,12 @@ void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info) mutex_unlock(&fs_devices->device_list_mutex); } -bool btrfs_zoned_should_reclaim(struct btrfs_fs_info *fs_info) +bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info) { struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; struct btrfs_device *device; + u64 total = btrfs_super_total_bytes(fs_info->super_copy); u64 used = 0; - u64 total = 0; u64 factor; ASSERT(btrfs_is_zoned(fs_info)); @@ -2364,7 +2371,6 @@ bool btrfs_zoned_should_reclaim(struct btrfs_fs_info *fs_info) if (!device->bdev) continue; - total += device->disk_total_bytes; used += device->bytes_used; } mutex_unlock(&fs_devices->device_list_mutex); @@ -2418,7 +2424,7 @@ int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info) spin_lock(&block_group->lock); if (block_group->reserved || block_group->alloc_offset == 0 || - (block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM) || + !(block_group->flags & BTRFS_BLOCK_GROUP_DATA) || test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { spin_unlock(&block_group->lock); continue; diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index b9cec523b7784..c18f31d3dc25f 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -71,13 +71,13 @@ int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, bool btrfs_zone_activate(struct btrfs_block_group *block_group); int btrfs_zone_finish(struct btrfs_block_group *block_group); bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags); -void btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, +int btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 length); void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg, struct extent_buffer *eb); void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg); void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info); -bool btrfs_zoned_should_reclaim(struct btrfs_fs_info *fs_info); +bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info); void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical, u64 length); int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info); @@ -227,8 +227,11 @@ static inline bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, return true; } -static inline void btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, - u64 logical, u64 length) { } +static inline int btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, + u64 logical, u64 length) +{ + return 0; +} static inline void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg, struct extent_buffer *eb) { } @@ -237,7 +240,7 @@ static inline void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg) { } static inline void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info) { } -static inline bool btrfs_zoned_should_reclaim(struct btrfs_fs_info *fs_info) +static inline bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info) { return false; } diff --git a/fs/buffer.c b/fs/buffer.c index 21a90dd865466..dff7ca9b508c8 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -157,8 +157,8 @@ static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) */ void end_buffer_read_sync(struct buffer_head *bh, int uptodate) { - __end_buffer_read_notouch(bh, uptodate); put_bh(bh); + __end_buffer_read_notouch(bh, uptodate); } EXPORT_SYMBOL(end_buffer_read_sync); diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c index 009d23cd435b5..239a6002083d7 100644 --- a/fs/cachefiles/io.c +++ b/fs/cachefiles/io.c @@ -346,8 +346,6 @@ int __cachefiles_write(struct cachefiles_object *object, default: ki->was_async = false; cachefiles_write_complete(&ki->iocb, ret); - if (ret > 0) - ret = 0; break; } diff --git a/fs/cachefiles/ondemand.c b/fs/cachefiles/ondemand.c index 3389a373faf68..cfa8f23fdfb65 100644 --- a/fs/cachefiles/ondemand.c +++ b/fs/cachefiles/ondemand.c @@ -84,10 +84,8 @@ static ssize_t cachefiles_ondemand_fd_write_iter(struct kiocb *kiocb, trace_cachefiles_ondemand_fd_write(object, file_inode(file), pos, len); ret = __cachefiles_write(object, file, pos, iter, NULL, NULL); - if (!ret) { - ret = len; + if (ret > 0) kiocb->ki_pos += ret; - } out: fput(file); diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 1395b71df5ccc..529dd07fa459f 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1213,8 +1213,7 @@ static void ceph_async_unlink_cb(struct ceph_mds_client *mdsc, spin_unlock(&fsc->async_unlink_conflict_lock); spin_lock(&dentry->d_lock); - di->flags &= ~CEPH_DENTRY_ASYNC_UNLINK; - wake_up_bit(&di->flags, CEPH_DENTRY_ASYNC_UNLINK_BIT); + clear_and_wake_up_bit(CEPH_DENTRY_ASYNC_UNLINK_BIT, &di->flags); spin_unlock(&dentry->d_lock); synchronize_rcu(); diff --git a/fs/ceph/file.c b/fs/ceph/file.c index a03b11cf78872..0ec78d87519ba 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -539,8 +539,7 @@ static void wake_async_create_waiters(struct inode *inode, spin_lock(&ci->i_ceph_lock); if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) { - ci->i_ceph_flags &= ~CEPH_I_ASYNC_CREATE; - wake_up_bit(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT); + clear_and_wake_up_bit(CEPH_ASYNC_CREATE_BIT, &ci->i_ceph_flags); if (ci->i_ceph_flags & CEPH_I_ASYNC_CHECK_CAPS) { ci->i_ceph_flags &= ~CEPH_I_ASYNC_CHECK_CAPS; @@ -716,8 +715,7 @@ static int ceph_finish_async_create(struct inode *dir, struct inode *inode, } spin_lock(&dentry->d_lock); - di->flags &= ~CEPH_DENTRY_ASYNC_CREATE; - wake_up_bit(&di->flags, CEPH_DENTRY_ASYNC_CREATE_BIT); + clear_and_wake_up_bit(CEPH_DENTRY_ASYNC_CREATE_BIT, &di->flags); spin_unlock(&dentry->d_lock); return ret; @@ -2513,7 +2511,7 @@ static int ceph_zero_objects(struct inode *inode, loff_t offset, loff_t length) s32 stripe_unit = ci->i_layout.stripe_unit; s32 stripe_count = ci->i_layout.stripe_count; s32 object_size = ci->i_layout.object_size; - u64 object_set_size = object_size * stripe_count; + u64 object_set_size = (u64) object_size * stripe_count; u64 nearly, t; /* round offset up to next period boundary */ diff --git a/fs/ceph/locks.c b/fs/ceph/locks.c index cb51c7e9c8e22..02f5fbe83aa46 100644 --- a/fs/ceph/locks.c +++ b/fs/ceph/locks.c @@ -219,7 +219,10 @@ static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc, if (err && err != -ERESTARTSYS) return err; - wait_for_completion_killable(&req->r_safe_completion); + err = wait_for_completion_killable(&req->r_safe_completion); + if (err) + return err; + return 0; } diff --git a/fs/ceph/super.c b/fs/ceph/super.c index af154600dc797..301a5b3fa8750 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -1220,6 +1220,7 @@ static int ceph_set_super(struct super_block *s, struct fs_context *fc) s->s_time_min = 0; s->s_time_max = U32_MAX; s->s_flags |= SB_NODIRATIME | SB_NOATIME; + s->s_magic = CEPH_SUPER_MAGIC; ceph_fscrypt_set_ops(s); diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c index 18677cd4e62f5..2df99cd1034d4 100644 --- a/fs/configfs/dir.c +++ b/fs/configfs/dir.c @@ -593,7 +593,7 @@ static int populate_attrs(struct config_item *item) break; } } - if (t->ct_bin_attrs) { + if (!error && t->ct_bin_attrs) { for (i = 0; (bin_attr = t->ct_bin_attrs[i]) != NULL; i++) { error = configfs_create_bin_file(item, bin_attr); if (error) diff --git a/fs/coredump.c b/fs/coredump.c index 9d235fa14ab98..f342f352b4784 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -502,7 +502,9 @@ static int umh_pipe_setup(struct subprocess_info *info, struct cred *new) { struct file *files[2]; struct coredump_params *cp = (struct coredump_params *)info->data; - int err = create_pipe_files(files, 0); + int err; + + err = create_pipe_files(files, 0); if (err) return err; @@ -510,10 +512,13 @@ static int umh_pipe_setup(struct subprocess_info *info, struct cred *new) err = replace_fd(0, files[0], 0); fput(files[0]); + if (err < 0) + return err; + /* and disallow core files too */ current->signal->rlim[RLIMIT_CORE] = (struct rlimit){1, 1}; - return err; + return 0; } void do_coredump(const kernel_siginfo_t *siginfo) diff --git a/fs/cramfs/inode.c b/fs/cramfs/inode.c index 167effab8d259..ec6b1b4f3a592 100644 --- a/fs/cramfs/inode.c +++ b/fs/cramfs/inode.c @@ -117,9 +117,18 @@ static struct inode *get_cramfs_inode(struct super_block *sb, inode_nohighmem(inode); inode->i_data.a_ops = &cramfs_aops; break; - default: + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: init_special_inode(inode, cramfs_inode->mode, old_decode_dev(cramfs_inode->size)); + break; + default: + printk(KERN_DEBUG "CRAMFS: Invalid file type 0%04o for inode %lu.\n", + inode->i_mode, inode->i_ino); + iget_failed(inode); + return ERR_PTR(-EIO); } inode->i_mode = cramfs_inode->mode; diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index a9fa6944878fc..32e8d9f5cbac7 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -48,6 +48,23 @@ */ #undef FSCRYPT_MAX_KEY_SIZE +/* + * This mask is passed as the third argument to the crypto_alloc_*() functions + * to prevent fscrypt from using the Crypto API drivers for non-inline crypto + * engines. Those drivers have been problematic for fscrypt. fscrypt users + * have reported hangs and even incorrect en/decryption with these drivers. + * Since going to the driver, off CPU, and back again is really slow, such + * drivers can be over 50 times slower than the CPU-based code for fscrypt's + * workload. Even on platforms that lack AES instructions on the CPU, using the + * offloads has been shown to be slower, even staying with AES. (Of course, + * Adiantum is faster still, and is the recommended option on such platforms...) + * + * Note that fscrypt also supports inline crypto engines. Those don't use the + * Crypto API and work much better than the old-style (non-inline) engines. + */ +#define FSCRYPT_CRYPTOAPI_MASK \ + (CRYPTO_ALG_ALLOCATES_MEMORY | CRYPTO_ALG_KERN_DRIVER_ONLY) + #define FSCRYPT_CONTEXT_V1 1 #define FSCRYPT_CONTEXT_V2 2 diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c index 7e007810e4346..06b18797d0b13 100644 --- a/fs/crypto/hkdf.c +++ b/fs/crypto/hkdf.c @@ -74,7 +74,7 @@ int fscrypt_init_hkdf(struct fscrypt_hkdf *hkdf, const u8 *master_key, u8 prk[HKDF_HASHLEN]; int err; - hmac_tfm = crypto_alloc_shash(HKDF_HMAC_ALG, 0, 0); + hmac_tfm = crypto_alloc_shash(HKDF_HMAC_ALG, 0, FSCRYPT_CRYPTOAPI_MASK); if (IS_ERR(hmac_tfm)) { fscrypt_err(NULL, "Error allocating " HKDF_HMAC_ALG ": %ld", PTR_ERR(hmac_tfm)); diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index 12b31b8eee51a..d55799dd7ee25 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -103,7 +103,8 @@ fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, struct crypto_skcipher *tfm; int err; - tfm = crypto_alloc_skcipher(mode->cipher_str, 0, 0); + tfm = crypto_alloc_skcipher(mode->cipher_str, 0, + FSCRYPT_CRYPTOAPI_MASK); if (IS_ERR(tfm)) { if (PTR_ERR(tfm) == -ENOENT) { fscrypt_warn(inode, diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index 452b545e6846f..a4122bff75432 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -52,7 +52,8 @@ static int derive_key_aes(const u8 *master_key, struct skcipher_request *req = NULL; DECLARE_CRYPTO_WAIT(wait); struct scatterlist src_sg, dst_sg; - struct crypto_skcipher *tfm = crypto_alloc_skcipher("ecb(aes)", 0, 0); + struct crypto_skcipher *tfm = + crypto_alloc_skcipher("ecb(aes)", 0, FSCRYPT_CRYPTOAPI_MASK); if (IS_ERR(tfm)) { res = PTR_ERR(tfm); diff --git a/fs/dax.c b/fs/dax.c index 8c09578fa0357..e1451efaab140 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1578,7 +1578,7 @@ dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, if (iov_iter_rw(iter) == WRITE) { lockdep_assert_held_write(&iomi.inode->i_rwsem); iomi.flags |= IOMAP_WRITE; - } else { + } else if (!sb_rdonly(iomi.inode->i_sb)) { lockdep_assert_held(&iomi.inode->i_rwsem); } diff --git a/fs/dcache.c b/fs/dcache.c index 1f4d21b43e591..7d3c283154872 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1861,6 +1861,8 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) __dget_dlock(parent); dentry->d_parent = parent; list_add(&dentry->d_child, &parent->d_subdirs); + if (parent->d_flags & DCACHE_DISCONNECTED) + dentry->d_flags |= DCACHE_DISCONNECTED; spin_unlock(&parent->d_lock); return dentry; diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index 0455dddb0797c..0b17657690d4d 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -802,7 +802,7 @@ static int release_lockspace(struct dlm_ls *ls, int force) dlm_device_deregister(ls); - if (force < 3 && dlm_user_daemon_available()) + if (force != 3 && dlm_user_daemon_available()) do_uevent(ls, 0); dlm_recoverd_stop(ls); diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index 0618af36f5506..3c9ab6461579c 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -1826,8 +1826,8 @@ static int dlm_tcp_listen_validate(void) { /* We don't support multi-homed hosts */ if (dlm_local_count > 1) { - log_print("TCP protocol can't handle multi-homed hosts, try SCTP"); - return -EINVAL; + log_print("Detect multi-homed hosts but use only the first IP address."); + log_print("Try SCTP, if you want to enable multi-link."); } return 0; diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c index 730746d1627ce..b877fad790186 100644 --- a/fs/efivarfs/super.c +++ b/fs/efivarfs/super.c @@ -90,6 +90,10 @@ static int efivarfs_d_compare(const struct dentry *dentry, { int guid = len - EFI_VARIABLE_GUID_LEN; + /* Parallel lookups may produce a temporary invalid filename */ + if (guid <= 0) + return 1; + if (name->len != len) return 1; diff --git a/fs/erofs/data.c b/fs/erofs/data.c index 19ab9bb3a9a0e..51250ac184a52 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -358,6 +358,8 @@ int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, */ static int erofs_read_folio(struct file *file, struct folio *folio) { + trace_erofs_read_folio(folio, true); + return iomap_read_folio(folio, &erofs_iomap_ops); } diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 4324dc776016f..c6fa905494c86 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -342,13 +342,11 @@ static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq, cur = min(cur, rq->outputsize); if (cur && rq->out[0]) { kin = kmap_local_page(rq->in[nrpages_in - 1]); - if (rq->out[0] == rq->in[nrpages_in - 1]) { + if (rq->out[0] == rq->in[nrpages_in - 1]) memmove(kin + rq->pageofs_out, kin + pi, cur); - flush_dcache_page(rq->out[0]); - } else { + else memcpy_to_page(rq->out[0], rq->pageofs_out, kin + pi, cur); - } kunmap_local(kin); } rq->outputsize -= cur; diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 1d9b2a61ca631..053d89fff8d4b 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -118,9 +118,11 @@ static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl) /* * bit 30: I/O error occurred on this page + * bit 29: CPU has dirty data in D-cache (needs aliasing handling); * bit 0 - 29: remaining parts to complete this page */ -#define Z_EROFS_PAGE_EIO (1 << 30) +#define Z_EROFS_ONLINEPAGE_EIO 30 +#define Z_EROFS_ONLINEPAGE_DIRTY 29 static inline void z_erofs_onlinepage_init(struct page *page) { @@ -139,7 +141,7 @@ static inline void z_erofs_onlinepage_split(struct page *page) atomic_inc((atomic_t *)&page->private); } -static void z_erofs_onlinepage_endio(struct page *page, int err) +static void z_erofs_onlinepage_end(struct page *page, int err, bool dirty) { int orig, v; @@ -147,16 +149,20 @@ static void z_erofs_onlinepage_endio(struct page *page, int err) do { orig = atomic_read((atomic_t *)&page->private); - v = (orig - 1) | (err ? Z_EROFS_PAGE_EIO : 0); + DBG_BUGON(orig <= 0); + v = dirty << Z_EROFS_ONLINEPAGE_DIRTY; + v |= (orig - 1) | (!!err << Z_EROFS_ONLINEPAGE_EIO); } while (atomic_cmpxchg((atomic_t *)&page->private, orig, v) != orig); - if (!(v & ~Z_EROFS_PAGE_EIO)) { - set_page_private(page, 0); - ClearPagePrivate(page); - if (!(v & Z_EROFS_PAGE_EIO)) - SetPageUptodate(page); - unlock_page(page); - } + if (v & (BIT(Z_EROFS_ONLINEPAGE_DIRTY) - 1)) + return; + set_page_private(page, 0); + ClearPagePrivate(page); + if (v & BIT(Z_EROFS_ONLINEPAGE_DIRTY)) + flush_dcache_page(page); + if (!(v & BIT(Z_EROFS_ONLINEPAGE_EIO))) + SetPageUptodate(page); + unlock_page(page); } #define Z_EROFS_ONSTACK_PAGES 32 @@ -336,6 +342,7 @@ static struct workqueue_struct *z_erofs_workqueue __read_mostly; #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD static struct kthread_worker __rcu **z_erofs_pcpu_workers; +static atomic_t erofs_percpu_workers_initialized = ATOMIC_INIT(0); static void erofs_destroy_percpu_workers(void) { @@ -381,12 +388,8 @@ static int erofs_init_percpu_workers(void) } return 0; } -#else -static inline void erofs_destroy_percpu_workers(void) {} -static inline int erofs_init_percpu_workers(void) { return 0; } -#endif -#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_EROFS_FS_PCPU_KTHREAD) +#ifdef CONFIG_HOTPLUG_CPU static DEFINE_SPINLOCK(z_erofs_pcpu_worker_lock); static enum cpuhp_state erofs_cpuhp_state; @@ -443,15 +446,53 @@ static void erofs_cpu_hotplug_destroy(void) if (erofs_cpuhp_state) cpuhp_remove_state_nocalls(erofs_cpuhp_state); } -#else /* !CONFIG_HOTPLUG_CPU || !CONFIG_EROFS_FS_PCPU_KTHREAD */ +#else /* !CONFIG_HOTPLUG_CPU */ static inline int erofs_cpu_hotplug_init(void) { return 0; } static inline void erofs_cpu_hotplug_destroy(void) {} -#endif +#endif/* CONFIG_HOTPLUG_CPU */ +static int z_erofs_init_pcpu_workers(struct super_block *sb) +{ + int err; -void z_erofs_exit_zip_subsystem(void) + if (atomic_xchg(&erofs_percpu_workers_initialized, 1)) + return 0; + + err = erofs_init_percpu_workers(); + if (err) { + erofs_err(sb, "per-cpu workers: failed to allocate."); + goto err_init_percpu_workers; + } + + err = erofs_cpu_hotplug_init(); + if (err < 0) { + erofs_err(sb, "per-cpu workers: failed CPU hotplug init."); + goto err_cpuhp_init; + } + erofs_info(sb, "initialized per-cpu workers successfully."); + return err; + +err_cpuhp_init: + erofs_destroy_percpu_workers(); +err_init_percpu_workers: + atomic_set(&erofs_percpu_workers_initialized, 0); + return err; +} + +static void z_erofs_destroy_pcpu_workers(void) { + if (!atomic_xchg(&erofs_percpu_workers_initialized, 0)) + return; erofs_cpu_hotplug_destroy(); erofs_destroy_percpu_workers(); +} +#else /* !CONFIG_EROFS_FS_PCPU_KTHREAD */ +static inline int z_erofs_init_pcpu_workers(struct super_block *sb) { return 0; } +static inline void z_erofs_destroy_pcpu_workers(void) {} +#endif/* CONFIG_EROFS_FS_PCPU_KTHREAD */ + +void z_erofs_exit_zip_subsystem(void) +{ + z_erofs_destroy_pcpu_workers(); destroy_workqueue(z_erofs_workqueue); z_erofs_destroy_pcluster_pool(); } @@ -467,23 +508,12 @@ int __init z_erofs_init_zip_subsystem(void) WQ_UNBOUND | WQ_HIGHPRI, num_possible_cpus()); if (!z_erofs_workqueue) { err = -ENOMEM; - goto out_error_workqueue_init; + goto out_err_workqueue_init; } - err = erofs_init_percpu_workers(); - if (err) - goto out_error_pcpu_worker; - - err = erofs_cpu_hotplug_init(); - if (err < 0) - goto out_error_cpuhp_init; return err; -out_error_cpuhp_init: - erofs_destroy_percpu_workers(); -out_error_pcpu_worker: - destroy_workqueue(z_erofs_workqueue); -out_error_workqueue_init: +out_err_workqueue_init: z_erofs_destroy_pcluster_pool(); out_error_pcluster_pool: return err; @@ -711,8 +741,14 @@ static const struct address_space_operations z_erofs_cache_aops = { int erofs_init_managed_cache(struct super_block *sb) { - struct inode *const inode = new_inode(sb); + struct inode *inode; + int err; + + err = z_erofs_init_pcpu_workers(sb); + if (err) + return err; + inode = new_inode(sb); if (!inode) return -ENOMEM; @@ -1057,7 +1093,7 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, goto repeat; out: - z_erofs_onlinepage_endio(page, err); + z_erofs_onlinepage_end(page, err, false); return err; } @@ -1160,7 +1196,7 @@ static void z_erofs_fill_other_copies(struct z_erofs_decompress_backend *be, cur += len; } kunmap_local(dst); - z_erofs_onlinepage_endio(bvi->bvec.page, err); + z_erofs_onlinepage_end(bvi->bvec.page, err, true); list_del(p); kfree(bvi); } @@ -1327,7 +1363,7 @@ static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be, /* recycle all individual short-lived pages */ if (z_erofs_put_shortlivedpage(be->pagepool, page)) continue; - z_erofs_onlinepage_endio(page, err); + z_erofs_onlinepage_end(page, err, true); } if (be->decompressed_pages != be->onstack_pages) @@ -1390,6 +1426,16 @@ static void z_erofs_decompressqueue_kthread_work(struct kthread_work *work) } #endif +/* Use (kthread_)work in atomic contexts to minimize scheduling overhead */ +static inline bool z_erofs_in_atomic(void) +{ + if (IS_ENABLED(CONFIG_PREEMPTION) && rcu_preempt_depth()) + return true; + if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) + return true; + return !preemptible(); +} + static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, int bios) { @@ -1404,8 +1450,7 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, if (atomic_add_return(bios, &io->pending_bios)) return; - /* Use (kthread_)work and sync decompression for atomic contexts only */ - if (!in_task() || irqs_disabled() || rcu_read_lock_any_held()) { + if (z_erofs_in_atomic()) { #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD struct kthread_worker *worker; @@ -1676,6 +1721,11 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, if (bio && (cur != last_pa || last_bdev != mdev.m_bdev)) { submit_bio_retry: + //#ifdef OPLUS_STORAGE_FS debug for bugid 7760993 + if (!bio->bi_iter.bi_size) + WARN(1, "erofs submit empty io, bio=0x%p, bi_opf=0x%x, bi_sector=%llu\n", + bio, bio->bi_opf, bio->bi_iter.bi_sector); + //#endif submit_bio(bio); if (memstall) { psi_memstall_leave(&pflags); @@ -1704,7 +1754,11 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, if (cur + bvec.bv_len > end) bvec.bv_len = end - cur; - DBG_BUGON(bvec.bv_len < sb->s_blocksize); + //#ifdef OPLUS_STORAGE_FS debug for bugid 7760993.origin code is DBG_BUGON + BUG_ON(bvec.bv_len < sb->s_blocksize); + //#else + //DBG_BUGON(bvec.bv_len < sb->s_blocksize); + //#endif if (!bio_add_page(bio, bvec.bv_page, bvec.bv_len, bvec.bv_offset)) goto submit_bio_retry; diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 76566c2cbf63e..e169ca5c7a902 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -393,6 +393,9 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, } out: map->m_plen = erofs_pos(sb, m->compressedblks); + //#ifdef OPLUS_STORAGE_FS debug for bugid 7760993 + WARN(!map->m_plen, "m_plen is 0 in out compressedblks:%u\n", m->compressedblks); + //#endif return 0; err_bonus_cblkcnt: erofs_err(sb, "bogus CBLKCNT @ lcn %lu of nid %llu", lcn, vi->nid); @@ -724,8 +727,14 @@ int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, err = z_erofs_do_map_blocks(inode, map, flags); out: - if (err) + if (err){ + //#ifdef OPLUS_STORAGE_FS debug for bugid 7760993 + erofs_err(inode->i_sb,"m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o err %d", + map->m_la, map->m_pa, + map->m_llen, map->m_plen, map->m_flags,err); + //#endif map->m_llen = 0; + } trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); return err; } diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 9ae458635eb74..6b531caf7f73e 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -47,10 +47,10 @@ * * 1) epnested_mutex (mutex) * 2) ep->mtx (mutex) - * 3) ep->lock (rwlock) + * 3) ep->lock (spinlock) * * The acquire order is the one listed above, from 1 to 3. - * We need a rwlock (ep->lock) because we manipulate objects + * We need a spinlock (ep->lock) because we manipulate objects * from inside the poll callback, that might be triggered from * a wake_up() that in turn might be called from IRQ context. * So we can't sleep inside the poll callback and hence we need @@ -196,7 +196,7 @@ struct eventpoll { struct list_head rdllist; /* Lock which protects rdllist and ovflist */ - rwlock_t lock; + spinlock_t lock; /* RB tree root used to store monitored fd structs */ struct rb_root_cached rbr; @@ -208,7 +208,7 @@ struct eventpoll { */ struct epitem *ovflist; - /* wakeup_source used when ep_scan_ready_list is running */ + /* wakeup_source used when ep_send_events or __ep_eventpoll_poll is running */ struct wakeup_source *ws; /* The user that created the eventpoll descriptor */ @@ -219,6 +219,7 @@ struct eventpoll { /* used to optimize loop detection check */ u64 gen; struct hlist_head refs; + u8 loop_check_depth; /* * usage count, used together with epitem->dying to @@ -626,10 +627,10 @@ static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist) * in a lockless way. */ lockdep_assert_irqs_enabled(); - write_lock_irq(&ep->lock); + spin_lock_irq(&ep->lock); list_splice_init(&ep->rdllist, txlist); WRITE_ONCE(ep->ovflist, NULL); - write_unlock_irq(&ep->lock); + spin_unlock_irq(&ep->lock); } static void ep_done_scan(struct eventpoll *ep, @@ -637,7 +638,7 @@ static void ep_done_scan(struct eventpoll *ep, { struct epitem *epi, *nepi; - write_lock_irq(&ep->lock); + spin_lock_irq(&ep->lock); /* * During the time we spent inside the "sproc" callback, some * other events might have been queued by the poll callback. @@ -678,7 +679,7 @@ static void ep_done_scan(struct eventpoll *ep, wake_up(&ep->wq); } - write_unlock_irq(&ep->lock); + spin_unlock_irq(&ep->lock); } static void epi_rcu_free(struct rcu_head *head) @@ -758,10 +759,10 @@ static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force) rb_erase_cached(&epi->rbn, &ep->rbr); - write_lock_irq(&ep->lock); + spin_lock_irq(&ep->lock); if (ep_is_linked(epi)) list_del_init(&epi->rdllink); - write_unlock_irq(&ep->lock); + spin_unlock_irq(&ep->lock); wakeup_source_unregister(ep_wakeup_source(epi)); /* @@ -774,7 +775,7 @@ static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force) call_rcu(&epi->rcu, epi_rcu_free); percpu_counter_dec(&ep->user->epoll_watches); - return ep_refcount_dec_and_test(ep); + return true; } /* @@ -782,14 +783,14 @@ static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force) */ static void ep_remove_safe(struct eventpoll *ep, struct epitem *epi) { - WARN_ON_ONCE(__ep_remove(ep, epi, false)); + if (__ep_remove(ep, epi, false)) + WARN_ON_ONCE(ep_refcount_dec_and_test(ep)); } static void ep_clear_and_put(struct eventpoll *ep) { struct rb_node *rbp, *next; struct epitem *epi; - bool dispose; /* We need to release all tasks waiting for these file */ if (waitqueue_active(&ep->poll_wait)) @@ -822,10 +823,8 @@ static void ep_clear_and_put(struct eventpoll *ep) cond_resched(); } - dispose = ep_refcount_dec_and_test(ep); mutex_unlock(&ep->mtx); - - if (dispose) + if (ep_refcount_dec_and_test(ep)) ep_free(ep); } @@ -1005,7 +1004,7 @@ void eventpoll_release_file(struct file *file) dispose = __ep_remove(ep, epi, true); mutex_unlock(&ep->mtx); - if (dispose) + if (dispose && ep_refcount_dec_and_test(ep)) ep_free(ep); goto again; } @@ -1021,7 +1020,7 @@ static int ep_alloc(struct eventpoll **pep) return -ENOMEM; mutex_init(&ep->mtx); - rwlock_init(&ep->lock); + spin_lock_init(&ep->lock); init_waitqueue_head(&ep->wq); init_waitqueue_head(&ep->poll_wait); INIT_LIST_HEAD(&ep->rdllist); @@ -1108,100 +1107,10 @@ struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, } #endif /* CONFIG_KCMP */ -/* - * Adds a new entry to the tail of the list in a lockless way, i.e. - * multiple CPUs are allowed to call this function concurrently. - * - * Beware: it is necessary to prevent any other modifications of the - * existing list until all changes are completed, in other words - * concurrent list_add_tail_lockless() calls should be protected - * with a read lock, where write lock acts as a barrier which - * makes sure all list_add_tail_lockless() calls are fully - * completed. - * - * Also an element can be locklessly added to the list only in one - * direction i.e. either to the tail or to the head, otherwise - * concurrent access will corrupt the list. - * - * Return: %false if element has been already added to the list, %true - * otherwise. - */ -static inline bool list_add_tail_lockless(struct list_head *new, - struct list_head *head) -{ - struct list_head *prev; - - /* - * This is simple 'new->next = head' operation, but cmpxchg() - * is used in order to detect that same element has been just - * added to the list from another CPU: the winner observes - * new->next == new. - */ - if (!try_cmpxchg(&new->next, &new, head)) - return false; - - /* - * Initially ->next of a new element must be updated with the head - * (we are inserting to the tail) and only then pointers are atomically - * exchanged. XCHG guarantees memory ordering, thus ->next should be - * updated before pointers are actually swapped and pointers are - * swapped before prev->next is updated. - */ - - prev = xchg(&head->prev, new); - - /* - * It is safe to modify prev->next and new->prev, because a new element - * is added only to the tail and new->next is updated before XCHG. - */ - - prev->next = new; - new->prev = prev; - - return true; -} - -/* - * Chains a new epi entry to the tail of the ep->ovflist in a lockless way, - * i.e. multiple CPUs are allowed to call this function concurrently. - * - * Return: %false if epi element has been already chained, %true otherwise. - */ -static inline bool chain_epi_lockless(struct epitem *epi) -{ - struct eventpoll *ep = epi->ep; - - /* Fast preliminary check */ - if (epi->next != EP_UNACTIVE_PTR) - return false; - - /* Check that the same epi has not been just chained from another CPU */ - if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR) - return false; - - /* Atomically exchange tail */ - epi->next = xchg(&ep->ovflist, epi); - - return true; -} - /* * This is the callback that is passed to the wait queue wakeup * mechanism. It is called by the stored file descriptors when they * have events to report. - * - * This callback takes a read lock in order not to contend with concurrent - * events from another file descriptor, thus all modifications to ->rdllist - * or ->ovflist are lockless. Read lock is paired with the write lock from - * ep_scan_ready_list(), which stops all list modifications and guarantees - * that lists state is seen correctly. - * - * Another thing worth to mention is that ep_poll_callback() can be called - * concurrently for the same @epi from different CPUs if poll table was inited - * with several wait queues entries. Plural wakeup from different CPUs of a - * single wait queue is serialized by wq.lock, but the case when multiple wait - * queues are used should be detected accordingly. This is detected using - * cmpxchg() operation. */ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) { @@ -1212,7 +1121,7 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v unsigned long flags; int ewake = 0; - read_lock_irqsave(&ep->lock, flags); + spin_lock_irqsave(&ep->lock, flags); ep_set_busy_poll_napi_id(epi); @@ -1241,12 +1150,15 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v * chained in ep->ovflist and requeued later on. */ if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) { - if (chain_epi_lockless(epi)) + if (epi->next == EP_UNACTIVE_PTR) { + epi->next = READ_ONCE(ep->ovflist); + WRITE_ONCE(ep->ovflist, epi); ep_pm_stay_awake_rcu(epi); + } } else if (!ep_is_linked(epi)) { /* In the usual case, add event to ready list. */ - if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist)) - ep_pm_stay_awake_rcu(epi); + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake_rcu(epi); } /* @@ -1279,7 +1191,7 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v pwake++; out_unlock: - read_unlock_irqrestore(&ep->lock, flags); + spin_unlock_irqrestore(&ep->lock, flags); /* We have to call this outside the lock */ if (pwake) @@ -1619,7 +1531,7 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, } /* We have to drop the new item inside our item list to keep track of it */ - write_lock_irq(&ep->lock); + spin_lock_irq(&ep->lock); /* record NAPI ID of new item if present */ ep_set_busy_poll_napi_id(epi); @@ -1636,7 +1548,7 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, pwake++; } - write_unlock_irq(&ep->lock); + spin_unlock_irq(&ep->lock); /* We have to call this outside the lock */ if (pwake) @@ -1700,7 +1612,7 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi, * list, push it inside. */ if (ep_item_poll(epi, &pt, 1)) { - write_lock_irq(&ep->lock); + spin_lock_irq(&ep->lock); if (!ep_is_linked(epi)) { list_add_tail(&epi->rdllink, &ep->rdllist); ep_pm_stay_awake(epi); @@ -1711,7 +1623,7 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi, if (waitqueue_active(&ep->poll_wait)) pwake++; } - write_unlock_irq(&ep->lock); + spin_unlock_irq(&ep->lock); } /* We have to call this outside the lock */ @@ -1800,7 +1712,7 @@ static int ep_send_events(struct eventpoll *ep, * availability. At this point, no one can insert * into ep->rdllist besides us. The epoll_ctl() * callers are locked out by - * ep_scan_ready_list() holding "mtx" and the + * ep_send_events() holding "mtx" and the * poll callback will queue them in ep->ovflist. */ list_add_tail(&epi->rdllink, &ep->rdllist); @@ -1944,7 +1856,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, init_wait(&wait); wait.func = ep_autoremove_wake_function; - write_lock_irq(&ep->lock); + spin_lock_irq(&ep->lock); /* * Barrierless variant, waitqueue_active() is called under * the same lock on wakeup ep_poll_callback() side, so it @@ -1953,7 +1865,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, __set_current_state(TASK_INTERRUPTIBLE); /* - * Do the final check under the lock. ep_scan_ready_list() + * Do the final check under the lock. ep_start/done_scan() * plays with two lists (->rdllist and ->ovflist) and there * is always a race when both lists are empty for short * period of time although events are pending, so lock is @@ -1963,7 +1875,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, if (!eavail) __add_wait_queue_exclusive(&ep->wq, &wait); - write_unlock_irq(&ep->lock); + spin_unlock_irq(&ep->lock); if (!eavail) timed_out = !schedule_hrtimeout_range(to, slack, @@ -1978,7 +1890,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, eavail = 1; if (!list_empty_careful(&wait.entry)) { - write_lock_irq(&ep->lock); + spin_lock_irq(&ep->lock); /* * If the thread timed out and is not on the wait queue, * it means that the thread was woken up after its @@ -1989,29 +1901,30 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, if (timed_out) eavail = list_empty(&wait.entry); __remove_wait_queue(&ep->wq, &wait); - write_unlock_irq(&ep->lock); + spin_unlock_irq(&ep->lock); } } } /** - * ep_loop_check_proc - verify that adding an epoll file inside another - * epoll structure does not violate the constraints, in - * terms of closed loops, or too deep chains (which can - * result in excessive stack usage). + * ep_loop_check_proc - verify that adding an epoll file @ep inside another + * epoll file does not create closed loops, and + * determine the depth of the subtree starting at @ep * * @ep: the &struct eventpoll to be currently checked. * @depth: Current depth of the path being checked. * - * Return: %zero if adding the epoll @file inside current epoll - * structure @ep does not violate the constraints, or %-1 otherwise. + * Return: depth of the subtree, or INT_MAX if we found a loop or went too deep. */ static int ep_loop_check_proc(struct eventpoll *ep, int depth) { - int error = 0; + int result = 0; struct rb_node *rbp; struct epitem *epi; + if (ep->gen == loop_check_gen) + return ep->loop_check_depth; + mutex_lock_nested(&ep->mtx, depth + 1); ep->gen = loop_check_gen; for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { @@ -2019,13 +1932,11 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth) if (unlikely(is_file_epoll(epi->ffd.file))) { struct eventpoll *ep_tovisit; ep_tovisit = epi->ffd.file->private_data; - if (ep_tovisit->gen == loop_check_gen) - continue; if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS) - error = -1; + result = INT_MAX; else - error = ep_loop_check_proc(ep_tovisit, depth + 1); - if (error != 0) + result = max(result, ep_loop_check_proc(ep_tovisit, depth + 1) + 1); + if (result > EP_MAX_NESTS) break; } else { /* @@ -2039,9 +1950,27 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth) list_file(epi->ffd.file); } } + ep->loop_check_depth = result; mutex_unlock(&ep->mtx); - return error; + return result; +} + +/** + * ep_get_upwards_depth_proc - determine depth of @ep when traversed upwards + */ +static int ep_get_upwards_depth_proc(struct eventpoll *ep, int depth) +{ + int result = 0; + struct epitem *epi; + + if (ep->gen == loop_check_gen) + return ep->loop_check_depth; + hlist_for_each_entry_rcu(epi, &ep->refs, fllink) + result = max(result, ep_get_upwards_depth_proc(epi->ep, depth + 1) + 1); + ep->gen = loop_check_gen; + ep->loop_check_depth = result; + return result; } /** @@ -2057,8 +1986,22 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth) */ static int ep_loop_check(struct eventpoll *ep, struct eventpoll *to) { + int depth, upwards_depth; + inserting_into = ep; - return ep_loop_check_proc(to, 0); + /* + * Check how deep down we can get from @to, and whether it is possible + * to loop up to @ep. + */ + depth = ep_loop_check_proc(to, 0); + if (depth > EP_MAX_NESTS) + return -1; + /* Check how far up we can go from @ep. */ + rcu_read_lock(); + upwards_depth = ep_get_upwards_depth_proc(ep, 0); + rcu_read_unlock(); + + return (depth+1+upwards_depth > EP_MAX_NESTS) ? -1 : 0; } static void clear_tfile_check_list(void) diff --git a/fs/exec.c b/fs/exec.c index cf487b0e49dd1..71bd99ce833e5 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -68,6 +68,10 @@ #include #include +#ifndef __GENKSYMS__ +#include +#endif + #include #include #include @@ -750,7 +754,7 @@ int setup_arg_pages(struct linux_binprm *bprm, unsigned long stack_top, int executable_stack) { - unsigned long ret; + int ret; unsigned long stack_shift; struct mm_struct *mm = current->mm; struct vm_area_struct *vma = bprm->vma; @@ -1250,6 +1254,7 @@ EXPORT_SYMBOL_GPL(__set_task_comm); int begin_new_exec(struct linux_binprm * bprm) { struct task_struct *me = current; + struct files_struct *old_files; int retval; /* Once we are committed compute the creds */ @@ -1273,11 +1278,22 @@ int begin_new_exec(struct linux_binprm * bprm) */ io_uring_task_cancel(); + /* + * unshare_files() may not do anything, but we still need to account dmabufs against the + * new_dmabuf_info even if it doesn't. We need to keep track of the original files_struct + * to handle task_dma_buf_info refcounting. + */ + old_files = me->files; + /* Ensure the files table is not shared. */ retval = unshare_files(); if (retval) goto out; + retval = dma_buf_begin_new_exec(old_files); + if (retval) + goto out; + /* * Must be called _before_ exec_mmap() as bprm->mm is * not visible until then. Doing it here also ensures @@ -1301,6 +1317,14 @@ int begin_new_exec(struct linux_binprm * bprm) goto out; bprm->mm = NULL; + /* + * New MM has just been installed. Use the task's new dmabuf_info (from + * dma_buf_begin_new_exec) for the new mm_struct. + */ + if (IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) { + refcount_inc(¤t->dmabuf_info->refcnt); + me->mm->dmabuf_info = current->dmabuf_info; + } retval = exec_task_namespaces(); if (retval) diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c index c3f8aeefbbd3d..86ecf07ac3d4e 100644 --- a/fs/exfat/dir.c +++ b/fs/exfat/dir.c @@ -1048,6 +1048,7 @@ int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, struct exfat_hint_femp candi_empty; struct exfat_sb_info *sbi = EXFAT_SB(sb); int num_entries = exfat_calc_num_entries(p_uniname); + unsigned int clu_count = 0; if (num_entries < 0) return num_entries; @@ -1185,6 +1186,10 @@ int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, } else { if (exfat_get_next_cluster(sb, &clu.dir)) return -EIO; + + /* break if the cluster chain includes a loop */ + if (unlikely(++clu_count > EXFAT_DATA_CLUSTER_COUNT(sbi))) + goto not_found; } } @@ -1268,6 +1273,7 @@ int exfat_count_dir_entries(struct super_block *sb, struct exfat_chain *p_dir) int i, count = 0; int dentries_per_clu; unsigned int entry_type; + unsigned int clu_count = 0; struct exfat_chain clu; struct exfat_dentry *ep; struct exfat_sb_info *sbi = EXFAT_SB(sb); @@ -1300,6 +1306,12 @@ int exfat_count_dir_entries(struct super_block *sb, struct exfat_chain *p_dir) } else { if (exfat_get_next_cluster(sb, &(clu.dir))) return -EIO; + + if (unlikely(++clu_count > sbi->used_clusters)) { + exfat_fs_error(sb, "FAT or bitmap is corrupted"); + return -EIO; + } + } } diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c index 24e1e05f9f34a..e5f4ce8c38e1a 100644 --- a/fs/exfat/fatent.c +++ b/fs/exfat/fatent.c @@ -89,35 +89,36 @@ int exfat_ent_get(struct super_block *sb, unsigned int loc, int err; if (!is_valid_cluster(sbi, loc)) { - exfat_fs_error(sb, "invalid access to FAT (entry 0x%08x)", + exfat_fs_error_ratelimit(sb, + "invalid access to FAT (entry 0x%08x)", loc); return -EIO; } err = __exfat_ent_get(sb, loc, content); if (err) { - exfat_fs_error(sb, + exfat_fs_error_ratelimit(sb, "failed to access to FAT (entry 0x%08x, err:%d)", loc, err); return err; } if (*content == EXFAT_FREE_CLUSTER) { - exfat_fs_error(sb, + exfat_fs_error_ratelimit(sb, "invalid access to FAT free cluster (entry 0x%08x)", loc); return -EIO; } if (*content == EXFAT_BAD_CLUSTER) { - exfat_fs_error(sb, + exfat_fs_error_ratelimit(sb, "invalid access to FAT bad cluster (entry 0x%08x)", loc); return -EIO; } if (*content != EXFAT_EOF_CLUSTER && !is_valid_cluster(sbi, *content)) { - exfat_fs_error(sb, + exfat_fs_error_ratelimit(sb, "invalid access to FAT (entry 0x%08x) bogus content (0x%08x)", loc, *content); return -EIO; @@ -461,5 +462,15 @@ int exfat_count_num_clusters(struct super_block *sb, } *ret_count = count; + + /* + * since exfat_count_used_clusters() is not called, sbi->used_clusters + * cannot be used here. + */ + if (unlikely(i == sbi->num_clusters && clu != EXFAT_EOF_CLUSTER)) { + exfat_fs_error(sb, "The cluster chain has a loop"); + return -EIO; + } + return 0; } diff --git a/fs/exfat/file.c b/fs/exfat/file.c index efd24e29f1195..272208708ffc4 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -610,9 +610,8 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) if (pos > valid_size) pos = valid_size; - if (iocb_is_dsync(iocb) && iocb->ki_pos > pos) { - ssize_t err = vfs_fsync_range(file, pos, iocb->ki_pos - 1, - iocb->ki_flags & IOCB_SYNC); + if (iocb->ki_pos > pos) { + ssize_t err = generic_write_sync(iocb, iocb->ki_pos - pos); if (err < 0) return err; } diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c index 0acd3ad717d84..57ee55c2b7b09 100644 --- a/fs/exfat/namei.c +++ b/fs/exfat/namei.c @@ -872,6 +872,7 @@ static int exfat_check_dir_empty(struct super_block *sb, { int i, dentries_per_clu; unsigned int type; + unsigned int clu_count = 0; struct exfat_chain clu; struct exfat_dentry *ep; struct exfat_sb_info *sbi = EXFAT_SB(sb); @@ -908,6 +909,10 @@ static int exfat_check_dir_empty(struct super_block *sb, } else { if (exfat_get_next_cluster(sb, &(clu.dir))) return -EIO; + + /* break if the cluster chain includes a loop */ + if (unlikely(++clu_count > EXFAT_DATA_CLUSTER_COUNT(sbi))) + break; } } diff --git a/fs/exfat/super.c b/fs/exfat/super.c index 1400a521af4e0..246e6807da199 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -370,13 +370,12 @@ static void exfat_hash_init(struct super_block *sb) INIT_HLIST_HEAD(&sbi->inode_hashtable[i]); } -static int exfat_read_root(struct inode *inode) +static int exfat_read_root(struct inode *inode, struct exfat_chain *root_clu) { struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); struct exfat_inode_info *ei = EXFAT_I(inode); - struct exfat_chain cdir; - int num_subdirs, num_clu = 0; + int num_subdirs; exfat_chain_set(&ei->dir, sbi->root_dir, 0, ALLOC_FAT_CHAIN); ei->entry = -1; @@ -389,12 +388,9 @@ static int exfat_read_root(struct inode *inode) ei->hint_stat.clu = sbi->root_dir; ei->hint_femp.eidx = EXFAT_HINT_NONE; - exfat_chain_set(&cdir, sbi->root_dir, 0, ALLOC_FAT_CHAIN); - if (exfat_count_num_clusters(sb, &cdir, &num_clu)) - return -EIO; - i_size_write(inode, num_clu << sbi->cluster_size_bits); + i_size_write(inode, EXFAT_CLU_TO_B(root_clu->size, sbi)); - num_subdirs = exfat_count_dir_entries(sb, &cdir); + num_subdirs = exfat_count_dir_entries(sb, root_clu); if (num_subdirs < 0) return -EIO; set_nlink(inode, num_subdirs + EXFAT_MIN_SUBDIR); @@ -456,7 +452,10 @@ static int exfat_read_boot_sector(struct super_block *sb) struct exfat_sb_info *sbi = EXFAT_SB(sb); /* set block size to read super block */ - sb_min_blocksize(sb, 512); + if (!sb_min_blocksize(sb, 512)) { + exfat_err(sb, "unable to set blocksize"); + return -EINVAL; + } /* read boot sector */ sbi->boot_bh = sb_bread(sb, 0); @@ -608,7 +607,8 @@ static int exfat_verify_boot_region(struct super_block *sb) } /* mount the file system volume */ -static int __exfat_fill_super(struct super_block *sb) +static int __exfat_fill_super(struct super_block *sb, + struct exfat_chain *root_clu) { int ret; struct exfat_sb_info *sbi = EXFAT_SB(sb); @@ -625,6 +625,18 @@ static int __exfat_fill_super(struct super_block *sb) goto free_bh; } + /* + * Call exfat_count_num_cluster() before searching for up-case and + * bitmap directory entries to avoid infinite loop if they are missing + * and the cluster chain includes a loop. + */ + exfat_chain_set(root_clu, sbi->root_dir, 0, ALLOC_FAT_CHAIN); + ret = exfat_count_num_clusters(sb, root_clu, &root_clu->size); + if (ret) { + exfat_err(sb, "failed to count the number of clusters in root"); + goto free_bh; + } + ret = exfat_create_upcase_table(sb); if (ret) { exfat_err(sb, "failed to load upcase table"); @@ -657,6 +669,7 @@ static int exfat_fill_super(struct super_block *sb, struct fs_context *fc) struct exfat_sb_info *sbi = sb->s_fs_info; struct exfat_mount_options *opts = &sbi->options; struct inode *root_inode; + struct exfat_chain root_clu; int err; if (opts->allow_utime == (unsigned short)-1) @@ -675,7 +688,7 @@ static int exfat_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_time_min = EXFAT_MIN_TIMESTAMP_SECS; sb->s_time_max = EXFAT_MAX_TIMESTAMP_SECS; - err = __exfat_fill_super(sb); + err = __exfat_fill_super(sb, &root_clu); if (err) { exfat_err(sb, "failed to recognize exfat type"); goto check_nls_io; @@ -710,7 +723,7 @@ static int exfat_fill_super(struct super_block *sb, struct fs_context *fc) root_inode->i_ino = EXFAT_ROOT_INO; inode_set_iversion(root_inode, 1); - err = exfat_read_root(root_inode); + err = exfat_read_root(root_inode, &root_clu); if (err) { exfat_err(sb, "failed to initialize root inode"); goto put_inode; diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 314b415ee5186..6ff1f8f29a3c6 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -895,9 +895,19 @@ int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { int ret; + loff_t i_size; inode_lock(inode); - len = min_t(u64, len, i_size_read(inode)); + i_size = i_size_read(inode); + /* + * iomap_fiemap() returns EINVAL for 0 length. Make sure we don't trim + * length to 0 but still trim the range as much as possible since + * ext2_get_blocks() iterates unmapped space block by block which is + * slow. + */ + if (i_size == 0) + i_size = 1; + len = min_t(u64, len, i_size); ret = iomap_fiemap(inode, fieinfo, start, len, &ext2_iomap_ops); inode_unlock(inode); diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c index 396474e9e2bff..3a2dfc59fb40f 100644 --- a/fs/ext4/balloc.c +++ b/fs/ext4/balloc.c @@ -641,8 +641,8 @@ static int ext4_has_free_clusters(struct ext4_sb_info *sbi, /* Hm, nope. Are (enough) root reserved clusters available? */ if (uid_eq(sbi->s_resuid, current_fsuid()) || (!gid_eq(sbi->s_resgid, GLOBAL_ROOT_GID) && in_group_p(sbi->s_resgid)) || - capable(CAP_SYS_RESOURCE) || - (flags & EXT4_MB_USE_ROOT_BLOCKS)) { + (flags & EXT4_MB_USE_ROOT_BLOCKS) || + capable(CAP_SYS_RESOURCE)) { if (free_clusters >= (nclusters + dirty_clusters + resv_clusters)) diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 60455c84a9374..7afce7b744c01 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -273,7 +273,8 @@ struct ext4_system_blocks { /* * Flags for ext4_io_end->flags */ -#define EXT4_IO_END_UNWRITTEN 0x0001 +#define EXT4_IO_END_UNWRITTEN 0x0001 +#define EXT4_IO_END_FAILED 0x0002 struct ext4_io_end_vec { struct list_head list; /* list of io_end_vec */ @@ -1959,6 +1960,16 @@ static inline bool ext4_verity_in_progress(struct inode *inode) #define NEXT_ORPHAN(inode) EXT4_I(inode)->i_dtime +/* + * Check whether the inode is tracked as orphan (either in orphan file or + * orphan list). + */ +static inline bool ext4_inode_orphan_tracked(struct inode *inode) +{ + return ext4_test_inode_state(inode, EXT4_STATE_ORPHAN_FILE) || + !list_empty(&EXT4_I(inode)->i_orphan); +} + /* * Codes for operating systems */ @@ -2994,6 +3005,8 @@ extern int ext4_inode_attach_jinode(struct inode *inode); extern int ext4_can_truncate(struct inode *inode); extern int ext4_truncate(struct inode *); extern int ext4_break_layouts(struct inode *); +extern int ext4_truncate_page_cache_block_range(struct inode *inode, + loff_t start, loff_t end); extern int ext4_punch_hole(struct file *file, loff_t offset, loff_t length); extern void ext4_set_inode_flags(struct inode *, bool init); extern int ext4_alloc_da_blocks(struct inode *inode); @@ -3348,6 +3361,13 @@ static inline unsigned int ext4_flex_bg_size(struct ext4_sb_info *sbi) return 1 << sbi->s_log_groups_per_flex; } +static inline loff_t ext4_get_maxbytes(struct inode *inode) +{ + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + return inode->i_sb->s_maxbytes; + return EXT4_SB(inode->i_sb)->s_bitmap_maxbytes; +} + #define ext4_std_error(sb, errno) \ do { \ if ((errno)) \ diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c index d1a2e66244017..32f59295cfa05 100644 --- a/fs/ext4/ext4_jbd2.c +++ b/fs/ext4/ext4_jbd2.c @@ -277,9 +277,16 @@ int __ext4_forget(const char *where, unsigned int line, handle_t *handle, bh, is_metadata, inode->i_mode, test_opt(inode->i_sb, DATA_FLAGS)); - /* In the no journal case, we can just do a bforget and return */ + /* + * In the no journal case, we should wait for the ongoing buffer + * to complete and do a forget. + */ if (!ext4_handle_valid(handle)) { - bforget(bh); + if (bh) { + clear_buffer_dirty(bh); + wait_on_buffer(bh); + __bforget(bh); + } return 0; } diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 32218ac7f50fe..a3d3c9fc64262 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -2374,18 +2374,19 @@ int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks, int ext4_ext_index_trans_blocks(struct inode *inode, int extents) { int index; - int depth; /* If we are converting the inline data, only one is needed here. */ if (ext4_has_inline_data(inode)) return 1; - depth = ext_depth(inode); - + /* + * Extent tree can change between the time we estimate credits and + * the time we actually modify the tree. Assume the worst case. + */ if (extents <= 1) - index = depth * 2; + index = EXT4_MAX_EXTENT_DEPTH * 2; else - index = depth * 3; + index = EXT4_MAX_EXTENT_DEPTH * 3; return index; } @@ -4660,22 +4661,13 @@ static long ext4_zero_range(struct file *file, loff_t offset, goto out_mutex; } - /* - * For journalled data we need to write (and checkpoint) pages - * before discarding page cache to avoid inconsitent data on - * disk in case of crash before zeroing trans is committed. - */ - if (ext4_should_journal_data(inode)) { - ret = filemap_write_and_wait_range(mapping, start, - end - 1); - if (ret) { - filemap_invalidate_unlock(mapping); - goto out_mutex; - } + /* Now release the pages and zero block aligned part of pages */ + ret = ext4_truncate_page_cache_block_range(inode, start, end); + if (ret) { + filemap_invalidate_unlock(mapping); + goto out_mutex; } - /* Now release the pages and zero block aligned part of pages */ - truncate_pagecache_range(inode, start, end - 1); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, @@ -4978,12 +4970,7 @@ static const struct iomap_ops ext4_iomap_xattr_ops = { static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len) { - u64 maxbytes; - - if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) - maxbytes = inode->i_sb->s_maxbytes; - else - maxbytes = EXT4_SB(inode->i_sb)->s_bitmap_maxbytes; + u64 maxbytes = ext4_get_maxbytes(inode); if (*len == 0) return -EINVAL; diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index b527f4ab47e02..62a6960242c5a 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -675,7 +675,7 @@ void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t star static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) { - blk_opf_t write_flags = REQ_SYNC; + blk_opf_t write_flags = JBD2_JOURNAL_REQ_FLAGS; struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh; /* Add REQ_FUA | REQ_PREFLUSH only its tail */ diff --git a/fs/ext4/file.c b/fs/ext4/file.c index c71af675e310a..0d387a5b5a678 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -354,7 +354,7 @@ static void ext4_inode_extension_cleanup(struct inode *inode, bool need_trunc) * to cleanup the orphan list in ext4_handle_inode_extension(). Do it * now. */ - if (!list_empty(&EXT4_I(inode)->i_orphan) && inode->i_nlink) { + if (ext4_inode_orphan_tracked(inode) && inode->i_nlink) { handle_t *handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); if (IS_ERR(handle)) { @@ -898,12 +898,7 @@ static int ext4_file_open(struct inode *inode, struct file *filp) loff_t ext4_llseek(struct file *file, loff_t offset, int whence) { struct inode *inode = file->f_mapping->host; - loff_t maxbytes; - - if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) - maxbytes = EXT4_SB(inode->i_sb)->s_bitmap_maxbytes; - else - maxbytes = inode->i_sb->s_maxbytes; + loff_t maxbytes = ext4_get_maxbytes(inode); switch (whence) { default: diff --git a/fs/ext4/fsmap.c b/fs/ext4/fsmap.c index 53a05b8292f03..c970f41c50483 100644 --- a/fs/ext4/fsmap.c +++ b/fs/ext4/fsmap.c @@ -74,7 +74,8 @@ static int ext4_getfsmap_dev_compare(const void *p1, const void *p2) static bool ext4_getfsmap_rec_before_low_key(struct ext4_getfsmap_info *info, struct ext4_fsmap *rec) { - return rec->fmr_physical < info->gfi_low.fmr_physical; + return rec->fmr_physical + rec->fmr_length <= + info->gfi_low.fmr_physical; } /* @@ -200,15 +201,18 @@ static int ext4_getfsmap_meta_helper(struct super_block *sb, ext4_group_first_block_no(sb, agno)); fs_end = fs_start + EXT4_C2B(sbi, len); - /* Return relevant extents from the meta_list */ + /* + * Return relevant extents from the meta_list. We emit all extents that + * partially/fully overlap with the query range + */ list_for_each_entry_safe(p, tmp, &info->gfi_meta_list, fmr_list) { - if (p->fmr_physical < info->gfi_next_fsblk) { + if (p->fmr_physical + p->fmr_length <= info->gfi_next_fsblk) { list_del(&p->fmr_list); kfree(p); continue; } - if (p->fmr_physical <= fs_start || - p->fmr_physical + p->fmr_length <= fs_end) { + if (p->fmr_physical <= fs_end && + p->fmr_physical + p->fmr_length > fs_start) { /* Emit the retained free extent record if present */ if (info->gfi_lastfree.fmr_owner) { error = ext4_getfsmap_helper(sb, info, @@ -393,6 +397,14 @@ static unsigned int ext4_getfsmap_find_sb(struct super_block *sb, /* Reserved GDT blocks */ if (!ext4_has_feature_meta_bg(sb) || metagroup < first_meta_bg) { len = le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks); + + /* + * mkfs.ext4 can set s_reserved_gdt_blocks as 0 in some cases, + * check for that. + */ + if (!len) + return 0; + error = ext4_getfsmap_fill(meta_list, fsb, len, EXT4_FMR_OWN_RESV_GDT); if (error) @@ -526,6 +538,7 @@ static int ext4_getfsmap_datadev(struct super_block *sb, ext4_group_t end_ag; ext4_grpblk_t first_cluster; ext4_grpblk_t last_cluster; + struct ext4_fsmap irec; int error = 0; bofs = le32_to_cpu(sbi->s_es->s_first_data_block); @@ -609,10 +622,18 @@ static int ext4_getfsmap_datadev(struct super_block *sb, goto err; } - /* Report any gaps at the end of the bg */ + /* + * The dummy record below will cause ext4_getfsmap_helper() to report + * any allocated blocks at the end of the range. + */ + irec.fmr_device = 0; + irec.fmr_physical = end_fsb + 1; + irec.fmr_length = 0; + irec.fmr_owner = EXT4_FMR_OWN_FREE; + irec.fmr_flags = 0; + info->gfi_last = true; - error = ext4_getfsmap_datadev_helper(sb, end_ag, last_cluster + 1, - 0, info); + error = ext4_getfsmap_helper(sb, info, &irec); if (error) goto err; diff --git a/fs/ext4/indirect.c b/fs/ext4/indirect.c index f2c495b745f1e..d18a5bee10215 100644 --- a/fs/ext4/indirect.c +++ b/fs/ext4/indirect.c @@ -539,7 +539,7 @@ int ext4_ind_map_blocks(handle_t *handle, struct inode *inode, int indirect_blks; int blocks_to_boundary = 0; int depth; - int count = 0; + u64 count = 0; ext4_fsblk_t first_block = 0; trace_ext4_ind_map_blocks_enter(inode, map->m_lblk, map->m_len, flags); @@ -588,7 +588,7 @@ int ext4_ind_map_blocks(handle_t *handle, struct inode *inode, count++; /* Fill in size of a hole we found */ map->m_pblk = 0; - map->m_len = min_t(unsigned int, map->m_len, count); + map->m_len = umin(map->m_len, count); goto cleanup; } diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 3f363276ddd36..5fa1dd58ac42c 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -298,7 +298,11 @@ static int ext4_create_inline_data(handle_t *handle, if (error) goto out; - BUG_ON(!is.s.not_found); + if (!is.s.not_found) { + EXT4_ERROR_INODE(inode, "unexpected inline data xattr"); + error = -EFSCORRUPTED; + goto out; + } error = ext4_xattr_ibody_set(handle, inode, &i, &is); if (error) { @@ -349,7 +353,11 @@ static int ext4_update_inline_data(handle_t *handle, struct inode *inode, if (error) goto out; - BUG_ON(is.s.not_found); + if (is.s.not_found) { + EXT4_ERROR_INODE(inode, "missing inline data xattr"); + error = -EFSCORRUPTED; + goto out; + } len -= EXT4_MIN_INLINE_DATA_SIZE; value = kzalloc(len, GFP_NOFS); @@ -392,7 +400,7 @@ static int ext4_update_inline_data(handle_t *handle, struct inode *inode, } static int ext4_prepare_inline_data(handle_t *handle, struct inode *inode, - unsigned int len) + loff_t len) { int ret, size, no_expand; struct ext4_inode_info *ei = EXT4_I(inode); @@ -1966,7 +1974,12 @@ int ext4_inline_data_truncate(struct inode *inode, int *has_inline) if ((err = ext4_xattr_ibody_find(inode, &i, &is)) != 0) goto out_error; - BUG_ON(is.s.not_found); + if (is.s.not_found) { + EXT4_ERROR_INODE(inode, + "missing inline data xattr"); + err = -EFSCORRUPTED; + goto out_error; + } value_len = le32_to_cpu(is.s.here->e_value_size); value = kmalloc(value_len, GFP_NOFS); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index ba6dda1d4025c..f7f2ede1a8f26 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -145,7 +146,7 @@ static int ext4_meta_trans_blocks(struct inode *inode, int lblocks, */ int ext4_inode_is_fast_symlink(struct inode *inode) { - if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) { + if (!ext4_has_feature_ea_inode(inode->i_sb)) { int ea_blocks = EXT4_I(inode)->i_file_acl ? EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0; @@ -1017,7 +1018,12 @@ int ext4_walk_page_buffers(handle_t *handle, struct inode *inode, */ static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh) { - folio_mark_dirty(bh->b_folio); + struct folio *folio = bh->b_folio; + struct inode *inode = folio->mapping->host; + + /* only regular files have a_ops */ + if (S_ISREG(inode->i_mode)) + folio_mark_dirty(folio); return ext4_handle_dirty_metadata(handle, NULL, bh); } @@ -3872,7 +3878,11 @@ int ext4_can_truncate(struct inode *inode) * We have to make sure i_disksize gets properly updated before we truncate * page cache due to hole punching or zero range. Otherwise i_disksize update * can get lost as it may have been postponed to submission of writeback but - * that will never happen after we truncate page cache. + * that will never happen if we remove the folio containing i_size from the + * page cache. Also if we punch hole within i_size but above i_disksize, + * following ext4_page_mkwrite() may mistakenly allocate written blocks over + * the hole and thus introduce allocated blocks beyond i_disksize which is + * not allowed (e2fsck would complain in case of crash). */ int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, loff_t len) @@ -3883,9 +3893,11 @@ int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, loff_t size = i_size_read(inode); WARN_ON(!inode_is_locked(inode)); - if (offset > size || offset + len < size) + if (offset > size) return 0; + if (offset + len < size) + size = offset + len; if (EXT4_I(inode)->i_disksize >= size) return 0; @@ -3899,6 +3911,68 @@ int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, return ret; } +static inline void ext4_truncate_folio(struct inode *inode, + loff_t start, loff_t end) +{ + unsigned long blocksize = i_blocksize(inode); + struct folio *folio; + + /* Nothing to be done if no complete block needs to be truncated. */ + if (round_up(start, blocksize) >= round_down(end, blocksize)) + return; + + folio = filemap_lock_folio(inode->i_mapping, start >> PAGE_SHIFT); + if (IS_ERR(folio)) + return; + + if (folio_mkclean(folio)) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); +} + +int ext4_truncate_page_cache_block_range(struct inode *inode, + loff_t start, loff_t end) +{ + unsigned long blocksize = i_blocksize(inode); + int ret; + + /* + * For journalled data we need to write (and checkpoint) pages + * before discarding page cache to avoid inconsitent data on disk + * in case of crash before freeing or unwritten converting trans + * is committed. + */ + if (ext4_should_journal_data(inode)) { + ret = filemap_write_and_wait_range(inode->i_mapping, start, + end - 1); + if (ret) + return ret; + goto truncate_pagecache; + } + + /* + * If the block size is less than the page size, the file's mapped + * blocks within one page could be freed or converted to unwritten. + * So it's necessary to remove writable userspace mappings, and then + * ext4_page_mkwrite() can be called during subsequent write access + * to these partial folios. + */ + if (!IS_ALIGNED(start | end, PAGE_SIZE) && + blocksize < PAGE_SIZE && start < inode->i_size) { + loff_t page_boundary = round_up(start, PAGE_SIZE); + + ext4_truncate_folio(inode, start, min(page_boundary, end)); + if (end > page_boundary) + ext4_truncate_folio(inode, + round_down(end, PAGE_SIZE), end); + } + +truncate_pagecache: + truncate_pagecache_range(inode, start, end - 1); + return 0; +} + static void ext4_wait_dax_page(struct inode *inode) { filemap_invalidate_unlock(inode->i_mapping); @@ -3953,17 +4027,6 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) trace_ext4_punch_hole(inode, offset, length, 0); - /* - * Write out all dirty pages to avoid race conditions - * Then release them. - */ - if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) { - ret = filemap_write_and_wait_range(mapping, offset, - offset + length - 1); - if (ret) - return ret; - } - inode_lock(inode); /* No need to punch hole beyond i_size */ @@ -4025,8 +4088,11 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) ret = ext4_update_disksize_before_punch(inode, offset, length); if (ret) goto out_dio; - truncate_pagecache_range(inode, first_block_offset, - last_block_offset); + + ret = ext4_truncate_page_cache_block_range(inode, + first_block_offset, last_block_offset + 1); + if (ret) + goto out_dio; } if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) @@ -4317,7 +4383,7 @@ static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode * old inodes get re-used with the upper 16 bits of the * uid/gid intact. */ - if (ei->i_dtime && list_empty(&ei->i_orphan)) { + if (ei->i_dtime && !ext4_inode_orphan_tracked(inode)) { raw_inode->i_uid_high = 0; raw_inode->i_gid_high = 0; } else { @@ -4885,13 +4951,22 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, } ei->i_flags = le32_to_cpu(raw_inode->i_flags); ext4_set_inode_flags(inode, true); + /* Detect invalid flag combination - can't have both inline data and extents */ + if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA) && + ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { + ext4_error_inode(inode, function, line, 0, + "inode has both inline data and extents flags"); + ret = -EFSCORRUPTED; + goto bad_inode; + } inode->i_blocks = ext4_inode_blocks(raw_inode, ei); ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo); if (ext4_has_feature_64bit(sb)) ei->i_file_acl |= ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; inode->i_size = ext4_isize(sb, raw_inode); - if ((size = i_size_read(inode)) < 0) { + size = i_size_read(inode); + if (size < 0 || size > ext4_get_maxbytes(inode)) { ext4_error_inode(inode, function, line, 0, "iget: bad i_size value: %lld", size); ret = -EFSCORRUPTED; diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 8a9f8c95c6f1e..c5f642096ab4e 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -841,30 +841,30 @@ static void mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) { struct ext4_sb_info *sbi = EXT4_SB(sb); - int new_order; + int new, old; - if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || grp->bb_fragments == 0) + if (!test_opt2(sb, MB_OPTIMIZE_SCAN)) return; - new_order = mb_avg_fragment_size_order(sb, - grp->bb_free / grp->bb_fragments); - if (new_order == grp->bb_avg_fragment_size_order) + old = grp->bb_avg_fragment_size_order; + new = grp->bb_fragments == 0 ? -1 : + mb_avg_fragment_size_order(sb, grp->bb_free / grp->bb_fragments); + if (new == old) return; - if (grp->bb_avg_fragment_size_order != -1) { - write_lock(&sbi->s_mb_avg_fragment_size_locks[ - grp->bb_avg_fragment_size_order]); + if (old >= 0) { + write_lock(&sbi->s_mb_avg_fragment_size_locks[old]); list_del(&grp->bb_avg_fragment_size_node); - write_unlock(&sbi->s_mb_avg_fragment_size_locks[ - grp->bb_avg_fragment_size_order]); + write_unlock(&sbi->s_mb_avg_fragment_size_locks[old]); + } + + grp->bb_avg_fragment_size_order = new; + if (new >= 0) { + write_lock(&sbi->s_mb_avg_fragment_size_locks[new]); + list_add_tail(&grp->bb_avg_fragment_size_node, + &sbi->s_mb_avg_fragment_size[new]); + write_unlock(&sbi->s_mb_avg_fragment_size_locks[new]); } - grp->bb_avg_fragment_size_order = new_order; - write_lock(&sbi->s_mb_avg_fragment_size_locks[ - grp->bb_avg_fragment_size_order]); - list_add_tail(&grp->bb_avg_fragment_size_node, - &sbi->s_mb_avg_fragment_size[grp->bb_avg_fragment_size_order]); - write_unlock(&sbi->s_mb_avg_fragment_size_locks[ - grp->bb_avg_fragment_size_order]); } /* @@ -1150,33 +1150,28 @@ static void mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) { struct ext4_sb_info *sbi = EXT4_SB(sb); - int i; + int new, old = grp->bb_largest_free_order; - for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--) - if (grp->bb_counters[i] > 0) + for (new = MB_NUM_ORDERS(sb) - 1; new >= 0; new--) + if (grp->bb_counters[new] > 0) break; + /* No need to move between order lists? */ - if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || - i == grp->bb_largest_free_order) { - grp->bb_largest_free_order = i; + if (new == old) return; - } - if (grp->bb_largest_free_order >= 0) { - write_lock(&sbi->s_mb_largest_free_orders_locks[ - grp->bb_largest_free_order]); + if (old >= 0 && !list_empty(&grp->bb_largest_free_order_node)) { + write_lock(&sbi->s_mb_largest_free_orders_locks[old]); list_del_init(&grp->bb_largest_free_order_node); - write_unlock(&sbi->s_mb_largest_free_orders_locks[ - grp->bb_largest_free_order]); + write_unlock(&sbi->s_mb_largest_free_orders_locks[old]); } - grp->bb_largest_free_order = i; - if (grp->bb_largest_free_order >= 0 && grp->bb_free) { - write_lock(&sbi->s_mb_largest_free_orders_locks[ - grp->bb_largest_free_order]); + + grp->bb_largest_free_order = new; + if (test_opt2(sb, MB_OPTIMIZE_SCAN) && new >= 0 && grp->bb_free) { + write_lock(&sbi->s_mb_largest_free_orders_locks[new]); list_add_tail(&grp->bb_largest_free_order_node, - &sbi->s_mb_largest_free_orders[grp->bb_largest_free_order]); - write_unlock(&sbi->s_mb_largest_free_orders_locks[ - grp->bb_largest_free_order]); + &sbi->s_mb_largest_free_orders[new]); + write_unlock(&sbi->s_mb_largest_free_orders_locks[new]); } } diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c index 5e6b07b349600..c566161127cd7 100644 --- a/fs/ext4/move_extent.c +++ b/fs/ext4/move_extent.c @@ -225,7 +225,7 @@ static int mext_page_mkuptodate(struct folio *folio, size_t from, size_t to) do { if (bh_offset(bh) + blocksize <= from) continue; - if (bh_offset(bh) > to) + if (bh_offset(bh) >= to) break; wait_on_buffer(bh); if (buffer_uptodate(bh)) diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index e5b47dda33175..05997b4d01203 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -109,11 +109,7 @@ int ext4_orphan_add(handle_t *handle, struct inode *inode) WARN_ON_ONCE(!(inode->i_state & (I_NEW | I_FREEING)) && !inode_is_locked(inode)); - /* - * Inode orphaned in orphan file or in orphan list? - */ - if (ext4_test_inode_state(inode, EXT4_STATE_ORPHAN_FILE) || - !list_empty(&EXT4_I(inode)->i_orphan)) + if (ext4_inode_orphan_tracked(inode)) return 0; /* @@ -517,7 +513,7 @@ void ext4_release_orphan_info(struct super_block *sb) return; for (i = 0; i < oi->of_blocks; i++) brelse(oi->of_binfo[i].ob_bh); - kfree(oi->of_binfo); + kvfree(oi->of_binfo); } static struct ext4_orphan_block_tail *ext4_orphan_block_tail( @@ -588,10 +584,22 @@ int ext4_init_orphan_info(struct super_block *sb) ext4_msg(sb, KERN_ERR, "get orphan inode failed"); return PTR_ERR(inode); } + /* + * This is just an artificial limit to prevent corrupted fs from + * consuming absurd amounts of memory when pinning blocks of orphan + * file in memory. + */ + if (inode->i_size > 8 << 20) { + ext4_msg(sb, KERN_ERR, "orphan file too big: %llu", + (unsigned long long)inode->i_size); + ret = -EFSCORRUPTED; + goto out_put; + } oi->of_blocks = inode->i_size >> sb->s_blocksize_bits; oi->of_csum_seed = EXT4_I(inode)->i_csum_seed; - oi->of_binfo = kmalloc(oi->of_blocks*sizeof(struct ext4_orphan_block), - GFP_KERNEL); + oi->of_binfo = kvmalloc_array(oi->of_blocks, + sizeof(struct ext4_orphan_block), + GFP_KERNEL); if (!oi->of_binfo) { ret = -ENOMEM; goto out_put; @@ -630,7 +638,7 @@ int ext4_init_orphan_info(struct super_block *sb) out_free: for (i--; i >= 0; i--) brelse(oi->of_binfo[i].ob_bh); - kfree(oi->of_binfo); + kvfree(oi->of_binfo); out_put: iput(inode); return ret; diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 7ab4f5a9bf5b8..7287dbfe13f12 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -181,14 +181,25 @@ static int ext4_end_io_end(ext4_io_end_t *io_end) "list->prev 0x%p\n", io_end, inode->i_ino, io_end->list.next, io_end->list.prev); - io_end->handle = NULL; /* Following call will use up the handle */ - ret = ext4_convert_unwritten_io_end_vec(handle, io_end); + /* + * Do not convert the unwritten extents if data writeback fails, + * or stale data may be exposed. + */ + io_end->handle = NULL; /* Following call will use up the handle */ + if (unlikely(io_end->flag & EXT4_IO_END_FAILED)) { + ret = -EIO; + if (handle) + jbd2_journal_free_reserved(handle); + } else { + ret = ext4_convert_unwritten_io_end_vec(handle, io_end); + } if (ret < 0 && !ext4_forced_shutdown(inode->i_sb)) { ext4_msg(inode->i_sb, KERN_EMERG, "failed to convert unwritten extents to written " "extents -- potential data loss! " "(inode %lu, error %d)", inode->i_ino, ret); } + ext4_clear_io_unwritten_flag(io_end); ext4_release_io_end(io_end); return ret; @@ -344,6 +355,7 @@ static void ext4_end_bio(struct bio *bio) bio->bi_status, inode->i_ino, (unsigned long long) bi_sector >> (inode->i_blkbits - 9)); + io_end->flag |= EXT4_IO_END_FAILED; mapping_set_error(inode->i_mapping, blk_status_to_errno(bio->bi_status)); } diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 9c280a4c482b8..a1c6006b0e79f 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1465,9 +1465,9 @@ static void ext4_free_in_core_inode(struct inode *inode) static void ext4_destroy_inode(struct inode *inode) { - if (!list_empty(&(EXT4_I(inode)->i_orphan))) { + if (ext4_inode_orphan_tracked(inode)) { ext4_msg(inode->i_sb, KERN_ERR, - "Inode %lu (%p): orphan list check failed!", + "Inode %lu (%p): inode tracked as orphan!", inode->i_ino, EXT4_I(inode)); print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 4, EXT4_I(inode), sizeof(struct ext4_inode_info), @@ -2029,6 +2029,9 @@ int ext4_init_fs_context(struct fs_context *fc) fc->fs_private = ctx; fc->ops = &ext4_context_ops; + /* i_version is always enabled now */ + fc->sb_flags |= SB_I_VERSION; + return 0; } @@ -2504,7 +2507,7 @@ static int parse_apply_sb_mount_options(struct super_block *sb, struct ext4_fs_context *m_ctx) { struct ext4_sb_info *sbi = EXT4_SB(sb); - char *s_mount_opts = NULL; + char s_mount_opts[65]; struct ext4_fs_context *s_ctx = NULL; struct fs_context *fc = NULL; int ret = -ENOMEM; @@ -2512,15 +2515,11 @@ static int parse_apply_sb_mount_options(struct super_block *sb, if (!sbi->s_es->s_mount_opts[0]) return 0; - s_mount_opts = kstrndup(sbi->s_es->s_mount_opts, - sizeof(sbi->s_es->s_mount_opts), - GFP_KERNEL); - if (!s_mount_opts) - return ret; + strscpy_pad(s_mount_opts, sbi->s_es->s_mount_opts, sizeof(s_mount_opts)); fc = kzalloc(sizeof(struct fs_context), GFP_KERNEL); if (!fc) - goto out_free; + return -ENOMEM; s_ctx = kzalloc(sizeof(struct ext4_fs_context), GFP_KERNEL); if (!s_ctx) @@ -2552,11 +2551,8 @@ static int parse_apply_sb_mount_options(struct super_block *sb, ret = 0; out_free: - if (fc) { - ext4_fc_free(fc); - kfree(fc); - } - kfree(s_mount_opts); + ext4_fc_free(fc); + kfree(fc); return ret; } @@ -2822,6 +2818,13 @@ static int ext4_check_opt_consistency(struct fs_context *fc, } if (is_remount) { + if (!sbi->s_journal && + ctx_test_mount_opt(ctx, EXT4_MOUNT_DATA_ERR_ABORT)) { + ext4_msg(NULL, KERN_WARNING, + "Remounting fs w/o journal so ignoring data_err option"); + ctx_clear_mount_opt(ctx, EXT4_MOUNT_DATA_ERR_ABORT); + } + if (ctx_test_mount_opt(ctx, EXT4_MOUNT_DAX_ALWAYS) && (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA)) { ext4_msg(NULL, KERN_ERR, "can't mount with " @@ -5301,9 +5304,6 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); - /* i_version is always enabled now */ - sb->s_flags |= SB_I_VERSION; - err = ext4_check_feature_compatibility(sb, es, silent); if (err) goto failed_mount; @@ -5394,6 +5394,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) err = ext4_load_and_init_journal(sb, es, ctx); if (err) goto failed_mount3a; + if (bdev_read_only(sb->s_bdev)) + needs_recovery = 0; } else if (test_opt(sb, NOLOAD) && !sb_rdonly(sb) && ext4_has_feature_journal_needs_recovery(sb)) { ext4_msg(sb, KERN_ERR, "required journal recovery " @@ -5424,6 +5426,11 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) "data=, fs mounted w/o journal"); goto failed_mount3a; } + if (test_opt(sb, DATA_ERR_ABORT)) { + ext4_msg(sb, KERN_ERR, + "can't mount with data_err=abort, fs mounted w/o journal"); + goto failed_mount3a; + } sbi->s_def_mount_opt &= ~EXT4_MOUNT_JOURNAL_CHECKSUM; clear_opt(sb, JOURNAL_CHECKSUM); clear_opt(sb, DATA_FLAGS); @@ -6774,6 +6781,7 @@ static int ext4_reconfigure(struct fs_context *fc) { struct super_block *sb = fc->root->d_sb; int ret; + bool old_ro = sb_rdonly(sb); fc->s_fs_info = EXT4_SB(sb); @@ -6785,9 +6793,9 @@ static int ext4_reconfigure(struct fs_context *fc) if (ret < 0) return ret; - ext4_msg(sb, KERN_INFO, "re-mounted %pU %s. Quota mode: %s.", - &sb->s_uuid, sb_rdonly(sb) ? "ro" : "r/w", - ext4_quota_mode(sb)); + ext4_msg(sb, KERN_INFO, "re-mounted %pU%s.", + &sb->s_uuid, + (old_ro != sb_rdonly(sb)) ? (sb_rdonly(sb) ? " ro" : " r/w") : ""); return 0; } diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index ca22aa9e04b4b..307081c994374 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -251,6 +251,10 @@ check_xattrs(struct inode *inode, struct buffer_head *bh, err_str = "invalid ea_ino"; goto errout; } + if (ea_ino && !size) { + err_str = "invalid size in ea xattr"; + goto errout; + } if (size > EXT4_XATTR_SIZE_MAX) { err_str = "e_value size too large"; goto errout; @@ -1036,7 +1040,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, int ref_change) { struct ext4_iloc iloc; - s64 ref_count; + u64 ref_count; int ret; inode_lock_nested(ea_inode, I_MUTEX_XATTR); @@ -1046,13 +1050,17 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, goto out; ref_count = ext4_xattr_inode_get_ref(ea_inode); + if ((ref_count == 0 && ref_change < 0) || (ref_count == U64_MAX && ref_change > 0)) { + ext4_error_inode(ea_inode, __func__, __LINE__, 0, + "EA inode %lu ref wraparound: ref_count=%lld ref_change=%d", + ea_inode->i_ino, ref_count, ref_change); + ret = -EFSCORRUPTED; + goto out; + } ref_count += ref_change; ext4_xattr_inode_set_ref(ea_inode, ref_count); if (ref_change > 0) { - WARN_ONCE(ref_count <= 0, "EA inode %lu ref_count=%lld", - ea_inode->i_ino, ref_count); - if (ref_count == 1) { WARN_ONCE(ea_inode->i_nlink, "EA inode %lu i_nlink=%u", ea_inode->i_ino, ea_inode->i_nlink); @@ -1061,9 +1069,6 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, ext4_orphan_del(handle, ea_inode); } } else { - WARN_ONCE(ref_count < 0, "EA inode %lu ref_count=%lld", - ea_inode->i_ino, ref_count); - if (ref_count == 0) { WARN_ONCE(ea_inode->i_nlink != 1, "EA inode %lu i_nlink=%u", @@ -1547,7 +1552,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, WARN_ON_ONCE(ext4_handle_valid(journal_current_handle()) && !(current->flags & PF_MEMALLOC_NOFS)); - ea_data = kvmalloc(value_len, GFP_KERNEL); + ea_data = kvmalloc(value_len, GFP_NOFS); if (!ea_data) { mb_cache_entry_put(ea_inode_cache, ce); return NULL; diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile index 8a7322d229e4d..c3a169950e25a 100644 --- a/fs/f2fs/Makefile +++ b/fs/f2fs/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_F2FS_FS) += f2fs.o f2fs-y := dir.o file.o inode.o namei.o hash.o super.o inline.o f2fs-y += checkpoint.o gc.o data.o node.o segment.o recovery.o -f2fs-y += shrinker.o extent_cache.o sysfs.o +f2fs-y += shrinker.o extent_cache.o sysfs.o lock.o f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 62046f0fb3514..0deccad195784 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -29,6 +29,16 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(f2fs_write_checkpoint); static struct kmem_cache *ino_entry_slab; struct kmem_cache *f2fs_inode_entry_slab; +static inline void f2fs_lock_all(struct f2fs_sb_info *sbi) +{ + f2fs_down_write(&sbi->cp_rwsem); +} + +static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi) +{ + f2fs_up_write(&sbi->cp_rwsem); +} + void _trace_android_rvh_f2fs_down_read(wait_queue_head_t *read_waiters, struct rw_semaphore *rwsem, bool *skip) { @@ -401,6 +411,7 @@ static int f2fs_write_meta_pages(struct address_space *mapping, struct writeback_control *wbc) { struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + struct f2fs_lock_context lc; long diff, written; if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) @@ -413,13 +424,13 @@ static int f2fs_write_meta_pages(struct address_space *mapping, goto skip_write; /* if locked failed, cp will flush dirty pages instead */ - if (!f2fs_down_write_trylock(&sbi->cp_global_sem)) + if (!f2fs_down_write_trylock_trace(&sbi->cp_global_sem, &lc)) goto skip_write; trace_f2fs_writepages(mapping->host, wbc, META); diff = nr_pages_to_write(sbi, META, wbc); written = f2fs_sync_meta_pages(sbi, META, wbc->nr_to_write, FS_META_IO); - f2fs_up_write(&sbi->cp_global_sem); + f2fs_up_write_trace(&sbi->cp_global_sem, &lc); wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff); return 0; @@ -1247,7 +1258,7 @@ static int block_operations(struct f2fs_sb_info *sbi) retry_flush_quotas: f2fs_lock_all(sbi); if (__need_flush_quota(sbi)) { - int locked; + bool need_lock = sbi->umount_lock_holder != current; if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) { set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH); @@ -1256,11 +1267,13 @@ static int block_operations(struct f2fs_sb_info *sbi) } f2fs_unlock_all(sbi); - /* only failed during mount/umount/freeze/quotactl */ - locked = down_read_trylock(&sbi->sb->s_umount); - f2fs_quota_sync(sbi->sb, -1); - if (locked) + /* don't grab s_umount lock during mount/umount/remount/freeze/quotactl */ + if (!need_lock) { + f2fs_do_quota_sync(sbi->sb, -1); + } else if (down_read_trylock(&sbi->sb->s_umount)) { + f2fs_do_quota_sync(sbi->sb, -1); up_read(&sbi->sb->s_umount); + } cond_resched(); goto retry_flush_quotas; } @@ -1639,6 +1652,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) { struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_lock_context lc; unsigned long long ckpt_ver; int err = 0; @@ -1651,7 +1665,7 @@ int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) f2fs_warn(sbi, "Start checkpoint disabled!"); } if (cpc->reason != CP_RESIZE) - f2fs_down_write(&sbi->cp_global_sem); + f2fs_down_write_trace(&sbi->cp_global_sem, &lc); if (!is_sbi_flag_set(sbi, SBI_IS_DIRTY) && ((cpc->reason & CP_FASTBOOT) || (cpc->reason & CP_SYNC) || @@ -1733,7 +1747,7 @@ int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish checkpoint"); out: if (cpc->reason != CP_RESIZE) - f2fs_up_write(&sbi->cp_global_sem); + f2fs_up_write_trace(&sbi->cp_global_sem, &lc); return err; } @@ -1779,11 +1793,12 @@ void f2fs_destroy_checkpoint_caches(void) static int __write_checkpoint_sync(struct f2fs_sb_info *sbi) { struct cp_control cpc = { .reason = CP_SYNC, }; + struct f2fs_lock_context lc; int err; - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &lc); err = f2fs_write_checkpoint(sbi, &cpc); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &lc); return err; } @@ -1868,12 +1883,14 @@ int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi) struct cp_control cpc; cpc.reason = __get_cp_reason(sbi); - if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC) { + if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC || + sbi->umount_lock_holder == current) { + struct f2fs_lock_context lc; int ret; - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &lc); ret = f2fs_write_checkpoint(sbi, &cpc); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &lc); return ret; } @@ -1930,6 +1947,8 @@ int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi) } set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio); + set_user_nice(cprc->f2fs_issue_ckpt, + PRIO_TO_NICE(sbi->critical_task_priority)); return 0; } diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 39fb7c5d2e9c1..0d5c51545327b 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -23,20 +23,18 @@ static struct kmem_cache *cic_entry_slab; static struct kmem_cache *dic_entry_slab; -static void *page_array_alloc(struct inode *inode, int nr) +static void *page_array_alloc(struct f2fs_sb_info *sbi, int nr) { - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); unsigned int size = sizeof(struct page *) * nr; if (likely(size <= sbi->page_array_slab_size)) return f2fs_kmem_cache_alloc(sbi->page_array_slab, - GFP_F2FS_ZERO, false, F2FS_I_SB(inode)); + GFP_F2FS_ZERO, false, sbi); return f2fs_kzalloc(sbi, size, GFP_NOFS); } -static void page_array_free(struct inode *inode, void *pages, int nr) +static void page_array_free(struct f2fs_sb_info *sbi, void *pages, int nr) { - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); unsigned int size = sizeof(struct page *) * nr; if (!pages) @@ -147,13 +145,13 @@ int f2fs_init_compress_ctx(struct compress_ctx *cc) if (cc->rpages) return 0; - cc->rpages = page_array_alloc(cc->inode, cc->cluster_size); + cc->rpages = page_array_alloc(F2FS_I_SB(cc->inode), cc->cluster_size); return cc->rpages ? 0 : -ENOMEM; } void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse) { - page_array_free(cc->inode, cc->rpages, cc->cluster_size); + page_array_free(F2FS_I_SB(cc->inode), cc->rpages, cc->cluster_size); cc->rpages = NULL; cc->nr_rpages = 0; cc->nr_cpages = 0; @@ -178,8 +176,7 @@ void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio) #ifdef CONFIG_F2FS_FS_LZO static int lzo_init_compress_ctx(struct compress_ctx *cc) { - cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), - LZO1X_MEM_COMPRESS, GFP_NOFS); + cc->private = f2fs_vmalloc(LZO1X_MEM_COMPRESS); if (!cc->private) return -ENOMEM; @@ -189,7 +186,7 @@ static int lzo_init_compress_ctx(struct compress_ctx *cc) static void lzo_destroy_compress_ctx(struct compress_ctx *cc) { - kvfree(cc->private); + vfree(cc->private); cc->private = NULL; } @@ -214,13 +211,13 @@ static int lzo_decompress_pages(struct decompress_io_ctx *dic) ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, dic->rbuf, &dic->rlen); if (ret != LZO_E_OK) { - f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + f2fs_err_ratelimited(dic->sbi, "lzo decompress failed, ret:%d", ret); return -EIO; } if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { - f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + f2fs_err_ratelimited(dic->sbi, "lzo invalid rlen:%zu, expected:%lu", dic->rlen, PAGE_SIZE << dic->log_cluster_size); return -EIO; @@ -246,7 +243,7 @@ static int lz4_init_compress_ctx(struct compress_ctx *cc) size = LZ4HC_MEM_COMPRESS; #endif - cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), size, GFP_NOFS); + cc->private = f2fs_vmalloc(size); if (!cc->private) return -ENOMEM; @@ -261,7 +258,7 @@ static int lz4_init_compress_ctx(struct compress_ctx *cc) static void lz4_destroy_compress_ctx(struct compress_ctx *cc) { - kvfree(cc->private); + vfree(cc->private); cc->private = NULL; } @@ -294,13 +291,13 @@ static int lz4_decompress_pages(struct decompress_io_ctx *dic) ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, dic->clen, dic->rlen); if (ret < 0) { - f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + f2fs_err_ratelimited(dic->sbi, "lz4 decompress failed, ret:%d", ret); return -EIO; } if (ret != PAGE_SIZE << dic->log_cluster_size) { - f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + f2fs_err_ratelimited(dic->sbi, "lz4 invalid ret:%d, expected:%lu", ret, PAGE_SIZE << dic->log_cluster_size); return -EIO; @@ -342,8 +339,7 @@ static int zstd_init_compress_ctx(struct compress_ctx *cc) params = zstd_get_params(level, cc->rlen); workspace_size = zstd_cstream_workspace_bound(¶ms.cParams); - workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), - workspace_size, GFP_NOFS); + workspace = f2fs_vmalloc(workspace_size); if (!workspace) return -ENOMEM; @@ -351,7 +347,7 @@ static int zstd_init_compress_ctx(struct compress_ctx *cc) if (!stream) { f2fs_err_ratelimited(F2FS_I_SB(cc->inode), "%s zstd_init_cstream failed", __func__); - kvfree(workspace); + vfree(workspace); return -EIO; } @@ -364,7 +360,7 @@ static int zstd_init_compress_ctx(struct compress_ctx *cc) static void zstd_destroy_compress_ctx(struct compress_ctx *cc) { - kvfree(cc->private); + vfree(cc->private); cc->private = NULL; cc->private2 = NULL; } @@ -423,16 +419,15 @@ static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) workspace_size = zstd_dstream_workspace_bound(max_window_size); - workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode), - workspace_size, GFP_NOFS); + workspace = f2fs_vmalloc(workspace_size); if (!workspace) return -ENOMEM; stream = zstd_init_dstream(max_window_size, workspace, workspace_size); if (!stream) { - f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + f2fs_err_ratelimited(dic->sbi, "%s zstd_init_dstream failed", __func__); - kvfree(workspace); + vfree(workspace); return -EIO; } @@ -444,7 +439,7 @@ static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) { - kvfree(dic->private); + vfree(dic->private); dic->private = NULL; dic->private2 = NULL; } @@ -466,14 +461,14 @@ static int zstd_decompress_pages(struct decompress_io_ctx *dic) ret = zstd_decompress_stream(stream, &outbuf, &inbuf); if (zstd_is_error(ret)) { - f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + f2fs_err_ratelimited(dic->sbi, "%s zstd_decompress_stream failed, ret: %d", __func__, zstd_get_error_code(ret)); return -EIO; } if (dic->rlen != outbuf.pos) { - f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + f2fs_err_ratelimited(dic->sbi, "%s ZSTD invalid rlen:%zu, expected:%lu", __func__, dic->rlen, PAGE_SIZE << dic->log_cluster_size); @@ -619,6 +614,7 @@ static void *f2fs_vmap(struct page **pages, unsigned int count) static int f2fs_compress_pages(struct compress_ctx *cc) { + struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); struct f2fs_inode_info *fi = F2FS_I(cc->inode); const struct f2fs_compress_ops *cops = f2fs_cops[fi->i_compress_algorithm]; @@ -639,7 +635,7 @@ static int f2fs_compress_pages(struct compress_ctx *cc) cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); cc->valid_nr_cpages = cc->nr_cpages; - cc->cpages = page_array_alloc(cc->inode, cc->nr_cpages); + cc->cpages = page_array_alloc(sbi, cc->nr_cpages); if (!cc->cpages) { ret = -ENOMEM; goto destroy_compress_ctx; @@ -714,7 +710,7 @@ static int f2fs_compress_pages(struct compress_ctx *cc) if (cc->cpages[i]) f2fs_compress_free_page(cc->cpages[i]); } - page_array_free(cc->inode, cc->cpages, cc->nr_cpages); + page_array_free(sbi, cc->cpages, cc->nr_cpages); cc->cpages = NULL; destroy_compress_ctx: if (cops->destroy_compress_ctx) @@ -732,7 +728,7 @@ static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) { - struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); + struct f2fs_sb_info *sbi = dic->sbi; struct f2fs_inode_info *fi = F2FS_I(dic->inode); const struct f2fs_compress_ops *cops = f2fs_cops[fi->i_compress_algorithm]; @@ -805,7 +801,7 @@ void f2fs_end_read_compressed_page(struct page *page, bool failed, { struct decompress_io_ctx *dic = (struct decompress_io_ctx *)page_private(page); - struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); + struct f2fs_sb_info *sbi = dic->sbi; dec_page_count(sbi, F2FS_RD_DATA); @@ -1243,7 +1239,7 @@ int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) for (i = cluster_size - 1; i >= 0; i--) { struct folio *folio = page_folio(rpages[i]); - loff_t start = folio->index << PAGE_SHIFT; + loff_t start = (loff_t)folio->index << PAGE_SHIFT; if (from <= start) { folio_zero_segment(folio, 0, folio_size(folio)); @@ -1286,6 +1282,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, struct dnode_of_data dn; struct node_info ni; struct compress_io_ctx *cic; + struct f2fs_lock_context lc; pgoff_t start_idx = start_idx_of_cluster(cc); unsigned int last_index = cc->cluster_size - 1; loff_t psize; @@ -1304,8 +1301,8 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, * checkpoint. This can only happen to quota writes which can cause * the below discard race condition. */ - f2fs_down_read(&sbi->node_write); - } else if (!f2fs_trylock_op(sbi)) { + f2fs_down_read_trace(&sbi->node_write, &lc); + } else if (!f2fs_trylock_op(sbi, &lc)) { goto out_free; } @@ -1337,7 +1334,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; cic->inode = inode; atomic_set(&cic->pending_pages, cc->valid_nr_cpages); - cic->rpages = page_array_alloc(cc->inode, cc->cluster_size); + cic->rpages = page_array_alloc(sbi, cc->cluster_size); if (!cic->rpages) goto out_put_cic; @@ -1429,9 +1426,9 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, f2fs_put_dnode(&dn); if (quota_inode) - f2fs_up_read(&sbi->node_write); + f2fs_up_read_trace(&sbi->node_write, &lc); else - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); spin_lock(&fi->i_size_lock); if (fi->last_disk_size < psize) @@ -1439,13 +1436,13 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, spin_unlock(&fi->i_size_lock); f2fs_put_rpages(cc); - page_array_free(cc->inode, cc->cpages, cc->nr_cpages); + page_array_free(sbi, cc->cpages, cc->nr_cpages); cc->cpages = NULL; f2fs_destroy_compress_ctx(cc, false); return 0; out_destroy_crypt: - page_array_free(cc->inode, cic->rpages, cc->cluster_size); + page_array_free(sbi, cic->rpages, cc->cluster_size); for (--i; i >= 0; i--) { if (!cc->cpages[i]) @@ -1458,15 +1455,15 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, f2fs_put_dnode(&dn); out_unlock_op: if (quota_inode) - f2fs_up_read(&sbi->node_write); + f2fs_up_read_trace(&sbi->node_write, &lc); else - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); out_free: for (i = 0; i < cc->valid_nr_cpages; i++) { f2fs_compress_free_page(cc->cpages[i]); cc->cpages[i] = NULL; } - page_array_free(cc->inode, cc->cpages, cc->nr_cpages); + page_array_free(sbi, cc->cpages, cc->nr_cpages); cc->cpages = NULL; return -EAGAIN; } @@ -1496,7 +1493,7 @@ void f2fs_compress_write_end_io(struct bio *bio, struct page *page) end_page_writeback(cic->rpages[i]); } - page_array_free(cic->inode, cic->rpages, cic->nr_rpages); + page_array_free(sbi, cic->rpages, cic->nr_rpages); kmem_cache_free(cic_entry_slab, cic); } @@ -1507,6 +1504,7 @@ static int f2fs_write_raw_pages(struct compress_ctx *cc, { struct address_space *mapping = cc->inode->i_mapping; struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + struct f2fs_lock_context lc; int submitted, compr_blocks, i; int ret = 0; @@ -1525,7 +1523,7 @@ static int f2fs_write_raw_pages(struct compress_ctx *cc, /* overwrite compressed cluster w/ normal cluster */ if (compr_blocks > 0) - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); for (i = 0; i < cc->cluster_size; i++) { if (!cc->rpages[i]) @@ -1580,7 +1578,7 @@ static int f2fs_write_raw_pages(struct compress_ctx *cc, out: if (compr_blocks > 0) - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_balance_fs(sbi, true); return ret; @@ -1629,14 +1627,13 @@ static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi, static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, bool pre_alloc) { - const struct f2fs_compress_ops *cops = - f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm]; + const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; int i; - if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc)) + if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) return 0; - dic->tpages = page_array_alloc(dic->inode, dic->cluster_size); + dic->tpages = page_array_alloc(dic->sbi, dic->cluster_size); if (!dic->tpages) return -ENOMEM; @@ -1666,10 +1663,9 @@ static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, bool bypass_destroy_callback, bool pre_alloc) { - const struct f2fs_compress_ops *cops = - f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm]; + const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; - if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc)) + if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) return; if (!bypass_destroy_callback && cops->destroy_decompress_ctx) @@ -1696,7 +1692,7 @@ struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) if (!dic) return ERR_PTR(-ENOMEM); - dic->rpages = page_array_alloc(cc->inode, cc->cluster_size); + dic->rpages = page_array_alloc(sbi, cc->cluster_size); if (!dic->rpages) { kmem_cache_free(dic_entry_slab, dic); return ERR_PTR(-ENOMEM); @@ -1704,6 +1700,8 @@ struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; dic->inode = cc->inode; + dic->sbi = sbi; + dic->compress_algorithm = F2FS_I(cc->inode)->i_compress_algorithm; atomic_set(&dic->remaining_pages, cc->nr_cpages); dic->cluster_idx = cc->cluster_idx; dic->cluster_size = cc->cluster_size; @@ -1717,7 +1715,7 @@ struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) dic->rpages[i] = cc->rpages[i]; dic->nr_rpages = cc->cluster_size; - dic->cpages = page_array_alloc(dic->inode, dic->nr_cpages); + dic->cpages = page_array_alloc(sbi, dic->nr_cpages); if (!dic->cpages) { ret = -ENOMEM; goto out_free; @@ -1747,6 +1745,8 @@ static void f2fs_free_dic(struct decompress_io_ctx *dic, bool bypass_destroy_callback) { int i; + /* use sbi in dic to avoid UFA of dic->inode*/ + struct f2fs_sb_info *sbi = dic->sbi; f2fs_release_decomp_mem(dic, bypass_destroy_callback, true); @@ -1758,7 +1758,7 @@ static void f2fs_free_dic(struct decompress_io_ctx *dic, continue; f2fs_compress_free_page(dic->tpages[i]); } - page_array_free(dic->inode, dic->tpages, dic->cluster_size); + page_array_free(sbi, dic->tpages, dic->cluster_size); } if (dic->cpages) { @@ -1767,10 +1767,10 @@ static void f2fs_free_dic(struct decompress_io_ctx *dic, continue; f2fs_compress_free_page(dic->cpages[i]); } - page_array_free(dic->inode, dic->cpages, dic->nr_cpages); + page_array_free(sbi, dic->cpages, dic->nr_cpages); } - page_array_free(dic->inode, dic->rpages, dic->nr_rpages); + page_array_free(sbi, dic->rpages, dic->nr_rpages); kmem_cache_free(dic_entry_slab, dic); } @@ -1789,8 +1789,7 @@ static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task) f2fs_free_dic(dic, false); } else { INIT_WORK(&dic->free_work, f2fs_late_free_dic); - queue_work(F2FS_I_SB(dic->inode)->post_read_wq, - &dic->free_work); + queue_work(dic->sbi->post_read_wq, &dic->free_work); } } } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 6cdc6985c427e..534f97ba35d7b 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -55,8 +55,8 @@ bool f2fs_is_cp_guaranteed(struct page *page) struct inode *inode; struct f2fs_sb_info *sbi; - if (!mapping) - return false; + if (fscrypt_is_bounce_page(page)) + return page_private_gcing(fscrypt_pagecache_page(page)); inode = mapping->host; sbi = F2FS_I_SB(inode); @@ -284,7 +284,7 @@ static void f2fs_read_end_io(struct bio *bio) { struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio)); struct bio_post_read_ctx *ctx; - bool intask = in_task(); + bool intask = in_task() && !irqs_disabled(); iostat_update_and_unbind_ctx(bio); ctx = bio->bi_private; @@ -599,7 +599,8 @@ int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi) for (j = HOT; j < n; j++) { struct f2fs_bio_info *io = &sbi->write_io[i][j]; - init_f2fs_rwsem(&io->io_rwsem); + init_f2fs_rwsem_trace(&io->io_rwsem, sbi, + LOCK_NAME_IO_RWSEM); io->sbi = sbi; io->bio = NULL; io->last_block_in_bio = 0; @@ -623,8 +624,9 @@ static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, { enum page_type btype = PAGE_TYPE_OF_BIO(type); struct f2fs_bio_info *io = sbi->write_io[btype] + temp; + struct f2fs_lock_context lc; - f2fs_down_write(&io->io_rwsem); + f2fs_down_write_trace(&io->io_rwsem, &lc); if (!io->bio) goto unlock_out; @@ -638,7 +640,7 @@ static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, } __submit_merged_bio(io); unlock_out: - f2fs_up_write(&io->io_rwsem); + f2fs_up_write_trace(&io->io_rwsem, &lc); } static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, @@ -652,10 +654,11 @@ static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, if (!force) { enum page_type btype = PAGE_TYPE_OF_BIO(type); struct f2fs_bio_info *io = sbi->write_io[btype] + temp; + struct f2fs_lock_context lc; - f2fs_down_read(&io->io_rwsem); + f2fs_down_read_trace(&io->io_rwsem, &lc); ret = __has_merged_page(io->bio, inode, page, ino); - f2fs_up_read(&io->io_rwsem); + f2fs_up_read_trace(&io->io_rwsem, &lc); } if (ret) __f2fs_submit_merged_write(sbi, type, temp); @@ -949,11 +952,12 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio) enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; struct page *bio_page; + struct f2fs_lock_context lc; enum count_type type; f2fs_bug_on(sbi, is_read_io(fio->op)); - f2fs_down_write(&io->io_rwsem); + f2fs_down_write_trace(&io->io_rwsem, &lc); next: #ifdef CONFIG_BLK_DEV_ZONED if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { @@ -1034,7 +1038,7 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio) if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || !f2fs_is_checkpoint_ready(sbi)) __submit_merged_bio(io); - f2fs_up_write(&io->io_rwsem); + f2fs_up_write_trace(&io->io_rwsem, &lc); } static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, @@ -1426,32 +1430,37 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) return 0; } -static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) +static void f2fs_map_lock(struct f2fs_sb_info *sbi, + struct f2fs_lock_context *lc, + int flag) { if (flag == F2FS_GET_BLOCK_PRE_AIO) - f2fs_down_read(&sbi->node_change); + f2fs_down_read_trace(&sbi->node_change, lc); else - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, lc); } -static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) +static void f2fs_map_unlock(struct f2fs_sb_info *sbi, + struct f2fs_lock_context *lc, + int flag) { if (flag == F2FS_GET_BLOCK_PRE_AIO) - f2fs_up_read(&sbi->node_change); + f2fs_up_read_trace(&sbi->node_change, lc); else - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, lc); } int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) { struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct f2fs_lock_context lc; int err = 0; - f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); + f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); if (!f2fs_lookup_read_extent_cache_block(dn->inode, index, &dn->data_blkaddr)) err = f2fs_reserve_block(dn, index); - f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); + f2fs_map_unlock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); return err; } @@ -1505,8 +1514,8 @@ static bool f2fs_map_blocks_cached(struct inode *inode, struct f2fs_dev_info *dev = &sbi->devs[bidx]; map->m_bdev = dev->bdev; - map->m_pblk -= dev->start_blk; map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); + map->m_pblk -= dev->start_blk; } else { map->m_bdev = inode->i_sb->s_bdev; } @@ -1542,6 +1551,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) unsigned int maxblocks = map->m_len; struct dnode_of_data dn; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE; pgoff_t pgofs, end_offset, end; int err = 0, ofs = 1; @@ -1570,8 +1580,11 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) end = pgofs + maxblocks; next_dnode: - if (map->m_may_create) - f2fs_map_lock(sbi, flag); + if (map->m_may_create) { + if (f2fs_lfs_mode(sbi)) + f2fs_balance_fs(sbi, true); + f2fs_map_lock(sbi, &lc, flag); + } /* When reading holes, we need its node page */ set_new_dnode(&dn, inode, NULL, NULL, 0); @@ -1739,7 +1752,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) f2fs_put_dnode(&dn); if (map->m_may_create) { - f2fs_map_unlock(sbi, flag); + f2fs_map_unlock(sbi, &lc, flag); f2fs_balance_fs(sbi, dn.node_changed); } goto next_dnode; @@ -1775,17 +1788,18 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) if (map->m_flags & F2FS_MAP_MAPPED) { unsigned int ofs = start_pgofs - map->m_lblk; - f2fs_update_read_extent_cache_range(&dn, - start_pgofs, map->m_pblk + ofs, - map->m_len - ofs); + if (map->m_len > ofs) + f2fs_update_read_extent_cache_range(&dn, + start_pgofs, map->m_pblk + ofs, + map->m_len - ofs); } if (map->m_next_extent) - *map->m_next_extent = pgofs + 1; + *map->m_next_extent = is_hole ? pgofs + 1 : pgofs; } f2fs_put_dnode(&dn); unlock_out: if (map->m_may_create) { - f2fs_map_unlock(sbi, flag); + f2fs_map_unlock(sbi, &lc, flag); f2fs_balance_fs(sbi, dn.node_changed); } out: @@ -2657,6 +2671,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) struct inode *inode = folio->mapping->host; struct dnode_of_data dn; struct node_info ni; + struct f2fs_lock_context lc; bool ipu_force = false; bool atomic_commit; int err = 0; @@ -2682,7 +2697,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) } /* Deadlock due to between page->lock and f2fs_lock_op */ - if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) + if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi, &lc)) return -EAGAIN; err = f2fs_get_dnode_of_data(&dn, folio->index, LOOKUP_NODE); @@ -2723,7 +2738,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) folio_start_writeback(folio); f2fs_put_dnode(&dn); if (fio->need_lock == LOCK_REQ) - f2fs_unlock_op(fio->sbi); + f2fs_unlock_op(fio->sbi, &lc); err = f2fs_inplace_write_data(fio); if (err) { if (fscrypt_inode_uses_fs_layer_crypto(inode)) @@ -2737,7 +2752,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) } if (fio->need_lock == LOCK_RETRY) { - if (!f2fs_trylock_op(fio->sbi)) { + if (!f2fs_trylock_op(fio->sbi, &lc)) { err = -EAGAIN; goto out_writepage; } @@ -2769,7 +2784,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) f2fs_put_dnode(&dn); out: if (fio->need_lock == LOCK_REQ) - f2fs_unlock_op(fio->sbi); + f2fs_unlock_op(fio->sbi, &lc); return err; } @@ -2850,19 +2865,21 @@ int f2fs_write_single_data_page(struct folio *folio, int *submitted, write: /* Dentry/quota blocks are controlled by checkpoint */ if (S_ISDIR(inode->i_mode) || quota_inode) { + struct f2fs_lock_context lc; + /* * We need to wait for node_write to avoid block allocation during * checkpoint. This can only happen to quota writes which can cause * the below discard race condition. */ if (quota_inode) - f2fs_down_read(&sbi->node_write); + f2fs_down_read_trace(&sbi->node_write, &lc); fio.need_lock = LOCK_DONE; err = f2fs_do_write_data_page(&fio); if (quota_inode) - f2fs_up_read(&sbi->node_write); + f2fs_up_read_trace(&sbi->node_write, &lc); goto done; } @@ -3282,6 +3299,16 @@ static inline bool __should_serialize_io(struct inode *inode, return false; } +static inline void update_skipped_write(struct f2fs_sb_info *sbi, + struct writeback_control *wbc) +{ + long skipped = wbc->pages_skipped; + + if (is_sbi_flag_set(sbi, SBI_ENABLE_CHECKPOINT) && skipped && + wbc->sync_mode == WB_SYNC_ALL) + atomic_add(skipped, &sbi->nr_pages[F2FS_SKIPPED_WRITE]); +} + static int __f2fs_write_data_pages(struct address_space *mapping, struct writeback_control *wbc, enum iostat_type io_type) @@ -3346,10 +3373,19 @@ static int __f2fs_write_data_pages(struct address_space *mapping, */ f2fs_remove_dirty_inode(inode); + + /* + * f2fs_write_cache_pages() has retry logic for EAGAIN case which is + * common when racing w/ checkpoint, so only update skipped write + * when ret is non-zero. + */ + if (ret) + update_skipped_write(sbi, wbc); return ret; skip_write: wbc->pages_skipped += get_dirty_pages(inode); + update_skipped_write(sbi, wbc); trace_f2fs_writepages(mapping->host, wbc, DATA); return 0; } @@ -3392,6 +3428,7 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, pgoff_t index = folio->index; struct dnode_of_data dn; struct page *ipage; + struct f2fs_lock_context lc; bool locked = false; int flag = F2FS_GET_BLOCK_PRE_AIO; int err = 0; @@ -3407,10 +3444,10 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, if (f2fs_has_inline_data(inode)) { if (pos + len > MAX_INLINE_DATA(inode)) flag = F2FS_GET_BLOCK_DEFAULT; - f2fs_map_lock(sbi, flag); + f2fs_map_lock(sbi, &lc, flag); locked = true; } else if ((pos & PAGE_MASK) >= i_size_read(inode)) { - f2fs_map_lock(sbi, flag); + f2fs_map_lock(sbi, &lc, flag); locked = true; } @@ -3454,7 +3491,7 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, if (!err && dn.data_blkaddr != NULL_ADDR) goto out; f2fs_put_dnode(&dn); - f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); + f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); locked = true; goto restart; @@ -3468,7 +3505,7 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, f2fs_put_dnode(&dn); unlock_out: if (locked) - f2fs_map_unlock(sbi, flag); + f2fs_map_unlock(sbi, &lc, flag); return err; } @@ -3505,9 +3542,10 @@ static int __reserve_data_block(struct inode *inode, pgoff_t index, struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; struct page *ipage; + struct f2fs_lock_context lc; int err = 0; - f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); + f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); ipage = f2fs_get_node_page(sbi, inode->i_ino); if (IS_ERR(ipage)) { @@ -3525,7 +3563,7 @@ static int __reserve_data_block(struct inode *inode, pgoff_t index, f2fs_put_dnode(&dn); unlock_out: - f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); + f2fs_map_unlock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); return err; } diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index 16c2dfb4f5953..308d92568cb93 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -422,6 +422,7 @@ static const char *s_flag[MAX_SBI_FLAG] = { [SBI_IS_RESIZEFS] = "resizefs", [SBI_IS_FREEZING] = "freezefs", [SBI_IS_WRITABLE] = "writable", + [SBI_ENABLE_CHECKPOINT] = "enable_checkpoint", }; static const char *ipu_mode_names[F2FS_IPU_MAX] = { diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index 11696d72cd8db..dc308b5119f1e 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -16,6 +16,21 @@ #include "xattr.h" #include +static inline bool f2fs_should_fallback_to_linear(struct inode *dir) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + + switch (f2fs_get_lookup_mode(sbi)) { + case LOOKUP_PERF: + return false; + case LOOKUP_COMPAT: + return true; + case LOOKUP_AUTO: + return !sb_no_casefold_compat_fallback(sbi->sb); + } + return false; +} + #if IS_ENABLED(CONFIG_UNICODE) extern struct kmem_cache *f2fs_cf_name_slab; #endif @@ -409,7 +424,7 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, out: #if IS_ENABLED(CONFIG_UNICODE) - if (!sb_no_casefold_compat_fallback(dir->i_sb) && + if (f2fs_should_fallback_to_linear(dir) && IS_CASEFOLDED(dir) && !de && use_hash) { use_hash = false; goto start_find_entry; diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 347b3b6478347..2cefe55a169ac 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -414,7 +414,7 @@ void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage) struct f2fs_extent *i_ext = &F2FS_INODE(ipage)->i_ext; struct extent_tree *et; struct extent_node *en; - struct extent_info ei; + struct extent_info ei = {0}; if (!__may_extent_tree(inode, EX_READ)) { /* drop largest read extent */ @@ -604,7 +604,13 @@ static struct extent_node *__insert_extent_tree(struct f2fs_sb_info *sbi, p = &(*p)->rb_right; leftmost = false; } else { + f2fs_err_ratelimited(sbi, "%s: corrupted extent, type: %d, " + "extent node in rb tree [%u, %u, %u], age [%llu, %llu], " + "extent node to insert [%u, %u, %u], age [%llu, %llu]", + __func__, et->type, en->ei.fofs, en->ei.blk, en->ei.len, en->ei.age, + en->ei.last_blocks, ei->fofs, ei->blk, ei->len, ei->age, ei->last_blocks); f2fs_bug_on(sbi, 1); + return NULL; } } diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index f0932cb5a18a1..ea628fa0f1dab 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -115,6 +115,7 @@ extern const char *f2fs_fault_name[FAULT_MAX]; #define F2FS_MOUNT_GC_MERGE 0x02000000 #define F2FS_MOUNT_COMPRESS_CACHE 0x04000000 #define F2FS_MOUNT_AGE_EXTENT_CACHE 0x08000000 +#define F2FS_MOUNT_RESERVE_NODE 0x80000000 #define F2FS_OPTION(sbi) ((sbi)->mount_opt) #define clear_opt(sbi, option) (F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option) @@ -139,6 +140,17 @@ enum blkzone_allocation_policy { BLKZONE_ALLOC_PRIOR_CONV, /* Prioritize writing to conventional zones */ }; +enum f2fs_lock_name { + LOCK_NAME_NONE, + LOCK_NAME_CP_RWSEM, + LOCK_NAME_NODE_CHANGE, + LOCK_NAME_NODE_WRITE, + LOCK_NAME_GC_LOCK, + LOCK_NAME_CP_GLOBAL, + LOCK_NAME_IO_RWSEM, + LOCK_NAME_MAX, +}; + /* * An implementation of an rwsem that is explicitly unfair to readers. This * prevents priority inversion when a low-priority reader acquires the read lock @@ -147,6 +159,8 @@ enum blkzone_allocation_policy { */ struct f2fs_rwsem { + struct f2fs_sb_info *sbi; + enum f2fs_lock_name name; struct rw_semaphore internal_rwsem; #ifdef CONFIG_F2FS_UNFAIR_RWSEM wait_queue_head_t read_waiters; @@ -156,6 +170,7 @@ struct f2fs_rwsem { struct f2fs_mount_info { unsigned int opt; block_t root_reserved_blocks; /* root reserved blocks */ + block_t root_reserved_nodes; /* root reserved nodes */ kuid_t s_resuid; /* reserved blocks for uid */ kgid_t s_resgid; /* reserved blocks for gid */ int active_logs; /* # of active logs */ @@ -596,6 +611,8 @@ enum { #define DEFAULT_RETRY_IO_COUNT 8 /* maximum retry read IO or flush count */ +#define MAX_FLUSH_RETRY_COUNT 3 /* maximum flush retry count in f2fs_enable_checkpoint() */ + /* congestion wait timeout value, default: 20ms */ #define DEFAULT_IO_TIMEOUT (msecs_to_jiffies(20)) @@ -856,6 +873,7 @@ struct f2fs_inode_info { /* linked in global inode list for cache donation */ struct list_head gdonate_list; pgoff_t donate_start, donate_end; /* inclusive */ + atomic_t open_count; /* # of open files */ struct task_struct *atomic_write_task; /* store atomic write task */ struct extent_tree *extent_tree[NR_EXTENT_CACHES]; @@ -1120,6 +1138,7 @@ enum count_type { F2FS_RD_META, F2FS_DIO_WRITE, F2FS_DIO_READ, + F2FS_SKIPPED_WRITE, /* skip or fail during f2fs_enable_checkpoint() */ NR_COUNT_TYPE, }; @@ -1266,7 +1285,7 @@ struct f2fs_bio_info { #define RDEV(i) (raw_super->devs[i]) struct f2fs_dev_info { struct block_device *bdev; - char path[MAX_PATH_LEN]; + char path[MAX_PATH_LEN + 1]; unsigned int total_segments; block_t start_blk; block_t end_blk; @@ -1304,6 +1323,27 @@ struct atgc_management { unsigned long long age_threshold; /* age threshold */ }; +struct f2fs_time_stat { + unsigned long long total_time; /* total wall clock time */ +#ifdef CONFIG_64BIT + unsigned long long running_time; /* running time */ +#endif +#if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS) + unsigned long long runnable_time; /* runnable(including preempted) time */ +#endif +#ifdef CONFIG_TASK_DELAY_ACCT + unsigned long long io_sleep_time; /* IO sleep time */ +#endif +}; + +struct f2fs_lock_context { + struct f2fs_time_stat ts; + int orig_nice; + int new_nice; + bool lock_trace; + bool need_restore; +}; + struct f2fs_gc_control { unsigned int victim_segno; /* target victim segment number */ int init_gc_type; /* FG_GC or BG_GC */ @@ -1312,6 +1352,7 @@ struct f2fs_gc_control { bool err_gc_skipped; /* return EAGAIN if GC skipped */ bool one_time; /* require one time GC in one migration unit */ unsigned int nr_free_secs; /* # of free sections to do GC */ + struct f2fs_lock_context lc; /* lock context for gc_lock */ }; /* @@ -1335,6 +1376,7 @@ enum { SBI_IS_RESIZEFS, /* resizefs is in process */ SBI_IS_FREEZING, /* freezefs is in process */ SBI_IS_WRITABLE, /* remove ro mountoption transiently */ + SBI_ENABLE_CHECKPOINT, /* indicate it's during f2fs_enable_checkpoint() */ MAX_SBI_FLAG, }; @@ -1423,6 +1465,12 @@ enum { TOTAL_CALL = FOREGROUND, }; +/* a threshold of maximum elapsed time in critical region to print tracepoint */ +#define MAX_LOCK_ELAPSED_TIME 500 + +#define F2FS_DEFAULT_TASK_PRIORITY (DEFAULT_PRIO) +#define F2FS_CRITICAL_TASK_PRIORITY NICE_TO_PRIO(0) + static inline int f2fs_test_bit(unsigned int nr, char *addr); static inline void f2fs_set_bit(unsigned int nr, char *addr); static inline void f2fs_clear_bit(unsigned int nr, char *addr); @@ -1516,6 +1564,7 @@ struct compress_io_ctx { struct decompress_io_ctx { u32 magic; /* magic number to indicate page is compressed */ struct inode *inode; /* inode the context belong to */ + struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */ pgoff_t cluster_idx; /* cluster index number */ unsigned int cluster_size; /* page count in cluster */ unsigned int log_cluster_size; /* log of cluster size */ @@ -1556,6 +1605,7 @@ struct decompress_io_ctx { bool failed; /* IO error occurred before decompression? */ bool need_verity; /* need fs-verity verification after decompression? */ + unsigned char compress_algorithm; /* backup algorithm type */ void *private; /* payload buffer for specified decompression algorithm */ void *private2; /* extra payload buffer */ struct work_struct verity_work; /* work to verify the decompressed pages */ @@ -1670,6 +1720,7 @@ struct f2fs_sb_info { unsigned int nquota_files; /* # of quota sysfile */ struct f2fs_rwsem quota_sem; /* blocking cp for flags */ + struct task_struct *umount_lock_holder; /* s_umount lock holder */ /* # of pages, see count_type */ atomic_t nr_pages[NR_COUNT_TYPE]; @@ -1820,6 +1871,18 @@ struct f2fs_sb_info { /* carve out reserved_blocks from total blocks */ bool carve_out; + /* max elapsed time threshold in critical region that lock covered */ + unsigned long long max_lock_elapsed_time; + + /* enable/disable to adjust task priority in critical region covered by lock */ + unsigned int adjust_lock_priority; + + /* adjust priority for task which is in critical region covered by lock */ + unsigned int lock_duration_priority; + + /* priority for critical task, e.g. f2fs_ckpt, f2fs_gc threads */ + long critical_task_priority; + #ifdef CONFIG_F2FS_FS_COMPRESSION struct kmem_cache *page_array_slab; /* page array entry */ unsigned int page_array_slab_size; /* default page array slab size */ @@ -2175,16 +2238,22 @@ static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f) spin_unlock_irqrestore(&sbi->cp_lock, flags); } -#define init_f2fs_rwsem(sem) \ +#define init_f2fs_rwsem(sem) __init_f2fs_rwsem(sem, NULL, LOCK_NAME_NONE) +#define init_f2fs_rwsem_trace __init_f2fs_rwsem + +#define __init_f2fs_rwsem(sem, sbi, name) \ do { \ static struct lock_class_key __key; \ \ - __init_f2fs_rwsem((sem), #sem, &__key); \ + do_init_f2fs_rwsem((sem), #sem, &__key, sbi, name); \ } while (0) -static inline void __init_f2fs_rwsem(struct f2fs_rwsem *sem, - const char *sem_name, struct lock_class_key *key) +static inline void do_init_f2fs_rwsem(struct f2fs_rwsem *sem, + const char *sem_name, struct lock_class_key *key, + struct f2fs_sb_info *sbi, enum f2fs_lock_name name) { + sem->sbi = sbi; + sem->name = name; __init_rwsem(&sem->internal_rwsem, sem_name, key); #ifdef CONFIG_F2FS_UNFAIR_RWSEM init_waitqueue_head(&sem->read_waiters); @@ -2206,11 +2275,11 @@ void _trace_android_rvh_f2fs_down_read(wait_queue_head_t *read_waiters, static inline void f2fs_down_read(struct f2fs_rwsem *sem) { +#ifdef CONFIG_F2FS_UNFAIR_RWSEM bool skip = false; _trace_android_rvh_f2fs_down_read(&sem->read_waiters, &sem->internal_rwsem, &skip); if (skip) return; -#ifdef CONFIG_F2FS_UNFAIR_RWSEM wait_event(sem->read_waiters, down_read_trylock(&sem->internal_rwsem)); #else down_read(&sem->internal_rwsem); @@ -2260,6 +2329,16 @@ static inline void f2fs_up_write(struct f2fs_rwsem *sem) #endif } +void f2fs_down_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc); +int f2fs_down_read_trylock_trace(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc); +void f2fs_up_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc); +void f2fs_down_write_trace(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc); +int f2fs_down_write_trylock_trace(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc); +void f2fs_up_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc); + static inline void disable_nat_bits(struct f2fs_sb_info *sbi, bool lock) { unsigned long flags; @@ -2290,33 +2369,6 @@ static inline bool enabled_nat_bits(struct f2fs_sb_info *sbi, return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set; } -static inline void f2fs_lock_op(struct f2fs_sb_info *sbi) -{ - f2fs_down_read(&sbi->cp_rwsem); -} - -static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi) -{ - if (time_to_inject(sbi, FAULT_LOCK_OP)) - return 0; - return f2fs_down_read_trylock(&sbi->cp_rwsem); -} - -static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi) -{ - f2fs_up_read(&sbi->cp_rwsem); -} - -static inline void f2fs_lock_all(struct f2fs_sb_info *sbi) -{ - f2fs_down_write(&sbi->cp_rwsem); -} - -static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi) -{ - f2fs_up_write(&sbi->cp_rwsem); -} - static inline int __get_cp_reason(struct f2fs_sb_info *sbi) { int reason = CP_SYNC; @@ -2354,13 +2406,11 @@ static inline bool f2fs_has_xattr_block(unsigned int ofs) return ofs == XATTR_NODE_OFFSET; } -static inline bool __allow_reserved_blocks(struct f2fs_sb_info *sbi, +static inline bool __allow_reserved_root(struct f2fs_sb_info *sbi, struct inode *inode, bool cap) { if (!inode) return true; - if (!test_opt(sbi, RESERVE_ROOT)) - return false; if (IS_NOQUOTA(inode)) return true; if (uid_eq(F2FS_OPTION(sbi).s_resuid, current_fsuid())) @@ -2381,7 +2431,7 @@ static inline unsigned int get_available_block_count(struct f2fs_sb_info *sbi, avail_user_block_count = sbi->user_block_count - sbi->current_reserved_blocks; - if (!__allow_reserved_blocks(sbi, inode, cap)) + if (test_opt(sbi, RESERVE_ROOT) && !__allow_reserved_root(sbi, inode, cap)) avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { @@ -2537,8 +2587,14 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi, blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK; spin_lock(&sbi->stat_lock); - f2fs_bug_on(sbi, sbi->total_valid_block_count < (block_t) count); - sbi->total_valid_block_count -= (block_t)count; + if (unlikely(sbi->total_valid_block_count < count)) { + f2fs_warn(sbi, "Inconsistent total_valid_block_count:%u, ino:%lu, count:%u", + sbi->total_valid_block_count, inode->i_ino, count); + sbi->total_valid_block_count = 0; + set_sbi_flag(sbi, SBI_NEED_FSCK); + } else { + sbi->total_valid_block_count -= count; + } if (sbi->reserved_blocks && sbi->current_reserved_blocks < sbi->reserved_blocks) sbi->current_reserved_blocks = min(sbi->reserved_blocks, @@ -2721,7 +2777,7 @@ static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, struct inode *inode, bool is_inode) { block_t valid_block_count; - unsigned int valid_node_count; + unsigned int valid_node_count, avail_user_node_count; unsigned int avail_user_block_count; int err; @@ -2743,15 +2799,20 @@ static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, spin_lock(&sbi->stat_lock); valid_block_count = sbi->total_valid_block_count + 1; - avail_user_block_count = get_available_block_count(sbi, inode, false); + avail_user_block_count = get_available_block_count(sbi, inode, + test_opt(sbi, RESERVE_NODE)); if (unlikely(valid_block_count > avail_user_block_count)) { spin_unlock(&sbi->stat_lock); goto enospc; } + avail_user_node_count = sbi->total_node_count - F2FS_RESERVED_NODE_NUM; + if (test_opt(sbi, RESERVE_NODE) && + !__allow_reserved_root(sbi, inode, true)) + avail_user_node_count -= F2FS_OPTION(sbi).root_reserved_nodes; valid_node_count = sbi->total_valid_node_count + 1; - if (unlikely(valid_node_count > sbi->total_node_count)) { + if (unlikely(valid_node_count > avail_user_node_count)) { spin_unlock(&sbi->stat_lock); goto enospc; } @@ -3518,6 +3579,11 @@ static inline void *f2fs_kvzalloc(struct f2fs_sb_info *sbi, return f2fs_kvmalloc(sbi, size, flags | __GFP_ZERO); } +static inline void *f2fs_vmalloc(size_t size) +{ + return vmalloc(size); +} + static inline int get_extra_isize(struct inode *inode) { return F2FS_I(inode)->i_extra_isize / sizeof(__le32); @@ -3602,8 +3668,9 @@ int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink); void f2fs_update_inode(struct inode *inode, struct page *node_page); void f2fs_update_inode_page(struct inode *inode); int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc); +void f2fs_remove_donate_inode(struct inode *inode); void f2fs_evict_inode(struct inode *inode); -void f2fs_handle_failed_inode(struct inode *inode); +void f2fs_handle_failed_inode(struct inode *inode, struct f2fs_lock_context *lc); /* * namei.c @@ -3679,7 +3746,7 @@ int f2fs_inode_dirtied(struct inode *inode, bool sync); void f2fs_inode_synced(struct inode *inode); int f2fs_dquot_initialize(struct inode *inode); int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly); -int f2fs_quota_sync(struct super_block *sb, int type); +int f2fs_do_quota_sync(struct super_block *sb, int type); loff_t max_file_blocks(struct inode *inode); void f2fs_quota_off_umount(struct super_block *sb); void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag); @@ -3848,6 +3915,9 @@ static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) /* * checkpoint.c */ +void f2fs_lock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc); +int f2fs_trylock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc); +void f2fs_unlock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc); void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, unsigned char reason); void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi); @@ -4811,6 +4881,47 @@ static inline void f2fs_invalidate_internal_cache(struct f2fs_sb_info *sbi, f2fs_invalidate_compress_pages_range(sbi, blkaddr, len); } +enum f2fs_lookup_mode { + LOOKUP_PERF, + LOOKUP_COMPAT, + LOOKUP_AUTO, +}; + +/* + * For bit-packing in f2fs_mount_info->alloc_mode + */ +#define ALLOC_MODE_BITS 1 +#define LOOKUP_MODE_BITS 2 + +#define ALLOC_MODE_SHIFT 0 +#define LOOKUP_MODE_SHIFT (ALLOC_MODE_SHIFT + ALLOC_MODE_BITS) + +#define ALLOC_MODE_MASK (((1 << ALLOC_MODE_BITS) - 1) << ALLOC_MODE_SHIFT) +#define LOOKUP_MODE_MASK (((1 << LOOKUP_MODE_BITS) - 1) << LOOKUP_MODE_SHIFT) + +static inline int f2fs_get_alloc_mode(struct f2fs_sb_info *sbi) +{ + return (F2FS_OPTION(sbi).alloc_mode & ALLOC_MODE_MASK) >> ALLOC_MODE_SHIFT; +} + +static inline void f2fs_set_alloc_mode(struct f2fs_sb_info *sbi, int mode) +{ + F2FS_OPTION(sbi).alloc_mode &= ~ALLOC_MODE_MASK; + F2FS_OPTION(sbi).alloc_mode |= (mode << ALLOC_MODE_SHIFT); +} + +static inline enum f2fs_lookup_mode f2fs_get_lookup_mode(struct f2fs_sb_info *sbi) +{ + return (F2FS_OPTION(sbi).alloc_mode & LOOKUP_MODE_MASK) >> LOOKUP_MODE_SHIFT; +} + +static inline void f2fs_set_lookup_mode(struct f2fs_sb_info *sbi, + enum f2fs_lookup_mode mode) +{ + F2FS_OPTION(sbi).alloc_mode &= ~LOOKUP_MODE_MASK; + F2FS_OPTION(sbi).alloc_mode |= (mode << LOOKUP_MODE_SHIFT); +} + #define EFSBADCRC EBADMSG /* Bad CRC detected */ #define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f8832212ee37f..6141a5794b4dd 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -618,7 +618,10 @@ static int f2fs_file_open(struct inode *inode, struct file *filp) if (err) return err; - return finish_preallocate_blocks(inode); + err = finish_preallocate_blocks(inode); + if (!err) + atomic_inc(&F2FS_I(inode)->open_count); + return err; } void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count) @@ -748,6 +751,7 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; + struct f2fs_lock_context lc; pgoff_t free_from; int count = 0, err = 0; struct page *ipage; @@ -766,7 +770,7 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) goto free_partial; if (lock) - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); ipage = f2fs_get_node_page(sbi, inode->i_ino); if (IS_ERR(ipage)) { @@ -817,7 +821,7 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) err = f2fs_truncate_inode_blocks(inode, free_from); out: if (lock) - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); free_partial: /* lastly zero out the first data page */ if (!err) @@ -1037,6 +1041,18 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) return -EIO; + err = setattr_prepare(idmap, dentry, attr); + if (err) + return err; + + err = fscrypt_prepare_setattr(dentry, attr); + if (err) + return err; + + err = fsverity_prepare_setattr(dentry, attr); + if (err) + return err; + if (unlikely(IS_IMMUTABLE(inode))) return -EPERM; @@ -1055,18 +1071,6 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, return -EINVAL; } - err = setattr_prepare(idmap, dentry, attr); - if (err) - return err; - - err = fscrypt_prepare_setattr(dentry, attr); - if (err) - return err; - - err = fsverity_prepare_setattr(dentry, attr); - if (err) - return err; - if (is_quota_modification(idmap, inode, attr)) { err = f2fs_dquot_initialize(inode); if (err) @@ -1074,12 +1078,14 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, } if (i_uid_needs_update(idmap, attr, inode) || i_gid_needs_update(idmap, attr, inode)) { - f2fs_lock_op(F2FS_I_SB(inode)); + struct f2fs_lock_context lc; + + f2fs_lock_op(F2FS_I_SB(inode), &lc); err = dquot_transfer(idmap, inode, attr); if (err) { set_sbi_flag(F2FS_I_SB(inode), SBI_QUOTA_NEED_REPAIR); - f2fs_unlock_op(F2FS_I_SB(inode)); + f2fs_unlock_op(F2FS_I_SB(inode), &lc); return err; } /* @@ -1089,7 +1095,7 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, i_uid_update(idmap, attr, inode); i_gid_update(idmap, attr, inode); f2fs_mark_inode_dirty_sync(inode, true); - f2fs_unlock_op(F2FS_I_SB(inode)); + f2fs_unlock_op(F2FS_I_SB(inode), &lc); } if (attr->ia_valid & ATTR_SIZE) { @@ -1171,15 +1177,16 @@ static int fill_zero(struct inode *inode, pgoff_t index, { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct page *page; + struct f2fs_lock_context lc; if (!len) return 0; f2fs_balance_fs(sbi, true); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); page = f2fs_get_new_data_page(inode, NULL, index, false); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (IS_ERR(page)) return PTR_ERR(page); @@ -1260,6 +1267,7 @@ static int f2fs_punch_hole(struct inode *inode, loff_t offset, loff_t len) if (pg_start < pg_end) { loff_t blk_start, blk_end; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; f2fs_balance_fs(sbi, true); @@ -1271,9 +1279,9 @@ static int f2fs_punch_hole(struct inode *inode, loff_t offset, loff_t len) truncate_pagecache_range(inode, blk_start, blk_end - 1); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); ret = f2fs_truncate_hole(inode, pg_start, pg_end); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); @@ -1505,6 +1513,7 @@ static int __exchange_data_block(struct inode *src_inode, static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; pgoff_t nrpages = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); pgoff_t start = offset >> PAGE_SHIFT; pgoff_t end = (offset + len) >> PAGE_SHIFT; @@ -1516,11 +1525,11 @@ static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); filemap_invalidate_lock(inode->i_mapping); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); f2fs_drop_extent_tree(inode); truncate_pagecache(inode, offset); ret = __exchange_data_block(inode, inode, end, start, nrpages - end, true); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); @@ -1663,6 +1672,7 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, for (index = pg_start; index < pg_end;) { struct dnode_of_data dn; + struct f2fs_lock_context lc; unsigned int end_offset; pgoff_t end; @@ -1673,12 +1683,12 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, (loff_t)index << PAGE_SHIFT, ((loff_t)pg_end << PAGE_SHIFT) - 1); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); set_new_dnode(&dn, inode, NULL, NULL, 0); ret = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE); if (ret) { - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); filemap_invalidate_unlock(mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); goto out; @@ -1690,7 +1700,7 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, ret = f2fs_do_zero_range(&dn, index, end); f2fs_put_dnode(&dn); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); filemap_invalidate_unlock(mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); @@ -1771,17 +1781,19 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) truncate_pagecache(inode, offset); while (!ret && idx > pg_start) { + struct f2fs_lock_context lc; + nr = idx - pg_start; if (nr > delta) nr = delta; idx -= nr; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); f2fs_drop_extent_tree(inode); ret = __exchange_data_block(inode, inode, idx, idx + delta, nr, false); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); } filemap_invalidate_unlock(mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); @@ -1861,7 +1873,7 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, if (has_not_enough_free_secs(sbi, 0, sbi->reserved_pin_section)) { - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); stat_inc_gc_call_count(sbi, FOREGROUND); err = f2fs_gc(sbi, &gc_control); if (err && err != -ENODATA) { @@ -1998,6 +2010,9 @@ static long f2fs_fallocate(struct file *file, int mode, static int f2fs_release_file(struct inode *inode, struct file *filp) { + if (atomic_dec_and_test(&F2FS_I(inode)->open_count)) + f2fs_remove_donate_inode(inode); + /* * f2fs_release_file is called at every close calls. So we should * not drop any inmemory pages by close called by other process. @@ -2708,12 +2723,13 @@ static int f2fs_ioc_gc(struct file *filp, unsigned long arg) return ret; if (!sync) { - if (!f2fs_down_write_trylock(&sbi->gc_lock)) { + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, + &gc_control.lc)) { ret = -EBUSY; goto out; } } else { - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); } gc_control.init_gc_type = sync ? FG_GC : BG_GC; @@ -2753,12 +2769,12 @@ static int __f2fs_ioc_gc_range(struct file *filp, struct f2fs_gc_range *range) do_more: if (!range->sync) { - if (!f2fs_down_write_trylock(&sbi->gc_lock)) { + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, &gc_control.lc)) { ret = -EBUSY; goto out; } } else { - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); } gc_control.victim_segno = GET_SEGNO(sbi, range->start); @@ -3031,6 +3047,7 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, struct inode *src = file_inode(file_in); struct inode *dst = file_inode(file_out); struct f2fs_sb_info *sbi = F2FS_I_SB(src); + struct f2fs_lock_context lc; size_t olen = len, dst_max_i_size = 0; size_t dst_osize; int ret; @@ -3126,7 +3143,7 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, goto out_src; } - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); ret = __exchange_data_block(src, dst, F2FS_BYTES_TO_BLK(pos_in), F2FS_BYTES_TO_BLK(pos_out), F2FS_BYTES_TO_BLK(len), false); @@ -3137,7 +3154,7 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, else if (dst_osize != dst->i_size) f2fs_i_size_write(dst, dst_osize); } - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (src != dst) f2fs_up_write(&F2FS_I(dst)->i_gc_rwsem[WRITE]); @@ -3253,7 +3270,7 @@ static int f2fs_ioc_flush_device(struct file *filp, unsigned long arg) end_segno = min(start_segno + range.segments, dev_end_segno); while (start_segno < end_segno) { - if (!f2fs_down_write_trylock(&sbi->gc_lock)) { + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, &gc_control.lc)) { ret = -EBUSY; goto out; } @@ -3310,6 +3327,7 @@ static int f2fs_ioc_setproject(struct inode *inode, __u32 projid) struct f2fs_inode_info *fi = F2FS_I(inode); struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode *ri = NULL; + struct f2fs_lock_context lc; kprojid_t kprojid; int err; @@ -3340,7 +3358,7 @@ static int f2fs_ioc_setproject(struct inode *inode, __u32 projid) if (err) return err; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_transfer_project_quota(inode, kprojid); if (err) goto out_unlock; @@ -3349,7 +3367,7 @@ static int f2fs_ioc_setproject(struct inode *inode, __u32 projid) inode_set_ctime_current(inode); f2fs_mark_inode_dirty_sync(inode, true); out_unlock: - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); return err; } #else @@ -3782,6 +3800,7 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) struct inode *inode = file_inode(filp); struct f2fs_inode_info *fi = F2FS_I(inode); struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; pgoff_t page_idx = 0, last_idx; unsigned int released_blocks = 0; int ret; @@ -3836,12 +3855,12 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) struct dnode_of_data dn; pgoff_t end_offset, count; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); set_new_dnode(&dn, inode, NULL, NULL, 0); ret = f2fs_get_dnode_of_data(&dn, page_idx, LOOKUP_NODE); if (ret) { - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (ret == -ENOENT) { page_idx = f2fs_get_next_page_offset(&dn, page_idx); @@ -3859,7 +3878,7 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) f2fs_put_dnode(&dn); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (ret < 0) break; @@ -4012,14 +4031,15 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) while (page_idx < last_idx) { struct dnode_of_data dn; + struct f2fs_lock_context lc; pgoff_t end_offset, count; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); set_new_dnode(&dn, inode, NULL, NULL, 0); ret = f2fs_get_dnode_of_data(&dn, page_idx, LOOKUP_NODE); if (ret) { - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (ret == -ENOENT) { page_idx = f2fs_get_next_page_offset(&dn, page_idx); @@ -4037,7 +4057,7 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) f2fs_put_dnode(&dn); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (ret < 0) break; @@ -4798,6 +4818,7 @@ static ssize_t f2fs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) struct inode *inode = file_inode(iocb->ki_filp); const loff_t pos = iocb->ki_pos; ssize_t ret; + bool dio; if (!f2fs_is_compress_backend_ready(inode)) return -EOPNOTSUPP; @@ -4806,12 +4827,15 @@ static ssize_t f2fs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) f2fs_trace_rw_file_path(iocb->ki_filp, iocb->ki_pos, iov_iter_count(to), READ); + dio = f2fs_should_use_dio(inode, iocb, to); + /* In LFS mode, if there is inflight dio, wait for its completion */ if (f2fs_lfs_mode(F2FS_I_SB(inode)) && - get_pages(F2FS_I_SB(inode), F2FS_DIO_WRITE)) + get_pages(F2FS_I_SB(inode), F2FS_DIO_WRITE) && + (!f2fs_is_pinned_file(inode) || !dio)) inode_dio_wait(inode); - if (f2fs_should_use_dio(inode, iocb, to)) { + if (dio) { ret = f2fs_dio_read_iter(iocb, to); } else { ret = filemap_read(iocb, to, 0); diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 0c3b379d2e7f7..9f7e1bc781a64 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -41,13 +41,14 @@ static int gc_thread_func(void *data) struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, .should_migrate_blocks = false, - .err_gc_skipped = false }; + .err_gc_skipped = false, + .one_time = false }; wait_ms = gc_th->min_sleep_time; set_freezable(); do { - bool sync_mode, foreground = false; + bool sync_mode, foreground = false, gc_boost = false; wait_event_freezable_timeout(*wq, kthread_should_stop() || @@ -55,8 +56,12 @@ static int gc_thread_func(void *data) gc_th->gc_wake, msecs_to_jiffies(wait_ms)); - if (test_opt(sbi, GC_MERGE) && waitqueue_active(fggc_wq)) + if (test_opt(sbi, GC_MERGE) && waitqueue_active(fggc_wq)) { foreground = true; + gc_control.one_time = false; + } else if (f2fs_sb_has_blkzoned(sbi)) { + gc_control.one_time = true; + } /* give it a try one time */ if (gc_th->gc_wake) @@ -84,8 +89,6 @@ static int gc_thread_func(void *data) continue; } - gc_control.one_time = false; - /* * [GC triggering condition] * 0. GC is not conducted currently. @@ -102,21 +105,22 @@ static int gc_thread_func(void *data) if (sbi->gc_mode == GC_URGENT_HIGH || sbi->gc_mode == GC_URGENT_MID) { wait_ms = gc_th->urgent_sleep_time; - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); goto do_gc; } if (foreground) { - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); goto do_gc; - } else if (!f2fs_down_write_trylock(&sbi->gc_lock)) { + } else if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, + &gc_control.lc)) { stat_other_skip_bggc_count(sbi); goto next; } if (!is_idle(sbi, GC_TIME)) { increase_sleep_time(gc_th, &wait_ms); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &gc_control.lc); stat_io_skip_bggc_count(sbi); goto next; } @@ -125,7 +129,8 @@ static int gc_thread_func(void *data) if (has_enough_free_blocks(sbi, gc_th->no_zoned_gc_percent)) { wait_ms = gc_th->no_gc_sleep_time; - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, + &gc_control.lc); goto next; } if (wait_ms == gc_th->no_gc_sleep_time) @@ -135,7 +140,7 @@ static int gc_thread_func(void *data) if (need_to_boost_gc(sbi)) { decrease_sleep_time(gc_th, &wait_ms); if (f2fs_sb_has_blkzoned(sbi)) - gc_control.one_time = true; + gc_boost = true; } else { increase_sleep_time(gc_th, &wait_ms); } @@ -144,10 +149,10 @@ static int gc_thread_func(void *data) FOREGROUND : BACKGROUND); sync_mode = (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC) || - gc_control.one_time; + gc_boost; /* foreground GC was been triggered via f2fs_balance_fs() */ - if (foreground) + if (foreground && !f2fs_sb_has_blkzoned(sbi)) sync_mode = false; gc_control.init_gc_type = sync_mode ? FG_GC : BG_GC; @@ -230,6 +235,8 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) return err; } + set_user_nice(gc_th->f2fs_gc_task, + PRIO_TO_NICE(sbi->critical_task_priority)); return 0; } @@ -392,14 +399,15 @@ static unsigned int get_cb_cost(struct f2fs_sb_info *sbi, unsigned int segno) } static inline unsigned int get_gc_cost(struct f2fs_sb_info *sbi, - unsigned int segno, struct victim_sel_policy *p) + unsigned int segno, struct victim_sel_policy *p, + unsigned int valid_thresh_ratio) { if (p->alloc_mode == SSR) return get_seg_entry(sbi, segno)->ckpt_valid_blocks; - if (p->one_time_gc && (get_valid_blocks(sbi, segno, true) >= - CAP_BLKS_PER_SEC(sbi) * sbi->gc_thread->valid_thresh_ratio / - 100)) + if (p->one_time_gc && (valid_thresh_ratio < 100) && + (get_valid_blocks(sbi, segno, true) >= + CAP_BLKS_PER_SEC(sbi) * valid_thresh_ratio / 100)) return UINT_MAX; /* alloc_mode == LFS */ @@ -780,6 +788,7 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, unsigned int secno, last_victim; unsigned int last_segment; unsigned int nsearched; + unsigned int valid_thresh_ratio = 100; bool is_atgc; int ret = 0; @@ -789,7 +798,11 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, p.alloc_mode = alloc_mode; p.age = age; p.age_threshold = sbi->am.age_threshold; - p.one_time_gc = one_time; + if (one_time) { + p.one_time_gc = one_time; + if (has_enough_free_secs(sbi, 0, NR_PERSISTENT_LOG)) + valid_thresh_ratio = sbi->gc_thread->valid_thresh_ratio; + } retry: select_policy(sbi, gc_type, type, &p); @@ -902,6 +915,9 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, if (!f2fs_segment_has_free_slot(sbi, segno)) goto next; } + + if (!get_valid_blocks(sbi, segno, true)) + goto next; } if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap)) @@ -915,7 +931,7 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, goto next; } - cost = get_gc_cost(sbi, segno, &p); + cost = get_gc_cost(sbi, segno, &p, valid_thresh_ratio); if (p.min_cost > cost) { p.min_segno = segno; @@ -1028,7 +1044,8 @@ static int check_valid_map(struct f2fs_sb_info *sbi, * ignore that. */ static int gc_node_segment(struct f2fs_sb_info *sbi, - struct f2fs_summary *sum, unsigned int segno, int gc_type) + struct f2fs_summary *sum, unsigned int segno, int gc_type, + struct blk_plug *plug) { struct f2fs_summary *entry; block_t start_addr; @@ -1097,8 +1114,11 @@ static int gc_node_segment(struct f2fs_sb_info *sbi, stat_inc_node_blk_count(sbi, 1, gc_type); } - if (++phase < 3) + if (++phase < 3) { + blk_finish_plug(plug); + blk_start_plug(plug); goto next_step; + } if (fggc) atomic_dec(&sbi->wb_sync_req[NODE]); @@ -1529,7 +1549,7 @@ static int move_data_page(struct inode *inode, block_t bidx, int gc_type, */ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, struct gc_inode_list *gc_list, unsigned int segno, int gc_type, - bool force_migrate) + bool force_migrate, struct blk_plug *plug) { struct super_block *sb = sbi->sb; struct f2fs_summary *entry; @@ -1697,8 +1717,11 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, } } - if (++phase < 5) + if (++phase < 5) { + blk_finish_plug(plug); + blk_start_plug(plug); goto next_step; + } return submitted; } @@ -1797,6 +1820,13 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, GET_SUM_BLOCK(sbi, segno)); f2fs_put_page(sum_page, 0); + if (IS_CURSEC(sbi, GET_SEC_FROM_SEG(sbi, segno))) { + f2fs_err(sbi, "%s: segment %u is used by log", + __func__, segno); + f2fs_bug_on(sbi, 1); + goto skip; + } + if (get_valid_blocks(sbi, segno, false) == 0) goto freed; if (gc_type == BG_GC && __is_large_section(sbi) && @@ -1807,7 +1837,7 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, sum = page_address(sum_page); if (type != GET_SUM_TYPE((&sum->footer))) { - f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SSA and SIT", + f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SIT and SSA", segno, type, GET_SUM_TYPE((&sum->footer))); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_SUMMARY); @@ -1823,11 +1853,11 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, */ if (type == SUM_TYPE_NODE) submitted += gc_node_segment(sbi, sum->entries, segno, - gc_type); + gc_type, &plug); else submitted += gc_data_segment(sbi, sum->entries, gc_list, segno, gc_type, - force_migrate); + force_migrate, &plug); stat_inc_gc_seg_count(sbi, data_type, gc_type); sbi->gc_reclaimed_segs[sbi->gc_mode]++; @@ -1844,8 +1874,11 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, segno + 1 : NULL_SEGNO; skip: f2fs_put_page(sum_page, 0); + if (unlikely(freezing(current))) + goto stop; } +stop: if (submitted) f2fs_submit_merged_write(sbi, data_type); @@ -1896,6 +1929,7 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control) /* Let's run FG_GC, if we don't have enough space. */ if (has_not_enough_free_secs(sbi, 0, 0)) { gc_type = FG_GC; + gc_control->one_time = false; /* * For example, if there are many prefree_segments below given @@ -1918,6 +1952,10 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control) goto stop; } retry: + if (unlikely(freezing(current))) { + ret = 0; + goto stop; + } ret = __get_victim(sbi, &segno, gc_type, gc_control->one_time); if (ret) { /* allow to search victim from sections has pinned data */ @@ -2002,7 +2040,7 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control) reserved_segments(sbi), prefree_segments(sbi)); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &gc_control->lc); put_gc_inode(&gc_list); @@ -2069,6 +2107,13 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS), }; + /* + * avoid migrating empty section, as it can be allocated by + * log in parallel. + */ + if (!get_valid_blocks(sbi, segno, true)) + continue; + if (IS_CURSEC(sbi, GET_SEC_FROM_SEG(sbi, segno))) continue; @@ -2206,6 +2251,9 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(filp)); __u64 old_block_count, shrunk_blocks; struct cp_control cpc = { CP_RESIZE, 0, 0, 0 }; + struct f2fs_lock_context lc; + struct f2fs_lock_context glc; + struct f2fs_lock_context clc; unsigned int secs; int err = 0; __u32 rem; @@ -2249,13 +2297,13 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) secs = div_u64(shrunk_blocks, BLKS_PER_SEC(sbi)); /* stop other GC */ - if (!f2fs_down_write_trylock(&sbi->gc_lock)) { + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, &glc)) { err = -EAGAIN; goto out_drop_write; } /* stop CP to protect MAIN_SEC in free_segment_range */ - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); spin_lock(&sbi->stat_lock); if (shrunk_blocks + valid_user_blocks(sbi) + @@ -2270,8 +2318,8 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) err = free_segment_range(sbi, secs, true); out_unlock: - f2fs_unlock_op(sbi); - f2fs_up_write(&sbi->gc_lock); + f2fs_unlock_op(sbi, &lc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); out_drop_write: mnt_drop_write_file(filp); if (err) @@ -2288,8 +2336,8 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) return -EROFS; } - f2fs_down_write(&sbi->gc_lock); - f2fs_down_write(&sbi->cp_global_sem); + f2fs_down_write_trace(&sbi->gc_lock, &glc); + f2fs_down_write_trace(&sbi->cp_global_sem, &clc); spin_lock(&sbi->stat_lock); if (shrunk_blocks + valid_user_blocks(sbi) + @@ -2337,8 +2385,8 @@ int f2fs_resize_fs(struct file *filp, __u64 block_count) spin_unlock(&sbi->stat_lock); } out_err: - f2fs_up_write(&sbi->cp_global_sem); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->cp_global_sem, &clc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); thaw_super(sbi->sb, FREEZE_HOLDER_USERSPACE); return err; } diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h index 5c1eaf55e1277..feb574c493450 100644 --- a/fs/f2fs/gc.h +++ b/fs/f2fs/gc.h @@ -25,7 +25,7 @@ #define DEF_GC_THREAD_CANDIDATE_RATIO 20 /* select 20% oldest sections as candidates */ #define DEF_GC_THREAD_MAX_CANDIDATE_COUNT 10 /* select at most 10 sections as candidates */ #define DEF_GC_THREAD_AGE_WEIGHT 60 /* age weight */ -#define DEF_GC_THREAD_VALID_THRESH_RATIO 95 /* do not GC over 95% valid block ratio for one time GC */ +#define DEF_GC_THREAD_VALID_THRESH_RATIO 80 /* do not GC over 80% valid block ratio for one time GC */ #define DEFAULT_ACCURACY_CLASS 10000 /* accuracy class */ #define LIMIT_INVALID_BLOCK 40 /* percentage over total user space */ diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index b1b85aaf8e360..867bbd291a8fa 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -219,6 +219,7 @@ int f2fs_convert_inline_inode(struct inode *inode) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; struct page *ipage, *page; + struct f2fs_lock_context lc; int err = 0; if (f2fs_hw_is_readonly(sbi) || f2fs_readonly(sbi->sb)) @@ -235,7 +236,7 @@ int f2fs_convert_inline_inode(struct inode *inode) if (!page) return -ENOMEM; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); ipage = f2fs_get_node_page(sbi, inode->i_ino); if (IS_ERR(ipage)) { @@ -250,7 +251,7 @@ int f2fs_convert_inline_inode(struct inode *inode) f2fs_put_dnode(&dn); out: - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_put_page(page, 1); @@ -597,13 +598,14 @@ int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry) struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct page *ipage; struct f2fs_filename fname; + struct f2fs_lock_context lc; void *inline_dentry = NULL; int err = 0; if (!f2fs_has_inline_dentry(dir)) return 0; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_setup_filename(dir, &dentry->d_name, 0, &fname); if (err) @@ -628,7 +630,7 @@ int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry) out_fname: f2fs_free_filename(&fname); out: - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); return err; } diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index da3bb7bf3d66c..c620bc080c5f3 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -34,7 +34,9 @@ void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync) if (f2fs_inode_dirtied(inode, sync)) return; - if (f2fs_is_atomic_file(inode)) + /* only atomic file w/ FI_ATOMIC_COMMITTED can be set vfs dirty */ + if (f2fs_is_atomic_file(inode) && + !is_inode_flag_set(inode, FI_ATOMIC_COMMITTED)) return; mark_inode_dirty_sync(inode); @@ -286,6 +288,12 @@ static bool sanity_check_inode(struct inode *inode, struct page *node_page) return false; } + if (ino_of_node(node_page) == fi->i_xattr_nid) { + f2fs_warn(sbi, "%s: corrupted inode i_ino=%lx, xnid=%x, run fsck to fix.", + __func__, inode->i_ino, fi->i_xattr_nid); + return false; + } + if (f2fs_has_extra_attr(inode)) { if (!f2fs_sb_has_extra_attr(sbi)) { f2fs_warn(sbi, "%s: inode (ino=%lx) is with extra_attr, but extra_attr feature is off", @@ -806,7 +814,7 @@ int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) return 0; } -static void f2fs_remove_donate_inode(struct inode *inode) +void f2fs_remove_donate_inode(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); @@ -885,9 +893,11 @@ void f2fs_evict_inode(struct inode *inode) err = -EIO; if (!err) { - f2fs_lock_op(sbi); + struct f2fs_lock_context lc; + + f2fs_lock_op(sbi, &lc); err = f2fs_remove_inode_page(inode); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (err == -ENOENT) { err = 0; @@ -919,6 +929,19 @@ void f2fs_evict_inode(struct inode *inode) f2fs_update_inode_page(inode); if (dquot_initialize_needed(inode)) set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + + /* + * If both f2fs_truncate() and f2fs_update_inode_page() failed + * due to fuzzed corrupted inode, call f2fs_inode_synced() to + * avoid triggering later f2fs_bug_on(). + */ + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { + f2fs_warn(sbi, + "f2fs_evict_inode: inode is dirty, ino:%lu", + inode->i_ino); + f2fs_inode_synced(inode); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } } if (freeze_protected) sb_end_intwrite(inode->i_sb); @@ -935,8 +958,12 @@ void f2fs_evict_inode(struct inode *inode) if (likely(!f2fs_cp_error(sbi) && !is_sbi_flag_set(sbi, SBI_CP_DISABLED))) f2fs_bug_on(sbi, is_inode_flag_set(inode, FI_DIRTY_INODE)); - else - f2fs_inode_synced(inode); + + /* + * anyway, it needs to remove the inode from sbi->inode_list[DIRTY_META] + * list to avoid UAF in f2fs_sync_inode_meta() during checkpoint. + */ + f2fs_inode_synced(inode); /* for the case f2fs_new_inode() was failed, .i_ino is zero, skip it */ if (inode->i_ino) @@ -967,7 +994,7 @@ void f2fs_evict_inode(struct inode *inode) } /* caller should call f2fs_lock_op() */ -void f2fs_handle_failed_inode(struct inode *inode) +void f2fs_handle_failed_inode(struct inode *inode, struct f2fs_lock_context *lc) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct node_info ni; @@ -1016,7 +1043,7 @@ void f2fs_handle_failed_inode(struct inode *inode) } out: - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, lc); /* iput will drop the inode object */ iput(inode); diff --git a/fs/f2fs/lock.c b/fs/f2fs/lock.c new file mode 100644 index 0000000000000..09d1839e7ce1b --- /dev/null +++ b/fs/f2fs/lock.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/checkpoint.c + * + * Copyright (c) 2026 Google + */ +#include +#include +#include +#include + +#include "f2fs.h" +#include "segment.h" +#include + +static inline void get_lock_elapsed_time(struct f2fs_time_stat *ts) +{ + ts->total_time = ktime_get(); +#ifdef CONFIG_64BIT + ts->running_time = current->se.sum_exec_runtime; +#endif +#if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS) + ts->runnable_time = current->sched_info.run_delay; +#endif +#ifdef CONFIG_TASK_DELAY_ACCT + if (current->delays) + ts->io_sleep_time = current->delays->blkio_delay; +#endif +} + +static inline void trace_lock_elapsed_time_start(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc) +{ + lc->lock_trace = trace_f2fs_lock_elapsed_time_enabled(); + if (!lc->lock_trace) + return; + + get_lock_elapsed_time(&lc->ts); +} + +static inline void trace_lock_elapsed_time_end(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc, bool is_write) +{ + struct f2fs_time_stat tts; + unsigned long long total_time; + unsigned long long running_time = 0; + unsigned long long runnable_time = 0; + unsigned long long io_sleep_time = 0; + unsigned long long other_time = 0; + unsigned npm = NSEC_PER_MSEC; + + if (!lc->lock_trace) + return; + + get_lock_elapsed_time(&tts); + + total_time = div_u64(tts.total_time - lc->ts.total_time, npm); + if (total_time <= sem->sbi->max_lock_elapsed_time) + return; + +#ifdef CONFIG_64BIT + running_time = div_u64(tts.running_time - lc->ts.running_time, npm); +#endif +#if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS) + runnable_time = div_u64(tts.runnable_time - lc->ts.runnable_time, npm); +#endif +#ifdef CONFIG_TASK_DELAY_ACCT + io_sleep_time = div_u64(tts.io_sleep_time - lc->ts.io_sleep_time, npm); +#endif + if (total_time > running_time + io_sleep_time + runnable_time) + other_time = total_time - running_time - + io_sleep_time - runnable_time; + + trace_f2fs_lock_elapsed_time(sem->sbi, sem->name, is_write, current, + get_current_ioprio(), total_time, running_time, + runnable_time, io_sleep_time, other_time); +} + +static bool need_uplift_priority(struct f2fs_rwsem *sem, bool is_write) +{ + if (!(sem->sbi->adjust_lock_priority & BIT(sem->name - 1))) + return false; + + switch (sem->name) { + /* + * writer is checkpoint which has high priority, let's just uplift + * priority for reader + */ + case LOCK_NAME_CP_RWSEM: + case LOCK_NAME_NODE_CHANGE: + case LOCK_NAME_NODE_WRITE: + return !is_write; + case LOCK_NAME_GC_LOCK: + case LOCK_NAME_CP_GLOBAL: + case LOCK_NAME_IO_RWSEM: + return true; + default: + f2fs_bug_on(sem->sbi, 1); + } + return false; +} + +static void uplift_priority(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc, + bool is_write) +{ + lc->need_restore = false; + if (!sem->sbi->adjust_lock_priority) + return; + if (rt_task(current)) + return; + if (!need_uplift_priority(sem, is_write)) + return; + lc->orig_nice = task_nice(current); + lc->new_nice = PRIO_TO_NICE(sem->sbi->lock_duration_priority); + if (lc->orig_nice <= lc->new_nice) + return; + set_user_nice(current, lc->new_nice); + lc->need_restore = true; + + trace_f2fs_priority_uplift(sem->sbi, sem->name, is_write, current, + NICE_TO_PRIO(lc->orig_nice), NICE_TO_PRIO(lc->new_nice)); +} + +static void restore_priority(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc, + bool is_write) +{ + if (!lc->need_restore) + return; + /* someone has updated the priority */ + if (task_nice(current) != lc->new_nice) + return; + set_user_nice(current, lc->orig_nice); + + trace_f2fs_priority_restore(sem->sbi, sem->name, is_write, current, + NICE_TO_PRIO(lc->orig_nice), NICE_TO_PRIO(lc->new_nice)); +} + +void f2fs_down_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, false); + f2fs_down_read(sem); + trace_lock_elapsed_time_start(sem, lc); +} + +int f2fs_down_read_trylock_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, false); + if (!f2fs_down_read_trylock(sem)) { + restore_priority(sem, lc, false); + return 0; + } + trace_lock_elapsed_time_start(sem, lc); + return 1; +} + +void f2fs_up_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + f2fs_up_read(sem); + restore_priority(sem, lc, false); + trace_lock_elapsed_time_end(sem, lc, false); +} + +void f2fs_down_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, true); + f2fs_down_write(sem); + trace_lock_elapsed_time_start(sem, lc); +} + +int f2fs_down_write_trylock_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, true); + if (!f2fs_down_write_trylock(sem)) { + restore_priority(sem, lc, true); + return 0; + } + trace_lock_elapsed_time_start(sem, lc); + return 1; +} + +void f2fs_up_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + f2fs_up_write(sem); + restore_priority(sem, lc, true); + trace_lock_elapsed_time_end(sem, lc, true); +} + +void f2fs_lock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc) +{ + f2fs_down_read_trace(&sbi->cp_rwsem, lc); +} + +int f2fs_trylock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc) +{ + if (time_to_inject(sbi, FAULT_LOCK_OP)) + return 0; + + return f2fs_down_read_trylock_trace(&sbi->cp_rwsem, lc); +} + +void f2fs_unlock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc) +{ + f2fs_up_read_trace(&sbi->cp_rwsem, lc); +} diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index f70eb188cf996..ffb3b5f5cfcc1 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -354,6 +354,7 @@ static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; struct inode *inode; nid_t ino = 0; int err; @@ -376,11 +377,11 @@ static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir, inode->i_mapping->a_ops = &f2fs_dblock_aops; ino = inode->i_ino; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_add_link(dentry, inode); if (err) goto out; - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_alloc_nid_done(sbi, ino); @@ -392,7 +393,7 @@ static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir, f2fs_balance_fs(sbi, true); return 0; out: - f2fs_handle_failed_inode(inode); + f2fs_handle_failed_inode(inode, &lc); return err; } @@ -401,6 +402,7 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir, { struct inode *inode = d_inode(old_dentry); struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; int err; if (unlikely(f2fs_cp_error(sbi))) @@ -414,7 +416,7 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir, if (is_inode_flag_set(dir, FI_PROJ_INHERIT) && (!projid_eq(F2FS_I(dir)->i_projid, - F2FS_I(old_dentry->d_inode)->i_projid))) + F2FS_I(inode)->i_projid))) return -EXDEV; err = f2fs_dquot_initialize(dir); @@ -427,11 +429,11 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir, ihold(inode); set_inode_flag(inode, FI_INC_LINK); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_add_link(dentry, inode); if (err) goto out; - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); d_instantiate(dentry, inode); @@ -441,7 +443,7 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir, out: clear_inode_flag(inode, FI_INC_LINK); iput(inode); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); return err; } @@ -539,6 +541,7 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode = d_inode(dentry); struct f2fs_dir_entry *de; + struct f2fs_lock_context lc; struct page *page; int err; @@ -563,17 +566,26 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) goto fail; } + if (unlikely(inode->i_nlink == 0)) { + f2fs_warn(F2FS_I_SB(inode), "%s: inode (ino=%lx) has zero i_nlink", + __func__, inode->i_ino); + err = -EFSCORRUPTED; + set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); + f2fs_put_page(page, 0); + goto fail; + } + f2fs_balance_fs(sbi, true); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_acquire_orphan_inode(sbi); if (err) { - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_put_page(page, 0); goto fail; } f2fs_delete_entry(de, page, dir, inode); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); #if IS_ENABLED(CONFIG_UNICODE) /* VFS negative dentries are incompatible with Encoding and @@ -611,6 +623,7 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, const char *symname) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; struct inode *inode; size_t len = strlen(symname); struct fscrypt_str disk_link; @@ -641,11 +654,11 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, inode_nohighmem(inode); inode->i_mapping->a_ops = &f2fs_dblock_aops; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_add_link(dentry, inode); if (err) goto out_f2fs_handle_failed_inode; - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_alloc_nid_done(sbi, inode->i_ino); err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link); @@ -680,7 +693,7 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, goto out_free_encrypted_link; out_f2fs_handle_failed_inode: - f2fs_handle_failed_inode(inode); + f2fs_handle_failed_inode(inode, &lc); out_free_encrypted_link: if (disk_link.name != (unsigned char *)symname) kfree(disk_link.name); @@ -691,6 +704,7 @@ static int f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; struct inode *inode; int err; @@ -711,11 +725,11 @@ static int f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir, mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); set_inode_flag(inode, FI_INC_LINK); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_add_link(dentry, inode); if (err) goto out_fail; - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_alloc_nid_done(sbi, inode->i_ino); @@ -729,7 +743,7 @@ static int f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir, out_fail: clear_inode_flag(inode, FI_INC_LINK); - f2fs_handle_failed_inode(inode); + f2fs_handle_failed_inode(inode, &lc); return err; } @@ -746,6 +760,7 @@ static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; struct inode *inode; int err = 0; @@ -765,11 +780,11 @@ static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir, init_special_inode(inode, inode->i_mode, rdev); inode->i_op = &f2fs_special_inode_operations; - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_add_link(dentry, inode); if (err) goto out; - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_alloc_nid_done(sbi, inode->i_ino); @@ -781,7 +796,7 @@ static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir, f2fs_balance_fs(sbi, true); return 0; out: - f2fs_handle_failed_inode(inode); + f2fs_handle_failed_inode(inode, &lc); return err; } @@ -790,6 +805,7 @@ static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, struct inode **new_inode, struct f2fs_filename *fname) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; struct inode *inode; int err; @@ -810,7 +826,7 @@ static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, inode->i_mapping->a_ops = &f2fs_dblock_aops; } - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_acquire_orphan_inode(sbi); if (err) goto out; @@ -839,7 +855,7 @@ static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, f2fs_i_links_write(inode, false); } /* link_count was changed by d_tmpfile as well. */ - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); unlock_new_inode(inode); if (new_inode) @@ -851,7 +867,7 @@ static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, release_out: f2fs_release_orphan_inode(sbi); out: - f2fs_handle_failed_inode(inode); + f2fs_handle_failed_inode(inode, &lc); return err; } @@ -899,6 +915,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, struct f2fs_dir_entry *old_dir_entry = NULL; struct f2fs_dir_entry *old_entry; struct f2fs_dir_entry *new_entry; + struct f2fs_lock_context lc; bool old_is_dir = S_ISDIR(old_inode->i_mode); int err; @@ -909,7 +926,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, if (is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && (!projid_eq(F2FS_I(new_dir)->i_projid, - F2FS_I(old_dentry->d_inode)->i_projid))) + F2FS_I(old_inode)->i_projid))) return -EXDEV; /* @@ -987,7 +1004,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, f2fs_balance_fs(sbi, true); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_acquire_orphan_inode(sbi); if (err) @@ -1010,11 +1027,11 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, } else { f2fs_balance_fs(sbi, true); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_add_link(new_dentry, old_inode); if (err) { - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); goto out_dir; } @@ -1061,7 +1078,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, TRANS_DIR_INO); } - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) f2fs_sync_fs(sbi->sb, 1); @@ -1070,7 +1087,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, return 0; put_out_dir: - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_put_page(new_page, 0); out_dir: if (old_dir_entry) @@ -1092,6 +1109,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, struct page *old_page, *new_page; struct f2fs_dir_entry *old_dir_entry = NULL, *new_dir_entry = NULL; struct f2fs_dir_entry *old_entry, *new_entry; + struct f2fs_lock_context lc; int old_nlink = 0, new_nlink = 0; int err; @@ -1102,10 +1120,10 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, if ((is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && !projid_eq(F2FS_I(new_dir)->i_projid, - F2FS_I(old_dentry->d_inode)->i_projid)) || - (is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && + F2FS_I(old_inode)->i_projid)) || + (is_inode_flag_set(old_dir, FI_PROJ_INHERIT) && !projid_eq(F2FS_I(old_dir)->i_projid, - F2FS_I(new_dentry->d_inode)->i_projid))) + F2FS_I(new_inode)->i_projid))) return -EXDEV; err = f2fs_dquot_initialize(old_dir); @@ -1171,7 +1189,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_balance_fs(sbi, true); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); /* update ".." directory entry info of old dentry */ if (old_dir_entry) @@ -1224,7 +1242,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_add_ino_entry(sbi, new_dir->i_ino, TRANS_DIR_INO); } - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) f2fs_sync_fs(sbi->sb, 1); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index a45ab780f416e..26a3a0c9f2ca4 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -800,6 +800,16 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) for (i = 1; i <= level; i++) { bool done = false; + if (nids[i] && nids[i] == dn->inode->i_ino) { + err = -EFSCORRUPTED; + f2fs_err_ratelimited(sbi, + "inode mapping table is corrupted, run fsck to fix it, " + "ino:%lu, nid:%u, level:%d, offset:%d", + dn->inode->i_ino, nids[i], level, offset[level]); + set_sbi_flag(sbi, SBI_NEED_FSCK); + goto release_pages; + } + if (!nids[i] && mode == ALLOC_NODE) { /* alloc new node */ if (!f2fs_alloc_nid(sbi, &(nids[i]))) { @@ -1645,6 +1655,7 @@ static int __write_node_page(struct page *page, bool atomic, bool *submitted, .io_type = io_type, .io_wbc = wbc, }; + struct f2fs_lock_context lc; unsigned int seq; trace_f2fs_writepage(folio, NODE); @@ -1675,17 +1686,17 @@ static int __write_node_page(struct page *page, bool atomic, bool *submitted, goto redirty_out; if (wbc->for_reclaim) { - if (!f2fs_down_read_trylock(&sbi->node_write)) + if (!f2fs_down_read_trylock_trace(&sbi->node_write, &lc)) goto redirty_out; } else { - f2fs_down_read(&sbi->node_write); + f2fs_down_read_trace(&sbi->node_write, &lc); } /* This page is already truncated */ if (unlikely(ni.blk_addr == NULL_ADDR)) { folio_clear_uptodate(folio); dec_page_count(sbi, F2FS_DIRTY_NODES); - f2fs_up_read(&sbi->node_write); + f2fs_up_read_trace(&sbi->node_write, &lc); folio_unlock(folio); return 0; } @@ -1693,7 +1704,7 @@ static int __write_node_page(struct page *page, bool atomic, bool *submitted, if (__is_valid_data_blkaddr(ni.blk_addr) && !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) { - f2fs_up_read(&sbi->node_write); + f2fs_up_read_trace(&sbi->node_write, &lc); goto redirty_out; } @@ -1713,7 +1724,7 @@ static int __write_node_page(struct page *page, bool atomic, bool *submitted, f2fs_do_write_node_page(nid, &fio); set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(page)); dec_page_count(sbi, F2FS_DIRTY_NODES); - f2fs_up_read(&sbi->node_write); + f2fs_up_read_trace(&sbi->node_write, &lc); if (wbc->for_reclaim) { f2fs_submit_merged_write_cond(sbi, NULL, page, 0, NODE); diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index db97ff59dff0a..a14e111f41dc9 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -855,6 +855,7 @@ int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only) { struct list_head inode_list, tmp_inode_list; struct list_head dir_list; + struct f2fs_lock_context lc; int err; int ret = 0; unsigned long s_flags = sbi->sb->s_flags; @@ -868,7 +869,7 @@ int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only) INIT_LIST_HEAD(&dir_list); /* prevent checkpoint */ - f2fs_down_write(&sbi->cp_global_sem); + f2fs_down_write_trace(&sbi->cp_global_sem, &lc); /* step #1: find fsynced inode numbers */ err = find_fsync_dnodes(sbi, &inode_list, check_only); @@ -912,7 +913,7 @@ int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only) if (!err) clear_sbi_flag(sbi, SBI_POR_DOING); - f2fs_up_write(&sbi->cp_global_sem); + f2fs_up_write_trace(&sbi->cp_global_sem, &lc); /* let's drop all the directory inodes for clean checkpoint */ destroy_fsync_dnodes(&dir_list, err); diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index b3f1ea01b5833..1c3321a9c813a 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -376,7 +376,13 @@ static int __f2fs_commit_atomic_write(struct inode *inode) } else { sbi->committed_atomic_block += fi->atomic_write_cnt; set_inode_flag(inode, FI_ATOMIC_COMMITTED); + + /* + * inode may has no FI_ATOMIC_DIRTIED flag due to no write + * before commit. + */ if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) { + /* clear atomic dirty status and set vfs dirty status */ clear_inode_flag(inode, FI_ATOMIC_DIRTIED); f2fs_mark_inode_dirty_sync(inode, true); } @@ -391,6 +397,7 @@ int f2fs_commit_atomic_write(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_lock_context lc; int err; err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); @@ -398,11 +405,11 @@ int f2fs_commit_atomic_write(struct inode *inode) return err; f2fs_down_write(&fi->i_gc_rwsem[WRITE]); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = __f2fs_commit_atomic_write(inode); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_up_write(&fi->i_gc_rwsem[WRITE]); return err; @@ -446,12 +453,13 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) } else { struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, - .init_gc_type = BG_GC, + .init_gc_type = f2fs_sb_has_blkzoned(sbi) ? + FG_GC : BG_GC, .no_bg_gc = true, .should_migrate_blocks = false, .err_gc_skipped = false, .nr_free_secs = 1 }; - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); stat_inc_gc_call_count(sbi, FOREGROUND); f2fs_gc(sbi, &gc_control); } @@ -1611,6 +1619,9 @@ static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, if (dc->state != D_PREP) goto next; + if (*issued > 0 && unlikely(freezing(current))) + break; + if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { io_interrupted = true; break; @@ -1650,6 +1661,7 @@ static int __issue_discard_cmd(struct f2fs_sb_info *sbi, struct blk_plug plug; int i, issued; bool io_interrupted = false; + bool suspended = false; if (dpolicy->timeout) f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); @@ -1680,6 +1692,11 @@ static int __issue_discard_cmd(struct f2fs_sb_info *sbi, list_for_each_entry_safe(dc, tmp, pend_list, list) { f2fs_bug_on(sbi, dc->state != D_PREP); + if (issued > 0 && unlikely(freezing(current))) { + suspended = true; + break; + } + if (dpolicy->timeout && f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) break; @@ -1699,7 +1716,8 @@ static int __issue_discard_cmd(struct f2fs_sb_info *sbi, next: mutex_unlock(&dcc->cmd_lock); - if (issued >= dpolicy->max_requests || io_interrupted) + if (issued >= dpolicy->max_requests || io_interrupted || + suspended) break; } @@ -2909,7 +2927,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) return SIT_I(sbi)->last_victim[ALLOC_NEXT]; /* find segments from 0 to reuse freed segments */ - if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) + if (f2fs_get_alloc_mode(sbi) == ALLOC_MODE_REUSE) return 0; return curseg->segno; @@ -3294,19 +3312,20 @@ int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi) { + struct f2fs_lock_context lc; int err; bool gc_required = true; retry: - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (f2fs_sb_has_blkzoned(sbi) && err == -EAGAIN && gc_required) { - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &lc); err = f2fs_gc_range(sbi, 0, GET_SEGNO(sbi, FDEV(0).end_blk), true, ZONED_PIN_SEC_REQUIRED_COUNT); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &lc); gc_required = false; if (!err) @@ -3426,6 +3445,7 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) block_t start_block, end_block; struct cp_control cpc; struct discard_policy dpolicy; + struct f2fs_lock_context lc; unsigned long long trimmed = 0; int err = 0; bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); @@ -3458,10 +3478,10 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) if (sbi->discard_blks == 0) goto out; - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &lc); stat_inc_cp_call_count(sbi, TOTAL_CALL); err = f2fs_write_checkpoint(sbi, &cpc); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &lc); if (err) goto out; diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index e905c1aff512a..0a822ee029bb6 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -616,8 +616,7 @@ static inline void __get_secs_required(struct f2fs_sb_info *sbi, unsigned int dent_blocks = total_dent_blocks % CAP_BLKS_PER_SEC(sbi); unsigned int data_blocks = 0; - if (f2fs_lfs_mode(sbi) && - unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (f2fs_lfs_mode(sbi)) { total_data_blocks = get_pages(sbi, F2FS_DIRTY_DATA); data_secs = total_data_blocks / CAP_BLKS_PER_SEC(sbi); data_blocks = total_data_blocks % CAP_BLKS_PER_SEC(sbi); @@ -626,7 +625,7 @@ static inline void __get_secs_required(struct f2fs_sb_info *sbi, if (lower_p) *lower_p = node_secs + dent_secs + data_secs; if (upper_p) - *upper_p = node_secs + dent_secs + + *upper_p = node_secs + dent_secs + data_secs + (node_blocks ? 1 : 0) + (dent_blocks ? 1 : 0) + (data_blocks ? 1 : 0); if (curseg_p) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index b5d23377166d8..0075ab84eb0fc 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -131,6 +131,7 @@ enum { Opt_noinline_data, Opt_data_flush, Opt_reserve_root, + Opt_reserve_node, Opt_resgid, Opt_resuid, Opt_mode, @@ -176,6 +177,7 @@ enum { Opt_memory_mode, Opt_age_extent_cache, Opt_errors, + Opt_lookup_mode, Opt_err, }; @@ -209,6 +211,7 @@ static match_table_t f2fs_tokens = { {Opt_noinline_data, "noinline_data"}, {Opt_data_flush, "data_flush"}, {Opt_reserve_root, "reserve_root=%u"}, + {Opt_reserve_node, "reserve_node=%u"}, {Opt_resgid, "resgid=%u"}, {Opt_resuid, "resuid=%u"}, {Opt_mode, "mode=%s"}, @@ -255,6 +258,7 @@ static match_table_t f2fs_tokens = { {Opt_memory_mode, "memory=%s"}, {Opt_age_extent_cache, "age_extent_cache"}, {Opt_errors, "errors=%s"}, + {Opt_lookup_mode, "lookup_mode=%s"}, {Opt_err, NULL}, }; @@ -321,22 +325,30 @@ static void f2fs_destroy_casefold_cache(void) { } static inline void limit_reserve_root(struct f2fs_sb_info *sbi) { - block_t limit = min((sbi->user_block_count >> 3), + block_t block_limit = min((sbi->user_block_count >> 3), sbi->user_block_count - sbi->reserved_blocks); + block_t node_limit = sbi->total_node_count >> 3; /* limit is 12.5% */ if (test_opt(sbi, RESERVE_ROOT) && - F2FS_OPTION(sbi).root_reserved_blocks > limit) { - F2FS_OPTION(sbi).root_reserved_blocks = limit; + F2FS_OPTION(sbi).root_reserved_blocks > block_limit) { + F2FS_OPTION(sbi).root_reserved_blocks = block_limit; f2fs_info(sbi, "Reduce reserved blocks for root = %u", F2FS_OPTION(sbi).root_reserved_blocks); } - if (!test_opt(sbi, RESERVE_ROOT) && + if (test_opt(sbi, RESERVE_NODE) && + F2FS_OPTION(sbi).root_reserved_nodes > node_limit) { + F2FS_OPTION(sbi).root_reserved_nodes = node_limit; + f2fs_info(sbi, "Reduce reserved nodes for root = %u", + F2FS_OPTION(sbi).root_reserved_nodes); + } + if (!test_opt(sbi, RESERVE_ROOT) && !test_opt(sbi, RESERVE_NODE) && (!uid_eq(F2FS_OPTION(sbi).s_resuid, make_kuid(&init_user_ns, F2FS_DEF_RESUID)) || !gid_eq(F2FS_OPTION(sbi).s_resgid, make_kgid(&init_user_ns, F2FS_DEF_RESGID)))) - f2fs_info(sbi, "Ignore s_resuid=%u, s_resgid=%u w/o reserve_root", + f2fs_info(sbi, "Ignore s_resuid=%u, s_resgid=%u w/o reserve_root" + " and reserve_node", from_kuid_munged(&init_user_ns, F2FS_OPTION(sbi).s_resuid), from_kgid_munged(&init_user_ns, @@ -847,6 +859,17 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) set_opt(sbi, RESERVE_ROOT); } break; + case Opt_reserve_node: + if (args->from && match_int(args, &arg)) + return -EINVAL; + if (test_opt(sbi, RESERVE_NODE)) { + f2fs_info(sbi, "Preserve previous reserve_node=%u", + F2FS_OPTION(sbi).root_reserved_nodes); + } else { + F2FS_OPTION(sbi).root_reserved_nodes = arg; + set_opt(sbi, RESERVE_NODE); + } + break; case Opt_resuid: if (args->from && match_int(args, &arg)) return -EINVAL; @@ -998,9 +1021,9 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) return -ENOMEM; if (!strcmp(name, "default")) { - F2FS_OPTION(sbi).alloc_mode = ALLOC_MODE_DEFAULT; + f2fs_set_alloc_mode(sbi, ALLOC_MODE_DEFAULT); } else if (!strcmp(name, "reuse")) { - F2FS_OPTION(sbi).alloc_mode = ALLOC_MODE_REUSE; + f2fs_set_alloc_mode(sbi, ALLOC_MODE_REUSE); } else { kfree(name); return -EINVAL; @@ -1300,6 +1323,22 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) } kfree(name); break; + case Opt_lookup_mode: + name = match_strdup(&args[0]); + if (!name) + return -ENOMEM; + if (!strcmp(name, "perf")) { + f2fs_set_lookup_mode(sbi, LOOKUP_PERF); + } else if (!strcmp(name, "compat")) { + f2fs_set_lookup_mode(sbi, LOOKUP_COMPAT); + } else if (!strcmp(name, "auto")) { + f2fs_set_lookup_mode(sbi, LOOKUP_AUTO); + } else { + kfree(name); + return -EINVAL; + } + kfree(name); + break; default: f2fs_err(sbi, "Unrecognized mount option \"%s\" or missing value", p); @@ -1415,6 +1454,7 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb) /* Initialize f2fs-specific inode info */ atomic_set(&fi->dirty_pages, 0); atomic_set(&fi->i_compr_blocks, 0); + atomic_set(&fi->open_count, 0); init_f2fs_rwsem(&fi->i_sem); spin_lock_init(&fi->i_size_lock); INIT_LIST_HEAD(&fi->dirty_list); @@ -1507,7 +1547,9 @@ int f2fs_inode_dirtied(struct inode *inode, bool sync) } spin_unlock(&sbi->inode_lock[DIRTY_META]); - if (!ret && f2fs_is_atomic_file(inode)) + /* if atomic write is not committed, set inode w/ atomic dirty */ + if (!ret && f2fs_is_atomic_file(inode) && + !is_inode_flag_set(inode, FI_ATOMIC_COMMITTED)) set_inode_flag(inode, FI_ATOMIC_DIRTIED); return ret; @@ -1722,22 +1764,28 @@ int f2fs_sync_fs(struct super_block *sb, int sync) static int f2fs_freeze(struct super_block *sb) { + struct f2fs_sb_info *sbi = F2FS_SB(sb); + if (f2fs_readonly(sb)) return 0; /* IO error happened before */ - if (unlikely(f2fs_cp_error(F2FS_SB(sb)))) + if (unlikely(f2fs_cp_error(sbi))) return -EIO; /* must be clean, since sync_filesystem() was already called */ - if (is_sbi_flag_set(F2FS_SB(sb), SBI_IS_DIRTY)) + if (is_sbi_flag_set(sbi, SBI_IS_DIRTY)) return -EINVAL; + sbi->umount_lock_holder = current; + /* Let's flush checkpoints and stop the thread. */ - f2fs_flush_ckpt_thread(F2FS_SB(sb)); + f2fs_flush_ckpt_thread(sbi); + + sbi->umount_lock_holder = NULL; /* to avoid deadlock on f2fs_evict_inode->SB_FREEZE_FS */ - set_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING); + set_sbi_flag(sbi, SBI_IS_FREEZING); return 0; } @@ -1776,26 +1824,32 @@ static int f2fs_statfs_project(struct super_block *sb, limit = min_not_zero(dquot->dq_dqb.dqb_bsoftlimit, dquot->dq_dqb.dqb_bhardlimit); - if (limit) - limit >>= sb->s_blocksize_bits; + limit >>= sb->s_blocksize_bits; + + if (limit) { + uint64_t remaining = 0; - if (limit && buf->f_blocks > limit) { curblock = (dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace) >> sb->s_blocksize_bits; - buf->f_blocks = limit; - buf->f_bfree = buf->f_bavail = - (buf->f_blocks > curblock) ? - (buf->f_blocks - curblock) : 0; + if (limit > curblock) + remaining = limit - curblock; + + buf->f_blocks = min(buf->f_blocks, limit); + buf->f_bfree = min(buf->f_bfree, remaining); + buf->f_bavail = min(buf->f_bavail, remaining); } limit = min_not_zero(dquot->dq_dqb.dqb_isoftlimit, dquot->dq_dqb.dqb_ihardlimit); - if (limit && buf->f_files > limit) { - buf->f_files = limit; - buf->f_ffree = - (buf->f_files > dquot->dq_dqb.dqb_curinodes) ? - (buf->f_files - dquot->dq_dqb.dqb_curinodes) : 0; + if (limit) { + uint64_t remaining = 0; + + if (limit > dquot->dq_dqb.dqb_curinodes) + remaining = limit - dquot->dq_dqb.dqb_curinodes; + + buf->f_files = min(buf->f_files, limit); + buf->f_ffree = min(buf->f_ffree, remaining); } spin_unlock(&dquot->dq_dqb_lock); @@ -1854,9 +1908,9 @@ static int f2fs_statfs(struct dentry *dentry, struct kstatfs *buf) buf->f_fsid = u64_to_fsid(id); #ifdef CONFIG_QUOTA - if (is_inode_flag_set(dentry->d_inode, FI_PROJ_INHERIT) && + if (is_inode_flag_set(d_inode(dentry), FI_PROJ_INHERIT) && sb_has_quota_limits_enabled(sb, PRJQUOTA)) { - f2fs_statfs_project(sb, F2FS_I(dentry->d_inode)->i_projid, buf); + f2fs_statfs_project(sb, F2FS_I(d_inode(dentry))->i_projid, buf); } #endif return 0; @@ -2044,9 +2098,11 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) seq_puts(seq, "fragment:block"); seq_printf(seq, ",active_logs=%u", F2FS_OPTION(sbi).active_logs); - if (test_opt(sbi, RESERVE_ROOT)) - seq_printf(seq, ",reserve_root=%u,resuid=%u,resgid=%u", + if (test_opt(sbi, RESERVE_ROOT) || test_opt(sbi, RESERVE_NODE)) + seq_printf(seq, ",reserve_root=%u,reserve_node=%u,resuid=%u," + "resgid=%u", F2FS_OPTION(sbi).root_reserved_blocks, + F2FS_OPTION(sbi).root_reserved_nodes, from_kuid_munged(&init_user_ns, F2FS_OPTION(sbi).s_resuid), from_kgid_munged(&init_user_ns, @@ -2076,9 +2132,9 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) if (sbi->sb->s_flags & SB_INLINECRYPT) seq_puts(seq, ",inlinecrypt"); - if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_DEFAULT) + if (f2fs_get_alloc_mode(sbi) == ALLOC_MODE_DEFAULT) seq_printf(seq, ",alloc_mode=%s", "default"); - else if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) + else if (f2fs_get_alloc_mode(sbi) == ALLOC_MODE_REUSE) seq_printf(seq, ",alloc_mode=%s", "reuse"); if (test_opt(sbi, DISABLE_CHECKPOINT)) @@ -2114,6 +2170,13 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) else if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_PANIC) seq_printf(seq, ",errors=%s", "panic"); + if (f2fs_get_lookup_mode(sbi) == LOOKUP_PERF) + seq_show_option(seq, "lookup_mode", "perf"); + else if (f2fs_get_lookup_mode(sbi) == LOOKUP_COMPAT) + seq_show_option(seq, "lookup_mode", "compat"); + else if (f2fs_get_lookup_mode(sbi) == LOOKUP_AUTO) + seq_show_option(seq, "lookup_mode", "auto"); + return 0; } @@ -2141,9 +2204,9 @@ static void default_options(struct f2fs_sb_info *sbi, bool remount) F2FS_OPTION(sbi).inline_xattr_size = DEFAULT_INLINE_XATTR_ADDRS; if (le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_main) <= SMALL_VOLUME_SEGMENTS) - F2FS_OPTION(sbi).alloc_mode = ALLOC_MODE_REUSE; + f2fs_set_alloc_mode(sbi, ALLOC_MODE_REUSE); else - F2FS_OPTION(sbi).alloc_mode = ALLOC_MODE_DEFAULT; + f2fs_set_alloc_mode(sbi, ALLOC_MODE_DEFAULT); F2FS_OPTION(sbi).fsync_mode = FSYNC_MODE_POSIX; F2FS_OPTION(sbi).s_resuid = make_kuid(&init_user_ns, F2FS_DEF_RESUID); F2FS_OPTION(sbi).s_resgid = make_kgid(&init_user_ns, F2FS_DEF_RESGID); @@ -2178,6 +2241,8 @@ static void default_options(struct f2fs_sb_info *sbi, bool remount) #endif f2fs_build_fault_attr(sbi, 0, 0); + + f2fs_set_lookup_mode(sbi, LOOKUP_PERF); } #ifdef CONFIG_QUOTA @@ -2188,6 +2253,7 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) { unsigned int s_flags = sbi->sb->s_flags; struct cp_control cpc; + struct f2fs_lock_context lc; unsigned int gc_mode = sbi->gc_mode; int err = 0; int ret; @@ -2217,7 +2283,7 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) .no_bg_gc = true, .nr_free_secs = 1 }; - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); stat_inc_gc_call_count(sbi, FOREGROUND); err = f2fs_gc(sbi, &gc_control); if (err == -ENODATA) { @@ -2241,7 +2307,7 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) } skip_gc: - f2fs_down_write(&sbi->gc_lock); + f2fs_down_write_trace(&sbi->gc_lock, &lc); cpc.reason = CP_PAUSE; set_sbi_flag(sbi, SBI_CP_DISABLED); stat_inc_cp_call_count(sbi, TOTAL_CALL); @@ -2254,37 +2320,89 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) spin_unlock(&sbi->stat_lock); out_unlock: - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &lc); restore_flag: sbi->gc_mode = gc_mode; sbi->sb->s_flags = s_flags; /* Restore SB_RDONLY status */ + f2fs_info(sbi, "f2fs_disable_checkpoint() finish, err:%d", err); return err; } -static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) +static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) { - int retry = DEFAULT_RETRY_IO_COUNT; + int retry = MAX_FLUSH_RETRY_COUNT; + long long start, writeback, end; + int ret; + long long skipped_write, dirty_data; + struct f2fs_lock_context lc; + + f2fs_info(sbi, "f2fs_enable_checkpoint() starts, meta: %lld, node: %lld, data: %lld", + get_pages(sbi, F2FS_DIRTY_META), + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DATA)); + + start = ktime_get(); + + set_sbi_flag(sbi, SBI_ENABLE_CHECKPOINT); /* we should flush all the data to keep data consistency */ do { + skipped_write = get_pages(sbi, F2FS_SKIPPED_WRITE); + dirty_data = get_pages(sbi, F2FS_DIRTY_DATA); + sync_inodes_sb(sbi->sb); f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); - } while (get_pages(sbi, F2FS_DIRTY_DATA) && retry--); - if (unlikely(retry < 0)) - f2fs_warn(sbi, "checkpoint=enable has some unwritten data."); + f2fs_info(sbi, "sync_inode_sb done, dirty_data: %lld, %lld, " + "skipped write: %lld, %lld, retry: %d", + get_pages(sbi, F2FS_DIRTY_DATA), + dirty_data, + get_pages(sbi, F2FS_SKIPPED_WRITE), + skipped_write, retry); - f2fs_down_write(&sbi->gc_lock); + /* + * sync_inodes_sb() has retry logic, so let's check dirty_data + * in prior to skipped_write in case there is no dirty data. + */ + if (!get_pages(sbi, F2FS_DIRTY_DATA)) + break; + if (get_pages(sbi, F2FS_SKIPPED_WRITE) == skipped_write) + break; + } while (retry--); + + clear_sbi_flag(sbi, SBI_ENABLE_CHECKPOINT); + + writeback = ktime_get(); + + if (unlikely(get_pages(sbi, F2FS_DIRTY_DATA) || + get_pages(sbi, F2FS_SKIPPED_WRITE))) + f2fs_warn(sbi, "checkpoint=enable unwritten data: %lld, skipped data: %lld, retry: %d", + get_pages(sbi, F2FS_DIRTY_DATA), + get_pages(sbi, F2FS_SKIPPED_WRITE), retry); + + if (get_pages(sbi, F2FS_SKIPPED_WRITE)) + atomic_set(&sbi->nr_pages[F2FS_SKIPPED_WRITE], 0); + + f2fs_down_write_trace(&sbi->gc_lock, &lc); f2fs_dirty_to_prefree(sbi); clear_sbi_flag(sbi, SBI_CP_DISABLED); set_sbi_flag(sbi, SBI_IS_DIRTY); - f2fs_up_write(&sbi->gc_lock); + f2fs_up_write_trace(&sbi->gc_lock, &lc); - f2fs_sync_fs(sbi->sb, 1); + ret = f2fs_sync_fs(sbi->sb, 1); + if (ret) + f2fs_err(sbi, "%s sync_fs failed, ret: %d", __func__, ret); /* Let's ensure there's no pending checkpoint anymore */ f2fs_flush_ckpt_thread(sbi); + + end = ktime_get(); + + f2fs_info(sbi, "f2fs_enable_checkpoint() finishes, writeback:%llu, sync:%llu", + ktime_ms_delta(writeback, start), + ktime_ms_delta(end, writeback)); + return ret; } static int f2fs_remount(struct super_block *sb, int *flags, char *data) @@ -2315,6 +2433,8 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) org_mount_opt = sbi->mount_opt; old_sb_flags = sb->s_flags; + sbi->umount_lock_holder = current; + #ifdef CONFIG_QUOTA org_mount_opt.s_jquota_fmt = F2FS_OPTION(sbi).s_jquota_fmt; for (i = 0; i < MAXQUOTAS; i++) { @@ -2500,7 +2620,9 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) goto restore_discard; need_enable_checkpoint = true; } else { - f2fs_enable_checkpoint(sbi); + err = f2fs_enable_checkpoint(sbi); + if (err) + goto restore_discard; need_disable_checkpoint = true; } } @@ -2538,10 +2660,13 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) limit_reserve_root(sbi); *flags = (*flags & ~SB_LAZYTIME) | (sb->s_flags & SB_LAZYTIME); + + sbi->umount_lock_holder = NULL; return 0; restore_checkpoint: if (need_enable_checkpoint) { - f2fs_enable_checkpoint(sbi); + if (f2fs_enable_checkpoint(sbi)) + f2fs_warn(sbi, "checkpoint has not been enabled"); } else if (need_disable_checkpoint) { if (f2fs_disable_checkpoint(sbi)) f2fs_warn(sbi, "checkpoint has not been disabled"); @@ -2578,6 +2703,8 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) #endif sbi->mount_opt = org_mount_opt; sb->s_flags = old_sb_flags; + + sbi->umount_lock_holder = NULL; return err; } @@ -2894,7 +3021,7 @@ static int f2fs_quota_sync_file(struct f2fs_sb_info *sbi, int type) return ret; } -int f2fs_quota_sync(struct super_block *sb, int type) +int f2fs_do_quota_sync(struct super_block *sb, int type) { struct f2fs_sb_info *sbi = F2FS_SB(sb); struct quota_info *dqopt = sb_dqopt(sb); @@ -2906,6 +3033,7 @@ int f2fs_quota_sync(struct super_block *sb, int type) * that userspace sees the changes. */ for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + struct f2fs_lock_context lc; if (type != -1 && cnt != type) continue; @@ -2925,13 +3053,13 @@ int f2fs_quota_sync(struct super_block *sb, int type) * block_operation * f2fs_down_read(quota_sem) */ - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); f2fs_down_read(&sbi->quota_sem); ret = f2fs_quota_sync_file(sbi, cnt); f2fs_up_read(&sbi->quota_sem); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); if (!f2fs_sb_has_quota_ino(sbi)) inode_unlock(dqopt->files[cnt]); @@ -2942,11 +3070,21 @@ int f2fs_quota_sync(struct super_block *sb, int type) return ret; } +static int f2fs_quota_sync(struct super_block *sb, int type) +{ + int ret; + + F2FS_SB(sb)->umount_lock_holder = current; + ret = f2fs_do_quota_sync(sb, type); + F2FS_SB(sb)->umount_lock_holder = NULL; + return ret; +} + static int f2fs_quota_on(struct super_block *sb, int type, int format_id, const struct path *path) { struct inode *inode; - int err; + int err = 0; /* if quota sysfile exists, deny enabling quota with specific file */ if (f2fs_sb_has_quota_ino(F2FS_SB(sb))) { @@ -2957,31 +3095,34 @@ static int f2fs_quota_on(struct super_block *sb, int type, int format_id, if (path->dentry->d_sb != sb) return -EXDEV; - err = f2fs_quota_sync(sb, type); + F2FS_SB(sb)->umount_lock_holder = current; + + err = f2fs_do_quota_sync(sb, type); if (err) - return err; + goto out; inode = d_inode(path->dentry); err = filemap_fdatawrite(inode->i_mapping); if (err) - return err; + goto out; err = filemap_fdatawait(inode->i_mapping); if (err) - return err; + goto out; err = dquot_quota_on(sb, type, format_id, path); if (err) - return err; + goto out; inode_lock(inode); F2FS_I(inode)->i_flags |= F2FS_QUOTA_DEFAULT_FL; f2fs_set_inode_flags(inode); inode_unlock(inode); f2fs_mark_inode_dirty_sync(inode, false); - - return 0; +out: + F2FS_SB(sb)->umount_lock_holder = NULL; + return err; } static int __f2fs_quota_off(struct super_block *sb, int type) @@ -2992,7 +3133,7 @@ static int __f2fs_quota_off(struct super_block *sb, int type) if (!inode || !igrab(inode)) return dquot_quota_off(sb, type); - err = f2fs_quota_sync(sb, type); + err = f2fs_do_quota_sync(sb, type); if (err) goto out_put; @@ -3015,6 +3156,8 @@ static int f2fs_quota_off(struct super_block *sb, int type) struct f2fs_sb_info *sbi = F2FS_SB(sb); int err; + F2FS_SB(sb)->umount_lock_holder = current; + err = __f2fs_quota_off(sb, type); /* @@ -3024,6 +3167,9 @@ static int f2fs_quota_off(struct super_block *sb, int type) */ if (is_journalled_quota(sbi)) set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + + F2FS_SB(sb)->umount_lock_holder = NULL; + return err; } @@ -3156,7 +3302,7 @@ int f2fs_dquot_initialize(struct inode *inode) return 0; } -int f2fs_quota_sync(struct super_block *sb, int type) +int f2fs_do_quota_sync(struct super_block *sb, int type) { return 0; } @@ -3650,6 +3796,7 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) block_t user_block_count, valid_user_blocks; block_t avail_node_count, valid_node_count; unsigned int nat_blocks, nat_bits_bytes, nat_bits_blocks; + unsigned int sit_blk_cnt; int i, j; total = le32_to_cpu(raw_super->segment_count); @@ -3761,6 +3908,13 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) return 1; } + sit_blk_cnt = DIV_ROUND_UP(main_segs, SIT_ENTRY_PER_BLOCK); + if (sit_bitmap_size * 8 < sit_blk_cnt) { + f2fs_err(sbi, "Wrong bitmap size: sit: %u, sit_blk_cnt:%u", + sit_bitmap_size, sit_blk_cnt); + return 1; + } + cp_pack_start_sum = __start_sum_addr(sbi); cp_payload = __cp_payload(sbi); if (cp_pack_start_sum < cp_payload + 1 || @@ -3831,6 +3985,10 @@ static void init_sb_info(struct f2fs_sb_info *sbi) sbi->max_fragment_hole = DEF_FRAGMENT_SIZE; spin_lock_init(&sbi->gc_remaining_trials_lock); atomic64_set(&sbi->current_atomic_write, 0); + sbi->max_lock_elapsed_time = MAX_LOCK_ELAPSED_TIME; + sbi->adjust_lock_priority = 0; + sbi->lock_duration_priority = F2FS_DEFAULT_TASK_PRIORITY; + sbi->critical_task_priority = F2FS_CRITICAL_TASK_PRIORITY; sbi->dir_level = DEF_DIR_LEVEL; sbi->interval_time[CP_TIME] = DEF_CP_INTERVAL; @@ -4437,13 +4595,13 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent) sbi->sb = sb; /* initialize locks within allocated memory */ - init_f2fs_rwsem(&sbi->gc_lock); + init_f2fs_rwsem_trace(&sbi->gc_lock, sbi, LOCK_NAME_GC_LOCK); mutex_init(&sbi->writepages); - init_f2fs_rwsem(&sbi->cp_global_sem); - init_f2fs_rwsem(&sbi->node_write); - init_f2fs_rwsem(&sbi->node_change); + init_f2fs_rwsem_trace(&sbi->cp_global_sem, sbi, LOCK_NAME_CP_GLOBAL); + init_f2fs_rwsem_trace(&sbi->node_write, sbi, LOCK_NAME_NODE_WRITE); + init_f2fs_rwsem_trace(&sbi->node_change, sbi, LOCK_NAME_NODE_CHANGE); spin_lock_init(&sbi->stat_lock); - init_f2fs_rwsem(&sbi->cp_rwsem); + init_f2fs_rwsem_trace(&sbi->cp_rwsem, sbi, LOCK_NAME_CP_RWSEM); init_f2fs_rwsem(&sbi->quota_sem); init_waitqueue_head(&sbi->cp_wait); spin_lock_init(&sbi->error_lock); @@ -4704,6 +4862,7 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent) if (err) goto free_compress_inode; + sbi->umount_lock_holder = current; #ifdef CONFIG_QUOTA /* Enable quota usage during mount */ if (f2fs_sb_has_quota_ino(sbi) && !f2fs_readonly(sb)) { @@ -4795,13 +4954,12 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent) if (err) goto sync_free_meta; - if (test_opt(sbi, DISABLE_CHECKPOINT)) { + if (test_opt(sbi, DISABLE_CHECKPOINT)) err = f2fs_disable_checkpoint(sbi); - if (err) - goto sync_free_meta; - } else if (is_set_ckpt_flags(sbi, CP_DISABLED_FLAG)) { - f2fs_enable_checkpoint(sbi); - } + else if (is_set_ckpt_flags(sbi, CP_DISABLED_FLAG)) + err = f2fs_enable_checkpoint(sbi); + if (err) + goto sync_free_meta; /* * If filesystem is not mounted as read-only then @@ -4834,6 +4992,8 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent) clear_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); cleancache_init_fs(sb); + + sbi->umount_lock_holder = NULL; return 0; sync_free_meta: @@ -4938,6 +5098,8 @@ static void kill_f2fs_super(struct super_block *sb) struct f2fs_sb_info *sbi = F2FS_SB(sb); if (sb->s_root) { + sbi->umount_lock_holder = current; + set_sbi_flag(sbi, SBI_IS_CLOSE); f2fs_stop_gc_thread(sbi); f2fs_stop_discard_thread(sbi); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 46216f0a203a9..0138b447f4e5f 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -281,6 +281,22 @@ static ssize_t encoding_flags_show(struct f2fs_attr *a, le16_to_cpu(F2FS_RAW_SUPER(sbi)->s_encoding_flags)); } +static ssize_t effective_lookup_mode_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + switch (f2fs_get_lookup_mode(sbi)) { + case LOOKUP_PERF: + return sysfs_emit(buf, "perf\n"); + case LOOKUP_COMPAT: + return sysfs_emit(buf, "compat\n"); + case LOOKUP_AUTO: + if (sb_no_casefold_compat_fallback(sbi->sb)) + return sysfs_emit(buf, "auto:perf\n"); + return sysfs_emit(buf, "auto:compat\n"); + } + return 0; +} + static ssize_t mounted_time_sec_show(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf) { @@ -831,6 +847,34 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } + if (!strcmp(a->attr.name, "adjust_lock_priority")) { + if (t >= BIT(LOCK_NAME_MAX - 1)) + return -EINVAL; + sbi->adjust_lock_priority = t; + return count; + } + + if (!strcmp(a->attr.name, "lock_duration_priority")) { + if (t < NICE_TO_PRIO(MIN_NICE) || t > NICE_TO_PRIO(MAX_NICE)) + return -EINVAL; + sbi->lock_duration_priority = t; + return count; + } + + if (!strcmp(a->attr.name, "critical_task_priority")) { + if (t < NICE_TO_PRIO(MIN_NICE) || t > NICE_TO_PRIO(MAX_NICE)) + return -EINVAL; + if (!capable(CAP_SYS_NICE)) + return -EPERM; + sbi->critical_task_priority = t; + if (sbi->cprc_info.f2fs_issue_ckpt) + set_user_nice(sbi->cprc_info.f2fs_issue_ckpt, + PRIO_TO_NICE(sbi->critical_task_priority)); + if (sbi->gc_thread && sbi->gc_thread->f2fs_gc_task) + set_user_nice(sbi->gc_thread->f2fs_gc_task, + PRIO_TO_NICE(sbi->critical_task_priority)); + return count; + } *ui = (unsigned int)t; return count; @@ -1138,6 +1182,10 @@ F2FS_SBI_GENERAL_RW_ATTR(blkzone_alloc_policy); #endif F2FS_SBI_GENERAL_RW_ATTR(carve_out); F2FS_SBI_GENERAL_RW_ATTR(reserved_pin_section); +F2FS_SBI_GENERAL_RW_ATTR(max_lock_elapsed_time); +F2FS_SBI_GENERAL_RW_ATTR(lock_duration_priority); +F2FS_SBI_GENERAL_RW_ATTR(adjust_lock_priority); +F2FS_SBI_GENERAL_RW_ATTR(critical_task_priority); /* STAT_INFO ATTR */ #ifdef CONFIG_F2FS_STAT_FS @@ -1174,6 +1222,7 @@ F2FS_GENERAL_RO_ATTR(current_reserved_blocks); F2FS_GENERAL_RO_ATTR(unusable); F2FS_GENERAL_RO_ATTR(encoding); F2FS_GENERAL_RO_ATTR(encoding_flags); +F2FS_GENERAL_RO_ATTR(effective_lookup_mode); F2FS_GENERAL_RO_ATTR(mounted_time_sec); F2FS_GENERAL_RO_ATTR(main_blkaddr); F2FS_GENERAL_RO_ATTR(pending_discard); @@ -1290,6 +1339,7 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(current_reserved_blocks), ATTR_LIST(encoding), ATTR_LIST(encoding_flags), + ATTR_LIST(effective_lookup_mode), ATTR_LIST(mounted_time_sec), #ifdef CONFIG_F2FS_STAT_FS ATTR_LIST(cp_foreground_calls), @@ -1332,6 +1382,10 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(max_read_extent_count), ATTR_LIST(carve_out), ATTR_LIST(reserved_pin_section), + ATTR_LIST(max_lock_elapsed_time), + ATTR_LIST(lock_duration_priority), + ATTR_LIST(adjust_lock_priority), + ATTR_LIST(critical_task_priority), NULL, }; ATTRIBUTE_GROUPS(f2fs); @@ -1703,6 +1757,69 @@ static int __maybe_unused disk_map_seq_show(struct seq_file *seq, return 0; } +static int __maybe_unused donation_list_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct inode *inode; + struct f2fs_inode_info *fi; + struct dentry *dentry; + char *buf, *path; + int i; + + buf = f2fs_getname(sbi); + if (!buf) + return 0; + + seq_printf(seq, "Donation List\n"); + seq_printf(seq, " # of files : %u\n", sbi->donate_files); + seq_printf(seq, " %-50s %10s %20s %20s %22s\n", + "File path", "Status", "Donation offset (kb)", + "Donation size (kb)", "File cached size (kb)"); + seq_printf(seq, "---\n"); + + for (i = 0; i < sbi->donate_files; i++) { + spin_lock(&sbi->inode_lock[DONATE_INODE]); + if (list_empty(&sbi->inode_list[DONATE_INODE])) { + spin_unlock(&sbi->inode_lock[DONATE_INODE]); + break; + } + fi = list_first_entry(&sbi->inode_list[DONATE_INODE], + struct f2fs_inode_info, gdonate_list); + list_move_tail(&fi->gdonate_list, &sbi->inode_list[DONATE_INODE]); + inode = igrab(&fi->vfs_inode); + spin_unlock(&sbi->inode_lock[DONATE_INODE]); + + if (!inode) + continue; + + inode_lock_shared(inode); + + dentry = d_find_alias(inode); + if (!dentry) { + path = NULL; + } else { + path = dentry_path_raw(dentry, buf, PATH_MAX); + if (IS_ERR(path)) + goto next; + } + seq_printf(seq, " %-50s %10s %20llu %20llu %22llu\n", + path ? path : "", + is_inode_flag_set(inode, FI_DONATE_FINISHED) ? + "Evicted" : "Donated", + (loff_t)fi->donate_start << (PAGE_SHIFT - 10), + (loff_t)(fi->donate_end + 1) << (PAGE_SHIFT - 10), + (loff_t)inode->i_mapping->nrpages << (PAGE_SHIFT - 10)); +next: + dput(dentry); + inode_unlock_shared(inode); + iput(inode); + } + f2fs_putname(buf); + return 0; +} + int __init f2fs_init_sysfs(void) { int ret; @@ -1794,6 +1911,8 @@ int f2fs_register_sysfs(struct f2fs_sb_info *sbi) discard_plist_seq_show, sb); proc_create_single_data("disk_map", 0444, sbi->s_proc, disk_map_seq_show, sb); + proc_create_single_data("donation_list", 0444, sbi->s_proc, + donation_list_seq_show, sb); return 0; put_feature_list_kobj: kobject_put(&sbi->s_feature_list_kobj); diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 54ab9caaae4de..3a6ea70dc3756 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -803,6 +803,7 @@ int f2fs_setxattr(struct inode *inode, int index, const char *name, struct page *ipage, int flags) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; int err; if (unlikely(f2fs_cp_error(sbi))) @@ -820,11 +821,11 @@ int f2fs_setxattr(struct inode *inode, int index, const char *name, size, ipage, flags); f2fs_balance_fs(sbi, true); - f2fs_lock_op(sbi); + f2fs_lock_op(sbi, &lc); f2fs_down_write(&F2FS_I(inode)->i_xattr_sem); err = __f2fs_setxattr(inode, index, name, value, size, ipage, flags); f2fs_up_write(&F2FS_I(inode)->i_xattr_sem); - f2fs_unlock_op(sbi); + f2fs_unlock_op(sbi, &lc); f2fs_update_time(sbi, REQ_TIME); return err; diff --git a/fs/file.c b/fs/file.c index 1f1181b189bf3..5a64619057850 100644 --- a/fs/file.c +++ b/fs/file.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "internal.h" @@ -89,18 +90,11 @@ static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt) * 'unsigned long' in some places, but simply because that is how the Linux * kernel bitmaps are defined to work: they are not "bits in an array of bytes", * they are very much "bits in an array of unsigned long". - * - * The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied - * by that "1024/sizeof(ptr)" before, we already know there are sufficient - * clear low bits. Clang seems to realize that, gcc ends up being confused. - * - * On a 128-bit machine, the ALIGN() would actually matter. In the meantime, - * let's consider it documentation (and maybe a test-case for gcc to improve - * its code generation ;) */ -static struct fdtable * alloc_fdtable(unsigned int nr) +static struct fdtable *alloc_fdtable(unsigned int slots_wanted) { struct fdtable *fdt; + unsigned int nr; void *data; /* @@ -108,22 +102,47 @@ static struct fdtable * alloc_fdtable(unsigned int nr) * Allocation steps are keyed to the size of the fdarray, since it * grows far faster than any of the other dynamic data. We try to fit * the fdarray into comfortable page-tuned chunks: starting at 1024B - * and growing in powers of two from there on. + * and growing in powers of two from there on. Since we called only + * with slots_wanted > BITS_PER_LONG (embedded instance in files->fdtab + * already gives BITS_PER_LONG slots), the above boils down to + * 1. use the smallest power of two large enough to give us that many + * slots. + * 2. on 32bit skip 64 and 128 - the minimal capacity we want there is + * 256 slots (i.e. 1Kb fd array). + * 3. on 64bit don't skip anything, 1Kb fd array means 128 slots there + * and we are never going to be asked for 64 or less. */ - nr /= (1024 / sizeof(struct file *)); - nr = roundup_pow_of_two(nr + 1); - nr *= (1024 / sizeof(struct file *)); - nr = ALIGN(nr, BITS_PER_LONG); + if (IS_ENABLED(CONFIG_32BIT) && slots_wanted < 256) + nr = 256; + else + nr = roundup_pow_of_two(slots_wanted); /* * Note that this can drive nr *below* what we had passed if sysctl_nr_open - * had been set lower between the check in expand_files() and here. Deal - * with that in caller, it's cheaper that way. + * had been set lower between the check in expand_files() and here. * * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise * bitmaps handling below becomes unpleasant, to put it mildly... */ - if (unlikely(nr > sysctl_nr_open)) - nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1; + if (unlikely(nr > sysctl_nr_open)) { + nr = round_down(sysctl_nr_open, BITS_PER_LONG); + if (nr < slots_wanted) + return ERR_PTR(-EMFILE); + } + + /* + * Check if the allocation size would exceed INT_MAX. kvmalloc_array() + * and kvmalloc() will warn if the allocation size is greater than + * INT_MAX, as filp_cachep objects are not __GFP_NOWARN. + * + * This can happen when sysctl_nr_open is set to a very high value and + * a process tries to use a file descriptor near that limit. For example, + * if sysctl_nr_open is set to 1073741816 (0x3ffffff8) - which is what + * systemd typically sets it to - then trying to use a file descriptor + * close to that value will require allocating a file descriptor table + * that exceeds 8GB in size. + */ + if (unlikely(nr > INT_MAX / sizeof(struct file *))) + return ERR_PTR(-EMFILE); fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT); if (!fdt) @@ -152,7 +171,7 @@ static struct fdtable * alloc_fdtable(unsigned int nr) out_fdt: kfree(fdt); out: - return NULL; + return ERR_PTR(-ENOMEM); } /* @@ -169,7 +188,7 @@ static int expand_fdtable(struct files_struct *files, unsigned int nr) struct fdtable *new_fdt, *cur_fdt; spin_unlock(&files->file_lock); - new_fdt = alloc_fdtable(nr); + new_fdt = alloc_fdtable(nr + 1); /* make sure all fd_install() have seen resize_in_progress * or have finished their rcu_read_lock_sched() section. @@ -178,16 +197,8 @@ static int expand_fdtable(struct files_struct *files, unsigned int nr) synchronize_rcu(); spin_lock(&files->file_lock); - if (!new_fdt) - return -ENOMEM; - /* - * extremely unlikely race - sysctl_nr_open decreased between the check in - * caller and alloc_fdtable(). Cheaper to catch it here... - */ - if (unlikely(new_fdt->max_fds <= nr)) { - __free_fdtable(new_fdt); - return -EMFILE; - } + if (IS_ERR(new_fdt)) + return PTR_ERR(new_fdt); cur_fdt = files_fdtable(files); BUG_ON(nr < cur_fdt->max_fds); copy_fdtable(new_fdt, cur_fdt); @@ -301,7 +312,6 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho struct file **old_fds, **new_fds; unsigned int open_files, i; struct fdtable *old_fdt, *new_fdt; - int error; newf = kmem_cache_alloc(files_cachep, GFP_KERNEL); if (!newf) @@ -319,6 +329,7 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho new_fdt->open_fds = newf->open_fds_init; new_fdt->full_fds_bits = newf->full_fds_bits_init; new_fdt->fd = &newf->fd_array[0]; + newf->dmabuf_info = NULL; spin_lock(&oldf->file_lock); old_fdt = files_fdtable(oldf); @@ -333,17 +344,10 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho if (new_fdt != &newf->fdtab) __free_fdtable(new_fdt); - new_fdt = alloc_fdtable(open_files - 1); - if (!new_fdt) { - error = -ENOMEM; - goto out_release; - } - - /* beyond sysctl_nr_open; nothing to do */ - if (unlikely(new_fdt->max_fds < open_files)) { - __free_fdtable(new_fdt); - error = -EMFILE; - goto out_release; + new_fdt = alloc_fdtable(open_files); + if (IS_ERR(new_fdt)) { + kmem_cache_free(files_cachep, newf); + return ERR_CAST(new_fdt); } /* @@ -392,10 +396,6 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho rcu_assign_pointer(newf->fdt, new_fdt); return newf; - -out_release: - kmem_cache_free(files_cachep, newf); - return ERR_PTR(error); } static struct fdtable *close_files(struct files_struct * files) @@ -418,6 +418,9 @@ static struct fdtable *close_files(struct files_struct * files) if (set & 1) { struct file * file = xchg(&fdt->fd[i], NULL); if (file) { + if (is_dma_buf_file(file)) + dma_buf_unaccount_task(file->private_data, + files->dmabuf_info); filp_close(file, files); cond_resched(); } @@ -438,6 +441,7 @@ void put_files_struct(struct files_struct *files) /* free the arrays if they are not embedded */ if (fdt != &files->fdtab) __free_fdtable(fdt); + put_dmabuf_info(files->dmabuf_info); kmem_cache_free(files_cachep, files); } } @@ -593,6 +597,14 @@ void fd_install(unsigned int fd, struct file *file) struct files_struct *files = current->files; struct fdtable *fdt; + if (is_dma_buf_file(file)) { + int err = dma_buf_account_task(file->private_data, files->dmabuf_info); + + if (err) + pr_err("dmabuf accounting failed during fd_install operation, err %d\n", + err); + } + rcu_read_lock_sched(); if (unlikely(files->resize_in_progress)) { @@ -634,6 +646,8 @@ static struct file *pick_file(struct files_struct *files, unsigned fd) fd = array_index_nospec(fd, fdt->max_fds); file = rcu_dereference_raw(fdt->fd[fd]); if (file) { + if (is_dma_buf_file(file)) + dma_buf_unaccount_task(file->private_data, files->dmabuf_info); rcu_assign_pointer(fdt->fd[fd], NULL); __put_unused_fd(files, fd); } @@ -743,6 +757,15 @@ int __close_range(unsigned fd, unsigned max_fd, unsigned int flags) fds = dup_fd(cur_fds, punch_hole); if (IS_ERR(fds)) return PTR_ERR(fds); + + /* + * This is a new partial sharing relationship, since we have a new files_struct. + * Since partial sharing is not supported for dmabuf accounting, we need to remove + * the accounting info from the task. Leave the cur_fds->dmabuf_info so any existing + * accounting can be unaccounted properly. + */ + put_dmabuf_info(current->dmabuf_info); + current->dmabuf_info = NULL; /* * We used to share our file descriptor table, and have now * created a private one, make sure we're using it below. @@ -821,6 +844,8 @@ void do_close_on_exec(struct files_struct *files) rcu_assign_pointer(fdt->fd[fd], NULL); __put_unused_fd(files, fd); spin_unlock(&files->file_lock); + if (is_dma_buf_file(file)) + dma_buf_unaccount_task(file->private_data, files->dmabuf_info); filp_close(file, files); cond_resched(); spin_lock(&files->file_lock); @@ -1114,8 +1139,11 @@ __releases(&files->file_lock) __clear_close_on_exec(fd, fdt); spin_unlock(&files->file_lock); - if (tofree) + if (tofree) { + if (is_dma_buf_file(tofree)) + dma_buf_unaccount_task(tofree->private_data, files->dmabuf_info); filp_close(tofree, files); + } return fd; @@ -1139,7 +1167,10 @@ int replace_fd(unsigned fd, struct file *file, unsigned flags) err = expand_files(files, fd); if (unlikely(err < 0)) goto out_unlock; - return do_dup2(files, file, fd, flags); + err = do_dup2(files, file, fd, flags); + if (err < 0) + return err; + return 0; out_unlock: spin_unlock(&files->file_lock); diff --git a/fs/filesystems.c b/fs/filesystems.c index 58b9067b2391c..95e5256821a53 100644 --- a/fs/filesystems.c +++ b/fs/filesystems.c @@ -156,15 +156,19 @@ static int fs_index(const char __user * __name) static int fs_name(unsigned int index, char __user * buf) { struct file_system_type * tmp; - int len, res; + int len, res = -EINVAL; read_lock(&file_systems_lock); - for (tmp = file_systems; tmp; tmp = tmp->next, index--) - if (index <= 0 && try_module_get(tmp->owner)) + for (tmp = file_systems; tmp; tmp = tmp->next, index--) { + if (index == 0) { + if (try_module_get(tmp->owner)) + res = 0; break; + } + } read_unlock(&file_systems_lock); - if (!tmp) - return -EINVAL; + if (res) + return res; /* OK, we got the reference, so we can safely block */ len = strlen(tmp->name) + 1; diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 3c9a6d48c5a87..3466b219f4ba1 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -422,22 +422,23 @@ static bool inode_do_switch_wbs(struct inode *inode, * Transfer to @new_wb's IO list if necessary. If the @inode is dirty, * the specific list @inode was on is ignored and the @inode is put on * ->b_dirty which is always correct including from ->b_dirty_time. - * The transfer preserves @inode->dirtied_when ordering. If the @inode - * was clean, it means it was on the b_attached list, so move it onto - * the b_attached list of @new_wb. + * If the @inode was clean, it means it was on the b_attached list, so + * move it onto the b_attached list of @new_wb. */ if (!list_empty(&inode->i_io_list)) { inode->i_wb = new_wb; if (inode->i_state & I_DIRTY_ALL) { - struct inode *pos; - - list_for_each_entry(pos, &new_wb->b_dirty, i_io_list) - if (time_after_eq(inode->dirtied_when, - pos->dirtied_when)) - break; + /* + * We need to keep b_dirty list sorted by + * dirtied_time_when. However properly sorting the + * inode in the list gets too expensive when switching + * many inodes. So just attach inode at the end of the + * dirty list and clobber the dirtied_time_when. + */ + inode->dirtied_time_when = jiffies; inode_io_list_move_locked(inode, new_wb, - pos->i_io_list.prev); + &new_wb->b_dirty); } else { inode_cgwb_move_to_attached(inode, new_wb); } @@ -479,6 +480,7 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) */ down_read(&bdi->wb_switch_rwsem); + inodep = isw->inodes; /* * By the time control reaches here, RCU grace period has passed * since I_WB_SWITCH assertion and all wb stat update transactions @@ -489,6 +491,7 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) * gives us exclusion against all wb related operations on @inode * including IO list manipulations and stat updates. */ +relock: if (old_wb < new_wb) { spin_lock(&old_wb->list_lock); spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING); @@ -497,10 +500,17 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); } - for (inodep = isw->inodes; *inodep; inodep++) { + while (*inodep) { WARN_ON_ONCE((*inodep)->i_wb != old_wb); if (inode_do_switch_wbs(*inodep, old_wb, new_wb)) nr_switched++; + inodep++; + if (*inodep && need_resched()) { + spin_unlock(&new_wb->list_lock); + spin_unlock(&old_wb->list_lock); + cond_resched(); + goto relock; + } } spin_unlock(&new_wb->list_lock); @@ -2536,10 +2546,6 @@ void __mark_inode_dirty(struct inode *inode, int flags) wakeup_bdi = inode_io_list_move_locked(inode, wb, dirty_list); - spin_unlock(&wb->list_lock); - spin_unlock(&inode->i_lock); - trace_writeback_dirty_inode_enqueue(inode); - /* * If this is the first dirty inode for this bdi, * we have to wake-up the corresponding bdi thread @@ -2549,6 +2555,11 @@ void __mark_inode_dirty(struct inode *inode, int flags) if (wakeup_bdi && (wb->bdi->capabilities & BDI_CAP_WRITEBACK)) wb_wakeup_delayed(wb); + + spin_unlock(&wb->list_lock); + spin_unlock(&inode->i_lock); + trace_writeback_dirty_inode_enqueue(inode); + return; } } diff --git a/fs/fsopen.c b/fs/fsopen.c index ce03f6521c887..3576b1084ec2f 100644 --- a/fs/fsopen.c +++ b/fs/fsopen.c @@ -18,50 +18,56 @@ #include "internal.h" #include "mount.h" +static inline const char *fetch_message_locked(struct fc_log *log, size_t len, + bool *need_free) +{ + const char *p; + int index; + + if (unlikely(log->head == log->tail)) + return ERR_PTR(-ENODATA); + + index = log->tail & (ARRAY_SIZE(log->buffer) - 1); + p = log->buffer[index]; + if (unlikely(strlen(p) > len)) + return ERR_PTR(-EMSGSIZE); + + log->buffer[index] = NULL; + *need_free = log->need_free & (1 << index); + log->need_free &= ~(1 << index); + log->tail++; + + return p; +} + /* * Allow the user to read back any error, warning or informational messages. + * Only one message is returned for each read(2) call. */ static ssize_t fscontext_read(struct file *file, char __user *_buf, size_t len, loff_t *pos) { struct fs_context *fc = file->private_data; - struct fc_log *log = fc->log.log; - unsigned int logsize = ARRAY_SIZE(log->buffer); - ssize_t ret; - char *p; + ssize_t err; + const char *p __free(kfree) = NULL, *message; bool need_free; - int index, n; + int n; - ret = mutex_lock_interruptible(&fc->uapi_mutex); - if (ret < 0) - return ret; - - if (log->head == log->tail) { - mutex_unlock(&fc->uapi_mutex); - return -ENODATA; - } - - index = log->tail & (logsize - 1); - p = log->buffer[index]; - need_free = log->need_free & (1 << index); - log->buffer[index] = NULL; - log->need_free &= ~(1 << index); - log->tail++; + err = mutex_lock_interruptible(&fc->uapi_mutex); + if (err < 0) + return err; + message = fetch_message_locked(fc->log.log, len, &need_free); mutex_unlock(&fc->uapi_mutex); + if (IS_ERR(message)) + return PTR_ERR(message); - ret = -EMSGSIZE; - n = strlen(p); - if (n > len) - goto err_free; - ret = -EFAULT; - if (copy_to_user(_buf, p, n) != 0) - goto err_free; - ret = n; - -err_free: if (need_free) - kfree(p); - return ret; + p = message; + + n = strlen(message); + if (copy_to_user(_buf, message, n)) + return -EFAULT; + return n; } static int fscontext_release(struct inode *inode, struct file *file) diff --git a/fs/fuse/backing.c b/fs/fuse/backing.c index 21fb9cdd20aa3..68f820d5188cb 100644 --- a/fs/fuse/backing.c +++ b/fs/fuse/backing.c @@ -406,23 +406,26 @@ int fuse_lseek_backing(struct fuse_bpf_args *fa, struct file *file, loff_t offse struct file *backing_file = fuse_file->backing_file; loff_t ret; - /* TODO: Handle changing of the file handle */ if (offset == 0) { if (whence == SEEK_CUR) { flo->offset = file->f_pos; - return flo->offset; + return 0; } if (whence == SEEK_SET) { flo->offset = vfs_setpos(file, 0, 0); - return flo->offset; + return 0; } } inode_lock(file->f_inode); backing_file->f_pos = file->f_pos; ret = vfs_llseek(backing_file, fli->offset, fli->whence); - flo->offset = ret; + + if (!IS_ERR(ERR_PTR(ret))) { + flo->offset = ret; + ret = 0; + } inode_unlock(file->f_inode); return ret; } @@ -2363,8 +2366,11 @@ static bool filldir(struct dir_context *ctx, const char *name, int namelen, return true; } -static int parse_dirfile(char *buf, size_t nbytes, struct dir_context *ctx) +static int parse_dirfile(char *buf, size_t nbytes, struct dir_context *ctx, + loff_t next_offset) { + char *buffstart = buf; + while (nbytes >= FUSE_NAME_OFFSET) { struct fuse_dirent *dirent = (struct fuse_dirent *) buf; size_t reclen = FUSE_DIRENT_SIZE(dirent); @@ -2378,12 +2384,18 @@ static int parse_dirfile(char *buf, size_t nbytes, struct dir_context *ctx) ctx->pos = dirent->off; if (!dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, - dirent->type)) - break; + dirent->type)) { + // If we can't make any progress, user buffer is too small + if (buf == buffstart) + return -EINVAL; + else + return 0; + } buf += reclen; nbytes -= reclen; } + ctx->pos = next_offset; return 0; } @@ -2430,13 +2442,12 @@ void *fuse_readdir_finalize(struct fuse_bpf_args *fa, struct file *backing_dir = ff->backing_file; int err = 0; - err = parse_dirfile(fa->out_args[1].value, fa->out_args[1].size, ctx); + err = parse_dirfile(fa->out_args[1].value, fa->out_args[1].size, ctx, fro->offset); *force_again = !!fro->again; if (*force_again && !*allow_force) err = -EINVAL; - ctx->pos = fro->offset; - backing_dir->f_pos = fro->offset; + backing_dir->f_pos = ctx->pos; free_page((unsigned long) fa->out_args[1].value); return ERR_PTR(err); diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index ba677377b226d..522bd0406bd1c 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1421,6 +1421,8 @@ static int fuse_link(struct dentry *entry, struct inode *newdir, else if (err == -EINTR) fuse_invalidate_attr(inode); + if (err == -ENOSYS) + err = -EPERM; return err; } @@ -2156,6 +2158,7 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, int err; bool trust_local_cmtime = is_wb; bool fault_blocked = false; + u64 attr_version; #ifdef CONFIG_FUSE_BPF struct fuse_err_ret fer; @@ -2250,6 +2253,8 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, if (fc->handle_killpriv_v2 && !capable(CAP_FSETID)) inarg.valid |= FATTR_KILL_SUIDGID; } + + attr_version = fuse_get_attr_version(fm->fc); fuse_setattr_fill(fc, &args, inode, &inarg, &outarg); err = fuse_simple_request(fm, &args); if (err) { @@ -2275,6 +2280,14 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, /* FIXME: clear I_DIRTY_SYNC? */ } + if (fi->attr_version > attr_version) { + /* + * Apply attributes, for example for fsnotify_change(), but set + * attribute timeout to zero. + */ + outarg.attr_valid = outarg.attr_valid_nsec = 0; + } + fuse_change_attributes_common(inode, &outarg.attr, NULL, ATTR_TIMEOUT(&outarg), fuse_get_cache_mask(inode)); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 98be790d26882..43854d5c5632d 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3292,7 +3292,7 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, .nodeid_out = ff_out->nodeid, .fh_out = ff_out->fh, .off_out = pos_out, - .len = len, + .len = min_t(size_t, len, UINT_MAX & PAGE_MASK), .flags = flags }; struct fuse_write_out outarg; @@ -3370,6 +3370,9 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, fc->no_copy_file_range = 1; err = -EOPNOTSUPP; } + if (!err && outarg.size > len) + err = -EIO; + if (err) goto out; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index baab5b6aebf8b..f60fca8ed4609 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1714,6 +1714,8 @@ int fuse_file_write_iter_backing(struct fuse_bpf_args *fa, struct kiocb *iocb, struct iov_iter *from); void *fuse_file_write_iter_finalize(struct fuse_bpf_args *fa, struct kiocb *iocb, struct iov_iter *from); +ssize_t fuse_splice_read_backing(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, unsigned long flags); ssize_t fuse_splice_read_backing(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned long flags); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 0ef81978dddb8..c1f634fc3142f 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -95,19 +95,11 @@ static struct inode *fuse_alloc_inode(struct super_block *sb) if (!fi) return NULL; - fi->i_time = 0; - fi->inval_mask = ~0; -#ifdef CONFIG_FUSE_BPF - fi->backing_inode = NULL; - fi->bpf = NULL; -#endif + /* Initialize private data (i.e. everything except fi->inode) */ + BUILD_BUG_ON(offsetof(struct fuse_inode, inode) != 0); + memset((void *) fi + sizeof(fi->inode), 0, sizeof(*fi) - sizeof(fi->inode)); - fi->nodeid = 0; - fi->nlookup = 0; - fi->attr_version = 0; - fi->orig_ino = 0; - fi->state = 0; - fi->submount_lookup = NULL; + fi->inval_mask = ~0; mutex_init(&fi->mutex); spin_lock_init(&fi->lock); fi->forget = fuse_alloc_forget(); diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 2c0908a302102..c4bc86c3535ba 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -811,8 +811,6 @@ __acquires(&gl->gl_lockref.lock) clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD); return; - } else { - clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags); } } @@ -853,11 +851,12 @@ static void run_queue(struct gfs2_glock *gl, const int nonblock) __releases(&gl->gl_lockref.lock) __acquires(&gl->gl_lockref.lock) { - struct gfs2_holder *gh = NULL; + struct gfs2_holder *gh; if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) return; + /* While a demote is in progress, the GLF_LOCK flag must be set. */ GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)); if (test_bit(GLF_DEMOTE, &gl->gl_flags) && @@ -869,18 +868,22 @@ __acquires(&gl->gl_lockref.lock) set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE); gl->gl_target = gl->gl_demote_state; + do_xmote(gl, NULL, gl->gl_target); + return; } else { if (test_bit(GLF_DEMOTE, &gl->gl_flags)) gfs2_demote_wake(gl); if (do_promote(gl)) goto out_unlock; gh = find_first_waiter(gl); + if (!gh) + goto out_unlock; gl->gl_target = gh->gh_state; if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) do_error(gl, 0); /* Fail queued try locks */ + do_xmote(gl, gh, gl->gl_target); + return; } - do_xmote(gl, gh, gl->gl_target); - return; out_sched: clear_bit(GLF_LOCK, &gl->gl_flags); diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 29085643ad104..1cb5ce63fbf69 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -658,7 +658,8 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, if (!IS_ERR(inode)) { if (S_ISDIR(inode->i_mode)) { iput(inode); - inode = ERR_PTR(-EISDIR); + inode = NULL; + error = -EISDIR; goto fail_gunlock; } d_instantiate(dentry, inode); diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c index e028e55e67d95..07aac73377d8f 100644 --- a/fs/gfs2/lock_dlm.c +++ b/fs/gfs2/lock_dlm.c @@ -959,14 +959,15 @@ static int control_mount(struct gfs2_sbd *sdp) if (sdp->sd_args.ar_spectator) { fs_info(sdp, "Recovery is required. Waiting for a " "non-spectator to mount.\n"); + spin_unlock(&ls->ls_recover_spin); msleep_interruptible(1000); } else { fs_info(sdp, "control_mount wait1 block %u start %u " "mount %u lvb %u flags %lx\n", block_gen, start_gen, mount_gen, lvb_gen, ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); } - spin_unlock(&ls->ls_recover_spin); goto restart; } diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index 1f42eae112fb8..b1a368fc089f3 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -93,6 +93,7 @@ const struct address_space_operations gfs2_meta_aops = { .invalidate_folio = block_invalidate_folio, .writepage = gfs2_aspace_writepage, .release_folio = gfs2_release_folio, + .migrate_folio = buffer_migrate_folio_norefs, }; const struct address_space_operations gfs2_rgrp_aops = { @@ -100,6 +101,7 @@ const struct address_space_operations gfs2_rgrp_aops = { .invalidate_folio = block_invalidate_folio, .writepage = gfs2_aspace_writepage, .release_folio = gfs2_release_folio, + .migrate_folio = buffer_migrate_folio_norefs, }; /** diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c index 268ff47b03963..fd921e07e12b2 100644 --- a/fs/gfs2/util.c +++ b/fs/gfs2/util.c @@ -232,32 +232,23 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) */ ret = gfs2_glock_nq(&sdp->sd_live_gh); + gfs2_glock_put(live_gl); /* drop extra reference we acquired */ + clear_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags); + /* * If we actually got the "live" lock in EX mode, there are no other - * nodes available to replay our journal. So we try to replay it - * ourselves. We hold the "live" glock to prevent other mounters - * during recovery, then just dequeue it and reacquire it in our - * normal SH mode. Just in case the problem that caused us to - * withdraw prevents us from recovering our journal (e.g. io errors - * and such) we still check if the journal is clean before proceeding - * but we may wait forever until another mounter does the recovery. + * nodes available to replay our journal. */ if (ret == 0) { - fs_warn(sdp, "No other mounters found. Trying to recover our " - "own journal jid %d.\n", sdp->sd_lockstruct.ls_jid); - if (gfs2_recover_journal(sdp->sd_jdesc, 1)) - fs_warn(sdp, "Unable to recover our journal jid %d.\n", - sdp->sd_lockstruct.ls_jid); - gfs2_glock_dq_wait(&sdp->sd_live_gh); - gfs2_holder_reinit(LM_ST_SHARED, - LM_FLAG_NOEXP | GL_EXACT | GL_NOPID, - &sdp->sd_live_gh); - gfs2_glock_nq(&sdp->sd_live_gh); + fs_warn(sdp, "No other mounters found.\n"); + /* + * We are about to release the lockspace. By keeping live_gl + * locked here, we ensure that the next mounter coming along + * will be a "first" mounter which will perform recovery. + */ + goto skip_recovery; } - gfs2_glock_put(live_gl); /* drop extra reference we acquired */ - clear_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags); - /* * At this point our journal is evicted, so we need to get a new inode * for it. Once done, we need to call gfs2_find_jhead which diff --git a/fs/hfs/bfind.c b/fs/hfs/bfind.c index ef9498a6e88ac..e46f650b5e9c2 100644 --- a/fs/hfs/bfind.c +++ b/fs/hfs/bfind.c @@ -16,9 +16,12 @@ int hfs_find_init(struct hfs_btree *tree, struct hfs_find_data *fd) { void *ptr; + if (!tree || !fd) + return -EINVAL; + fd->tree = tree; fd->bnode = NULL; - ptr = kmalloc(tree->max_key_len * 2 + 4, GFP_KERNEL); + ptr = kzalloc(tree->max_key_len * 2 + 4, GFP_KERNEL); if (!ptr) return -ENOMEM; fd->search_key = ptr; @@ -112,6 +115,12 @@ int hfs_brec_find(struct hfs_find_data *fd) __be32 data; int height, res; + fd->record = -1; + fd->keyoffset = -1; + fd->keylength = -1; + fd->entryoffset = -1; + fd->entrylength = -1; + tree = fd->tree; if (fd->bnode) hfs_bnode_put(fd->bnode); diff --git a/fs/hfs/bnode.c b/fs/hfs/bnode.c index cb823a8a6ba96..e8cd1a31f2470 100644 --- a/fs/hfs/bnode.c +++ b/fs/hfs/bnode.c @@ -15,6 +15,48 @@ #include "btree.h" +static inline +bool is_bnode_offset_valid(struct hfs_bnode *node, int off) +{ + bool is_valid = off < node->tree->node_size; + + if (!is_valid) { + pr_err("requested invalid offset: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d\n", + node->this, node->type, node->height, + node->tree->node_size, off); + } + + return is_valid; +} + +static inline +int check_and_correct_requested_length(struct hfs_bnode *node, int off, int len) +{ + unsigned int node_size; + + if (!is_bnode_offset_valid(node, off)) + return 0; + + node_size = node->tree->node_size; + + if ((off + len) > node_size) { + int new_len = (int)node_size - off; + + pr_err("requested length has been corrected: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, " + "requested_len %d, corrected_len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len, new_len); + + return new_len; + } + + return len; +} + void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len) { struct page *page; @@ -22,6 +64,20 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len) int bytes_read; int bytes_to_read; + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + off += node->page_offset; pagenum = off >> PAGE_SHIFT; off &= ~PAGE_MASK; /* compute page offset for the first page */ @@ -80,6 +136,20 @@ void hfs_bnode_write(struct hfs_bnode *node, void *buf, int off, int len) { struct page *page; + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + off += node->page_offset; page = node->page[0]; @@ -104,6 +174,20 @@ void hfs_bnode_clear(struct hfs_bnode *node, int off, int len) { struct page *page; + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + off += node->page_offset; page = node->page[0]; @@ -119,6 +203,10 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst, hfs_dbg(BNODE_MOD, "copybytes: %u,%u,%u\n", dst, src, len); if (!len) return; + + len = check_and_correct_requested_length(src_node, src, len); + len = check_and_correct_requested_length(dst_node, dst, len); + src += src_node->page_offset; dst += dst_node->page_offset; src_page = src_node->page[0]; @@ -136,6 +224,10 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) hfs_dbg(BNODE_MOD, "movebytes: %u,%u,%u\n", dst, src, len); if (!len) return; + + len = check_and_correct_requested_length(node, src, len); + len = check_and_correct_requested_length(node, dst, len); + src += node->page_offset; dst += node->page_offset; page = node->page[0]; @@ -482,6 +574,7 @@ void hfs_bnode_put(struct hfs_bnode *node) if (test_bit(HFS_BNODE_DELETED, &node->flags)) { hfs_bnode_unhash(node); spin_unlock(&tree->hash_lock); + hfs_bnode_clear(node, 0, tree->node_size); hfs_bmap_free(node); hfs_bnode_free(node); return; diff --git a/fs/hfs/brec.c b/fs/hfs/brec.c index 896396554bcc1..b01db1fae147c 100644 --- a/fs/hfs/brec.c +++ b/fs/hfs/brec.c @@ -179,6 +179,7 @@ int hfs_brec_remove(struct hfs_find_data *fd) struct hfs_btree *tree; struct hfs_bnode *node, *parent; int end_off, rec_off, data_off, size; + int src, dst, len; tree = fd->tree; node = fd->bnode; @@ -208,10 +209,14 @@ int hfs_brec_remove(struct hfs_find_data *fd) } hfs_bnode_write_u16(node, offsetof(struct hfs_bnode_desc, num_recs), node->num_recs); - if (rec_off == end_off) - goto skip; size = fd->keylength + fd->entrylength; + if (rec_off == end_off) { + src = fd->keyoffset; + hfs_bnode_clear(node, src, size); + goto skip; + } + do { data_off = hfs_bnode_read_u16(node, rec_off); hfs_bnode_write_u16(node, rec_off + 2, data_off - size); @@ -219,9 +224,23 @@ int hfs_brec_remove(struct hfs_find_data *fd) } while (rec_off >= end_off); /* fill hole */ - hfs_bnode_move(node, fd->keyoffset, fd->keyoffset + size, - data_off - fd->keyoffset - size); + dst = fd->keyoffset; + src = fd->keyoffset + size; + len = data_off - src; + + hfs_bnode_move(node, dst, src, len); + + src = dst + len; + len = data_off - src; + + hfs_bnode_clear(node, src, len); + skip: + /* + * Remove the obsolete offset to free space. + */ + hfs_bnode_write_u16(node, end_off, 0); + hfs_bnode_dump(node); if (!fd->record) hfs_brec_update_parent(fd); diff --git a/fs/hfs/btree.c b/fs/hfs/btree.c index 2fa4b1f8cc7fb..e86e1e235658f 100644 --- a/fs/hfs/btree.c +++ b/fs/hfs/btree.c @@ -21,8 +21,12 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, btree_keycmp ke struct hfs_btree *tree; struct hfs_btree_header_rec *head; struct address_space *mapping; - struct page *page; + struct folio *folio; + struct buffer_head *bh; unsigned int size; + u16 dblock; + sector_t start_block; + loff_t offset; tree = kzalloc(sizeof(*tree), GFP_KERNEL); if (!tree) @@ -75,12 +79,40 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, btree_keycmp ke unlock_new_inode(tree->inode); mapping = tree->inode->i_mapping; - page = read_mapping_page(mapping, 0, NULL); - if (IS_ERR(page)) + folio = filemap_grab_folio(mapping, 0); + if (IS_ERR(folio)) goto free_inode; + folio_zero_range(folio, 0, folio_size(folio)); + + dblock = hfs_ext_find_block(HFS_I(tree->inode)->first_extents, 0); + start_block = HFS_SB(sb)->fs_start + (dblock * HFS_SB(sb)->fs_div); + + size = folio_size(folio); + offset = 0; + while (size > 0) { + size_t len; + + bh = sb_bread(sb, start_block); + if (!bh) { + pr_err("unable to read tree header\n"); + goto put_folio; + } + + len = min_t(size_t, folio_size(folio), sb->s_blocksize); + memcpy_to_folio(folio, offset, bh->b_data, sb->s_blocksize); + + brelse(bh); + + start_block++; + offset += len; + size -= len; + } + + folio_mark_uptodate(folio); + /* Load the header */ - head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + + head = (struct hfs_btree_header_rec *)(kmap_local_folio(folio, 0) + sizeof(struct hfs_bnode_desc)); tree->root = be32_to_cpu(head->root); tree->leaf_count = be32_to_cpu(head->leaf_count); @@ -95,22 +127,22 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, btree_keycmp ke size = tree->node_size; if (!is_power_of_2(size)) - goto fail_page; + goto fail_folio; if (!tree->node_count) - goto fail_page; + goto fail_folio; switch (id) { case HFS_EXT_CNID: if (tree->max_key_len != HFS_MAX_EXT_KEYLEN) { pr_err("invalid extent max_key_len %d\n", tree->max_key_len); - goto fail_page; + goto fail_folio; } break; case HFS_CAT_CNID: if (tree->max_key_len != HFS_MAX_CAT_KEYLEN) { pr_err("invalid catalog max_key_len %d\n", tree->max_key_len); - goto fail_page; + goto fail_folio; } break; default: @@ -121,12 +153,15 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, btree_keycmp ke tree->pages_per_bnode = (tree->node_size + PAGE_SIZE - 1) >> PAGE_SHIFT; kunmap_local(head); - put_page(page); + folio_unlock(folio); + folio_put(folio); return tree; -fail_page: +fail_folio: kunmap_local(head); - put_page(page); +put_folio: + folio_unlock(folio); + folio_put(folio); free_inode: tree->inode->i_mapping->a_ops = &hfs_aops; iput(tree->inode); diff --git a/fs/hfs/extent.c b/fs/hfs/extent.c index 6d1878b99b305..941c92525815e 100644 --- a/fs/hfs/extent.c +++ b/fs/hfs/extent.c @@ -71,7 +71,7 @@ int hfs_ext_keycmp(const btree_key *key1, const btree_key *key2) * * Find a block within an extent record */ -static u16 hfs_ext_find_block(struct hfs_extent *ext, u16 off) +u16 hfs_ext_find_block(struct hfs_extent *ext, u16 off) { int i; u16 count; diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h index 49d02524e6671..f1402d71b092b 100644 --- a/fs/hfs/hfs_fs.h +++ b/fs/hfs/hfs_fs.h @@ -190,6 +190,7 @@ extern const struct inode_operations hfs_dir_inode_operations; /* extent.c */ extern int hfs_ext_keycmp(const btree_key *, const btree_key *); +extern u16 hfs_ext_find_block(struct hfs_extent *ext, u16 off); extern int hfs_free_fork(struct super_block *, struct hfs_cat_file *, int); extern int hfs_ext_write_extent(struct inode *); extern int hfs_extend_file(struct inode *); diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index 61ed76d103927..6a89f22d89677 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -697,6 +697,7 @@ static const struct file_operations hfs_file_operations = { .write_iter = generic_file_write_iter, .mmap = generic_file_mmap, .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, .fsync = hfs_file_fsync, .open = hfs_file_open, .release = hfs_file_release, diff --git a/fs/hfs/mdb.c b/fs/hfs/mdb.c index 8082eb01127cd..bf811347bb07d 100644 --- a/fs/hfs/mdb.c +++ b/fs/hfs/mdb.c @@ -172,7 +172,7 @@ int hfs_mdb_get(struct super_block *sb) pr_warn("continuing without an alternate MDB\n"); } - HFS_SB(sb)->bitmap = kmalloc(8192, GFP_KERNEL); + HFS_SB(sb)->bitmap = kzalloc(8192, GFP_KERNEL); if (!HFS_SB(sb)->bitmap) goto out; diff --git a/fs/hfsplus/bfind.c b/fs/hfsplus/bfind.c index 901e83d65d202..26ebac4c60424 100644 --- a/fs/hfsplus/bfind.c +++ b/fs/hfsplus/bfind.c @@ -18,7 +18,7 @@ int hfs_find_init(struct hfs_btree *tree, struct hfs_find_data *fd) fd->tree = tree; fd->bnode = NULL; - ptr = kmalloc(tree->max_key_len * 2 + 4, GFP_KERNEL); + ptr = kzalloc(tree->max_key_len * 2 + 4, GFP_KERNEL); if (!ptr) return -ENOMEM; fd->search_key = ptr; @@ -158,6 +158,12 @@ int hfs_brec_find(struct hfs_find_data *fd, search_strategy_t do_key_compare) __be32 data; int height, res; + fd->record = -1; + fd->keyoffset = -1; + fd->keylength = -1; + fd->entryoffset = -1; + fd->entrylength = -1; + tree = fd->tree; if (fd->bnode) hfs_bnode_put(fd->bnode); diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index 079ea80534f7d..407d5152eb411 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -18,12 +18,27 @@ #include "hfsplus_fs.h" #include "hfsplus_raw.h" + /* Copy a specified range of bytes from the raw data of a node */ void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len) { struct page **pagep; int l; + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); off &= ~PAGE_MASK; @@ -81,6 +96,20 @@ void hfs_bnode_write(struct hfs_bnode *node, void *buf, int off, int len) struct page **pagep; int l; + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); off &= ~PAGE_MASK; @@ -109,6 +138,20 @@ void hfs_bnode_clear(struct hfs_bnode *node, int off, int len) struct page **pagep; int l; + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); off &= ~PAGE_MASK; @@ -133,6 +176,10 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst, hfs_dbg(BNODE_MOD, "copybytes: %u,%u,%u\n", dst, src, len); if (!len) return; + + len = check_and_correct_requested_length(src_node, src, len); + len = check_and_correct_requested_length(dst_node, dst, len); + src += src_node->page_offset; dst += dst_node->page_offset; src_page = src_node->page + (src >> PAGE_SHIFT); @@ -187,6 +234,10 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) hfs_dbg(BNODE_MOD, "movebytes: %u,%u,%u\n", dst, src, len); if (!len) return; + + len = check_and_correct_requested_length(node, src, len); + len = check_and_correct_requested_length(node, dst, len); + src += node->page_offset; dst += node->page_offset; if (dst > src) { diff --git a/fs/hfsplus/btree.c b/fs/hfsplus/btree.c index 9e1732a2b92a8..fe6a54c4083c3 100644 --- a/fs/hfsplus/btree.c +++ b/fs/hfsplus/btree.c @@ -393,6 +393,12 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) len = hfs_brec_lenoff(node, 2, &off16); off = off16; + if (!is_bnode_offset_valid(node, off)) { + hfs_bnode_put(node); + return ERR_PTR(-EIO); + } + len = check_and_correct_requested_length(node, off, len); + off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); data = kmap_local_page(*pagep); diff --git a/fs/hfsplus/extents.c b/fs/hfsplus/extents.c index 9c51867dddc51..f6ac98b0d44ff 100644 --- a/fs/hfsplus/extents.c +++ b/fs/hfsplus/extents.c @@ -342,9 +342,6 @@ static int hfsplus_free_extents(struct super_block *sb, int i; int err = 0; - /* Mapping the allocation file may lock the extent tree */ - WARN_ON(mutex_is_locked(&HFSPLUS_SB(sb)->ext_tree->tree_lock)); - hfsplus_dump_extent(extent); for (i = 0; i < 8; extent++, i++) { count = be32_to_cpu(extent->block_count); diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index 1473b04fc0f31..e67b35cb5ccc7 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -574,6 +574,48 @@ hfsplus_btree_lock_class(struct hfs_btree *tree) return class; } +static inline +bool is_bnode_offset_valid(struct hfs_bnode *node, int off) +{ + bool is_valid = off < node->tree->node_size; + + if (!is_valid) { + pr_err("requested invalid offset: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d\n", + node->this, node->type, node->height, + node->tree->node_size, off); + } + + return is_valid; +} + +static inline +int check_and_correct_requested_length(struct hfs_bnode *node, int off, int len) +{ + unsigned int node_size; + + if (!is_bnode_offset_valid(node, off)) + return 0; + + node_size = node->tree->node_size; + + if ((off + len) > node_size) { + int new_len = (int)node_size - off; + + pr_err("requested length has been corrected: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %d, " + "requested_len %d, corrected_len %d\n", + node->this, node->type, node->height, + node->tree->node_size, off, len, new_len); + + return new_len; + } + + return len; +} + /* compatibility */ #define hfsp_mt2ut(t) (struct timespec64){ .tv_sec = __hfsp_mt2ut(t) } #define hfsp_ut2mt(t) __hfsp_ut2mt((t).tv_sec) diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index c65c8c4b03dd3..73ff191ff2adc 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -373,6 +373,7 @@ static const struct file_operations hfsplus_file_operations = { .write_iter = generic_file_write_iter, .mmap = generic_file_mmap, .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, .fsync = hfsplus_file_fsync, .open = hfsplus_file_open, .release = hfsplus_file_release, diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 548dd322ed6c3..3626938d09b58 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -67,13 +67,26 @@ struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino) if (!(inode->i_state & I_NEW)) return inode; - INIT_LIST_HEAD(&HFSPLUS_I(inode)->open_dir_list); - spin_lock_init(&HFSPLUS_I(inode)->open_dir_lock); - mutex_init(&HFSPLUS_I(inode)->extents_lock); - HFSPLUS_I(inode)->flags = 0; + atomic_set(&HFSPLUS_I(inode)->opencnt, 0); + HFSPLUS_I(inode)->first_blocks = 0; + HFSPLUS_I(inode)->clump_blocks = 0; + HFSPLUS_I(inode)->alloc_blocks = 0; + HFSPLUS_I(inode)->cached_start = U32_MAX; + HFSPLUS_I(inode)->cached_blocks = 0; + memset(HFSPLUS_I(inode)->first_extents, 0, sizeof(hfsplus_extent_rec)); + memset(HFSPLUS_I(inode)->cached_extents, 0, sizeof(hfsplus_extent_rec)); HFSPLUS_I(inode)->extent_state = 0; + mutex_init(&HFSPLUS_I(inode)->extents_lock); HFSPLUS_I(inode)->rsrc_inode = NULL; - atomic_set(&HFSPLUS_I(inode)->opencnt, 0); + HFSPLUS_I(inode)->create_date = 0; + HFSPLUS_I(inode)->linkid = 0; + HFSPLUS_I(inode)->flags = 0; + HFSPLUS_I(inode)->fs_blocks = 0; + HFSPLUS_I(inode)->userflags = 0; + HFSPLUS_I(inode)->subfolders = 0; + INIT_LIST_HEAD(&HFSPLUS_I(inode)->open_dir_list); + spin_lock_init(&HFSPLUS_I(inode)->open_dir_lock); + HFSPLUS_I(inode)->phys_size = 0; if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID || inode->i_ino == HFSPLUS_ROOT_CNID) { @@ -525,7 +538,7 @@ static int hfsplus_fill_super(struct super_block *sb, void *data, int silent) if (!hfs_brec_read(&fd, &entry, sizeof(entry))) { hfs_find_exit(&fd); if (entry.type != cpu_to_be16(HFSPLUS_FOLDER)) { - err = -EINVAL; + err = -EIO; goto out_put_root; } inode = hfsplus_iget(sb, be32_to_cpu(entry.folder.id)); diff --git a/fs/hfsplus/unicode.c b/fs/hfsplus/unicode.c index 73342c925a4b6..ebd326799f35a 100644 --- a/fs/hfsplus/unicode.c +++ b/fs/hfsplus/unicode.c @@ -40,6 +40,18 @@ int hfsplus_strcasecmp(const struct hfsplus_unistr *s1, p1 = s1->unicode; p2 = s2->unicode; + if (len1 > HFSPLUS_MAX_STRLEN) { + len1 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s1->length), len1); + } + + if (len2 > HFSPLUS_MAX_STRLEN) { + len2 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s2->length), len2); + } + while (1) { c1 = c2 = 0; @@ -74,6 +86,18 @@ int hfsplus_strcmp(const struct hfsplus_unistr *s1, p1 = s1->unicode; p2 = s2->unicode; + if (len1 > HFSPLUS_MAX_STRLEN) { + len1 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s1->length), len1); + } + + if (len2 > HFSPLUS_MAX_STRLEN) { + len2 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s2->length), len2); + } + for (len = min(len1, len2); len > 0; len--) { c1 = be16_to_cpu(*p1); c2 = be16_to_cpu(*p2); @@ -132,7 +156,14 @@ int hfsplus_uni2asc(struct super_block *sb, op = astr; ip = ustr->unicode; + ustrlen = be16_to_cpu(ustr->length); + if (ustrlen > HFSPLUS_MAX_STRLEN) { + ustrlen = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(ustr->length), ustrlen); + } + len = *len_p; ce1 = NULL; compose = !test_bit(HFSPLUS_SB_NODECOMPOSE, &HFSPLUS_SB(sb)->flags); diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c index f7f9d0889df34..d5fd8e068486e 100644 --- a/fs/hfsplus/xattr.c +++ b/fs/hfsplus/xattr.c @@ -172,7 +172,11 @@ static int hfsplus_create_attributes_file(struct super_block *sb) return PTR_ERR(attr_file); } - BUG_ON(i_size_read(attr_file) != 0); + if (i_size_read(attr_file) != 0) { + err = -EIO; + pr_err("detected inconsistent attributes file, running fsck.hfsplus is recommended.\n"); + goto end_attr_file_creation; + } hip = HFSPLUS_I(attr_file); diff --git a/fs/hpfs/namei.c b/fs/hpfs/namei.c index f4eb8d6f59894..6cf17fe239c9a 100644 --- a/fs/hpfs/namei.c +++ b/fs/hpfs/namei.c @@ -52,8 +52,10 @@ static int hpfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, dee.fnode = cpu_to_le32(fno); dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); result = new_inode(dir->i_sb); - if (!result) + if (!result) { + err = -ENOMEM; goto bail2; + } hpfs_init_inode(result); result->i_ino = fno; hpfs_i(result)->i_parent_dir = dir->i_ino; @@ -153,9 +155,10 @@ static int hpfs_create(struct mnt_idmap *idmap, struct inode *dir, dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); result = new_inode(dir->i_sb); - if (!result) + if (!result) { + err = -ENOMEM; goto bail1; - + } hpfs_init_inode(result); result->i_ino = fno; result->i_mode |= S_IFREG; @@ -239,9 +242,10 @@ static int hpfs_mknod(struct mnt_idmap *idmap, struct inode *dir, dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); result = new_inode(dir->i_sb); - if (!result) + if (!result) { + err = -ENOMEM; goto bail1; - + } hpfs_init_inode(result); result->i_ino = fno; hpfs_i(result)->i_parent_dir = dir->i_ino; @@ -314,8 +318,10 @@ static int hpfs_symlink(struct mnt_idmap *idmap, struct inode *dir, dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); result = new_inode(dir->i_sb); - if (!result) + if (!result) { + err = -ENOMEM; goto bail1; + } result->i_ino = fno; hpfs_init_inode(result); hpfs_i(result)->i_parent_dir = dir->i_ino; diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index ac519515ef6c0..914e850b9cbb7 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -136,7 +136,7 @@ static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) vm_flags_set(vma, VM_HUGETLB | VM_DONTEXPAND); vma->vm_ops = &hugetlb_vm_ops; - ret = seal_check_future_write(info->seals, vma); + ret = seal_check_write(info->seals, vma); if (ret) return ret; @@ -619,14 +619,16 @@ static bool remove_inode_single_folio(struct hstate *h, struct inode *inode, /* * If folio is mapped, it was faulted in after being - * unmapped in caller. Unmap (again) while holding - * the fault mutex. The mutex will prevent faults - * until we finish removing the folio. + * unmapped in caller or hugetlb_vmdelete_list() skips + * unmapping it due to fail to grab lock. Unmap (again) + * while holding the fault mutex. The mutex will prevent + * faults until we finish removing the folio. Hold folio + * lock to guarantee no concurrent migration. */ + folio_lock(folio); if (unlikely(folio_mapped(folio))) hugetlb_unmap_file_folio(h, mapping, folio, index); - folio_lock(folio); /* * We must remove the folio from page cache before removing * the region/ reserve map (hugetlb_unreserve_pages). In diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index 0e69fa751c303..c87ac996da979 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -1486,9 +1486,16 @@ static int isofs_read_inode(struct inode *inode, int relocated) inode->i_op = &page_symlink_inode_operations; inode_nohighmem(inode); inode->i_data.a_ops = &isofs_symlink_aops; - } else + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { /* XXX - parse_rock_ridge_inode() had already set i_rdev. */ init_special_inode(inode, inode->i_mode, inode->i_rdev); + } else { + printk(KERN_DEBUG "ISOFS: Invalid file type 0%04o for inode %lu.\n", + inode->i_mode, inode->i_ino); + ret = -EIO; + goto fail; + } ret = 0; out: diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 8fda66c98a610..368ae50d8a59e 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -285,6 +285,7 @@ int jbd2_log_do_checkpoint(journal_t *journal) retry: if (batch_count) __flush_batch(journal, &batch_count); + cond_resched(); spin_lock(&journal->j_list_lock); goto restart; } diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c index 421c0d360836e..19ec325374833 100644 --- a/fs/jbd2/recovery.c +++ b/fs/jbd2/recovery.c @@ -286,21 +286,22 @@ static int fc_do_one_pass(journal_t *journal, int jbd2_journal_recover(journal_t *journal) { int err, err2; - journal_superblock_t * sb; - struct recovery_info info; errseq_t wb_err; struct address_space *mapping; memset(&info, 0, sizeof(info)); - sb = journal->j_superblock; /* * The journal superblock's s_start field (the current log head) * is always zero if, and only if, the journal was cleanly - * unmounted. + * unmounted. We use its in-memory version j_tail here because + * jbd2_journal_wipe() could have updated it without updating journal + * superblock. */ - if (!sb->s_start) { + if (!journal->j_tail) { + journal_superblock_t *sb = journal->j_superblock; + jbd2_debug(1, "No recovery required, last transaction %d, head block %u\n", be32_to_cpu(sb->s_sequence), be32_to_cpu(sb->s_head)); journal->j_transaction_sequence = be32_to_cpu(sb->s_sequence) + 1; diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c index 76adab83cac36..82bd29c170855 100644 --- a/fs/jbd2/transaction.c +++ b/fs/jbd2/transaction.c @@ -1499,7 +1499,7 @@ int jbd2_journal_dirty_metadata(handle_t *handle, struct buffer_head *bh) jh->b_next_transaction == transaction); spin_unlock(&jh->b_state_lock); } - if (jh->b_modified == 1) { + if (data_race(jh->b_modified == 1)) { /* If it's in our transaction it must be in BJ_Metadata list. */ if (data_race(jh->b_transaction == transaction && jh->b_jlist != BJ_Metadata)) { @@ -1518,7 +1518,6 @@ int jbd2_journal_dirty_metadata(handle_t *handle, struct buffer_head *bh) goto out; } - journal = transaction->t_journal; spin_lock(&jh->b_state_lock); if (is_handle_aborted(handle)) { @@ -1533,6 +1532,8 @@ int jbd2_journal_dirty_metadata(handle_t *handle, struct buffer_head *bh) goto out_unlock_bh; } + journal = transaction->t_journal; + if (jh->b_modified == 0) { /* * This buffer's got modified and becoming part @@ -1648,6 +1649,7 @@ int jbd2_journal_forget(handle_t *handle, struct buffer_head *bh) int drop_reserve = 0; int err = 0; int was_modified = 0; + int wait_for_writeback = 0; if (is_handle_aborted(handle)) return -EROFS; @@ -1771,18 +1773,22 @@ int jbd2_journal_forget(handle_t *handle, struct buffer_head *bh) } /* - * The buffer is still not written to disk, we should - * attach this buffer to current transaction so that the - * buffer can be checkpointed only after the current - * transaction commits. + * The buffer has not yet been written to disk. We should + * either clear the buffer or ensure that the ongoing I/O + * is completed, and attach this buffer to current + * transaction so that the buffer can be checkpointed only + * after the current transaction commits. */ clear_buffer_dirty(bh); + wait_for_writeback = 1; __jbd2_journal_file_buffer(jh, transaction, BJ_Forget); spin_unlock(&journal->j_list_lock); } drop: __brelse(bh); spin_unlock(&jh->b_state_lock); + if (wait_for_writeback) + wait_on_buffer(bh); jbd2_journal_put_journal_head(jh); if (drop_reserve) { /* no need to reserve log space for this block -bzzz */ diff --git a/fs/jffs2/erase.c b/fs/jffs2/erase.c index ef3a1e1b6cb06..fda9f4d6093f9 100644 --- a/fs/jffs2/erase.c +++ b/fs/jffs2/erase.c @@ -425,7 +425,9 @@ static void jffs2_mark_erased_block(struct jffs2_sb_info *c, struct jffs2_eraseb .totlen = cpu_to_je32(c->cleanmarker_size) }; - jffs2_prealloc_raw_node_refs(c, jeb, 1); + ret = jffs2_prealloc_raw_node_refs(c, jeb, 1); + if (ret) + goto filebad; marker.hdr_crc = cpu_to_je32(crc32(0, &marker, sizeof(struct jffs2_unknown_node)-4)); diff --git a/fs/jffs2/scan.c b/fs/jffs2/scan.c index 29671e33a1714..62879c218d4b1 100644 --- a/fs/jffs2/scan.c +++ b/fs/jffs2/scan.c @@ -256,7 +256,9 @@ int jffs2_scan_medium(struct jffs2_sb_info *c) jffs2_dbg(1, "%s(): Skipping %d bytes in nextblock to ensure page alignment\n", __func__, skip); - jffs2_prealloc_raw_node_refs(c, c->nextblock, 1); + ret = jffs2_prealloc_raw_node_refs(c, c->nextblock, 1); + if (ret) + goto out; jffs2_scan_dirty_space(c, c->nextblock, skip); } #endif diff --git a/fs/jffs2/summary.c b/fs/jffs2/summary.c index 4fe64519870f1..d83372d3e1a07 100644 --- a/fs/jffs2/summary.c +++ b/fs/jffs2/summary.c @@ -858,7 +858,10 @@ int jffs2_sum_write_sumnode(struct jffs2_sb_info *c) spin_unlock(&c->erase_completion_lock); jeb = c->nextblock; - jffs2_prealloc_raw_node_refs(c, jeb, 1); + ret = jffs2_prealloc_raw_node_refs(c, jeb, 1); + + if (ret) + goto out; if (!c->summary->sum_num || !c->summary->sum_list_head) { JFFS2_WARNING("Empty summary info!!!\n"); @@ -872,6 +875,8 @@ int jffs2_sum_write_sumnode(struct jffs2_sb_info *c) datasize += padsize; ret = jffs2_sum_write_data(c, jeb, infosize, datasize, padsize); + +out: spin_lock(&c->erase_completion_lock); return ret; } diff --git a/fs/jfs/file.c b/fs/jfs/file.c index 01b6912e60f80..742cadd1f37e8 100644 --- a/fs/jfs/file.c +++ b/fs/jfs/file.c @@ -44,6 +44,9 @@ static int jfs_open(struct inode *inode, struct file *file) { int rc; + if (S_ISREG(inode->i_mode) && inode->i_size < 0) + return -EIO; + if ((rc = dquot_file_open(inode, file))) return rc; diff --git a/fs/jfs/inode.c b/fs/jfs/inode.c index 920d58a1566bf..1e6c1d1a15a6a 100644 --- a/fs/jfs/inode.c +++ b/fs/jfs/inode.c @@ -59,9 +59,15 @@ struct inode *jfs_iget(struct super_block *sb, unsigned long ino) */ inode->i_link[inode->i_size] = '\0'; } - } else { + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { inode->i_op = &jfs_file_inode_operations; init_special_inode(inode, inode->i_mode, inode->i_rdev); + } else { + printk(KERN_DEBUG "JFS: Invalid file type 0%04o for inode %lu.\n", + inode->i_mode, inode->i_ino); + iget_failed(inode); + return ERR_PTR(-EIO); } unlock_new_inode(inode); return inode; @@ -145,9 +151,9 @@ void jfs_evict_inode(struct inode *inode) if (!inode->i_nlink && !is_bad_inode(inode)) { dquot_initialize(inode); + truncate_inode_pages_final(&inode->i_data); if (JFS_IP(inode)->fileset == FILESYSTEM_I) { struct inode *ipimap = JFS_SBI(inode->i_sb)->ipimap; - truncate_inode_pages_final(&inode->i_data); if (test_cflag(COMMIT_Freewmap, inode)) jfs_free_zero_link(inode); diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c index 0e1019382cf51..cdfa699cd7c8f 100644 --- a/fs/jfs/jfs_dmap.c +++ b/fs/jfs/jfs_dmap.c @@ -178,45 +178,30 @@ int dbMount(struct inode *ipbmap) dbmp_le = (struct dbmap_disk *) mp->data; bmp->db_mapsize = le64_to_cpu(dbmp_le->dn_mapsize); bmp->db_nfree = le64_to_cpu(dbmp_le->dn_nfree); - bmp->db_l2nbperpage = le32_to_cpu(dbmp_le->dn_l2nbperpage); - if (bmp->db_l2nbperpage > L2PSIZE - L2MINBLOCKSIZE || - bmp->db_l2nbperpage < 0) { - err = -EINVAL; - goto err_release_metapage; - } - bmp->db_numag = le32_to_cpu(dbmp_le->dn_numag); - if (!bmp->db_numag || bmp->db_numag > MAXAG) { - err = -EINVAL; - goto err_release_metapage; - } - bmp->db_maxlevel = le32_to_cpu(dbmp_le->dn_maxlevel); bmp->db_maxag = le32_to_cpu(dbmp_le->dn_maxag); bmp->db_agpref = le32_to_cpu(dbmp_le->dn_agpref); - if (bmp->db_maxag >= MAXAG || bmp->db_maxag < 0 || - bmp->db_agpref >= MAXAG || bmp->db_agpref < 0) { - err = -EINVAL; - goto err_release_metapage; - } - bmp->db_aglevel = le32_to_cpu(dbmp_le->dn_aglevel); bmp->db_agheight = le32_to_cpu(dbmp_le->dn_agheight); bmp->db_agwidth = le32_to_cpu(dbmp_le->dn_agwidth); - if (!bmp->db_agwidth) { - err = -EINVAL; - goto err_release_metapage; - } bmp->db_agstart = le32_to_cpu(dbmp_le->dn_agstart); bmp->db_agl2size = le32_to_cpu(dbmp_le->dn_agl2size); - if (bmp->db_agl2size > L2MAXL2SIZE - L2MAXAG || - bmp->db_agl2size < 0) { - err = -EINVAL; - goto err_release_metapage; - } - if (((bmp->db_mapsize - 1) >> bmp->db_agl2size) > MAXAG) { + if ((bmp->db_l2nbperpage > L2PSIZE - L2MINBLOCKSIZE) || + (bmp->db_l2nbperpage < 0) || + !bmp->db_numag || (bmp->db_numag > MAXAG) || + (bmp->db_maxag >= MAXAG) || (bmp->db_maxag < 0) || + (bmp->db_agpref >= MAXAG) || (bmp->db_agpref < 0) || + (bmp->db_agheight < 0) || (bmp->db_agheight > (L2LPERCTL >> 1)) || + (bmp->db_agwidth < 1) || (bmp->db_agwidth > (LPERCTL / MAXAG)) || + (bmp->db_agwidth > (1 << (L2LPERCTL - (bmp->db_agheight << 1)))) || + (bmp->db_agstart < 0) || + (bmp->db_agstart > (CTLTREESIZE - 1 - bmp->db_agwidth * (MAXAG - 1))) || + (bmp->db_agl2size > L2MAXL2SIZE - L2MAXAG) || + (bmp->db_agl2size < 0) || + ((bmp->db_mapsize - 1) >> bmp->db_agl2size) > MAXAG) { err = -EINVAL; goto err_release_metapage; } @@ -1404,6 +1389,12 @@ dbAllocAG(struct bmap * bmp, int agno, s64 nblocks, int l2nb, s64 * results) (1 << (L2LPERCTL - (bmp->db_agheight << 1))) / bmp->db_agwidth; ti = bmp->db_agstart + bmp->db_agwidth * (agno & (agperlev - 1)); + if (ti < 0 || ti >= le32_to_cpu(dcp->nleafs)) { + jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmapctl page\n"); + release_metapage(mp); + return -EIO; + } + /* dmap control page trees fan-out by 4 and a single allocation * group may be described by 1 or 2 subtrees within the ag level * dmap control page, depending upon the ag size. examine the ag's @@ -1824,8 +1815,10 @@ dbAllocCtl(struct bmap * bmp, s64 nblocks, int l2nb, s64 blkno, s64 * results) return -EIO; dp = (struct dmap *) mp->data; - if (dp->tree.budmin < 0) + if (dp->tree.budmin < 0) { + release_metapage(mp); return -EIO; + } /* try to allocate the blocks. */ diff --git a/fs/jfs/jfs_imap.c b/fs/jfs/jfs_imap.c index 9a6d504228e7c..35f3144d703b5 100644 --- a/fs/jfs/jfs_imap.c +++ b/fs/jfs/jfs_imap.c @@ -3029,14 +3029,23 @@ static void duplicateIXtree(struct super_block *sb, s64 blkno, * * RETURN VALUES: * 0 - success - * -ENOMEM - insufficient memory + * -EINVAL - unexpected inode type */ static int copy_from_dinode(struct dinode * dip, struct inode *ip) { struct jfs_inode_info *jfs_ip = JFS_IP(ip); struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + int fileset = le32_to_cpu(dip->di_fileset); + + switch (fileset) { + case AGGR_RESERVED_I: case AGGREGATE_I: case BMAP_I: + case LOG_I: case BADBLOCK_I: case FILESYSTEM_I: + break; + default: + return -EINVAL; + } - jfs_ip->fileset = le32_to_cpu(dip->di_fileset); + jfs_ip->fileset = fileset; jfs_ip->mode2 = le32_to_cpu(dip->di_mode); jfs_set_inode_flags(ip); diff --git a/fs/jfs/jfs_txnmgr.c b/fs/jfs/jfs_txnmgr.c index dccc8b3f10459..42fb833ef2834 100644 --- a/fs/jfs/jfs_txnmgr.c +++ b/fs/jfs/jfs_txnmgr.c @@ -272,14 +272,15 @@ int txInit(void) if (TxBlock == NULL) return -ENOMEM; - for (k = 1; k < nTxBlock - 1; k++) { - TxBlock[k].next = k + 1; + for (k = 0; k < nTxBlock; k++) { init_waitqueue_head(&TxBlock[k].gcwait); init_waitqueue_head(&TxBlock[k].waitor); } + + for (k = 1; k < nTxBlock - 1; k++) { + TxBlock[k].next = k + 1; + } TxBlock[k].next = 0; - init_waitqueue_head(&TxBlock[k].gcwait); - init_waitqueue_head(&TxBlock[k].waitor); TxAnchor.freetid = 1; init_waitqueue_head(&TxAnchor.freewait); diff --git a/fs/kernel_read_file.c b/fs/kernel_read_file.c index c429c42a68679..f6d74db373db8 100644 --- a/fs/kernel_read_file.c +++ b/fs/kernel_read_file.c @@ -187,3 +187,13 @@ ssize_t kernel_read_file_from_fd(int fd, loff_t offset, void **buf, return ret; } EXPORT_SYMBOL_GPL(kernel_read_file_from_fd); + +ssize_t read_comp_algo_dictionary(void **dict, const char *dict_path) +{ + return kernel_read_file_from_path(dict_path, 0, + dict, + INT_MAX, + NULL, + READING_POLICY); +} +EXPORT_SYMBOL_GPL(read_comp_algo_dictionary); diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index b068ed32d7b32..f6e2a4523f7e6 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -1560,8 +1560,9 @@ void kernfs_break_active_protection(struct kernfs_node *kn) * invoked before finishing the kernfs operation. Note that while this * function restores the active reference, it doesn't and can't actually * restore the active protection - @kn may already or be in the process of - * being removed. Once kernfs_break_active_protection() is invoked, that - * protection is irreversibly gone for the kernfs operation instance. + * being drained and removed. Once kernfs_break_active_protection() is + * invoked, that protection is irreversibly gone for the kernfs operation + * instance. * * While this function may be called at any point after * kernfs_break_active_protection() is invoked, its most useful location diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 332d08d2fe0d5..257ba5398387b 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -70,6 +70,24 @@ static struct kernfs_open_node *of_on(struct kernfs_open_file *of) !list_empty(&of->list)); } +/* Get active reference to kernfs node for an open file */ +static struct kernfs_open_file *kernfs_get_active_of(struct kernfs_open_file *of) +{ + /* Skip if file was already released */ + if (unlikely(of->released)) + return NULL; + + if (!kernfs_get_active(of->kn)) + return NULL; + + return of; +} + +static void kernfs_put_active_of(struct kernfs_open_file *of) +{ + return kernfs_put_active(of->kn); +} + /** * kernfs_deref_open_node_locked - Get kernfs_open_node corresponding to @kn * @@ -139,7 +157,7 @@ static void kernfs_seq_stop_active(struct seq_file *sf, void *v) if (ops->seq_stop) ops->seq_stop(sf, v); - kernfs_put_active(of->kn); + kernfs_put_active_of(of); } static void *kernfs_seq_start(struct seq_file *sf, loff_t *ppos) @@ -152,7 +170,7 @@ static void *kernfs_seq_start(struct seq_file *sf, loff_t *ppos) * the ops aren't called concurrently for the same open file. */ mutex_lock(&of->mutex); - if (!kernfs_get_active(of->kn)) + if (!kernfs_get_active_of(of)) return ERR_PTR(-ENODEV); ops = kernfs_ops(of->kn); @@ -238,7 +256,7 @@ static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) * the ops aren't called concurrently for the same open file. */ mutex_lock(&of->mutex); - if (!kernfs_get_active(of->kn)) { + if (!kernfs_get_active_of(of)) { len = -ENODEV; mutex_unlock(&of->mutex); goto out_free; @@ -252,7 +270,7 @@ static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) else len = -EINVAL; - kernfs_put_active(of->kn); + kernfs_put_active_of(of); mutex_unlock(&of->mutex); if (len < 0) @@ -323,7 +341,7 @@ static ssize_t kernfs_fop_write_iter(struct kiocb *iocb, struct iov_iter *iter) * the ops aren't called concurrently for the same open file. */ mutex_lock(&of->mutex); - if (!kernfs_get_active(of->kn)) { + if (!kernfs_get_active_of(of)) { mutex_unlock(&of->mutex); len = -ENODEV; goto out_free; @@ -335,7 +353,7 @@ static ssize_t kernfs_fop_write_iter(struct kiocb *iocb, struct iov_iter *iter) else len = -EINVAL; - kernfs_put_active(of->kn); + kernfs_put_active_of(of); mutex_unlock(&of->mutex); if (len > 0) @@ -357,13 +375,13 @@ static void kernfs_vma_open(struct vm_area_struct *vma) if (!of->vm_ops) return; - if (!kernfs_get_active(of->kn)) + if (!kernfs_get_active_of(of)) return; if (of->vm_ops->open) of->vm_ops->open(vma); - kernfs_put_active(of->kn); + kernfs_put_active_of(of); } static vm_fault_t kernfs_vma_fault(struct vm_fault *vmf) @@ -375,14 +393,14 @@ static vm_fault_t kernfs_vma_fault(struct vm_fault *vmf) if (!of->vm_ops) return VM_FAULT_SIGBUS; - if (!kernfs_get_active(of->kn)) + if (!kernfs_get_active_of(of)) return VM_FAULT_SIGBUS; ret = VM_FAULT_SIGBUS; if (of->vm_ops->fault) ret = of->vm_ops->fault(vmf); - kernfs_put_active(of->kn); + kernfs_put_active_of(of); return ret; } @@ -395,7 +413,7 @@ static vm_fault_t kernfs_vma_page_mkwrite(struct vm_fault *vmf) if (!of->vm_ops) return VM_FAULT_SIGBUS; - if (!kernfs_get_active(of->kn)) + if (!kernfs_get_active_of(of)) return VM_FAULT_SIGBUS; ret = 0; @@ -404,7 +422,7 @@ static vm_fault_t kernfs_vma_page_mkwrite(struct vm_fault *vmf) else file_update_time(file); - kernfs_put_active(of->kn); + kernfs_put_active_of(of); return ret; } @@ -418,14 +436,14 @@ static int kernfs_vma_access(struct vm_area_struct *vma, unsigned long addr, if (!of->vm_ops) return -EINVAL; - if (!kernfs_get_active(of->kn)) + if (!kernfs_get_active_of(of)) return -EINVAL; ret = -EINVAL; if (of->vm_ops->access) ret = of->vm_ops->access(vma, addr, buf, len, write); - kernfs_put_active(of->kn); + kernfs_put_active_of(of); return ret; } @@ -504,7 +522,7 @@ static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma) mutex_lock(&of->mutex); rc = -ENODEV; - if (!kernfs_get_active(of->kn)) + if (!kernfs_get_active_of(of)) goto out_unlock; ops = kernfs_ops(of->kn); @@ -539,7 +557,7 @@ static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma) } vma->vm_ops = &kernfs_vm_ops; out_put: - kernfs_put_active(of->kn); + kernfs_put_active_of(of); out_unlock: mutex_unlock(&of->mutex); @@ -820,8 +838,9 @@ bool kernfs_should_drain_open_files(struct kernfs_node *kn) /* * @kn being deactivated guarantees that @kn->attr.open can't change * beneath us making the lockless test below safe. + * Callers post kernfs_unbreak_active_protection may be counted in + * kn->active by now, do not WARN_ON because of them. */ - WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); rcu_read_lock(); on = rcu_dereference(kn->attr.open); @@ -893,7 +912,7 @@ static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait) struct kernfs_node *kn = kernfs_dentry_node(filp->f_path.dentry); __poll_t ret; - if (!kernfs_get_active(kn)) + if (!kernfs_get_active_of(of)) return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI; if (kn->attr.ops->poll) @@ -901,7 +920,7 @@ static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait) else ret = kernfs_generic_poll(of, wait); - kernfs_put_active(kn); + kernfs_put_active_of(of); return ret; } diff --git a/fs/libfs.c b/fs/libfs.c index b5fff0b28aa2a..c42f7d49d0f6e 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -610,7 +610,7 @@ void simple_recursive_removal(struct dentry *dentry, struct dentry *victim = NULL, *child; struct inode *inode = this->d_inode; - inode_lock(inode); + inode_lock_nested(inode, I_MUTEX_CHILD); if (d_is_dir(this)) inode->i_flags |= S_DEAD; while ((child = find_next_child(this, victim)) == NULL) { @@ -622,7 +622,7 @@ void simple_recursive_removal(struct dentry *dentry, victim = this; this = this->d_parent; inode = this->d_inode; - inode_lock(inode); + inode_lock_nested(inode, I_MUTEX_CHILD); if (simple_positive(victim)) { d_invalidate(victim); // avoid lost mounts if (d_is_dir(victim)) diff --git a/fs/minix/inode.c b/fs/minix/inode.c index a0e495acc73af..d3827682fa7fb 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -470,8 +470,14 @@ void minix_set_inode(struct inode *inode, dev_t rdev) inode->i_op = &minix_symlink_inode_operations; inode_nohighmem(inode); inode->i_mapping->a_ops = &minix_aops; - } else + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { init_special_inode(inode, inode->i_mode, rdev); + } else { + printk(KERN_DEBUG "MINIX-fs: Invalid file type 0%04o for inode %lu.\n", + inode->i_mode, inode->i_ino); + make_bad_inode(inode); + } } /* diff --git a/fs/namei.c b/fs/namei.c index 5f45d9b2f84de..b31b583f18789 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -1364,6 +1364,10 @@ static int follow_automount(struct path *path, int *count, unsigned lookup_flags dentry->d_inode) return -EISDIR; + /* No need to trigger automounts if mountpoint crossing is disabled. */ + if (lookup_flags & LOOKUP_NO_XDEV) + return -EXDEV; + if (count && (*count)++ >= MAXSYMLINKS) return -ELOOP; @@ -1387,6 +1391,10 @@ static int __traverse_mounts(struct path *path, unsigned flags, bool *jumped, /* Allow the filesystem to manage the transit without i_mutex * being held. */ if (flags & DCACHE_MANAGE_TRANSIT) { + if (lookup_flags & LOOKUP_NO_XDEV) { + ret = -EXDEV; + break; + } ret = path->dentry->d_op->d_manage(path, false); flags = smp_load_acquire(&path->dentry->d_flags); if (ret < 0) diff --git a/fs/namespace.c b/fs/namespace.c index 68e22ccc50c2d..97fc4227773ff 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -66,6 +66,15 @@ static int __init set_mphash_entries(char *str) } __setup("mphash_entries=", set_mphash_entries); +static char * __initdata initramfs_options; +static int __init initramfs_options_setup(char *str) +{ + initramfs_options = str; + return 1; +} + +__setup("initramfs_options=", initramfs_options_setup); + static u64 event; static DEFINE_IDA(mnt_id_ida); static DEFINE_IDA(mnt_group_ida); @@ -635,15 +644,11 @@ int __legitimize_mnt(struct vfsmount *bastard, unsigned seq) return 0; mnt = real_mount(bastard); mnt_add_count(mnt, 1); - smp_mb(); // see mntput_no_expire() + smp_mb(); // see mntput_no_expire() and do_umount() if (likely(!read_seqretry(&mount_lock, seq))) return 0; - if (bastard->mnt_flags & MNT_SYNC_UMOUNT) { - mnt_add_count(mnt, -1); - return 1; - } lock_mount_hash(); - if (unlikely(bastard->mnt_flags & MNT_DOOMED)) { + if (unlikely(bastard->mnt_flags & (MNT_SYNC_UMOUNT | MNT_DOOMED))) { mnt_add_count(mnt, -1); unlock_mount_hash(); return 1; @@ -1788,6 +1793,7 @@ static int do_umount(struct mount *mnt, int flags) umount_tree(mnt, UMOUNT_PROPAGATE); retval = 0; } else { + smp_mb(); // paired with __legitimize_mnt() shrink_submounts(mnt); retval = -EBUSY; if (!propagate_mount_busy(mnt, 2)) { @@ -2116,6 +2122,11 @@ struct vfsmount *clone_private_mount(const struct path *path) if (!check_mnt(old_mnt)) goto invalid; + if (!ns_capable(old_mnt->mnt_ns->user_ns, CAP_SYS_ADMIN)) { + up_read(&namespace_sem); + return ERR_PTR(-EPERM); + } + if (has_locked_children(old_mnt, path->dentry)) goto invalid; @@ -2370,14 +2381,14 @@ static int attach_recursive_mnt(struct mount *source_mnt, hlist_for_each_entry_safe(child, n, &tree_list, mnt_hash) { struct mount *q; hlist_del_init(&child->mnt_hash); - q = __lookup_mnt(&child->mnt_parent->mnt, - child->mnt_mountpoint); - if (q) - mnt_change_mountpoint(child, smp, q); /* Notice when we are propagating across user namespaces */ if (child->mnt_parent->mnt_ns->user_ns != user_ns) lock_mnt_tree(child); child->mnt.mnt_flags &= ~MNT_LOCKED; + q = __lookup_mnt(&child->mnt_parent->mnt, + child->mnt_mountpoint); + if (q) + mnt_change_mountpoint(child, smp, q); commit_tree(child); } put_mountpoint(smp); @@ -2527,6 +2538,19 @@ static int graft_tree(struct mount *mnt, struct mount *p, struct mountpoint *mp) return attach_recursive_mnt(mnt, p, mp, 0); } +static int may_change_propagation(const struct mount *m) +{ + struct mnt_namespace *ns = m->mnt_ns; + + // it must be mounted in some namespace + if (IS_ERR_OR_NULL(ns)) // is_mounted() + return -EINVAL; + // and the caller must be admin in userns of that namespace + if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) + return -EPERM; + return 0; +} + /* * Sanity check the flags to change_mnt_propagation. */ @@ -2563,6 +2587,10 @@ static int do_change_type(struct path *path, int ms_flags) return -EINVAL; namespace_lock(); + err = may_change_propagation(mnt); + if (err) + goto out_unlock; + if (type == MS_SHARED) { err = invent_group_ids(mnt, recurse); if (err) @@ -2957,18 +2985,11 @@ static int do_set_group(struct path *from_path, struct path *to_path) namespace_lock(); - err = -EINVAL; - /* To and From must be mounted */ - if (!is_mounted(&from->mnt)) - goto out; - if (!is_mounted(&to->mnt)) - goto out; - - err = -EPERM; - /* We should be allowed to modify mount namespaces of both mounts */ - if (!ns_capable(from->mnt_ns->user_ns, CAP_SYS_ADMIN)) + err = may_change_propagation(from); + if (err) goto out; - if (!ns_capable(to->mnt_ns->user_ns, CAP_SYS_ADMIN)) + err = may_change_propagation(to); + if (err) goto out; err = -EINVAL; @@ -3001,7 +3022,7 @@ static int do_set_group(struct path *from_path, struct path *to_path) if (IS_MNT_SLAVE(from)) { struct mount *m = from->mnt_master; - list_add(&to->mnt_slave, &m->mnt_slave_list); + list_add(&to->mnt_slave, &from->mnt_slave); to->mnt_master = m; } @@ -3026,18 +3047,25 @@ static int do_set_group(struct path *from_path, struct path *to_path) * Check if path is overmounted, i.e., if there's a mount on top of * @path->mnt with @path->dentry as mountpoint. * - * Context: This function expects namespace_lock() to be held. + * Context: namespace_sem must be held at least shared. + * MUST NOT be called under lock_mount_hash() (there one should just + * call __lookup_mnt() and check if it returns NULL). * Return: If path is overmounted true is returned, false if not. */ static inline bool path_overmounted(const struct path *path) { + unsigned seq = read_seqbegin(&mount_lock); + bool no_child; + rcu_read_lock(); - if (unlikely(__lookup_mnt(path->mnt, path->dentry))) { - rcu_read_unlock(); - return true; - } + no_child = !__lookup_mnt(path->mnt, path->dentry); rcu_read_unlock(); - return false; + if (need_seqretry(&mount_lock, seq)) { + read_seqlock_excl(&mount_lock); + no_child = !__lookup_mnt(path->mnt, path->dentry); + read_sequnlock_excl(&mount_lock); + } + return unlikely(!no_child); } /** @@ -4714,7 +4742,7 @@ static void __init init_mount_tree(void) struct mnt_namespace *ns; struct path root; - mnt = vfs_kern_mount(&rootfs_fs_type, 0, "rootfs", NULL); + mnt = vfs_kern_mount(&rootfs_fs_type, 0, "rootfs", initramfs_options); if (IS_ERR(mnt)) panic("Can't create rootfs"); diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index 6be13e0ec170d..e498aade8c479 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c @@ -149,8 +149,8 @@ do_add_page_to_bio(struct bio *bio, int npg, enum req_op op, sector_t isect, /* limit length to what the device mapping allows */ end = disk_addr + *len; - if (end >= map->start + map->len) - *len = map->start + map->len - disk_addr; + if (end >= map->disk_offset + map->len) + *len = map->disk_offset + map->len - disk_addr; retry: if (!bio) { diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index 65cbb5607a5fc..61ee0b6c0fba2 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -199,10 +199,11 @@ static bool bl_map_stripe(struct pnfs_block_dev *dev, u64 offset, struct pnfs_block_dev *child; u64 chunk; u32 chunk_idx; + u64 disk_chunk; u64 disk_offset; chunk = div_u64(offset, dev->chunk_size); - div_u64_rem(chunk, dev->nr_children, &chunk_idx); + disk_chunk = div_u64_rem(chunk, dev->nr_children, &chunk_idx); if (chunk_idx >= dev->nr_children) { dprintk("%s: invalid chunk idx %d (%lld/%lld)\n", @@ -215,7 +216,7 @@ static bool bl_map_stripe(struct pnfs_block_dev *dev, u64 offset, offset = chunk * dev->chunk_size; /* disk offset of the stripe */ - disk_offset = div_u64(offset, dev->nr_children); + disk_offset = disk_chunk * dev->chunk_size; child = &dev->children[chunk_idx]; child->map(child, disk_offset, map); diff --git a/fs/nfs/blocklayout/extent_tree.c b/fs/nfs/blocklayout/extent_tree.c index 8f7cff7a42938..0add0f329816b 100644 --- a/fs/nfs/blocklayout/extent_tree.c +++ b/fs/nfs/blocklayout/extent_tree.c @@ -552,6 +552,15 @@ static int ext_tree_encode_commit(struct pnfs_block_layout *bl, __be32 *p, return ret; } +/** + * ext_tree_prepare_commit - encode extents that need to be committed + * @arg: layout commit data + * + * Return values: + * %0: Success, all required extents are encoded + * %-ENOSPC: Some extents are encoded, but not all, due to RPC size limit + * %-ENOMEM: Out of memory, extents not encoded + */ int ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg) { @@ -568,12 +577,12 @@ ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg) start_p = page_address(arg->layoutupdate_page); arg->layoutupdate_pages = &arg->layoutupdate_page; -retry: - ret = ext_tree_encode_commit(bl, start_p + 1, buffer_size, &count, &arg->lastbytewritten); + ret = ext_tree_encode_commit(bl, start_p + 1, buffer_size, + &count, &arg->lastbytewritten); if (unlikely(ret)) { ext_tree_free_commitdata(arg, buffer_size); - buffer_size = ext_tree_layoutupdate_size(bl, count); + buffer_size = NFS_SERVER(arg->inode)->wsize; count = 0; arg->layoutupdate_pages = @@ -588,7 +597,8 @@ ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg) return -ENOMEM; } - goto retry; + ret = ext_tree_encode_commit(bl, start_p + 1, buffer_size, + &count, &arg->lastbytewritten); } *start_p = cpu_to_be32(count); @@ -608,7 +618,7 @@ ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg) } dprintk("%s found %zu ranges\n", __func__, count); - return 0; + return ret; } void diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 62607d52bfa5e..1bcdaee7e856f 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -668,6 +668,44 @@ struct nfs_client *nfs_init_client(struct nfs_client *clp, } EXPORT_SYMBOL_GPL(nfs_init_client); +static void nfs4_server_set_init_caps(struct nfs_server *server) +{ +#if IS_ENABLED(CONFIG_NFS_V4) + /* Set the basic capabilities */ + server->caps = server->nfs_client->cl_mvops->init_caps; + if (server->flags & NFS_MOUNT_NORDIRPLUS) + server->caps &= ~NFS_CAP_READDIRPLUS; + if (server->nfs_client->cl_proto == XPRT_TRANSPORT_RDMA) + server->caps &= ~NFS_CAP_READ_PLUS; + + /* + * Don't use NFS uid/gid mapping if we're using AUTH_SYS or lower + * authentication. + */ + if (nfs4_disable_idmapping && + server->client->cl_auth->au_flavor == RPC_AUTH_UNIX) + server->caps |= NFS_CAP_UIDGID_NOMAP; +#endif +} + +void nfs_server_set_init_caps(struct nfs_server *server) +{ + switch (server->nfs_client->rpc_ops->version) { + case 2: + server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; + break; + case 3: + server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; + if (!(server->flags & NFS_MOUNT_NORDIRPLUS)) + server->caps |= NFS_CAP_READDIRPLUS; + break; + default: + nfs4_server_set_init_caps(server); + break; + } +} +EXPORT_SYMBOL_GPL(nfs_server_set_init_caps); + /* * Create a version 2 or 3 client */ @@ -709,7 +747,6 @@ static int nfs_init_server(struct nfs_server *server, /* Initialise the client representation from the mount data */ server->flags = ctx->flags; server->options = ctx->options; - server->caps |= NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; switch (clp->rpc_ops->version) { case 2: @@ -745,6 +782,8 @@ static int nfs_init_server(struct nfs_server *server, if (error < 0) goto error; + nfs_server_set_init_caps(server); + /* Preserve the values of mount_server-related mount options */ if (ctx->mount_server.addrlen) { memcpy(&server->mountd_address, &ctx->mount_server.address, @@ -834,6 +873,8 @@ static void nfs_server_set_fsinfo(struct nfs_server *server, if (fsinfo->xattr_support) server->caps |= NFS_CAP_XATTR; + else + server->caps &= ~NFS_CAP_XATTR; #endif } @@ -919,7 +960,6 @@ void nfs_server_copy_userdata(struct nfs_server *target, struct nfs_server *sour target->acregmax = source->acregmax; target->acdirmin = source->acdirmin; target->acdirmax = source->acdirmax; - target->caps = source->caps; target->options = source->options; target->auth_info = source->auth_info; target->port = source->port; @@ -1080,6 +1120,8 @@ struct nfs_server *nfs_create_server(struct fs_context *fc) if (server->namelen == 0 || server->namelen > NFS2_MAXNAMLEN) server->namelen = NFS2_MAXNAMLEN; } + /* Linux 'subtree_check' borkenness mandates this setting */ + server->fh_expire_type = NFS_FH_VOL_RENAME; if (!(fattr->valid & NFS_ATTR_FATTR)) { error = ctx->nfs_mod->rpc_ops->getattr(server, ctx->mntfh, @@ -1143,6 +1185,8 @@ struct nfs_server *nfs_clone_server(struct nfs_server *source, if (error < 0) goto out_free_server; + nfs_server_set_init_caps(server); + /* probe the filesystem info for this server filesystem */ error = nfs_probe_server(server, fh); if (error < 0) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 55cfa1c4e0a65..bbd582d8a7dc9 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -297,7 +297,8 @@ nfs_start_delegation_return_locked(struct nfs_inode *nfsi) if (delegation == NULL) goto out; spin_lock(&delegation->lock); - if (!test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) { + if (delegation->inode && + !test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) { clear_bit(NFS_DELEGATION_RETURN_DELAYED, &delegation->flags); /* Refcount matched in nfs_end_delegation_return() */ ret = nfs_get_delegation(delegation); diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 39f7549afcf5b..385baf871800c 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -1835,9 +1835,7 @@ static void block_revalidate(struct dentry *dentry) static void unblock_revalidate(struct dentry *dentry) { - /* store_release ensures wait_var_event() sees the update */ - smp_store_release(&dentry->d_fsdata, NULL); - wake_up_var(&dentry->d_fsdata); + store_release_wake_up(&dentry->d_fsdata, NULL); } /* @@ -2642,6 +2640,18 @@ nfs_unblock_rename(struct rpc_task *task, struct nfs_renamedata *data) unblock_revalidate(new_dentry); } +static bool nfs_rename_is_unsafe_cross_dir(struct dentry *old_dentry, + struct dentry *new_dentry) +{ + struct nfs_server *server = NFS_SB(old_dentry->d_sb); + + if (old_dentry->d_parent != new_dentry->d_parent) + return false; + if (server->fh_expire_type & NFS_FH_RENAME_UNSAFE) + return !(server->fh_expire_type & NFS_FH_NOEXPIRE_WITH_OPEN); + return true; +} + /* * RENAME * FIXME: Some nfsds, like the Linux user space nfsd, may generate a @@ -2729,7 +2739,8 @@ int nfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, } - if (S_ISREG(old_inode->i_mode)) + if (S_ISREG(old_inode->i_mode) && + nfs_rename_is_unsafe_cross_dir(old_dentry, new_dentry)) nfs_sync_inode(old_inode); task = nfs_async_rename(old_dir, new_dir, old_dentry, new_dentry, must_unblock ? nfs_unblock_rename : NULL); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index a1ff4a4f5380e..4e53708dfcf43 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -469,8 +469,16 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, if (user_backed_iter(iter)) dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; - if (!swap) - nfs_start_io_direct(inode); + if (!swap) { + result = nfs_start_io_direct(inode); + if (result) { + /* release the reference that would usually be + * consumed by nfs_direct_read_schedule_iovec() + */ + nfs_direct_req_release(dreq); + goto out_release; + } + } NFS_I(inode)->read_io += count; requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); @@ -1023,7 +1031,14 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter, requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, FLUSH_STABLE); } else { - nfs_start_io_direct(inode); + result = nfs_start_io_direct(inode); + if (result) { + /* release the reference that would usually be + * consumed by nfs_direct_write_schedule_iovec() + */ + nfs_direct_req_release(dreq); + goto out_release; + } requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, FLUSH_COND_STABLE); diff --git a/fs/nfs/export.c b/fs/nfs/export.c index be686b8e0c546..aeb17adcb2b64 100644 --- a/fs/nfs/export.c +++ b/fs/nfs/export.c @@ -66,14 +66,21 @@ nfs_fh_to_dentry(struct super_block *sb, struct fid *fid, { struct nfs_fattr *fattr = NULL; struct nfs_fh *server_fh = nfs_exp_embedfh(fid->raw); - size_t fh_size = offsetof(struct nfs_fh, data) + server_fh->size; + size_t fh_size = offsetof(struct nfs_fh, data); const struct nfs_rpc_ops *rpc_ops; struct dentry *dentry; struct inode *inode; - int len = EMBED_FH_OFF + XDR_QUADLEN(fh_size); + int len = EMBED_FH_OFF; u32 *p = fid->raw; int ret; + /* Initial check of bounds */ + if (fh_len < len + XDR_QUADLEN(fh_size) || + fh_len > XDR_QUADLEN(NFS_MAXFHSIZE)) + return NULL; + /* Calculate embedded filehandle size */ + fh_size += server_fh->size; + len += XDR_QUADLEN(fh_size); /* NULL translates to ESTALE */ if (fh_len < len || fh_type != len) return NULL; diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 003dda0018403..2f4db026f8d67 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -167,7 +167,10 @@ nfs_file_read(struct kiocb *iocb, struct iov_iter *to) iocb->ki_filp, iov_iter_count(to), (unsigned long) iocb->ki_pos); - nfs_start_io_read(inode); + result = nfs_start_io_read(inode); + if (result) + return result; + result = nfs_revalidate_mapping(inode, iocb->ki_filp->f_mapping); if (!result) { result = generic_file_read_iter(iocb, to); @@ -188,7 +191,10 @@ nfs_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe dprintk("NFS: splice_read(%pD2, %zu@%llu)\n", in, len, *ppos); - nfs_start_io_read(inode); + result = nfs_start_io_read(inode); + if (result) + return result; + result = nfs_revalidate_mapping(inode, in->f_mapping); if (!result) { result = filemap_splice_read(in, ppos, pipe, len, flags); @@ -668,7 +674,9 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) nfs_clear_invalid_mapping(file->f_mapping); since = filemap_sample_wb_err(file->f_mapping); - nfs_start_io_write(inode); + error = nfs_start_io_write(inode); + if (error) + return error; result = generic_write_checks(iocb, from); if (result > 0) result = generic_perform_write(iocb, from); diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c index acf4b88889dc3..d5f1fbfd9a0c7 100644 --- a/fs/nfs/filelayout/filelayoutdev.c +++ b/fs/nfs/filelayout/filelayoutdev.c @@ -75,6 +75,7 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, struct page *scratch; struct list_head dsaddrs; struct nfs4_pnfs_ds_addr *da; + struct net *net = server->nfs_client->cl_net; /* set up xdr stream */ scratch = alloc_page(gfp_flags); @@ -158,8 +159,7 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, mp_count = be32_to_cpup(p); /* multipath count */ for (j = 0; j < mp_count; j++) { - da = nfs4_decode_mp_ds_addr(server->nfs_client->cl_net, - &stream, gfp_flags); + da = nfs4_decode_mp_ds_addr(net, &stream, gfp_flags); if (da) list_add_tail(&da->da_node, &dsaddrs); } @@ -169,7 +169,7 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, goto out_err_free_deviceid; } - dsaddr->ds_list[i] = nfs4_pnfs_ds_add(&dsaddrs, gfp_flags); + dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); if (!dsaddr->ds_list[i]) goto out_err_drain_dsaddrs; diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 2b3c5eea1f134..42c73c647a27f 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -276,7 +276,7 @@ ff_lseg_match_mirrors(struct pnfs_layout_segment *l1, struct pnfs_layout_segment *l2) { const struct nfs4_ff_layout_segment *fl1 = FF_LAYOUT_LSEG(l1); - const struct nfs4_ff_layout_segment *fl2 = FF_LAYOUT_LSEG(l1); + const struct nfs4_ff_layout_segment *fl2 = FF_LAYOUT_LSEG(l2); u32 i; if (fl1->mirror_array_cnt != fl2->mirror_array_cnt) @@ -745,25 +745,28 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg, { struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); struct nfs4_ff_layout_mirror *mirror; - struct nfs4_pnfs_ds *ds; + struct nfs4_pnfs_ds *ds = ERR_PTR(-EAGAIN); u32 idx; /* mirrors are initially sorted by efficiency */ for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) { mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false); - if (!ds) + if (IS_ERR(ds)) continue; if (check_device && - nfs4_test_deviceid_unavailable(&mirror->mirror_ds->id_node)) + nfs4_test_deviceid_unavailable(&mirror->mirror_ds->id_node)) { + // reinitialize the error state in case if this is the last iteration + ds = ERR_PTR(-EINVAL); continue; + } *best_idx = idx; - return ds; + break; } - return NULL; + return ds; } static struct nfs4_pnfs_ds * @@ -787,7 +790,7 @@ ff_layout_choose_best_ds_for_read(struct pnfs_layout_segment *lseg, struct nfs4_pnfs_ds *ds; ds = ff_layout_choose_valid_ds_for_read(lseg, start_idx, best_idx); - if (ds) + if (!IS_ERR(ds)) return ds; return ff_layout_choose_any_ds_for_read(lseg, start_idx, best_idx); } @@ -801,7 +804,7 @@ ff_layout_get_ds_for_read(struct nfs_pageio_descriptor *pgio, ds = ff_layout_choose_best_ds_for_read(lseg, pgio->pg_mirror_idx, best_idx); - if (ds || !pgio->pg_mirror_idx) + if (!IS_ERR(ds) || !pgio->pg_mirror_idx) return ds; return ff_layout_choose_best_ds_for_read(lseg, 0, best_idx); } @@ -859,7 +862,7 @@ ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio, req->wb_nio = 0; ds = ff_layout_get_ds_for_read(pgio, &ds_idx); - if (!ds) { + if (IS_ERR(ds)) { if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg)) goto out_mds; pnfs_generic_pg_cleanup(pgio); @@ -933,7 +936,7 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio, for (i = 0; i < pgio->pg_mirror_count; i++) { mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i); ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror, true); - if (!ds) { + if (IS_ERR(ds)) { if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg)) goto out_mds; pnfs_generic_pg_cleanup(pgio); @@ -1063,11 +1066,13 @@ static void ff_layout_resend_pnfs_read(struct nfs_pgio_header *hdr) { u32 idx = hdr->pgio_mirror_idx + 1; u32 new_idx = 0; + struct nfs4_pnfs_ds *ds; - if (ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx)) - ff_layout_send_layouterror(hdr->lseg); - else + ds = ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx); + if (IS_ERR(ds)) pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg); + else + ff_layout_send_layouterror(hdr->lseg); pnfs_read_resend_pnfs(hdr, new_idx); } @@ -1096,6 +1101,7 @@ static void ff_layout_reset_read(struct nfs_pgio_header *hdr) } static int ff_layout_async_handle_error_v4(struct rpc_task *task, + u32 op_status, struct nfs4_state *state, struct nfs_client *clp, struct pnfs_layout_segment *lseg, @@ -1106,32 +1112,42 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx); struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table; - switch (task->tk_status) { - case -NFS4ERR_BADSESSION: - case -NFS4ERR_BADSLOT: - case -NFS4ERR_BAD_HIGH_SLOT: - case -NFS4ERR_DEADSESSION: - case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: - case -NFS4ERR_SEQ_FALSE_RETRY: - case -NFS4ERR_SEQ_MISORDERED: + switch (op_status) { + case NFS4_OK: + case NFS4ERR_NXIO: + break; + case NFSERR_PERM: + if (!task->tk_xprt) + break; + xprt_force_disconnect(task->tk_xprt); + goto out_retry; + case NFS4ERR_BADSESSION: + case NFS4ERR_BADSLOT: + case NFS4ERR_BAD_HIGH_SLOT: + case NFS4ERR_DEADSESSION: + case NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case NFS4ERR_SEQ_FALSE_RETRY: + case NFS4ERR_SEQ_MISORDERED: dprintk("%s ERROR %d, Reset session. Exchangeid " "flags 0x%x\n", __func__, task->tk_status, clp->cl_exchange_flags); nfs4_schedule_session_recovery(clp->cl_session, task->tk_status); - break; - case -NFS4ERR_DELAY: - case -NFS4ERR_GRACE: + goto out_retry; + case NFS4ERR_DELAY: + nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY); + fallthrough; + case NFS4ERR_GRACE: rpc_delay(task, FF_LAYOUT_POLL_RETRY_MAX); - break; - case -NFS4ERR_RETRY_UNCACHED_REP: - break; + goto out_retry; + case NFS4ERR_RETRY_UNCACHED_REP: + goto out_retry; /* Invalidate Layout errors */ - case -NFS4ERR_PNFS_NO_LAYOUT: - case -ESTALE: /* mapped NFS4ERR_STALE */ - case -EBADHANDLE: /* mapped NFS4ERR_BADHANDLE */ - case -EISDIR: /* mapped NFS4ERR_ISDIR */ - case -NFS4ERR_FHEXPIRED: - case -NFS4ERR_WRONG_TYPE: + case NFS4ERR_PNFS_NO_LAYOUT: + case NFS4ERR_STALE: + case NFS4ERR_BADHANDLE: + case NFS4ERR_ISDIR: + case NFS4ERR_FHEXPIRED: + case NFS4ERR_WRONG_TYPE: dprintk("%s Invalid layout error %d\n", __func__, task->tk_status); /* @@ -1144,6 +1160,11 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, pnfs_destroy_layout(NFS_I(inode)); rpc_wake_up(&tbl->slot_tbl_waitq); goto reset; + default: + break; + } + + switch (task->tk_status) { /* RPC connection errors */ case -ECONNREFUSED: case -EHOSTDOWN: @@ -1159,26 +1180,56 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, nfs4_delete_deviceid(devid->ld, devid->nfs_client, &devid->deviceid); rpc_wake_up(&tbl->slot_tbl_waitq); - fallthrough; + break; default: - if (ff_layout_avoid_mds_available_ds(lseg)) - return -NFS4ERR_RESET_TO_PNFS; -reset: - dprintk("%s Retry through MDS. Error %d\n", __func__, - task->tk_status); - return -NFS4ERR_RESET_TO_MDS; + break; } + + if (ff_layout_avoid_mds_available_ds(lseg)) + return -NFS4ERR_RESET_TO_PNFS; +reset: + dprintk("%s Retry through MDS. Error %d\n", __func__, + task->tk_status); + return -NFS4ERR_RESET_TO_MDS; + +out_retry: task->tk_status = 0; return -EAGAIN; } /* Retry all errors through either pNFS or MDS except for -EJUKEBOX */ static int ff_layout_async_handle_error_v3(struct rpc_task *task, + u32 op_status, + struct nfs_client *clp, struct pnfs_layout_segment *lseg, u32 idx) { struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx); + switch (op_status) { + case NFS_OK: + case NFSERR_NXIO: + break; + case NFSERR_PERM: + if (!task->tk_xprt) + break; + xprt_force_disconnect(task->tk_xprt); + goto out_retry; + case NFSERR_ACCES: + case NFSERR_BADHANDLE: + case NFSERR_FBIG: + case NFSERR_IO: + case NFSERR_NOSPC: + case NFSERR_ROFS: + case NFSERR_STALE: + goto out_reset_to_pnfs; + case NFSERR_JUKEBOX: + nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY); + goto out_retry; + default: + break; + } + switch (task->tk_status) { /* File access problems. Don't mark the device as unavailable */ case -EACCES: @@ -1197,6 +1248,7 @@ static int ff_layout_async_handle_error_v3(struct rpc_task *task, nfs4_delete_deviceid(devid->ld, devid->nfs_client, &devid->deviceid); } +out_reset_to_pnfs: /* FIXME: Need to prevent infinite looping here. */ return -NFS4ERR_RESET_TO_PNFS; out_retry: @@ -1207,6 +1259,7 @@ static int ff_layout_async_handle_error_v3(struct rpc_task *task, } static int ff_layout_async_handle_error(struct rpc_task *task, + u32 op_status, struct nfs4_state *state, struct nfs_client *clp, struct pnfs_layout_segment *lseg, @@ -1225,10 +1278,11 @@ static int ff_layout_async_handle_error(struct rpc_task *task, switch (vers) { case 3: - return ff_layout_async_handle_error_v3(task, lseg, idx); - case 4: - return ff_layout_async_handle_error_v4(task, state, clp, + return ff_layout_async_handle_error_v3(task, op_status, clp, lseg, idx); + case 4: + return ff_layout_async_handle_error_v4(task, op_status, state, + clp, lseg, idx); default: /* should never happen */ WARN_ON_ONCE(1); @@ -1255,6 +1309,7 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, case -ECONNRESET: case -EHOSTDOWN: case -EHOSTUNREACH: + case -ENETDOWN: case -ENETUNREACH: case -EADDRINUSE: case -ENOBUFS: @@ -1280,6 +1335,7 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, switch (status) { case NFS4ERR_DELAY: case NFS4ERR_GRACE: + case NFS4ERR_PERM: break; case NFS4ERR_NXIO: ff_layout_mark_ds_unreachable(lseg, idx); @@ -1312,7 +1368,8 @@ static int ff_layout_read_done_cb(struct rpc_task *task, trace_ff_layout_read_error(hdr); } - err = ff_layout_async_handle_error(task, hdr->args.context->state, + err = ff_layout_async_handle_error(task, hdr->res.op_status, + hdr->args.context->state, hdr->ds_clp, hdr->lseg, hdr->pgio_mirror_idx); @@ -1482,7 +1539,8 @@ static int ff_layout_write_done_cb(struct rpc_task *task, trace_ff_layout_write_error(hdr); } - err = ff_layout_async_handle_error(task, hdr->args.context->state, + err = ff_layout_async_handle_error(task, hdr->res.op_status, + hdr->args.context->state, hdr->ds_clp, hdr->lseg, hdr->pgio_mirror_idx); @@ -1528,8 +1586,9 @@ static int ff_layout_commit_done_cb(struct rpc_task *task, trace_ff_layout_commit_error(data); } - err = ff_layout_async_handle_error(task, NULL, data->ds_clp, - data->lseg, data->ds_commit_index); + err = ff_layout_async_handle_error(task, data->res.op_status, + NULL, data->ds_clp, data->lseg, + data->ds_commit_index); trace_nfs4_pnfs_commit_ds(data, err); switch (err) { @@ -1785,6 +1844,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) u32 idx = hdr->pgio_mirror_idx; int vers; struct nfs_fh *fh; + bool ds_fatal_error = false; dprintk("--> %s ino %lu pgbase %u req %zu@%llu\n", __func__, hdr->inode->i_ino, @@ -1792,8 +1852,10 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false); - if (!ds) + if (IS_ERR(ds)) { + ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); goto out_failed; + } ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, hdr->inode); @@ -1834,7 +1896,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) return PNFS_ATTEMPTED; out_failed: - if (ff_layout_avoid_mds_available_ds(lseg)) + if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; trace_pnfs_mds_fallback_read_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, @@ -1855,11 +1917,14 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) int vers; struct nfs_fh *fh; u32 idx = hdr->pgio_mirror_idx; + bool ds_fatal_error = false; mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true); - if (!ds) + if (IS_ERR(ds)) { + ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); goto out_failed; + } ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, hdr->inode); @@ -1902,7 +1967,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) return PNFS_ATTEMPTED; out_failed: - if (ff_layout_avoid_mds_available_ds(lseg)) + if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; trace_pnfs_mds_fallback_write_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, @@ -1944,7 +2009,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) idx = calc_ds_index_from_commit(lseg, data->ds_commit_index); mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true); - if (!ds) + if (IS_ERR(ds)) goto out_err; ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index e028f5a0ef5f6..95d5dca671456 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -49,6 +49,7 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, struct nfs4_pnfs_ds_addr *da; struct nfs4_ff_layout_ds *new_ds = NULL; struct nfs4_ff_ds_version *ds_versions = NULL; + struct net *net = server->nfs_client->cl_net; u32 mp_count; u32 version_count; __be32 *p; @@ -80,8 +81,7 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, for (i = 0; i < mp_count; i++) { /* multipath ds */ - da = nfs4_decode_mp_ds_addr(server->nfs_client->cl_net, - &stream, gfp_flags); + da = nfs4_decode_mp_ds_addr(net, &stream, gfp_flags); if (da) list_add_tail(&da->da_node, &dsaddrs); } @@ -149,7 +149,7 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, new_ds->ds_versions = ds_versions; new_ds->ds_versions_cnt = version_count; - new_ds->ds = nfs4_pnfs_ds_add(&dsaddrs, gfp_flags); + new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); if (!new_ds->ds) goto out_err_drain_dsaddrs; @@ -370,11 +370,11 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, struct nfs4_ff_layout_mirror *mirror, bool fail_return) { - struct nfs4_pnfs_ds *ds = NULL; + struct nfs4_pnfs_ds *ds; struct inode *ino = lseg->pls_layout->plh_inode; struct nfs_server *s = NFS_SERVER(ino); unsigned int max_payload; - int status; + int status = -EAGAIN; if (!ff_layout_init_mirror_ds(lseg->pls_layout, mirror)) goto noconnect; @@ -412,7 +412,7 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, ff_layout_send_layouterror(lseg); if (fail_return || !ff_layout_has_available_ds(lseg)) pnfs_error_mark_layout_for_return(ino, lseg); - ds = NULL; + ds = ERR_PTR(status); out: return ds; } diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 238bf0896bf43..870bda3d4dbd9 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -74,6 +74,8 @@ nfs_fattr_to_ino_t(struct nfs_fattr *fattr) int nfs_wait_bit_killable(struct wait_bit_key *key, int mode) { + if (unlikely(nfs_current_task_exiting())) + return -EINTR; schedule(); if (signal_pending_state(mode, current)) return -ERESTARTSYS; @@ -553,6 +555,8 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) set_nlink(inode, fattr->nlink); else if (fattr_supported & NFS_ATTR_FATTR_NLINK) nfs_set_cache_invalid(inode, NFS_INO_INVALID_NLINK); + else + set_nlink(inode, 1); if (fattr->valid & NFS_ATTR_FATTR_OWNER) inode->i_uid = fattr->uid; else if (fattr_supported & NFS_ATTR_FATTR_OWNER) @@ -641,8 +645,10 @@ nfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, trace_nfs_setattr_enter(inode); /* Write all dirty data */ - if (S_ISREG(inode->i_mode)) + if (S_ISREG(inode->i_mode)) { + nfs_file_block_o_direct(NFS_I(inode)); nfs_sync_inode(inode); + } fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode)); if (fattr == NULL) { @@ -2438,15 +2444,26 @@ EXPORT_SYMBOL_GPL(nfs_net_id); static int nfs_net_init(struct net *net) { struct nfs_net *nn = net_generic(net, nfs_net_id); + int err; nfs_clients_init(net); if (!rpc_proc_register(net, &nn->rpcstats)) { - nfs_clients_exit(net); - return -ENOMEM; + err = -ENOMEM; + goto err_proc_rpc; } - return nfs_fs_proc_net_init(net); + err = nfs_fs_proc_net_init(net); + if (err) + goto err_proc_nfs; + + return 0; + +err_proc_nfs: + rpc_proc_unregister(net, "nfs"); +err_proc_rpc: + nfs_clients_exit(net); + return err; } static void nfs_net_exit(struct net *net) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index ca49d999159eb..bde81e0abf0ae 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -6,6 +6,7 @@ #include "nfs4_fs.h" #include #include +#include #include #include #include @@ -223,7 +224,7 @@ extern struct nfs_client * nfs4_find_client_sessionid(struct net *, const struct sockaddr *, struct nfs4_sessionid *, u32); extern struct nfs_server *nfs_create_server(struct fs_context *); -extern void nfs4_server_set_init_caps(struct nfs_server *); +extern void nfs_server_set_init_caps(struct nfs_server *); extern struct nfs_server *nfs4_create_server(struct fs_context *); extern struct nfs_server *nfs4_create_referral_server(struct fs_context *); extern int nfs4_update_server(struct nfs_server *server, const char *hostname, @@ -461,11 +462,11 @@ extern const struct netfs_request_ops nfs_netfs_ops; #endif /* io.c */ -extern void nfs_start_io_read(struct inode *inode); +extern __must_check int nfs_start_io_read(struct inode *inode); extern void nfs_end_io_read(struct inode *inode); -extern void nfs_start_io_write(struct inode *inode); +extern __must_check int nfs_start_io_write(struct inode *inode); extern void nfs_end_io_write(struct inode *inode); -extern void nfs_start_io_direct(struct inode *inode); +extern __must_check int nfs_start_io_direct(struct inode *inode); extern void nfs_end_io_direct(struct inode *inode); static inline bool nfs_file_io_is_buffered(struct nfs_inode *nfsi) @@ -473,6 +474,16 @@ static inline bool nfs_file_io_is_buffered(struct nfs_inode *nfsi) return test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0; } +/* Must be called with exclusively locked inode->i_rwsem */ +static inline void nfs_file_block_o_direct(struct nfs_inode *nfsi) +{ + if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { + clear_bit(NFS_INO_ODIRECT, &nfsi->flags); + inode_dio_wait(&nfsi->vfs_inode); + } +} + + /* namespace.c */ #define NFS_PATH_CANONICAL 1 extern char *nfs_path(char **p, struct dentry *dentry, @@ -613,9 +624,12 @@ nfs_write_match_verf(const struct nfs_writeverf *verf, static inline gfp_t nfs_io_gfp_mask(void) { - if (current->flags & PF_WQ_WORKER) - return GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; - return GFP_KERNEL; + gfp_t ret = current_gfp_context(GFP_KERNEL); + + /* For workers __GFP_NORETRY only with __GFP_IO or __GFP_FS */ + if ((current->flags & PF_WQ_WORKER) && ret == GFP_KERNEL) + ret |= __GFP_NORETRY | __GFP_NOWARN; + return ret; } /* @@ -865,6 +879,11 @@ static inline u32 nfs_stateid_hash(const nfs4_stateid *stateid) NFS4_STATEID_OTHER_SIZE); } +static inline bool nfs_current_task_exiting(void) +{ + return (current->flags & PF_EXITING) != 0; +} + static inline bool nfs_error_is_fatal(int err) { switch (err) { diff --git a/fs/nfs/io.c b/fs/nfs/io.c index b5551ed8f648b..d275b0a250bf3 100644 --- a/fs/nfs/io.c +++ b/fs/nfs/io.c @@ -14,15 +14,6 @@ #include "internal.h" -/* Call with exclusively locked inode->i_rwsem */ -static void nfs_block_o_direct(struct nfs_inode *nfsi, struct inode *inode) -{ - if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { - clear_bit(NFS_INO_ODIRECT, &nfsi->flags); - inode_dio_wait(inode); - } -} - /** * nfs_start_io_read - declare the file is being used for buffered reads * @inode: file inode @@ -39,19 +30,28 @@ static void nfs_block_o_direct(struct nfs_inode *nfsi, struct inode *inode) * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. */ -void +int nfs_start_io_read(struct inode *inode) { struct nfs_inode *nfsi = NFS_I(inode); + int err; + /* Be an optimist! */ - down_read(&inode->i_rwsem); + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0) - return; + return 0; up_read(&inode->i_rwsem); + /* Slow path.... */ - down_write(&inode->i_rwsem); - nfs_block_o_direct(nfsi, inode); + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; + nfs_file_block_o_direct(nfsi); downgrade_write(&inode->i_rwsem); + + return 0; } /** @@ -74,11 +74,15 @@ nfs_end_io_read(struct inode *inode) * Declare that a buffered read operation is about to start, and ensure * that we block all direct I/O. */ -void +int nfs_start_io_write(struct inode *inode) { - down_write(&inode->i_rwsem); - nfs_block_o_direct(NFS_I(inode), inode); + int err; + + err = down_write_killable(&inode->i_rwsem); + if (!err) + nfs_file_block_o_direct(NFS_I(inode)); + return err; } /** @@ -119,19 +123,28 @@ static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode) * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. */ -void +int nfs_start_io_direct(struct inode *inode) { struct nfs_inode *nfsi = NFS_I(inode); + int err; + /* Be an optimist! */ - down_read(&inode->i_rwsem); + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0) - return; + return 0; up_read(&inode->i_rwsem); + /* Slow path.... */ - down_write(&inode->i_rwsem); + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; nfs_block_buffered(nfsi, inode); downgrade_write(&inode->i_rwsem); + + return 0; } /** diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c index 674c012868b1a..1aa4c43c9b3b4 100644 --- a/fs/nfs/nfs3client.c +++ b/fs/nfs/nfs3client.c @@ -2,6 +2,7 @@ #include #include #include +#include #include "internal.h" #include "nfs3_fs.h" #include "netns.h" @@ -98,7 +99,11 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, .net = mds_clp->cl_net, .timeparms = &ds_timeout, .cred = mds_srv->cred, - .xprtsec = mds_clp->cl_xprtsec, + .xprtsec = { + .policy = RPC_XPRTSEC_NONE, + .cert_serial = TLS_NO_CERT, + .privkey_serial = TLS_NO_PRIVKEY, + }, .connect_timeout = connect_timeout, .reconnect_timeout = connect_timeout, }; @@ -111,8 +116,14 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, cl_init.hostname = buf; switch (ds_proto) { - case XPRT_TRANSPORT_TCP: case XPRT_TRANSPORT_TCP_TLS: + if (mds_clp->cl_xprtsec.policy != RPC_XPRTSEC_NONE) + cl_init.xprtsec = mds_clp->cl_xprtsec; + else + ds_proto = XPRT_TRANSPORT_TCP; + fallthrough; + case XPRT_TRANSPORT_RDMA: + case XPRT_TRANSPORT_TCP: if (mds_clp->cl_nconnect > 1) cl_init.nconnect = mds_clp->cl_nconnect; } diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index 4bf208a0a8e99..715753f41fb07 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -39,7 +39,7 @@ nfs3_rpc_wrapper(struct rpc_clnt *clnt, struct rpc_message *msg, int flags) __set_current_state(TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); schedule_timeout(NFS_JUKEBOX_RETRY_TIME); res = -ERESTARTSYS; - } while (!fatal_signal_pending(current)); + } while (!fatal_signal_pending(current) && !nfs_current_task_exiting()); return res; } diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index 9f0d69e652644..582cf8a469560 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -112,6 +112,7 @@ static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, exception.inode = inode; exception.state = lock->open_context->state; + nfs_file_block_o_direct(NFS_I(inode)); err = nfs_sync_inode(inode); if (err) goto out; @@ -355,6 +356,7 @@ static ssize_t _nfs42_proc_copy(struct file *src, return status; } + nfs_file_block_o_direct(NFS_I(dst_inode)); status = nfs_sync_inode(dst_inode); if (status) return status; diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index ac80f87cb9d99..b14688da814d6 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "internal.h" #include "callback.h" #include "delegation.h" @@ -222,6 +223,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) clp->cl_state = 1 << NFS4CLNT_LEASE_EXPIRED; clp->cl_mvops = nfs_v4_minor_ops[cl_init->minorversion]; clp->cl_mig_gen = 1; + clp->cl_last_renewal = jiffies; #if IS_ENABLED(CONFIG_NFS_V4_1) init_waitqueue_head(&clp->cl_lock_waitq); #endif @@ -923,6 +925,7 @@ static int nfs4_set_client(struct nfs_server *server, else cl_init.max_connect = max_connect; switch (proto) { + case XPRT_TRANSPORT_RDMA: case XPRT_TRANSPORT_TCP: case XPRT_TRANSPORT_TCP_TLS: cl_init.nconnect = nconnect; @@ -990,7 +993,11 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, .net = mds_clp->cl_net, .timeparms = &ds_timeout, .cred = mds_srv->cred, - .xprtsec = mds_srv->nfs_client->cl_xprtsec, + .xprtsec = { + .policy = RPC_XPRTSEC_NONE, + .cert_serial = TLS_NO_CERT, + .privkey_serial = TLS_NO_PRIVKEY, + }, }; char buf[INET6_ADDRSTRLEN + 1]; @@ -999,8 +1006,14 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, cl_init.hostname = buf; switch (ds_proto) { - case XPRT_TRANSPORT_TCP: case XPRT_TRANSPORT_TCP_TLS: + if (mds_srv->nfs_client->cl_xprtsec.policy != RPC_XPRTSEC_NONE) + cl_init.xprtsec = mds_srv->nfs_client->cl_xprtsec; + else + ds_proto = XPRT_TRANSPORT_TCP; + fallthrough; + case XPRT_TRANSPORT_RDMA: + case XPRT_TRANSPORT_TCP: if (mds_clp->cl_nconnect > 1) { cl_init.nconnect = mds_clp->cl_nconnect; cl_init.max_connect = NFS_MAX_TRANSPORTS; @@ -1079,24 +1092,6 @@ static void nfs4_session_limit_xasize(struct nfs_server *server) #endif } -void nfs4_server_set_init_caps(struct nfs_server *server) -{ - /* Set the basic capabilities */ - server->caps |= server->nfs_client->cl_mvops->init_caps; - if (server->flags & NFS_MOUNT_NORDIRPLUS) - server->caps &= ~NFS_CAP_READDIRPLUS; - if (server->nfs_client->cl_proto == XPRT_TRANSPORT_RDMA) - server->caps &= ~NFS_CAP_READ_PLUS; - - /* - * Don't use NFS uid/gid mapping if we're using AUTH_SYS or lower - * authentication. - */ - if (nfs4_disable_idmapping && - server->client->cl_auth->au_flavor == RPC_AUTH_UNIX) - server->caps |= NFS_CAP_UIDGID_NOMAP; -} - static int nfs4_server_common_setup(struct nfs_server *server, struct nfs_fh *mntfh, bool auth_probe) { @@ -1111,7 +1106,7 @@ static int nfs4_server_common_setup(struct nfs_server *server, if (error < 0) goto out; - nfs4_server_set_init_caps(server); + nfs_server_set_init_caps(server); /* Probe the root fh to retrieve its FSID and filehandle */ error = nfs4_get_rootfh(server, mntfh, auth_probe); diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c index 02788c3c85e5b..befdb0f4e6dc3 100644 --- a/fs/nfs/nfs4file.c +++ b/fs/nfs/nfs4file.c @@ -282,9 +282,11 @@ static loff_t nfs42_remap_file_range(struct file *src_file, loff_t src_off, /* flush all pending writes on both src and dst so that server * has the latest data */ + nfs_file_block_o_direct(NFS_I(src_inode)); ret = nfs_sync_inode(src_inode); if (ret) goto out_unlock; + nfs_file_block_o_direct(NFS_I(dst_inode)); ret = nfs_sync_inode(dst_inode); if (ret) goto out_unlock; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 4b12e45f57539..a0a71a163ffed 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -362,7 +362,9 @@ static void nfs4_setup_readdir(u64 cookie, __be32 *verifier, struct dentry *dent *p++ = htonl(attrs); /* bitmap */ *p++ = htonl(12); /* attribute buffer length */ *p++ = htonl(NF4DIR); + spin_lock(&dentry->d_lock); p = xdr_encode_hyper(p, NFS_FILEID(d_inode(dentry->d_parent))); + spin_unlock(&dentry->d_lock); readdir->pgbase = (char *)p - (char *)start; readdir->count -= readdir->pgbase; @@ -422,6 +424,8 @@ static int nfs4_delay_killable(long *timeout) { might_sleep(); + if (unlikely(nfs_current_task_exiting())) + return -EINTR; __set_current_state(TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); schedule_timeout(nfs4_update_delay(timeout)); if (!__fatal_signal_pending(current)) @@ -433,6 +437,8 @@ static int nfs4_delay_interruptible(long *timeout) { might_sleep(); + if (unlikely(nfs_current_task_exiting())) + return -EINTR; __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE_UNSAFE); schedule_timeout(nfs4_update_delay(timeout)); if (!signal_pending(current)) @@ -1712,7 +1718,8 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state, rcu_read_unlock(); trace_nfs4_open_stateid_update_wait(state->inode, stateid, 0); - if (!fatal_signal_pending(current)) { + if (!fatal_signal_pending(current) && + !nfs_current_task_exiting()) { if (schedule_timeout(5*HZ) == 0) status = -EAGAIN; else @@ -3494,7 +3501,7 @@ static bool nfs4_refresh_open_old_stateid(nfs4_stateid *dst, write_sequnlock(&state->seqlock); trace_nfs4_close_stateid_update_wait(state->inode, dst, 0); - if (fatal_signal_pending(current)) + if (fatal_signal_pending(current) || nfs_current_task_exiting()) status = -EINTR; else if (schedule_timeout(5*HZ) != 0) @@ -3877,8 +3884,9 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f res.attr_bitmask[2] &= FATTR4_WORD2_NFS42_MASK; } memcpy(server->attr_bitmask, res.attr_bitmask, sizeof(server->attr_bitmask)); - server->caps &= ~(NFS_CAP_ACLS | NFS_CAP_HARDLINKS | - NFS_CAP_SYMLINKS| NFS_CAP_SECURITY_LABEL); + server->caps &= + ~(NFS_CAP_ACLS | NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS | + NFS_CAP_SECURITY_LABEL | NFS_CAP_FS_LOCATIONS); server->fattr_valid = NFS_ATTR_FATTR_V4; if (res.attr_bitmask[0] & FATTR4_WORD0_ACL && res.acl_bitmask & ACL4_SUPPORT_ALLOW_ACL) @@ -3946,7 +3954,6 @@ int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle) }; int err; - nfs4_server_set_init_caps(server); do { err = nfs4_handle_exception(server, _nfs4_server_capabilities(server, fhandle), @@ -4571,16 +4578,19 @@ static int _nfs4_proc_lookupp(struct inode *inode, }; unsigned short task_flags = 0; - if (NFS_SERVER(inode)->flags & NFS_MOUNT_SOFTREVAL) + if (server->flags & NFS_MOUNT_SOFTREVAL) task_flags |= RPC_TASK_TIMEOUT; + if (server->caps & NFS_CAP_MOVEABLE) + task_flags |= RPC_TASK_MOVEABLE; args.bitmask = nfs4_bitmask(server, fattr->label); nfs_fattr_init(fattr); + nfs4_init_sequence(&args.seq_args, &res.seq_res, 0, 0); dprintk("NFS call lookupp ino=0x%lx\n", inode->i_ino); - status = nfs4_call_sync(clnt, server, &msg, &args.seq_args, - &res.seq_res, task_flags); + status = nfs4_do_call_sync(clnt, server, &msg, &args.seq_args, + &res.seq_res, task_flags); dprintk("NFS reply lookupp: %d\n", status); return status; } @@ -6054,6 +6064,8 @@ static ssize_t nfs4_proc_get_acl(struct inode *inode, void *buf, size_t buflen, struct nfs_server *server = NFS_SERVER(inode); int ret; + if (unlikely(NFS_FH(inode)->size == 0)) + return -ENODATA; if (!nfs4_server_supports_acls(server, type)) return -EOPNOTSUPP; ret = nfs_revalidate_inode(inode, NFS_INO_INVALID_CHANGE); @@ -6128,6 +6140,9 @@ static int nfs4_proc_set_acl(struct inode *inode, const void *buf, { struct nfs4_exception exception = { }; int err; + + if (unlikely(NFS_FH(inode)->size == 0)) + return -ENODATA; do { err = __nfs4_proc_set_acl(inode, buf, buflen, type); trace_nfs4_set_acl(inode, err); @@ -6880,10 +6895,18 @@ static struct nfs4_unlockdata *nfs4_alloc_unlockdata(struct file_lock *fl, struct nfs4_unlockdata *p; struct nfs4_state *state = lsp->ls_state; struct inode *inode = state->inode; + struct nfs_lock_context *l_ctx; p = kzalloc(sizeof(*p), GFP_KERNEL); if (p == NULL) return NULL; + l_ctx = nfs_get_lock_context(ctx); + if (!IS_ERR(l_ctx)) { + p->l_ctx = l_ctx; + } else { + kfree(p); + return NULL; + } p->arg.fh = NFS_FH(inode); p->arg.fl = &p->fl; p->arg.seqid = seqid; @@ -6891,7 +6914,6 @@ static struct nfs4_unlockdata *nfs4_alloc_unlockdata(struct file_lock *fl, p->lsp = lsp; /* Ensure we don't close file until we're done freeing locks! */ p->ctx = get_nfs_open_context(ctx); - p->l_ctx = nfs_get_lock_context(ctx); locks_init_lock(&p->fl); locks_copy_lock(&p->fl, fl); p->server = NFS_SERVER(inode); @@ -7641,10 +7663,10 @@ int nfs4_lock_delegation_recall(struct file_lock *fl, struct nfs4_state *state, return err; do { err = _nfs4_do_setlk(state, F_SETLK, fl, NFS_LOCK_NEW); - if (err != -NFS4ERR_DELAY) + if (err != -NFS4ERR_DELAY && err != -NFS4ERR_GRACE) break; ssleep(1); - } while (err == -NFS4ERR_DELAY); + } while (err == -NFS4ERR_DELAY || err == -NFSERR_GRACE); return nfs4_handle_delegation_recall_error(server, state, stateid, fl, err); } @@ -9209,7 +9231,7 @@ static int nfs4_verify_back_channel_attrs(struct nfs41_create_session_args *args goto out; if (rcvd->max_rqst_sz > sent->max_rqst_sz) return -EINVAL; - if (rcvd->max_resp_sz < sent->max_resp_sz) + if (rcvd->max_resp_sz > sent->max_resp_sz) return -EINVAL; if (rcvd->max_resp_sz_cached > sent->max_resp_sz_cached) return -EINVAL; @@ -10613,7 +10635,7 @@ const struct nfs4_minor_version_ops *nfs_v4_minor_ops[] = { static ssize_t nfs4_listxattr(struct dentry *dentry, char *list, size_t size) { - ssize_t error, error2, error3; + ssize_t error, error2, error3, error4 = 0; size_t left = size; error = generic_listxattr(dentry, list, left); @@ -10636,8 +10658,18 @@ static ssize_t nfs4_listxattr(struct dentry *dentry, char *list, size_t size) error3 = nfs4_listxattr_nfs4_user(d_inode(dentry), list, left); if (error3 < 0) return error3; + if (list) { + list += error3; + left -= error3; + } + + if (!nfs_server_capable(d_inode(dentry), NFS_CAP_SECURITY_LABEL)) { + error4 = security_inode_listsecurity(d_inode(dentry), list, left); + if (error4 < 0) + return error4; + } - error += error2 + error3; + error += error2 + error3 + error4; if (size && error > size) return -ERANGE; return error; diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index 794bb4aa588d3..78d52c2006dcc 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -2741,7 +2741,18 @@ static void nfs4_state_manager(struct nfs_client *clp) pr_warn_ratelimited("NFS: state manager%s%s failed on NFSv4 server %s" " with error %d\n", section_sep, section, clp->cl_hostname, -status); - ssleep(1); + switch (status) { + case -ENETDOWN: + case -ENETUNREACH: + nfs_mark_client_ready(clp, -EIO); + break; + case -EINVAL: + nfs_mark_client_ready(clp, status); + break; + default: + ssleep(1); + break; + } out_drain: memalloc_nofs_restore(memflags); nfs4_end_drain_session(clp); diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 040b6b79c75e5..0ea3916ed1dcb 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -206,83 +206,6 @@ nfs_page_group_lock_head(struct nfs_page *req) return head; } -/* - * nfs_unroll_locks - unlock all newly locked reqs and wait on @req - * @head: head request of page group, must be holding head lock - * @req: request that couldn't lock and needs to wait on the req bit lock - * - * This is a helper function for nfs_lock_and_join_requests - * returns 0 on success, < 0 on error. - */ -static void -nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req) -{ - struct nfs_page *tmp; - - /* relinquish all the locks successfully grabbed this run */ - for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { - if (!kref_read(&tmp->wb_kref)) - continue; - nfs_unlock_and_release_request(tmp); - } -} - -/* - * nfs_page_group_lock_subreq - try to lock a subrequest - * @head: head request of page group - * @subreq: request to lock - * - * This is a helper function for nfs_lock_and_join_requests which - * must be called with the head request and page group both locked. - * On error, it returns with the page group unlocked. - */ -static int -nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq) -{ - int ret; - - if (!kref_get_unless_zero(&subreq->wb_kref)) - return 0; - while (!nfs_lock_request(subreq)) { - nfs_page_group_unlock(head); - ret = nfs_wait_on_request(subreq); - if (!ret) - ret = nfs_page_group_lock(head); - if (ret < 0) { - nfs_unroll_locks(head, subreq); - nfs_release_request(subreq); - return ret; - } - } - return 0; -} - -/* - * nfs_page_group_lock_subrequests - try to lock the subrequests - * @head: head request of page group - * - * This is a helper function for nfs_lock_and_join_requests which - * must be called with the head request locked. - */ -int nfs_page_group_lock_subrequests(struct nfs_page *head) -{ - struct nfs_page *subreq; - int ret; - - ret = nfs_page_group_lock(head); - if (ret < 0) - return ret; - /* lock each request in the page group */ - for (subreq = head->wb_this_page; subreq != head; - subreq = subreq->wb_this_page) { - ret = nfs_page_group_lock_subreq(head, subreq); - if (ret < 0) - return ret; - } - nfs_page_group_unlock(head); - return 0; -} - /* * nfs_page_set_headlock - set the request PG_HEADLOCK * @req: request that is to be locked @@ -349,13 +272,14 @@ nfs_page_group_unlock(struct nfs_page *req) nfs_page_clear_headlock(req); } -/* - * nfs_page_group_sync_on_bit_locked +/** + * nfs_page_group_sync_on_bit_locked - Test if all requests have @bit set + * @req: request in page group + * @bit: PG_* bit that is used to sync page group * * must be called with page group lock held */ -static bool -nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit) +bool nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit) { struct nfs_page *head = req->wb_head; struct nfs_page *tmp; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index fe83c681e3fe0..b40c20bd364b0 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -732,6 +732,14 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, return remaining; } +static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) +{ + struct pnfs_layout_segment *lseg; + + list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) + pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); +} + static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, struct list_head *free_me, @@ -1180,6 +1188,7 @@ void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); pnfs_free_returned_lsegs(lo, &freeme, range, seq); pnfs_set_layout_stateid(lo, stateid, NULL, true); + pnfs_reset_return_info(lo); } else pnfs_mark_layout_stateid_invalid(lo, &freeme); out_unlock: @@ -1921,8 +1930,10 @@ static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) { if (atomic_dec_and_test(&lo->plh_outstanding) && - test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) + test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { + smp_mb__after_atomic(); wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); + } } static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) @@ -3205,6 +3216,7 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) struct nfs_inode *nfsi = NFS_I(inode); loff_t end_pos; int status; + bool mark_as_dirty = false; if (!pnfs_layoutcommit_outstanding(inode)) return 0; @@ -3256,19 +3268,23 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) if (ld->prepare_layoutcommit) { status = ld->prepare_layoutcommit(&data->args); if (status) { - put_cred(data->cred); + if (status != -ENOSPC) + put_cred(data->cred); spin_lock(&inode->i_lock); set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); if (end_pos > nfsi->layout->plh_lwb) nfsi->layout->plh_lwb = end_pos; - goto out_unlock; + if (status != -ENOSPC) + goto out_unlock; + spin_unlock(&inode->i_lock); + mark_as_dirty = true; } } status = nfs4_proc_layoutcommit(data, sync); out: - if (status) + if (status || mark_as_dirty) mark_inode_dirty_sync(inode); dprintk("<-- %s status %d\n", __func__, status); return status; diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index d886c8226d8fe..79996d7dad0f7 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -59,6 +59,7 @@ struct nfs4_pnfs_ds { struct list_head ds_node; /* nfs4_pnfs_dev_hlist dev_dslist */ char *ds_remotestr; /* comma sep list of addrs */ struct list_head ds_addrs; + const struct net *ds_net; struct nfs_client *ds_clp; refcount_t ds_count; unsigned long ds_state; @@ -405,7 +406,8 @@ int pnfs_generic_commit_pagelist(struct inode *inode, int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max); void pnfs_generic_write_commit_done(struct rpc_task *task, void *data); void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds); -struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(struct list_head *dsaddrs, +struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net, + struct list_head *dsaddrs, gfp_t gfp_flags); void nfs4_pnfs_v3_ds_connect_unload(void); int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 88e061bd711b7..5314dabb725d8 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -651,12 +651,12 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1, * Lookup DS by addresses. nfs4_ds_cache_lock is held */ static struct nfs4_pnfs_ds * -_data_server_lookup_locked(const struct list_head *dsaddrs) +_data_server_lookup_locked(const struct net *net, const struct list_head *dsaddrs) { struct nfs4_pnfs_ds *ds; list_for_each_entry(ds, &nfs4_data_server_cache, ds_node) - if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) + if (ds->ds_net == net && _same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) return ds; return NULL; } @@ -763,7 +763,7 @@ nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags) * uncached and return cached struct nfs4_pnfs_ds. */ struct nfs4_pnfs_ds * -nfs4_pnfs_ds_add(struct list_head *dsaddrs, gfp_t gfp_flags) +nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_flags) { struct nfs4_pnfs_ds *tmp_ds, *ds = NULL; char *remotestr; @@ -781,13 +781,14 @@ nfs4_pnfs_ds_add(struct list_head *dsaddrs, gfp_t gfp_flags) remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags); spin_lock(&nfs4_ds_cache_lock); - tmp_ds = _data_server_lookup_locked(dsaddrs); + tmp_ds = _data_server_lookup_locked(net, dsaddrs); if (tmp_ds == NULL) { INIT_LIST_HEAD(&ds->ds_addrs); list_splice_init(dsaddrs, &ds->ds_addrs); ds->ds_remotestr = remotestr; refcount_set(&ds->ds_count, 1); INIT_LIST_HEAD(&ds->ds_node); + ds->ds_net = net; ds->ds_clp = NULL; list_add(&ds->ds_node, &nfs4_data_server_cache); dprintk("%s add new data server %s\n", __func__, @@ -913,7 +914,10 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, u32 minor_version) { struct nfs_client *clp = ERR_PTR(-EIO); + struct nfs_client *mds_clp = mds_srv->nfs_client; + enum xprtsec_policies xprtsec_policy = mds_clp->cl_xprtsec.policy; struct nfs4_pnfs_ds_addr *da; + int ds_proto; int status = 0; dprintk("--> %s DS %s\n", __func__, ds->ds_remotestr); @@ -941,12 +945,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, .data = &xprtdata, }; - if (da->da_transport != clp->cl_proto && - clp->cl_proto != XPRT_TRANSPORT_TCP_TLS) - continue; - if (da->da_transport == XPRT_TRANSPORT_TCP && - mds_srv->nfs_client->cl_proto == - XPRT_TRANSPORT_TCP_TLS) { + if (xprt_args.ident == XPRT_TRANSPORT_TCP && + clp->cl_proto == XPRT_TRANSPORT_TCP_TLS) { struct sockaddr *addr = (struct sockaddr *)&da->da_addr; struct sockaddr_in *sin = @@ -977,7 +977,10 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, xprt_args.ident = XPRT_TRANSPORT_TCP_TLS; xprt_args.servername = servername; } - if (da->da_addr.ss_family != clp->cl_addr.ss_family) + if (xprt_args.ident != clp->cl_proto) + continue; + if (xprt_args.dstaddr->sa_family != + clp->cl_addr.ss_family) continue; /** @@ -991,15 +994,14 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, if (xprtdata.cred) put_cred(xprtdata.cred); } else { - if (da->da_transport == XPRT_TRANSPORT_TCP && - mds_srv->nfs_client->cl_proto == - XPRT_TRANSPORT_TCP_TLS) - da->da_transport = XPRT_TRANSPORT_TCP_TLS; - clp = nfs4_set_ds_client(mds_srv, - &da->da_addr, - da->da_addrlen, - da->da_transport, timeo, - retrans, minor_version); + ds_proto = da->da_transport; + if (ds_proto == XPRT_TRANSPORT_TCP && + xprtsec_policy != RPC_XPRTSEC_NONE) + ds_proto = XPRT_TRANSPORT_TCP_TLS; + + clp = nfs4_set_ds_client(mds_srv, &da->da_addr, + da->da_addrlen, ds_proto, + timeo, retrans, minor_version); if (IS_ERR(clp)) continue; @@ -1010,7 +1012,6 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, clp = ERR_PTR(-EIO); continue; } - } } diff --git a/fs/nfs/read.c b/fs/nfs/read.c index a142287d86f68..688a24e9bc8bb 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -56,7 +56,8 @@ static int nfs_return_empty_folio(struct folio *folio) { folio_zero_segment(folio, 0, folio_size(folio)); folio_mark_uptodate(folio); - folio_unlock(folio); + if (nfs_netfs_folio_unlock(folio)) + folio_unlock(folio); return 0; } diff --git a/fs/nfs/super.c b/fs/nfs/super.c index e1bcad5906ae7..4e72ee57fc8fc 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -1020,6 +1020,16 @@ int nfs_reconfigure(struct fs_context *fc) sync_filesystem(sb); + /* + * The SB_RDONLY flag has been removed from the superblock during + * mounts to prevent interference between different filesystems. + * Similarly, it is also necessary to ignore the SB_RDONLY flag + * during reconfiguration; otherwise, it may also result in the + * creation of redundant superblocks when mounting a directory with + * different rw and ro flags multiple times. + */ + fc->sb_flags_mask &= ~SB_RDONLY; + /* * Userspace mount programs that send binary options generally send * them populated with default values. We have no way to know which @@ -1277,8 +1287,17 @@ int nfs_get_tree_common(struct fs_context *fc) if (IS_ERR(server)) return PTR_ERR(server); + /* + * When NFS_MOUNT_UNSHARED is not set, NFS forces the sharing of a + * superblock among each filesystem that mounts sub-directories + * belonging to a single exported root path. + * To prevent interference between different filesystems, the + * SB_RDONLY flag should be removed from the superblock. + */ if (server->flags & NFS_MOUNT_UNSHARED) compare_super = NULL; + else + fc->sb_flags &= ~SB_RDONLY; /* -o noac implies -o sync */ if (server->flags & NFS_MOUNT_NOAC) diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c index 784f7c1d003bf..53d4cdf28ee00 100644 --- a/fs/nfs/sysfs.c +++ b/fs/nfs/sysfs.c @@ -189,6 +189,7 @@ static struct nfs_netns_client *nfs_netns_client_alloc(struct kobject *parent, return p; kobject_put(&p->kobject); + kobject_put(&p->nfs_net_kobj); } return NULL; } diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 7d03811f44a4b..ef69b15aa72e5 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -156,20 +156,10 @@ nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode) } } -static int -nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode) +static void nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode) { - int ret; - - if (!test_bit(PG_REMOVE, &req->wb_flags)) - return 0; - ret = nfs_page_group_lock(req); - if (ret) - return ret; if (test_and_clear_bit(PG_REMOVE, &req->wb_flags)) nfs_page_set_inode_ref(req, inode); - nfs_page_group_unlock(req); - return 0; } static struct nfs_page *nfs_folio_private_request(struct folio *folio) @@ -238,36 +228,6 @@ static struct nfs_page *nfs_folio_find_head_request(struct folio *folio) return req; } -static struct nfs_page *nfs_folio_find_and_lock_request(struct folio *folio) -{ - struct inode *inode = folio_file_mapping(folio)->host; - struct nfs_page *req, *head; - int ret; - - for (;;) { - req = nfs_folio_find_head_request(folio); - if (!req) - return req; - head = nfs_page_group_lock_head(req); - if (head != req) - nfs_release_request(req); - if (IS_ERR(head)) - return head; - ret = nfs_cancel_remove_inode(head, inode); - if (ret < 0) { - nfs_unlock_and_release_request(head); - return ERR_PTR(ret); - } - /* Ensure that nobody removed the request before we locked it */ - if (head == nfs_folio_private_request(folio)) - break; - if (folio_test_swapcache(folio)) - break; - nfs_unlock_and_release_request(head); - } - return head; -} - /* Adjust the file length if we're writing beyond the end */ static void nfs_grow_file(struct folio *folio, unsigned int offset, unsigned int count) @@ -548,6 +508,57 @@ void nfs_join_page_group(struct nfs_page *head, struct nfs_commit_info *cinfo, nfs_destroy_unlinked_subrequests(destroy_list, head, inode); } +/* + * nfs_unroll_locks - unlock all newly locked reqs and wait on @req + * @head: head request of page group, must be holding head lock + * @req: request that couldn't lock and needs to wait on the req bit lock + * + * This is a helper function for nfs_lock_and_join_requests + * returns 0 on success, < 0 on error. + */ +static void +nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req) +{ + struct nfs_page *tmp; + + /* relinquish all the locks successfully grabbed this run */ + for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { + if (!kref_read(&tmp->wb_kref)) + continue; + nfs_unlock_and_release_request(tmp); + } +} + +/* + * nfs_page_group_lock_subreq - try to lock a subrequest + * @head: head request of page group + * @subreq: request to lock + * + * This is a helper function for nfs_lock_and_join_requests which + * must be called with the head request and page group both locked. + * On error, it returns with the page group unlocked. + */ +static int +nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq) +{ + int ret; + + if (!kref_get_unless_zero(&subreq->wb_kref)) + return 0; + while (!nfs_lock_request(subreq)) { + nfs_page_group_unlock(head); + ret = nfs_wait_on_request(subreq); + if (!ret) + ret = nfs_page_group_lock(head); + if (ret < 0) { + nfs_unroll_locks(head, subreq); + nfs_release_request(subreq); + return ret; + } + } + return 0; +} + /* * nfs_lock_and_join_requests - join all subreqs to the head req * @folio: the folio used to lookup the "page group" of nfs_page structures @@ -566,30 +577,59 @@ void nfs_join_page_group(struct nfs_page *head, struct nfs_commit_info *cinfo, static struct nfs_page *nfs_lock_and_join_requests(struct folio *folio) { struct inode *inode = folio_file_mapping(folio)->host; - struct nfs_page *head; + struct nfs_page *head, *subreq; struct nfs_commit_info cinfo; int ret; - nfs_init_cinfo_from_inode(&cinfo, inode); /* * A reference is taken only on the head request which acts as a * reference to the whole page group - the group will not be destroyed * until the head reference is released. */ - head = nfs_folio_find_and_lock_request(folio); - if (IS_ERR_OR_NULL(head)) - return head; +retry: + head = nfs_folio_find_head_request(folio); + if (!head) + return NULL; - /* lock each request in the page group */ - ret = nfs_page_group_lock_subrequests(head); - if (ret < 0) { + while (!nfs_lock_request(head)) { + ret = nfs_wait_on_request(head); + if (ret < 0) { + nfs_release_request(head); + return ERR_PTR(ret); + } + } + + ret = nfs_page_group_lock(head); + if (ret < 0) + goto out_unlock; + + /* Ensure that nobody removed the request before we locked it */ + if (head != folio->private && !folio_test_swapcache(folio)) { + nfs_page_group_unlock(head); nfs_unlock_and_release_request(head); - return ERR_PTR(ret); + goto retry; } - nfs_join_page_group(head, &cinfo, inode); + nfs_cancel_remove_inode(head, inode); + + /* lock each request in the page group */ + for (subreq = head->wb_this_page; + subreq != head; + subreq = subreq->wb_this_page) { + ret = nfs_page_group_lock_subreq(head, subreq); + if (ret < 0) + goto out_unlock; + } + nfs_page_group_unlock(head); + + nfs_init_cinfo_from_inode(&cinfo, inode); + nfs_join_page_group(head, &cinfo, inode); return head; + +out_unlock: + nfs_unlock_and_release_request(head); + return ERR_PTR(ret); } static void nfs_write_error(struct nfs_page *req, int error) @@ -792,7 +832,8 @@ static void nfs_inode_remove_request(struct nfs_page *req) { struct nfs_inode *nfsi = NFS_I(nfs_page_to_inode(req)); - if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) { + nfs_page_group_lock(req); + if (nfs_page_group_sync_on_bit_locked(req, PG_REMOVE)) { struct folio *folio = nfs_page_to_folio(req->wb_head); struct address_space *mapping = folio_file_mapping(folio); @@ -804,6 +845,7 @@ static void nfs_inode_remove_request(struct nfs_page *req) } spin_unlock(&mapping->private_lock); } + nfs_page_group_unlock(req); if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) { atomic_long_dec(&nfsi->nrequests); @@ -1596,7 +1638,8 @@ static int nfs_writeback_done(struct rpc_task *task, /* Deal with the suid/sgid bit corner case */ if (nfs_should_remove_suid(inode)) { spin_lock(&inode->i_lock); - nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE + | NFS_INO_REVAL_FORCED); spin_unlock(&inode->i_lock); } return 0; diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c index 01d7fd108cf3d..59f119cce3dc6 100644 --- a/fs/nfsd/blocklayout.c +++ b/fs/nfsd/blocklayout.c @@ -117,7 +117,6 @@ static __be32 nfsd4_block_commit_blocks(struct inode *inode, struct nfsd4_layoutcommit *lcp, struct iomap *iomaps, int nr_iomaps) { - loff_t new_size = lcp->lc_last_wr + 1; struct iattr iattr = { .ia_valid = 0 }; int error; @@ -127,9 +126,9 @@ nfsd4_block_commit_blocks(struct inode *inode, struct nfsd4_layoutcommit *lcp, iattr.ia_valid |= ATTR_ATIME | ATTR_CTIME | ATTR_MTIME; iattr.ia_atime = iattr.ia_ctime = iattr.ia_mtime = lcp->lc_mtime; - if (new_size > i_size_read(inode)) { + if (lcp->lc_size_chg) { iattr.ia_valid |= ATTR_SIZE; - iattr.ia_size = new_size; + iattr.ia_size = lcp->lc_newsize; } error = inode->i_sb->s_export_op->commit_blocks(inode, iomaps, diff --git a/fs/nfsd/blocklayoutxdr.c b/fs/nfsd/blocklayoutxdr.c index 1ed2f691ebb90..dd35c472eb37d 100644 --- a/fs/nfsd/blocklayoutxdr.c +++ b/fs/nfsd/blocklayoutxdr.c @@ -29,8 +29,7 @@ nfsd4_block_encode_layoutget(struct xdr_stream *xdr, *p++ = cpu_to_be32(len); *p++ = cpu_to_be32(1); /* we always return a single extent */ - p = xdr_encode_opaque_fixed(p, &b->vol_id, - sizeof(struct nfsd4_deviceid)); + p = svcxdr_encode_deviceid4(p, &b->vol_id); p = xdr_encode_hyper(p, b->foff); p = xdr_encode_hyper(p, b->len); p = xdr_encode_hyper(p, b->soff); @@ -145,9 +144,7 @@ nfsd4_block_decode_layoutupdate(__be32 *p, u32 len, struct iomap **iomapp, for (i = 0; i < nr_iomaps; i++) { struct pnfs_block_extent bex; - memcpy(&bex.vol_id, p, sizeof(struct nfsd4_deviceid)); - p += XDR_QUADLEN(sizeof(struct nfsd4_deviceid)); - + p = svcxdr_decode_deviceid4(p, &bex.vol_id); p = xdr_decode_hyper(p, &bex.foff); if (bex.foff & (block_size - 1)) { dprintk("%s: unaligned offset 0x%llx\n", diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 4b5d998cbc2f4..f4e77859aa859 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -1071,28 +1071,62 @@ static struct svc_export *exp_find(struct cache_detail *cd, return exp; } -__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp) +/** + * check_xprtsec_policy - check if access to export is allowed by the + * xprtsec policy + * @exp: svc_export that is being accessed. + * @rqstp: svc_rqst attempting to access @exp. + * + * Helper function for check_nfsd_access(). Note that callers should be + * using check_nfsd_access() instead of calling this function directly. The + * one exception is fh_verify() since it has logic that may result in one + * or both of the helpers being skipped. + * + * Return values: + * %nfs_ok if access is granted, or + * %nfserr_acces or %nfserr_wrongsec if access is denied + */ +__be32 check_xprtsec_policy(struct svc_export *exp, struct svc_rqst *rqstp) { - struct exp_flavor_info *f, *end = exp->ex_flavors + exp->ex_nflavors; struct svc_xprt *xprt = rqstp->rq_xprt; if (exp->ex_xprtsec_modes & NFSEXP_XPRTSEC_NONE) { if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) - goto ok; + return nfs_ok; } if (exp->ex_xprtsec_modes & NFSEXP_XPRTSEC_TLS) { if (test_bit(XPT_TLS_SESSION, &xprt->xpt_flags) && !test_bit(XPT_PEER_AUTH, &xprt->xpt_flags)) - goto ok; + return nfs_ok; } if (exp->ex_xprtsec_modes & NFSEXP_XPRTSEC_MTLS) { if (test_bit(XPT_TLS_SESSION, &xprt->xpt_flags) && test_bit(XPT_PEER_AUTH, &xprt->xpt_flags)) - goto ok; + return nfs_ok; } - goto denied; -ok: + return rqstp->rq_vers < 4 ? nfserr_acces : nfserr_wrongsec; +} + +/** + * check_security_flavor - check if access to export is allowed by the + * xprtsec policy + * @exp: svc_export that is being accessed. + * @rqstp: svc_rqst attempting to access @exp. + * + * Helper function for check_nfsd_access(). Note that callers should be + * using check_nfsd_access() instead of calling this function directly. The + * one exception is fh_verify() since it has logic that may result in one + * or both of the helpers being skipped. + * + * Return values: + * %nfs_ok if access is granted, or + * %nfserr_acces or %nfserr_wrongsec if access is denied + */ +__be32 check_security_flavor(struct svc_export *exp, struct svc_rqst *rqstp) +{ + struct exp_flavor_info *f, *end = exp->ex_flavors + exp->ex_nflavors; + /* legacy gss-only clients are always OK: */ if (exp->ex_client == rqstp->rq_gssclient) return 0; @@ -1117,10 +1151,20 @@ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp) if (nfsd4_spo_must_allow(rqstp)) return 0; -denied: return rqstp->rq_vers < 4 ? nfserr_acces : nfserr_wrongsec; } +__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp) +{ + __be32 status; + + status = check_xprtsec_policy(exp, rqstp); + if (status != nfs_ok) + return status; + + return check_security_flavor(exp, rqstp); +} + /* * Uses rq_client and rq_gssclient to find an export; uses rq_client (an * auth_unix client) if it's available and has secinfo information; diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index ca9dc230ae3d0..4a48b2ad56067 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -100,6 +100,8 @@ struct svc_expkey { #define EX_WGATHER(exp) ((exp)->ex_flags & NFSEXP_GATHERED_WRITES) int nfsexp_flags(struct svc_rqst *rqstp, struct svc_export *exp); +__be32 check_xprtsec_policy(struct svc_export *exp, struct svc_rqst *rqstp); +__be32 check_security_flavor(struct svc_export *exp, struct svc_rqst *rqstp); __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp); /* diff --git a/fs/nfsd/flexfilelayout.c b/fs/nfsd/flexfilelayout.c index 3ca5304440ff0..0bc52e6bec394 100644 --- a/fs/nfsd/flexfilelayout.c +++ b/fs/nfsd/flexfilelayout.c @@ -125,6 +125,13 @@ nfsd4_ff_proc_getdeviceinfo(struct super_block *sb, struct svc_rqst *rqstp, return 0; } +static __be32 +nfsd4_ff_proc_layoutcommit(struct inode *inode, + struct nfsd4_layoutcommit *lcp) +{ + return nfs_ok; +} + const struct nfsd4_layout_ops ff_layout_ops = { .notify_types = NOTIFY_DEVICEID4_DELETE | NOTIFY_DEVICEID4_CHANGE, @@ -133,4 +140,5 @@ const struct nfsd4_layout_ops ff_layout_ops = { .encode_getdeviceinfo = nfsd4_ff_encode_getdeviceinfo, .proc_layoutget = nfsd4_ff_proc_layoutget, .encode_layoutget = nfsd4_ff_encode_layoutget, + .proc_layoutcommit = nfsd4_ff_proc_layoutcommit, }; diff --git a/fs/nfsd/flexfilelayoutxdr.c b/fs/nfsd/flexfilelayoutxdr.c index bb205328e043d..223a10f37898e 100644 --- a/fs/nfsd/flexfilelayoutxdr.c +++ b/fs/nfsd/flexfilelayoutxdr.c @@ -54,8 +54,7 @@ nfsd4_ff_encode_layoutget(struct xdr_stream *xdr, *p++ = cpu_to_be32(1); /* single mirror */ *p++ = cpu_to_be32(1); /* single data server */ - p = xdr_encode_opaque_fixed(p, &fl->deviceid, - sizeof(struct nfsd4_deviceid)); + p = svcxdr_encode_deviceid4(p, &fl->deviceid); *p++ = cpu_to_be32(1); /* efficiency */ diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c index 46a7f9b813e52..b02886f38925f 100644 --- a/fs/nfsd/lockd.c +++ b/fs/nfsd/lockd.c @@ -48,6 +48,21 @@ nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp, switch (nfserr) { case nfs_ok: return 0; + case nfserr_jukebox: + /* this error can indicate a presence of a conflicting + * delegation to an NLM lock request. Options are: + * (1) For now, drop this request and make the client + * retry. When delegation is returned, client's lock retry + * will complete. + * (2) NLM4_DENIED as per "spec" signals to the client + * that the lock is unavailable now but client can retry. + * Linux client implementation does not. It treats + * NLM4_DENIED same as NLM4_FAILED and errors the request. + * (3) For the future, treat this as blocked lock and try + * to callback when the delegation is returned but might + * not have a proper lock request to block on. + */ + fallthrough; case nfserr_dropit: return nlm_drop_reply; case nfserr_stale: diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index e8a80052cb1ba..308214378fd35 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -120,7 +120,6 @@ nfsd4_set_deviceid(struct nfsd4_deviceid *id, const struct svc_fh *fhp, id->fsid_idx = fhp->fh_export->ex_devid_map->idx; id->generation = device_generation; - id->pad = 0; return 0; } diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index b3eca08f15b13..886c092675442 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -978,10 +978,11 @@ nfsd4_read(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, static void nfsd4_read_release(union nfsd4_op_u *u) { - if (u->read.rd_nf) + if (u->read.rd_nf) { + trace_nfsd_read_done(u->read.rd_rqstp, u->read.rd_fhp, + u->read.rd_offset, u->read.rd_length); nfsd_file_put(u->read.rd_nf); - trace_nfsd_read_done(u->read.rd_rqstp, u->read.rd_fhp, - u->read.rd_offset, u->read.rd_length); + } } static __be32 @@ -1131,6 +1132,7 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, }; struct inode *inode; __be32 status = nfs_ok; + bool save_no_wcc; int err; if (setattr->sa_iattr.ia_valid & ATTR_SIZE) { @@ -1156,8 +1158,11 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) goto out; + save_no_wcc = cstate->current_fh.fh_no_wcc; + cstate->current_fh.fh_no_wcc = true; status = nfsd_setattr(rqstp, &cstate->current_fh, &attrs, 0, (time64_t)0); + cstate->current_fh.fh_no_wcc = save_no_wcc; if (!status) status = nfserrno(attrs.na_labelerr); if (!status) @@ -1364,7 +1369,7 @@ static __be32 nfsd4_ssc_setup_dul(struct nfsd_net *nn, char *ipaddr, return 0; } if (work) { - strscpy(work->nsui_ipaddr, ipaddr, sizeof(work->nsui_ipaddr) - 1); + strscpy(work->nsui_ipaddr, ipaddr, sizeof(work->nsui_ipaddr)); refcount_set(&work->nsui_refcnt, 2); work->nsui_busy = true; list_add_tail(&work->nsui_list, &nn->nfsd_ssc_mount_list); @@ -2304,7 +2309,6 @@ nfsd4_layoutcommit(struct svc_rqst *rqstp, const struct nfsd4_layout_seg *seg = &lcp->lc_seg; struct svc_fh *current_fh = &cstate->current_fh; const struct nfsd4_layout_ops *ops; - loff_t new_size = lcp->lc_last_wr + 1; struct inode *inode; struct nfs4_layout_stateid *ls; __be32 nfserr; @@ -2320,18 +2324,20 @@ nfsd4_layoutcommit(struct svc_rqst *rqstp, goto out; inode = d_inode(current_fh->fh_dentry); - nfserr = nfserr_inval; - if (new_size <= seg->offset) { - dprintk("pnfsd: last write before layout segment\n"); - goto out; - } - if (new_size > seg->offset + seg->length) { - dprintk("pnfsd: last write beyond layout segment\n"); - goto out; - } - if (!lcp->lc_newoffset && new_size > i_size_read(inode)) { - dprintk("pnfsd: layoutcommit beyond EOF\n"); - goto out; + lcp->lc_size_chg = false; + if (lcp->lc_newoffset) { + loff_t new_size = lcp->lc_last_wr + 1; + + nfserr = nfserr_inval; + if (new_size <= seg->offset) + goto out; + if (new_size > seg->offset + seg->length) + goto out; + + if (new_size > i_size_read(inode)) { + lcp->lc_size_chg = true; + lcp->lc_newsize = new_size; + } } nfserr = nfsd4_preprocess_layout_stateid(rqstp, cstate, &lcp->lc_sid, @@ -2348,13 +2354,6 @@ nfsd4_layoutcommit(struct svc_rqst *rqstp, /* LAYOUTCOMMIT does not require any serialization */ mutex_unlock(&ls->ls_mutex); - if (new_size > i_size_read(inode)) { - lcp->lc_size_chg = 1; - lcp->lc_newsize = new_size; - } else { - lcp->lc_size_chg = 0; - } - nfserr = ops->proc_layoutcommit(inode, lcp); nfs4_put_stid(&ls->ls_stid); out: @@ -3580,7 +3579,8 @@ bool nfsd4_spo_must_allow(struct svc_rqst *rqstp) struct nfs4_op_map *allow = &cstate->clp->cl_spo_must_allow; u32 opiter; - if (!cstate->minorversion) + if (rqstp->rq_procinfo != &nfsd_version4.vs_proc[NFSPROC4_COMPOUND] || + cstate->minorversion == 0) return false; if (cstate->spo_must_allowed) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index e2875706e6bfd..e6d0c3b1169aa 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1496,7 +1496,8 @@ static void nfs4_free_ol_stateid(struct nfs4_stid *stid) release_all_access(stp); if (stp->st_stateowner) nfs4_put_stateowner(stp->st_stateowner); - WARN_ON(!list_empty(&stid->sc_cp_list)); + if (!list_empty(&stid->sc_cp_list)) + nfs4_free_cpntf_statelist(stid->sc_client->net, stid); kmem_cache_free(stateid_slab, stid); } @@ -4282,10 +4283,16 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp, } status = nfs_ok; if (conf) { - old = unconf; - unhash_client_locked(old); - nfsd4_change_callback(conf, &unconf->cl_cb_conn); - } else { + if (get_client_locked(conf) == nfs_ok) { + old = unconf; + unhash_client_locked(old); + nfsd4_change_callback(conf, &unconf->cl_cb_conn); + } else { + conf = NULL; + } + } + + if (!conf) { old = find_confirmed_client_by_name(&unconf->cl_name, nn); if (old) { status = nfserr_clid_inuse; @@ -4302,10 +4309,14 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp, } trace_nfsd_clid_replaced(&old->cl_clientid); } + status = get_client_locked(unconf); + if (status != nfs_ok) { + old = NULL; + goto out; + } move_to_confirmed(unconf); conf = unconf; } - get_client_locked(conf); spin_unlock(&nn->client_lock); if (conf == unconf) fsnotify_dentry(conf->cl_nfsd_info_dentry, FS_MODIFY); @@ -5765,6 +5776,20 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf status = nfs4_check_deleg(cl, open, &dp); if (status) goto out; + if (dp && nfsd4_is_deleg_cur(open) && + (dp->dl_stid.sc_file != fp)) { + /* + * RFC8881 section 8.2.4 mandates the server to return + * NFS4ERR_BAD_STATEID if the selected table entry does + * not match the current filehandle. However returning + * NFS4ERR_BAD_STATEID in the OPEN can cause the client + * to repeatedly retry the operation with the same + * stateid, since the stateid itself is valid. To avoid + * this situation NFSD returns NFS4ERR_INVAL instead. + */ + status = nfserr_inval; + goto out; + } stp = nfsd4_find_and_lock_existing_open(fp, open); } else { open->op_file = NULL; diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 76dfbb99277f0..3eff780fd8da0 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -566,18 +566,6 @@ nfsd4_decode_state_owner4(struct nfsd4_compoundargs *argp, } #ifdef CONFIG_NFSD_PNFS -static __be32 -nfsd4_decode_deviceid4(struct nfsd4_compoundargs *argp, - struct nfsd4_deviceid *devid) -{ - __be32 *p; - - p = xdr_inline_decode(argp->xdr, NFS4_DEVICEID4_SIZE); - if (!p) - return nfserr_bad_xdr; - memcpy(devid, p, sizeof(*devid)); - return nfs_ok; -} static __be32 nfsd4_decode_layoutupdate4(struct nfsd4_compoundargs *argp, @@ -1733,7 +1721,7 @@ nfsd4_decode_getdeviceinfo(struct nfsd4_compoundargs *argp, __be32 status; memset(gdev, 0, sizeof(*gdev)); - status = nfsd4_decode_deviceid4(argp, &gdev->gd_devid); + status = nfsd4_decode_deviceid4(argp->xdr, &gdev->gd_devid); if (status) return status; if (xdr_stream_decode_u32(argp->xdr, &gdev->gd_layout_type) < 0) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index c2495d98c1892..283c1a60c8460 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -370,6 +370,16 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) if (error) goto out; + /* + * NLM is allowed to bypass the xprtsec policy check because lockd + * doesn't support xprtsec. + */ + if (!(access & NFSD_MAY_LOCK)) { + error = check_xprtsec_policy(exp, rqstp); + if (error) + goto out; + } + /* * pseudoflavor restrictions are not enforced on NLM, * which clients virtually always use auth_sys for, @@ -386,7 +396,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) && exp->ex_path.dentry == dentry) goto skip_pseudoflavor_check; - error = check_nfsd_access(exp, rqstp); + error = check_security_flavor(exp, rqstp); if (error) goto out; diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 710a54c7dffc5..623f522b15659 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -432,13 +432,13 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) if (ret) goto out_filecache; +#ifdef CONFIG_NFSD_V4_2_INTER_SSC + nfsd4_ssc_init_umount_work(nn); +#endif ret = nfs4_state_start_net(net); if (ret) goto out_reply_cache; -#ifdef CONFIG_NFSD_V4_2_INTER_SSC - nfsd4_ssc_init_umount_work(nn); -#endif nn->nfsd_net_up = true; return 0; diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index b3e51d88faff4..5ee7149ceaa5a 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -480,7 +480,7 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, int accmode = NFSD_MAY_SATTR; umode_t ftype = 0; __be32 err; - int host_err; + int host_err = 0; bool get_write_count; bool size_change = (iap->ia_valid & ATTR_SIZE); int retries; @@ -538,6 +538,9 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, } inode_lock(inode); + err = fh_fill_pre_attrs(fhp); + if (err) + goto out_unlock; for (retries = 1;;) { struct iattr attrs; @@ -565,13 +568,15 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, attr->na_aclerr = set_posix_acl(&nop_mnt_idmap, dentry, ACL_TYPE_DEFAULT, attr->na_dpacl); + fh_fill_post_attrs(fhp); +out_unlock: inode_unlock(inode); if (size_change) put_write_access(inode); out: if (!host_err) host_err = commit_metadata(fhp); - return nfserrno(host_err); + return err != 0 ? err : nfserrno(host_err); } #if defined(CONFIG_NFSD_V4) @@ -1965,11 +1970,9 @@ nfsd_unlink(struct svc_rqst *rqstp, struct svc_fh *fhp, int type, err = nfserr_file_open; else err = nfserr_acces; - } else { - err = nfserrno(host_err); } out: - return err; + return err != nfs_ok ? err : nfserrno(host_err); out_unlock: inode_unlock(dirp); goto out_drop_write; diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 144e05efd14c3..a83a52b0f5181 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -459,9 +459,43 @@ struct nfsd4_reclaim_complete { struct nfsd4_deviceid { u64 fsid_idx; u32 generation; - u32 pad; }; +static inline __be32 * +svcxdr_encode_deviceid4(__be32 *p, const struct nfsd4_deviceid *devid) +{ + __be64 *q = (__be64 *)p; + + *q = (__force __be64)devid->fsid_idx; + p += 2; + *p++ = (__force __be32)devid->generation; + *p++ = xdr_zero; + return p; +} + +static inline __be32 * +svcxdr_decode_deviceid4(__be32 *p, struct nfsd4_deviceid *devid) +{ + __be64 *q = (__be64 *)p; + + devid->fsid_idx = (__force u64)(*q); + p += 2; + devid->generation = (__force u32)(*p++); + p++; /* NFSD does not use the remaining octets */ + return p; +} + +static inline __be32 +nfsd4_decode_deviceid4(struct xdr_stream *xdr, struct nfsd4_deviceid *devid) +{ + __be32 *p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE); + + if (unlikely(!p)) + return nfserr_bad_xdr; + svcxdr_decode_deviceid4(p, devid); + return nfs_ok; +} + struct nfsd4_layout_seg { u32 iomode; u64 offset; diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c index dbd27a44632fa..5e70a3478afe0 100644 --- a/fs/nilfs2/btree.c +++ b/fs/nilfs2/btree.c @@ -2094,11 +2094,13 @@ static int nilfs_btree_propagate(struct nilfs_bmap *btree, ret = nilfs_btree_do_lookup(btree, path, key, NULL, level + 1, 0); if (ret < 0) { - if (unlikely(ret == -ENOENT)) + if (unlikely(ret == -ENOENT)) { nilfs_crit(btree->b_inode->i_sb, "writing node/leaf block does not appear in b-tree (ino=%lu) at key=%llu, level=%d", btree->b_inode->i_ino, (unsigned long long)key, level); + ret = -EINVAL; + } goto out; } diff --git a/fs/nilfs2/direct.c b/fs/nilfs2/direct.c index 893ab36824cc2..2d8dc6b35b547 100644 --- a/fs/nilfs2/direct.c +++ b/fs/nilfs2/direct.c @@ -273,6 +273,9 @@ static int nilfs_direct_propagate(struct nilfs_bmap *bmap, dat = nilfs_bmap_get_dat(bmap); key = nilfs_bmap_data_get_key(bmap, bh); ptr = nilfs_direct_get_ptr(bmap, key); + if (ptr == NILFS_BMAP_INVALID_PTR) + return -EINVAL; + if (!buffer_nilfs_volatile(bh)) { oldreq.pr_entry_nr = ptr; newreq.pr_entry_nr = ptr; diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 5f2d73f36e0d2..8c47c13c8e716 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -517,11 +517,18 @@ static int __nilfs_read_inode(struct super_block *sb, inode->i_op = &nilfs_symlink_inode_operations; inode_nohighmem(inode); inode->i_mapping->a_ops = &nilfs_aops; - } else { + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { inode->i_op = &nilfs_special_inode_operations; init_special_inode( inode, inode->i_mode, huge_decode_dev(le64_to_cpu(raw_inode->i_device_code))); + } else { + nilfs_error(sb, + "invalid file type bits in mode 0%o for inode %lu", + inode->i_mode, ino); + err = -EIO; + goto failed_unmap; } nilfs_ifile_unmap_inode(root->ifile, ino, bh); brelse(bh); diff --git a/fs/nilfs2/sysfs.c b/fs/nilfs2/sysfs.c index 905c7eadf9676..59fda8ce0790b 100644 --- a/fs/nilfs2/sysfs.c +++ b/fs/nilfs2/sysfs.c @@ -1075,7 +1075,7 @@ void nilfs_sysfs_delete_device_group(struct the_nilfs *nilfs) ************************************************************************/ static ssize_t nilfs_feature_revision_show(struct kobject *kobj, - struct attribute *attr, char *buf) + struct kobj_attribute *attr, char *buf) { return sysfs_emit(buf, "%d.%d\n", NILFS_CURRENT_REV, NILFS_MINOR_REV); @@ -1087,7 +1087,7 @@ static const char features_readme_str[] = "(1) revision\n\tshow current revision of NILFS file system driver.\n"; static ssize_t nilfs_feature_README_show(struct kobject *kobj, - struct attribute *attr, + struct kobj_attribute *attr, char *buf) { return sysfs_emit(buf, features_readme_str); diff --git a/fs/nilfs2/sysfs.h b/fs/nilfs2/sysfs.h index 78a87a016928b..d370cd5cce3f5 100644 --- a/fs/nilfs2/sysfs.h +++ b/fs/nilfs2/sysfs.h @@ -50,16 +50,16 @@ struct nilfs_sysfs_dev_subgroups { struct completion sg_segments_kobj_unregister; }; -#define NILFS_COMMON_ATTR_STRUCT(name) \ +#define NILFS_KOBJ_ATTR_STRUCT(name) \ struct nilfs_##name##_attr { \ struct attribute attr; \ - ssize_t (*show)(struct kobject *, struct attribute *, \ + ssize_t (*show)(struct kobject *, struct kobj_attribute *, \ char *); \ - ssize_t (*store)(struct kobject *, struct attribute *, \ + ssize_t (*store)(struct kobject *, struct kobj_attribute *, \ const char *, size_t); \ } -NILFS_COMMON_ATTR_STRUCT(feature); +NILFS_KOBJ_ATTR_STRUCT(feature); #define NILFS_DEV_ATTR_STRUCT(name) \ struct nilfs_##name##_attr { \ diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index be41e26b78246..05fdbbc63e1f5 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -680,8 +680,6 @@ int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb, char *data) int blocksize; int err; - down_write(&nilfs->ns_sem); - blocksize = sb_min_blocksize(sb, NILFS_MIN_BLOCK_SIZE); if (!blocksize) { nilfs_err(sb, "unable to set blocksize"); @@ -757,7 +755,6 @@ int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb, char *data) set_nilfs_init(nilfs); err = 0; out: - up_write(&nilfs->ns_sem); return err; failed_sbh: diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index 9dac7f6e72d2b..723ff9cad9ed9 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -445,7 +445,13 @@ static int fanotify_encode_fh(struct fanotify_fh *fh, struct inode *inode, dwords = fh_len >> 2; type = exportfs_encode_fid(inode, buf, &dwords); err = -EINVAL; - if (type <= 0 || type == FILEID_INVALID || fh_len != dwords << 2) + /* + * Unlike file_handle, type and len of struct fanotify_fh are u8. + * Traditionally, filesystem return handle_type < 0xff, but there + * is no enforecement for that in vfs. + */ + BUILD_BUG_ON(MAX_HANDLE_SZ > 0xff || FILEID_INVALID > 0xff); + if (type <= 0 || type >= FILEID_INVALID || fh_len != dwords << 2) goto out_err; fh->type = type; diff --git a/fs/notify/fdinfo.c b/fs/notify/fdinfo.c index 26655572975d3..1aa7de55094cd 100644 --- a/fs/notify/fdinfo.c +++ b/fs/notify/fdinfo.c @@ -17,6 +17,7 @@ #include "fanotify/fanotify.h" #include "fdinfo.h" #include "fsnotify.h" +#include "../internal.h" #if defined(CONFIG_PROC_FS) @@ -50,7 +51,12 @@ static void show_mark_fhandle(struct seq_file *m, struct inode *inode) f.handle.handle_bytes = sizeof(f.pad); size = f.handle.handle_bytes >> 2; + if (!super_trylock_shared(inode->i_sb)) + return; + ret = exportfs_encode_fid(inode, (struct fid *)f.handle.f_handle, &size); + up_read(&inode->i_sb->s_umount); + if ((ret == FILEID_INVALID) || (ret < 0)) return; diff --git a/fs/ntfs3/bitmap.c b/fs/ntfs3/bitmap.c index cf4fe21a50399..29b585f443f3e 100644 --- a/fs/ntfs3/bitmap.c +++ b/fs/ntfs3/bitmap.c @@ -1399,6 +1399,7 @@ int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits) mark_buffer_dirty(bh); unlock_buffer(bh); /* err = sync_dirty_buffer(bh); */ + put_bh(bh); b0 = 0; bits -= op; diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index e1b856ecce61d..6b93c909bdc9e 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -304,6 +304,9 @@ static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi, if (sbi->options->nohidden && (fname->dup.fa & FILE_ATTRIBUTE_HIDDEN)) return true; + if (fname->name_len + sizeof(struct NTFS_DE) > le16_to_cpu(e->size)) + return true; + name_len = ntfs_utf16_to_nls(sbi, fname->name, fname->name_len, name, PATH_MAX); if (name_len <= 0) { diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 4aea458216117..a7fe2e02c32ee 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -299,7 +299,10 @@ static int ntfs_file_mmap(struct file *file, struct vm_area_struct *vma) } if (ni->i_valid < to) { - inode_lock(inode); + if (!inode_trylock(inode)) { + err = -EAGAIN; + goto out; + } err = ntfs_extend_initialized_size(file, ni, ni->i_valid, to); inode_unlock(inode); diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 3c876c468c2c4..5c45fec832b00 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -3058,8 +3058,7 @@ int ni_add_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, * ni_rename - Remove one name and insert new name. */ int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, - struct ntfs_inode *ni, struct NTFS_DE *de, struct NTFS_DE *new_de, - bool *is_bad) + struct ntfs_inode *ni, struct NTFS_DE *de, struct NTFS_DE *new_de) { int err; struct NTFS_DE *de2 = NULL; @@ -3082,8 +3081,8 @@ int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, err = ni_add_name(new_dir_ni, ni, new_de); if (!err) { err = ni_remove_name(dir_ni, ni, de, &de2, &undo); - if (err && ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)) - *is_bad = true; + WARN_ON(err && ni_remove_name(new_dir_ni, ni, new_de, &de2, + &undo)); } /* diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 28aae6ea1e615..f227db9f76c2b 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -1510,6 +1510,16 @@ static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, bmp_size = bmp_size_v = le32_to_cpu(bmp->res.data_size); } + /* + * Index blocks exist, but $BITMAP has zero valid bits. + * This implies an on-disk corruption and must be rejected. + */ + if (in->name == I30_NAME && + unlikely(bmp_size_v == 0 && indx->alloc_run.count)) { + err = -EINVAL; + goto out1; + } + bit = bmp_size << 3; } @@ -2184,6 +2194,10 @@ static int indx_get_entry_to_replace(struct ntfs_index *indx, e = hdr_first_de(&n->index->ihdr); fnd_push(fnd, n, e); + if (!e) { + err = -EINVAL; + goto out; + } if (!de_is_last(e)) { /* @@ -2205,6 +2219,10 @@ static int indx_get_entry_to_replace(struct ntfs_index *indx, n = fnd->nodes[level]; te = hdr_first_de(&n->index->ihdr); + if (!te) { + err = -EINVAL; + goto out; + } /* Copy the candidate entry into the replacement entry buffer. */ re = kmalloc(le16_to_cpu(te->size) + sizeof(u64), GFP_NOFS); if (!re) { diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index af7c0cbba74e3..a967babea8a59 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -464,6 +464,7 @@ static struct inode *ntfs_read_mft(struct inode *inode, fname->home.seq == cpu_to_le16(MFT_REC_EXTEND)) { /* Records in $Extend are not a files or general directories. */ inode->i_op = &ntfs_file_inode_operations; + mode = S_IFREG; } else { err = -EINVAL; goto out; @@ -1130,10 +1131,10 @@ int inode_write_data(struct inode *inode, const void *data, size_t bytes) * Number of bytes for REPARSE_DATA_BUFFER(IO_REPARSE_TAG_SYMLINK) * for unicode string of @uni_len length. */ -static inline u32 ntfs_reparse_bytes(u32 uni_len) +static inline u32 ntfs_reparse_bytes(u32 uni_len, bool is_absolute) { /* Header + unicode string + decorated unicode string. */ - return sizeof(short) * (2 * uni_len + 4) + + return sizeof(short) * (2 * uni_len + (is_absolute ? 4 : 0)) + offsetof(struct REPARSE_DATA_BUFFER, SymbolicLinkReparseBuffer.PathBuffer); } @@ -1146,8 +1147,11 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, struct REPARSE_DATA_BUFFER *rp; __le16 *rp_name; typeof(rp->SymbolicLinkReparseBuffer) *rs; + bool is_absolute; - rp = kzalloc(ntfs_reparse_bytes(2 * size + 2), GFP_NOFS); + is_absolute = (strlen(symname) > 1 && symname[1] == ':'); + + rp = kzalloc(ntfs_reparse_bytes(2 * size + 2, is_absolute), GFP_NOFS); if (!rp) return ERR_PTR(-ENOMEM); @@ -1162,7 +1166,7 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, goto out; /* err = the length of unicode name of symlink. */ - *nsize = ntfs_reparse_bytes(err); + *nsize = ntfs_reparse_bytes(err, is_absolute); if (*nsize > sbi->reparse.max_size) { err = -EFBIG; @@ -1182,7 +1186,7 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, /* PrintName + SubstituteName. */ rs->SubstituteNameOffset = cpu_to_le16(sizeof(short) * err); - rs->SubstituteNameLength = cpu_to_le16(sizeof(short) * err + 8); + rs->SubstituteNameLength = cpu_to_le16(sizeof(short) * err + (is_absolute ? 8 : 0)); rs->PrintNameLength = rs->SubstituteNameOffset; /* @@ -1190,16 +1194,18 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, * parse this path. * 0-absolute path 1- relative path (SYMLINK_FLAG_RELATIVE). */ - rs->Flags = 0; + rs->Flags = cpu_to_le32(is_absolute ? 0 : SYMLINK_FLAG_RELATIVE); - memmove(rp_name + err + 4, rp_name, sizeof(short) * err); + memmove(rp_name + err + (is_absolute ? 4 : 0), rp_name, sizeof(short) * err); - /* Decorate SubstituteName. */ - rp_name += err; - rp_name[0] = cpu_to_le16('\\'); - rp_name[1] = cpu_to_le16('?'); - rp_name[2] = cpu_to_le16('?'); - rp_name[3] = cpu_to_le16('\\'); + if (is_absolute) { + /* Decorate SubstituteName. */ + rp_name += err; + rp_name[0] = cpu_to_le16('\\'); + rp_name[1] = cpu_to_le16('?'); + rp_name[2] = cpu_to_le16('?'); + rp_name[3] = cpu_to_le16('\\'); + } return rp; out: diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c index 61c4da8e6c3de..f5901c23ab937 100644 --- a/fs/ntfs3/namei.c +++ b/fs/ntfs3/namei.c @@ -261,7 +261,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, struct ntfs_inode *ni = ntfs_i(inode); struct inode *new_inode = d_inode(new_dentry); struct NTFS_DE *de, *new_de; - bool is_same, is_bad; + bool is_same; /* * de - memory of PATH_MAX bytes: * [0-1024) - original name (dentry->d_name) @@ -330,12 +330,8 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, if (dir_ni != new_dir_ni) ni_lock_dir2(new_dir_ni); - is_bad = false; - err = ni_rename(dir_ni, new_dir_ni, ni, de, new_de, &is_bad); - if (is_bad) { - /* Restore after failed rename failed too. */ - _ntfs_bad_inode(inode); - } else if (!err) { + err = ni_rename(dir_ni, new_dir_ni, ni, de, new_de); + if (!err) { simple_rename_timestamp(dir, dentry, new_dir, new_dentry); mark_inode_dirty(inode); mark_inode_dirty(dir); diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index 6556b943582c8..9627d5436920e 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -578,8 +578,7 @@ int ni_add_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, struct NTFS_DE *de); int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, - struct ntfs_inode *ni, struct NTFS_DE *de, struct NTFS_DE *new_de, - bool *is_bad); + struct ntfs_inode *ni, struct NTFS_DE *de, struct NTFS_DE *new_de); bool ni_is_dirty(struct inode *inode); diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 44e93ad491ba7..0139124578d9e 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "debug.h" #include "ntfs.h" @@ -982,12 +983,16 @@ int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, if (!dlcn) return -EINVAL; - lcn = prev_lcn + dlcn; + + if (check_add_overflow(prev_lcn, dlcn, &lcn)) + return -EINVAL; prev_lcn = lcn; } else return -EINVAL; - next_vcn = vcn64 + len; + if (check_add_overflow(vcn64, len, &next_vcn)) + return -EINVAL; + /* Check boundary. */ if (next_vcn > evcn + 1) return -EINVAL; @@ -1151,7 +1156,8 @@ int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn) return -EINVAL; run_buf += size_size + offset_size; - vcn64 += len; + if (check_add_overflow(vcn64, len, &vcn64)) + return -EINVAL; #ifndef CONFIG_NTFS3_64BIT_CLUSTER if (vcn64 > 0x100000000ull) diff --git a/fs/ocfs2/extent_map.c b/fs/ocfs2/extent_map.c index f7672472fa827..5e86c7e2c8212 100644 --- a/fs/ocfs2/extent_map.c +++ b/fs/ocfs2/extent_map.c @@ -696,6 +696,8 @@ int ocfs2_extent_map_get_blocks(struct inode *inode, u64 v_blkno, u64 *p_blkno, * it not only handles the fiemap for inlined files, but also deals * with the fast symlink, cause they have no difference for extent * mapping per se. + * + * Must be called with ip_alloc_sem semaphore held. */ static int ocfs2_fiemap_inline(struct inode *inode, struct buffer_head *di_bh, struct fiemap_extent_info *fieinfo, @@ -707,6 +709,7 @@ static int ocfs2_fiemap_inline(struct inode *inode, struct buffer_head *di_bh, u64 phys; u32 flags = FIEMAP_EXTENT_DATA_INLINE|FIEMAP_EXTENT_LAST; struct ocfs2_inode_info *oi = OCFS2_I(inode); + lockdep_assert_held_read(&oi->ip_alloc_sem); di = (struct ocfs2_dinode *)di_bh->b_data; if (ocfs2_inode_is_fast_symlink(inode)) @@ -722,8 +725,11 @@ static int ocfs2_fiemap_inline(struct inode *inode, struct buffer_head *di_bh, phys += offsetof(struct ocfs2_dinode, id2.i_data.id_data); + /* Release the ip_alloc_sem to prevent deadlock on page fault */ + up_read(&OCFS2_I(inode)->ip_alloc_sem); ret = fiemap_fill_next_extent(fieinfo, 0, phys, id_count, flags); + down_read(&OCFS2_I(inode)->ip_alloc_sem); if (ret < 0) return ret; } @@ -792,9 +798,11 @@ int ocfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, len_bytes = (u64)le16_to_cpu(rec.e_leaf_clusters) << osb->s_clustersize_bits; phys_bytes = le64_to_cpu(rec.e_blkno) << osb->sb->s_blocksize_bits; virt_bytes = (u64)le32_to_cpu(rec.e_cpos) << osb->s_clustersize_bits; - + /* Release the ip_alloc_sem to prevent deadlock on page fault */ + up_read(&OCFS2_I(inode)->ip_alloc_sem); ret = fiemap_fill_next_extent(fieinfo, virt_bytes, phys_bytes, len_bytes, fe_flags); + down_read(&OCFS2_I(inode)->ip_alloc_sem); if (ret) break; diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index 999111bfc2717..c561a8a6493e7 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -1205,6 +1205,9 @@ static void ocfs2_clear_inode(struct inode *inode) * the journal is flushed before journal shutdown. Thus it is safe to * have inodes get cleaned up after journal shutdown. */ + if (!osb->journal) + return; + jbd2_journal_release_jbd_inode(osb->journal->j_journal, &oi->ip_jinode); } diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c index cbe3c12ff5f75..b246e53271114 100644 --- a/fs/ocfs2/journal.c +++ b/fs/ocfs2/journal.c @@ -174,7 +174,7 @@ int ocfs2_recovery_init(struct ocfs2_super *osb) struct ocfs2_recovery_map *rm; mutex_init(&osb->recovery_lock); - osb->disable_recovery = 0; + osb->recovery_state = OCFS2_REC_ENABLED; osb->recovery_thread_task = NULL; init_waitqueue_head(&osb->recovery_event); @@ -190,31 +190,53 @@ int ocfs2_recovery_init(struct ocfs2_super *osb) return 0; } -/* we can't grab the goofy sem lock from inside wait_event, so we use - * memory barriers to make sure that we'll see the null task before - * being woken up */ static int ocfs2_recovery_thread_running(struct ocfs2_super *osb) { - mb(); return osb->recovery_thread_task != NULL; } -void ocfs2_recovery_exit(struct ocfs2_super *osb) +static void ocfs2_recovery_disable(struct ocfs2_super *osb, + enum ocfs2_recovery_state state) { - struct ocfs2_recovery_map *rm; - - /* disable any new recovery threads and wait for any currently - * running ones to exit. Do this before setting the vol_state. */ mutex_lock(&osb->recovery_lock); - osb->disable_recovery = 1; + /* + * If recovery thread is not running, we can directly transition to + * final state. + */ + if (!ocfs2_recovery_thread_running(osb)) { + osb->recovery_state = state + 1; + goto out_lock; + } + osb->recovery_state = state; + /* Wait for recovery thread to acknowledge state transition */ + wait_event_cmd(osb->recovery_event, + !ocfs2_recovery_thread_running(osb) || + osb->recovery_state >= state + 1, + mutex_unlock(&osb->recovery_lock), + mutex_lock(&osb->recovery_lock)); +out_lock: mutex_unlock(&osb->recovery_lock); - wait_event(osb->recovery_event, !ocfs2_recovery_thread_running(osb)); - /* At this point, we know that no more recovery threads can be - * launched, so wait for any recovery completion work to - * complete. */ + /* + * At this point we know that no more recovery work can be queued so + * wait for any recovery completion work to complete. + */ if (osb->ocfs2_wq) flush_workqueue(osb->ocfs2_wq); +} + +void ocfs2_recovery_disable_quota(struct ocfs2_super *osb) +{ + ocfs2_recovery_disable(osb, OCFS2_REC_QUOTA_WANT_DISABLE); +} + +void ocfs2_recovery_exit(struct ocfs2_super *osb) +{ + struct ocfs2_recovery_map *rm; + + /* disable any new recovery threads and wait for any currently + * running ones to exit. Do this before setting the vol_state. */ + ocfs2_recovery_disable(osb, OCFS2_REC_WANT_DISABLE); /* * Now that recovery is shut down, and the osb is about to be @@ -1472,6 +1494,18 @@ static int __ocfs2_recovery_thread(void *arg) } } restart: + if (quota_enabled) { + mutex_lock(&osb->recovery_lock); + /* Confirm that recovery thread will no longer recover quotas */ + if (osb->recovery_state == OCFS2_REC_QUOTA_WANT_DISABLE) { + osb->recovery_state = OCFS2_REC_QUOTA_DISABLED; + wake_up(&osb->recovery_event); + } + if (osb->recovery_state >= OCFS2_REC_QUOTA_DISABLED) + quota_enabled = 0; + mutex_unlock(&osb->recovery_lock); + } + status = ocfs2_super_lock(osb, 1); if (status < 0) { mlog_errno(status); @@ -1569,27 +1603,29 @@ static int __ocfs2_recovery_thread(void *arg) ocfs2_free_replay_slots(osb); osb->recovery_thread_task = NULL; - mb(); /* sync with ocfs2_recovery_thread_running */ + if (osb->recovery_state == OCFS2_REC_WANT_DISABLE) + osb->recovery_state = OCFS2_REC_DISABLED; wake_up(&osb->recovery_event); mutex_unlock(&osb->recovery_lock); - if (quota_enabled) - kfree(rm_quota); + kfree(rm_quota); return status; } void ocfs2_recovery_thread(struct ocfs2_super *osb, int node_num) { + int was_set = -1; + mutex_lock(&osb->recovery_lock); + if (osb->recovery_state < OCFS2_REC_WANT_DISABLE) + was_set = ocfs2_recovery_map_set(osb, node_num); trace_ocfs2_recovery_thread(node_num, osb->node_num, - osb->disable_recovery, osb->recovery_thread_task, - osb->disable_recovery ? - -1 : ocfs2_recovery_map_set(osb, node_num)); + osb->recovery_state, osb->recovery_thread_task, was_set); - if (osb->disable_recovery) + if (osb->recovery_state >= OCFS2_REC_WANT_DISABLE) goto out; if (osb->recovery_thread_task) diff --git a/fs/ocfs2/journal.h b/fs/ocfs2/journal.h index e3c3a35dc5e0e..6397170f302f2 100644 --- a/fs/ocfs2/journal.h +++ b/fs/ocfs2/journal.h @@ -148,6 +148,7 @@ void ocfs2_wait_for_recovery(struct ocfs2_super *osb); int ocfs2_recovery_init(struct ocfs2_super *osb); void ocfs2_recovery_exit(struct ocfs2_super *osb); +void ocfs2_recovery_disable_quota(struct ocfs2_super *osb); int ocfs2_compute_replay_slots(struct ocfs2_super *osb); void ocfs2_free_replay_slots(struct ocfs2_super *osb); diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c index 1f9ed117e78b6..2f34074f0078b 100644 --- a/fs/ocfs2/move_extents.c +++ b/fs/ocfs2/move_extents.c @@ -868,6 +868,11 @@ static int __ocfs2_move_extents_range(struct buffer_head *di_bh, mlog_errno(ret); goto out; } + /* + * Invalidate extent cache after moving/defragging to prevent + * stale cached data with outdated extent flags. + */ + ocfs2_extent_map_trunc(inode, cpos); context->clusters_moved += alloc_size; next: diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h index 8fe826143d7bf..5e3ebbab698ad 100644 --- a/fs/ocfs2/ocfs2.h +++ b/fs/ocfs2/ocfs2.h @@ -308,6 +308,21 @@ enum ocfs2_journal_trigger_type { void ocfs2_initialize_journal_triggers(struct super_block *sb, struct ocfs2_triggers triggers[]); +enum ocfs2_recovery_state { + OCFS2_REC_ENABLED = 0, + OCFS2_REC_QUOTA_WANT_DISABLE, + /* + * Must be OCFS2_REC_QUOTA_WANT_DISABLE + 1 for + * ocfs2_recovery_disable_quota() to work. + */ + OCFS2_REC_QUOTA_DISABLED, + OCFS2_REC_WANT_DISABLE, + /* + * Must be OCFS2_REC_WANT_DISABLE + 1 for ocfs2_recovery_exit() to work + */ + OCFS2_REC_DISABLED, +}; + struct ocfs2_journal; struct ocfs2_slot_info; struct ocfs2_recovery_map; @@ -370,7 +385,7 @@ struct ocfs2_super struct ocfs2_recovery_map *recovery_map; struct ocfs2_replay_map *replay_map; struct task_struct *recovery_thread_task; - int disable_recovery; + enum ocfs2_recovery_state recovery_state; wait_queue_head_t checkpoint_event; struct ocfs2_journal *journal; unsigned long osb_commit_interval; diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c index 4b4fa58cd32ff..c7bda48b5fb21 100644 --- a/fs/ocfs2/quota_local.c +++ b/fs/ocfs2/quota_local.c @@ -453,8 +453,7 @@ struct ocfs2_quota_recovery *ocfs2_begin_quota_recovery( /* Sync changes in local quota file into global quota file and * reinitialize local quota file. - * The function expects local quota file to be already locked and - * s_umount locked in shared mode. */ + * The function expects local quota file to be already locked. */ static int ocfs2_recover_local_quota_file(struct inode *lqinode, int type, struct ocfs2_quota_recovery *rec) @@ -585,7 +584,6 @@ int ocfs2_finish_quota_recovery(struct ocfs2_super *osb, { unsigned int ino[OCFS2_MAXQUOTAS] = { LOCAL_USER_QUOTA_SYSTEM_INODE, LOCAL_GROUP_QUOTA_SYSTEM_INODE }; - struct super_block *sb = osb->sb; struct ocfs2_local_disk_dqinfo *ldinfo; struct buffer_head *bh; handle_t *handle; @@ -597,7 +595,6 @@ int ocfs2_finish_quota_recovery(struct ocfs2_super *osb, printk(KERN_NOTICE "ocfs2: Finishing quota recovery on device (%s) for " "slot %u\n", osb->dev_str, slot_num); - down_read(&sb->s_umount); for (type = 0; type < OCFS2_MAXQUOTAS; type++) { if (list_empty(&(rec->r_list[type]))) continue; @@ -674,8 +671,7 @@ int ocfs2_finish_quota_recovery(struct ocfs2_super *osb, break; } out: - up_read(&sb->s_umount); - kfree(rec); + ocfs2_free_quota_recovery(rec); return status; } @@ -840,8 +836,7 @@ static int ocfs2_local_free_info(struct super_block *sb, int type) ocfs2_release_local_quota_bitmaps(&oinfo->dqi_chunk); /* - * s_umount held in exclusive mode protects us against racing with - * recovery thread... + * ocfs2_dismount_volume() has already aborted quota recovery... */ if (oinfo->dqi_rec) { ocfs2_free_quota_recovery(oinfo->dqi_rec); diff --git a/fs/ocfs2/stack_user.c b/fs/ocfs2/stack_user.c index 9b76ee66aeb2f..38c0d6f64fce3 100644 --- a/fs/ocfs2/stack_user.c +++ b/fs/ocfs2/stack_user.c @@ -1018,6 +1018,7 @@ static int user_cluster_connect(struct ocfs2_cluster_connection *conn) printk(KERN_ERR "ocfs2: Could not determine" " locking version\n"); user_cluster_disconnect(conn); + lc = NULL; goto out; } wait_event(lc->oc_wait, (atomic_read(&lc->oc_this_node) > 0)); diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index d3578d5459aca..674fd6292c0b2 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1573,15 +1573,13 @@ static int __init ocfs2_init(void) ocfs2_set_locking_protocol(); - status = register_quota_format(&ocfs2_quota_format); - if (status < 0) - goto out3; + register_quota_format(&ocfs2_quota_format); + status = register_filesystem(&ocfs2_fs_type); if (!status) return 0; unregister_quota_format(&ocfs2_quota_format); -out3: debugfs_remove(ocfs2_debugfs_root); ocfs2_free_mem_caches(); out2: @@ -1872,6 +1870,9 @@ static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err) /* Orphan scan should be stopped as early as possible */ ocfs2_orphan_scan_stop(osb); + /* Stop quota recovery so that we can disable quotas */ + ocfs2_recovery_disable_quota(osb); + ocfs2_disable_quotas(osb); /* All dquots should be freed by now */ diff --git a/fs/open.c b/fs/open.c index f69210a888fd8..0641cb3fdd6bf 100644 --- a/fs/open.c +++ b/fs/open.c @@ -1031,18 +1031,20 @@ EXPORT_SYMBOL(finish_open); * finish_no_open - finish ->atomic_open() without opening the file * * @file: file pointer - * @dentry: dentry or NULL (as returned from ->lookup()) + * @dentry: dentry, ERR_PTR(-E...) or NULL (as returned from ->lookup()) * - * This can be used to set the result of a successful lookup in ->atomic_open(). + * This can be used to set the result of a lookup in ->atomic_open(). * * NB: unlike finish_open() this function does consume the dentry reference and * the caller need not dput() it. * - * Returns "0" which must be the return value of ->atomic_open() after having - * called this function. + * Returns 0 or -E..., which must be the return value of ->atomic_open() after + * having called this function. */ int finish_no_open(struct file *file, struct dentry *dentry) { + if (IS_ERR(dentry)) + return PTR_ERR(dentry); file->f_path.dentry = dentry; return 0; } diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index 0859122684425..dd4dc70e4aaab 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -23,9 +23,9 @@ static int orangefs_writepage_locked(struct page *page, struct orangefs_write_range *wr = NULL; struct iov_iter iter; struct bio_vec bv; - size_t len, wlen; + size_t wlen; ssize_t ret; - loff_t off; + loff_t len, off; set_page_writeback(page); @@ -92,8 +92,7 @@ static int orangefs_writepages_work(struct orangefs_writepages *ow, struct orangefs_write_range *wrp, wr; struct iov_iter iter; ssize_t ret; - size_t len; - loff_t off; + loff_t len, off; int i; len = i_size_read(inode); diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c index fa41db0884880..cd4bfd92ebd6e 100644 --- a/fs/orangefs/orangefs-debugfs.c +++ b/fs/orangefs/orangefs-debugfs.c @@ -354,7 +354,7 @@ static ssize_t orangefs_debug_read(struct file *file, goto out; mutex_lock(&orangefs_debug_lock); - sprintf_ret = sprintf(buf, "%s", (char *)file->private_data); + sprintf_ret = scnprintf(buf, ORANGEFS_MAX_DEBUG_STRING_LEN, "%s", (char *)file->private_data); mutex_unlock(&orangefs_debug_lock); read_ret = simple_read_from_buffer(ubuf, count, ppos, buf, sprintf_ret); @@ -728,8 +728,8 @@ static void do_k_string(void *k_mask, int index) if (*mask & s_kmod_keyword_mask_map[index].mask_val) { if ((strlen(kernel_debug_string) + - strlen(s_kmod_keyword_mask_map[index].keyword)) - < ORANGEFS_MAX_DEBUG_STRING_LEN - 1) { + strlen(s_kmod_keyword_mask_map[index].keyword) + 1) + < ORANGEFS_MAX_DEBUG_STRING_LEN) { strcat(kernel_debug_string, s_kmod_keyword_mask_map[index].keyword); strcat(kernel_debug_string, ","); @@ -756,7 +756,7 @@ static void do_c_string(void *c_mask, int index) (mask->mask2 & cdm_array[index].mask2)) { if ((strlen(client_debug_string) + strlen(cdm_array[index].keyword) + 1) - < ORANGEFS_MAX_DEBUG_STRING_LEN - 2) { + < ORANGEFS_MAX_DEBUG_STRING_LEN) { strcat(client_debug_string, cdm_array[index].keyword); strcat(client_debug_string, ","); diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c index 68b62689a63e2..15738379f5511 100644 --- a/fs/orangefs/xattr.c +++ b/fs/orangefs/xattr.c @@ -54,7 +54,9 @@ static inline int convert_to_internal_xattr_flags(int setxattr_flags) static unsigned int xattr_key(const char *key) { unsigned int i = 0; - while (key) + if (!key) + return 0; + while (*key) i += *key++; return i % 16; } @@ -175,8 +177,8 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name, cx->length = -1; cx->timeout = jiffies + orangefs_getattr_timeout_msecs*HZ/1000; - hash_add(orangefs_inode->xattr_cache, &cx->node, - xattr_key(cx->key)); + hlist_add_head( &cx->node, + &orangefs_inode->xattr_cache[xattr_key(cx->key)]); } } goto out_release_op; @@ -229,8 +231,8 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name, memcpy(cx->val, buffer, length); cx->length = length; cx->timeout = jiffies + HZ; - hash_add(orangefs_inode->xattr_cache, &cx->node, - xattr_key(cx->key)); + hlist_add_head(&cx->node, + &orangefs_inode->xattr_cache[xattr_key(cx->key)]); } } diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 6c60141fec4f5..5bdf8dc51666b 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -283,7 +283,9 @@ enum ovl_path_type ovl_path_realdata(struct dentry *dentry, struct path *path) struct dentry *ovl_dentry_upper(struct dentry *dentry) { - return ovl_upperdentry_dereference(OVL_I(d_inode(dentry))); + struct inode *inode = d_inode(dentry); + + return inode ? ovl_upperdentry_dereference(OVL_I(inode)) : NULL; } struct dentry *ovl_dentry_lower(struct dentry *dentry) diff --git a/fs/proc/base.c b/fs/proc/base.c index 7cff02bc816e2..68d01c9ab7c6c 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -100,6 +100,7 @@ #include #include #include +#include #include #include #include "internal.h" @@ -3304,6 +3305,121 @@ static int proc_stack_depth(struct seq_file *m, struct pid_namespace *ns, } #endif /* CONFIG_STACKLEAK_METRICS */ +#ifdef CONFIG_DMA_SHARED_BUFFER +static int proc_dmabuf_rss_show(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct task_dma_buf_info *dmabuf_info = task->dmabuf_info; + + if (dmabuf_info) { + unsigned long rss; + + spin_lock(&dmabuf_info->lock); + rss = dmabuf_info->rss; + spin_unlock(&dmabuf_info->lock); + seq_printf(m, "%lu\n", rss); + } + + return 0; +} + +static int proc_dmabuf_rss_hwm_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *task; + int ret = 0; + + task = get_proc_task(inode); + if (!task) + return -ESRCH; + + if (task->dmabuf_info) { + unsigned long rss_hwm; + + spin_lock(&task->dmabuf_info->lock); + rss_hwm = task->dmabuf_info->rss_hwm; + spin_unlock(&task->dmabuf_info->lock); + seq_printf(m, "%lu\n", rss_hwm); + } + + put_task_struct(task); + + return ret; +} + +static int proc_dmabuf_rss_hwm_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, proc_dmabuf_rss_hwm_show, inode); +} + +static ssize_t +proc_dmabuf_rss_hwm_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *task; + unsigned long long val; + int ret; + + ret = kstrtoull_from_user(buf, count, 10, &val); + if (ret) + return ret; + + if (val != 0) + return -EINVAL; + + task = get_proc_task(inode); + if (!task) + return -ESRCH; + + if (!task->dmabuf_info) { + ret = -ENOENT; + } else { + spin_lock(&task->dmabuf_info->lock); + task->dmabuf_info->rss_hwm = task->dmabuf_info->rss; + spin_unlock(&task->dmabuf_info->lock); + } + + put_task_struct(task); + + return ret < 0 ? ret : count; +} + +static const struct file_operations proc_dmabuf_rss_hwm_operations = { + .open = proc_dmabuf_rss_hwm_open, + .write = proc_dmabuf_rss_hwm_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int proc_dmabuf_pss_show(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct task_dma_buf_record *rec; + + if (task->dmabuf_info) { + unsigned long pss = 0; + + spin_lock(&task->dmabuf_info->lock); + list_for_each_entry(rec, &task->dmabuf_info->dmabufs, node) { + s64 refs = atomic64_read(&rec->dmabuf->nr_task_refs); + + if (refs <= 0) { + pr_err("dmabuf has refs <= 0 %lld\n", refs); + continue; + } + + pss += rec->dmabuf->size / (size_t)refs; + } + spin_unlock(&task->dmabuf_info->lock); + seq_printf(m, "%lu\n", pss); + } + + return 0; +} +#endif + /* * Thread groups */ @@ -3427,6 +3543,11 @@ static const struct pid_entry tgid_base_stuff[] = { ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), #endif +#ifdef CONFIG_DMA_SHARED_BUFFER + ONE("dmabuf_rss", 0444, proc_dmabuf_rss_show), + REG("dmabuf_rss_hwm", 0644, proc_dmabuf_rss_hwm_operations), + ONE("dmabuf_pss", 0444, proc_dmabuf_pss_show), +#endif }; static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx) diff --git a/fs/proc/generic.c b/fs/proc/generic.c index c8785d68e870b..9eff57c7aef3a 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -362,6 +362,25 @@ static const struct inode_operations proc_dir_inode_operations = { .setattr = proc_notify_change, }; +static void pde_set_flags(struct proc_dir_entry *pde) +{ + const struct proc_ops *proc_ops = pde->proc_ops; + + if (!proc_ops) + return; + + if (proc_ops->proc_flags & PROC_ENTRY_PERMANENT) + pde->flags |= PROC_ENTRY_PERMANENT; + if (proc_ops->proc_read_iter) + pde->flags |= PROC_ENTRY_proc_read_iter; +#ifdef CONFIG_COMPAT + if (proc_ops->proc_compat_ioctl) + pde->flags |= PROC_ENTRY_proc_compat_ioctl; +#endif + if (proc_ops->proc_lseek) + pde->flags |= PROC_ENTRY_proc_lseek; +} + /* returns the registered entry, or frees dp and returns NULL on failure */ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, struct proc_dir_entry *dp) @@ -369,6 +388,9 @@ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, if (proc_alloc_inum(&dp->low_ino)) goto out_free_entry; + if (!S_ISDIR(dp->mode)) + pde_set_flags(dp); + write_lock(&proc_subdir_lock); dp->parent = dir; if (pde_subdir_insert(dir, dp) == false) { @@ -557,18 +579,6 @@ struct proc_dir_entry *proc_create_reg(const char *name, umode_t mode, return p; } -static void pde_set_flags(struct proc_dir_entry *pde) -{ - if (pde->proc_ops->proc_flags & PROC_ENTRY_PERMANENT) - pde->flags |= PROC_ENTRY_PERMANENT; - if (pde->proc_ops->proc_read_iter) - pde->flags |= PROC_ENTRY_proc_read_iter; -#ifdef CONFIG_COMPAT - if (pde->proc_ops->proc_compat_ioctl) - pde->flags |= PROC_ENTRY_proc_compat_ioctl; -#endif -} - struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, struct proc_dir_entry *parent, const struct proc_ops *proc_ops, void *data) @@ -579,7 +589,6 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, if (!p) return NULL; p->proc_ops = proc_ops; - pde_set_flags(p); return proc_register(parent, p); } EXPORT_SYMBOL(proc_create_data); @@ -630,7 +639,6 @@ struct proc_dir_entry *proc_create_seq_private(const char *name, umode_t mode, p->proc_ops = &proc_seq_ops; p->seq_ops = ops; p->state_size = state_size; - pde_set_flags(p); return proc_register(parent, p); } EXPORT_SYMBOL(proc_create_seq_private); @@ -661,7 +669,6 @@ struct proc_dir_entry *proc_create_single_data(const char *name, umode_t mode, return NULL; p->proc_ops = &proc_single_ops; p->single_show = show; - pde_set_flags(p); return proc_register(parent, p); } EXPORT_SYMBOL(proc_create_single_data); @@ -687,6 +694,12 @@ void pde_put(struct proc_dir_entry *pde) } } +static void pde_erase(struct proc_dir_entry *pde, struct proc_dir_entry *parent) +{ + rb_erase(&pde->subdir_node, &parent->subdir); + RB_CLEAR_NODE(&pde->subdir_node); +} + /* * Remove a /proc entry and free it if it's not currently in use. */ @@ -709,7 +722,7 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent) WARN(1, "removing permanent /proc entry '%s'", de->name); de = NULL; } else { - rb_erase(&de->subdir_node, &parent->subdir); + pde_erase(de, parent); if (S_ISDIR(de->mode)) parent->nlink--; } @@ -753,7 +766,7 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent) root->parent->name, root->name); return -EINVAL; } - rb_erase(&root->subdir_node, &parent->subdir); + pde_erase(root, parent); de = root; while (1) { @@ -765,7 +778,7 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent) next->parent->name, next->name); return -EINVAL; } - rb_erase(&next->subdir_node, &de->subdir); + pde_erase(next, de); de = next; continue; } diff --git a/fs/proc/inode.c b/fs/proc/inode.c index fc2ba55137f4d..85a09918507e9 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -42,7 +42,7 @@ static void proc_evict_inode(struct inode *inode) head = ei->sysctl; if (head) { - RCU_INIT_POINTER(ei->sysctl, NULL); + WRITE_ONCE(ei->sysctl, NULL); proc_sys_evict_inode(inode, head); } } @@ -491,7 +491,7 @@ static int proc_reg_open(struct inode *inode, struct file *file) typeof_member(struct proc_ops, proc_release) release; struct pde_opener *pdeo; - if (!pde->proc_ops->proc_lseek) + if (!pde_has_proc_lseek(pde)) file->f_mode &= ~FMODE_LSEEK; if (pde_is_permanent(pde)) { diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 445c74a39a931..fe37813601207 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -98,6 +98,11 @@ static inline bool pde_has_proc_compat_ioctl(const struct proc_dir_entry *pde) #endif } +static inline bool pde_has_proc_lseek(const struct proc_dir_entry *pde) +{ + return pde->flags & PROC_ENTRY_proc_lseek; +} + extern struct kmem_cache *proc_dir_entry_cache; void pde_free(struct proc_dir_entry *pde); diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 1d665f1c8f349..6a14ab9d3d0a2 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -920,17 +920,21 @@ static int proc_sys_compare(const struct dentry *dentry, struct ctl_table_header *head; struct inode *inode; - /* Although proc doesn't have negative dentries, rcu-walk means - * that inode here can be NULL */ - /* AV: can it, indeed? */ - inode = d_inode_rcu(dentry); - if (!inode) - return 1; if (name->len != len) return 1; if (memcmp(name->name, str, len)) return 1; - head = rcu_dereference(PROC_I(inode)->sysctl); + + // false positive is fine here - we'll recheck anyway + if (d_in_lookup(dentry)) + return 0; + + inode = d_inode_rcu(dentry); + // we just might have run into dentry in the middle of __dentry_kill() + if (!inode) + return 1; + + head = READ_ONCE(PROC_I(inode)->sysctl); return !head || !sysctl_is_seen(head); } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index ad94f3ee623db..e4380b37fc316 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -36,9 +36,9 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long text, lib, swap, anon, file, shmem; unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss; - anon = get_mm_counter(mm, MM_ANONPAGES); - file = get_mm_counter(mm, MM_FILEPAGES); - shmem = get_mm_counter(mm, MM_SHMEMPAGES); + anon = get_mm_counter_sum(mm, MM_ANONPAGES); + file = get_mm_counter_sum(mm, MM_FILEPAGES); + shmem = get_mm_counter_sum(mm, MM_SHMEMPAGES); /* * Note: to minimize their overhead, mm maintains hiwater_vm and @@ -59,7 +59,7 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) text = min(text, mm->exec_vm << PAGE_SHIFT); lib = (mm->exec_vm << PAGE_SHIFT) - text; - swap = get_mm_counter(mm, MM_SWAPENTS); + swap = get_mm_counter_sum(mm, MM_SWAPENTS); SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); @@ -92,12 +92,12 @@ unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, unsigned long *text, unsigned long *data, unsigned long *resident) { - *shared = get_mm_counter(mm, MM_FILEPAGES) + - get_mm_counter(mm, MM_SHMEMPAGES); + *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + + get_mm_counter_sum(mm, MM_SHMEMPAGES); *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> PAGE_SHIFT; *data = mm->data_vm + mm->stack_vm; - *resident = *shared + get_mm_counter(mm, MM_ANONPAGES); + *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); return mm->total_vm; } diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c index 7dbbf3b6d98d3..a1c97cd2720a6 100644 --- a/fs/pstore/inode.c +++ b/fs/pstore/inode.c @@ -264,7 +264,7 @@ static void parse_options(char *options) static int pstore_show_options(struct seq_file *m, struct dentry *root) { if (kmsg_bytes != CONFIG_PSTORE_DEFAULT_KMSG_BYTES) - seq_printf(m, ",kmsg_bytes=%lu", kmsg_bytes); + seq_printf(m, ",kmsg_bytes=%u", kmsg_bytes); return 0; } diff --git a/fs/pstore/internal.h b/fs/pstore/internal.h index 801d6c0b170c3..a0fc511969100 100644 --- a/fs/pstore/internal.h +++ b/fs/pstore/internal.h @@ -6,7 +6,7 @@ #include #include -extern unsigned long kmsg_bytes; +extern unsigned int kmsg_bytes; #ifdef CONFIG_PSTORE_FTRACE extern void pstore_register_ftrace(void); @@ -35,7 +35,7 @@ static inline void pstore_unregister_pmsg(void) {} extern struct pstore_info *psinfo; -extern void pstore_set_kmsg_bytes(int); +extern void pstore_set_kmsg_bytes(unsigned int bytes); extern void pstore_get_records(int); extern void pstore_get_backend_records(struct pstore_info *psi, struct dentry *root, int quiet); diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c index 03425928d2fb3..ef62389212b60 100644 --- a/fs/pstore/platform.c +++ b/fs/pstore/platform.c @@ -92,8 +92,8 @@ module_param(compress, charp, 0444); MODULE_PARM_DESC(compress, "compression to use"); /* How much of the kernel log to snapshot */ -unsigned long kmsg_bytes = CONFIG_PSTORE_DEFAULT_KMSG_BYTES; -module_param(kmsg_bytes, ulong, 0444); +unsigned int kmsg_bytes = CONFIG_PSTORE_DEFAULT_KMSG_BYTES; +module_param(kmsg_bytes, uint, 0444); MODULE_PARM_DESC(kmsg_bytes, "amount of kernel log to snapshot (in bytes)"); static void *compress_workspace; @@ -107,9 +107,9 @@ static void *compress_workspace; static char *big_oops_buf; static size_t max_compressed_size; -void pstore_set_kmsg_bytes(int bytes) +void pstore_set_kmsg_bytes(unsigned int bytes) { - kmsg_bytes = bytes; + WRITE_ONCE(kmsg_bytes, bytes); } /* Tag each group of saved records with a sequence number */ @@ -278,6 +278,7 @@ static void pstore_dump(struct kmsg_dumper *dumper, enum kmsg_dump_reason reason) { struct kmsg_dump_iter iter; + unsigned int remaining = READ_ONCE(kmsg_bytes); unsigned long total = 0; const char *why; unsigned int part = 1; @@ -300,7 +301,7 @@ static void pstore_dump(struct kmsg_dumper *dumper, kmsg_dump_rewind(&iter); oopscount++; - while (total < kmsg_bytes) { + while (total < remaining) { char *dst; size_t dst_size; int header_size; diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 67562c78e57d5..42a7d0a71b22e 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -163,13 +163,15 @@ static struct quota_module_name module_names[] = INIT_QUOTA_MODULE_NAMES; /* SLAB cache for dquot structures */ static struct kmem_cache *dquot_cachep; -int register_quota_format(struct quota_format_type *fmt) +/* workqueue for work quota_release_work*/ +static struct workqueue_struct *quota_unbound_wq; + +void register_quota_format(struct quota_format_type *fmt) { spin_lock(&dq_list_lock); fmt->qf_next = quota_formats; quota_formats = fmt; spin_unlock(&dq_list_lock); - return 0; } EXPORT_SYMBOL(register_quota_format); @@ -892,7 +894,7 @@ void dqput(struct dquot *dquot) put_releasing_dquots(dquot); atomic_dec(&dquot->dq_count); spin_unlock(&dq_list_lock); - queue_delayed_work(system_unbound_wq, "a_release_work, 1); + queue_delayed_work(quota_unbound_wq, "a_release_work, 1); } EXPORT_SYMBOL(dqput); @@ -3047,6 +3049,11 @@ static int __init dquot_init(void) if (register_shrinker(&dqcache_shrinker, "dquota-cache")) panic("Cannot register dquot shrinker"); + quota_unbound_wq = alloc_workqueue("quota_events_unbound", + WQ_UNBOUND | WQ_MEM_RECLAIM, WQ_MAX_ACTIVE); + if (!quota_unbound_wq) + panic("Cannot create quota_unbound_wq\n"); + return 0; } fs_initcall(dquot_init); diff --git a/fs/quota/quota_v1.c b/fs/quota/quota_v1.c index a0db3f195e951..8aaf4a501fc0d 100644 --- a/fs/quota/quota_v1.c +++ b/fs/quota/quota_v1.c @@ -229,7 +229,8 @@ static struct quota_format_type v1_quota_format = { static int __init init_v1_quota_format(void) { - return register_quota_format(&v1_quota_format); + register_quota_format(&v1_quota_format); + return 0; } static void __exit exit_v1_quota_format(void) diff --git a/fs/quota/quota_v2.c b/fs/quota/quota_v2.c index 7978ab671e0c6..d73f443292771 100644 --- a/fs/quota/quota_v2.c +++ b/fs/quota/quota_v2.c @@ -422,12 +422,9 @@ static struct quota_format_type v2r1_quota_format = { static int __init init_v2_quota_format(void) { - int ret; - - ret = register_quota_format(&v2r0_quota_format); - if (ret) - return ret; - return register_quota_format(&v2r1_quota_format); + register_quota_format(&v2r0_quota_format); + register_quota_format(&v2r1_quota_format); + return 0; } static void __exit exit_v2_quota_format(void) diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index 9c0ef4195b582..e92a61e934e44 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -16,6 +16,7 @@ static struct cached_fid *init_cached_dir(const char *path); static void free_cached_dir(struct cached_fid *cfid); static void smb2_close_cached_fid(struct kref *ref); static void cfids_laundromat_worker(struct work_struct *work); +static void close_cached_dir_locked(struct cached_fid *cfid); struct cached_dir_dentry { struct list_head entry; @@ -29,7 +30,6 @@ static struct cached_fid *find_or_create_cached_dir(struct cached_fids *cfids, { struct cached_fid *cfid; - spin_lock(&cfids->cfid_list_lock); list_for_each_entry(cfid, &cfids->entries, entry) { if (!strcmp(cfid->path, path)) { /* @@ -38,25 +38,20 @@ static struct cached_fid *find_or_create_cached_dir(struct cached_fids *cfids, * being deleted due to a lease break. */ if (!cfid->time || !cfid->has_lease) { - spin_unlock(&cfids->cfid_list_lock); return NULL; } kref_get(&cfid->refcount); - spin_unlock(&cfids->cfid_list_lock); return cfid; } } if (lookup_only) { - spin_unlock(&cfids->cfid_list_lock); return NULL; } if (cfids->num_entries >= max_cached_dirs) { - spin_unlock(&cfids->cfid_list_lock); return NULL; } cfid = init_cached_dir(path); if (cfid == NULL) { - spin_unlock(&cfids->cfid_list_lock); return NULL; } cfid->cfids = cfids; @@ -74,7 +69,6 @@ static struct cached_fid *find_or_create_cached_dir(struct cached_fids *cfids, */ cfid->has_lease = true; - spin_unlock(&cfids->cfid_list_lock); return cfid; } @@ -185,8 +179,10 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, if (!utf16_path) return -ENOMEM; + spin_lock(&cfids->cfid_list_lock); cfid = find_or_create_cached_dir(cfids, path, lookup_only, tcon->max_cached_dirs); if (cfid == NULL) { + spin_unlock(&cfids->cfid_list_lock); kfree(utf16_path); return -ENOENT; } @@ -195,7 +191,6 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, * Otherwise, it is either a new entry or laundromat worker removed it * from @cfids->entries. Caller will put last reference if the latter. */ - spin_lock(&cfids->cfid_list_lock); if (cfid->has_lease && cfid->time) { spin_unlock(&cfids->cfid_list_lock); *ret_cfid = cfid; @@ -368,11 +363,11 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, * lease. Release one here, and the second below. */ cfid->has_lease = false; - kref_put(&cfid->refcount, smb2_close_cached_fid); + close_cached_dir_locked(cfid); } spin_unlock(&cfids->cfid_list_lock); - kref_put(&cfid->refcount, smb2_close_cached_fid); + close_cached_dir(cfid); } else { *ret_cfid = cfid; atomic_inc(&tcon->num_remote_opens); @@ -412,12 +407,14 @@ int open_cached_dir_by_dentry(struct cifs_tcon *tcon, static void smb2_close_cached_fid(struct kref *ref) +__releases(&cfid->cfids->cfid_list_lock) { struct cached_fid *cfid = container_of(ref, struct cached_fid, refcount); int rc; - spin_lock(&cfid->cfids->cfid_list_lock); + lockdep_assert_held(&cfid->cfids->cfid_list_lock); + if (cfid->on_list) { list_del(&cfid->entry); cfid->on_list = false; @@ -452,15 +449,49 @@ void drop_cached_dir_by_name(const unsigned int xid, struct cifs_tcon *tcon, spin_lock(&cfid->cfids->cfid_list_lock); if (cfid->has_lease) { cfid->has_lease = false; - kref_put(&cfid->refcount, smb2_close_cached_fid); + close_cached_dir_locked(cfid); } spin_unlock(&cfid->cfids->cfid_list_lock); close_cached_dir(cfid); } - +/** + * close_cached_dir - drop a reference of a cached dir + * + * The release function will be called with cfid_list_lock held to remove the + * cached dirs from the list before any other thread can take another @cfid + * ref. Must not be called with cfid_list_lock held; use + * close_cached_dir_locked() called instead. + * + * @cfid: cached dir + */ void close_cached_dir(struct cached_fid *cfid) { + lockdep_assert_not_held(&cfid->cfids->cfid_list_lock); + kref_put_lock(&cfid->refcount, smb2_close_cached_fid, &cfid->cfids->cfid_list_lock); +} + +/** + * close_cached_dir_locked - put a reference of a cached dir with + * cfid_list_lock held + * + * Calling close_cached_dir() with cfid_list_lock held has the potential effect + * of causing a deadlock if the invariant of refcount >= 2 is false. + * + * This function is used in paths that hold cfid_list_lock and expect at least + * two references. If that invariant is violated, WARNs and returns without + * dropping a reference; the final put must still go through + * close_cached_dir(). + * + * @cfid: cached dir + */ +static void close_cached_dir_locked(struct cached_fid *cfid) +{ + lockdep_assert_held(&cfid->cfids->cfid_list_lock); + + if (WARN_ON(kref_read(&cfid->refcount) < 2)) + return; + kref_put(&cfid->refcount, smb2_close_cached_fid); } @@ -490,8 +521,17 @@ void close_all_cached_dirs(struct cifs_sb_info *cifs_sb) spin_lock(&cfids->cfid_list_lock); list_for_each_entry(cfid, &cfids->entries, entry) { tmp_list = kmalloc(sizeof(*tmp_list), GFP_ATOMIC); - if (tmp_list == NULL) - break; + if (tmp_list == NULL) { + /* + * If the malloc() fails, we won't drop all + * dentries, and unmounting is likely to trigger + * a 'Dentry still in use' error. + */ + cifs_tcon_dbg(VFS, "Out of memory while dropping dentries\n"); + spin_unlock(&cfids->cfid_list_lock); + spin_unlock(&cifs_sb->tlink_tree_lock); + goto done; + } spin_lock(&cfid->fid_lock); tmp_list->dentry = cfid->dentry; cfid->dentry = NULL; @@ -503,6 +543,7 @@ void close_all_cached_dirs(struct cifs_sb_info *cifs_sb) } spin_unlock(&cifs_sb->tlink_tree_lock); +done: list_for_each_entry_safe(tmp_list, q, &entry, entry) { list_del(&tmp_list->entry); dput(tmp_list->dentry); @@ -562,7 +603,7 @@ cached_dir_offload_close(struct work_struct *work) WARN_ON(cfid->on_list); - kref_put(&cfid->refcount, smb2_close_cached_fid); + close_cached_dir(cfid); cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cached_close); } @@ -686,7 +727,7 @@ static void cfids_invalidation_worker(struct work_struct *work) list_for_each_entry_safe(cfid, q, &entry, entry) { list_del(&cfid->entry); /* Drop the ref-count acquired in invalidate_all_cached_dirs */ - kref_put(&cfid->refcount, smb2_close_cached_fid); + close_cached_dir(cfid); } } @@ -739,7 +780,7 @@ static void cfids_laundromat_worker(struct work_struct *work) * Drop the ref-count from above, either the lease-ref (if there * was one) or the extra one acquired. */ - kref_put(&cfid->refcount, smb2_close_cached_fid); + close_cached_dir(cfid); } queue_delayed_work(cfid_put_wq, &cfids->laundromat_work, dir_cache_timeout * HZ); diff --git a/fs/smb/client/cached_dir.h b/fs/smb/client/cached_dir.h index 1dfe79d947a62..bc8a812ff95f8 100644 --- a/fs/smb/client/cached_dir.h +++ b/fs/smb/client/cached_dir.h @@ -21,10 +21,10 @@ struct cached_dirent { struct cached_dirents { bool is_valid:1; bool is_failed:1; - struct dir_context *ctx; /* - * Only used to make sure we only take entries - * from a single context. Never dereferenced. - */ + struct file *file; /* + * Used to associate the cache with a single + * open file instance. + */ struct mutex de_mutex; int pos; /* Expected ctx->pos */ struct list_head entries; diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c index 4a20e92474b23..50ad8246ed187 100644 --- a/fs/smb/client/cifs_debug.c +++ b/fs/smb/client/cifs_debug.c @@ -384,7 +384,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) seq_printf(m, "\nSMBDirect (in hex) protocol version: %x " "transport status: %x", server->smbd_conn->protocol, - server->smbd_conn->transport_status); + server->smbd_conn->socket.status); seq_printf(m, "\nConn receive_credit_max: %x " "send_credit_target: %x max_send_size: %x", server->smbd_conn->receive_credit_max, diff --git a/fs/smb/client/cifs_unicode.c b/fs/smb/client/cifs_unicode.c index 4cc6e0896fad3..f8659d36793f1 100644 --- a/fs/smb/client/cifs_unicode.c +++ b/fs/smb/client/cifs_unicode.c @@ -629,6 +629,9 @@ cifs_strndup_to_utf16(const char *src, const int maxlen, int *utf16_len, int len; __le16 *dst; + if (!src) + return NULL; + len = cifs_local_to_utf16_bytes(src, maxlen, cp); len += 2; /* NULL */ dst = kmalloc(len, GFP_KERNEL); diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index db9076da2182a..7bd29e827c8f1 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -187,7 +187,7 @@ compare_sids(const struct smb_sid *ctsid, const struct smb_sid *cwsid) /* compare all of the subauth values if any */ num_sat = ctsid->num_subauth; num_saw = cwsid->num_subauth; - num_subauth = num_sat < num_saw ? num_sat : num_saw; + num_subauth = min(num_sat, num_saw); if (num_subauth) { for (i = 0; i < num_subauth; ++i) { if (ctsid->sub_auth[i] != cwsid->sub_auth[i]) { @@ -811,7 +811,23 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, return; for (i = 0; i < num_aces; ++i) { + if (end_of_acl - acl_base < acl_size) + break; + ppace[i] = (struct smb_ace *) (acl_base + acl_size); + acl_base = (char *)ppace[i]; + acl_size = offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth); + + if (end_of_acl - acl_base < acl_size || + ppace[i]->sid.num_subauth == 0 || + ppace[i]->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + (end_of_acl - acl_base < + acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth) || + (le16_to_cpu(ppace[i]->size) < + acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth)) + break; + #ifdef CONFIG_CIFS_DEBUG2 dump_ace(ppace[i], end_of_acl); #endif @@ -855,7 +871,6 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, (void *)ppace[i], sizeof(struct smb_ace)); */ - acl_base = (char *)ppace[i]; acl_size = le16_to_cpu(ppace[i]->size); } diff --git a/fs/smb/client/cifsacl.h b/fs/smb/client/cifsacl.h index cbaed8038e365..05b3650ba0aec 100644 --- a/fs/smb/client/cifsacl.h +++ b/fs/smb/client/cifsacl.h @@ -144,7 +144,7 @@ struct smb3_sd { #define ACL_CONTROL_SI 0x0800 /* SACL Auto-Inherited */ #define ACL_CONTROL_DI 0x0400 /* DACL Auto-Inherited */ #define ACL_CONTROL_SC 0x0200 /* SACL computed through inheritance */ -#define ACL_CONTROL_DC 0x0100 /* DACL computed through inheritence */ +#define ACL_CONTROL_DC 0x0100 /* DACL computed through inheritance */ #define ACL_CONTROL_SS 0x0080 /* Create server ACL */ #define ACL_CONTROL_DT 0x0040 /* DACL provided by trusted source */ #define ACL_CONTROL_SD 0x0020 /* SACL defaulted */ diff --git a/fs/smb/client/cifsencrypt.c b/fs/smb/client/cifsencrypt.c index b0473c2567fe6..da3d003cb43db 100644 --- a/fs/smb/client/cifsencrypt.c +++ b/fs/smb/client/cifsencrypt.c @@ -353,7 +353,7 @@ int cifs_verify_signature(struct smb_rqst *rqst, cifs_dbg(FYI, "dummy signature received for smb command 0x%x\n", cifs_pdu->Command); - /* save off the origiginal signature so we can modify the smb and check + /* save off the original signature so we can modify the smb and check its signature against what the server sent */ memcpy(server_response_sig, cifs_pdu->Signature.SecuritySignature, 8); diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index dfe648e94b4f1..277921dabf98e 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -133,7 +133,7 @@ module_param(enable_oplocks, bool, 0644); MODULE_PARM_DESC(enable_oplocks, "Enable or disable oplocks. Default: y/Y/1"); module_param(enable_gcm_256, bool, 0644); -MODULE_PARM_DESC(enable_gcm_256, "Enable requesting strongest (256 bit) GCM encryption. Default: y/Y/0"); +MODULE_PARM_DESC(enable_gcm_256, "Enable requesting strongest (256 bit) GCM encryption. Default: y/Y/1"); module_param(require_gcm_256, bool, 0644); MODULE_PARM_DESC(require_gcm_256, "Require strongest (256 bit) GCM encryption. Default: n/N/0"); @@ -161,7 +161,7 @@ __u32 cifs_lock_secret; /* * Bumps refcount for cifs super block. - * Note that it should be only called if a referece to VFS super block is + * Note that it should be only called if a reference to VFS super block is * already held, e.g. in open-type syscalls context. Otherwise it can race with * atomic_dec_and_test in deactivate_locked_super. */ @@ -289,7 +289,7 @@ static void cifs_kill_sb(struct super_block *sb) struct cifs_sb_info *cifs_sb = CIFS_SB(sb); /* - * We ned to release all dentries for the cached directories + * We need to release all dentries for the cached directories * before we kill the sb. */ if (cifs_sb->root) { @@ -1371,6 +1371,20 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, netfs_resize_file(&target_cifsi->netfs, new_size); fscache_resize_cookie(cifs_inode_cookie(target_inode), new_size); + } else if (rc == -EOPNOTSUPP) { + /* + * copy_file_range syscall man page indicates EINVAL + * is returned e.g when "fd_in and fd_out refer to the + * same file and the source and target ranges overlap." + * Test generic/157 was what showed these cases where + * we need to remap EOPNOTSUPP to EINVAL + */ + if (off >= src_inode->i_size) { + rc = -EINVAL; + } else if (src_inode == target_inode) { + if (off + len > destoff) + rc = -EINVAL; + } } } diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h index 39117343b703f..6168c6d62b5e5 100644 --- a/fs/smb/client/cifsglob.h +++ b/fs/smb/client/cifsglob.h @@ -677,6 +677,7 @@ inc_rfc1001_len(void *buf, int count) struct TCP_Server_Info { struct list_head tcp_ses_list; struct list_head smb_ses_list; + struct list_head rlist; /* reconnect list */ spinlock_t srv_lock; /* protect anything here that is not protected */ __u64 conn_id; /* connection identifier (useful for debugging) */ int srv_count; /* reference counter */ @@ -702,7 +703,7 @@ struct TCP_Server_Info { bool nosharesock; bool tcp_nodelay; bool terminate; - unsigned int credits; /* send no more requests at once */ + int credits; /* send no more requests at once */ unsigned int max_credits; /* can override large 32000 default at mnt */ unsigned int in_flight; /* number of requests on the wire to server */ unsigned int max_in_flight; /* max number of requests that were on wire */ @@ -739,8 +740,10 @@ struct TCP_Server_Info { char workstation_RFC1001_name[RFC1001_NAME_LEN_WITH_NULL]; __u32 sequence_number; /* for signing, protected by srv_mutex */ __u32 reconnect_instance; /* incremented on each reconnect */ + __le32 session_key_id; /* retrieved from negotiate response and send in session setup request */ struct session_key session_key; unsigned long lstrp; /* when we got last response from this server */ + unsigned long neg_start; /* when negotiate started (jiffies) */ struct cifs_secmech secmech; /* crypto sec mech functs, descriptors */ #define CIFS_NEGFLAVOR_UNENCAP 1 /* wct == 17, but no ext_sec */ #define CIFS_NEGFLAVOR_EXTENDED 2 /* wct == 17, ext_sec bit set */ @@ -782,7 +785,7 @@ struct TCP_Server_Info { } compression; __u16 signing_algorithm; __le16 cipher_type; - /* save initital negprot hash */ + /* save initial negprot hash */ __u8 preauth_sha_hash[SMB2_PREAUTH_HASH_SIZE]; bool signing_negotiated; /* true if valid signing context rcvd from server */ bool posix_ext_supported; @@ -827,6 +830,7 @@ struct TCP_Server_Info { * format: \\HOST\SHARE[\OPTIONAL PATH] */ char *leaf_fullpath; + bool dfs_conn:1; }; static inline bool is_smb1(struct TCP_Server_Info *server) @@ -1053,6 +1057,7 @@ struct cifs_chan { }; #define CIFS_SES_FLAG_SCALE_CHANNELS (0x1) +#define CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES (0x2) /* * Session structure. One of these for each uid session with a particular host @@ -1061,6 +1066,7 @@ struct cifs_ses { struct list_head smb_ses_list; struct list_head rlist; /* reconnect list */ struct list_head tcon_list; + struct list_head dlist; /* dfs list */ struct cifs_tcon *tcon_ipc; spinlock_t ses_lock; /* protect anything here that is not protected */ struct mutex session_mutex; @@ -1265,6 +1271,7 @@ struct cifs_tcon { bool use_persistent:1; /* use persistent instead of durable handles */ bool no_lease:1; /* Do not request leases on files or directories */ bool use_witness:1; /* use witness protocol */ + bool dummy:1; /* dummy tcon used for reconnecting channels */ __le32 capabilities; __u32 share_flags; __u32 maximal_access; @@ -1289,6 +1296,7 @@ struct cifs_tcon { /* BB add field for back pointer to sb struct(s)? */ #ifdef CONFIG_CIFS_DFS_UPCALL struct delayed_work dfs_cache_work; + struct list_head dfs_ses_list; #endif struct delayed_work query_interfaces; /* query interfaces workqueue job */ char *origin_fullpath; /* canonical copy of smb3_fs_context::source */ diff --git a/fs/smb/client/cifspdu.h b/fs/smb/client/cifspdu.h index c46d418c1c0c3..f4cfb082dfd14 100644 --- a/fs/smb/client/cifspdu.h +++ b/fs/smb/client/cifspdu.h @@ -557,7 +557,7 @@ typedef union smb_com_session_setup_andx { __le16 MaxBufferSize; __le16 MaxMpxCount; __le16 VcNumber; - __u32 SessionKey; + __le32 SessionKey; __le16 SecurityBlobLength; __u32 Reserved; __le32 Capabilities; /* see below */ @@ -576,7 +576,7 @@ typedef union smb_com_session_setup_andx { __le16 MaxBufferSize; __le16 MaxMpxCount; __le16 VcNumber; - __u32 SessionKey; + __le32 SessionKey; __le16 CaseInsensitivePasswordLength; /* ASCII password len */ __le16 CaseSensitivePasswordLength; /* Unicode password length*/ __u32 Reserved; /* see below */ @@ -614,7 +614,7 @@ typedef union smb_com_session_setup_andx { __le16 MaxBufferSize; __le16 MaxMpxCount; __le16 VcNumber; - __u32 SessionKey; + __le32 SessionKey; __le16 PasswordLength; __u32 Reserved; /* encrypt key len and offset */ __le16 ByteCount; @@ -781,7 +781,7 @@ typedef struct smb_com_logoff_andx_rsp { __u16 ByteCount; } __attribute__((packed)) LOGOFF_ANDX_RSP; -typedef union smb_com_tree_disconnect { /* as an altetnative can use flag on +typedef union smb_com_tree_disconnect { /* as an alternative can use flag on tree_connect PDU to effect disconnect */ /* tdis is probably simplest SMB PDU */ struct { @@ -1226,10 +1226,9 @@ typedef struct smb_com_query_information_rsp { typedef struct smb_com_setattr_req { struct smb_hdr hdr; /* wct = 8 */ __le16 attr; - __le16 time_low; - __le16 time_high; + __le32 last_write_time; __le16 reserved[5]; /* must be zero */ - __u16 ByteCount; + __le16 ByteCount; __u8 BufferFormat; /* 4 = ASCII */ unsigned char fileName[]; } __attribute__((packed)) SETATTR_REQ; @@ -2406,7 +2405,7 @@ struct cifs_posix_ace { /* access control entry (ACE) */ __le64 cifs_uid; /* or gid */ } __attribute__((packed)); -struct cifs_posix_acl { /* access conrol list (ACL) */ +struct cifs_posix_acl { /* access control list (ACL) */ __le16 version; __le16 access_entry_count; /* access ACL - count of entries */ __le16 default_entry_count; /* default ACL - count of entries */ diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h index 7f97e54686524..5ab877e480abc 100644 --- a/fs/smb/client/cifsproto.h +++ b/fs/smb/client/cifsproto.h @@ -31,6 +31,9 @@ extern void cifs_small_buf_release(void *); extern void free_rsp_buf(int, void *); extern int smb_send(struct TCP_Server_Info *, struct smb_hdr *, unsigned int /* length */); +extern int smb_send_kvec(struct TCP_Server_Info *server, + struct msghdr *msg, + size_t *sent); extern unsigned int _get_xid(void); extern void _free_xid(unsigned int); #define get_xid() \ @@ -133,6 +136,7 @@ extern int SendReceiveBlockingLock(const unsigned int xid, struct smb_hdr *out_buf, int *bytes_returned); +void smb2_query_server_interfaces(struct work_struct *work); void cifs_signal_cifsd_for_reconnect(struct TCP_Server_Info *server, bool all_channels); @@ -395,6 +399,10 @@ extern int CIFSSMBQFSUnixInfo(const unsigned int xid, struct cifs_tcon *tcon); extern int CIFSSMBQFSPosixInfo(const unsigned int xid, struct cifs_tcon *tcon, struct kstatfs *FSData); +extern int SMBSetInformation(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, __le32 attributes, __le64 write_time, + const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb); extern int CIFSSMBSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon, const char *fileName, const FILE_BASIC_INFO *data, const struct nls_table *nls_codepage, @@ -730,15 +738,9 @@ static inline int cifs_create_options(struct cifs_sb_info *cifs_sb, int options) int cifs_wait_for_server_reconnect(struct TCP_Server_Info *server, bool retry); -/* Put references of @ses and its children */ static inline void cifs_put_smb_ses(struct cifs_ses *ses) { - struct cifs_ses *next; - - do { - next = ses->dfs_root_ses; - __cifs_put_smb_ses(ses); - } while ((ses = next)); + __cifs_put_smb_ses(ses); } /* Get an active reference of @ses and its children. @@ -752,9 +754,7 @@ static inline void cifs_put_smb_ses(struct cifs_ses *ses) static inline void cifs_smb_ses_inc_refcount(struct cifs_ses *ses) { lockdep_assert_held(&cifs_tcp_ses_lock); - - for (; ses; ses = ses->dfs_root_ses) - ses->ses_count++; + ses->ses_count++; } static inline bool dfs_src_pathname_equal(const char *s1, const char *s2) diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c index 769950adb7763..91f4e50af1e94 100644 --- a/fs/smb/client/cifssmb.c +++ b/fs/smb/client/cifssmb.c @@ -479,6 +479,7 @@ CIFSSMBNegotiate(const unsigned int xid, server->max_rw = le32_to_cpu(pSMBr->MaxRawSize); cifs_dbg(NOISY, "Max buf = %d\n", ses->server->maxBuf); server->capabilities = le32_to_cpu(pSMBr->Capabilities); + server->session_key_id = pSMBr->SessionKey; server->timeAdj = (int)(__s16)le16_to_cpu(pSMBr->ServerTimeZone); server->timeAdj *= 60; @@ -1213,7 +1214,7 @@ CIFS_open(const unsigned int xid, struct cifs_open_parms *oparms, int *oplock, req->CreateDisposition = cpu_to_le32(disposition); req->CreateOptions = cpu_to_le32(create_options & CREATE_OPTIONS_MASK); - /* BB Expirement with various impersonation levels and verify */ + /* BB Experiment with various impersonation levels and verify */ req->ImpersonationLevel = cpu_to_le32(SECURITY_IMPERSONATION); req->SecurityFlags = SECURITY_CONTEXT_TRACKING|SECURITY_EFFECTIVE_ONLY; @@ -2738,10 +2739,10 @@ int cifs_query_reparse_point(const unsigned int xid, io_req->TotalParameterCount = 0; io_req->TotalDataCount = 0; - io_req->MaxParameterCount = cpu_to_le32(2); + io_req->MaxParameterCount = cpu_to_le32(0); /* BB find exact data count max from sess structure BB */ io_req->MaxDataCount = cpu_to_le32(CIFSMaxBufSize & 0xFFFFFF00); - io_req->MaxSetupCount = 4; + io_req->MaxSetupCount = 1; io_req->Reserved = 0; io_req->ParameterOffset = 0; io_req->DataCount = 0; @@ -2768,6 +2769,22 @@ int cifs_query_reparse_point(const unsigned int xid, goto error; } + /* SetupCount must be 1, otherwise offset to ByteCount is incorrect. */ + if (io_rsp->SetupCount != 1) { + rc = -EIO; + goto error; + } + + /* + * ReturnedDataLen is output length of executed IOCTL. + * DataCount is output length transferred over network. + * Check that we have full FSCTL_GET_REPARSE_POINT buffer. + */ + if (data_count != le16_to_cpu(io_rsp->ReturnedDataLen)) { + rc = -EIO; + goto error; + } + end = 2 + get_bcc(&io_rsp->hdr) + (__u8 *)&io_rsp->ByteCount; start = (__u8 *)&io_rsp->hdr.Protocol + data_offset; if (start >= end) { @@ -2976,7 +2993,7 @@ static void cifs_init_ace(struct cifs_posix_ace *cifs_ace, /** * posix_acl_to_cifs - convert ACLs from POSIX ACL to cifs format - * @parm_data: ACLs in cifs format to conver to + * @parm_data: ACLs in cifs format to convert to * @acl: ACLs in POSIX ACL format to convert from * @acl_type: the type of POSIX ACLs stored in @acl * @@ -3953,7 +3970,7 @@ CIFSFindFirst(const unsigned int xid, struct cifs_tcon *tcon, name_len = cifsConvertToUTF16((__le16 *) pSMB->FileName, searchName, PATH_MAX, nls_codepage, remap); - /* We can not add the asterik earlier in case + /* We can not add the asterisk earlier in case it got remapped to 0xF03A as if it were part of the directory name instead of a wildcard */ name_len *= 2; @@ -3967,6 +3984,12 @@ CIFSFindFirst(const unsigned int xid, struct cifs_tcon *tcon, pSMB->FileName[name_len] = 0; pSMB->FileName[name_len+1] = 0; name_len += 2; + } else if (!searchName[0]) { + pSMB->FileName[0] = CIFS_DIR_SEP(cifs_sb); + pSMB->FileName[1] = 0; + pSMB->FileName[2] = 0; + pSMB->FileName[3] = 0; + name_len = 4; } } else { name_len = copy_path_name(pSMB->FileName, searchName); @@ -3978,6 +4001,10 @@ CIFSFindFirst(const unsigned int xid, struct cifs_tcon *tcon, pSMB->FileName[name_len] = '*'; pSMB->FileName[name_len+1] = 0; name_len += 2; + } else if (!searchName[0]) { + pSMB->FileName[0] = CIFS_DIR_SEP(cifs_sb); + pSMB->FileName[1] = 0; + name_len = 2; } } @@ -5157,6 +5184,63 @@ CIFSSMBSetFileSize(const unsigned int xid, struct cifs_tcon *tcon, return rc; } +int +SMBSetInformation(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, __le32 attributes, __le64 write_time, + const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb) +{ + SETATTR_REQ *pSMB; + SETATTR_RSP *pSMBr; + struct timespec64 ts; + int bytes_returned; + int name_len; + int rc; + + cifs_dbg(FYI, "In %s path %s\n", __func__, fileName); + +retry: + rc = smb_init(SMB_COM_SETATTR, 8, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc) + return rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->fileName, + fileName, PATH_MAX, nls_codepage, + cifs_remap(cifs_sb)); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->fileName, fileName); + } + /* Only few attributes can be set by this command, others are not accepted by Win9x. */ + pSMB->attr = cpu_to_le16(le32_to_cpu(attributes) & + (ATTR_READONLY | ATTR_HIDDEN | ATTR_SYSTEM | ATTR_ARCHIVE)); + /* Zero write time value (in both NT and SETATTR formats) means to not change it. */ + if (le64_to_cpu(write_time) != 0) { + ts = cifs_NTtimeToUnix(write_time); + pSMB->last_write_time = cpu_to_le32(ts.tv_sec); + } + pSMB->BufferFormat = 0x04; + name_len++; /* account for buffer type byte */ + inc_rfc1001_len(pSMB, (__u16)name_len); + pSMB->ByteCount = cpu_to_le16(name_len); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "Send error in %s = %d\n", __func__, rc); + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto retry; + + return rc; +} + /* Some legacy servers such as NT4 require that the file times be set on an open handle, rather than by pathname - this is awkward due to potential access conflicts on the open, but it is unavoidable for these diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index 54aba8d642ee7..986286ef40466 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -113,7 +113,7 @@ static int reconn_set_ipaddr_from_hostname(struct TCP_Server_Info *server) return rc; } -static void smb2_query_server_interfaces(struct work_struct *work) +void smb2_query_server_interfaces(struct work_struct *work) { int rc; int xid; @@ -132,18 +132,22 @@ static void smb2_query_server_interfaces(struct work_struct *work) rc = server->ops->query_server_interfaces(xid, tcon, false); free_xid(xid); - if (rc) { - if (rc == -EOPNOTSUPP) - return; - + if (rc) cifs_dbg(FYI, "%s: failed to query server interfaces: %d\n", __func__, rc); - } queue_delayed_work(cifsiod_wq, &tcon->query_interfaces, (SMB_INTERFACE_POLL_INTERVAL * HZ)); } +#define set_need_reco(server) \ +do { \ + spin_lock(&server->srv_lock); \ + if (server->tcpStatus != CifsExiting) \ + server->tcpStatus = CifsNeedReconnect; \ + spin_unlock(&server->srv_lock); \ +} while (0) + /* * Update the tcpStatus for the server. * This is used to signal the cifsd thread to call cifs_reconnect @@ -157,39 +161,45 @@ void cifs_signal_cifsd_for_reconnect(struct TCP_Server_Info *server, bool all_channels) { - struct TCP_Server_Info *pserver; + struct TCP_Server_Info *nserver; struct cifs_ses *ses; + LIST_HEAD(reco); int i; - /* If server is a channel, select the primary channel */ - pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; - /* if we need to signal just this channel */ if (!all_channels) { - spin_lock(&server->srv_lock); - if (server->tcpStatus != CifsExiting) - server->tcpStatus = CifsNeedReconnect; - spin_unlock(&server->srv_lock); + set_need_reco(server); return; } - spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { - if (cifs_ses_exiting(ses)) - continue; - spin_lock(&ses->chan_lock); - for (i = 0; i < ses->chan_count; i++) { - if (!ses->chans[i].server) + if (SERVER_IS_CHAN(server)) + server = server->primary_server; + scoped_guard(spinlock, &cifs_tcp_ses_lock) { + set_need_reco(server); + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); continue; - - spin_lock(&ses->chans[i].server->srv_lock); - if (ses->chans[i].server->tcpStatus != CifsExiting) - ses->chans[i].server->tcpStatus = CifsNeedReconnect; - spin_unlock(&ses->chans[i].server->srv_lock); + } + spin_lock(&ses->chan_lock); + for (i = 1; i < ses->chan_count; i++) { + nserver = ses->chans[i].server; + if (!nserver) + continue; + nserver->srv_count++; + list_add(&nserver->rlist, &reco); + } + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); } - spin_unlock(&ses->chan_lock); } - spin_unlock(&cifs_tcp_ses_lock); + + list_for_each_entry_safe(server, nserver, &reco, rlist) { + list_del_init(&server->rlist); + set_need_reco(server); + cifs_put_tcp_session(server, 0); + } } /* @@ -393,6 +403,13 @@ static int __cifs_reconnect(struct TCP_Server_Info *server, if (!cifs_tcp_ses_needs_reconnect(server, 1)) return 0; + /* + * if smb session has been marked for reconnect, also reconnect all + * connections. This way, the other connections do not end up bad. + */ + if (mark_smb_session) + cifs_signal_cifsd_for_reconnect(server, mark_smb_session); + cifs_mark_tcp_ses_conns_for_reconnect(server, mark_smb_session); cifs_abort_connection(server); @@ -401,7 +418,8 @@ static int __cifs_reconnect(struct TCP_Server_Info *server, try_to_freeze(); cifs_server_lock(server); - if (!cifs_swn_set_server_dstaddr(server)) { + if (!cifs_swn_set_server_dstaddr(server) && + !SERVER_IS_CHAN(server)) { /* resolve the hostname again to make sure that IP address is up-to-date */ rc = reconn_set_ipaddr_from_hostname(server); cifs_dbg(FYI, "%s: reconn_set_ipaddr_from_hostname: rc=%d\n", __func__, rc); @@ -659,12 +677,12 @@ server_unresponsive(struct TCP_Server_Info *server) /* * If we're in the process of mounting a share or reconnecting a session * and the server abruptly shut down (e.g. socket wasn't closed, packet - * had been ACK'ed but no SMB response), don't wait longer than 20s to - * negotiate protocol. + * had been ACK'ed but no SMB response), don't wait longer than 20s from + * when negotiate actually started. */ spin_lock(&server->srv_lock); if (server->tcpStatus == CifsInNegotiate && - time_after(jiffies, server->lstrp + 20 * HZ)) { + time_after(jiffies, server->neg_start + 20 * HZ)) { spin_unlock(&server->srv_lock); cifs_reconnect(server, false); return true; @@ -1533,6 +1551,9 @@ static int match_server(struct TCP_Server_Info *server, if (server->nosharesock) return 0; + if (!match_super && (ctx->dfs_conn || server->dfs_conn)) + return 0; + /* If multidialect negotiation see if existing sessions match one */ if (strcmp(ctx->vals->version_string, SMB3ANY_VERSION_STRING) == 0) { if (server->vals->protocol_id < SMB30_PROT_ID) @@ -1722,6 +1743,7 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, if (ctx->nosharesock) tcp_ses->nosharesock = true; + tcp_ses->dfs_conn = ctx->dfs_conn; tcp_ses->ops = ctx->ops; tcp_ses->vals = ctx->vals; @@ -1872,12 +1894,14 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, } /* this function must be called with ses_lock and chan_lock held */ -static int match_session(struct cifs_ses *ses, struct smb3_fs_context *ctx) +static int match_session(struct cifs_ses *ses, + struct smb3_fs_context *ctx, + bool match_super) { struct TCP_Server_Info *server = ses->server; enum securityEnum ctx_sec, ses_sec; - if (ctx->dfs_root_ses != ses->dfs_root_ses) + if (!match_super && ctx->dfs_root_ses != ses->dfs_root_ses) return 0; /* @@ -2029,7 +2053,7 @@ cifs_find_smb_ses(struct TCP_Server_Info *server, struct smb3_fs_context *ctx) continue; } spin_lock(&ses->chan_lock); - if (match_session(ses, ctx)) { + if (match_session(ses, ctx, false)) { spin_unlock(&ses->chan_lock); spin_unlock(&ses->ses_lock); ret = ses; @@ -2432,8 +2456,6 @@ cifs_get_smb_ses(struct TCP_Server_Info *server, struct smb3_fs_context *ctx) * need to lock before changing something in the session. */ spin_lock(&cifs_tcp_ses_lock); - if (ctx->dfs_root_ses) - cifs_smb_ses_inc_refcount(ctx->dfs_root_ses); ses->dfs_root_ses = ctx->dfs_root_ses; list_add(&ses->smb_ses_list, &server->smb_ses_list); spin_unlock(&cifs_tcp_ses_lock); @@ -2510,6 +2532,7 @@ cifs_put_tcon(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace) { unsigned int xid; struct cifs_ses *ses; + LIST_HEAD(ses_list); /* * IPC tcon share the lifetime of their session and are @@ -2541,6 +2564,7 @@ cifs_put_tcon(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace) cancel_delayed_work_sync(&tcon->query_interfaces); #ifdef CONFIG_CIFS_DFS_UPCALL cancel_delayed_work_sync(&tcon->dfs_cache_work); + list_replace_init(&tcon->dfs_ses_list, &ses_list); #endif if (tcon->use_witness) { @@ -2561,6 +2585,9 @@ cifs_put_tcon(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace) cifs_fscache_release_super_cookie(tcon); tconInfoFree(tcon, netfs_trace_tcon_ref_free); cifs_put_smb_ses(ses); +#ifdef CONFIG_CIFS_DFS_UPCALL + dfs_put_root_smb_sessions(&ses_list); +#endif } /** @@ -2789,20 +2816,14 @@ cifs_get_tcon(struct cifs_ses *ses, struct smb3_fs_context *ctx) tcon->max_cached_dirs = ctx->max_cached_dirs; tcon->nodelete = ctx->nodelete; tcon->local_lease = ctx->local_lease; - INIT_LIST_HEAD(&tcon->pending_opens); tcon->status = TID_GOOD; - INIT_DELAYED_WORK(&tcon->query_interfaces, - smb2_query_server_interfaces); if (ses->server->dialect >= SMB30_PROT_ID && (ses->server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL)) { /* schedule query interfaces poll */ queue_delayed_work(cifsiod_wq, &tcon->query_interfaces, (SMB_INTERFACE_POLL_INTERVAL * HZ)); } -#ifdef CONFIG_CIFS_DFS_UPCALL - INIT_DELAYED_WORK(&tcon->dfs_cache_work, dfs_cache_refresh); -#endif spin_lock(&cifs_tcp_ses_lock); list_add(&tcon->tcon_list, &ses->tcon_list); spin_unlock(&cifs_tcp_ses_lock); @@ -2944,7 +2965,7 @@ cifs_match_super(struct super_block *sb, void *data) spin_lock(&ses->chan_lock); spin_lock(&tcon->tc_lock); if (!match_server(tcp_srv, ctx, true) || - !match_session(ses, ctx) || + !match_session(ses, ctx, true) || !match_tcon(tcon, ctx) || !match_prepath(sb, tcon, mnt_data)) { rc = 0; @@ -3051,8 +3072,10 @@ ip_rfc1001_connect(struct TCP_Server_Info *server) * sessinit is sent but no second negprot */ struct rfc1002_session_packet req = {}; - struct smb_hdr *smb_buf = (struct smb_hdr *)&req; + struct msghdr msg = {}; + struct kvec iov = {}; unsigned int len; + size_t sent; req.trailer.session_req.called_len = sizeof(req.trailer.session_req.called_name); @@ -3081,10 +3104,18 @@ ip_rfc1001_connect(struct TCP_Server_Info *server) * As per rfc1002, @len must be the number of bytes that follows the * length field of a rfc1002 session request payload. */ - len = sizeof(req) - offsetof(struct rfc1002_session_packet, trailer.session_req); + len = sizeof(req.trailer.session_req); + req.type = RFC1002_SESSION_REQUEST; + req.flags = 0; + req.length = cpu_to_be16(len); + len += offsetof(typeof(req), trailer.session_req); + iov.iov_base = &req; + iov.iov_len = len; + iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &iov, 1, len); + rc = smb_send_kvec(server, &msg, &sent); + if (rc < 0 || len != sent) + return (rc == -EINTR || rc == -EAGAIN) ? rc : -ECONNABORTED; - smb_buf->smb_buf_length = cpu_to_be32((RFC1002_SESSION_REQUEST << 24) | len); - rc = smb_send(server, smb_buf, len); /* * RFC1001 layer in at least one server requires very short break before * negprot presumably because not expecting negprot to follow so fast. @@ -3093,7 +3124,7 @@ ip_rfc1001_connect(struct TCP_Server_Info *server) */ usleep_range(1000, 2000); - return rc; + return 0; } static int @@ -3131,18 +3162,15 @@ generic_ip_connect(struct TCP_Server_Info *server) struct net *net = cifs_net_ns(server); struct sock *sk; - rc = __sock_create(net, sfamily, SOCK_STREAM, - IPPROTO_TCP, &server->ssocket, 1); + rc = sock_create_kern(net, sfamily, SOCK_STREAM, + IPPROTO_TCP, &server->ssocket); if (rc < 0) { cifs_server_dbg(VFS, "Error %d creating socket\n", rc); return rc; } sk = server->ssocket->sk; - __netns_tracker_free(net, &sk->ns_tracker, false); - sk->sk_net_refcnt = 1; - get_net_track(net, &sk->ns_tracker, GFP_KERNEL); - sock_inuse_add(net, 1); + sk_net_refcnt_upgrade(sk); /* BB other socket options to set KEEPALIVE, NODELAY? */ cifs_dbg(FYI, "Socket created\n"); @@ -3684,13 +3712,12 @@ int cifs_is_path_remote(struct cifs_mount_ctx *mnt_ctx) int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) { struct cifs_mount_ctx mnt_ctx = { .cifs_sb = cifs_sb, .fs_ctx = ctx, }; - bool isdfs; int rc; - rc = dfs_mount_share(&mnt_ctx, &isdfs); + rc = dfs_mount_share(&mnt_ctx); if (rc) goto error; - if (!isdfs) + if (!ctx->dfs_conn) goto out; /* @@ -3946,11 +3973,13 @@ int cifs_negotiate_protocol(const unsigned int xid, struct cifs_ses *ses, struct TCP_Server_Info *server) { + bool in_retry = false; int rc = 0; if (!server->ops->need_neg || !server->ops->negotiate) return -ENOSYS; +retry: /* only send once per connect */ spin_lock(&server->srv_lock); if (server->tcpStatus != CifsGood && @@ -3967,9 +3996,18 @@ cifs_negotiate_protocol(const unsigned int xid, struct cifs_ses *ses, } server->tcpStatus = CifsInNegotiate; + server->neg_start = jiffies; spin_unlock(&server->srv_lock); rc = server->ops->negotiate(xid, ses, server); + if (rc == -EAGAIN) { + /* Allow one retry attempt */ + if (!in_retry) { + in_retry = true; + goto retry; + } + rc = -EHOSTDOWN; + } if (rc == 0) { spin_lock(&server->srv_lock); if (server->tcpStatus == CifsInNegotiate) @@ -4095,7 +4133,7 @@ cifs_set_vol_auth(struct smb3_fs_context *ctx, struct cifs_ses *ses) } static struct cifs_tcon * -__cifs_construct_tcon(struct cifs_sb_info *cifs_sb, kuid_t fsuid) +cifs_construct_tcon(struct cifs_sb_info *cifs_sb, kuid_t fsuid) { int rc; struct cifs_tcon *master_tcon = cifs_sb_master_tcon(cifs_sb); @@ -4193,17 +4231,6 @@ __cifs_construct_tcon(struct cifs_sb_info *cifs_sb, kuid_t fsuid) return tcon; } -static struct cifs_tcon * -cifs_construct_tcon(struct cifs_sb_info *cifs_sb, kuid_t fsuid) -{ - struct cifs_tcon *ret; - - cifs_mount_lock(); - ret = __cifs_construct_tcon(cifs_sb, fsuid); - cifs_mount_unlock(); - return ret; -} - struct cifs_tcon * cifs_sb_master_tcon(struct cifs_sb_info *cifs_sb) { diff --git a/fs/smb/client/dfs.c b/fs/smb/client/dfs.c index bd259b04cdede..c35953843373e 100644 --- a/fs/smb/client/dfs.c +++ b/fs/smb/client/dfs.c @@ -69,7 +69,7 @@ static int get_session(struct cifs_mount_ctx *mnt_ctx, const char *full_path) * Get an active reference of @ses so that next call to cifs_put_tcon() won't * release it as any new DFS referrals must go through its IPC tcon. */ -static void add_root_smb_session(struct cifs_mount_ctx *mnt_ctx) +static void set_root_smb_session(struct cifs_mount_ctx *mnt_ctx) { struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; struct cifs_ses *ses = mnt_ctx->ses; @@ -95,7 +95,7 @@ static inline int parse_dfs_target(struct smb3_fs_context *ctx, return rc; } -static int set_ref_paths(struct cifs_mount_ctx *mnt_ctx, +static int setup_dfs_ref(struct cifs_mount_ctx *mnt_ctx, struct dfs_info3_param *tgt, struct dfs_ref_walk *rw) { @@ -120,6 +120,7 @@ static int set_ref_paths(struct cifs_mount_ctx *mnt_ctx, } ref_walk_path(rw) = ref_path; ref_walk_fpath(rw) = full_path; + ref_walk_ses(rw) = ctx->dfs_root_ses; return 0; } @@ -128,11 +129,11 @@ static int __dfs_referral_walk(struct cifs_mount_ctx *mnt_ctx, { struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; struct dfs_info3_param tgt = {}; - bool is_refsrv; int rc = -ENOENT; again: do { + ctx->dfs_root_ses = ref_walk_ses(rw); if (ref_walk_empty(rw)) { rc = dfs_get_referral(mnt_ctx, ref_walk_path(rw) + 1, NULL, ref_walk_tl(rw)); @@ -158,10 +159,7 @@ static int __dfs_referral_walk(struct cifs_mount_ctx *mnt_ctx, if (rc) continue; - is_refsrv = tgt.server_type == DFS_TYPE_ROOT || - DFS_INTERLINK(tgt.flags); ref_walk_set_tgt_hint(rw); - if (tgt.flags & DFSREF_STORAGE_SERVER) { rc = cifs_mount_get_tcon(mnt_ctx); if (!rc) @@ -172,12 +170,10 @@ static int __dfs_referral_walk(struct cifs_mount_ctx *mnt_ctx, continue; } - if (is_refsrv) - add_root_smb_session(mnt_ctx); - + set_root_smb_session(mnt_ctx); rc = ref_walk_advance(rw); if (!rc) { - rc = set_ref_paths(mnt_ctx, &tgt, rw); + rc = setup_dfs_ref(mnt_ctx, &tgt, rw); if (!rc) { rc = -EREMOTE; goto again; @@ -193,20 +189,22 @@ static int __dfs_referral_walk(struct cifs_mount_ctx *mnt_ctx, return rc; } -static int dfs_referral_walk(struct cifs_mount_ctx *mnt_ctx) +static int dfs_referral_walk(struct cifs_mount_ctx *mnt_ctx, + struct dfs_ref_walk **rw) { - struct dfs_ref_walk *rw; int rc; - rw = ref_walk_alloc(); - if (IS_ERR(rw)) - return PTR_ERR(rw); + *rw = ref_walk_alloc(); + if (IS_ERR(*rw)) { + rc = PTR_ERR(*rw); + *rw = NULL; + return rc; + } - ref_walk_init(rw); - rc = set_ref_paths(mnt_ctx, NULL, rw); + ref_walk_init(*rw); + rc = setup_dfs_ref(mnt_ctx, NULL, *rw); if (!rc) - rc = __dfs_referral_walk(mnt_ctx, rw); - ref_walk_free(rw); + rc = __dfs_referral_walk(mnt_ctx, *rw); return rc; } @@ -214,16 +212,16 @@ static int __dfs_mount_share(struct cifs_mount_ctx *mnt_ctx) { struct cifs_sb_info *cifs_sb = mnt_ctx->cifs_sb; struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; + struct dfs_ref_walk *rw = NULL; struct cifs_tcon *tcon; char *origin_fullpath; - bool new_tcon = true; int rc; origin_fullpath = dfs_get_path(cifs_sb, ctx->source); if (IS_ERR(origin_fullpath)) return PTR_ERR(origin_fullpath); - rc = dfs_referral_walk(mnt_ctx); + rc = dfs_referral_walk(mnt_ctx, &rw); if (!rc) { /* * Prevent superblock from being created with any missing @@ -241,21 +239,16 @@ static int __dfs_mount_share(struct cifs_mount_ctx *mnt_ctx) tcon = mnt_ctx->tcon; spin_lock(&tcon->tc_lock); - if (!tcon->origin_fullpath) { - tcon->origin_fullpath = origin_fullpath; - origin_fullpath = NULL; - } else { - new_tcon = false; - } + tcon->origin_fullpath = origin_fullpath; + origin_fullpath = NULL; + ref_walk_set_tcon(rw, tcon); spin_unlock(&tcon->tc_lock); - - if (new_tcon) { - queue_delayed_work(dfscache_wq, &tcon->dfs_cache_work, - dfs_cache_get_ttl() * HZ); - } + queue_delayed_work(dfscache_wq, &tcon->dfs_cache_work, + dfs_cache_get_ttl() * HZ); out: kfree(origin_fullpath); + ref_walk_free(rw); return rc; } @@ -279,7 +272,7 @@ static int update_fs_context_dstaddr(struct smb3_fs_context *ctx) return rc; } -int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx, bool *isdfs) +int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx) { struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; bool nodfs = ctx->nodfs; @@ -289,7 +282,6 @@ int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx, bool *isdfs) if (rc) return rc; - *isdfs = false; rc = get_session(mnt_ctx, NULL); if (rc) return rc; @@ -317,10 +309,15 @@ int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx, bool *isdfs) return rc; } - *isdfs = true; - add_root_smb_session(mnt_ctx); - rc = __dfs_mount_share(mnt_ctx); - dfs_put_root_smb_sessions(mnt_ctx); + if (!ctx->dfs_conn) { + ctx->dfs_conn = true; + cifs_mount_put_conns(mnt_ctx); + rc = get_session(mnt_ctx, NULL); + } + if (!rc) { + set_root_smb_session(mnt_ctx); + rc = __dfs_mount_share(mnt_ctx); + } return rc; } diff --git a/fs/smb/client/dfs.h b/fs/smb/client/dfs.h index e5c4dcf837503..1aa2bc65b3bc2 100644 --- a/fs/smb/client/dfs.h +++ b/fs/smb/client/dfs.h @@ -19,6 +19,7 @@ struct dfs_ref { char *path; char *full_path; + struct cifs_ses *ses; struct dfs_cache_tgt_list tl; struct dfs_cache_tgt_iterator *tit; }; @@ -38,6 +39,7 @@ struct dfs_ref_walk { #define ref_walk_path(w) (ref_walk_cur(w)->path) #define ref_walk_fpath(w) (ref_walk_cur(w)->full_path) #define ref_walk_tl(w) (&ref_walk_cur(w)->tl) +#define ref_walk_ses(w) (ref_walk_cur(w)->ses) static inline struct dfs_ref_walk *ref_walk_alloc(void) { @@ -60,14 +62,19 @@ static inline void __ref_walk_free(struct dfs_ref *ref) kfree(ref->path); kfree(ref->full_path); dfs_cache_free_tgts(&ref->tl); + if (ref->ses) + cifs_put_smb_ses(ref->ses); memset(ref, 0, sizeof(*ref)); } static inline void ref_walk_free(struct dfs_ref_walk *rw) { - struct dfs_ref *ref = ref_walk_start(rw); + struct dfs_ref *ref; - for (; ref <= ref_walk_end(rw); ref++) + if (!rw) + return; + + for (ref = ref_walk_start(rw); ref <= ref_walk_end(rw); ref++) __ref_walk_free(ref); kfree(rw); } @@ -116,9 +123,22 @@ static inline void ref_walk_set_tgt_hint(struct dfs_ref_walk *rw) ref_walk_tit(rw)); } +static inline void ref_walk_set_tcon(struct dfs_ref_walk *rw, + struct cifs_tcon *tcon) +{ + struct dfs_ref *ref = ref_walk_start(rw); + + for (; ref <= ref_walk_cur(rw); ref++) { + if (WARN_ON_ONCE(!ref->ses)) + continue; + list_add(&ref->ses->dlist, &tcon->dfs_ses_list); + ref->ses = NULL; + } +} + int dfs_parse_target_referral(const char *full_path, const struct dfs_info3_param *ref, struct smb3_fs_context *ctx); -int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx, bool *isdfs); +int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx); static inline char *dfs_get_path(struct cifs_sb_info *cifs_sb, const char *path) { @@ -142,20 +162,14 @@ static inline int dfs_get_referral(struct cifs_mount_ctx *mnt_ctx, const char *p * references of all DFS root sessions that were used across the mount process * in dfs_mount_share(). */ -static inline void dfs_put_root_smb_sessions(struct cifs_mount_ctx *mnt_ctx) +static inline void dfs_put_root_smb_sessions(struct list_head *head) { - const struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; - struct cifs_ses *ses = ctx->dfs_root_ses; - struct cifs_ses *cur; - - if (!ses) - return; + struct cifs_ses *ses, *n; - for (cur = ses; cur; cur = cur->dfs_root_ses) { - if (cur->dfs_root_ses) - cifs_put_smb_ses(cur->dfs_root_ses); + list_for_each_entry_safe(ses, n, head, dlist) { + list_del_init(&ses->dlist); + cifs_put_smb_ses(ses); } - cifs_put_smb_ses(ses); } #endif /* _CIFS_DFS_H */ diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c index 11c8efecf7aa1..433f546055b97 100644 --- a/fs/smb/client/dfs_cache.c +++ b/fs/smb/client/dfs_cache.c @@ -1095,16 +1095,18 @@ int dfs_cache_get_tgt_share(char *path, const struct dfs_cache_tgt_iterator *it, return 0; } -static bool target_share_equal(struct TCP_Server_Info *server, const char *s1, const char *s2) +static bool target_share_equal(struct cifs_tcon *tcon, const char *s1) { - char unc[sizeof("\\\\") + SERVER_NAME_LENGTH] = {0}; + struct TCP_Server_Info *server = tcon->ses->server; + struct sockaddr_storage ss; const char *host; + const char *s2 = &tcon->tree_name[1]; size_t hostlen; - struct sockaddr_storage ss; + char unc[sizeof("\\\\") + SERVER_NAME_LENGTH] = {0}; bool match; int rc; - if (strcasecmp(s1, s2)) + if (strcasecmp(s2, s1)) return false; /* @@ -1128,34 +1130,6 @@ static bool target_share_equal(struct TCP_Server_Info *server, const char *s1, c return match; } -/* - * Mark dfs tcon for reconnecting when the currently connected tcon does not match any of the new - * target shares in @refs. - */ -static void mark_for_reconnect_if_needed(struct TCP_Server_Info *server, - const char *path, - struct dfs_cache_tgt_list *old_tl, - struct dfs_cache_tgt_list *new_tl) -{ - struct dfs_cache_tgt_iterator *oit, *nit; - - for (oit = dfs_cache_get_tgt_iterator(old_tl); oit; - oit = dfs_cache_get_next_tgt(old_tl, oit)) { - for (nit = dfs_cache_get_tgt_iterator(new_tl); nit; - nit = dfs_cache_get_next_tgt(new_tl, nit)) { - if (target_share_equal(server, - dfs_cache_get_tgt_name(oit), - dfs_cache_get_tgt_name(nit))) { - dfs_cache_noreq_update_tgthint(path, nit); - return; - } - } - } - - cifs_dbg(FYI, "%s: no cached or matched targets. mark dfs share for reconnect.\n", __func__); - cifs_signal_cifsd_for_reconnect(server, true); -} - static bool is_ses_good(struct cifs_ses *ses) { struct TCP_Server_Info *server = ses->server; @@ -1172,41 +1146,35 @@ static bool is_ses_good(struct cifs_ses *ses) return ret; } -/* Refresh dfs referral of @ses and mark it for reconnect if needed */ -static void __refresh_ses_referral(struct cifs_ses *ses, bool force_refresh) +static char *get_ses_refpath(struct cifs_ses *ses) { struct TCP_Server_Info *server = ses->server; - DFS_CACHE_TGT_LIST(old_tl); - DFS_CACHE_TGT_LIST(new_tl); - bool needs_refresh = false; - struct cache_entry *ce; - unsigned int xid; - char *path = NULL; - int rc = 0; - - xid = get_xid(); + char *path = ERR_PTR(-ENOENT); mutex_lock(&server->refpath_lock); if (server->leaf_fullpath) { path = kstrdup(server->leaf_fullpath + 1, GFP_ATOMIC); if (!path) - rc = -ENOMEM; + path = ERR_PTR(-ENOMEM); } mutex_unlock(&server->refpath_lock); - if (!path) - goto out; + return path; +} - down_read(&htable_rw_lock); - ce = lookup_cache_entry(path); - needs_refresh = force_refresh || IS_ERR(ce) || cache_entry_expired(ce); - if (!IS_ERR(ce)) { - rc = get_targets(ce, &old_tl); - cifs_dbg(FYI, "%s: get_targets: %d\n", __func__, rc); - } - up_read(&htable_rw_lock); +/* Refresh dfs referral of @ses */ +static void refresh_ses_referral(struct cifs_ses *ses) +{ + struct cache_entry *ce; + unsigned int xid; + char *path; + int rc = 0; - if (!needs_refresh) { - rc = 0; + xid = get_xid(); + + path = get_ses_refpath(ses); + if (IS_ERR(path)) { + rc = PTR_ERR(path); + path = NULL; goto out; } @@ -1217,29 +1185,106 @@ static void __refresh_ses_referral(struct cifs_ses *ses, bool force_refresh) goto out; } - ce = cache_refresh_path(xid, ses, path, true); - if (!IS_ERR(ce)) { - rc = get_targets(ce, &new_tl); + ce = cache_refresh_path(xid, ses, path, false); + if (!IS_ERR(ce)) up_read(&htable_rw_lock); - cifs_dbg(FYI, "%s: get_targets: %d\n", __func__, rc); - mark_for_reconnect_if_needed(server, path, &old_tl, &new_tl); - } + else + rc = PTR_ERR(ce); out: free_xid(xid); - dfs_cache_free_tgts(&old_tl); - dfs_cache_free_tgts(&new_tl); kfree(path); } -static inline void refresh_ses_referral(struct cifs_ses *ses) +static int __refresh_tcon_referral(struct cifs_tcon *tcon, + const char *path, + struct dfs_info3_param *refs, + int numrefs, bool force_refresh) { - __refresh_ses_referral(ses, false); + struct cache_entry *ce; + bool reconnect = force_refresh; + int rc = 0; + int i; + + if (unlikely(!numrefs)) + return 0; + + if (force_refresh) { + for (i = 0; i < numrefs; i++) { + /* TODO: include prefix paths in the matching */ + if (target_share_equal(tcon, refs[i].node_name)) { + reconnect = false; + break; + } + } + } + + down_write(&htable_rw_lock); + ce = lookup_cache_entry(path); + if (!IS_ERR(ce)) { + if (force_refresh || cache_entry_expired(ce)) + rc = update_cache_entry_locked(ce, refs, numrefs); + } else if (PTR_ERR(ce) == -ENOENT) { + ce = add_cache_entry_locked(refs, numrefs); + } + up_write(&htable_rw_lock); + + if (IS_ERR(ce)) + rc = PTR_ERR(ce); + if (reconnect) { + cifs_tcon_dbg(FYI, "%s: mark for reconnect\n", __func__); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); + } + return rc; } -static inline void force_refresh_ses_referral(struct cifs_ses *ses) +static void refresh_tcon_referral(struct cifs_tcon *tcon, bool force_refresh) { - __refresh_ses_referral(ses, true); + struct dfs_info3_param *refs = NULL; + struct cache_entry *ce; + struct cifs_ses *ses; + unsigned int xid; + bool needs_refresh; + char *path; + int numrefs = 0; + int rc = 0; + + xid = get_xid(); + ses = tcon->ses; + + path = get_ses_refpath(ses); + if (IS_ERR(path)) { + rc = PTR_ERR(path); + path = NULL; + goto out; + } + + down_read(&htable_rw_lock); + ce = lookup_cache_entry(path); + needs_refresh = force_refresh || IS_ERR(ce) || cache_entry_expired(ce); + if (!needs_refresh) { + up_read(&htable_rw_lock); + goto out; + } + up_read(&htable_rw_lock); + + ses = CIFS_DFS_ROOT_SES(ses); + if (!is_ses_good(ses)) { + cifs_dbg(FYI, "%s: skip cache refresh due to disconnected ipc\n", + __func__); + goto out; + } + + rc = get_dfs_referral(xid, ses, path, &refs, &numrefs); + if (!rc) { + rc = __refresh_tcon_referral(tcon, path, refs, + numrefs, force_refresh); + } + +out: + free_xid(xid); + kfree(path); + free_dfs_info_array(refs, numrefs); } /** @@ -1280,7 +1325,7 @@ int dfs_cache_remount_fs(struct cifs_sb_info *cifs_sb) */ cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_USE_PREFIX_PATH; - force_refresh_ses_referral(tcon->ses); + refresh_tcon_referral(tcon, true); return 0; } @@ -1292,8 +1337,9 @@ void dfs_cache_refresh(struct work_struct *work) tcon = container_of(work, struct cifs_tcon, dfs_cache_work.work); - for (ses = tcon->ses; ses; ses = ses->dfs_root_ses) + list_for_each_entry(ses, &tcon->dfs_ses_list, dlist) refresh_ses_referral(ses); + refresh_tcon_referral(tcon, false); queue_delayed_work(dfscache_wq, &tcon->dfs_cache_work, atomic_read(&dfs_cache_ttl) * HZ); diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index d883ed75022c4..92e43589fd83f 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -37,6 +37,81 @@ #include "cifs_ioctl.h" #include "cached_dir.h" +/* + * Allocate a bio_vec array and extract up to sg_max pages from a KVEC-type + * iterator and add them to the array. This can deal with vmalloc'd buffers as + * well as kmalloc'd or static buffers. The pages are not pinned. + */ +static ssize_t extract_kvec_to_bvec(struct iov_iter *iter, ssize_t maxsize, + unsigned int bc_max, + struct bio_vec **_bv, unsigned int *_bc) +{ + const struct kvec *kv = iter->kvec; + struct bio_vec *bv; + unsigned long start = iter->iov_offset; + unsigned int i, bc = 0; + ssize_t ret = 0; + + bc_max = iov_iter_npages(iter, bc_max); + if (bc_max == 0) { + *_bv = NULL; + *_bc = 0; + return 0; + } + + bv = kvmalloc(array_size(bc_max, sizeof(*bv)), GFP_NOFS); + if (!bv) { + *_bv = NULL; + *_bc = 0; + return -ENOMEM; + } + *_bv = bv; + + for (i = 0; i < iter->nr_segs; i++) { + struct page *page; + unsigned long kaddr; + size_t off, len, seg; + + len = kv[i].iov_len; + if (start >= len) { + start -= len; + continue; + } + + kaddr = (unsigned long)kv[i].iov_base + start; + off = kaddr & ~PAGE_MASK; + len = min_t(size_t, maxsize, len - start); + kaddr &= PAGE_MASK; + + maxsize -= len; + ret += len; + do { + seg = umin(len, PAGE_SIZE - off); + if (is_vmalloc_or_module_addr((void *)kaddr)) + page = vmalloc_to_page((void *)kaddr); + else + page = virt_to_page((void *)kaddr); + + bvec_set_page(bv, page, len, off); + bv++; + bc++; + + len -= seg; + kaddr += PAGE_SIZE; + off = 0; + } while (len > 0 && bc < bc_max); + + if (maxsize <= 0 || bc >= bc_max) + break; + start = 0; + } + + if (ret > 0) + iov_iter_advance(iter, ret); + *_bc = bc; + return ret; +} + /* * Remove the dirty flags from a span of pages. */ @@ -2421,7 +2496,7 @@ cifs_get_writable_file(struct cifsInodeInfo *cifs_inode, int flags, } } } - /* couldn't find useable FH with same pid, try any available */ + /* couldn't find usable FH with same pid, try any available */ if (!any_available) { any_available = true; goto refind_writable; @@ -2747,8 +2822,10 @@ static void cifs_extend_writeback(struct address_space *mapping, loff_t start, int max_pages, loff_t max_len, - size_t *_len) + size_t *_len, + unsigned long long i_size) { + struct inode *inode = mapping->host; struct folio_batch batch; struct folio *folio; unsigned int nr_pages; @@ -2779,7 +2856,7 @@ static void cifs_extend_writeback(struct address_space *mapping, if (!folio_try_get(folio)) { xas_reset(xas); - continue; + break; } nr_pages = folio_nr_pages(folio); if (nr_pages > max_pages) { @@ -2799,6 +2876,15 @@ static void cifs_extend_writeback(struct address_space *mapping, xas_reset(xas); break; } + + /* if file size is changing, stop extending */ + if (i_size_read(inode) != i_size) { + folio_unlock(folio); + folio_put(folio); + xas_reset(xas); + break; + } + if (!folio_test_dirty(folio) || folio_test_writeback(folio)) { folio_unlock(folio); @@ -2884,17 +2970,21 @@ static ssize_t cifs_write_back_from_locked_folio(struct address_space *mapping, rc = cifs_get_writable_file(CIFS_I(inode), FIND_WR_ANY, &cfile); if (rc) { cifs_dbg(VFS, "No writable handle in writepages rc=%d\n", rc); + folio_unlock(folio); goto err_xid; } rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->wsize, &wsize, credits); - if (rc != 0) + if (rc != 0) { + folio_unlock(folio); goto err_close; + } wdata = cifs_writedata_alloc(cifs_writev_complete); if (!wdata) { rc = -ENOMEM; + folio_unlock(folio); goto err_uncredit; } @@ -2930,7 +3020,8 @@ static ssize_t cifs_write_back_from_locked_folio(struct address_space *mapping, if (max_pages > 0) cifs_extend_writeback(mapping, xas, &count, start, - max_pages, max_len, &len); + max_pages, max_len, &len, + i_size); } } len = min_t(unsigned long long, len, i_size - start); @@ -3041,17 +3132,22 @@ static ssize_t cifs_writepages_begin(struct address_space *mapping, lock_again: if (wbc->sync_mode != WB_SYNC_NONE) { ret = folio_lock_killable(folio); - if (ret < 0) + if (ret < 0) { + folio_put(folio); return ret; + } } else { - if (!folio_trylock(folio)) + if (!folio_trylock(folio)) { + folio_put(folio); goto search_again; + } } if (folio->mapping != mapping || !folio_test_dirty(folio)) { start += folio_size(folio); folio_unlock(folio); + folio_put(folio); goto search_again; } @@ -3081,6 +3177,7 @@ static ssize_t cifs_writepages_begin(struct address_space *mapping, out: if (ret > 0) *_start = start + ret; + folio_put(folio); return ret; } @@ -4308,11 +4405,27 @@ static ssize_t __cifs_readv( ctx->bv = (void *)ctx->iter.bvec; ctx->bv_need_unpin = iov_iter_extract_will_pin(to); ctx->should_dirty = true; - } else if ((iov_iter_is_bvec(to) || iov_iter_is_kvec(to)) && - !is_sync_kiocb(iocb)) { + } else if (iov_iter_is_kvec(to)) { + /* + * Extract a KVEC-type iterator into a BVEC-type iterator. We + * assume that the storage will be retained by the caller; in + * any case, we may or may not be able to pin the pages, so we + * don't try. + */ + unsigned int bc; + + rc = extract_kvec_to_bvec(to, iov_iter_count(to), INT_MAX, + &ctx->bv, &bc); + if (rc < 0) { + kref_put(&ctx->refcount, cifs_aio_ctx_release); + return rc; + } + + iov_iter_bvec(&ctx->iter, ITER_DEST, ctx->bv, bc, rc); + } else if (iov_iter_is_bvec(to) && !is_sync_kiocb(iocb)) { /* * If the op is asynchronous, we need to copy the list attached - * to a BVEC/KVEC-type iterator, but we assume that the storage + * to a BVEC-type iterator, but we assume that the storage * will be retained by the caller; in any case, we may or may * not be able to pin the pages, so we don't try. */ @@ -5042,7 +5155,8 @@ void cifs_oplock_break(struct work_struct *work) struct cifsFileInfo *cfile = container_of(work, struct cifsFileInfo, oplock_break); struct inode *inode = d_inode(cfile->dentry); - struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct super_block *sb = inode->i_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); struct cifsInodeInfo *cinode = CIFS_I(inode); struct cifs_tcon *tcon; struct TCP_Server_Info *server; @@ -5052,6 +5166,12 @@ void cifs_oplock_break(struct work_struct *work) __u64 persistent_fid, volatile_fid; __u16 net_fid; + /* + * Hold a reference to the superblock to prevent it and its inodes from + * being freed while we are accessing cinode. Otherwise, _cifsFileInfo_put() + * may release the last reference to the sb and trigger inode eviction. + */ + cifs_sb_active(sb); wait_on_bit(&cinode->flags, CIFS_INODE_PENDING_WRITERS, TASK_UNINTERRUPTIBLE); @@ -5124,6 +5244,7 @@ void cifs_oplock_break(struct work_struct *work) cifs_put_tlink(tlink); out: cifs_done_oplock_break(cinode); + cifs_sb_deactive(sb); } /* diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index d2e291ef104ec..a64c0b0dbec78 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -1249,6 +1249,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, case Opt_rsize: ctx->rsize = result.uint_32; ctx->got_rsize = true; + ctx->vol_rsize = ctx->rsize; break; case Opt_wsize: ctx->wsize = result.uint_32; @@ -1264,6 +1265,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, ctx->wsize, PAGE_SIZE); } } + ctx->vol_wsize = ctx->wsize; break; case Opt_acregmax: if (result.uint_32 > CIFS_MAX_ACTIMEO / HZ) { @@ -1359,12 +1361,14 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, cifs_errorf(fc, "Unknown error parsing devname\n"); goto cifs_parse_mount_err; } + kfree(ctx->source); ctx->source = smb3_fs_context_fullpath(ctx, '/'); if (IS_ERR(ctx->source)) { ctx->source = NULL; cifs_errorf(fc, "OOM when copying UNC string\n"); goto cifs_parse_mount_err; } + kfree(fc->source); fc->source = kstrdup(ctx->source, GFP_KERNEL); if (fc->source == NULL) { cifs_errorf(fc, "OOM when copying UNC string\n"); @@ -1722,6 +1726,10 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, ctx->password = NULL; kfree_sensitive(ctx->password2); ctx->password2 = NULL; + kfree(ctx->source); + ctx->source = NULL; + kfree(fc->source); + fc->source = NULL; return -EINVAL; } diff --git a/fs/smb/client/fs_context.h b/fs/smb/client/fs_context.h index bbd2063ab838d..90ebff5d0199d 100644 --- a/fs/smb/client/fs_context.h +++ b/fs/smb/client/fs_context.h @@ -253,6 +253,9 @@ struct smb3_fs_context { bool use_client_guid:1; /* reuse existing guid for multichannel */ u8 client_guid[SMB2_CLIENT_GUID_SIZE]; + /* User-specified original r/wsize value */ + unsigned int vol_rsize; + unsigned int vol_wsize; unsigned int bsize; unsigned int rasize; unsigned int rsize; @@ -260,7 +263,7 @@ struct smb3_fs_context { unsigned int min_offload; unsigned int retrans; bool sockopt_tcp_nodelay:1; - /* attribute cache timemout for files and directories in jiffies */ + /* attribute cache timeout for files and directories in jiffies */ unsigned long acregmax; unsigned long acdirmax; /* timeout for deferred close of files in jiffies */ @@ -284,6 +287,7 @@ struct smb3_fs_context { struct cifs_ses *dfs_root_ses; bool dfs_automount:1; /* set for dfs automount only */ enum cifs_reparse_type reparse_type; + bool dfs_conn:1; /* set for dfs mounts */ }; extern const struct fs_parameter_spec smb3_fs_parameters[]; diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index d93ebd58ecae1..e55852aa49f91 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -1856,15 +1856,24 @@ int cifs_unlink(struct inode *dir, struct dentry *dentry) struct cifs_sb_info *cifs_sb = CIFS_SB(sb); struct tcon_link *tlink; struct cifs_tcon *tcon; + __u32 dosattr = 0, origattr = 0; struct TCP_Server_Info *server; struct iattr *attrs = NULL; - __u32 dosattr = 0, origattr = 0; + bool rehash = false; cifs_dbg(FYI, "cifs_unlink, dir=0x%p, dentry=0x%p\n", dir, dentry); if (unlikely(cifs_forced_shutdown(cifs_sb))) return -EIO; + /* Unhash dentry in advance to prevent any concurrent opens */ + spin_lock(&dentry->d_lock); + if (!d_unhashed(dentry)) { + __d_drop(dentry); + rehash = true; + } + spin_unlock(&dentry->d_lock); + tlink = cifs_sb_tlink(cifs_sb); if (IS_ERR(tlink)) return PTR_ERR(tlink); @@ -1915,7 +1924,8 @@ int cifs_unlink(struct inode *dir, struct dentry *dentry) cifs_drop_nlink(inode); } } else if (rc == -ENOENT) { - d_drop(dentry); + if (simple_positive(dentry)) + d_delete(dentry); } else if (rc == -EBUSY) { if (server->ops->rename_pending_delete) { rc = server->ops->rename_pending_delete(full_path, @@ -1968,6 +1978,8 @@ int cifs_unlink(struct inode *dir, struct dentry *dentry) kfree(attrs); free_xid(xid); cifs_put_tlink(tlink); + if (rehash) + d_rehash(dentry); return rc; } @@ -2307,8 +2319,10 @@ cifs_do_rename(const unsigned int xid, struct dentry *from_dentry, tcon = tlink_tcon(tlink); server = tcon->ses->server; - if (!server->ops->rename) - return -ENOSYS; + if (!server->ops->rename) { + rc = -ENOSYS; + goto do_rename_exit; + } /* try path-based rename first */ rc = server->ops->rename(xid, tcon, from_dentry, @@ -2367,6 +2381,7 @@ cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir, struct cifs_sb_info *cifs_sb; struct tcon_link *tlink; struct cifs_tcon *tcon; + bool rehash = false; unsigned int xid; int rc, tmprc; int retry_count = 0; @@ -2382,6 +2397,17 @@ cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir, if (unlikely(cifs_forced_shutdown(cifs_sb))) return -EIO; + /* + * Prevent any concurrent opens on the target by unhashing the dentry. + * VFS already unhashes the target when renaming directories. + */ + if (d_is_positive(target_dentry) && !d_is_dir(target_dentry)) { + if (!d_unhashed(target_dentry)) { + d_drop(target_dentry); + rehash = true; + } + } + tlink = cifs_sb_tlink(cifs_sb); if (IS_ERR(tlink)) return PTR_ERR(tlink); @@ -2421,6 +2447,8 @@ cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir, } } + if (!rc) + rehash = false; /* * No-replace is the natural behavior for CIFS, so skip unlink hacks. */ @@ -2479,12 +2507,16 @@ cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir, goto cifs_rename_exit; rc = cifs_do_rename(xid, source_dentry, from_name, target_dentry, to_name); + if (!rc) + rehash = false; } /* force revalidate to go get info when needed */ CIFS_I(source_dir)->time = CIFS_I(target_dir)->time = 0; cifs_rename_exit: + if (rehash) + d_rehash(target_dentry); kfree(info_buf_source); free_dentry_path(page2); free_dentry_path(page1); diff --git a/fs/smb/client/link.c b/fs/smb/client/link.c index d86da949a9190..007da0a699cb0 100644 --- a/fs/smb/client/link.c +++ b/fs/smb/client/link.c @@ -257,7 +257,7 @@ cifs_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, struct cifs_open_parms oparms; struct cifs_io_parms io_parms = {0}; int buf_type = CIFS_NO_BUFFER; - FILE_ALL_INFO file_info; + struct cifs_open_info_data query_data; oparms = (struct cifs_open_parms) { .tcon = tcon, @@ -269,11 +269,11 @@ cifs_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, .fid = &fid, }; - rc = CIFS_open(xid, &oparms, &oplock, &file_info); + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, &query_data); if (rc) return rc; - if (file_info.EndOfFile != cpu_to_le64(CIFS_MF_SYMLINK_FILE_SIZE)) { + if (query_data.fi.EndOfFile != cpu_to_le64(CIFS_MF_SYMLINK_FILE_SIZE)) { rc = -ENOENT; /* it's not a symlink */ goto out; @@ -312,7 +312,7 @@ cifs_create_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, .fid = &fid, }; - rc = CIFS_open(xid, &oparms, &oplock, NULL); + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, NULL); if (rc) return rc; diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c index 65d4b72b4d51a..9b5aeafe220bd 100644 --- a/fs/smb/client/misc.c +++ b/fs/smb/client/misc.c @@ -145,6 +145,15 @@ tcon_info_alloc(bool dir_leases_enabled, enum smb3_tcon_ref_trace trace) mutex_init(&ret_buf->fscache_lock); #endif trace_smb3_tcon_ref(ret_buf->debug_id, ret_buf->tc_count, trace); +#ifdef CONFIG_CIFS_DFS_UPCALL + INIT_LIST_HEAD(&ret_buf->dfs_ses_list); +#endif + INIT_LIST_HEAD(&ret_buf->pending_opens); + INIT_DELAYED_WORK(&ret_buf->query_interfaces, + smb2_query_server_interfaces); +#ifdef CONFIG_CIFS_DFS_UPCALL + INIT_DELAYED_WORK(&ret_buf->dfs_cache_work, dfs_cache_refresh); +#endif return ret_buf; } @@ -251,7 +260,7 @@ free_rsp_buf(int resp_buftype, void *rsp) } /* NB: MID can not be set if treeCon not passed in, in that - case it is responsbility of caller to set the mid */ + case it is responsibility of caller to set the mid */ void header_assemble(struct smb_hdr *buffer, char smb_command /* command */ , const struct cifs_tcon *treeCon, int word_count @@ -320,6 +329,14 @@ check_smb_hdr(struct smb_hdr *smb) if (smb->Command == SMB_COM_LOCKING_ANDX) return 0; + /* + * Windows NT server returns error resposne (e.g. STATUS_DELETE_PENDING + * or STATUS_OBJECT_NAME_NOT_FOUND or ERRDOS/ERRbadfile or any other) + * for some TRANS2 requests without the RESPONSE flag set in header. + */ + if (smb->Command == SMB_COM_TRANSACTION2 && smb->Status.CifsError != 0) + return 0; + cifs_dbg(VFS, "Server sent request, not response. mid=%u\n", get_mid(smb)); return 1; @@ -905,6 +922,14 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, char *data_end; struct dfs_referral_level_3 *ref; + if (rsp_size < sizeof(*rsp)) { + cifs_dbg(VFS | ONCE, + "%s: header is malformed (size is %u, must be %zu)\n", + __func__, rsp_size, sizeof(*rsp)); + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + *num_of_nodes = le16_to_cpu(rsp->NumberOfReferrals); if (*num_of_nodes < 1) { @@ -914,6 +939,15 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, goto parse_DFS_referrals_exit; } + if (sizeof(*rsp) + *num_of_nodes * sizeof(REFERRAL3) > rsp_size) { + cifs_dbg(VFS | ONCE, + "%s: malformed buffer (size is %u, must be at least %zu)\n", + __func__, rsp_size, + sizeof(*rsp) + *num_of_nodes * sizeof(REFERRAL3)); + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + ref = (struct dfs_referral_level_3 *) &(rsp->referrals); if (ref->VersionNumber != cpu_to_le16(3)) { cifs_dbg(VFS, "Referrals of V%d version are not supported, should be V3\n", diff --git a/fs/smb/client/namespace.c b/fs/smb/client/namespace.c index 830f2a292bb00..a6655807c0865 100644 --- a/fs/smb/client/namespace.c +++ b/fs/smb/client/namespace.c @@ -146,6 +146,9 @@ static char *automount_fullpath(struct dentry *dentry, void *page) } spin_unlock(&tcon->tc_lock); + if (unlikely(!page)) + return ERR_PTR(-ENOMEM); + s = dentry_path_raw(dentry, page, PATH_MAX); if (IS_ERR(s)) return s; @@ -257,7 +260,7 @@ static struct vfsmount *cifs_do_automount(struct path *path) ctx->source = NULL; goto out; } - ctx->dfs_automount = is_dfs_mount(mntpt); + ctx->dfs_automount = ctx->dfs_conn = is_dfs_mount(mntpt); cifs_dbg(FYI, "%s: ctx: source=%s UNC=%s prepath=%s dfs_automount=%d\n", __func__, ctx->source, ctx->UNC, ctx->prepath, ctx->dfs_automount); diff --git a/fs/smb/client/netmisc.c b/fs/smb/client/netmisc.c index 1b52e6ac431cb..2a8d71221e5e7 100644 --- a/fs/smb/client/netmisc.c +++ b/fs/smb/client/netmisc.c @@ -1003,7 +1003,7 @@ struct timespec64 cnvrtDosUnixTm(__le16 le_date, __le16 le_time, int offset) year is 2**7, the last year is 1980+127, which means we need only consider 2 special case years, ie the years 2000 and 2100, and only adjust for the lack of leap year for the year 2100, as 2000 was a - leap year (divisable by 400) */ + leap year (divisible by 400) */ if (year >= 120) /* the year 2100 */ days = days - 1; /* do not count leap year for the year 2100 */ diff --git a/fs/smb/client/readdir.c b/fs/smb/client/readdir.c index 75929a0a56f96..5febf8afaab04 100644 --- a/fs/smb/client/readdir.c +++ b/fs/smb/client/readdir.c @@ -263,7 +263,7 @@ cifs_posix_to_fattr(struct cifs_fattr *fattr, struct smb2_posix_info *info, /* The Mode field in the response can now include the file type as well */ fattr->cf_mode = wire_mode_to_posix(le32_to_cpu(info->Mode), fattr->cf_cifsattrs & ATTR_DIRECTORY); - fattr->cf_dtype = S_DT(le32_to_cpu(info->Mode)); + fattr->cf_dtype = S_DT(fattr->cf_mode); switch (fattr->cf_mode & S_IFMT) { case S_IFLNK: @@ -567,7 +567,7 @@ static void cifs_fill_dirent_std(struct cifs_dirent *de, const FIND_FILE_STANDARD_INFO *info) { de->name = &info->FileName[0]; - /* one byte length, no endianess conversion */ + /* one byte length, no endianness conversion */ de->namelen = info->FileNameLength; de->resume_key = info->ResumeKey; } @@ -733,7 +733,10 @@ find_cifs_entry(const unsigned int xid, struct cifs_tcon *tcon, loff_t pos, else cifs_buf_release(cfile->srch_inf. ntwrk_buf_start); + /* Reset all pointers to the network buffer to prevent stale references */ cfile->srch_inf.ntwrk_buf_start = NULL; + cfile->srch_inf.srch_entries_start = NULL; + cfile->srch_inf.last_entry = NULL; } rc = initiate_cifs_search(xid, file, full_path); if (rc) { @@ -756,11 +759,11 @@ find_cifs_entry(const unsigned int xid, struct cifs_tcon *tcon, loff_t pos, rc = server->ops->query_dir_next(xid, tcon, &cfile->fid, search_flags, &cfile->srch_inf); + if (rc) + return -ENOENT; /* FindFirst/Next set last_entry to NULL on malformed reply */ if (cfile->srch_inf.last_entry) cifs_save_resume_key(cfile->srch_inf.last_entry, cfile); - if (rc) - return -ENOENT; } if (index_to_find < cfile->srch_inf.index_of_last_entry) { /* we found the buffer that contains the entry */ @@ -829,7 +832,7 @@ static bool emit_cached_dirents(struct cached_dirents *cde, * However, this sequence of ->pos values may have holes * in it, for example dot-dirs returned from the server * are suppressed. - * Handle this bu forcing ctx->pos to be the same as the + * Handle this by forcing ctx->pos to be the same as the * ->pos of the current dirent we emit from the cache. * This means that when we emit these entries from the cache * we now emit them with the same ->pos value as in the @@ -847,9 +850,9 @@ static bool emit_cached_dirents(struct cached_dirents *cde, } static void update_cached_dirents_count(struct cached_dirents *cde, - struct dir_context *ctx) + struct file *file) { - if (cde->ctx != ctx) + if (cde->file != file) return; if (cde->is_valid || cde->is_failed) return; @@ -858,9 +861,9 @@ static void update_cached_dirents_count(struct cached_dirents *cde, } static void finished_cached_dirents_count(struct cached_dirents *cde, - struct dir_context *ctx) + struct dir_context *ctx, struct file *file) { - if (cde->ctx != ctx) + if (cde->file != file) return; if (cde->is_valid || cde->is_failed) return; @@ -873,11 +876,12 @@ static void finished_cached_dirents_count(struct cached_dirents *cde, static void add_cached_dirent(struct cached_dirents *cde, struct dir_context *ctx, const char *name, int namelen, - struct cifs_fattr *fattr) + struct cifs_fattr *fattr, + struct file *file) { struct cached_dirent *de; - if (cde->ctx != ctx) + if (cde->file != file) return; if (cde->is_valid || cde->is_failed) return; @@ -907,7 +911,8 @@ static void add_cached_dirent(struct cached_dirents *cde, static bool cifs_dir_emit(struct dir_context *ctx, const char *name, int namelen, struct cifs_fattr *fattr, - struct cached_fid *cfid) + struct cached_fid *cfid, + struct file *file) { bool rc; ino_t ino = cifs_uniqueid_to_ino_t(fattr->cf_uniqueid); @@ -919,7 +924,7 @@ static bool cifs_dir_emit(struct dir_context *ctx, if (cfid) { mutex_lock(&cfid->dirents.de_mutex); add_cached_dirent(&cfid->dirents, ctx, name, namelen, - fattr); + fattr, file); mutex_unlock(&cfid->dirents.de_mutex); } @@ -1019,7 +1024,7 @@ static int cifs_filldir(char *find_entry, struct file *file, cifs_prime_dcache(file_dentry(file), &name, &fattr); return !cifs_dir_emit(ctx, name.name, name.len, - &fattr, cfid); + &fattr, cfid, file); } @@ -1070,8 +1075,8 @@ int cifs_readdir(struct file *file, struct dir_context *ctx) * we need to initialize scanning and storing the * directory content. */ - if (ctx->pos == 0 && cfid->dirents.ctx == NULL) { - cfid->dirents.ctx = ctx; + if (ctx->pos == 0 && cfid->dirents.file == NULL) { + cfid->dirents.file = file; cfid->dirents.pos = 2; } /* @@ -1139,7 +1144,7 @@ int cifs_readdir(struct file *file, struct dir_context *ctx) } else { if (cfid) { mutex_lock(&cfid->dirents.de_mutex); - finished_cached_dirents_count(&cfid->dirents, ctx); + finished_cached_dirents_count(&cfid->dirents, ctx, file); mutex_unlock(&cfid->dirents.de_mutex); } cifs_dbg(FYI, "Could not find entry\n"); @@ -1180,7 +1185,7 @@ int cifs_readdir(struct file *file, struct dir_context *ctx) ctx->pos++; if (cfid) { mutex_lock(&cfid->dirents.de_mutex); - update_cached_dirents_count(&cfid->dirents, ctx); + update_cached_dirents_count(&cfid->dirents, file); mutex_unlock(&cfid->dirents.de_mutex); } diff --git a/fs/smb/client/reparse.c b/fs/smb/client/reparse.c index b6556fe3dfa11..4d45c31336df1 100644 --- a/fs/smb/client/reparse.c +++ b/fs/smb/client/reparse.c @@ -738,7 +738,6 @@ static bool wsl_to_fattr(struct cifs_open_info_data *data, if (!have_xattr_dev && (tag == IO_REPARSE_TAG_LX_CHR || tag == IO_REPARSE_TAG_LX_BLK)) return false; - fattr->cf_dtype = S_DT(fattr->cf_mode); return true; } diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c index f04922eb45d4c..bbde7180a90ac 100644 --- a/fs/smb/client/sess.c +++ b/fs/smb/client/sess.c @@ -372,6 +372,7 @@ cifs_chan_update_iface(struct cifs_ses *ses, struct TCP_Server_Info *server) struct cifs_server_iface *old_iface = NULL; struct cifs_server_iface *last_iface = NULL; struct sockaddr_storage ss; + int retry = 0; spin_lock(&ses->chan_lock); chan_index = cifs_ses_get_chan_index(ses, server); @@ -400,6 +401,7 @@ cifs_chan_update_iface(struct cifs_ses *ses, struct TCP_Server_Info *server) return; } +try_again: last_iface = list_last_entry(&ses->iface_list, struct cifs_server_iface, iface_head); iface_min_speed = last_iface->speed; @@ -437,6 +439,13 @@ cifs_chan_update_iface(struct cifs_ses *ses, struct TCP_Server_Info *server) } if (list_entry_is_head(iface, &ses->iface_list, iface_head)) { + list_for_each_entry(iface, &ses->iface_list, iface_head) + iface->weight_fulfilled = 0; + + /* see if it can be satisfied in second attempt */ + if (!retry++) + goto try_again; + iface = NULL; cifs_dbg(FYI, "unable to find a suitable iface\n"); } @@ -485,6 +494,10 @@ cifs_chan_update_iface(struct cifs_ses *ses, struct TCP_Server_Info *server) ses->chans[chan_index].iface = iface; spin_unlock(&ses->chan_lock); + + spin_lock(&server->srv_lock); + memcpy(&server->dstaddr, &iface->sockaddr, sizeof(server->dstaddr)); + spin_unlock(&server->srv_lock); } /* @@ -554,8 +567,7 @@ cifs_ses_add_channel(struct cifs_ses *ses, ctx->domainauto = ses->domainAuto; ctx->domainname = ses->domainName; - /* no hostname for extra channels */ - ctx->server_hostname = ""; + ctx->server_hostname = ses->server->hostname; ctx->username = ses->user_name; ctx->password = ses->password; @@ -680,6 +692,7 @@ static __u32 cifs_ssetup_hdr(struct cifs_ses *ses, USHRT_MAX)); pSMB->req.MaxMpxCount = cpu_to_le16(server->maxReq); pSMB->req.VcNumber = cpu_to_le16(1); + pSMB->req.SessionKey = server->session_key_id; /* Now no need to set SMBFLG_CASELESS or obsolete CANONICAL PATH */ @@ -1736,22 +1749,22 @@ _sess_auth_rawntlmssp_assemble_req(struct sess_data *sess_data) pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; capabilities = cifs_ssetup_hdr(ses, server, pSMB); - if ((pSMB->req.hdr.Flags2 & SMBFLG2_UNICODE) == 0) { - cifs_dbg(VFS, "NTLMSSP requires Unicode support\n"); - return -ENOSYS; - } - pSMB->req.hdr.Flags2 |= SMBFLG2_EXT_SEC; capabilities |= CAP_EXTENDED_SECURITY; pSMB->req.Capabilities |= cpu_to_le32(capabilities); bcc_ptr = sess_data->iov[2].iov_base; - /* unicode strings must be word aligned */ - if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) { - *bcc_ptr = 0; - bcc_ptr++; + + if (pSMB->req.hdr.Flags2 & SMBFLG2_UNICODE) { + /* unicode strings must be word aligned */ + if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) { + *bcc_ptr = 0; + bcc_ptr++; + } + unicode_oslm_strings(&bcc_ptr, sess_data->nls_cp); + } else { + ascii_oslm_strings(&bcc_ptr, sess_data->nls_cp); } - unicode_oslm_strings(&bcc_ptr, sess_data->nls_cp); sess_data->iov[2].iov_len = (long) bcc_ptr - (long) sess_data->iov[2].iov_base; diff --git a/fs/smb/client/smb1ops.c b/fs/smb/client/smb1ops.c index caa1d852ece49..c80b291a14a57 100644 --- a/fs/smb/client/smb1ops.c +++ b/fs/smb/client/smb1ops.c @@ -426,13 +426,6 @@ cifs_negotiate(const unsigned int xid, { int rc; rc = CIFSSMBNegotiate(xid, ses, server); - if (rc == -EAGAIN) { - /* retry only once on 1st time connection */ - set_credits(server, 1); - rc = CIFSSMBNegotiate(xid, ses, server); - if (rc == -EAGAIN) - rc = -EHOSTDOWN; - } return rc; } @@ -444,8 +437,8 @@ cifs_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) unsigned int wsize; /* start with specified wsize, or default */ - if (ctx->wsize) - wsize = ctx->wsize; + if (ctx->got_wsize) + wsize = ctx->vol_wsize; else if (tcon->unix_ext && (unix_cap & CIFS_UNIX_LARGE_WRITE_CAP)) wsize = CIFS_DEFAULT_IOSIZE; else @@ -497,7 +490,7 @@ cifs_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) else defsize = server->maxBuf - sizeof(READ_RSP); - rsize = ctx->rsize ? ctx->rsize : defsize; + rsize = ctx->got_rsize ? ctx->vol_rsize : defsize; /* * no CAP_LARGE_READ_X? Then MS-CIFS states that we must limit this to @@ -548,24 +541,104 @@ static int cifs_query_path_info(const unsigned int xid, const char *full_path, struct cifs_open_info_data *data) { - int rc; + int rc = -EOPNOTSUPP; FILE_ALL_INFO fi = {}; + struct cifs_search_info search_info = {}; + bool non_unicode_wildcard = false; data->symlink = false; data->adjust_tz = false; - /* could do find first instead but this returns more info */ - rc = CIFSSMBQPathInfo(xid, tcon, full_path, &fi, 0 /* not legacy */, cifs_sb->local_nls, - cifs_remap(cifs_sb)); /* - * BB optimize code so we do not make the above call when server claims - * no NT SMB support and the above call failed at least once - set flag - * in tcon or mount. + * First try CIFSSMBQPathInfo() function which returns more info + * (NumberOfLinks) than CIFSFindFirst() fallback function. + * Some servers like Win9x do not support SMB_QUERY_FILE_ALL_INFO over + * TRANS2_QUERY_PATH_INFORMATION, but supports it with filehandle over + * TRANS2_QUERY_FILE_INFORMATION (function CIFSSMBQFileInfo(). But SMB + * Open command on non-NT servers works only for files, does not work + * for directories. And moreover Win9x SMB server returns bogus data in + * SMB_QUERY_FILE_ALL_INFO Attributes field. So for non-NT servers, + * do not even use CIFSSMBQPathInfo() or CIFSSMBQFileInfo() function. + */ + if (tcon->ses->capabilities & CAP_NT_SMBS) + rc = CIFSSMBQPathInfo(xid, tcon, full_path, &fi, 0 /* not legacy */, + cifs_sb->local_nls, cifs_remap(cifs_sb)); + + /* + * Non-UNICODE variant of fallback functions below expands wildcards, + * so they cannot be used for querying paths with wildcard characters. + */ + if (rc && !(tcon->ses->capabilities & CAP_UNICODE) && strpbrk(full_path, "*?\"><")) + non_unicode_wildcard = true; + + /* + * Then fallback to CIFSFindFirst() which works also with non-NT servers + * but does not does not provide NumberOfLinks. */ - if ((rc == -EOPNOTSUPP) || (rc == -EINVAL)) { + if ((rc == -EOPNOTSUPP || rc == -EINVAL) && + !non_unicode_wildcard) { + if (!(tcon->ses->capabilities & tcon->ses->server->vals->cap_nt_find)) + search_info.info_level = SMB_FIND_FILE_INFO_STANDARD; + else + search_info.info_level = SMB_FIND_FILE_FULL_DIRECTORY_INFO; + rc = CIFSFindFirst(xid, tcon, full_path, cifs_sb, NULL, + CIFS_SEARCH_CLOSE_ALWAYS | CIFS_SEARCH_CLOSE_AT_END, + &search_info, false); + if (rc == 0) { + if (!(tcon->ses->capabilities & tcon->ses->server->vals->cap_nt_find)) { + FIND_FILE_STANDARD_INFO *di; + int offset = tcon->ses->server->timeAdj; + + di = (FIND_FILE_STANDARD_INFO *)search_info.srch_entries_start; + fi.CreationTime = cpu_to_le64(cifs_UnixTimeToNT(cnvrtDosUnixTm( + di->CreationDate, di->CreationTime, offset))); + fi.LastAccessTime = cpu_to_le64(cifs_UnixTimeToNT(cnvrtDosUnixTm( + di->LastAccessDate, di->LastAccessTime, offset))); + fi.LastWriteTime = cpu_to_le64(cifs_UnixTimeToNT(cnvrtDosUnixTm( + di->LastWriteDate, di->LastWriteTime, offset))); + fi.ChangeTime = fi.LastWriteTime; + fi.Attributes = cpu_to_le32(le16_to_cpu(di->Attributes)); + fi.AllocationSize = cpu_to_le64(le32_to_cpu(di->AllocationSize)); + fi.EndOfFile = cpu_to_le64(le32_to_cpu(di->DataSize)); + } else { + FILE_FULL_DIRECTORY_INFO *di; + + di = (FILE_FULL_DIRECTORY_INFO *)search_info.srch_entries_start; + fi.CreationTime = di->CreationTime; + fi.LastAccessTime = di->LastAccessTime; + fi.LastWriteTime = di->LastWriteTime; + fi.ChangeTime = di->ChangeTime; + fi.Attributes = di->ExtFileAttributes; + fi.AllocationSize = di->AllocationSize; + fi.EndOfFile = di->EndOfFile; + fi.EASize = di->EaSize; + } + fi.NumberOfLinks = cpu_to_le32(1); + fi.DeletePending = 0; + fi.Directory = !!(le32_to_cpu(fi.Attributes) & ATTR_DIRECTORY); + cifs_buf_release(search_info.ntwrk_buf_start); + } else if (!full_path[0]) { + /* + * CIFSFindFirst() does not work on root path if the + * root path was exported on the server from the top + * level path (drive letter). + */ + rc = -EOPNOTSUPP; + } + } + + /* + * If everything failed then fallback to the legacy SMB command + * SMB_COM_QUERY_INFORMATION which works with all servers, but + * provide just few information. + */ + if ((rc == -EOPNOTSUPP || rc == -EINVAL) && !non_unicode_wildcard) { rc = SMBQueryInformation(xid, tcon, full_path, &fi, cifs_sb->local_nls, cifs_remap(cifs_sb)); data->adjust_tz = true; + } else if ((rc == -EOPNOTSUPP || rc == -EINVAL) && non_unicode_wildcard) { + /* Path with non-UNICODE wildcard character cannot exist. */ + rc = -ENOENT; } if (!rc) { @@ -598,14 +671,72 @@ static int cifs_query_path_info(const unsigned int xid, } #ifdef CONFIG_CIFS_XATTR + /* + * For non-symlink WSL reparse points it is required to fetch + * EA $LXMOD which contains in its S_DT part the mandatory file type. + */ + if (!rc && data->reparse_point) { + struct smb2_file_full_ea_info *ea; + u32 next = 0; + + ea = (struct smb2_file_full_ea_info *)data->wsl.eas; + do { + ea = (void *)((u8 *)ea + next); + next = le32_to_cpu(ea->next_entry_offset); + } while (next); + if (le16_to_cpu(ea->ea_value_length)) { + ea->next_entry_offset = cpu_to_le32(ALIGN(sizeof(*ea) + + ea->ea_name_length + 1 + + le16_to_cpu(ea->ea_value_length), 4)); + ea = (void *)((u8 *)ea + le32_to_cpu(ea->next_entry_offset)); + } + + rc = CIFSSMBQAllEAs(xid, tcon, full_path, SMB2_WSL_XATTR_MODE, + &ea->ea_data[SMB2_WSL_XATTR_NAME_LEN + 1], + SMB2_WSL_XATTR_MODE_SIZE, cifs_sb); + if (rc == SMB2_WSL_XATTR_MODE_SIZE) { + ea->next_entry_offset = cpu_to_le32(0); + ea->flags = 0; + ea->ea_name_length = SMB2_WSL_XATTR_NAME_LEN; + ea->ea_value_length = cpu_to_le16(SMB2_WSL_XATTR_MODE_SIZE); + memcpy(&ea->ea_data[0], SMB2_WSL_XATTR_MODE, SMB2_WSL_XATTR_NAME_LEN + 1); + data->wsl.eas_len += ALIGN(sizeof(*ea) + SMB2_WSL_XATTR_NAME_LEN + 1 + + SMB2_WSL_XATTR_MODE_SIZE, 4); + rc = 0; + } else if (rc >= 0) { + /* It is an error if EA $LXMOD has wrong size. */ + rc = -EINVAL; + } else { + /* + * In all other cases ignore error if fetching + * of EA $LXMOD failed. It is needed only for + * non-symlink WSL reparse points and wsl_to_fattr() + * handle the case when EA is missing. + */ + rc = 0; + } + } + /* * For WSL CHR and BLK reparse points it is required to fetch * EA $LXDEV which contains major and minor device numbers. */ if (!rc && data->reparse_point) { struct smb2_file_full_ea_info *ea; + u32 next = 0; ea = (struct smb2_file_full_ea_info *)data->wsl.eas; + do { + ea = (void *)((u8 *)ea + next); + next = le32_to_cpu(ea->next_entry_offset); + } while (next); + if (le16_to_cpu(ea->ea_value_length)) { + ea->next_entry_offset = cpu_to_le32(ALIGN(sizeof(*ea) + + ea->ea_name_length + 1 + + le16_to_cpu(ea->ea_value_length), 4)); + ea = (void *)((u8 *)ea + le32_to_cpu(ea->next_entry_offset)); + } + rc = CIFSSMBQAllEAs(xid, tcon, full_path, SMB2_WSL_XATTR_DEV, &ea->ea_data[SMB2_WSL_XATTR_NAME_LEN + 1], SMB2_WSL_XATTR_DEV_SIZE, cifs_sb); @@ -615,8 +746,8 @@ static int cifs_query_path_info(const unsigned int xid, ea->ea_name_length = SMB2_WSL_XATTR_NAME_LEN; ea->ea_value_length = cpu_to_le16(SMB2_WSL_XATTR_DEV_SIZE); memcpy(&ea->ea_data[0], SMB2_WSL_XATTR_DEV, SMB2_WSL_XATTR_NAME_LEN + 1); - data->wsl.eas_len = sizeof(*ea) + SMB2_WSL_XATTR_NAME_LEN + 1 + - SMB2_WSL_XATTR_DEV_SIZE; + data->wsl.eas_len += ALIGN(sizeof(*ea) + SMB2_WSL_XATTR_NAME_LEN + 1 + + SMB2_WSL_XATTR_MODE_SIZE, 4); rc = 0; } else if (rc >= 0) { /* It is an error if EA $LXDEV has wrong size. */ @@ -662,6 +793,13 @@ static int cifs_query_file_info(const unsigned int xid, struct cifs_tcon *tcon, int rc; FILE_ALL_INFO fi = {}; + /* + * CIFSSMBQFileInfo() for non-NT servers returns bogus data in + * Attributes fields. So do not use this command for non-NT servers. + */ + if (!(tcon->ses->capabilities & CAP_NT_SMBS)) + return -EOPNOTSUPP; + if (cfile->symlink_target) { data->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); if (!data->symlink_target) @@ -832,6 +970,9 @@ smb_set_file_info(struct inode *inode, const char *full_path, struct cifs_fid fid; struct cifs_open_parms oparms; struct cifsFileInfo *open_file; + FILE_BASIC_INFO new_buf; + struct cifs_open_info_data query_data; + __le64 write_time = buf->LastWriteTime; struct cifsInodeInfo *cinode = CIFS_I(inode); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); struct tcon_link *tlink = NULL; @@ -839,20 +980,58 @@ smb_set_file_info(struct inode *inode, const char *full_path, /* if the file is already open for write, just use that fileid */ open_file = find_writable_file(cinode, FIND_WR_FSUID_ONLY); + if (open_file) { fid.netfid = open_file->fid.netfid; netpid = open_file->pid; tcon = tlink_tcon(open_file->tlink); - goto set_via_filehandle; + } else { + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + tlink = NULL; + goto out; + } + tcon = tlink_tcon(tlink); } - tlink = cifs_sb_tlink(cifs_sb); - if (IS_ERR(tlink)) { - rc = PTR_ERR(tlink); - tlink = NULL; - goto out; + /* + * Non-NT servers interprets zero time value in SMB_SET_FILE_BASIC_INFO + * over TRANS2_SET_FILE_INFORMATION as a valid time value. NT servers + * interprets zero time value as do not change existing value on server. + * API of ->set_file_info() callback expects that zero time value has + * the NT meaning - do not change. Therefore if server is non-NT and + * some time values in "buf" are zero, then fetch missing time values. + */ + if (!(tcon->ses->capabilities & CAP_NT_SMBS) && + (!buf->CreationTime || !buf->LastAccessTime || + !buf->LastWriteTime || !buf->ChangeTime)) { + rc = cifs_query_path_info(xid, tcon, cifs_sb, full_path, &query_data); + if (rc) { + if (open_file) { + cifsFileInfo_put(open_file); + open_file = NULL; + } + goto out; + } + /* + * Original write_time from buf->LastWriteTime is preserved + * as SMBSetInformation() interprets zero as do not change. + */ + new_buf = *buf; + buf = &new_buf; + if (!buf->CreationTime) + buf->CreationTime = query_data.fi.CreationTime; + if (!buf->LastAccessTime) + buf->LastAccessTime = query_data.fi.LastAccessTime; + if (!buf->LastWriteTime) + buf->LastWriteTime = query_data.fi.LastWriteTime; + if (!buf->ChangeTime) + buf->ChangeTime = query_data.fi.ChangeTime; } - tcon = tlink_tcon(tlink); + + if (open_file) + goto set_via_filehandle; rc = CIFSSMBSetPathInfo(xid, tcon, full_path, buf, cifs_sb->local_nls, cifs_sb); @@ -873,8 +1052,45 @@ smb_set_file_info(struct inode *inode, const char *full_path, .fid = &fid, }; - cifs_dbg(FYI, "calling SetFileInfo since SetPathInfo for times not supported by this server\n"); - rc = CIFS_open(xid, &oparms, &oplock, NULL); + if (S_ISDIR(inode->i_mode) && !(tcon->ses->capabilities & CAP_NT_SMBS)) { + /* Opening directory path is not possible on non-NT servers. */ + rc = -EOPNOTSUPP; + } else { + /* + * Use cifs_open_file() instead of CIFS_open() as the + * cifs_open_file() selects the correct function which + * works also on non-NT servers. + */ + rc = cifs_open_file(xid, &oparms, &oplock, NULL); + /* + * Opening path for writing on non-NT servers is not + * possible when the read-only attribute is already set. + * Non-NT server in this case returns -EACCES. For those + * servers the only possible way how to clear the read-only + * bit is via SMB_COM_SETATTR command. + */ + if (rc == -EACCES && + (cinode->cifsAttrs & ATTR_READONLY) && + le32_to_cpu(buf->Attributes) != 0 && /* 0 = do not change attrs */ + !(le32_to_cpu(buf->Attributes) & ATTR_READONLY) && + !(tcon->ses->capabilities & CAP_NT_SMBS)) + rc = -EOPNOTSUPP; + } + + /* Fallback to SMB_COM_SETATTR command when absolutelty needed. */ + if (rc == -EOPNOTSUPP) { + cifs_dbg(FYI, "calling SetInformation since SetPathInfo for attrs/times not supported by this server\n"); + rc = SMBSetInformation(xid, tcon, full_path, + buf->Attributes != 0 ? buf->Attributes : cpu_to_le32(cinode->cifsAttrs), + write_time, + cifs_sb->local_nls, cifs_sb); + if (rc == 0) + cinode->cifsAttrs = le32_to_cpu(buf->Attributes); + else + rc = -EACCES; + goto out; + } + if (rc != 0) { if (rc == -EIO) rc = -EINVAL; @@ -882,6 +1098,7 @@ smb_set_file_info(struct inode *inode, const char *full_path, } netpid = current->tgid; + cifs_dbg(FYI, "calling SetFileInfo since SetPathInfo for attrs/times not supported by this server\n"); set_via_filehandle: rc = CIFSSMBSetFileInfo(xid, tcon, buf, fid.netfid, netpid); @@ -892,6 +1109,21 @@ smb_set_file_info(struct inode *inode, const char *full_path, CIFSSMBClose(xid, tcon, fid.netfid); else cifsFileInfo_put(open_file); + + /* + * Setting the read-only bit is not honered on non-NT servers when done + * via open-semantics. So for setting it, use SMB_COM_SETATTR command. + * This command works only after the file is closed, so use it only when + * operation was called without the filehandle. + */ + if (open_file == NULL && + !(tcon->ses->capabilities & CAP_NT_SMBS) && + le32_to_cpu(buf->Attributes) & ATTR_READONLY) { + SMBSetInformation(xid, tcon, full_path, + buf->Attributes, + 0 /* do not change write time */, + cifs_sb->local_nls, cifs_sb); + } out: if (tlink != NULL) cifs_put_tlink(tlink); diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index db9c807115c60..d7f2835e0b1cc 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -107,16 +107,25 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 int err_buftype = CIFS_NO_BUFFER; struct cifs_fid *fid = oparms->fid; struct network_resiliency_req nr_ioctl_req; + bool retry_without_read_attributes = false; smb2_path = cifs_convert_path_to_utf16(oparms->path, oparms->cifs_sb); if (smb2_path == NULL) return -ENOMEM; - oparms->desired_access |= FILE_READ_ATTRIBUTES; + if (!(oparms->desired_access & FILE_READ_ATTRIBUTES)) { + oparms->desired_access |= FILE_READ_ATTRIBUTES; + retry_without_read_attributes = true; + } smb2_oplock = SMB2_OPLOCK_LEVEL_BATCH; rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, &err_buftype); + if (rc == -EACCES && retry_without_read_attributes) { + oparms->desired_access &= ~FILE_READ_ATTRIBUTES; + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, + &err_buftype); + } if (rc && data) { struct smb2_hdr *hdr = err_iov.iov_base; diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index e1078a1decdfa..d6086394d0b84 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -206,8 +206,10 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, server = cifs_pick_channel(ses); vars = kzalloc(sizeof(*vars), GFP_ATOMIC); - if (vars == NULL) - return -ENOMEM; + if (vars == NULL) { + rc = -ENOMEM; + goto out; + } rqst = &vars->rqst[0]; rsp_iov = &vars->rsp_iov[0]; @@ -639,7 +641,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, tmp_rc = rc; for (i = 0; i < num_cmds; i++) { - char *buf = rsp_iov[i + i].iov_base; + char *buf = rsp_iov[i + 1].iov_base; if (buf && resp_buftype[i + 1] != CIFS_NO_BUFFER) rc = server->ops->map_error(buf, false); @@ -828,6 +830,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, smb2_should_replay(tcon, &retries, &cur_sleep)) goto replay_again; +out: if (cfile) cifsFileInfo_put(cfile); @@ -1125,6 +1128,8 @@ static int smb2_set_path_attr(const unsigned int xid, struct cifs_tcon *tcon, smb2_to_name = cifs_convert_path_to_utf16(to_name, cifs_sb); if (smb2_to_name == NULL) { rc = -ENOMEM; + if (cfile) + cifsFileInfo_put(cfile); goto smb2_rename_path; } in_iov.iov_base = smb2_to_name; @@ -1213,31 +1218,33 @@ int smb2_set_file_info(struct inode *inode, const char *full_path, FILE_BASIC_INFO *buf, const unsigned int xid) { - struct cifs_open_parms oparms; + struct kvec in_iov = { .iov_base = buf, .iov_len = sizeof(*buf), }; struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifsFileInfo *cfile = NULL; + struct cifs_open_parms oparms; struct tcon_link *tlink; struct cifs_tcon *tcon; - struct cifsFileInfo *cfile; - struct kvec in_iov = { .iov_base = buf, .iov_len = sizeof(*buf), }; - int rc; - - if ((buf->CreationTime == 0) && (buf->LastAccessTime == 0) && - (buf->LastWriteTime == 0) && (buf->ChangeTime == 0) && - (buf->Attributes == 0)) - return 0; /* would be a no op, no sense sending this */ + int rc = 0; tlink = cifs_sb_tlink(cifs_sb); if (IS_ERR(tlink)) return PTR_ERR(tlink); tcon = tlink_tcon(tlink); - cifs_get_writable_path(tcon, full_path, FIND_WR_ANY, &cfile); + if ((buf->CreationTime == 0) && (buf->LastAccessTime == 0) && + (buf->LastWriteTime == 0) && (buf->ChangeTime == 0)) { + if (buf->Attributes == 0) + goto out; /* would be a no op, no sense sending this */ + cifs_get_writable_path(tcon, full_path, FIND_WR_ANY, &cfile); + } + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, FILE_WRITE_ATTRIBUTES, FILE_OPEN, 0, ACL_NO_MODE); rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, &oparms, &in_iov, &(int){SMB2_OP_SET_INFO}, 1, cfile, NULL, NULL, NULL); +out: cifs_put_tlink(tlink); return rc; } diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index b809a616728f2..138b3ed08217c 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -428,12 +428,20 @@ smb2_negotiate(const unsigned int xid, server->CurrentMid = 0; spin_unlock(&server->mid_lock); rc = SMB2_negotiate(xid, ses, server); - /* BB we probably don't need to retry with modern servers */ - if (rc == -EAGAIN) - rc = -EHOSTDOWN; return rc; } +static inline unsigned int +prevent_zero_iosize(unsigned int size, const char *type) +{ + if (size == 0) { + cifs_dbg(VFS, "SMB: Zero %ssize calculated, using minimum value %u\n", + type, CIFS_MIN_DEFAULT_IOSIZE); + return CIFS_MIN_DEFAULT_IOSIZE; + } + return size; +} + static unsigned int smb2_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) { @@ -441,12 +449,12 @@ smb2_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) unsigned int wsize; /* start with specified wsize, or default */ - wsize = ctx->wsize ? ctx->wsize : CIFS_DEFAULT_IOSIZE; + wsize = ctx->got_wsize ? ctx->vol_wsize : CIFS_DEFAULT_IOSIZE; wsize = min_t(unsigned int, wsize, server->max_write); if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE); - return wsize; + return prevent_zero_iosize(wsize, "w"); } static unsigned int @@ -456,7 +464,7 @@ smb3_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) unsigned int wsize; /* start with specified wsize, or default */ - wsize = ctx->wsize ? ctx->wsize : SMB3_DEFAULT_IOSIZE; + wsize = ctx->got_wsize ? ctx->vol_wsize : SMB3_DEFAULT_IOSIZE; wsize = min_t(unsigned int, wsize, server->max_write); #ifdef CONFIG_CIFS_SMB_DIRECT if (server->rdma) { @@ -478,7 +486,7 @@ smb3_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE); - return wsize; + return prevent_zero_iosize(wsize, "w"); } static unsigned int @@ -488,13 +496,13 @@ smb2_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) unsigned int rsize; /* start with specified rsize, or default */ - rsize = ctx->rsize ? ctx->rsize : CIFS_DEFAULT_IOSIZE; + rsize = ctx->got_rsize ? ctx->vol_rsize : CIFS_DEFAULT_IOSIZE; rsize = min_t(unsigned int, rsize, server->max_read); if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) rsize = min_t(unsigned int, rsize, SMB2_MAX_BUFFER_SIZE); - return rsize; + return prevent_zero_iosize(rsize, "r"); } static unsigned int @@ -504,7 +512,7 @@ smb3_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) unsigned int rsize; /* start with specified rsize, or default */ - rsize = ctx->rsize ? ctx->rsize : SMB3_DEFAULT_IOSIZE; + rsize = ctx->got_rsize ? ctx->vol_rsize : SMB3_DEFAULT_IOSIZE; rsize = min_t(unsigned int, rsize, server->max_read); #ifdef CONFIG_CIFS_SMB_DIRECT if (server->rdma) { @@ -527,7 +535,7 @@ smb3_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) rsize = min_t(unsigned int, rsize, SMB2_MAX_BUFFER_SIZE); - return rsize; + return prevent_zero_iosize(rsize, "r"); } /* @@ -722,6 +730,13 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, bytes_left -= sizeof(*p); break; } + /* Validate that Next doesn't point beyond the buffer */ + if (next > bytes_left) { + cifs_dbg(VFS, "%s: invalid Next pointer %zu > %zd\n", + __func__, next, bytes_left); + rc = -EINVAL; + goto out; + } p = (struct network_interface_info_ioctl_rsp *)((u8 *)p+next); bytes_left -= next; } @@ -733,7 +748,9 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, } /* Azure rounds the buffer size up 8, to a 16 byte boundary */ - if ((bytes_left > 8) || p->Next) + if ((bytes_left > 8) || + (bytes_left >= offsetof(struct network_interface_info_ioctl_rsp, Next) + + sizeof(p->Next) && p->Next)) cifs_dbg(VFS, "%s: incomplete interface info\n", __func__); ses->iface_last_update = jiffies; @@ -2124,7 +2141,7 @@ smb3_enum_snapshots(const unsigned int xid, struct cifs_tcon *tcon, NULL, 0 /* no input data */, max_response_size, (char **)&retbuf, &ret_data_len); - cifs_dbg(FYI, "enum snaphots ioctl returned %d and ret buflen is %d\n", + cifs_dbg(FYI, "enum snapshots ioctl returned %d and ret buflen is %d\n", rc, ret_data_len); if (rc) return rc; @@ -2646,11 +2663,12 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_fid fid; int rc; __le16 *utf16_path; - struct cached_fid *cfid = NULL; + struct cached_fid *cfid; int retries = 0, cur_sleep = 1; replay_again: /* reinitialize for possible replay */ + cfid = NULL; flags = CIFS_CP_CREATE_CLOSE_OP; oplock = SMB2_OPLOCK_LEVEL_NONE; server = cifs_pick_channel(ses); @@ -3055,8 +3073,7 @@ get_smb2_acl_by_path(struct cifs_sb_info *cifs_sb, utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); if (!utf16_path) { rc = -ENOMEM; - free_xid(xid); - return ERR_PTR(rc); + goto put_tlink; } oparms = (struct cifs_open_parms) { @@ -3088,6 +3105,7 @@ get_smb2_acl_by_path(struct cifs_sb_info *cifs_sb, SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); } +put_tlink: cifs_put_tlink(tlink); free_xid(xid); @@ -3128,8 +3146,7 @@ set_smb2_acl(struct smb_ntsd *pnntsd, __u32 acllen, utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); if (!utf16_path) { rc = -ENOMEM; - free_xid(xid); - return rc; + goto put_tlink; } oparms = (struct cifs_open_parms) { @@ -3150,6 +3167,7 @@ set_smb2_acl(struct smb_ntsd *pnntsd, __u32 acllen, SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); } +put_tlink: cifs_put_tlink(tlink); free_xid(xid); return rc; @@ -3532,7 +3550,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, /* * At this point, we are trying to fallocate an internal * regions of a sparse file. Since smb2 does not have a - * fallocate command we have two otions on how to emulate this. + * fallocate command we have two options on how to emulate this. * We can either turn the entire file to become non-sparse * which we only do if the fallocate is for virtually * the whole file, or we can overwrite the region with zeroes @@ -4135,7 +4153,7 @@ fill_transform_hdr(struct smb2_transform_hdr *tr_hdr, unsigned int orig_len, static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst *rqst, int num_rqst, const u8 *sig, u8 **iv, struct aead_request **req, struct sg_table *sgt, - unsigned int *num_sgs, size_t *sensitive_size) + unsigned int *num_sgs) { unsigned int req_size = sizeof(**req) + crypto_aead_reqsize(tfm); unsigned int iv_size = crypto_aead_ivsize(tfm); @@ -4152,9 +4170,8 @@ static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst len += req_size; len = ALIGN(len, __alignof__(struct scatterlist)); len += array_size(*num_sgs, sizeof(struct scatterlist)); - *sensitive_size = len; - p = kvzalloc(len, GFP_NOFS); + p = kzalloc(len, GFP_NOFS); if (!p) return ERR_PTR(-ENOMEM); @@ -4168,16 +4185,14 @@ static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst static void *smb2_get_aead_req(struct crypto_aead *tfm, struct smb_rqst *rqst, int num_rqst, const u8 *sig, u8 **iv, - struct aead_request **req, struct scatterlist **sgl, - size_t *sensitive_size) + struct aead_request **req, struct scatterlist **sgl) { struct sg_table sgtable = {}; unsigned int skip, num_sgs, i, j; ssize_t rc; void *p; - p = smb2_aead_req_alloc(tfm, rqst, num_rqst, sig, iv, req, &sgtable, - &num_sgs, sensitive_size); + p = smb2_aead_req_alloc(tfm, rqst, num_rqst, sig, iv, req, &sgtable, &num_sgs); if (IS_ERR(p)) return ERR_CAST(p); @@ -4263,9 +4278,9 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, u8 key[SMB3_ENC_DEC_KEY_SIZE]; struct aead_request *req; u8 *iv; + DECLARE_CRYPTO_WAIT(wait); unsigned int crypt_len = le32_to_cpu(tr_hdr->OriginalMessageSize); void *creq; - size_t sensitive_size; rc = smb2_get_enc_key(server, le64_to_cpu(tr_hdr->SessionId), enc, key); if (rc) { @@ -4291,8 +4306,7 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, return rc; } - creq = smb2_get_aead_req(tfm, rqst, num_rqst, sign, &iv, &req, &sg, - &sensitive_size); + creq = smb2_get_aead_req(tfm, rqst, num_rqst, sign, &iv, &req, &sg); if (IS_ERR(creq)) return PTR_ERR(creq); @@ -4313,12 +4327,16 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, aead_request_set_crypt(req, sg, sg, crypt_len, iv); aead_request_set_ad(req, assoc_data_len); - rc = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + rc = crypto_wait_req(enc ? crypto_aead_encrypt(req) + : crypto_aead_decrypt(req), &wait); if (!rc && enc) memcpy(&tr_hdr->Signature, sign, SMB2_SIGNATURE_SIZE); - kvfree_sensitive(creq, sensitive_size); + kfree_sensitive(creq); return rc; } diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 0af3535e08f30..a8890ae217144 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -425,14 +425,23 @@ smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon, if (!rc && (server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL) && server->ops->query_server_interfaces) { - mutex_unlock(&ses->session_mutex); - /* - * query server network interfaces, in case they change + * query server network interfaces, in case they change. + * Also mark the session as pending this update while the query + * is in progress. This will be used to avoid calling + * smb2_reconnect recursively. */ + ses->flags |= CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES; xid = get_xid(); rc = server->ops->query_server_interfaces(xid, tcon, false); free_xid(xid); + ses->flags &= ~CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES; + + if (!tcon->ipc && !tcon->dummy) + queue_delayed_work(cifsiod_wq, &tcon->query_interfaces, + (SMB_INTERFACE_POLL_INTERVAL * HZ)); + + mutex_unlock(&ses->session_mutex); if (rc == -EOPNOTSUPP && ses->chan_count > 1) { /* @@ -452,11 +461,8 @@ smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon, if (ses->chan_max > ses->chan_count && ses->iface_count && !SERVER_IS_CHAN(server)) { - if (ses->chan_count == 1) { + if (ses->chan_count == 1) cifs_server_dbg(VFS, "supports multichannel now\n"); - queue_delayed_work(cifsiod_wq, &tcon->query_interfaces, - (SMB_INTERFACE_POLL_INTERVAL * HZ)); - } cifs_try_adding_channels(ses); } @@ -574,11 +580,18 @@ static int smb2_ioctl_req_init(u32 opcode, struct cifs_tcon *tcon, struct TCP_Server_Info *server, void **request_buf, unsigned int *total_len) { - /* Skip reconnect only for FSCTL_VALIDATE_NEGOTIATE_INFO IOCTLs */ - if (opcode == FSCTL_VALIDATE_NEGOTIATE_INFO) { + /* + * Skip reconnect in one of the following cases: + * 1. For FSCTL_VALIDATE_NEGOTIATE_INFO IOCTLs + * 2. For FSCTL_QUERY_NETWORK_INTERFACE_INFO IOCTL when called from + * smb2_reconnect (indicated by CIFS_SES_FLAG_SCALE_CHANNELS ses flag) + */ + if (opcode == FSCTL_VALIDATE_NEGOTIATE_INFO || + (opcode == FSCTL_QUERY_NETWORK_INTERFACE_INFO && + (tcon->ses->flags & CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES))) return __smb2_plain_req_init(SMB2_IOCTL, tcon, server, request_buf, total_len); - } + return smb2_plain_req_init(SMB2_IOCTL, tcon, server, request_buf, total_len); } @@ -2932,6 +2945,7 @@ int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, req->CreateContextsOffset = cpu_to_le32( sizeof(struct smb2_create_req) + iov[1].iov_len); + le32_add_cpu(&req->CreateContextsLength, iov[n_iov-1].iov_len); pc_buf = iov[n_iov-1].iov_base; } @@ -2975,10 +2989,10 @@ int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, SMB2_close(xid, tcon, rsp->PersistentFileId, rsp->VolatileFileId); - /* Eventually save off posix specific response info and timestaps */ + /* Eventually save off posix specific response info and timestamps */ err_free_rsp_buf: - free_rsp_buf(resp_buftype, rsp); + free_rsp_buf(resp_buftype, rsp_iov.iov_base); kfree(pc_buf); err_free_req: cifs_small_buf_release(req); @@ -4054,9 +4068,12 @@ SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, smb_rsp = (struct smb2_change_notify_rsp *)rsp_iov.iov_base; - smb2_validate_iov(le16_to_cpu(smb_rsp->OutputBufferOffset), - le32_to_cpu(smb_rsp->OutputBufferLength), &rsp_iov, + rc = smb2_validate_iov(le16_to_cpu(smb_rsp->OutputBufferOffset), + le32_to_cpu(smb_rsp->OutputBufferLength), + &rsp_iov, sizeof(struct file_notify_information)); + if (rc) + goto cnotify_exit; *out_data = kmemdup((char *)smb_rsp + le16_to_cpu(smb_rsp->OutputBufferOffset), le32_to_cpu(smb_rsp->OutputBufferLength), GFP_KERNEL); @@ -4214,10 +4231,8 @@ void smb2_reconnect_server(struct work_struct *work) } goto done; } - tcon->status = TID_GOOD; - tcon->retry = false; - tcon->need_reconnect = false; + tcon->dummy = true; /* now reconnect sessions for necessary channels */ list_for_each_entry_safe(ses, ses2, &tmp_ses_list, rlist) { @@ -4562,7 +4577,7 @@ smb2_readv_callback(struct mid_q_entry *mid) } #ifdef CONFIG_CIFS_SMB_DIRECT /* - * If this rdata has a memmory registered, the MR can be freed + * If this rdata has a memory registered, the MR can be freed * MR needs to be freed as soon as I/O finishes to prevent deadlock * because they have limited number and are used for future I/Os */ diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c index 4a43802375b3a..99081e9d6283a 100644 --- a/fs/smb/client/smb2transport.c +++ b/fs/smb/client/smb2transport.c @@ -720,7 +720,7 @@ smb2_verify_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server) shdr->Command); /* - * Save off the origiginal signature so we can modify the smb and check + * Save off the original signature so we can modify the smb and check * our calculated signature against what the server sent. */ memcpy(server_response_sig, shdr->Signature, SMB2_SIGNATURE_SIZE); diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index d74e829de51c2..be9be8f363319 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -164,10 +164,11 @@ static void smbd_disconnect_rdma_work(struct work_struct *work) { struct smbd_connection *info = container_of(work, struct smbd_connection, disconnect_work); + struct smbdirect_socket *sc = &info->socket; - if (info->transport_status == SMBD_CONNECTED) { - info->transport_status = SMBD_DISCONNECTING; - rdma_disconnect(info->id); + if (sc->status == SMBDIRECT_SOCKET_CONNECTED) { + sc->status = SMBDIRECT_SOCKET_DISCONNECTING; + rdma_disconnect(sc->rdma.cm_id); } } @@ -181,6 +182,7 @@ static int smbd_conn_upcall( struct rdma_cm_id *id, struct rdma_cm_event *event) { struct smbd_connection *info = id->context; + struct smbdirect_socket *sc = &info->socket; log_rdma_event(INFO, "event=%d status=%d\n", event->event, event->status); @@ -204,7 +206,7 @@ static int smbd_conn_upcall( case RDMA_CM_EVENT_ESTABLISHED: log_rdma_event(INFO, "connected event=%d\n", event->event); - info->transport_status = SMBD_CONNECTED; + sc->status = SMBDIRECT_SOCKET_CONNECTED; wake_up_interruptible(&info->conn_wait); break; @@ -212,20 +214,20 @@ static int smbd_conn_upcall( case RDMA_CM_EVENT_UNREACHABLE: case RDMA_CM_EVENT_REJECTED: log_rdma_event(INFO, "connecting failed event=%d\n", event->event); - info->transport_status = SMBD_DISCONNECTED; + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; wake_up_interruptible(&info->conn_wait); break; case RDMA_CM_EVENT_DEVICE_REMOVAL: case RDMA_CM_EVENT_DISCONNECTED: - /* This happenes when we fail the negotiation */ - if (info->transport_status == SMBD_NEGOTIATE_FAILED) { - info->transport_status = SMBD_DISCONNECTED; + /* This happens when we fail the negotiation */ + if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_FAILED) { + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; wake_up(&info->conn_wait); break; } - info->transport_status = SMBD_DISCONNECTED; + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; wake_up_interruptible(&info->disconn_wait); wake_up_interruptible(&info->wait_reassembly_queue); wake_up_interruptible_all(&info->wait_send_queue); @@ -274,22 +276,26 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) int i; struct smbd_request *request = container_of(wc->wr_cqe, struct smbd_request, cqe); + struct smbd_connection *info = request->info; + struct smbdirect_socket *sc = &info->socket; log_rdma_send(INFO, "smbd_request 0x%p completed wc->status=%d\n", request, wc->status); - if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { - log_rdma_send(ERR, "wc->status=%d wc->opcode=%d\n", - wc->status, wc->opcode); - smbd_disconnect_rdma_connection(request->info); - } - for (i = 0; i < request->num_sge; i++) - ib_dma_unmap_single(request->info->id->device, + ib_dma_unmap_single(sc->ib.dev, request->sge[i].addr, request->sge[i].length, DMA_TO_DEVICE); + if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { + log_rdma_send(ERR, "wc->status=%d wc->opcode=%d\n", + wc->status, wc->opcode); + mempool_free(request, info->request_mempool); + smbd_disconnect_rdma_connection(info); + return; + } + if (atomic_dec_and_test(&request->info->send_pending)) wake_up(&request->info->wait_send_pending); @@ -392,8 +398,9 @@ static void smbd_post_send_credits(struct work_struct *work) struct smbd_connection *info = container_of(work, struct smbd_connection, post_send_credits_work); + struct smbdirect_socket *sc = &info->socket; - if (info->transport_status != SMBD_CONNECTED) { + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { wake_up(&info->wait_receive_queues); return; } @@ -460,7 +467,6 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { log_rdma_recv(INFO, "wc->status=%d opcode=%d\n", wc->status, wc->opcode); - smbd_disconnect_rdma_connection(info); goto error; } @@ -477,8 +483,9 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) info->full_packet_received = true; info->negotiate_done = process_negotiation_response(response, wc->byte_len); + put_receive_buffer(info, response); complete(&info->negotiate_completion); - break; + return; /* SMBD data transfer packet */ case SMBD_TRANSFER_DATA: @@ -535,14 +542,16 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } return; - - default: - log_rdma_recv(ERR, - "unexpected response type=%d\n", response->type); } + /* + * This is an internal error! + */ + log_rdma_recv(ERR, "unexpected response type=%d\n", response->type); + WARN_ON_ONCE(response->type != SMBD_TRANSFER_DATA); error: put_receive_buffer(info, response); + smbd_disconnect_rdma_connection(info); } static struct rdma_cm_id *smbd_create_id( @@ -634,32 +643,34 @@ static int smbd_ia_open( struct smbd_connection *info, struct sockaddr *dstaddr, int port) { + struct smbdirect_socket *sc = &info->socket; int rc; - info->id = smbd_create_id(info, dstaddr, port); - if (IS_ERR(info->id)) { - rc = PTR_ERR(info->id); + sc->rdma.cm_id = smbd_create_id(info, dstaddr, port); + if (IS_ERR(sc->rdma.cm_id)) { + rc = PTR_ERR(sc->rdma.cm_id); goto out1; } + sc->ib.dev = sc->rdma.cm_id->device; - if (!frwr_is_supported(&info->id->device->attrs)) { + if (!frwr_is_supported(&sc->ib.dev->attrs)) { log_rdma_event(ERR, "Fast Registration Work Requests (FRWR) is not supported\n"); log_rdma_event(ERR, "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", - info->id->device->attrs.device_cap_flags, - info->id->device->attrs.max_fast_reg_page_list_len); + sc->ib.dev->attrs.device_cap_flags, + sc->ib.dev->attrs.max_fast_reg_page_list_len); rc = -EPROTONOSUPPORT; goto out2; } info->max_frmr_depth = min_t(int, smbd_max_frmr_depth, - info->id->device->attrs.max_fast_reg_page_list_len); + sc->ib.dev->attrs.max_fast_reg_page_list_len); info->mr_type = IB_MR_TYPE_MEM_REG; - if (info->id->device->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) + if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) info->mr_type = IB_MR_TYPE_SG_GAPS; - info->pd = ib_alloc_pd(info->id->device, 0); - if (IS_ERR(info->pd)) { - rc = PTR_ERR(info->pd); + sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); + if (IS_ERR(sc->ib.pd)) { + rc = PTR_ERR(sc->ib.pd); log_rdma_event(ERR, "ib_alloc_pd() returned %d\n", rc); goto out2; } @@ -667,8 +678,8 @@ static int smbd_ia_open( return 0; out2: - rdma_destroy_id(info->id); - info->id = NULL; + rdma_destroy_id(sc->rdma.cm_id); + sc->rdma.cm_id = NULL; out1: return rc; @@ -682,6 +693,7 @@ static int smbd_ia_open( */ static int smbd_post_send_negotiate_req(struct smbd_connection *info) { + struct smbdirect_socket *sc = &info->socket; struct ib_send_wr send_wr; int rc = -ENOMEM; struct smbd_request *request; @@ -705,18 +717,18 @@ static int smbd_post_send_negotiate_req(struct smbd_connection *info) request->num_sge = 1; request->sge[0].addr = ib_dma_map_single( - info->id->device, (void *)packet, + sc->ib.dev, (void *)packet, sizeof(*packet), DMA_TO_DEVICE); - if (ib_dma_mapping_error(info->id->device, request->sge[0].addr)) { + if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) { rc = -EIO; goto dma_mapping_failed; } request->sge[0].length = sizeof(*packet); - request->sge[0].lkey = info->pd->local_dma_lkey; + request->sge[0].lkey = sc->ib.pd->local_dma_lkey; ib_dma_sync_single_for_device( - info->id->device, request->sge[0].addr, + sc->ib.dev, request->sge[0].addr, request->sge[0].length, DMA_TO_DEVICE); request->cqe.done = send_done; @@ -733,14 +745,14 @@ static int smbd_post_send_negotiate_req(struct smbd_connection *info) request->sge[0].length, request->sge[0].lkey); atomic_inc(&info->send_pending); - rc = ib_post_send(info->id->qp, &send_wr, NULL); + rc = ib_post_send(sc->ib.qp, &send_wr, NULL); if (!rc) return 0; /* if we reach here, post send failed */ log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); atomic_dec(&info->send_pending); - ib_dma_unmap_single(info->id->device, request->sge[0].addr, + ib_dma_unmap_single(sc->ib.dev, request->sge[0].addr, request->sge[0].length, DMA_TO_DEVICE); smbd_disconnect_rdma_connection(info); @@ -792,6 +804,7 @@ static int manage_keep_alive_before_sending(struct smbd_connection *info) static int smbd_post_send(struct smbd_connection *info, struct smbd_request *request) { + struct smbdirect_socket *sc = &info->socket; struct ib_send_wr send_wr; int rc, i; @@ -800,7 +813,7 @@ static int smbd_post_send(struct smbd_connection *info, "rdma_request sge[%d] addr=0x%llx length=%u\n", i, request->sge[i].addr, request->sge[i].length); ib_dma_sync_single_for_device( - info->id->device, + sc->ib.dev, request->sge[i].addr, request->sge[i].length, DMA_TO_DEVICE); @@ -815,7 +828,7 @@ static int smbd_post_send(struct smbd_connection *info, send_wr.opcode = IB_WR_SEND; send_wr.send_flags = IB_SEND_SIGNALED; - rc = ib_post_send(info->id->qp, &send_wr, NULL); + rc = ib_post_send(sc->ib.qp, &send_wr, NULL); if (rc) { log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); smbd_disconnect_rdma_connection(info); @@ -832,6 +845,7 @@ static int smbd_post_send_iter(struct smbd_connection *info, struct iov_iter *iter, int *_remaining_data_length) { + struct smbdirect_socket *sc = &info->socket; int i, rc; int header_length; int data_length; @@ -843,11 +857,11 @@ static int smbd_post_send_iter(struct smbd_connection *info, /* Wait for send credits. A SMBD packet needs one credit */ rc = wait_event_interruptible(info->wait_send_queue, atomic_read(&info->send_credits) > 0 || - info->transport_status != SMBD_CONNECTED); + sc->status != SMBDIRECT_SOCKET_CONNECTED); if (rc) goto err_wait_credit; - if (info->transport_status != SMBD_CONNECTED) { + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { log_outgoing(ERR, "disconnected not sending on wait_credit\n"); rc = -EAGAIN; goto err_wait_credit; @@ -860,9 +874,9 @@ static int smbd_post_send_iter(struct smbd_connection *info, wait_send_queue: wait_event(info->wait_post_send, atomic_read(&info->send_pending) < info->send_credit_target || - info->transport_status != SMBD_CONNECTED); + sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (info->transport_status != SMBD_CONNECTED) { + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { log_outgoing(ERR, "disconnected not sending on wait_send_queue\n"); rc = -EAGAIN; goto err_wait_send_queue; @@ -889,8 +903,8 @@ static int smbd_post_send_iter(struct smbd_connection *info, .nr_sge = 1, .max_sge = SMBDIRECT_MAX_SEND_SGE, .sge = request->sge, - .device = info->id->device, - .local_dma_lkey = info->pd->local_dma_lkey, + .device = sc->ib.dev, + .local_dma_lkey = sc->ib.pd->local_dma_lkey, .direction = DMA_TO_DEVICE, }; @@ -942,18 +956,18 @@ static int smbd_post_send_iter(struct smbd_connection *info, if (!data_length) header_length = offsetof(struct smbd_data_transfer, padding); - request->sge[0].addr = ib_dma_map_single(info->id->device, + request->sge[0].addr = ib_dma_map_single(sc->ib.dev, (void *)packet, header_length, DMA_TO_DEVICE); - if (ib_dma_mapping_error(info->id->device, request->sge[0].addr)) { + if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) { rc = -EIO; request->sge[0].addr = 0; goto err_dma; } request->sge[0].length = header_length; - request->sge[0].lkey = info->pd->local_dma_lkey; + request->sge[0].lkey = sc->ib.pd->local_dma_lkey; rc = smbd_post_send(info, request); if (!rc) @@ -962,7 +976,7 @@ static int smbd_post_send_iter(struct smbd_connection *info, err_dma: for (i = 0; i < request->num_sge; i++) if (request->sge[i].addr) - ib_dma_unmap_single(info->id->device, + ib_dma_unmap_single(sc->ib.dev, request->sge[i].addr, request->sge[i].length, DMA_TO_DEVICE); @@ -1007,17 +1021,18 @@ static int smbd_post_send_empty(struct smbd_connection *info) static int smbd_post_recv( struct smbd_connection *info, struct smbd_response *response) { + struct smbdirect_socket *sc = &info->socket; struct ib_recv_wr recv_wr; int rc = -EIO; response->sge.addr = ib_dma_map_single( - info->id->device, response->packet, + sc->ib.dev, response->packet, info->max_receive_size, DMA_FROM_DEVICE); - if (ib_dma_mapping_error(info->id->device, response->sge.addr)) + if (ib_dma_mapping_error(sc->ib.dev, response->sge.addr)) return rc; response->sge.length = info->max_receive_size; - response->sge.lkey = info->pd->local_dma_lkey; + response->sge.lkey = sc->ib.pd->local_dma_lkey; response->cqe.done = recv_done; @@ -1026,10 +1041,11 @@ static int smbd_post_recv( recv_wr.sg_list = &response->sge; recv_wr.num_sge = 1; - rc = ib_post_recv(info->id->qp, &recv_wr, NULL); + rc = ib_post_recv(sc->ib.qp, &recv_wr, NULL); if (rc) { - ib_dma_unmap_single(info->id->device, response->sge.addr, + ib_dma_unmap_single(sc->ib.dev, response->sge.addr, response->sge.length, DMA_FROM_DEVICE); + response->sge.length = 0; smbd_disconnect_rdma_connection(info); log_rdma_recv(ERR, "ib_post_recv failed rc=%d\n", rc); } @@ -1048,8 +1064,10 @@ static int smbd_negotiate(struct smbd_connection *info) log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=0x%llx iov.length=%u iov.lkey=0x%x\n", rc, response->sge.addr, response->sge.length, response->sge.lkey); - if (rc) + if (rc) { + put_receive_buffer(info, response); return rc; + } init_completion(&info->negotiate_completion); info->negotiate_done = false; @@ -1186,10 +1204,16 @@ static struct smbd_response *get_receive_buffer(struct smbd_connection *info) static void put_receive_buffer( struct smbd_connection *info, struct smbd_response *response) { + struct smbdirect_socket *sc = &info->socket; unsigned long flags; - ib_dma_unmap_single(info->id->device, response->sge.addr, - response->sge.length, DMA_FROM_DEVICE); + if (likely(response->sge.length != 0)) { + ib_dma_unmap_single(sc->ib.dev, + response->sge.addr, + response->sge.length, + DMA_FROM_DEVICE); + response->sge.length = 0; + } spin_lock_irqsave(&info->receive_queue_lock, flags); list_add_tail(&response->list, &info->receive_queue); @@ -1227,6 +1251,7 @@ static int allocate_receive_buffers(struct smbd_connection *info, int num_buf) goto allocate_failed; response->info = info; + response->sge.length = 0; list_add_tail(&response->list, &info->receive_queue); info->count_receive_queue++; } @@ -1288,6 +1313,7 @@ static void idle_connection_timer(struct work_struct *work) void smbd_destroy(struct TCP_Server_Info *server) { struct smbd_connection *info = server->smbd_conn; + struct smbdirect_socket *sc; struct smbd_response *response; unsigned long flags; @@ -1295,27 +1321,25 @@ void smbd_destroy(struct TCP_Server_Info *server) log_rdma_event(INFO, "rdma session already destroyed\n"); return; } + sc = &info->socket; log_rdma_event(INFO, "destroying rdma session\n"); - if (info->transport_status != SMBD_DISCONNECTED) { - rdma_disconnect(server->smbd_conn->id); + if (sc->status != SMBDIRECT_SOCKET_DISCONNECTED) { + rdma_disconnect(sc->rdma.cm_id); log_rdma_event(INFO, "wait for transport being disconnected\n"); wait_event_interruptible( info->disconn_wait, - info->transport_status == SMBD_DISCONNECTED); + sc->status == SMBDIRECT_SOCKET_DISCONNECTED); } log_rdma_event(INFO, "destroying qp\n"); - ib_drain_qp(info->id->qp); - rdma_destroy_qp(info->id); + ib_drain_qp(sc->ib.qp); + rdma_destroy_qp(sc->rdma.cm_id); + sc->ib.qp = NULL; log_rdma_event(INFO, "cancelling idle timer\n"); cancel_delayed_work_sync(&info->idle_timer_work); - log_rdma_event(INFO, "wait for all send posted to IB to finish\n"); - wait_event(info->wait_send_pending, - atomic_read(&info->send_pending) == 0); - /* It's not possible for upper layer to get to reassembly */ log_rdma_event(INFO, "drain the reassembly queue\n"); do { @@ -1343,7 +1367,7 @@ void smbd_destroy(struct TCP_Server_Info *server) * are not locked by srv_mutex. It is possible some processes are * blocked on transport srv_mutex while holding memory registration. * Release the transport srv_mutex to allow them to hit the failure - * path when sending data, and then release memory registartions. + * path when sending data, and then release memory registrations. */ log_rdma_event(INFO, "freeing mr list\n"); wake_up_interruptible_all(&info->wait_mr); @@ -1354,10 +1378,10 @@ void smbd_destroy(struct TCP_Server_Info *server) } destroy_mr_list(info); - ib_free_cq(info->send_cq); - ib_free_cq(info->recv_cq); - ib_dealloc_pd(info->pd); - rdma_destroy_id(info->id); + ib_free_cq(sc->ib.send_cq); + ib_free_cq(sc->ib.recv_cq); + ib_dealloc_pd(sc->ib.pd); + rdma_destroy_id(sc->rdma.cm_id); /* free mempools */ mempool_destroy(info->request_mempool); @@ -1366,7 +1390,7 @@ void smbd_destroy(struct TCP_Server_Info *server) mempool_destroy(info->response_mempool); kmem_cache_destroy(info->response_cache); - info->transport_status = SMBD_DESTROYED; + sc->status = SMBDIRECT_SOCKET_DESTROYED; destroy_workqueue(info->workqueue); log_rdma_event(INFO, "rdma session destroyed\n"); @@ -1391,7 +1415,7 @@ int smbd_reconnect(struct TCP_Server_Info *server) * This is possible if transport is disconnected and we haven't received * notification from RDMA, but upper layer has detected timeout */ - if (server->smbd_conn->transport_status == SMBD_CONNECTED) { + if (server->smbd_conn->socket.status == SMBDIRECT_SOCKET_CONNECTED) { log_rdma_event(INFO, "disconnecting transport\n"); smbd_destroy(server); } @@ -1490,6 +1514,7 @@ static struct smbd_connection *_smbd_get_connection( { int rc; struct smbd_connection *info; + struct smbdirect_socket *sc; struct rdma_conn_param conn_param; struct ib_qp_init_attr qp_attr; struct sockaddr_in *addr_in = (struct sockaddr_in *) dstaddr; @@ -1499,29 +1524,30 @@ static struct smbd_connection *_smbd_get_connection( info = kzalloc(sizeof(struct smbd_connection), GFP_KERNEL); if (!info) return NULL; + sc = &info->socket; - info->transport_status = SMBD_CONNECTING; + sc->status = SMBDIRECT_SOCKET_CONNECTING; rc = smbd_ia_open(info, dstaddr, port); if (rc) { log_rdma_event(INFO, "smbd_ia_open rc=%d\n", rc); goto create_id_failed; } - if (smbd_send_credit_target > info->id->device->attrs.max_cqe || - smbd_send_credit_target > info->id->device->attrs.max_qp_wr) { + if (smbd_send_credit_target > sc->ib.dev->attrs.max_cqe || + smbd_send_credit_target > sc->ib.dev->attrs.max_qp_wr) { log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", smbd_send_credit_target, - info->id->device->attrs.max_cqe, - info->id->device->attrs.max_qp_wr); + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); goto config_failed; } - if (smbd_receive_credit_max > info->id->device->attrs.max_cqe || - smbd_receive_credit_max > info->id->device->attrs.max_qp_wr) { + if (smbd_receive_credit_max > sc->ib.dev->attrs.max_cqe || + smbd_receive_credit_max > sc->ib.dev->attrs.max_qp_wr) { log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", smbd_receive_credit_max, - info->id->device->attrs.max_cqe, - info->id->device->attrs.max_qp_wr); + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); goto config_failed; } @@ -1532,32 +1558,30 @@ static struct smbd_connection *_smbd_get_connection( info->max_receive_size = smbd_max_receive_size; info->keep_alive_interval = smbd_keep_alive_interval; - if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SEND_SGE || - info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_RECV_SGE) { + if (sc->ib.dev->attrs.max_send_sge < SMBDIRECT_MAX_SEND_SGE || + sc->ib.dev->attrs.max_recv_sge < SMBDIRECT_MAX_RECV_SGE) { log_rdma_event(ERR, "device %.*s max_send_sge/max_recv_sge = %d/%d too small\n", IB_DEVICE_NAME_MAX, - info->id->device->name, - info->id->device->attrs.max_send_sge, - info->id->device->attrs.max_recv_sge); + sc->ib.dev->name, + sc->ib.dev->attrs.max_send_sge, + sc->ib.dev->attrs.max_recv_sge); goto config_failed; } - info->send_cq = NULL; - info->recv_cq = NULL; - info->send_cq = - ib_alloc_cq_any(info->id->device, info, + sc->ib.send_cq = + ib_alloc_cq_any(sc->ib.dev, info, info->send_credit_target, IB_POLL_SOFTIRQ); - if (IS_ERR(info->send_cq)) { - info->send_cq = NULL; + if (IS_ERR(sc->ib.send_cq)) { + sc->ib.send_cq = NULL; goto alloc_cq_failed; } - info->recv_cq = - ib_alloc_cq_any(info->id->device, info, + sc->ib.recv_cq = + ib_alloc_cq_any(sc->ib.dev, info, info->receive_credit_max, IB_POLL_SOFTIRQ); - if (IS_ERR(info->recv_cq)) { - info->recv_cq = NULL; + if (IS_ERR(sc->ib.recv_cq)) { + sc->ib.recv_cq = NULL; goto alloc_cq_failed; } @@ -1571,31 +1595,30 @@ static struct smbd_connection *_smbd_get_connection( qp_attr.cap.max_inline_data = 0; qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; qp_attr.qp_type = IB_QPT_RC; - qp_attr.send_cq = info->send_cq; - qp_attr.recv_cq = info->recv_cq; + qp_attr.send_cq = sc->ib.send_cq; + qp_attr.recv_cq = sc->ib.recv_cq; qp_attr.port_num = ~0; - rc = rdma_create_qp(info->id, info->pd, &qp_attr); + rc = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); if (rc) { log_rdma_event(ERR, "rdma_create_qp failed %i\n", rc); goto create_qp_failed; } + sc->ib.qp = sc->rdma.cm_id->qp; memset(&conn_param, 0, sizeof(conn_param)); conn_param.initiator_depth = 0; conn_param.responder_resources = - info->id->device->attrs.max_qp_rd_atom - < SMBD_CM_RESPONDER_RESOURCES ? - info->id->device->attrs.max_qp_rd_atom : - SMBD_CM_RESPONDER_RESOURCES; + min(sc->ib.dev->attrs.max_qp_rd_atom, + SMBD_CM_RESPONDER_RESOURCES); info->responder_resources = conn_param.responder_resources; log_rdma_mr(INFO, "responder_resources=%d\n", info->responder_resources); /* Need to send IRD/ORD in private data for iWARP */ - info->id->device->ops.get_port_immutable( - info->id->device, info->id->port_num, &port_immutable); + sc->ib.dev->ops.get_port_immutable( + sc->ib.dev, sc->rdma.cm_id->port_num, &port_immutable); if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) { ird_ord_hdr[0] = info->responder_resources; ird_ord_hdr[1] = 1; @@ -1616,16 +1639,18 @@ static struct smbd_connection *_smbd_get_connection( init_waitqueue_head(&info->conn_wait); init_waitqueue_head(&info->disconn_wait); init_waitqueue_head(&info->wait_reassembly_queue); - rc = rdma_connect(info->id, &conn_param); + rc = rdma_connect(sc->rdma.cm_id, &conn_param); if (rc) { log_rdma_event(ERR, "rdma_connect() failed with %i\n", rc); goto rdma_connect_failed; } - wait_event_interruptible( - info->conn_wait, info->transport_status != SMBD_CONNECTING); + wait_event_interruptible_timeout( + info->conn_wait, + sc->status != SMBDIRECT_SOCKET_CONNECTING, + msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT)); - if (info->transport_status != SMBD_CONNECTED) { + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { log_rdma_event(ERR, "rdma_connect failed port=%d\n", port); goto rdma_connect_failed; } @@ -1676,26 +1701,26 @@ static struct smbd_connection *_smbd_get_connection( negotiation_failed: cancel_delayed_work_sync(&info->idle_timer_work); destroy_caches_and_workqueue(info); - info->transport_status = SMBD_NEGOTIATE_FAILED; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; init_waitqueue_head(&info->conn_wait); - rdma_disconnect(info->id); + rdma_disconnect(sc->rdma.cm_id); wait_event(info->conn_wait, - info->transport_status == SMBD_DISCONNECTED); + sc->status == SMBDIRECT_SOCKET_DISCONNECTED); allocate_cache_failed: rdma_connect_failed: - rdma_destroy_qp(info->id); + rdma_destroy_qp(sc->rdma.cm_id); create_qp_failed: alloc_cq_failed: - if (info->send_cq) - ib_free_cq(info->send_cq); - if (info->recv_cq) - ib_free_cq(info->recv_cq); + if (sc->ib.send_cq) + ib_free_cq(sc->ib.send_cq); + if (sc->ib.recv_cq) + ib_free_cq(sc->ib.recv_cq); config_failed: - ib_dealloc_pd(info->pd); - rdma_destroy_id(info->id); + ib_dealloc_pd(sc->ib.pd); + rdma_destroy_id(sc->rdma.cm_id); create_id_failed: kfree(info); @@ -1735,6 +1760,7 @@ struct smbd_connection *smbd_get_connection( static int smbd_recv_buf(struct smbd_connection *info, char *buf, unsigned int size) { + struct smbdirect_socket *sc = &info->socket; struct smbd_response *response; struct smbd_data_transfer *data_transfer; int to_copy, to_read, data_read, offset; @@ -1849,12 +1875,12 @@ static int smbd_recv_buf(struct smbd_connection *info, char *buf, rc = wait_event_interruptible( info->wait_reassembly_queue, info->reassembly_data_length >= size || - info->transport_status != SMBD_CONNECTED); + sc->status != SMBDIRECT_SOCKET_CONNECTED); /* Don't return any data if interrupted */ if (rc) return rc; - if (info->transport_status != SMBD_CONNECTED) { + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { log_read(ERR, "disconnected\n"); return -ECONNABORTED; } @@ -1872,6 +1898,7 @@ static int smbd_recv_page(struct smbd_connection *info, struct page *page, unsigned int page_offset, unsigned int to_read) { + struct smbdirect_socket *sc = &info->socket; int ret; char *to_address; void *page_address; @@ -1880,7 +1907,7 @@ static int smbd_recv_page(struct smbd_connection *info, ret = wait_event_interruptible( info->wait_reassembly_queue, info->reassembly_data_length >= to_read || - info->transport_status != SMBD_CONNECTED); + sc->status != SMBDIRECT_SOCKET_CONNECTED); if (ret) return ret; @@ -1955,12 +1982,13 @@ int smbd_send(struct TCP_Server_Info *server, int num_rqst, struct smb_rqst *rqst_array) { struct smbd_connection *info = server->smbd_conn; + struct smbdirect_socket *sc = &info->socket; struct smb_rqst *rqst; struct iov_iter iter; unsigned int remaining_data_length, klen; int rc, i, rqst_idx; - if (info->transport_status != SMBD_CONNECTED) + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -EAGAIN; /* @@ -2023,7 +2051,11 @@ int smbd_send(struct TCP_Server_Info *server, */ wait_event(info->wait_send_pending, - atomic_read(&info->send_pending) == 0); + atomic_read(&info->send_pending) == 0 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED && rc == 0) + rc = -EAGAIN; return rc; } @@ -2054,6 +2086,7 @@ static void smbd_mr_recovery_work(struct work_struct *work) { struct smbd_connection *info = container_of(work, struct smbd_connection, mr_recovery_work); + struct smbdirect_socket *sc = &info->socket; struct smbd_mr *smbdirect_mr; int rc; @@ -2071,7 +2104,7 @@ static void smbd_mr_recovery_work(struct work_struct *work) } smbdirect_mr->mr = ib_alloc_mr( - info->pd, info->mr_type, + sc->ib.pd, info->mr_type, info->max_frmr_depth); if (IS_ERR(smbdirect_mr->mr)) { log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n", @@ -2100,12 +2133,13 @@ static void smbd_mr_recovery_work(struct work_struct *work) static void destroy_mr_list(struct smbd_connection *info) { + struct smbdirect_socket *sc = &info->socket; struct smbd_mr *mr, *tmp; cancel_work_sync(&info->mr_recovery_work); list_for_each_entry_safe(mr, tmp, &info->mr_list, list) { if (mr->state == MR_INVALIDATED) - ib_dma_unmap_sg(info->id->device, mr->sgt.sgl, + ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); ib_dereg_mr(mr->mr); kfree(mr->sgt.sgl); @@ -2122,6 +2156,7 @@ static void destroy_mr_list(struct smbd_connection *info) */ static int allocate_mr_list(struct smbd_connection *info) { + struct smbdirect_socket *sc = &info->socket; int i; struct smbd_mr *smbdirect_mr, *tmp; @@ -2137,7 +2172,7 @@ static int allocate_mr_list(struct smbd_connection *info) smbdirect_mr = kzalloc(sizeof(*smbdirect_mr), GFP_KERNEL); if (!smbdirect_mr) goto cleanup_entries; - smbdirect_mr->mr = ib_alloc_mr(info->pd, info->mr_type, + smbdirect_mr->mr = ib_alloc_mr(sc->ib.pd, info->mr_type, info->max_frmr_depth); if (IS_ERR(smbdirect_mr->mr)) { log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n", @@ -2182,20 +2217,20 @@ static int allocate_mr_list(struct smbd_connection *info) */ static struct smbd_mr *get_mr(struct smbd_connection *info) { + struct smbdirect_socket *sc = &info->socket; struct smbd_mr *ret; int rc; again: rc = wait_event_interruptible(info->wait_mr, atomic_read(&info->mr_ready_count) || - info->transport_status != SMBD_CONNECTED); + sc->status != SMBDIRECT_SOCKET_CONNECTED); if (rc) { log_rdma_mr(ERR, "wait_event_interruptible rc=%x\n", rc); return NULL; } - if (info->transport_status != SMBD_CONNECTED) { - log_rdma_mr(ERR, "info->transport_status=%x\n", - info->transport_status); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + log_rdma_mr(ERR, "sc->status=%x\n", sc->status); return NULL; } @@ -2248,6 +2283,7 @@ struct smbd_mr *smbd_register_mr(struct smbd_connection *info, struct iov_iter *iter, bool writing, bool need_invalidate) { + struct smbdirect_socket *sc = &info->socket; struct smbd_mr *smbdirect_mr; int rc, num_pages; enum dma_data_direction dir; @@ -2277,7 +2313,7 @@ struct smbd_mr *smbd_register_mr(struct smbd_connection *info, num_pages, iov_iter_count(iter), info->max_frmr_depth); smbd_iter_to_mr(info, iter, &smbdirect_mr->sgt, info->max_frmr_depth); - rc = ib_dma_map_sg(info->id->device, smbdirect_mr->sgt.sgl, + rc = ib_dma_map_sg(sc->ib.dev, smbdirect_mr->sgt.sgl, smbdirect_mr->sgt.nents, dir); if (!rc) { log_rdma_mr(ERR, "ib_dma_map_sg num_pages=%x dir=%x rc=%x\n", @@ -2313,7 +2349,7 @@ struct smbd_mr *smbd_register_mr(struct smbd_connection *info, * on IB_WR_REG_MR. Hardware enforces a barrier and order of execution * on the next ib_post_send when we actaully send I/O to remote peer */ - rc = ib_post_send(info->id->qp, ®_wr->wr, NULL); + rc = ib_post_send(sc->ib.qp, ®_wr->wr, NULL); if (!rc) return smbdirect_mr; @@ -2322,7 +2358,7 @@ struct smbd_mr *smbd_register_mr(struct smbd_connection *info, /* If all failed, attempt to recover this MR by setting it MR_ERROR*/ map_mr_error: - ib_dma_unmap_sg(info->id->device, smbdirect_mr->sgt.sgl, + ib_dma_unmap_sg(sc->ib.dev, smbdirect_mr->sgt.sgl, smbdirect_mr->sgt.nents, smbdirect_mr->dir); dma_map_error: @@ -2360,6 +2396,7 @@ int smbd_deregister_mr(struct smbd_mr *smbdirect_mr) { struct ib_send_wr *wr; struct smbd_connection *info = smbdirect_mr->conn; + struct smbdirect_socket *sc = &info->socket; int rc = 0; if (smbdirect_mr->need_invalidate) { @@ -2373,7 +2410,7 @@ int smbd_deregister_mr(struct smbd_mr *smbdirect_mr) wr->send_flags = IB_SEND_SIGNALED; init_completion(&smbdirect_mr->invalidate_done); - rc = ib_post_send(info->id->qp, wr, NULL); + rc = ib_post_send(sc->ib.qp, wr, NULL); if (rc) { log_rdma_mr(ERR, "ib_post_send failed rc=%x\n", rc); smbd_disconnect_rdma_connection(info); @@ -2390,7 +2427,7 @@ int smbd_deregister_mr(struct smbd_mr *smbdirect_mr) if (smbdirect_mr->state == MR_INVALIDATED) { ib_dma_unmap_sg( - info->id->device, smbdirect_mr->sgt.sgl, + sc->ib.dev, smbdirect_mr->sgt.sgl, smbdirect_mr->sgt.nents, smbdirect_mr->dir); smbdirect_mr->state = MR_READY; diff --git a/fs/smb/client/smbdirect.h b/fs/smb/client/smbdirect.h index 83f239f376f06..c881e58c639de 100644 --- a/fs/smb/client/smbdirect.h +++ b/fs/smb/client/smbdirect.h @@ -15,6 +15,8 @@ #include #include +#include "../common/smbdirect/smbdirect_socket.h" + extern int rdma_readwrite_threshold; extern int smbd_max_frmr_depth; extern int smbd_keep_alive_interval; @@ -50,14 +52,8 @@ enum smbd_connection_status { * 5. mempools for allocating packets */ struct smbd_connection { - enum smbd_connection_status transport_status; - - /* RDMA related */ - struct rdma_cm_id *id; - struct ib_qp_init_attr qp_attr; - struct ib_pd *pd; - struct ib_cq *send_cq, *recv_cq; - struct ib_device_attr dev_attr; + struct smbdirect_socket socket; + int ri_rc; struct completion ri_done; wait_queue_head_t conn_wait; @@ -111,7 +107,7 @@ struct smbd_connection { /* Used by transport to wait until all MRs are returned */ wait_queue_head_t wait_for_mr_cleanup; - /* Activity accoutning */ + /* Activity accounting */ atomic_t send_pending; wait_queue_head_t wait_send_pending; wait_queue_head_t wait_post_send; diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index 563cb4d8edf0c..4dfdc521c5c98 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -114,7 +114,7 @@ DECLARE_EVENT_CLASS(smb3_rw_err_class, __entry->len = len; __entry->rc = rc; ), - TP_printk("\txid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx len=0x%x rc=%d", + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx len=0x%x rc=%d", __entry->xid, __entry->sesid, __entry->tid, __entry->fid, __entry->offset, __entry->len, __entry->rc) ) @@ -247,7 +247,7 @@ DECLARE_EVENT_CLASS(smb3_fd_class, __entry->tid = tid; __entry->sesid = sesid; ), - TP_printk("\txid=%u sid=0x%llx tid=0x%x fid=0x%llx", + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx", __entry->xid, __entry->sesid, __entry->tid, __entry->fid) ) @@ -286,7 +286,7 @@ DECLARE_EVENT_CLASS(smb3_fd_err_class, __entry->sesid = sesid; __entry->rc = rc; ), - TP_printk("\txid=%u sid=0x%llx tid=0x%x fid=0x%llx rc=%d", + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx rc=%d", __entry->xid, __entry->sesid, __entry->tid, __entry->fid, __entry->rc) ) @@ -558,7 +558,7 @@ DECLARE_EVENT_CLASS(smb3_cmd_err_class, __entry->status = status; __entry->rc = rc; ), - TP_printk("\tsid=0x%llx tid=0x%x cmd=%u mid=%llu status=0x%x rc=%d", + TP_printk("sid=0x%llx tid=0x%x cmd=%u mid=%llu status=0x%x rc=%d", __entry->sesid, __entry->tid, __entry->cmd, __entry->mid, __entry->status, __entry->rc) ) @@ -593,7 +593,7 @@ DECLARE_EVENT_CLASS(smb3_cmd_done_class, __entry->cmd = cmd; __entry->mid = mid; ), - TP_printk("\tsid=0x%llx tid=0x%x cmd=%u mid=%llu", + TP_printk("sid=0x%llx tid=0x%x cmd=%u mid=%llu", __entry->sesid, __entry->tid, __entry->cmd, __entry->mid) ) @@ -631,7 +631,7 @@ DECLARE_EVENT_CLASS(smb3_mid_class, __entry->when_sent = when_sent; __entry->when_received = when_received; ), - TP_printk("\tcmd=%u mid=%llu pid=%u, when_sent=%lu when_rcv=%lu", + TP_printk("cmd=%u mid=%llu pid=%u, when_sent=%lu when_rcv=%lu", __entry->cmd, __entry->mid, __entry->pid, __entry->when_sent, __entry->when_received) ) @@ -662,7 +662,7 @@ DECLARE_EVENT_CLASS(smb3_exit_err_class, __assign_str(func_name, func_name); __entry->rc = rc; ), - TP_printk("\t%s: xid=%u rc=%d", + TP_printk("%s: xid=%u rc=%d", __get_str(func_name), __entry->xid, __entry->rc) ) @@ -688,7 +688,7 @@ DECLARE_EVENT_CLASS(smb3_sync_err_class, __entry->ino = ino; __entry->rc = rc; ), - TP_printk("\tino=%lu rc=%d", + TP_printk("ino=%lu rc=%d", __entry->ino, __entry->rc) ) @@ -714,7 +714,7 @@ DECLARE_EVENT_CLASS(smb3_enter_exit_class, __entry->xid = xid; __assign_str(func_name, func_name); ), - TP_printk("\t%s: xid=%u", + TP_printk("%s: xid=%u", __get_str(func_name), __entry->xid) ) diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index ddf1a3aafee5c..7fee8e2c723a8 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "cifspdu.h" #include "cifsglob.h" #include "cifsproto.h" @@ -178,7 +179,7 @@ delete_mid(struct mid_q_entry *mid) * Our basic "send data to server" function. Should be called with srv_mutex * held. The caller is responsible for handling the results. */ -static int +int smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg, size_t *sent) { @@ -211,9 +212,16 @@ smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg, * send a packet. In most cases if we fail to send * after the retries we will kill the socket and * reconnect which may clear the network problem. + * + * Even if regular signals are masked, EINTR might be + * propagated from sk_stream_wait_memory() to here when + * TIF_NOTIFY_SIGNAL is used for task work. For example, + * certain io_uring completions will use that. Treat + * having EINTR with pending task work the same as EAGAIN + * to avoid unnecessary reconnects. */ rc = sock_sendmsg(ssocket, smb_msg); - if (rc == -EAGAIN) { + if (rc == -EAGAIN || unlikely(rc == -EINTR && task_work_pending(current))) { retries++; if (retries >= 14 || (!server->noblocksnd && (retries > 2))) { @@ -1025,18 +1033,20 @@ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) uint index = 0; unsigned int min_in_flight = UINT_MAX, max_in_flight = 0; struct TCP_Server_Info *server = NULL; - int i; + int i, start, cur; if (!ses) return NULL; spin_lock(&ses->chan_lock); + start = atomic_inc_return(&ses->chan_seq); for (i = 0; i < ses->chan_count; i++) { - server = ses->chans[i].server; + cur = (start + i) % ses->chan_count; + server = ses->chans[cur].server; if (!server || server->terminate) continue; - if (CIFS_CHAN_NEEDS_RECONNECT(ses, i)) + if (CIFS_CHAN_NEEDS_RECONNECT(ses, cur)) continue; /* @@ -1049,17 +1059,15 @@ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) */ if (server->in_flight < min_in_flight) { min_in_flight = server->in_flight; - index = i; + index = cur; } if (server->in_flight > max_in_flight) max_in_flight = server->in_flight; } /* if all channels are equally loaded, fall back to round-robin */ - if (min_in_flight == max_in_flight) { - index = (uint)atomic_inc_return(&ses->chan_seq); - index %= ses->chan_count; - } + if (min_in_flight == max_in_flight) + index = (uint)start % ses->chan_count; server = ses->chans[index].server; spin_unlock(&ses->chan_lock); diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h index c3ee42188d252..1af827ae757e0 100644 --- a/fs/smb/common/smb2pdu.h +++ b/fs/smb/common/smb2pdu.h @@ -95,6 +95,9 @@ */ #define SMB3_DEFAULT_IOSIZE (4 * 1024 * 1024) +/* According to MS-SMB2 specification The minimum recommended value is 65536.*/ +#define CIFS_MIN_DEFAULT_IOSIZE (65536) + /* * SMB2 Header Definition * diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h new file mode 100644 index 0000000000000..69a55561f91ae --- /dev/null +++ b/fs/smb/common/smbdirect/smbdirect_socket.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2025 Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ +#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ + +enum smbdirect_socket_status { + SMBDIRECT_SOCKET_CREATED, + SMBDIRECT_SOCKET_CONNECTING, + SMBDIRECT_SOCKET_CONNECTED, + SMBDIRECT_SOCKET_NEGOTIATE_FAILED, + SMBDIRECT_SOCKET_DISCONNECTING, + SMBDIRECT_SOCKET_DISCONNECTED, + SMBDIRECT_SOCKET_DESTROYED +}; + +struct smbdirect_socket { + enum smbdirect_socket_status status; + + /* RDMA related */ + struct { + struct rdma_cm_id *cm_id; + } rdma; + + /* IB verbs related */ + struct { + struct ib_pd *pd; + struct ib_cq *send_cq; + struct ib_cq *recv_cq; + + /* + * shortcuts for rdma.cm_id->{qp,device}; + */ + struct ib_qp *qp; + struct ib_device *dev; + } ib; +}; + +#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */ diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index 5345d2417c7fc..f4b20b80af062 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -546,7 +546,19 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, retval = -ENOMEM; goto out; } - sess->user = user; + + if (!sess->user) { + /* First successful authentication */ + sess->user = user; + } else { + if (!ksmbd_compare_user(sess->user, user)) { + ksmbd_debug(AUTH, "different user tried to reuse session\n"); + retval = -EPERM; + ksmbd_free_user(user); + goto out; + } + ksmbd_free_user(user); + } memcpy(sess->sess_key, resp->payload, resp->session_key_len); memcpy(out_blob, resp->payload + resp->session_key_len, diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index e7bcc38300310..f5ebc200dd738 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -39,8 +39,10 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) xa_destroy(&conn->sessions); kvfree(conn->request_buf); kfree(conn->preauth_info); - if (atomic_dec_and_test(&conn->refcnt)) + if (atomic_dec_and_test(&conn->refcnt)) { + conn->transport->ops->free_transport(conn->transport); kfree(conn); + } } /** @@ -501,7 +503,8 @@ void ksmbd_conn_transport_destroy(void) { mutex_lock(&init_lock); ksmbd_tcp_destroy(); - ksmbd_rdma_destroy(); + ksmbd_rdma_stop_listening(); stop_sessions(); + ksmbd_rdma_destroy(); mutex_unlock(&init_lock); } diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h index 4fdd76ce53b90..45421269ddd88 100644 --- a/fs/smb/server/connection.h +++ b/fs/smb/server/connection.h @@ -27,6 +27,7 @@ enum { KSMBD_SESS_EXITING, KSMBD_SESS_NEED_RECONNECT, KSMBD_SESS_NEED_NEGOTIATE, + KSMBD_SESS_NEED_SETUP, KSMBD_SESS_RELEASING }; @@ -45,6 +46,12 @@ struct ksmbd_conn { struct mutex srv_mutex; int status; unsigned int cli_cap; + union { + __be32 inet_addr; +#if IS_ENABLED(CONFIG_IPV6) + u8 inet6_addr[16]; +#endif + }; char *request_buf; struct ksmbd_transport *transport; struct nls_table *local_nls; @@ -107,6 +114,7 @@ struct ksmbd_conn { __le16 signing_algorithm; bool binding; atomic_t refcnt; + bool is_aapl; }; struct ksmbd_conn_ops { @@ -131,6 +139,7 @@ struct ksmbd_transport_ops { void *buf, unsigned int len, struct smb2_buffer_desc_v1 *desc, unsigned int desc_len); + void (*free_transport)(struct ksmbd_transport *kt); }; struct ksmbd_transport { @@ -187,6 +196,11 @@ static inline bool ksmbd_conn_need_negotiate(struct ksmbd_conn *conn) return READ_ONCE(conn->status) == KSMBD_SESS_NEED_NEGOTIATE; } +static inline bool ksmbd_conn_need_setup(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_NEED_SETUP; +} + static inline bool ksmbd_conn_need_reconnect(struct ksmbd_conn *conn) { return READ_ONCE(conn->status) == KSMBD_SESS_NEED_RECONNECT; @@ -217,6 +231,11 @@ static inline void ksmbd_conn_set_need_negotiate(struct ksmbd_conn *conn) WRITE_ONCE(conn->status, KSMBD_SESS_NEED_NEGOTIATE); } +static inline void ksmbd_conn_set_need_setup(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_NEED_SETUP); +} + static inline void ksmbd_conn_set_need_reconnect(struct ksmbd_conn *conn) { WRITE_ONCE(conn->status, KSMBD_SESS_NEED_RECONNECT); diff --git a/fs/smb/server/ksmbd_netlink.h b/fs/smb/server/ksmbd_netlink.h index f4e55199938d5..363501fc308aa 100644 --- a/fs/smb/server/ksmbd_netlink.h +++ b/fs/smb/server/ksmbd_netlink.h @@ -108,10 +108,12 @@ struct ksmbd_startup_request { __u32 smb2_max_credits; /* MAX credits */ __u32 smbd_max_io_size; /* smbd read write size */ __u32 max_connections; /* Number of maximum simultaneous connections */ - __u32 reserved[126]; /* Reserved room */ + __s8 bind_interfaces_only; + __u32 max_ip_connections; /* Number of maximum connection per ip address */ + __s8 reserved[499]; /* Reserved room */ __u32 ifc_list_sz; /* interfaces list size */ __s8 ____payload[]; -}; +} __packed; #define KSMBD_STARTUP_CONFIG_INTERFACES(s) ((s)->____payload) diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index 82dcc86a32c57..408f47220c07b 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -373,12 +373,12 @@ void destroy_previous_session(struct ksmbd_conn *conn, ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_RECONNECT); err = ksmbd_conn_wait_idle_sess_id(conn, id); if (err) { - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_NEGOTIATE); + ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); goto out; } ksmbd_destroy_file_table(&prev_sess->file_table); prev_sess->state = SMB2_SESSION_EXPIRED; - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_NEGOTIATE); + ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); out: up_write(&conn->session_lock); up_write(&sessions_table_lock); diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index 5a5277b4b53b1..2a3ef29ac0eb7 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -146,12 +146,9 @@ static struct oplock_info *opinfo_get_list(struct ksmbd_inode *ci) { struct oplock_info *opinfo; - if (list_empty(&ci->m_op_list)) - return NULL; - down_read(&ci->m_lock); - opinfo = list_first_entry(&ci->m_op_list, struct oplock_info, - op_entry); + opinfo = list_first_entry_or_null(&ci->m_op_list, struct oplock_info, + op_entry); if (opinfo) { if (opinfo->conn == NULL || !atomic_inc_not_zero(&opinfo->refcount)) @@ -1105,8 +1102,10 @@ void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, if (!atomic_inc_not_zero(&opinfo->refcount)) continue; - if (ksmbd_conn_releasing(opinfo->conn)) + if (ksmbd_conn_releasing(opinfo->conn)) { + opinfo_put(opinfo); continue; + } oplock_break(opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL); opinfo_put(opinfo); @@ -1142,8 +1141,11 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) if (!atomic_inc_not_zero(&opinfo->refcount)) continue; - if (ksmbd_conn_releasing(opinfo->conn)) + if (ksmbd_conn_releasing(opinfo->conn)) { + opinfo_put(opinfo); continue; + } + oplock_break(opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL); opinfo_put(opinfo); } @@ -1346,8 +1348,10 @@ void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, if (!atomic_inc_not_zero(&brk_op->refcount)) continue; - if (ksmbd_conn_releasing(brk_op->conn)) + if (ksmbd_conn_releasing(brk_op->conn)) { + opinfo_put(brk_op); continue; + } if (brk_op->is_lease && (brk_op->o_lease->state & (~(SMB2_LEASE_READ_CACHING_LE | @@ -1496,7 +1500,7 @@ struct lease_ctx_info *parse_lease_state(void *open_req) if (le16_to_cpu(cc->DataOffset) + le32_to_cpu(cc->DataLength) < sizeof(struct create_lease_v2) - 4) - return NULL; + goto err_out; memcpy(lreq->lease_key, lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); lreq->req_state = lc->lcontext.LeaseState; @@ -1512,7 +1516,7 @@ struct lease_ctx_info *parse_lease_state(void *open_req) if (le16_to_cpu(cc->DataOffset) + le32_to_cpu(cc->DataLength) < sizeof(struct create_lease)) - return NULL; + goto err_out; memcpy(lreq->lease_key, lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); lreq->req_state = lc->lcontext.LeaseState; @@ -1521,6 +1525,9 @@ struct lease_ctx_info *parse_lease_state(void *open_req) lreq->version = 1; } return lreq; +err_out: + kfree(lreq); + return NULL; } /** diff --git a/fs/smb/server/server.h b/fs/smb/server/server.h index 4d06f2eb0d6ad..48bd203abb441 100644 --- a/fs/smb/server/server.h +++ b/fs/smb/server/server.h @@ -43,8 +43,10 @@ struct ksmbd_server_config { unsigned int auth_mechs; unsigned int max_connections; unsigned int max_inflight_req; + unsigned int max_ip_connections; char *conf[SERVER_CONF_WORK_GROUP + 1]; + bool bind_interfaces_only; }; extern struct ksmbd_server_config server_conf; diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index d41d67ec5ee51..9f64808c7917c 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -38,6 +38,7 @@ #include "mgmt/user_session.h" #include "mgmt/ksmbd_ida.h" #include "ndr.h" +#include "transport_tcp.h" static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) { @@ -632,6 +633,11 @@ smb2_get_name(const char *src, const int maxlen, struct nls_table *local_nls) return name; } + if (*name == '\0') { + kfree(name); + return ERR_PTR(-EINVAL); + } + if (*name == '\\') { pr_err("not allow directory name included leading slash\n"); kfree(name); @@ -1247,7 +1253,7 @@ int smb2_handle_negotiate(struct ksmbd_work *work) } conn->srv_sec_mode = le16_to_cpu(rsp->SecurityMode); - ksmbd_conn_set_need_negotiate(conn); + ksmbd_conn_set_need_setup(conn); err_out: if (rc) @@ -1268,6 +1274,9 @@ static int alloc_preauth_hash(struct ksmbd_session *sess, if (sess->Preauth_HashValue) return 0; + if (!conn->preauth_info) + return -ENOMEM; + sess->Preauth_HashValue = kmemdup(conn->preauth_info->Preauth_HashValue, PREAUTH_HASHVALUE_SIZE, GFP_KERNEL); if (!sess->Preauth_HashValue) @@ -1340,8 +1349,7 @@ static int ntlm_negotiate(struct ksmbd_work *work, return rc; sz = le16_to_cpu(rsp->SecurityBufferOffset); - chgblob = - (struct challenge_message *)((char *)&rsp->hdr.ProtocolId + sz); + chgblob = (struct challenge_message *)rsp->Buffer; memset(chgblob, 0, sizeof(struct challenge_message)); if (!work->conn->use_spnego) { @@ -1374,8 +1382,7 @@ static int ntlm_negotiate(struct ksmbd_work *work, goto out; } - sz = le16_to_cpu(rsp->SecurityBufferOffset); - memcpy((char *)&rsp->hdr.ProtocolId + sz, spnego_blob, spnego_blob_len); + memcpy(rsp->Buffer, spnego_blob, spnego_blob_len); rsp->SecurityBufferLength = cpu_to_le16(spnego_blob_len); out: @@ -1457,8 +1464,7 @@ static int ntlm_authenticate(struct ksmbd_work *work, if (rc) return -ENOMEM; - sz = le16_to_cpu(rsp->SecurityBufferOffset); - memcpy((char *)&rsp->hdr.ProtocolId + sz, spnego_blob, spnego_blob_len); + memcpy(rsp->Buffer, spnego_blob, spnego_blob_len); rsp->SecurityBufferLength = cpu_to_le16(spnego_blob_len); kfree(spnego_blob); } @@ -1594,29 +1600,38 @@ static int krb5_authenticate(struct ksmbd_work *work, out_len = work->response_sz - (le16_to_cpu(rsp->SecurityBufferOffset) + 4); - /* Check previous session */ - prev_sess_id = le64_to_cpu(req->PreviousSessionId); - if (prev_sess_id && prev_sess_id != sess->id) - destroy_previous_session(conn, sess->user, prev_sess_id); - - if (sess->state == SMB2_SESSION_VALID) { - ksmbd_free_user(sess->user); - sess->user = NULL; - } - retval = ksmbd_krb5_authenticate(sess, in_blob, in_len, out_blob, &out_len); if (retval) { ksmbd_debug(SMB, "krb5 authentication failed\n"); return -EINVAL; } + + /* Check previous session */ + prev_sess_id = le64_to_cpu(req->PreviousSessionId); + if (prev_sess_id && prev_sess_id != sess->id) + destroy_previous_session(conn, sess->user, prev_sess_id); + rsp->SecurityBufferLength = cpu_to_le16(out_len); - if ((conn->sign || server_conf.enforced_signing) || + /* + * If session state is SMB2_SESSION_VALID, We can assume + * that it is reauthentication. And the user/password + * has been verified, so return it here. + */ + if (sess->state == SMB2_SESSION_VALID) { + if (conn->binding) + goto binding_session; + return 0; + } + + if ((rsp->SessionFlags != SMB2_SESSION_FLAG_IS_GUEST_LE && + (conn->sign || server_conf.enforced_signing)) || (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED)) sess->sign = true; - if (smb3_encryption_negotiated(conn)) { + if (smb3_encryption_negotiated(conn) && + !(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { retval = conn->ops->generate_encryptionkey(conn, sess); if (retval) { ksmbd_debug(SMB, @@ -1629,6 +1644,7 @@ static int krb5_authenticate(struct ksmbd_work *work, sess->sign = false; } +binding_session: if (conn->dialect >= SMB30_PROT_ID) { chann = lookup_chann_list(sess, conn); if (!chann) { @@ -1676,6 +1692,11 @@ int smb2_sess_setup(struct ksmbd_work *work) ksmbd_debug(SMB, "Received request for session setup\n"); + if (!ksmbd_conn_need_setup(conn) && !ksmbd_conn_good(conn)) { + work->send_no_response = 1; + return rc; + } + WORK_BUFFERS(work, req, rsp); rsp->StructureSize = cpu_to_le16(9); @@ -1772,6 +1793,7 @@ int smb2_sess_setup(struct ksmbd_work *work) if (ksmbd_conn_need_reconnect(conn)) { rc = -EFAULT; + ksmbd_user_session_put(sess); sess = NULL; goto out_err; } @@ -1819,8 +1841,6 @@ int smb2_sess_setup(struct ksmbd_work *work) ksmbd_conn_set_good(conn); sess->state = SMB2_SESSION_VALID; } - kfree(sess->Preauth_HashValue); - sess->Preauth_HashValue = NULL; } else if (conn->preferred_auth_mech == KSMBD_AUTH_NTLMSSP) { if (negblob->MessageType == NtLmNegotiate) { rc = ntlm_negotiate(work, negblob, negblob_len, rsp); @@ -1847,8 +1867,6 @@ int smb2_sess_setup(struct ksmbd_work *work) kfree(preauth_sess); } } - kfree(sess->Preauth_HashValue); - sess->Preauth_HashValue = NULL; } else { pr_info_ratelimited("Unknown NTLMSSP message type : 0x%x\n", le32_to_cpu(negblob->MessageType)); @@ -1911,7 +1929,7 @@ int smb2_sess_setup(struct ksmbd_work *work) if (try_delay) { ksmbd_conn_set_need_reconnect(conn); ssleep(5); - ksmbd_conn_set_need_negotiate(conn); + ksmbd_conn_set_need_setup(conn); } } smb2_set_err_rsp(work); @@ -2241,7 +2259,7 @@ int smb2_session_logoff(struct ksmbd_work *work) ksmbd_free_user(sess->user); sess->user = NULL; } - ksmbd_all_conn_set_status(sess_id, KSMBD_SESS_NEED_NEGOTIATE); + ksmbd_all_conn_set_status(sess_id, KSMBD_SESS_NEED_SETUP); rsp->StructureSize = cpu_to_le16(4); err = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_logoff_rsp)); @@ -2861,7 +2879,7 @@ int smb2_open(struct ksmbd_work *work) int req_op_level = 0, open_flags = 0, may_flags = 0, file_info = 0; int rc = 0; int contxt_cnt = 0, query_disk_id = 0; - int maximal_access_ctxt = 0, posix_ctxt = 0; + bool maximal_access_ctxt = false, posix_ctxt = false; int s_type = 0; int next_off = 0; char *name = NULL; @@ -2888,6 +2906,27 @@ int smb2_open(struct ksmbd_work *work) return create_smb2_pipe(work); } + if (req->CreateContextsOffset && tcon->posix_extensions) { + context = smb2_find_context_vals(req, SMB2_CREATE_TAG_POSIX, 16); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out2; + } else if (context) { + struct create_posix *posix = (struct create_posix *)context; + + if (le16_to_cpu(context->DataOffset) + + le32_to_cpu(context->DataLength) < + sizeof(struct create_posix) - 4) { + rc = -EINVAL; + goto err_out2; + } + ksmbd_debug(SMB, "get posix context\n"); + + posix_mode = le32_to_cpu(posix->Mode); + posix_ctxt = true; + } + } + if (req->NameLength) { name = smb2_get_name((char *)req + le16_to_cpu(req->NameOffset), le16_to_cpu(req->NameLength), @@ -2910,9 +2949,11 @@ int smb2_open(struct ksmbd_work *work) goto err_out2; } - rc = ksmbd_validate_filename(name); - if (rc < 0) - goto err_out2; + if (posix_ctxt == false) { + rc = ksmbd_validate_filename(name); + if (rc < 0) + goto err_out2; + } if (ksmbd_share_veto_filename(share, name)) { rc = -ENOENT; @@ -3070,28 +3111,6 @@ int smb2_open(struct ksmbd_work *work) rc = -EBADF; goto err_out2; } - - if (tcon->posix_extensions) { - context = smb2_find_context_vals(req, - SMB2_CREATE_TAG_POSIX, 16); - if (IS_ERR(context)) { - rc = PTR_ERR(context); - goto err_out2; - } else if (context) { - struct create_posix *posix = - (struct create_posix *)context; - if (le16_to_cpu(context->DataOffset) + - le32_to_cpu(context->DataLength) < - sizeof(struct create_posix) - 4) { - rc = -EINVAL; - goto err_out2; - } - ksmbd_debug(SMB, "get posix context\n"); - - posix_mode = le32_to_cpu(posix->Mode); - posix_ctxt = 1; - } - } } if (ksmbd_override_fsids(work)) { @@ -3525,6 +3544,15 @@ int smb2_open(struct ksmbd_work *work) ksmbd_debug(SMB, "get query on disk id context\n"); query_disk_id = 1; } + + if (conn->is_aapl == false) { + context = smb2_find_context_vals(req, SMB2_CREATE_AAPL, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out1; + } else if (context) + conn->is_aapl = true; + } } rc = ksmbd_vfs_getattr(&path, &stat); @@ -3963,7 +3991,10 @@ static int smb2_populate_readdir_entry(struct ksmbd_conn *conn, int info_level, if (dinfo->EaSize) dinfo->ExtFileAttributes = FILE_ATTRIBUTE_REPARSE_POINT_LE; dinfo->Reserved = 0; - dinfo->UniqueId = cpu_to_le64(ksmbd_kstat->kstat->ino); + if (conn->is_aapl) + dinfo->UniqueId = 0; + else + dinfo->UniqueId = cpu_to_le64(ksmbd_kstat->kstat->ino); if (d_info->hide_dot_file && d_info->name[0] == '.') dinfo->ExtFileAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; memcpy(dinfo->FileName, conv_name, conv_len); @@ -3980,7 +4011,10 @@ static int smb2_populate_readdir_entry(struct ksmbd_conn *conn, int info_level, smb2_get_reparse_tag_special_file(ksmbd_kstat->kstat->mode); if (fibdinfo->EaSize) fibdinfo->ExtFileAttributes = FILE_ATTRIBUTE_REPARSE_POINT_LE; - fibdinfo->UniqueId = cpu_to_le64(ksmbd_kstat->kstat->ino); + if (conn->is_aapl) + fibdinfo->UniqueId = 0; + else + fibdinfo->UniqueId = cpu_to_le64(ksmbd_kstat->kstat->ino); fibdinfo->ShortNameLength = 0; fibdinfo->Reserved = 0; fibdinfo->Reserved2 = cpu_to_le16(0); @@ -5564,7 +5598,8 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, if (!work->tcon->posix_extensions) { pr_err("client doesn't negotiate with SMB3.1.1 POSIX Extensions\n"); - rc = -EOPNOTSUPP; + path_put(&path); + return -EOPNOTSUPP; } else { info = (struct filesystem_posix_info *)(rsp->Buffer); info->OptimalTransferSize = cpu_to_le32(stfs.f_bsize); @@ -5987,7 +6022,6 @@ static int smb2_create_link(struct ksmbd_work *work, { char *link_name = NULL, *target_name = NULL, *pathname = NULL; struct path path, parent_path; - bool file_present = false; int rc; if (buf_len < (u64)sizeof(struct smb2_file_link_info) + @@ -6020,11 +6054,8 @@ static int smb2_create_link(struct ksmbd_work *work, if (rc) { if (rc != -ENOENT) goto out; - } else - file_present = true; - - if (file_info->ReplaceIfExists) { - if (file_present) { + } else { + if (file_info->ReplaceIfExists) { rc = ksmbd_vfs_remove_file(work, &path); if (rc) { rc = -EINVAL; @@ -6032,21 +6063,17 @@ static int smb2_create_link(struct ksmbd_work *work, link_name); goto out; } - } - } else { - if (file_present) { + } else { rc = -EEXIST; ksmbd_debug(SMB, "link already exists\n"); goto out; } + ksmbd_vfs_kern_path_unlock(&parent_path, &path); } - rc = ksmbd_vfs_link(work, target_name, link_name); if (rc) rc = -EINVAL; out: - if (file_present) - ksmbd_vfs_kern_path_unlock(&parent_path, &path); if (!IS_ERR(link_name)) kfree(link_name); @@ -6746,6 +6773,7 @@ int smb2_read(struct ksmbd_work *work) nbytes = ksmbd_vfs_read(work, fp, length, &offset, aux_payload_buf); if (nbytes < 0) { + kvfree(aux_payload_buf); err = nbytes; goto out; } @@ -7765,6 +7793,9 @@ static int fsctl_query_iface_info_ioctl(struct ksmbd_conn *conn, if (netdev->type == ARPHRD_LOOPBACK) continue; + if (!ksmbd_find_netdev_name_iface_list(netdev->name)) + continue; + flags = dev_get_flags(netdev); if (!(flags & IFF_RUNNING)) continue; @@ -8489,11 +8520,6 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) goto err_out; } - opinfo->op_state = OPLOCK_STATE_NONE; - wake_up_interruptible_all(&opinfo->oplock_q); - opinfo_put(opinfo); - ksmbd_fd_put(work, fp); - rsp->StructureSize = cpu_to_le16(24); rsp->OplockLevel = rsp_oplevel; rsp->Reserved = 0; @@ -8501,16 +8527,15 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) rsp->VolatileFid = volatile_id; rsp->PersistentFid = persistent_id; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_oplock_break)); - if (!ret) - return; - + if (ret) { err_out: + smb2_set_err_rsp(work); + } + opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); - opinfo_put(opinfo); ksmbd_fd_put(work, fp); - smb2_set_err_rsp(work); } static int check_lease_state(struct lease *lease, __le32 req_state) @@ -8640,11 +8665,6 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) } lease_state = lease->state; - opinfo->op_state = OPLOCK_STATE_NONE; - wake_up_interruptible_all(&opinfo->oplock_q); - atomic_dec(&opinfo->breaking_cnt); - wake_up_interruptible_all(&opinfo->oplock_brk); - opinfo_put(opinfo); rsp->StructureSize = cpu_to_le16(36); rsp->Reserved = 0; @@ -8653,16 +8673,16 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) rsp->LeaseState = lease_state; rsp->LeaseDuration = 0; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); - if (!ret) - return; - + if (ret) { err_out: + smb2_set_err_rsp(work); + } + + opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); atomic_dec(&opinfo->breaking_cnt); wake_up_interruptible_all(&opinfo->oplock_brk); - opinfo_put(opinfo); - smb2_set_err_rsp(work); } /** diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h index 25cc81aac350f..2821e6c8298f4 100644 --- a/fs/smb/server/smb2pdu.h +++ b/fs/smb/server/smb2pdu.h @@ -63,6 +63,9 @@ struct preauth_integrity_info { #define SMB2_SESSION_TIMEOUT (10 * HZ) +/* Apple Defined Contexts */ +#define SMB2_CREATE_AAPL "AAPL" + struct create_durable_req_v2 { struct create_context_hdr ccontext; __u8 Name[8]; diff --git a/fs/smb/server/smb_common.c b/fs/smb/server/smb_common.c index 23537e1b34685..2839c704110cd 100644 --- a/fs/smb/server/smb_common.c +++ b/fs/smb/server/smb_common.c @@ -515,7 +515,7 @@ int ksmbd_extract_shortname(struct ksmbd_conn *conn, const char *longname, p = strrchr(longname, '.'); if (p == longname) { /*name starts with a dot*/ - strscpy(extension, "___", strlen("___")); + strscpy(extension, "___", sizeof(extension)); } else { if (p) { p++; diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c index 281101fd1f76f..36e1e52c30a0f 100644 --- a/fs/smb/server/transport_ipc.c +++ b/fs/smb/server/transport_ipc.c @@ -249,10 +249,16 @@ static void ipc_msg_handle_free(int handle) static int handle_response(int type, void *payload, size_t sz) { - unsigned int handle = *(unsigned int *)payload; + unsigned int handle; struct ipc_msg_table_entry *entry; int ret = 0; + /* Prevent 4-byte read beyond declared payload size */ + if (sz < sizeof(unsigned int)) + return -EINVAL; + + handle = *(unsigned int *)payload; + ipc_update_last_active(); down_read(&ipc_msg_table_lock); hash_for_each_possible(ipc_msg_table, entry, ipc_table_hlist, handle) { @@ -321,9 +327,13 @@ static int ipc_server_config_on_startup(struct ksmbd_startup_request *req) if (req->max_connections) server_conf.max_connections = req->max_connections; + if (req->max_ip_connections) + server_conf.max_ip_connections = req->max_ip_connections; + ret = ksmbd_set_netbios_name(req->netbios_name); ret |= ksmbd_set_server_string(req->server_string); ret |= ksmbd_set_work_group(req->work_group); + server_conf.bind_interfaces_only = req->bind_interfaces_only; ret |= ksmbd_tcp_set_interfaces(KSMBD_STARTUP_CONFIG_INTERFACES(req), req->ifc_list_sz); out: diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index 8faa25c6e129b..91e85a1a154fd 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -128,9 +128,6 @@ struct smb_direct_transport { spinlock_t recvmsg_queue_lock; struct list_head recvmsg_queue; - spinlock_t empty_recvmsg_queue_lock; - struct list_head empty_recvmsg_queue; - int send_credit_target; atomic_t send_credits; spinlock_t lock_new_recv_credits; @@ -150,15 +147,20 @@ struct smb_direct_transport { wait_queue_head_t wait_send_pending; atomic_t send_pending; - struct delayed_work post_recv_credits_work; + struct work_struct post_recv_credits_work; struct work_struct send_immediate_work; struct work_struct disconnect_work; bool negotiation_requested; + + bool legacy_iwarp; + u8 initiator_depth; + u8 responder_resources; }; #define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport)) - +#define SMBD_TRANS(t) ((struct smb_direct_transport *)container_of(t, \ + struct smb_direct_transport, transport)) enum { SMB_DIRECT_MSG_NEGOTIATE_REQ = 0, SMB_DIRECT_MSG_DATA_TRANSFER @@ -266,40 +268,19 @@ smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t) static void put_recvmsg(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg) { - ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, - recvmsg->sge.length, DMA_FROM_DEVICE); + if (likely(recvmsg->sge.length != 0)) { + ib_dma_unmap_single(t->cm_id->device, + recvmsg->sge.addr, + recvmsg->sge.length, + DMA_FROM_DEVICE); + recvmsg->sge.length = 0; + } spin_lock(&t->recvmsg_queue_lock); list_add(&recvmsg->list, &t->recvmsg_queue); spin_unlock(&t->recvmsg_queue_lock); } -static struct -smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t) -{ - struct smb_direct_recvmsg *recvmsg = NULL; - - spin_lock(&t->empty_recvmsg_queue_lock); - if (!list_empty(&t->empty_recvmsg_queue)) { - recvmsg = list_first_entry(&t->empty_recvmsg_queue, - struct smb_direct_recvmsg, list); - list_del(&recvmsg->list); - } - spin_unlock(&t->empty_recvmsg_queue_lock); - return recvmsg; -} - -static void put_empty_recvmsg(struct smb_direct_transport *t, - struct smb_direct_recvmsg *recvmsg) -{ - ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, - recvmsg->sge.length, DMA_FROM_DEVICE); - - spin_lock(&t->empty_recvmsg_queue_lock); - list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue); - spin_unlock(&t->empty_recvmsg_queue_lock); -} - static void enqueue_reassembly(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg, int data_length) @@ -369,6 +350,9 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) t->cm_id = cm_id; cm_id->context = t; + t->initiator_depth = SMB_DIRECT_CM_INITIATOR_DEPTH; + t->responder_resources = 1; + t->status = SMB_DIRECT_CS_NEW; init_waitqueue_head(&t->wait_status); @@ -384,16 +368,13 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) spin_lock_init(&t->recvmsg_queue_lock); INIT_LIST_HEAD(&t->recvmsg_queue); - spin_lock_init(&t->empty_recvmsg_queue_lock); - INIT_LIST_HEAD(&t->empty_recvmsg_queue); - init_waitqueue_head(&t->wait_send_pending); atomic_set(&t->send_pending, 0); spin_lock_init(&t->lock_new_recv_credits); - INIT_DELAYED_WORK(&t->post_recv_credits_work, - smb_direct_post_recv_credits); + INIT_WORK(&t->post_recv_credits_work, + smb_direct_post_recv_credits); INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work); INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work); @@ -409,6 +390,11 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) return NULL; } +static void smb_direct_free_transport(struct ksmbd_transport *kt) +{ + kfree(SMBD_TRANS(kt)); +} + static void free_transport(struct smb_direct_transport *t) { struct smb_direct_recvmsg *recvmsg; @@ -420,13 +406,14 @@ static void free_transport(struct smb_direct_transport *t) atomic_read(&t->send_pending) == 0); cancel_work_sync(&t->disconnect_work); - cancel_delayed_work_sync(&t->post_recv_credits_work); + cancel_work_sync(&t->post_recv_credits_work); cancel_work_sync(&t->send_immediate_work); if (t->qp) { ib_drain_qp(t->qp); ib_mr_pool_destroy(t->qp, &t->qp->rdma_mrs); - ib_destroy_qp(t->qp); + t->qp = NULL; + rdma_destroy_qp(t->cm_id); } ksmbd_debug(RDMA, "drain the reassembly queue\n"); @@ -454,7 +441,6 @@ static void free_transport(struct smb_direct_transport *t) smb_direct_destroy_pools(t); ksmbd_conn_free(KSMBD_TRANS(t)->conn); - kfree(t); } static struct smb_direct_sendmsg @@ -541,13 +527,13 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) t = recvmsg->transport; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { + put_recvmsg(t, recvmsg); if (wc->status != IB_WC_WR_FLUSH_ERR) { pr_err("Recv error. status='%s (%d)' opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); smb_direct_disconnect_rdma_connection(t); } - put_empty_recvmsg(t, recvmsg); return; } @@ -561,7 +547,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) switch (recvmsg->type) { case SMB_DIRECT_MSG_NEGOTIATE_REQ: if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) { - put_empty_recvmsg(t, recvmsg); + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); return; } t->negotiation_requested = true; @@ -569,27 +556,39 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) t->status = SMB_DIRECT_CS_CONNECTED; enqueue_reassembly(t, recvmsg, 0); wake_up_interruptible(&t->wait_status); - break; + return; case SMB_DIRECT_MSG_DATA_TRANSFER: { struct smb_direct_data_transfer *data_transfer = (struct smb_direct_data_transfer *)recvmsg->packet; - unsigned int data_length; + u32 remaining_data_length, data_offset, data_length; int avail_recvmsg_count, receive_credits; if (wc->byte_len < offsetof(struct smb_direct_data_transfer, padding)) { - put_empty_recvmsg(t, recvmsg); + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); return; } + remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length); data_length = le32_to_cpu(data_transfer->data_length); - if (data_length) { - if (wc->byte_len < sizeof(struct smb_direct_data_transfer) + - (u64)data_length) { - put_empty_recvmsg(t, recvmsg); - return; - } + data_offset = le32_to_cpu(data_transfer->data_offset); + if (wc->byte_len < data_offset || + wc->byte_len < (u64)data_offset + data_length) { + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); + return; + } + if (remaining_data_length > t->max_fragmented_recv_size || + data_length > t->max_fragmented_recv_size || + (u64)remaining_data_length + (u64)data_length > + (u64)t->max_fragmented_recv_size) { + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); + return; + } + if (data_length) { if (t->full_packet_received) recvmsg->first_segment = true; @@ -598,16 +597,11 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) else t->full_packet_received = true; - enqueue_reassembly(t, recvmsg, (int)data_length); - wake_up_interruptible(&t->wait_reassembly_queue); - spin_lock(&t->receive_credit_lock); receive_credits = --(t->recv_credits); avail_recvmsg_count = t->count_avail_recvmsg; spin_unlock(&t->receive_credit_lock); } else { - put_empty_recvmsg(t, recvmsg); - spin_lock(&t->receive_credit_lock); receive_credits = --(t->recv_credits); avail_recvmsg_count = ++(t->count_avail_recvmsg); @@ -627,13 +621,24 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) wake_up_interruptible(&t->wait_send_credits); if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count)) - mod_delayed_work(smb_direct_wq, - &t->post_recv_credits_work, 0); - break; + queue_work(smb_direct_wq, &t->post_recv_credits_work); + + if (data_length) { + enqueue_reassembly(t, recvmsg, (int)data_length); + wake_up_interruptible(&t->wait_reassembly_queue); + } else + put_recvmsg(t, recvmsg); + + return; } - default: - break; } + + /* + * This is an internal error! + */ + WARN_ON_ONCE(recvmsg->type != SMB_DIRECT_MSG_DATA_TRANSFER); + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); } static int smb_direct_post_recv(struct smb_direct_transport *t, @@ -663,6 +668,7 @@ static int smb_direct_post_recv(struct smb_direct_transport *t, ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, recvmsg->sge.length, DMA_FROM_DEVICE); + recvmsg->sge.length = 0; smb_direct_disconnect_rdma_connection(t); return ret; } @@ -772,8 +778,7 @@ static int smb_direct_read(struct ksmbd_transport *t, char *buf, st->count_avail_recvmsg += queue_removed; if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) { spin_unlock(&st->receive_credit_lock); - mod_delayed_work(smb_direct_wq, - &st->post_recv_credits_work, 0); + queue_work(smb_direct_wq, &st->post_recv_credits_work); } else { spin_unlock(&st->receive_credit_lock); } @@ -800,11 +805,10 @@ static int smb_direct_read(struct ksmbd_transport *t, char *buf, static void smb_direct_post_recv_credits(struct work_struct *work) { struct smb_direct_transport *t = container_of(work, - struct smb_direct_transport, post_recv_credits_work.work); + struct smb_direct_transport, post_recv_credits_work); struct smb_direct_recvmsg *recvmsg; int receive_credits, credits = 0; int ret; - int use_free = 1; spin_lock(&t->receive_credit_lock); receive_credits = t->recv_credits; @@ -812,18 +816,9 @@ static void smb_direct_post_recv_credits(struct work_struct *work) if (receive_credits < t->recv_credit_target) { while (true) { - if (use_free) - recvmsg = get_free_recvmsg(t); - else - recvmsg = get_empty_recvmsg(t); - if (!recvmsg) { - if (use_free) { - use_free = 0; - continue; - } else { - break; - } - } + recvmsg = get_free_recvmsg(t); + if (!recvmsg) + break; recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER; recvmsg->first_segment = false; @@ -943,12 +938,15 @@ static int smb_direct_flush_send_list(struct smb_direct_transport *t, struct smb_direct_sendmsg, list); + if (send_ctx->need_invalidate_rkey) { + first->wr.opcode = IB_WR_SEND_WITH_INV; + first->wr.ex.invalidate_rkey = send_ctx->remote_key; + send_ctx->need_invalidate_rkey = false; + send_ctx->remote_key = 0; + } + last->wr.send_flags = IB_SEND_SIGNALED; last->wr.wr_cqe = &last->cqe; - if (is_last && send_ctx->need_invalidate_rkey) { - last->wr.opcode = IB_WR_SEND_WITH_INV; - last->wr.ex.invalidate_rkey = send_ctx->remote_key; - } ret = smb_direct_post_send(t, &first->wr); if (!ret) { @@ -1635,21 +1633,21 @@ static int smb_direct_send_negotiate_response(struct smb_direct_transport *t, static int smb_direct_accept_client(struct smb_direct_transport *t) { struct rdma_conn_param conn_param; - struct ib_port_immutable port_immutable; - u32 ird_ord_hdr[2]; + __be32 ird_ord_hdr[2]; int ret; + /* + * smb_direct_handle_connect_request() + * already negotiated t->initiator_depth + * and t->responder_resources + */ memset(&conn_param, 0, sizeof(conn_param)); - conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom, - SMB_DIRECT_CM_INITIATOR_DEPTH); - conn_param.responder_resources = 0; - - t->cm_id->device->ops.get_port_immutable(t->cm_id->device, - t->cm_id->port_num, - &port_immutable); - if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) { - ird_ord_hdr[0] = conn_param.responder_resources; - ird_ord_hdr[1] = 1; + conn_param.initiator_depth = t->initiator_depth; + conn_param.responder_resources = t->responder_resources; + + if (t->legacy_iwarp) { + ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); + ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); conn_param.private_data = ird_ord_hdr; conn_param.private_data_len = sizeof(ird_ord_hdr); } else { @@ -1691,7 +1689,7 @@ static int smb_direct_prepare_negotiation(struct smb_direct_transport *t) goto out_err; } - smb_direct_post_recv_credits(&t->post_recv_credits_work.work); + smb_direct_post_recv_credits(&t->post_recv_credits_work); return 0; out_err: put_recvmsg(t, recvmsg); @@ -1799,8 +1797,6 @@ static void smb_direct_destroy_pools(struct smb_direct_transport *t) while ((recvmsg = get_free_recvmsg(t))) mempool_free(recvmsg, t->recvmsg_mempool); - while ((recvmsg = get_empty_recvmsg(t))) - mempool_free(recvmsg, t->recvmsg_mempool); mempool_destroy(t->recvmsg_mempool); t->recvmsg_mempool = NULL; @@ -1856,6 +1852,7 @@ static int smb_direct_create_pools(struct smb_direct_transport *t) if (!recvmsg) goto err; recvmsg->transport = t; + recvmsg->sge.length = 0; list_add(&recvmsg->list, &t->recvmsg_queue); } t->count_avail_recvmsg = t->recv_credit_max; @@ -1934,8 +1931,8 @@ static int smb_direct_create_qpair(struct smb_direct_transport *t, return 0; err: if (t->qp) { - ib_destroy_qp(t->qp); t->qp = NULL; + rdma_destroy_qp(t->cm_id); } if (t->recv_cq) { ib_destroy_cq(t->recv_cq); @@ -2036,10 +2033,13 @@ static bool rdma_frwr_is_supported(struct ib_device_attr *attrs) return true; } -static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id) +static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id, + struct rdma_cm_event *event) { struct smb_direct_transport *t; struct task_struct *handler; + u8 peer_initiator_depth; + u8 peer_responder_resources; int ret; if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) { @@ -2053,6 +2053,67 @@ static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id) if (!t) return -ENOMEM; + peer_initiator_depth = event->param.conn.initiator_depth; + peer_responder_resources = event->param.conn.responder_resources; + if (rdma_protocol_iwarp(new_cm_id->device, new_cm_id->port_num) && + event->param.conn.private_data_len == 8) { + /* + * Legacy clients with only iWarp MPA v1 support + * need a private blob in order to negotiate + * the IRD/ORD values. + */ + const __be32 *ird_ord_hdr = event->param.conn.private_data; + u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); + u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); + + /* + * cifs.ko sends the legacy IRD/ORD negotiation + * event if iWarp MPA v2 was used. + * + * Here we check that the values match and only + * mark the client as legacy if they don't match. + */ + if ((u32)event->param.conn.initiator_depth != ird32 || + (u32)event->param.conn.responder_resources != ord32) { + /* + * There are broken clients (old cifs.ko) + * using little endian and also + * struct rdma_conn_param only uses u8 + * for initiator_depth and responder_resources, + * so we truncate the value to U8_MAX. + * + * smb_direct_accept_client() will then + * do the real negotiation in order to + * select the minimum between client and + * server. + */ + ird32 = min_t(u32, ird32, U8_MAX); + ord32 = min_t(u32, ord32, U8_MAX); + + t->legacy_iwarp = true; + peer_initiator_depth = (u8)ird32; + peer_responder_resources = (u8)ord32; + } + } + + /* + * First set what the we as server are able to support + */ + t->initiator_depth = min_t(u8, t->initiator_depth, + new_cm_id->device->attrs.max_qp_rd_atom); + + /* + * negotiate the value by using the minimum + * between client and server if the client provided + * non 0 values. + */ + if (peer_initiator_depth != 0) + t->initiator_depth = min_t(u8, t->initiator_depth, + peer_initiator_depth); + if (peer_responder_resources != 0) + t->responder_resources = min_t(u8, t->responder_resources, + peer_responder_resources); + ret = smb_direct_connect(t); if (ret) goto out_err; @@ -2077,7 +2138,7 @@ static int smb_direct_listen_handler(struct rdma_cm_id *cm_id, { switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: { - int ret = smb_direct_handle_connect_request(cm_id); + int ret = smb_direct_handle_connect_request(cm_id, event); if (ret) { pr_err("Can't create transport: %d\n", ret); @@ -2214,7 +2275,7 @@ int ksmbd_rdma_init(void) return 0; } -void ksmbd_rdma_destroy(void) +void ksmbd_rdma_stop_listening(void) { if (!smb_direct_listener.cm_id) return; @@ -2223,7 +2284,10 @@ void ksmbd_rdma_destroy(void) rdma_destroy_id(smb_direct_listener.cm_id); smb_direct_listener.cm_id = NULL; +} +void ksmbd_rdma_destroy(void) +{ if (smb_direct_wq) { destroy_workqueue(smb_direct_wq); smb_direct_wq = NULL; @@ -2300,4 +2364,5 @@ static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = { .read = smb_direct_read, .rdma_read = smb_direct_rdma_read, .rdma_write = smb_direct_rdma_write, + .free_transport = smb_direct_free_transport, }; diff --git a/fs/smb/server/transport_rdma.h b/fs/smb/server/transport_rdma.h index 77aee4e5c9dcd..a2291b77488a1 100644 --- a/fs/smb/server/transport_rdma.h +++ b/fs/smb/server/transport_rdma.h @@ -54,13 +54,15 @@ struct smb_direct_data_transfer { #ifdef CONFIG_SMB_SERVER_SMBDIRECT int ksmbd_rdma_init(void); +void ksmbd_rdma_stop_listening(void); void ksmbd_rdma_destroy(void); bool ksmbd_rdma_capable_netdev(struct net_device *netdev); void init_smbd_max_io_size(unsigned int sz); unsigned int get_smbd_max_read_write_size(void); #else static inline int ksmbd_rdma_init(void) { return 0; } -static inline int ksmbd_rdma_destroy(void) { return 0; } +static inline void ksmbd_rdma_stop_listening(void) { } +static inline void ksmbd_rdma_destroy(void) { } static inline bool ksmbd_rdma_capable_netdev(struct net_device *netdev) { return false; } static inline void init_smbd_max_io_size(unsigned int sz) { } static inline unsigned int get_smbd_max_read_write_size(void) { return 0; } diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 2ce7f75059cb3..08275db6446c2 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -87,23 +87,35 @@ static struct tcp_transport *alloc_transport(struct socket *client_sk) return NULL; } +#if IS_ENABLED(CONFIG_IPV6) + if (client_sk->sk->sk_family == AF_INET6) + memcpy(&conn->inet6_addr, &client_sk->sk->sk_v6_daddr, 16); + else + conn->inet_addr = inet_sk(client_sk->sk)->inet_daddr; +#else + conn->inet_addr = inet_sk(client_sk->sk)->inet_daddr; +#endif conn->transport = KSMBD_TRANS(t); KSMBD_TRANS(t)->conn = conn; KSMBD_TRANS(t)->ops = &ksmbd_tcp_transport_ops; return t; } -static void free_transport(struct tcp_transport *t) +static void ksmbd_tcp_free_transport(struct ksmbd_transport *kt) { - kernel_sock_shutdown(t->sock, SHUT_RDWR); - sock_release(t->sock); - t->sock = NULL; + struct tcp_transport *t = TCP_TRANS(kt); - ksmbd_conn_free(KSMBD_TRANS(t)->conn); + sock_release(t->sock); kfree(t->iov); kfree(t); } +static void free_transport(struct tcp_transport *t) +{ + kernel_sock_shutdown(t->sock, SHUT_RDWR); + ksmbd_conn_free(KSMBD_TRANS(t)->conn); +} + /** * kvec_array_init() - initialize a IO vector segment * @new: IO vector to be initialized @@ -226,7 +238,9 @@ static int ksmbd_kthread_fn(void *p) { struct socket *client_sk = NULL; struct interface *iface = (struct interface *)p; + struct ksmbd_conn *conn; int ret; + unsigned int max_ip_conns; while (!kthread_should_stop()) { mutex_lock(&iface->sock_release_lock); @@ -244,6 +258,42 @@ static int ksmbd_kthread_fn(void *p) continue; } + if (!server_conf.max_ip_connections) + goto skip_max_ip_conns_limit; + + /* + * Limits repeated connections from clients with the same IP. + */ + max_ip_conns = 0; + down_read(&conn_list_lock); + list_for_each_entry(conn, &conn_list, conns_list) { +#if IS_ENABLED(CONFIG_IPV6) + if (client_sk->sk->sk_family == AF_INET6) { + if (memcmp(&client_sk->sk->sk_v6_daddr, + &conn->inet6_addr, 16) == 0) + max_ip_conns++; + } else if (inet_sk(client_sk->sk)->inet_daddr == + conn->inet_addr) + max_ip_conns++; +#else + if (inet_sk(client_sk->sk)->inet_daddr == + conn->inet_addr) + max_ip_conns++; +#endif + if (server_conf.max_ip_connections <= max_ip_conns) { + ret = -EAGAIN; + break; + } + } + up_read(&conn_list_lock); + if (ret == -EAGAIN) { + /* Per-IP limit hit: release the just-accepted socket. */ + sock_release(client_sk); + continue; + } + +skip_max_ip_conns_limit: + if (server_conf.max_connections && atomic_inc_return(&active_num_conn) >= server_conf.max_connections) { pr_info_ratelimited("Limit the maximum number of connections(%u)\n", @@ -429,12 +479,13 @@ static int create_socket(struct interface *iface) struct socket *ksmbd_socket; bool ipv4 = false; - ret = sock_create(PF_INET6, SOCK_STREAM, IPPROTO_TCP, &ksmbd_socket); + ret = sock_create_kern(current->nsproxy->net_ns, PF_INET6, SOCK_STREAM, + IPPROTO_TCP, &ksmbd_socket); if (ret) { if (ret != -EAFNOSUPPORT) pr_err("Can't create socket for ipv6, fallback to ipv4: %d\n", ret); - ret = sock_create(PF_INET, SOCK_STREAM, IPPROTO_TCP, - &ksmbd_socket); + ret = sock_create_kern(current->nsproxy->net_ns, PF_INET, + SOCK_STREAM, IPPROTO_TCP, &ksmbd_socket); if (ret) { pr_err("Can't create socket for ipv4: %d\n", ret); goto out_clear; @@ -504,30 +555,37 @@ static int create_socket(struct interface *iface) return ret; } +struct interface *ksmbd_find_netdev_name_iface_list(char *netdev_name) +{ + struct interface *iface; + + list_for_each_entry(iface, &iface_list, entry) + if (!strcmp(iface->name, netdev_name)) + return iface; + return NULL; +} + static int ksmbd_netdev_event(struct notifier_block *nb, unsigned long event, void *ptr) { struct net_device *netdev = netdev_notifier_info_to_dev(ptr); struct interface *iface; - int ret, found = 0; + int ret; switch (event) { case NETDEV_UP: if (netif_is_bridge_port(netdev)) return NOTIFY_OK; - list_for_each_entry(iface, &iface_list, entry) { - if (!strcmp(iface->name, netdev->name)) { - found = 1; - if (iface->state != IFACE_STATE_DOWN) - break; - ret = create_socket(iface); - if (ret) - return NOTIFY_OK; - break; - } + iface = ksmbd_find_netdev_name_iface_list(netdev->name); + if (iface && iface->state == IFACE_STATE_DOWN) { + ksmbd_debug(CONN, "netdev-up event: netdev(%s) is going up\n", + iface->name); + ret = create_socket(iface); + if (ret) + return NOTIFY_OK; } - if (!found && bind_additional_ifaces) { + if (!iface && bind_additional_ifaces) { iface = alloc_iface(kstrdup(netdev->name, GFP_KERNEL)); if (!iface) return NOTIFY_OK; @@ -537,19 +595,19 @@ static int ksmbd_netdev_event(struct notifier_block *nb, unsigned long event, } break; case NETDEV_DOWN: - list_for_each_entry(iface, &iface_list, entry) { - if (!strcmp(iface->name, netdev->name) && - iface->state == IFACE_STATE_CONFIGURED) { - tcp_stop_kthread(iface->ksmbd_kthread); - iface->ksmbd_kthread = NULL; - mutex_lock(&iface->sock_release_lock); - tcp_destroy_socket(iface->ksmbd_socket); - iface->ksmbd_socket = NULL; - mutex_unlock(&iface->sock_release_lock); - - iface->state = IFACE_STATE_DOWN; - break; - } + iface = ksmbd_find_netdev_name_iface_list(netdev->name); + if (iface && iface->state == IFACE_STATE_CONFIGURED) { + ksmbd_debug(CONN, "netdev-down event: netdev(%s) is going down\n", + iface->name); + tcp_stop_kthread(iface->ksmbd_kthread); + iface->ksmbd_kthread = NULL; + mutex_lock(&iface->sock_release_lock); + tcp_destroy_socket(iface->ksmbd_socket); + iface->ksmbd_socket = NULL; + mutex_unlock(&iface->sock_release_lock); + + iface->state = IFACE_STATE_DOWN; + break; } break; } @@ -618,18 +676,6 @@ int ksmbd_tcp_set_interfaces(char *ifc_list, int ifc_list_sz) int sz = 0; if (!ifc_list_sz) { - struct net_device *netdev; - - rtnl_lock(); - for_each_netdev(&init_net, netdev) { - if (netif_is_bridge_port(netdev)) - continue; - if (!alloc_iface(kstrdup(netdev->name, GFP_KERNEL))) { - rtnl_unlock(); - return -ENOMEM; - } - } - rtnl_unlock(); bind_additional_ifaces = 1; return 0; } @@ -655,4 +701,5 @@ static struct ksmbd_transport_ops ksmbd_tcp_transport_ops = { .read = ksmbd_tcp_read, .writev = ksmbd_tcp_writev, .disconnect = ksmbd_tcp_disconnect, + .free_transport = ksmbd_tcp_free_transport, }; diff --git a/fs/smb/server/transport_tcp.h b/fs/smb/server/transport_tcp.h index e338bebe322f1..bf6a3d71f7a04 100644 --- a/fs/smb/server/transport_tcp.h +++ b/fs/smb/server/transport_tcp.h @@ -7,6 +7,8 @@ #define __KSMBD_TRANSPORT_TCP_H__ int ksmbd_tcp_set_interfaces(char *ifc_list, int ifc_list_sz); +void ksmbd_free_transport(struct ksmbd_transport *kt); +struct interface *ksmbd_find_netdev_name_iface_list(char *netdev_name); int ksmbd_tcp_init(void); void ksmbd_tcp_destroy(void); diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index fa5b7e63eb832..067b346921b63 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -426,10 +426,15 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, ksmbd_debug(VFS, "write stream data pos : %llu, count : %zd\n", *pos, count); + if (*pos >= XATTR_SIZE_MAX) { + pr_err("stream write position %lld is out of bounds\n", *pos); + return -EINVAL; + } + size = *pos + count; if (size > XATTR_SIZE_MAX) { size = XATTR_SIZE_MAX; - count = (*pos + count) - XATTR_SIZE_MAX; + count = XATTR_SIZE_MAX - *pos; } v_len = ksmbd_vfs_getcasexattr(idmap, @@ -558,7 +563,8 @@ int ksmbd_vfs_getattr(const struct path *path, struct kstat *stat) { int err; - err = vfs_getattr(path, stat, STATX_BTIME, AT_STATX_SYNC_AS_STAT); + err = vfs_getattr(path, stat, STATX_BASIC_STATS | STATX_BTIME, + AT_STATX_SYNC_AS_STAT); if (err) pr_err("getattr failed, err %d\n", err); return err; @@ -1288,6 +1294,7 @@ int ksmbd_vfs_kern_path_locked(struct ksmbd_work *work, char *name, err = ksmbd_vfs_lock_parent(parent_path->dentry, path->dentry); if (err) { + mnt_drop_write(parent_path->mnt); path_put(path); path_put(parent_path); } diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index 271a23abc82fd..002f0864abee6 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -644,21 +644,40 @@ __close_file_table_ids(struct ksmbd_file_table *ft, bool (*skip)(struct ksmbd_tree_connect *tcon, struct ksmbd_file *fp)) { - unsigned int id; - struct ksmbd_file *fp; - int num = 0; + struct ksmbd_file *fp; + unsigned int id = 0; + int num = 0; + + while (1) { + write_lock(&ft->lock); + fp = idr_get_next(ft->idr, &id); + if (!fp) { + write_unlock(&ft->lock); + break; + } - idr_for_each_entry(ft->idr, fp, id) { - if (skip(tcon, fp)) + if (skip(tcon, fp) || + !atomic_dec_and_test(&fp->refcount)) { + id++; + write_unlock(&ft->lock); continue; + } set_close_state_blocked_works(fp); + idr_remove(ft->idr, fp->volatile_id); + fp->volatile_id = KSMBD_NO_FID; + write_unlock(&ft->lock); + + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); - if (!atomic_dec_and_test(&fp->refcount)) - continue; __ksmbd_close_fd(ft, fp); + num++; + id++; } + return num; } diff --git a/fs/squashfs/inode.c b/fs/squashfs/inode.c index d5918eba27e37..f5dcb8353f862 100644 --- a/fs/squashfs/inode.c +++ b/fs/squashfs/inode.c @@ -140,8 +140,17 @@ int squashfs_read_inode(struct inode *inode, long long ino) if (err < 0) goto failed_read; + inode->i_size = le32_to_cpu(sqsh_ino->file_size); frag = le32_to_cpu(sqsh_ino->fragment); if (frag != SQUASHFS_INVALID_FRAG) { + /* + * the file cannot have a fragment (tailend) and have a + * file size a multiple of the block size + */ + if ((inode->i_size & (msblk->block_size - 1)) == 0) { + err = -EINVAL; + goto failed_read; + } frag_offset = le32_to_cpu(sqsh_ino->offset); frag_size = squashfs_frag_lookup(sb, frag, &frag_blk); if (frag_size < 0) { @@ -155,7 +164,6 @@ int squashfs_read_inode(struct inode *inode, long long ino) } set_nlink(inode, 1); - inode->i_size = le32_to_cpu(sqsh_ino->file_size); inode->i_fop = &generic_ro_fops; inode->i_mode |= S_IFREG; inode->i_blocks = ((inode->i_size - 1) >> 9) + 1; @@ -165,6 +173,7 @@ int squashfs_read_inode(struct inode *inode, long long ino) squashfs_i(inode)->start = le32_to_cpu(sqsh_ino->start_block); squashfs_i(inode)->block_list_start = block; squashfs_i(inode)->offset = offset; + squashfs_i(inode)->parent = 0; inode->i_data.a_ops = &squashfs_aops; TRACE("File inode %x:%x, start_block %llx, block_list_start " @@ -183,8 +192,21 @@ int squashfs_read_inode(struct inode *inode, long long ino) if (err < 0) goto failed_read; + inode->i_size = le64_to_cpu(sqsh_ino->file_size); + if (inode->i_size < 0) { + err = -EINVAL; + goto failed_read; + } frag = le32_to_cpu(sqsh_ino->fragment); if (frag != SQUASHFS_INVALID_FRAG) { + /* + * the file cannot have a fragment (tailend) and have a + * file size a multiple of the block size + */ + if ((inode->i_size & (msblk->block_size - 1)) == 0) { + err = -EINVAL; + goto failed_read; + } frag_offset = le32_to_cpu(sqsh_ino->offset); frag_size = squashfs_frag_lookup(sb, frag, &frag_blk); if (frag_size < 0) { @@ -199,7 +221,6 @@ int squashfs_read_inode(struct inode *inode, long long ino) xattr_id = le32_to_cpu(sqsh_ino->xattr); set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); - inode->i_size = le64_to_cpu(sqsh_ino->file_size); inode->i_op = &squashfs_inode_ops; inode->i_fop = &generic_ro_fops; inode->i_mode |= S_IFREG; @@ -212,6 +233,7 @@ int squashfs_read_inode(struct inode *inode, long long ino) squashfs_i(inode)->start = le64_to_cpu(sqsh_ino->start_block); squashfs_i(inode)->block_list_start = block; squashfs_i(inode)->offset = offset; + squashfs_i(inode)->parent = 0; inode->i_data.a_ops = &squashfs_aops; TRACE("File inode %x:%x, start_block %llx, block_list_start " @@ -292,6 +314,7 @@ int squashfs_read_inode(struct inode *inode, long long ino) inode->i_mode |= S_IFLNK; squashfs_i(inode)->start = block; squashfs_i(inode)->offset = offset; + squashfs_i(inode)->parent = 0; if (type == SQUASHFS_LSYMLINK_TYPE) { __le32 xattr; @@ -329,6 +352,7 @@ int squashfs_read_inode(struct inode *inode, long long ino) set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); rdev = le32_to_cpu(sqsh_ino->rdev); init_special_inode(inode, inode->i_mode, new_decode_dev(rdev)); + squashfs_i(inode)->parent = 0; TRACE("Device inode %x:%x, rdev %x\n", SQUASHFS_INODE_BLK(ino), offset, rdev); @@ -353,6 +377,7 @@ int squashfs_read_inode(struct inode *inode, long long ino) set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); rdev = le32_to_cpu(sqsh_ino->rdev); init_special_inode(inode, inode->i_mode, new_decode_dev(rdev)); + squashfs_i(inode)->parent = 0; TRACE("Device inode %x:%x, rdev %x\n", SQUASHFS_INODE_BLK(ino), offset, rdev); @@ -373,6 +398,7 @@ int squashfs_read_inode(struct inode *inode, long long ino) inode->i_mode |= S_IFSOCK; set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); init_special_inode(inode, inode->i_mode, 0); + squashfs_i(inode)->parent = 0; break; } case SQUASHFS_LFIFO_TYPE: @@ -392,6 +418,7 @@ int squashfs_read_inode(struct inode *inode, long long ino) inode->i_op = &squashfs_inode_ops; set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); init_special_inode(inode, inode->i_mode, 0); + squashfs_i(inode)->parent = 0; break; } default: diff --git a/fs/squashfs/squashfs_fs_i.h b/fs/squashfs/squashfs_fs_i.h index 2c82d6f2a4561..8e497ac07b9a8 100644 --- a/fs/squashfs/squashfs_fs_i.h +++ b/fs/squashfs/squashfs_fs_i.h @@ -16,6 +16,7 @@ struct squashfs_inode_info { u64 xattr; unsigned int xattr_size; int xattr_count; + int parent; union { struct { u64 fragment_block; @@ -27,7 +28,6 @@ struct squashfs_inode_info { u64 dir_idx_start; int dir_idx_offset; int dir_idx_cnt; - int parent; }; }; struct inode vfs_inode; diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c index 641a12eda9292..fc79e0a58864b 100644 --- a/fs/squashfs/super.c +++ b/fs/squashfs/super.c @@ -187,10 +187,15 @@ static int squashfs_fill_super(struct super_block *sb, struct fs_context *fc) unsigned short flags; unsigned int fragments; u64 lookup_table_start, xattr_id_table_start, next_table; - int err; + int err, devblksize = sb_min_blocksize(sb, SQUASHFS_DEVBLK_SIZE); TRACE("Entered squashfs_fill_superblock\n"); + if (!devblksize) { + errorf(fc, "squashfs: unable to set blocksize\n"); + return -EINVAL; + } + sb->s_fs_info = kzalloc(sizeof(*msblk), GFP_KERNEL); if (sb->s_fs_info == NULL) { ERROR("Failed to allocate squashfs_sb_info\n"); @@ -201,7 +206,7 @@ static int squashfs_fill_super(struct super_block *sb, struct fs_context *fc) msblk->panic_on_errors = (opts->errors == Opt_errors_panic); - msblk->devblksize = sb_min_blocksize(sb, SQUASHFS_DEVBLK_SIZE); + msblk->devblksize = devblksize; msblk->devblksize_log2 = ffz(~msblk->devblksize); mutex_init(&msblk->meta_index_mutex); diff --git a/fs/super.c b/fs/super.c index 7cca243b17be6..2d8303d32ccf6 100644 --- a/fs/super.c +++ b/fs/super.c @@ -40,6 +40,8 @@ #include #include "internal.h" +#include + static int thaw_super_locked(struct super_block *sb, enum freeze_holder who); static LIST_HEAD(super_blocks); @@ -692,6 +694,7 @@ void generic_shutdown_super(struct super_block *sb) sb->s_dio_done_wq = NULL; } + trace_android_vh_put_super(sb); if (sop->put_super) sop->put_super(sb); diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 859442b5371e5..56264b3e5e283 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -485,9 +485,20 @@ static int tracefs_d_revalidate(struct dentry *dentry, unsigned int flags) return !(ei && ei->is_freed); } +static int tracefs_d_delete(const struct dentry *dentry) +{ + /* + * We want to keep eventfs dentries around but not tracefs + * ones. eventfs dentries have content in d_fsdata. + * Use d_fsdata to determine if it's a eventfs dentry or not. + */ + return dentry->d_fsdata == NULL; +} + static const struct dentry_operations tracefs_dentry_operations = { .d_revalidate = tracefs_d_revalidate, .d_release = tracefs_d_release, + .d_delete = tracefs_d_delete, }; static int trace_fill_super(struct super_block *sb, void *data, int silent) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 2f73119c7ec98..848c46c5e9d3c 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -2265,6 +2265,9 @@ int udf_current_aext(struct inode *inode, struct extent_position *epos, if (check_add_overflow(sizeof(struct allocExtDesc), le32_to_cpu(header->lengthAllocDescs), &alen)) return -1; + + if (alen > epos->bh->b_size) + return -1; } switch (iinfo->i_alloc_type) { diff --git a/fs/udf/super.c b/fs/udf/super.c index 4494783d18a74..c209d16c0dc7c 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1409,7 +1409,7 @@ static int udf_load_logicalvol(struct super_block *sb, sector_t block, struct genericPartitionMap *gpm; uint16_t ident; struct buffer_head *bh; - unsigned int table_len; + unsigned int table_len, part_map_count; int ret; bh = udf_read_tagged(sb, block, block, &ident); @@ -1430,7 +1430,16 @@ static int udf_load_logicalvol(struct super_block *sb, sector_t block, "logical volume"); if (ret) goto out_bh; - ret = udf_sb_alloc_partition_maps(sb, le32_to_cpu(lvd->numPartitionMaps)); + + part_map_count = le32_to_cpu(lvd->numPartitionMaps); + if (part_map_count > table_len / sizeof(struct genericPartitionMap1)) { + udf_err(sb, "error loading logical volume descriptor: " + "Too many partition maps (%u > %u)\n", part_map_count, + table_len / (unsigned)sizeof(struct genericPartitionMap1)); + ret = -EIO; + goto out_bh; + } + ret = udf_sb_alloc_partition_maps(sb, part_map_count); if (ret) goto out_bh; diff --git a/fs/udf/truncate.c b/fs/udf/truncate.c index 4f33a4a488861..b4071c9cf8c95 100644 --- a/fs/udf/truncate.c +++ b/fs/udf/truncate.c @@ -115,7 +115,7 @@ void udf_truncate_tail_extent(struct inode *inode) } /* This inode entry is in-memory only and thus we don't have to mark * the inode dirty */ - if (ret == 0) + if (ret >= 0) iinfo->i_lenExtents = inode->i_size; brelse(epos.bh); } diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 785bda3f2d0fc..3b9e96a3af0cf 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1755,8 +1755,11 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, user_uffdio_copy = (struct uffdio_copy __user *) arg; ret = -EAGAIN; - if (atomic_read(&ctx->mmap_changing)) + if (unlikely(atomic_read(&ctx->mmap_changing))) { + if (unlikely(put_user(ret, &user_uffdio_copy->copy))) + return -EFAULT; goto out; + } ret = -EFAULT; if (copy_from_user(&uffdio_copy, user_uffdio_copy, @@ -1811,8 +1814,11 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg; ret = -EAGAIN; - if (atomic_read(&ctx->mmap_changing)) + if (unlikely(atomic_read(&ctx->mmap_changing))) { + if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage))) + return -EFAULT; goto out; + } ret = -EFAULT; if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage, @@ -1914,8 +1920,11 @@ static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) user_uffdio_continue = (struct uffdio_continue __user *)arg; ret = -EAGAIN; - if (atomic_read(&ctx->mmap_changing)) + if (unlikely(atomic_read(&ctx->mmap_changing))) { + if (unlikely(put_user(ret, &user_uffdio_continue->mapped))) + return -EFAULT; goto out; + } ret = -EFAULT; if (copy_from_user(&uffdio_continue, user_uffdio_continue, @@ -1971,8 +1980,11 @@ static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long user_uffdio_poison = (struct uffdio_poison __user *)arg; ret = -EAGAIN; - if (atomic_read(&ctx->mmap_changing)) + if (unlikely(atomic_read(&ctx->mmap_changing))) { + if (unlikely(put_user(ret, &user_uffdio_poison->updated))) + return -EFAULT; goto out; + } ret = -EFAULT; if (copy_from_user(&uffdio_poison, user_uffdio_poison, @@ -2035,8 +2047,12 @@ static int userfaultfd_move(struct userfaultfd_ctx *ctx, user_uffdio_move = (struct uffdio_move __user *) arg; - if (atomic_read(&ctx->mmap_changing)) - return -EAGAIN; + ret = -EAGAIN; + if (unlikely(atomic_read(&ctx->mmap_changing))) { + if (unlikely(put_user(ret, &user_uffdio_move->move))) + return -EFAULT; + goto out; + } if (copy_from_user(&uffdio_move, user_uffdio_move, /* don't copy "move" last field */ diff --git a/fs/xattr.c b/fs/xattr.c index a6eb8cbc7b4cd..c49df4a3961d7 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -1291,6 +1291,15 @@ static bool xattr_is_trusted(const char *name) return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN); } +static bool xattr_is_maclabel(const char *name) +{ + const char *suffix = name + XATTR_SECURITY_PREFIX_LEN; + + return !strncmp(name, XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN) && + security_ismaclabel(suffix); +} + /** * simple_xattr_list - list all xattr objects * @inode: inode from which to get the xattrs @@ -1323,6 +1332,18 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, if (err) return err; + err = security_inode_listsecurity(inode, buffer, remaining_size); + if (err < 0) + return err; + + if (buffer) { + if (remaining_size < err) + return -ERANGE; + buffer += err; + } + remaining_size -= err; + err = 0; + read_lock(&xattrs->lock); for (rbp = rb_first(&xattrs->rb_root); rbp; rbp = rb_next(rbp)) { xattr = rb_entry(rbp, struct simple_xattr, rb_node); @@ -1331,6 +1352,10 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, if (!trusted && xattr_is_trusted(xattr->name)) continue; + /* skip MAC labels; these are provided by LSM above */ + if (xattr_is_maclabel(xattr->name)) + continue; + err = xattr_list_one(&buffer, &remaining_size, xattr->name); if (err) break; diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index 54de405cbab5a..4d369876487bd 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -418,6 +418,13 @@ xfs_attr_rmtval_get( dblkcnt = XFS_FSB_TO_BB(mp, map[i].br_blockcount); error = xfs_buf_read(mp->m_ddev_targp, dblkno, dblkcnt, 0, &bp, &xfs_attr3_rmt_buf_ops); + /* + * ENODATA from disk implies a disk medium failure; + * ENODATA for xattrs means attribute not found, so + * disambiguate that here. + */ + if (error == -ENODATA) + error = -EIO; if (error) return error; diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 28bbfc31039c0..1efd45076ee2a 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -2649,6 +2649,12 @@ xfs_da_read_buf( error = xfs_trans_read_buf_map(mp, tp, mp->m_ddev_targp, mapp, nmap, 0, &bp, ops); + /* + * ENODATA from disk implies a disk medium failure; ENODATA for + * xattrs means attribute not found, so disambiguate that here. + */ + if (error == -ENODATA && whichfork == XFS_ATTR_FORK) + error = -EIO; if (error) goto out_free; diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 269573c828085..e267fc9d3108f 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -171,12 +171,40 @@ typedef struct xlog_rec_header { __be32 h_prev_block; /* block number to previous LR : 4 */ __be32 h_num_logops; /* number of log operations in this LR : 4 */ __be32 h_cycle_data[XLOG_HEADER_CYCLE_SIZE / BBSIZE]; - /* new fields */ + + /* fields added by the Linux port: */ __be32 h_fmt; /* format of log record : 4 */ uuid_t h_fs_uuid; /* uuid of FS : 16 */ + + /* fields added for log v2: */ __be32 h_size; /* iclog size : 4 */ + + /* + * When h_size added for log v2 support, it caused structure to have + * a different size on i386 vs all other architectures because the + * sum of the size ofthe member is not aligned by that of the largest + * __be64-sized member, and i386 has really odd struct alignment rules. + * + * Due to the way the log headers are placed out on-disk that alone is + * not a problem becaue the xlog_rec_header always sits alone in a + * BBSIZEs area, and the rest of that area is padded with zeroes. + * But xlog_cksum used to calculate the checksum based on the structure + * size, and thus gives different checksums for i386 vs the rest. + * We now do two checksum validation passes for both sizes to allow + * moving v5 file systems with unclean logs between i386 and other + * (little-endian) architectures. + */ + __u32 h_pad0; } xlog_rec_header_t; +#ifdef __i386__ +#define XLOG_REC_SIZE offsetofend(struct xlog_rec_header, h_size) +#define XLOG_REC_SIZE_OTHER sizeof(struct xlog_rec_header) +#else +#define XLOG_REC_SIZE sizeof(struct xlog_rec_header) +#define XLOG_REC_SIZE_OTHER offsetofend(struct xlog_rec_header, h_size) +#endif /* __i386__ */ + typedef struct xlog_rec_ext_header { __be32 xh_cycle; /* write cycle of log : 4 */ __be32 xh_cycle_data[XLOG_HEADER_CYCLE_SIZE / BBSIZE]; /* : 256 */ diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index 822f5adf7f7cc..b968f7bc202cf 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -20,6 +20,7 @@ #include "xfs_ialloc_btree.h" #include "xfs_rmap.h" #include "xfs_rmap_btree.h" +#include "xfs_refcount.h" #include "xfs_refcount_btree.h" #include "xfs_extent_busy.h" #include "xfs_ag.h" @@ -376,9 +377,21 @@ xreap_agextent_iter( if (crosslinked) { trace_xreap_dispose_unmap_extent(sc->sa.pag, agbno, *aglenp); - rs->force_roll = true; - return xfs_rmap_free(sc->tp, sc->sa.agf_bp, sc->sa.pag, agbno, - *aglenp, rs->oinfo); + if (rs->oinfo == &XFS_RMAP_OINFO_COW) { + /* + * If we're unmapping CoW staging extents, remove the + * records from the refcountbt, which will remove the + * rmap record as well. + */ + xfs_refcount_free_cow_extent(sc->tp, fsbno, *aglenp); + rs->force_roll = true; + return 0; + } + + xfs_rmap_free_extent(sc->tp, sc->sa.pag->pag_agno, agbno, + *aglenp, rs->oinfo->oi_owner); + rs->deferred++; + return 0; } trace_xreap_dispose_free_extent(sc->sa.pag, agbno, *aglenp); diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index f5377ba5967a7..68af6ae3b5d5b 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -422,11 +422,15 @@ xfs_inumbers( .breq = breq, }; struct xfs_trans *tp; + unsigned int iwalk_flags = 0; int error = 0; if (xfs_bulkstat_already_done(breq->mp, breq->startino)) return 0; + if (breq->flags & XFS_IBULK_SAME_AG) + iwalk_flags |= XFS_IWALK_SAME_AG; + /* * Grab an empty transaction so that we can use its recursive buffer * locking abilities to detect cycles in the inobt without deadlocking. @@ -435,7 +439,7 @@ xfs_inumbers( if (error) goto out; - error = xfs_inobt_walk(breq->mp, tp, breq->startino, breq->flags, + error = xfs_inobt_walk(breq->mp, tp, breq->startino, iwalk_flags, xfs_inumbers_walk, breq->icount, &ic); xfs_trans_cancel(tp); out: diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index a1650fc81382f..d03976b821802 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1807,13 +1807,13 @@ xlog_cksum( struct xlog *log, struct xlog_rec_header *rhead, char *dp, - int size) + unsigned int hdrsize, + unsigned int size) { uint32_t crc; /* first generate the crc for the record header ... */ - crc = xfs_start_cksum_update((char *)rhead, - sizeof(struct xlog_rec_header), + crc = xfs_start_cksum_update((char *)rhead, hdrsize, offsetof(struct xlog_rec_header, h_crc)); /* ... then for additional cycle data for v2 logs ... */ @@ -2077,7 +2077,7 @@ xlog_sync( /* calculcate the checksum */ iclog->ic_header.h_crc = xlog_cksum(log, &iclog->ic_header, - iclog->ic_datap, size); + iclog->ic_datap, XLOG_REC_SIZE, size); /* * Intentionally corrupt the log record CRC based on the error injection * frequency, if defined. This facilitates testing log recovery in the diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index e30c06ec20e33..a183bfb36470a 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -503,8 +503,8 @@ xlog_recover_finish( extern void xlog_recover_cancel(struct xlog *); -extern __le32 xlog_cksum(struct xlog *log, struct xlog_rec_header *rhead, - char *dp, int size); +__le32 xlog_cksum(struct xlog *log, struct xlog_rec_header *rhead, + char *dp, unsigned int hdrsize, unsigned int size); extern struct kmem_cache *xfs_log_ticket_cache; struct xlog_ticket *xlog_ticket_alloc(struct xlog *log, int unit_bytes, diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 60382eb499610..f708bf4104b1a 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2860,20 +2860,34 @@ xlog_recover_process( int pass, struct list_head *buffer_list) { - __le32 old_crc = rhead->h_crc; - __le32 crc; + __le32 expected_crc = rhead->h_crc, crc, other_crc; - crc = xlog_cksum(log, rhead, dp, be32_to_cpu(rhead->h_len)); + crc = xlog_cksum(log, rhead, dp, XLOG_REC_SIZE, + be32_to_cpu(rhead->h_len)); + + /* + * Look at the end of the struct xlog_rec_header definition in + * xfs_log_format.h for the glory details. + */ + if (expected_crc && crc != expected_crc) { + other_crc = xlog_cksum(log, rhead, dp, XLOG_REC_SIZE_OTHER, + be32_to_cpu(rhead->h_len)); + if (other_crc == expected_crc) { + xfs_notice_once(log->l_mp, + "Fixing up incorrect CRC due to padding."); + crc = other_crc; + } + } /* * Nothing else to do if this is a CRC verification pass. Just return * if this a record with a non-zero crc. Unfortunately, mkfs always - * sets old_crc to 0 so we must consider this valid even on v5 supers. - * Otherwise, return EFSBADCRC on failure so the callers up the stack - * know precisely what failed. + * sets expected_crc to 0 so we must consider this valid even on v5 + * supers. Otherwise, return EFSBADCRC on failure so the callers up the + * stack know precisely what failed. */ if (pass == XLOG_RECOVER_CRCPASS) { - if (old_crc && crc != old_crc) + if (expected_crc && crc != expected_crc) return -EFSBADCRC; return 0; } @@ -2884,11 +2898,11 @@ xlog_recover_process( * zero CRC check prevents warnings from being emitted when upgrading * the kernel from one that does not add CRCs by default. */ - if (crc != old_crc) { - if (old_crc || xfs_has_crc(log->l_mp)) { + if (crc != expected_crc) { + if (expected_crc || xfs_has_crc(log->l_mp)) { xfs_alert(log->l_mp, "log record CRC mismatch: found 0x%x, expected 0x%x.", - le32_to_cpu(old_crc), + le32_to_cpu(expected_crc), le32_to_cpu(crc)); xfs_hex_dump(dp, 32); } diff --git a/fs/xfs/xfs_ondisk.h b/fs/xfs/xfs_ondisk.h index c4cc99b70dd30..618bf4f03f280 100644 --- a/fs/xfs/xfs_ondisk.h +++ b/fs/xfs/xfs_ondisk.h @@ -143,6 +143,8 @@ xfs_check_ondisk_structs(void) XFS_CHECK_STRUCT_SIZE(struct xfs_rud_log_format, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_map_extent, 32); XFS_CHECK_STRUCT_SIZE(struct xfs_phys_extent, 16); + XFS_CHECK_STRUCT_SIZE(struct xlog_rec_header, 328); + XFS_CHECK_STRUCT_SIZE(struct xlog_rec_ext_header, 260); XFS_CHECK_OFFSET(struct xfs_bui_log_format, bui_extents, 16); XFS_CHECK_OFFSET(struct xfs_cui_log_format, cui_extents, 16); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 581469c43097a..f6b53b1c1da11 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1230,16 +1230,25 @@ suffix_kstrtoint( static inline void xfs_fs_warn_deprecated( struct fs_context *fc, - struct fs_parameter *param, - uint64_t flag, - bool value) + struct fs_parameter *param) { - /* Don't print the warning if reconfiguring and current mount point - * already had the flag set + /* + * Always warn about someone passing in a deprecated mount option. + * Previously we wouldn't print the warning if we were reconfiguring + * and current mount point already had the flag set, but that was not + * the right thing to do. + * + * Many distributions mount the root filesystem with no options in the + * initramfs and rely on mount -a to remount the root fs with the + * options in fstab. However, the old behavior meant that there would + * never be a warning about deprecated mount options for the root fs in + * /etc/fstab. On a single-fs system, that means no warning at all. + * + * Compounding this problem are distribution scripts that copy + * /proc/mounts to fstab, which means that we can't remove mount + * options unless we're 100% sure they have only ever been advertised + * in /proc/mounts in response to explicitly provided mount options. */ - if ((fc->purpose & FS_CONTEXT_FOR_RECONFIGURE) && - !!(XFS_M(fc->root->d_sb)->m_features & flag) == value) - return; xfs_warn(fc->s_fs_info, "%s mount option is deprecated.", param->key); } @@ -1378,19 +1387,19 @@ xfs_fs_parse_param( #endif /* Following mount options will be removed in September 2025 */ case Opt_ikeep: - xfs_fs_warn_deprecated(fc, param, XFS_FEAT_IKEEP, true); + xfs_fs_warn_deprecated(fc, param); parsing_mp->m_features |= XFS_FEAT_IKEEP; return 0; case Opt_noikeep: - xfs_fs_warn_deprecated(fc, param, XFS_FEAT_IKEEP, false); + xfs_fs_warn_deprecated(fc, param); parsing_mp->m_features &= ~XFS_FEAT_IKEEP; return 0; case Opt_attr2: - xfs_fs_warn_deprecated(fc, param, XFS_FEAT_ATTR2, true); + xfs_fs_warn_deprecated(fc, param); parsing_mp->m_features |= XFS_FEAT_ATTR2; return 0; case Opt_noattr2: - xfs_fs_warn_deprecated(fc, param, XFS_FEAT_NOATTR2, true); + xfs_fs_warn_deprecated(fc, param); parsing_mp->m_features |= XFS_FEAT_NOATTR2; return 0; default: diff --git a/include/acpi/acpixf.h b/include/acpi/acpixf.h index 7e9e7e7690436..2b5891515f3d8 100644 --- a/include/acpi/acpixf.h +++ b/include/acpi/acpixf.h @@ -213,6 +213,12 @@ ACPI_INIT_GLOBAL(u8, acpi_gbl_osi_data, 0); */ ACPI_INIT_GLOBAL(u8, acpi_gbl_reduced_hardware, FALSE); +/* + * ACPI Global Lock is mainly used for systems with SMM, so no-SMM systems + * (such as loong_arch) may not have and not use Global Lock. + */ +ACPI_INIT_GLOBAL(u8, acpi_gbl_use_global_lock, TRUE); + /* * Maximum timeout for While() loop iterations before forced method abort. * This mechanism is intended to prevent infinite loops during interpreter diff --git a/include/acpi/actypes.h b/include/acpi/actypes.h index 85c2dcf2b7048..0a24200852415 100644 --- a/include/acpi/actypes.h +++ b/include/acpi/actypes.h @@ -527,7 +527,7 @@ typedef u64 acpi_integer; /* Support for the special RSDP signature (8 characters) */ -#define ACPI_VALIDATE_RSDP_SIG(a) (!strncmp (ACPI_CAST_PTR (char, (a)), ACPI_SIG_RSDP, 8)) +#define ACPI_VALIDATE_RSDP_SIG(a) (!strncmp (ACPI_CAST_PTR (char, (a)), ACPI_SIG_RSDP, (sizeof(a) < 8) ? ACPI_NAMESEG_SIZE : 8)) #define ACPI_MAKE_RSDP_SIG(dest) (memcpy (ACPI_CAST_PTR (char, (dest)), ACPI_SIG_RSDP, 8)) /* Support for OEMx signature (x can be any character) */ diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 941be574bbe00..ca2be8eaba5ea 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -11,6 +11,7 @@ mandatory-y += bitops.h mandatory-y += bug.h mandatory-y += bugs.h mandatory-y += cacheflush.h +mandatory-y += cfi.h mandatory-y += checksum.h mandatory-y += compat.h mandatory-y += current.h diff --git a/include/asm-generic/cfi.h b/include/asm-generic/cfi.h new file mode 100644 index 0000000000000..41fac3537bf94 --- /dev/null +++ b/include/asm-generic/cfi.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_CFI_H +#define __ASM_GENERIC_CFI_H + +#endif /* __ASM_GENERIC_CFI_H */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h old mode 100644 new mode 100755 index cf3f8b9bf43f0..7ddf330f1f819 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -131,6 +131,7 @@ *(__dl_sched_class) \ *(__rt_sched_class) \ *(__fair_sched_class) \ + *(__ext_sched_class) \ *(__idle_sched_class) \ __sched_class_lowest = .; @@ -351,6 +352,7 @@ __start_once = .; \ *(.data..once) \ __end_once = .; \ + *(.data..do_once) \ STRUCT_ALIGN(); \ *(__tracepoints) \ /* implement dynamic printk debug */ \ diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 08b803a4fcde4..2fe6abb2ca803 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -134,6 +134,7 @@ struct af_alg_async_req { * SG? * @enc: Cryptographic operation to be performed when * recvmsg is invoked. + * @write: True if we are in the middle of a write. * @init: True if metadata has been sent. * @len: Length of memory allocated for this data structure. * @inflight: Non-zero when AIO requests are in flight. @@ -149,10 +150,11 @@ struct af_alg_ctx { size_t used; atomic_t rcvused; - bool more; - bool merge; - bool enc; - bool init; + bool more:1, + merge:1, + enc:1, + write:1, + init:1; unsigned int len; diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index ab904d82236fb..0f5f3a6f79f0c 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -44,7 +44,7 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx, sctx->count += len; if (unlikely((partial + len) >= SHA256_BLOCK_SIZE)) { - int blocks; + unsigned int blocks; if (partial) { int p = SHA256_BLOCK_SIZE - partial; diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h index 9a022caacf936..f3e7e3e5078db 100644 --- a/include/drm/drm_atomic.h +++ b/include/drm/drm_atomic.h @@ -372,8 +372,27 @@ struct drm_atomic_state { * * Allow full modeset. This is used by the ATOMIC IOCTL handler to * implement the DRM_MODE_ATOMIC_ALLOW_MODESET flag. Drivers should - * never consult this flag, instead looking at the output of - * drm_atomic_crtc_needs_modeset(). + * generally not consult this flag, but instead look at the output of + * drm_atomic_crtc_needs_modeset(). The detailed rules are: + * + * - Drivers must not consult @allow_modeset in the atomic commit path. + * Use drm_atomic_crtc_needs_modeset() instead. + * + * - Drivers must consult @allow_modeset before adding unrelated struct + * drm_crtc_state to this commit by calling + * drm_atomic_get_crtc_state(). See also the warning in the + * documentation for that function. + * + * - Drivers must never change this flag, it is under the exclusive + * control of userspace. + * + * - Drivers may consult @allow_modeset in the atomic check path, if + * they have the choice between an optimal hardware configuration + * which requires a modeset, and a less optimal configuration which + * can be committed without a modeset. An example would be suboptimal + * scanout FIFO allocation resulting in increased idle power + * consumption. This allows userspace to avoid flickering and delays + * for the normal composition loop at reasonable cost. */ bool allow_modeset : 1; /** diff --git a/include/drm/drm_file.h b/include/drm/drm_file.h index cc61f6a2b2ad6..0cb5edd9c4671 100644 --- a/include/drm/drm_file.h +++ b/include/drm/drm_file.h @@ -302,6 +302,9 @@ struct drm_file { * * Mapping of mm object handles to object pointers. Used by the GEM * subsystem. Protected by @table_lock. + * + * Note that allocated entries might be NULL as a transient state when + * creating or deleting a handle. */ struct idr object_idr; diff --git a/include/drm/drm_framebuffer.h b/include/drm/drm_framebuffer.h index 0dcc07b686548..9622e3df7dfaa 100644 --- a/include/drm/drm_framebuffer.h +++ b/include/drm/drm_framebuffer.h @@ -23,6 +23,7 @@ #ifndef __DRM_FRAMEBUFFER_H__ #define __DRM_FRAMEBUFFER_H__ +#include #include #include #include @@ -100,6 +101,8 @@ struct drm_framebuffer_funcs { unsigned num_clips); }; +#define DRM_FRAMEBUFFER_HAS_HANDLE_REF(_i) BIT(0u + (_i)) + /** * struct drm_framebuffer - frame buffer object * @@ -200,6 +203,12 @@ struct drm_framebuffer { * universal plane. */ int hot_y; +#ifndef __GENKSYMS__ //Placing this in a hole in the structure + /** + * @internal_flags: Framebuffer flags like DRM_FRAMEBUFFER_HAS_HANDLE_REF. + */ + unsigned int internal_flags; +#endif /** * @filp_head: Placed on &drm_file.fbs, protected by &drm_file.fbs_lock. */ diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h index 7c2ec139c464a..5cc02f444a566 100644 --- a/include/drm/drm_gem.h +++ b/include/drm/drm_gem.h @@ -35,6 +35,9 @@ */ #include +#ifndef __GENKSYMS__ +#include +#endif #include #include #include @@ -557,6 +560,18 @@ static inline bool drm_gem_object_is_shared_for_memory_stats(struct drm_gem_obje return (obj->handle_count > 1) || obj->dma_buf; } +/** + * drm_gem_is_imported() - Tests if GEM object's buffer has been imported + * @obj: the GEM object + * + * Returns: + * True if the GEM object's buffer has been imported, false otherwise + */ +static inline bool drm_gem_is_imported(const struct drm_gem_object *obj) +{ + return !!obj->import_attach; +} + #ifdef CONFIG_LOCKDEP /** * drm_gem_gpuva_set_lock() - Set the lock protecting accesses to the gpuva list. diff --git a/include/drm/spsc_queue.h b/include/drm/spsc_queue.h index 125f096c88cb9..ee9df8cc67b73 100644 --- a/include/drm/spsc_queue.h +++ b/include/drm/spsc_queue.h @@ -70,9 +70,11 @@ static inline bool spsc_queue_push(struct spsc_queue *queue, struct spsc_node *n preempt_disable(); + atomic_inc(&queue->job_count); + smp_mb__after_atomic(); + tail = (struct spsc_node **)atomic_long_xchg(&queue->tail, (long)&node->next); WRITE_ONCE(*tail, node); - atomic_inc(&queue->job_count); /* * In case of first element verify new node will be visible to the consumer diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 1b76d2f83eac6..7c6f4006389da 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1098,13 +1098,13 @@ void acpi_os_set_prepare_extended_sleep(int (*func)(u8 sleep_state, acpi_status acpi_os_prepare_extended_sleep(u8 sleep_state, u32 val_a, u32 val_b); -#if defined(CONFIG_SUSPEND) && defined(CONFIG_X86) struct acpi_s2idle_dev_ops { struct list_head list_node; void (*prepare)(void); void (*check)(void); void (*restore)(void); }; +#if defined(CONFIG_SUSPEND) && defined(CONFIG_X86) int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg); void acpi_unregister_lps0_dev(struct acpi_s2idle_dev_ops *arg); int acpi_get_lps0_constraint(struct acpi_device *adev); @@ -1113,6 +1113,13 @@ static inline int acpi_get_lps0_constraint(struct device *dev) { return ACPI_STATE_UNKNOWN; } +static inline int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg) +{ + return -ENODEV; +} +static inline void acpi_unregister_lps0_dev(struct acpi_s2idle_dev_ops *arg) +{ +} #endif /* CONFIG_SUSPEND && CONFIG_X86 */ #ifndef CONFIG_IA64 void arch_reserve_mem_area(acpi_physical_address addr, size_t size); diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 838780aa182ce..0d1efecc5e6c5 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -27,6 +27,13 @@ static inline unsigned long topology_get_cpu_scale(int cpu) void topology_set_cpu_scale(unsigned int cpu, unsigned long capacity); +DECLARE_PER_CPU(unsigned long, capacity_freq_ref); + +static inline unsigned long topology_get_freq_ref(int cpu) +{ + return per_cpu(capacity_freq_ref, cpu); +} + DECLARE_PER_CPU(unsigned long, arch_freq_scale); static inline unsigned long topology_get_freq_scale(int cpu) @@ -93,6 +100,7 @@ void update_siblings_masks(unsigned int cpu); void remove_cpu_topology(unsigned int cpuid); void reset_cpu_topology(void); int parse_acpi_topology(void); +void freq_inv_set_max_ratio(int cpu, u64 max_rate); #endif extern bool topology_update_done; diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h index 255701e1251b4..f652a5028b590 100644 --- a/include/linux/arm_sdei.h +++ b/include/linux/arm_sdei.h @@ -46,12 +46,12 @@ int sdei_unregister_ghes(struct ghes *ghes); /* For use by arch code when CPU hotplug notifiers are not appropriate. */ int sdei_mask_local_cpu(void); int sdei_unmask_local_cpu(void); -void __init sdei_init(void); +void __init acpi_sdei_init(void); void sdei_handler_abort(void); #else static inline int sdei_mask_local_cpu(void) { return 0; } static inline int sdei_unmask_local_cpu(void) { return 0; } -static inline void sdei_init(void) { } +static inline void acpi_sdei_init(void) { } static inline void sdei_handler_abort(void) { } #endif /* CONFIG_ARM_SDE_INTERFACE */ diff --git a/include/linux/array_size.h b/include/linux/array_size.h new file mode 100644 index 0000000000000..06d7d83196ca3 --- /dev/null +++ b/include/linux/array_size.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_ARRAY_SIZE_H +#define _LINUX_ARRAY_SIZE_H + +#include + +/** + * ARRAY_SIZE - get the number of elements in array @arr + * @arr: array to be sized + */ +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr)) + +#endif /* _LINUX_ARRAY_SIZE_H */ diff --git a/include/linux/ata.h b/include/linux/ata.h index 792e10a09787f..c9013e472aa3d 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -566,6 +566,7 @@ struct ata_bmdma_prd { #define ata_id_has_ncq(id) ((id)[ATA_ID_SATA_CAPABILITY] & (1 << 8)) #define ata_id_queue_depth(id) (((id)[ATA_ID_QUEUE_DEPTH] & 0x1f) + 1) #define ata_id_removable(id) ((id)[ATA_ID_CONFIG] & (1 << 7)) +#define ata_id_is_locked(id) (((id)[ATA_ID_DLF] & 0x7) == 0x7) #define ata_id_has_atapi_AN(id) \ ((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \ ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \ diff --git a/include/linux/atmdev.h b/include/linux/atmdev.h index 9b02961d65ee6..70807c679f1ab 100644 --- a/include/linux/atmdev.h +++ b/include/linux/atmdev.h @@ -185,6 +185,7 @@ struct atmdev_ops { /* only send is required */ int (*compat_ioctl)(struct atm_dev *dev,unsigned int cmd, void __user *arg); #endif + int (*pre_send)(struct atm_vcc *vcc, struct sk_buff *skb); int (*send)(struct atm_vcc *vcc,struct sk_buff *skb); int (*send_bh)(struct atm_vcc *vcc, struct sk_buff *skb); int (*send_oam)(struct atm_vcc *vcc,void *cell,int flags); @@ -249,6 +250,12 @@ static inline void atm_account_tx(struct atm_vcc *vcc, struct sk_buff *skb) ATM_SKB(skb)->atm_options = vcc->atm_options; } +static inline void atm_return_tx(struct atm_vcc *vcc, struct sk_buff *skb) +{ + WARN_ON_ONCE(refcount_sub_and_test(ATM_SKB(skb)->acct_truesize, + &sk_atm(vcc)->sk_wmem_alloc)); +} + static inline void atm_force_charge(struct atm_vcc *vcc,int truesize) { atomic_add(truesize, &sk_atm(vcc)->sk_rmem_alloc); diff --git a/include/linux/audit.h b/include/linux/audit.h index 51b1b7054a233..34bddef1f9e8e 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -526,7 +526,7 @@ static inline void audit_log_kern_module(char *name) static inline void audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar) { - if (!audit_dummy_context()) + if (audit_enabled) __audit_fanotify(response, friar); } diff --git a/include/linux/bio.h b/include/linux/bio.h index 0286bada25ce7..f193aef4fac08 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -11,6 +11,7 @@ #include #define BIO_MAX_VECS 256U +#define BIO_MAX_INLINE_VECS UIO_MAXIOV struct queue_limits; @@ -293,7 +294,7 @@ static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio, fi->folio = page_folio(bvec->bv_page); fi->offset = bvec->bv_offset + - PAGE_SIZE * (bvec->bv_page - &fi->folio->page); + PAGE_SIZE * folio_page_idx(fi->folio, bvec->bv_page); fi->_seg_count = bvec->bv_len; fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count); fi->_next = folio_next(fi->folio); diff --git a/include/linux/bitops.h b/include/linux/bitops.h index f7f5a783da2aa..b2342eebc8d22 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -15,7 +15,6 @@ # define aligned_byte_mask(n) (~0xffUL << (BITS_PER_LONG - 8 - 8*(n))) #endif -#define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE) #define BITS_TO_LONGS(nr) __KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(long)) #define BITS_TO_U64(nr) __KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(u64)) #define BITS_TO_U32(nr) __KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(u32)) diff --git a/include/linux/bits.h b/include/linux/bits.h index 7c0cf5031abe8..09e167bc45304 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -11,6 +11,7 @@ #define BIT_ULL_MASK(nr) (ULL(1) << ((nr) % BITS_PER_LONG_LONG)) #define BIT_ULL_WORD(nr) ((nr) / BITS_PER_LONG_LONG) #define BITS_PER_BYTE 8 +#define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE) /* * Create a contiguous bitmask starting at bit position @l and ending at @@ -18,17 +19,50 @@ * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000. */ #if !defined(__ASSEMBLY__) + +/* + * Missing asm support + * + * GENMASK_U*() depend on BITS_PER_TYPE() which relies on sizeof(), + * something not available in asm. Nevertheless, fixed width integers is a C + * concept. Assembly code can rely on the long and long long versions instead. + */ + #include +#include #define GENMASK_INPUT_CHECK(h, l) \ (BUILD_BUG_ON_ZERO(__builtin_choose_expr( \ __is_constexpr((l) > (h)), (l) > (h), 0))) -#else + +/* + * Generate a mask for the specified type @t. Additional checks are made to + * guarantee the value returned fits in that type, relying on + * -Wshift-count-overflow compiler check to detect incompatible arguments. + * For example, all these create build errors or warnings: + * + * - GENMASK(15, 20): wrong argument order + * - GENMASK(72, 15): doesn't fit unsigned long + * - GENMASK_U32(33, 15): doesn't fit in a u32 + */ +#define GENMASK_TYPE(t, h, l) \ + ((t)(GENMASK_INPUT_CHECK(h, l) + \ + (type_max(t) << (l) & \ + type_max(t) >> (BITS_PER_TYPE(t) - 1 - (h))))) + +#define GENMASK_U8(h, l) GENMASK_TYPE(u8, h, l) +#define GENMASK_U16(h, l) GENMASK_TYPE(u16, h, l) +#define GENMASK_U32(h, l) GENMASK_TYPE(u32, h, l) +#define GENMASK_U64(h, l) GENMASK_TYPE(u64, h, l) + +#else /* defined(__ASSEMBLY__) */ + /* * BUILD_BUG_ON_ZERO is not available in h files included from asm files, * disable the input check if that is the case. */ #define GENMASK_INPUT_CHECK(h, l) 0 -#endif + +#endif /* !defined(__ASSEMBLY__) */ #define __GENMASK(h, l) \ (((~UL(0)) - (UL(1) << (l)) + 1) & \ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 6d7856414bb79..88475453b7936 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -527,6 +527,7 @@ static inline bool op_is_zone_mgmt(enum req_op op) { switch (op & REQ_OP_MASK) { case REQ_OP_ZONE_RESET: + case REQ_OP_ZONE_RESET_ALL: case REQ_OP_ZONE_OPEN: case REQ_OP_ZONE_CLOSE: case REQ_OP_ZONE_FINISH: diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index d4f2c8706042c..2331cd8174fe3 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -106,6 +106,7 @@ struct bpf_prog_list { struct bpf_prog *prog; struct bpf_cgroup_link *link; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]; + u32 flags; }; int cgroup_bpf_inherit(struct cgroup *cgrp); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e719857effa1a..47c41a9dacecf 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -29,6 +29,7 @@ #include #include #include +#include #include struct bpf_verifier_env; @@ -679,6 +680,13 @@ enum bpf_type_flag { /* DYNPTR points to xdp_buff */ DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS), + /* Memory must be aligned on some architectures, used in combination with + * MEM_FIXED_SIZE. + */ +#ifndef __GENKSYMS__ + MEM_ALIGNED = BIT(17 + BPF_BASE_TYPE_BITS), +#endif + __BPF_TYPE_FLAG_MAX, __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1, }; @@ -715,8 +723,8 @@ enum bpf_arg_type { ARG_ANYTHING, /* any (initialized) argument is ok */ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ - ARG_PTR_TO_INT, /* pointer to int */ - ARG_PTR_TO_LONG, /* pointer to long */ + ARG_PTR_TO_INT, /* pointer to int ANDROID: NOT USED, kept for ABI stability */ + ARG_PTR_TO_LONG, /* pointer to long ANDROID: NOT USED, kept for ABI stability */ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */ @@ -1056,6 +1064,17 @@ struct btf_func_model { */ #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7) +/* + * Indicate the trampoline should be suitable to receive indirect calls; + * without this indirectly calling the generated code can result in #UD/#CP, + * depending on the CFI options. + * + * Used by bpf_struct_ops. + * + * Incompatible with FENTRY usage, overloads @func_addr argument. + */ +#define BPF_TRAMP_F_INDIRECT BIT(8) + /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. */ @@ -1202,7 +1221,11 @@ struct bpf_dispatcher { ANDROID_KABI_RESERVE(1); }; -static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( +#ifndef __bpfcall +#define __bpfcall __nocfi +#endif + +static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func( const void *ctx, const struct bpf_insn *insnsi, bpf_func_t bpf_func) @@ -1292,7 +1315,7 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func #define DEFINE_BPF_DISPATCHER(name) \ __BPF_DISPATCHER_SC(name); \ - noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ + noinline __bpfcall unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ bpf_func_t bpf_func) \ @@ -1437,6 +1460,9 @@ struct bpf_prog_aux { struct bpf_kfunc_desc_tab *kfunc_tab; struct bpf_kfunc_btf_tab *kfunc_btf_tab; u32 size_poke_tab; +#ifdef CONFIG_FINEIBT + struct bpf_ksym ksym_prefix; +#endif struct bpf_ksym ksym; const struct bpf_prog_ops *ops; struct bpf_map **used_maps; @@ -1663,6 +1689,7 @@ struct bpf_struct_ops { struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; u32 type_id; u32 value_id; + void *cfi_stubs; ANDROID_KABI_RESERVE(1); }; @@ -1677,6 +1704,7 @@ int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, + void *stub_func, void *image, void *image_end); static inline bool bpf_try_module_get(const void *data, struct module *owner) { diff --git a/include/linux/bvec.h b/include/linux/bvec.h index bd1e361b351c5..99ab7b2bba27c 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -57,9 +57,12 @@ static inline void bvec_set_page(struct bio_vec *bv, struct page *page, * @offset: offset into the folio */ static inline void bvec_set_folio(struct bio_vec *bv, struct folio *folio, - unsigned int len, unsigned int offset) + size_t len, size_t offset) { - bvec_set_page(bv, &folio->page, len, offset); + unsigned long nr = offset / PAGE_SIZE; + + WARN_ON_ONCE(len > UINT_MAX); + bvec_set_page(bv, folio_page(folio, nr), len, offset % PAGE_SIZE); } /** diff --git a/include/linux/cfi.h b/include/linux/cfi.h index 3552ec82b7256..aaf34eda22493 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -9,8 +9,10 @@ #include #include +#include #ifdef CONFIG_CFI_CLANG + enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, unsigned long *target, u32 type); @@ -19,6 +21,44 @@ static inline enum bug_trap_type report_cfi_failure_noaddr(struct pt_regs *regs, { return report_cfi_failure(regs, addr, NULL, 0); } + +#ifndef cfi_get_offset +/* + * Returns the CFI prefix offset. By default, the compiler emits only + * a 4-byte CFI type hash before the function. If an architecture + * uses -fpatchable-function-entry=N,M where M>0 to change the prefix + * offset, they must override this function. + */ +static inline int cfi_get_offset(void) +{ + return 4; +} +#endif + +#ifndef cfi_get_func_hash +static inline u32 cfi_get_func_hash(void *func) +{ + u32 hash; + + if (get_kernel_nofault(hash, func - cfi_get_offset())) + return 0; + + return hash; +} +#endif + +/* CFI type hashes for BPF function types */ +extern u32 cfi_bpf_hash; +extern u32 cfi_bpf_subprog_hash; + +#else /* CONFIG_CFI_CLANG */ + +static inline int cfi_get_offset(void) { return 0; } +static inline u32 cfi_get_func_hash(void *func) { return 0; } + +#define cfi_bpf_hash 0U +#define cfi_bpf_subprog_hash 0U + #endif /* CONFIG_CFI_CLANG */ #ifdef CONFIG_ARCH_USES_CFI_TRAPS @@ -38,4 +78,8 @@ static inline void module_cfi_finalize(const Elf_Ehdr *hdr, #endif /* CONFIG_ARCH_USES_CFI_TRAPS */ #endif /* CONFIG_MODULES */ +#ifndef CFI_NOSEAL +#define CFI_NOSEAL(x) +#endif + #endif /* _LINUX_CFI_H */ diff --git a/include/linux/cfi_types.h b/include/linux/cfi_types.h index 6b87136757655..685f7181780f9 100644 --- a/include/linux/cfi_types.h +++ b/include/linux/cfi_types.h @@ -41,5 +41,28 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) #endif +#else /* __ASSEMBLY__ */ + +#ifdef CONFIG_CFI_CLANG +#define DEFINE_CFI_TYPE(name, func) \ + /* \ + * Force a reference to the function so the compiler generates \ + * __kcfi_typeid_. \ + */ \ + __ADDRESSABLE(func); \ + /* u32 name __ro_after_init = __kcfi_typeid_ */ \ + extern u32 name; \ + asm ( \ + " .pushsection .data..ro_after_init,\"aw\",\%progbits \n" \ + " .type " #name ",\%object \n" \ + " .globl " #name " \n" \ + " .p2align 2, 0x0 \n" \ + #name ": \n" \ + " .4byte __kcfi_typeid_" #func " \n" \ + " .size " #name ", 4 \n" \ + " .popsection \n" \ + ); +#endif + #endif /* __ASSEMBLY__ */ #endif /* _LINUX_CFI_TYPES_H */ diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 87c57b18b899b..b04bc9b2fbe9e 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -41,6 +41,24 @@ struct poll_table_struct; /* define the enumeration of all cgroup subsystems */ #define SUBSYS(_x) _x ## _cgrp_id, + +#define CSS_COUNTERS_SIZE (CGROUP_SUBSYS_COUNT * sizeof(atomic_t)) + +/* + * This should just use max(), but max() doesn't work in struct definitions. + * + * Originally, the space was reserved for per cgroup subsystem counters, where each counter was + * the size of an atomic_t variable. However, it was later reused to fit a struct rcu_head + * which is why the calculation considers the size of struct rcu_head. + * + * This macro is provided to ANDROID_BACKPORT_USE_ARRAY() which needs to reserve at least + * enough memory to accommodate struct rcu_head. However, if we only reserve CSS_COUNTERS_SIZE, + * that may not be enough space on kernels with a small amount of cgroup subsystems enabled. So, + * we take the max between the two values to use in ANDROID_BACKPORT_USE_ARRAY(). + */ +#define CGROUP_ROOT_BACKPORT_PADDING_SIZE \ + (CSS_COUNTERS_SIZE > sizeof(struct rcu_head) ? CSS_COUNTERS_SIZE : sizeof(struct rcu_head)) + enum cgroup_subsys_id { #include CGROUP_SUBSYS_COUNT, @@ -388,7 +406,45 @@ struct cgroup_freezer_state { * frozen, SIGSTOPped, and PTRACEd. */ int nr_frozen_tasks; + +}; + +/** + * struct cgroup_kmi_ext_info is meant to hold extensions to struct cgroup while + * maintaining KMI stability. + * + * Hide the definition of struct cgroup_kmi_ext_info from MODVERSIONS so that it + * can be modified to accommodate additional backports in the future. This type + * is meant to be opaque to vendor modules. + */ +#ifdef __GENKSYMS__ +struct cgroup_kmi_ext_info; +#else +struct cgroup_kmi_ext_info { + /* + * Metadata for cgroup v2 freeze time. Writes protected + * by css_set_lock. + */ + struct { + /* Freeze time data consistency protection */ + seqcount_t freeze_seq; + + /* + * Most recent time the cgroup was requested to freeze. + * Accesses guarded by freeze_seq counter. Writes serialized + * by css_set_lock. + */ + u64 freeze_start_nsec; + + /* + * Total duration the cgroup has spent freezing. + * Accesses guarded by freeze_seq counter. Writes serialized + * by css_set_lock. + */ + u64 frozen_nsec; + } freezer; }; +#endif /* __GENKSYMS__ */ struct cgroup { /* self css with NULL ->ss, points back to this cgroup */ @@ -540,7 +596,12 @@ struct cgroup { struct bpf_local_storage __rcu *bpf_cgrp_storage; #endif - ANDROID_BACKPORT_RESERVE(1); + /* + * Used to store KMI-compliant extensions to struct cgroup. + * For further additions, modify the definition for struct + * cgroup_kmi_ext_info. + */ + ANDROID_BACKPORT_USE(1, struct cgroup_kmi_ext_info *kmi_ext_info); /* All ancestors including self */ struct cgroup *ancestors[]; @@ -585,8 +646,12 @@ struct cgroup_root { /* The name for this hierarchy - may be empty */ char name[MAX_CGROUP_ROOT_NAMELEN]; - ANDROID_BACKPORT_USE_ARRAY(1, CGROUP_SUBSYS_COUNT * sizeof(atomic_t), - struct rcu_head rcu); + /* Use the original calculation to preserve the CRC value for the ABI. */ +#ifndef __GENKSYMS__ + ANDROID_BACKPORT_USE_ARRAY(1, CGROUP_ROOT_BACKPORT_PADDING_SIZE, struct rcu_head rcu); +#else + ANDROID_BACKPORT_USE_ARRAY(1, CGROUP_SUBSYS_COUNT * sizeof(atomic_t), struct rcu_head rcu); +#endif }; /* diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f09a1606fa680..97cc4d0a76dee 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -636,6 +636,7 @@ static inline void cgroup_kthread_ready(void) } void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen); +struct cgroup *__cgroup_get_from_id(u64 id); struct cgroup *cgroup_get_from_id(u64 id); #else /* !CONFIG_CGROUPS */ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 9b673fefcef8a..f9de53fff3acc 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -23,23 +23,42 @@ #define KASAN_ABI_VERSION 5 /* + * Clang 22 added preprocessor macros to match GCC, in hopes of eventually + * dropping __has_feature support for sanitizers: + * https://github.com/llvm/llvm-project/commit/568c23bbd3303518c5056d7f03444dae4fdc8a9c + * Create these macros for older versions of clang so that it is easy to clean + * up once the minimum supported version of LLVM for building the kernel always + * creates these macros. + * * Note: Checking __has_feature(*_sanitizer) is only true if the feature is * enabled. Therefore it is not required to additionally check defined(CONFIG_*) * to avoid adding redundant attributes in other configurations. */ +#if __has_feature(address_sanitizer) && !defined(__SANITIZE_ADDRESS__) +#define __SANITIZE_ADDRESS__ +#endif +#if __has_feature(hwaddress_sanitizer) && !defined(__SANITIZE_HWADDRESS__) +#define __SANITIZE_HWADDRESS__ +#endif +#if __has_feature(thread_sanitizer) && !defined(__SANITIZE_THREAD__) +#define __SANITIZE_THREAD__ +#endif -#if __has_feature(address_sanitizer) || __has_feature(hwaddress_sanitizer) -/* Emulate GCC's __SANITIZE_ADDRESS__ flag */ +/* + * Treat __SANITIZE_HWADDRESS__ the same as __SANITIZE_ADDRESS__ in the kernel. + */ +#ifdef __SANITIZE_HWADDRESS__ #define __SANITIZE_ADDRESS__ +#endif + +#ifdef __SANITIZE_ADDRESS__ #define __no_sanitize_address \ __attribute__((no_sanitize("address", "hwaddress"))) #else #define __no_sanitize_address #endif -#if __has_feature(thread_sanitizer) -/* emulate gcc's __SANITIZE_THREAD__ flag */ -#define __SANITIZE_THREAD__ +#ifdef __SANITIZE_THREAD__ #define __no_sanitize_thread \ __attribute__((no_sanitize("thread"))) #else diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 5a4054f17cbc6..1d4f23ea3105d 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -234,14 +234,6 @@ static inline void *offset_to_ptr(const int *off) #define __ADDRESSABLE(sym) \ ___ADDRESSABLE(sym, __section(".discard.addressable")) -#define __ADDRESSABLE_ASM(sym) \ - .pushsection .discard.addressable,"aw"; \ - .align ARCH_SEL(8,4); \ - ARCH_SEL(.quad, .long) __stringify(sym); \ - .popsection; - -#define __ADDRESSABLE_ASM_STR(sym) __stringify(__ADDRESSABLE_ASM(sym)) - /* &a[0] degrades to a pointer: a different type from an array */ #define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0])) @@ -252,6 +244,15 @@ static inline void *offset_to_ptr(const int *off) #define is_signed_type(type) (((type)(-1)) < (__force type)1) #define is_unsigned_type(type) (!is_signed_type(type)) +/* + * Useful shorthand for "is this condition known at compile-time?" + * + * Note that the condition may involve non-constant values, + * but the compiler may know enough about the details of the + * values to determine that the condition is statically true. + */ +#define statically_true(x) (__builtin_constant_p(x) && (x)) + /* * This is needed in functions which generate the stack canary, see * arch/x86/kernel/smpboot.c::start_secondary() for an example. diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 02f616dfb15f4..5c8aca9c34566 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -229,10 +229,9 @@ struct ftrace_likely_data { /* * GCC does not warn about unused static inline functions for -Wunused-function. * Suppress the warning in clang as well by using __maybe_unused, but enable it - * for W=1 build. This will allow clang to find unused functions. Remove the - * __inline_maybe_unused entirely after fixing most of -Wunused-function warnings. + * for W=2 build. This will allow clang to find unused functions. */ -#ifdef KBUILD_EXTRA_WARN1 +#ifdef KBUILD_EXTRA_WARN2 #define __inline_maybe_unused #else #define __inline_maybe_unused __maybe_unused diff --git a/include/linux/console.h b/include/linux/console.h index 7de11c763eb35..38571607065d7 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -36,9 +36,14 @@ enum vc_intensity; /** * struct consw - callbacks for consoles * + * @con_init: initialize the console on @vc. @init is true for the very first + * call on this @vc. + * @con_clear: erase @count characters at [@x, @y] on @vc. @count >= 1. * @con_scroll: move lines from @top to @bottom in direction @dir by @lines. * Return true if no generic handling should be done. * Invoked by csi_M and printing to the console. + * @con_switch: notifier about the console switch; it is supposed to return + * true if a redraw is needed. * @con_set_palette: sets the palette of the console to @table (optional) * @con_scrolldelta: the contents of the console should be scrolled by @lines. * Invoked by user. (optional) @@ -46,10 +51,10 @@ enum vc_intensity; struct consw { struct module *owner; const char *(*con_startup)(void); - void (*con_init)(struct vc_data *vc, int init); + void (*con_init)(struct vc_data *vc, bool init); void (*con_deinit)(struct vc_data *vc); - void (*con_clear)(struct vc_data *vc, int sy, int sx, int height, - int width); + void (*con_clear)(struct vc_data *vc, unsigned int y, + unsigned int x, unsigned int count); void (*con_putc)(struct vc_data *vc, int c, int ypos, int xpos); void (*con_putcs)(struct vc_data *vc, const unsigned short *s, int count, int ypos, int xpos); @@ -57,7 +62,7 @@ struct consw { bool (*con_scroll)(struct vc_data *vc, unsigned int top, unsigned int bottom, enum con_scroll dir, unsigned int lines); - int (*con_switch)(struct vc_data *vc); + bool (*con_switch)(struct vc_data *vc); int (*con_blank)(struct vc_data *vc, int blank, int mode_switch); int (*con_font_set)(struct vc_data *vc, struct console_font *font, unsigned int vpitch, unsigned int flags); diff --git a/include/linux/cpu.h b/include/linux/cpu.h index a7d91a167a8b6..1e3d1b2ac4c1d 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -77,6 +77,10 @@ extern ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf); extern ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf); +extern ssize_t cpu_show_indirect_target_selection(struct device *dev, + struct device_attribute *attr, char *buf); +extern ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf); +extern ssize_t cpu_show_vmscape(struct device *dev, struct device_attribute *attr, char *buf); extern __printf(4, 5) struct device *cpu_device_create(struct device *parent, void *drvdata, diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h old mode 100644 new mode 100755 index 63452733a2f93..b398cce36bc79 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -32,6 +32,9 @@ */ #define CPUFREQ_ETERNAL (-1) + +#define CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS NSEC_PER_MSEC + #define CPUFREQ_NAME_LEN 16 /* Print length for names. Extra 1 space for accommodating '\n' in prints */ #define CPUFREQ_NAME_PLEN (CPUFREQ_NAME_LEN + 1) @@ -44,6 +47,10 @@ enum cpufreq_table_sorting { CPUFREQ_TABLE_SORTED_DESCENDING }; +ssize_t store_scaling_governor(struct cpufreq_policy *policy, + const char *buf, size_t count); +ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf); + struct cpufreq_cpuinfo { unsigned int max_freq; unsigned int min_freq; @@ -1212,6 +1219,7 @@ void arch_set_freq_scale(const struct cpumask *cpus, { } #endif + /* the following are really really optional */ extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs; extern struct freq_attr cpufreq_freq_attr_scaling_boost_freqs; diff --git a/include/linux/device.h b/include/linux/device.h index cad394750ec8d..5b7629c6ec6c4 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -979,6 +979,9 @@ static inline bool device_pm_not_required(struct device *dev) static inline void device_set_pm_not_required(struct device *dev) { dev->power.no_pm = true; +#ifdef CONFIG_PM + dev->power.no_callbacks = true; +#endif } static inline void dev_pm_syscore_device(struct device *dev, bool val) diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index 64d67293d76b3..734ae3c345af0 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -24,6 +24,8 @@ #include #include #include +#include +#include struct device; struct dma_buf; @@ -531,7 +533,13 @@ struct dma_buf { } *sysfs_entry; #endif - ANDROID_KABI_RESERVE(1); + /** + * @nr_task_refs: + * + * The number of tasks that reference this buffer. For calculating PSS. + */ + ANDROID_KABI_USE(1, atomic64_t nr_task_refs); + ANDROID_KABI_RESERVE(2); }; @@ -639,6 +647,63 @@ struct dma_buf_export_info { ANDROID_KABI_RESERVE(2); }; +/** + * struct task_dma_buf_record and struct task_dma_buf_info will NEVER be exposed + * to vendor modules, except possibly via an opaque pointer. Their definitions + * can therefore be hidden from MODVERSIONS CRC machinery, allowing arbitrary + * future changes. + */ +#ifdef __GENKSYMS__ + +struct task_dma_buf_record; +struct task_dma_buf_info; + +#else + +/** + * struct task_dma_buf_record - Holds the number of (VMA and FD) references to a + * dmabuf by a collection of tasks that share both mm_struct and files_struct. + * This is the list entry type for @task_dma_buf_info dmabufs list. + * + * @node: Stores the list this record is on. + * @dmabuf: The dmabuf this record is for. + * @refcnt: The number of VMAs and FDs that reference @dmabuf by the tasks that + * share this record. + */ +struct task_dma_buf_record { + struct list_head node; + struct dma_buf *dmabuf; + unsigned long refcnt; +}; + +/** + * struct task_dma_buf_info - Holds RSS and RSS HWM counters, and a list of + * dmabufs for alltasks that share both mm_struct and files_struct. + * + * @rss: The sum of all dmabuf memory referenced by the task(s) via memory + * mappings or file descriptors in bytes. Buffers referenced more than + * once by the process (multiple mmaps, multiple FDs, or any combination + * of both mmaps and FDs) only cause the buffer to be accounted to the + * process once. Partial mappings cause the full size of the buffer to be + * accounted, regardless of the size of the mapping. + * @rss_hwm: The maximum value of @rss over the lifetime of this struct. (Unless + * reset by userspace.) + * @refcnt: The number of tasks sharing this struct. + * @lock: Lock protecting @rss, @dmabufs, and @dmabuf_count. + * @dmabufs: List of all dmabufs referenced by the task(s). + * @dmabuf_count: The number of elements on the @dmabufs list. + */ +struct task_dma_buf_info { + unsigned long rss; + unsigned long rss_hwm; + refcount_t refcnt; + spinlock_t lock; + struct list_head dmabufs; + unsigned int dmabuf_count; +}; + +#endif + /** * DEFINE_DMA_BUF_EXPORT_INFO - helper macro for exporters * @name: export-info name @@ -691,7 +756,6 @@ dma_buf_attachment_is_dynamic(struct dma_buf_attachment *attach) return !!attach->importer_ops; } -int is_dma_buf_file(struct file *file); int dma_buf_get_each(int (*callback)(const struct dma_buf *dmabuf, void *private), void *private); struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf, @@ -741,4 +805,30 @@ int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map); void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map); long dma_buf_set_name(struct dma_buf *dmabuf, const char *name); int dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags); +struct dma_buf *dma_buf_iter_begin(void); +struct dma_buf *dma_buf_iter_next(struct dma_buf *dmbuf); + +#ifdef CONFIG_DMA_SHARED_BUFFER + +int is_dma_buf_file(struct file *file); +int dma_buf_account_task(struct dma_buf *dmabuf, struct task_dma_buf_info *dmabuf_info); +void dma_buf_unaccount_task(struct dma_buf *dmabuf, struct task_dma_buf_info *dmabuf_info); +int copy_dmabuf_info(u64 clone_flags, struct task_struct *task); +void put_dmabuf_info(struct task_dma_buf_info *dmabuf_info); +int dma_buf_begin_new_exec(struct files_struct *old_files); + +#else /* CONFIG_DMA_SHARED_BUFFER */ + +static inline int is_dma_buf_file(struct file *file) { return 0; } +static inline int dma_buf_account_task(struct dma_buf *dmabuf, + struct task_dma_buf_info *dmabuf_info) { return 0; } +static inline void dma_buf_unaccount_task(struct dma_buf *dmabuf, + struct task_dma_buf_info *dmabuf_info) {} +static inline int copy_dmabuf_info(u64 clone_flags, + struct task_struct *task) { return 0; } +static inline void put_dmabuf_info(struct task_dma_buf_info *dmabuf_info) {} +static inline int dma_buf_begin_new_exec(struct files_struct *old_files) { return 0; } + +#endif /* CONFIG_DMA_SHARED_BUFFER */ + #endif /* __DMA_BUF_H__ */ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index f0ccca16a0aca..608e8296ba206 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -600,10 +600,14 @@ static inline int dma_mmap_wc(struct device *dev, #else #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME) #define DEFINE_DMA_UNMAP_LEN(LEN_NAME) -#define dma_unmap_addr(PTR, ADDR_NAME) (0) -#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) do { } while (0) -#define dma_unmap_len(PTR, LEN_NAME) (0) -#define dma_unmap_len_set(PTR, LEN_NAME, VAL) do { } while (0) +#define dma_unmap_addr(PTR, ADDR_NAME) \ + ({ typeof(PTR) __p __maybe_unused = PTR; 0; }) +#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) \ + do { typeof(PTR) __p __maybe_unused = PTR; } while (0) +#define dma_unmap_len(PTR, LEN_NAME) \ + ({ typeof(PTR) __p __maybe_unused = PTR; 0; }) +#define dma_unmap_len_set(PTR, LEN_NAME, VAL) \ + do { typeof(PTR) __p __maybe_unused = PTR; } while (0) #endif #endif /* _LINUX_DMA_MAPPING_H */ diff --git a/include/linux/export.h b/include/linux/export.h index 9911508a9604f..06f7a4eb64928 100644 --- a/include/linux/export.h +++ b/include/linux/export.h @@ -42,11 +42,17 @@ extern struct module __this_module; .long sym #endif -#define ___EXPORT_SYMBOL(sym, license, ns) \ +/* + * LLVM integrated assembler cam merge adjacent string literals (like + * C and GNU-as) passed to '.ascii', but not to '.asciz' and chokes on: + * + * .asciz "MODULE_" "kvm" ; + */ +#define ___EXPORT_SYMBOL(sym, license, ns...) \ .section ".export_symbol","a" ASM_NL \ __export_symbol_##sym: ASM_NL \ .asciz license ASM_NL \ - .asciz ns ASM_NL \ + .ascii ns "\0" ASM_NL \ __EXPORT_SYMBOL_REF(sym) ASM_NL \ .previous @@ -88,4 +94,6 @@ extern struct module __this_module; #define EXPORT_SYMBOL_NS(sym, ns) __EXPORT_SYMBOL(sym, "", __stringify(ns)) #define EXPORT_SYMBOL_NS_GPL(sym, ns) __EXPORT_SYMBOL(sym, "GPL", __stringify(ns)) +#define EXPORT_SYMBOL_GPL_FOR_MODULES(sym, mods) __EXPORT_SYMBOL(sym, "GPL", "module:" mods) + #endif /* _LINUX_EXPORT_H */ diff --git a/include/linux/fbcon.h b/include/linux/fbcon.h index 2382dec6d6ab8..fb0fc2736b801 100644 --- a/include/linux/fbcon.h +++ b/include/linux/fbcon.h @@ -11,6 +11,7 @@ void fbcon_suspended(struct fb_info *info); void fbcon_resumed(struct fb_info *info); int fbcon_mode_deleted(struct fb_info *info, struct fb_videomode *mode); +void fbcon_delete_modelist(struct list_head *head); void fbcon_new_modelist(struct fb_info *info); void fbcon_get_requirement(struct fb_info *info, struct fb_blit_caps *caps); @@ -31,6 +32,7 @@ static inline void fbcon_suspended(struct fb_info *info) {} static inline void fbcon_resumed(struct fb_info *info) {} static inline int fbcon_mode_deleted(struct fb_info *info, struct fb_videomode *mode) { return 0; } +static inline void fbcon_delete_modelist(struct list_head *head) {} static inline void fbcon_new_modelist(struct fb_info *info) {} static inline void fbcon_get_requirement(struct fb_info *info, struct fb_blit_caps *caps) {} diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h index fcff696860b8f..c31a8910c3833 100644 --- a/include/linux/fdtable.h +++ b/include/linux/fdtable.h @@ -55,6 +55,9 @@ struct files_struct { struct fdtable __rcu *fdt; struct fdtable fdtab; +#ifndef __GENKSYMS__ + struct task_dma_buf_info *dmabuf_info; +#endif /* * written part on a separate cache line in SMP */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 5090e940ba3e4..a345405ea6011 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -685,6 +685,26 @@ static inline void bpf_compute_data_pointers(struct sk_buff *skb) cb->data_end = skb->data + skb_headlen(skb); } +static inline int bpf_prog_run_data_pointers( + const struct bpf_prog *prog, + struct sk_buff *skb) +{ + struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; + void *save_data_meta, *save_data_end; + int res; + + save_data_meta = cb->data_meta; + save_data_end = cb->data_end; + + bpf_compute_data_pointers(skb); + res = bpf_prog_run(prog, skb); + + cb->data_meta = save_data_meta; + cb->data_end = save_data_end; + + return res; +} + /* Similar to bpf_compute_data_pointers(), except that save orginal * data in cb->data and cb->meta_data for restore. */ @@ -1065,7 +1085,7 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other); static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen, u32 pass, void *image) { - pr_err("flen=%u proglen=%u pass=%u image=%pK from=%s pid=%d\n", flen, + pr_err("flen=%u proglen=%u pass=%u image=%p from=%s pid=%d\n", flen, proglen, pass, image, current->comm, task_pid_nr(current)); if (image) diff --git a/include/linux/freezer.h b/include/linux/freezer.h index b303472255be4..32884c9721e56 100644 --- a/include/linux/freezer.h +++ b/include/linux/freezer.h @@ -47,6 +47,7 @@ extern int freeze_processes(void); extern int freeze_kernel_threads(void); extern void thaw_processes(void); extern void thaw_kernel_threads(void); +extern void thaw_process(struct task_struct *p); static inline bool try_to_freeze(void) { @@ -80,6 +81,7 @@ static inline int freeze_processes(void) { return -ENOSYS; } static inline int freeze_kernel_threads(void) { return -ENOSYS; } static inline void thaw_processes(void) {} static inline void thaw_kernel_threads(void) {} +static inline void thaw_process(struct task_struct *p) {} static inline bool try_to_freeze(void) { return false; } diff --git a/include/linux/fs.h b/include/linux/fs.h index 41477536a1639..6065d6d85e183 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -444,7 +444,7 @@ extern const struct address_space_operations empty_aops; * It is also used to block modification of page cache contents through * memory mappings. * @gfp_mask: Memory allocation flags to use for allocating pages. - * @i_mmap_writable: Number of VM_SHARED mappings. + * @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings. * @nr_thps: Number of THPs in the pagecache (non-shmem only). * @i_mmap: Tree of private and shared mappings. * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable. @@ -547,7 +547,7 @@ static inline int mapping_mapped(struct address_space *mapping) /* * Might pages of this file have been modified in userspace? - * Note that i_mmap_writable counts all VM_SHARED vmas: do_mmap + * Note that i_mmap_writable counts all VM_SHARED, VM_MAYWRITE vmas: do_mmap * marks vma as VM_SHARED if it is shared, and the file was opened for * writing i.e. vma may be mprotected writable even if now readonly. * @@ -3174,6 +3174,8 @@ extern int simple_write_begin(struct file *file, struct address_space *mapping, extern const struct address_space_operations ram_aops; extern int always_delete_dentry(const struct dentry *); extern struct inode *alloc_anon_inode(struct super_block *); +struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *name, + const struct inode *context_inode); extern int simple_nosetlease(struct file *, int, struct file_lock **, void **); extern const struct dentry_operations simple_dentry_operations; diff --git a/include/linux/fs_context.h b/include/linux/fs_context.h index c13e99cbbf816..c9eae71170013 100644 --- a/include/linux/fs_context.h +++ b/include/linux/fs_context.h @@ -196,7 +196,7 @@ void logfc(struct fc_log *log, const char *prefix, char level, const char *fmt, */ #define infof(fc, fmt, ...) __logfc(fc, 'i', fmt, ## __VA_ARGS__) #define info_plog(p, fmt, ...) __plog(p, 'i', fmt, ## __VA_ARGS__) -#define infofc(p, fmt, ...) __plog((&(fc)->log), 'i', fmt, ## __VA_ARGS__) +#define infofc(fc, fmt, ...) __plog((&(fc)->log), 'i', fmt, ## __VA_ARGS__) /** * warnf - Store supplementary warning message diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 43dc4463388b4..808caa5c170cd 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -139,9 +139,9 @@ static inline enum zone_type __gfp_zone(gfp_t flags) if (z == ZONE_MOVABLE) return LAST_VIRT_ZONE; - /* Allow dma-buf etc to use virtual zones */ + /* Allow dma-buf etc to use virtual zones, if there is no movable zone */ if ((flags & __GFP_COMP) && (flags & __GFP_HIGHMEM) && - !static_branch_unlikely(&movablecore_enabled)) + !static_branch_unlikely(&movablecore_enabled) && !movable_node_is_enabled()) return LAST_VIRT_ZONE; return z; diff --git a/include/linux/gpio/regmap.h b/include/linux/gpio/regmap.h index a9f7b7faf57b0..cf55202aaec26 100644 --- a/include/linux/gpio/regmap.h +++ b/include/linux/gpio/regmap.h @@ -37,9 +37,18 @@ struct regmap; * offset to a register/bitmask pair. If not * given the default gpio_regmap_simple_xlate() * is used. + * @fixed_direction_output: + * (Optional) Bitmap representing the fixed direction of + * the GPIO lines. Useful when there are GPIO lines with a + * fixed direction mixed together in the same register. * @drvdata: (Optional) Pointer to driver specific data which is * not used by gpio-remap but is provided "as is" to the * driver callback(s). + * @regmap_irq_chip: (Optional) Pointer on an regmap_irq_chip structure. If + * set, a regmap-irq device will be created and the IRQ + * domain will be set accordingly. + * @regmap_irq_line (Optional) The IRQ the device uses to signal interrupts. + * @regmap_irq_flags (Optional) The IRQF_ flags to use for the interrupt. * * The ->reg_mask_xlate translates a given base address and GPIO offset to * register and mask pair. The base address is one of the given register @@ -77,6 +86,13 @@ struct gpio_regmap_config { int reg_stride; int ngpio_per_reg; struct irq_domain *irq_domain; + unsigned long *fixed_direction_output; + +#ifdef CONFIG_REGMAP_IRQ + struct regmap_irq_chip *regmap_irq_chip; + int regmap_irq_line; + unsigned long regmap_irq_flags; +#endif int (*reg_mask_xlate)(struct gpio_regmap *gpio, unsigned int base, unsigned int offset, unsigned int *reg, diff --git a/include/linux/gunyah.h b/include/linux/gunyah.h index dbd5b0251b491..30696e36fe021 100644 --- a/include/linux/gunyah.h +++ b/include/linux/gunyah.h @@ -463,6 +463,8 @@ struct gunyah_hypercall_vcpu_run_resp { GUNYAH_VCPU_ADDRSPACE_VMMIO_WRITE = 5, /* VCPU blocked on fault where we can demand page */ GUNYAH_VCPU_ADDRSPACE_PAGE_FAULT = 7, + /* VCPU is powered off due to some system event/reset */ + GUNYAH_VCPU_STATE_SYSTEM_OFF = 0x100, /* clang-format on */ } state; u64 sized_state; diff --git a/include/linux/hid.h b/include/linux/hid.h index b1e6e06cd855e..26e0098083b47 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -744,8 +744,9 @@ struct hid_descriptor { __le16 bcdHID; __u8 bCountryCode; __u8 bNumDescriptors; + struct hid_class_descriptor rpt_desc; - struct hid_class_descriptor desc[1]; + struct hid_class_descriptor opt_descs[]; } __attribute__ ((packed)); #define HID_DEVICE(b, g, ven, prod) \ diff --git a/include/linux/highmem.h b/include/linux/highmem.h index ccef475c0d444..05019f1e48e43 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -448,7 +448,7 @@ static inline void memcpy_from_folio(char *to, struct folio *folio, const char *from = kmap_local_folio(folio, offset); size_t chunk = len; - if (folio_test_highmem(folio) && + if (folio_test_partial_kmap(folio) && chunk > PAGE_SIZE - offset_in_page(offset)) chunk = PAGE_SIZE - offset_in_page(offset); memcpy(to, from, chunk); @@ -469,7 +469,7 @@ static inline void memcpy_to_folio(struct folio *folio, size_t offset, char *to = kmap_local_folio(folio, offset); size_t chunk = len; - if (folio_test_highmem(folio) && + if (folio_test_partial_kmap(folio) && chunk > PAGE_SIZE - offset_in_page(offset)) chunk = PAGE_SIZE - offset_in_page(offset); memcpy(to, from, chunk); @@ -501,7 +501,7 @@ static inline size_t memcpy_from_file_folio(char *to, struct folio *folio, size_t offset = offset_in_folio(folio, pos); char *from = kmap_local_folio(folio, offset); - if (folio_test_highmem(folio)) { + if (folio_test_partial_kmap(folio)) { offset = offset_in_page(offset); len = min_t(size_t, len, PAGE_SIZE - offset); } else diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index fc2023d07f693..8b051b8c40345 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -281,6 +281,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, bool is_hugetlb_entry_migration(pte_t pte); void hugetlb_unshare_all_pmds(struct vm_area_struct *vma); +void hugetlb_split(struct vm_area_struct *vma, unsigned long addr); #else /* !CONFIG_HUGETLB_PAGE */ @@ -491,6 +492,8 @@ static inline vm_fault_t hugetlb_fault(struct mm_struct *mm, static inline void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { } +static inline void hugetlb_split(struct vm_area_struct *vma, unsigned long addr) {} + #endif /* !CONFIG_HUGETLB_PAGE */ /* * hugepages at page global directory. If arch support diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 3e7fc90547898..e4ad9760774e1 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -820,6 +820,8 @@ struct vmbus_requestor { #define VMBUS_RQST_RESET (U64_MAX - 3) struct vmbus_device { + /* preferred ring buffer size in KB, 0 means no preferred size for this device */ + size_t pref_ring_size; u16 dev_type; guid_t guid; bool perf_device; @@ -1224,13 +1226,6 @@ extern int vmbus_sendpacket(struct vmbus_channel *channel, enum vmbus_packet_type type, u32 flags); -extern int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel, - struct hv_page_buffer pagebuffers[], - u32 pagecount, - void *buffer, - u32 bufferlen, - u64 requestid); - extern int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, struct vmbus_packet_mpb_array *mpb, u32 desc_size, diff --git a/include/linux/hypervisor.h b/include/linux/hypervisor.h index 9efbc54e35e59..be5417303ecf6 100644 --- a/include/linux/hypervisor.h +++ b/include/linux/hypervisor.h @@ -37,6 +37,9 @@ static inline bool hypervisor_isolated_pci_functions(void) if (IS_ENABLED(CONFIG_S390)) return true; + if (IS_ENABLED(CONFIG_LOONGARCH)) + return true; + return jailhouse_paravirt(); } diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index f537228a84954..c048f9512ac4d 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2687,7 +2687,6 @@ static inline bool ieee80211_he_capa_size_ok(const u8 *data, u8 len) #define IEEE80211_6GHZ_CTRL_REG_LPI_AP 0 #define IEEE80211_6GHZ_CTRL_REG_SP_AP 1 -#define IEEE80211_6GHZ_CTRL_REG_VLP_AP 2 /** * struct ieee80211_he_6ghz_oper - HE 6 GHz operation Information field diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 430749a0f362a..272d9ad739607 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -79,11 +79,6 @@ static inline struct vlan_ethhdr *skb_vlan_eth_hdr(const struct sk_buff *skb) /* found in socket.c */ extern void vlan_ioctl_set(int (*hook)(struct net *, void __user *)); -static inline bool is_vlan_dev(const struct net_device *dev) -{ - return dev->priv_flags & IFF_802_1Q_VLAN; -} - #define skb_vlan_tag_present(__skb) (!!(__skb)->vlan_all) #define skb_vlan_tag_get(__skb) ((__skb)->vlan_tci) #define skb_vlan_tag_get_id(__skb) ((__skb)->vlan_tci & VLAN_VID_MASK) @@ -199,6 +194,11 @@ struct vlan_dev_priv { #endif }; +static inline bool is_vlan_dev(const struct net_device *dev) +{ + return dev->priv_flags & IFF_802_1Q_VLAN; +} + static inline struct vlan_dev_priv *vlan_dev_priv(const struct net_device *dev) { return netdev_priv(dev); @@ -236,6 +236,11 @@ extern void vlan_vids_del_by_dev(struct net_device *dev, extern bool vlan_uses_dev(const struct net_device *dev); #else +static inline bool is_vlan_dev(const struct net_device *dev) +{ + return false; +} + static inline struct net_device * __vlan_find_dev_deep_rcu(struct net_device *real_dev, __be16 vlan_proto, u16 vlan_id) @@ -253,19 +258,19 @@ vlan_for_each(struct net_device *dev, static inline struct net_device *vlan_dev_real_dev(const struct net_device *dev) { - BUG(); + WARN_ON_ONCE(1); return NULL; } static inline u16 vlan_dev_vlan_id(const struct net_device *dev) { - BUG(); + WARN_ON_ONCE(1); return 0; } static inline __be16 vlan_dev_vlan_proto(const struct net_device *dev) { - BUG(); + WARN_ON_ONCE(1); return 0; } diff --git a/include/linux/iio/frequency/adf4350.h b/include/linux/iio/frequency/adf4350.h index de45cf2ee1e4f..ce2086f97e3fc 100644 --- a/include/linux/iio/frequency/adf4350.h +++ b/include/linux/iio/frequency/adf4350.h @@ -51,7 +51,7 @@ /* REG3 Bit Definitions */ #define ADF4350_REG3_12BIT_CLKDIV(x) ((x) << 3) -#define ADF4350_REG3_12BIT_CLKDIV_MODE(x) ((x) << 16) +#define ADF4350_REG3_12BIT_CLKDIV_MODE(x) ((x) << 15) #define ADF4350_REG3_12BIT_CSR_EN (1 << 18) #define ADF4351_REG3_CHARGE_CANCELLATION_EN (1 << 21) #define ADF4351_REG3_ANTI_BACKLASH_3ns_EN (1 << 22) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 1838dee484ec8..591a7479503a0 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -290,6 +290,9 @@ struct io_ring_ctx { atomic_t cq_wait_nr; atomic_t cq_timeouts; struct wait_queue_head cq_wait; +#ifndef __GENKSYMS__ + struct llist_head retry_llist; +#endif } ____cacheline_aligned_in_smp; /* timeouts */ @@ -420,6 +423,7 @@ enum { /* keep async read/write and isreg together and in order */ REQ_F_SUPPORT_NOWAIT_BIT, REQ_F_ISREG_BIT, + REQ_F_CAN_POLL_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -487,6 +491,8 @@ enum { REQ_F_CLEAR_POLLIN = BIT(REQ_F_CLEAR_POLLIN_BIT), /* hashed into ->cancel_hash_locked, protected by ->uring_lock */ REQ_F_HASH_LOCKED = BIT(REQ_F_HASH_LOCKED_BIT), + /* file is pollable */ + REQ_F_CAN_POLL = BIT(REQ_F_CAN_POLL_BIT), }; typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts); diff --git a/include/linux/iosys-map.h b/include/linux/iosys-map.h index cb71aa616bd37..631d58d0b8385 100644 --- a/include/linux/iosys-map.h +++ b/include/linux/iosys-map.h @@ -264,12 +264,7 @@ static inline bool iosys_map_is_set(const struct iosys_map *map) */ static inline void iosys_map_clear(struct iosys_map *map) { - if (map->is_iomem) { - map->vaddr_iomem = NULL; - map->is_iomem = false; - } else { - map->vaddr = NULL; - } + memset(map, 0, sizeof(*map)); } /** diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index d03a893897669..91a6617299546 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -90,7 +90,7 @@ struct ipv6_devconf { ANDROID_KABI_RESERVE(1); ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); - ANDROID_KABI_BACKPORT_OK(4); + ANDROID_KABI_BACKPORT_USE(4, struct { __u8 ra_honor_pio_pflag; __u8 padding4[7]; }); }; struct ipv6_params { diff --git a/include/linux/ism.h b/include/linux/ism.h index 9a4c204df3da1..04e2fc1973ce4 100644 --- a/include/linux/ism.h +++ b/include/linux/ism.h @@ -28,6 +28,7 @@ struct ism_dmb { struct ism_dev { spinlock_t lock; /* protects the ism device */ + spinlock_t cmd_lock; /* serializes cmds */ struct list_head list; struct pci_dev *pdev; diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 5447c1d8c81ec..e0531145e63ea 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -50,12 +51,6 @@ #define READ 0 #define WRITE 1 -/** - * ARRAY_SIZE - get the number of elements in array @arr - * @arr: array to be sized - */ -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr)) - #define PTR_IF(cond, ptr) ((cond) ? (ptr) : NULL) #define u64_to_user_ptr(x) ( \ diff --git a/include/linux/kernel_read_file.h b/include/linux/kernel_read_file.h index 90451e2e12bd1..c608ea992bbc8 100644 --- a/include/linux/kernel_read_file.h +++ b/include/linux/kernel_read_file.h @@ -52,4 +52,6 @@ ssize_t kernel_read_file_from_fd(int fd, loff_t offset, size_t *file_size, enum kernel_read_file_id id); +ssize_t read_comp_algo_dictionary(void **dict, const char *dict_path); + #endif /* _LINUX_KERNEL_READ_FILE_H */ diff --git a/include/linux/ksm.h b/include/linux/ksm.h index f701b57fc64bf..05f2f3364b5f7 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -59,6 +59,12 @@ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) int ret; if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) { + long nr_ksm_zero_pages = atomic_long_read(&mm->ksm_zero_pages); + + mm->ksm_merging_pages = 0; + mm->ksm_rmap_items = 0; + atomic_long_add(nr_ksm_zero_pages, &ksm_zero_pages); + ret = __ksm_enter(mm); if (ret) return ret; diff --git a/include/linux/libata.h b/include/linux/libata.h index 91c4e11cb6abb..285d709cbbde4 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -1305,7 +1305,7 @@ int ata_acpi_stm(struct ata_port *ap, const struct ata_acpi_gtm *stm); int ata_acpi_gtm(struct ata_port *ap, struct ata_acpi_gtm *stm); unsigned int ata_acpi_gtm_xfermask(struct ata_device *dev, const struct ata_acpi_gtm *gtm); -int ata_acpi_cbl_80wire(struct ata_port *ap, const struct ata_acpi_gtm *gtm); +int ata_acpi_cbl_pata_type(struct ata_port *ap); #else static inline const struct ata_acpi_gtm *ata_acpi_init_gtm(struct ata_port *ap) { @@ -1330,10 +1330,9 @@ static inline unsigned int ata_acpi_gtm_xfermask(struct ata_device *dev, return 0; } -static inline int ata_acpi_cbl_80wire(struct ata_port *ap, - const struct ata_acpi_gtm *gtm) +static inline int ata_acpi_cbl_pata_type(struct ata_port *ap) { - return 0; + return ATA_CBL_PATA40; } #endif diff --git a/include/linux/local_lock.h b/include/linux/local_lock.h index e55010fa73296..16a2ee4f8310b 100644 --- a/include/linux/local_lock.h +++ b/include/linux/local_lock.h @@ -51,4 +51,53 @@ #define local_unlock_irqrestore(lock, flags) \ __local_unlock_irqrestore(lock, flags) +/** + * local_lock_init - Runtime initialize a lock instance + */ +#define local_trylock_init(lock) __local_trylock_init(lock) + +/** + * local_trylock - Try to acquire a per CPU local lock + * @lock: The lock variable + * + * The function can be used in any context such as NMI or HARDIRQ. Due to + * locking constrains it will _always_ fail to acquire the lock in NMI or + * HARDIRQ context on PREEMPT_RT. + */ +#define local_trylock(lock) __local_trylock(lock) + +/** + * local_trylock_irqsave - Try to acquire a per CPU local lock, save and disable + * interrupts if acquired + * @lock: The lock variable + * @flags: Storage for interrupt flags + * + * The function can be used in any context such as NMI or HARDIRQ. Due to + * locking constrains it will _always_ fail to acquire the lock in NMI or + * HARDIRQ context on PREEMPT_RT. + */ +#define local_trylock_irqsave(lock, flags) \ + __local_trylock_irqsave(lock, flags) + +DEFINE_GUARD(local_lock, local_lock_t __percpu*, + local_lock(_T), + local_unlock(_T)) +DEFINE_GUARD(local_lock_irq, local_lock_t __percpu*, + local_lock_irq(_T), + local_unlock_irq(_T)) +DEFINE_LOCK_GUARD_1(local_lock_irqsave, local_lock_t __percpu, + local_lock_irqsave(_T->lock, _T->flags), + local_unlock_irqrestore(_T->lock, _T->flags), + unsigned long flags) + +#define local_lock_nested_bh(_lock) \ + __local_lock_nested_bh(_lock) + +#define local_unlock_nested_bh(_lock) \ + __local_unlock_nested_bh(_lock) + +DEFINE_GUARD(local_lock_nested_bh, local_lock_t __percpu*, + local_lock_nested_bh(_T), + local_unlock_nested_bh(_T)) + #endif diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 975e33b793a77..85c2e1b1af6bf 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -15,6 +15,15 @@ typedef struct { #endif } local_lock_t; +/* local_trylock() and local_trylock_irqsave() only work with local_trylock_t */ +typedef struct { +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; + struct task_struct *owner; +#endif + u8 acquired; +} local_trylock_t; + #ifdef CONFIG_DEBUG_LOCK_ALLOC # define LOCAL_LOCK_DEBUG_INIT(lockname) \ .dep_map = { \ @@ -24,6 +33,9 @@ typedef struct { }, \ .owner = NULL, +# define LOCAL_TRYLOCK_DEBUG_INIT(lockname) \ + LOCAL_LOCK_DEBUG_INIT(lockname) + static inline void local_lock_acquire(local_lock_t *l) { lock_map_acquire(&l->dep_map); @@ -31,6 +43,13 @@ static inline void local_lock_acquire(local_lock_t *l) l->owner = current; } +static inline void local_trylock_acquire(local_lock_t *l) +{ + lock_map_acquire_try(&l->dep_map); + DEBUG_LOCKS_WARN_ON(l->owner); + l->owner = current; +} + static inline void local_lock_release(local_lock_t *l) { DEBUG_LOCKS_WARN_ON(l->owner != current); @@ -44,12 +63,15 @@ static inline void local_lock_debug_init(local_lock_t *l) } #else /* CONFIG_DEBUG_LOCK_ALLOC */ # define LOCAL_LOCK_DEBUG_INIT(lockname) +# define LOCAL_TRYLOCK_DEBUG_INIT(lockname) static inline void local_lock_acquire(local_lock_t *l) { } +static inline void local_trylock_acquire(local_lock_t *l) { } static inline void local_lock_release(local_lock_t *l) { } static inline void local_lock_debug_init(local_lock_t *l) { } #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ #define INIT_LOCAL_LOCK(lockname) { LOCAL_LOCK_DEBUG_INIT(lockname) } +#define INIT_LOCAL_TRYLOCK(lockname) { LOCAL_TRYLOCK_DEBUG_INIT(lockname) } #define __local_lock_init(lock) \ do { \ @@ -62,42 +84,130 @@ do { \ local_lock_debug_init(lock); \ } while (0) +#define __local_trylock_init(lock) __local_lock_init((local_lock_t *)lock) + +#define __spinlock_nested_bh_init(lock) \ +do { \ + static struct lock_class_key __key; \ + \ + debug_check_no_locks_freed((void *)lock, sizeof(*lock));\ + lockdep_init_map_type(&(lock)->dep_map, #lock, &__key, \ + 0, LD_WAIT_CONFIG, LD_WAIT_INV, \ + LD_LOCK_NORMAL); \ + local_lock_debug_init(lock); \ +} while (0) + +#define __local_lock_acquire(lock) \ + do { \ + local_trylock_t *tl; \ + local_lock_t *l; \ + \ + l = (local_lock_t *)this_cpu_ptr(lock); \ + tl = (local_trylock_t *)l; \ + _Generic((lock), \ + __percpu local_trylock_t *: ({ \ + lockdep_assert(tl->acquired == 0); \ + WRITE_ONCE(tl->acquired, 1); \ + }), \ + __percpu local_lock_t *: (void)0); \ + local_lock_acquire(l); \ + } while (0) + #define __local_lock(lock) \ do { \ preempt_disable(); \ - local_lock_acquire(this_cpu_ptr(lock)); \ + __local_lock_acquire(lock); \ } while (0) #define __local_lock_irq(lock) \ do { \ local_irq_disable(); \ - local_lock_acquire(this_cpu_ptr(lock)); \ + __local_lock_acquire(lock); \ } while (0) #define __local_lock_irqsave(lock, flags) \ do { \ local_irq_save(flags); \ - local_lock_acquire(this_cpu_ptr(lock)); \ + __local_lock_acquire(lock); \ + } while (0) + +#define __local_trylock(lock) \ + ({ \ + local_trylock_t *tl; \ + \ + preempt_disable(); \ + tl = this_cpu_ptr(lock); \ + if (READ_ONCE(tl->acquired)) { \ + preempt_enable(); \ + tl = NULL; \ + } else { \ + WRITE_ONCE(tl->acquired, 1); \ + local_trylock_acquire( \ + (local_lock_t *)tl); \ + } \ + !!tl; \ + }) + +#define __local_trylock_irqsave(lock, flags) \ + ({ \ + local_trylock_t *tl; \ + \ + local_irq_save(flags); \ + tl = this_cpu_ptr(lock); \ + if (READ_ONCE(tl->acquired)) { \ + local_irq_restore(flags); \ + tl = NULL; \ + } else { \ + WRITE_ONCE(tl->acquired, 1); \ + local_trylock_acquire( \ + (local_lock_t *)tl); \ + } \ + !!tl; \ + }) + +#define __local_lock_release(lock) \ + do { \ + local_trylock_t *tl; \ + local_lock_t *l; \ + \ + l = (local_lock_t *)this_cpu_ptr(lock); \ + tl = (local_trylock_t *)l; \ + local_lock_release(l); \ + _Generic((lock), \ + __percpu local_trylock_t *: ({ \ + lockdep_assert(tl->acquired == 1); \ + WRITE_ONCE(tl->acquired, 0); \ + }), \ + __percpu local_lock_t *: (void)0); \ } while (0) #define __local_unlock(lock) \ do { \ - local_lock_release(this_cpu_ptr(lock)); \ + __local_lock_release(lock); \ preempt_enable(); \ } while (0) #define __local_unlock_irq(lock) \ do { \ - local_lock_release(this_cpu_ptr(lock)); \ + __local_lock_release(lock); \ local_irq_enable(); \ } while (0) #define __local_unlock_irqrestore(lock, flags) \ do { \ - local_lock_release(this_cpu_ptr(lock)); \ + __local_lock_release(lock); \ local_irq_restore(flags); \ } while (0) +#define __local_lock_nested_bh(lock) \ + do { \ + lockdep_assert_in_softirq(); \ + local_lock_acquire(this_cpu_ptr(lock)); \ + } while (0) + +#define __local_unlock_nested_bh(lock) \ + local_lock_release(this_cpu_ptr(lock)) + #else /* !CONFIG_PREEMPT_RT */ /* @@ -105,14 +215,18 @@ do { \ * critical section while staying preemptible. */ typedef spinlock_t local_lock_t; +typedef spinlock_t local_trylock_t; #define INIT_LOCAL_LOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname)) +#define INIT_LOCAL_TRYLOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname)) #define __local_lock_init(l) \ do { \ local_spin_lock_init((l)); \ } while (0) +#define __local_trylock_init(l) __local_lock_init(l) + #define __local_lock(__lock) \ do { \ migrate_disable(); \ @@ -138,4 +252,37 @@ typedef spinlock_t local_lock_t; #define __local_unlock_irqrestore(lock, flags) __local_unlock(lock) +#define __local_lock_nested_bh(lock) \ +do { \ + lockdep_assert_in_softirq_func(); \ + spin_lock(this_cpu_ptr(lock)); \ +} while (0) + +#define __local_unlock_nested_bh(lock) \ +do { \ + spin_unlock(this_cpu_ptr((lock))); \ +} while (0) + +#define __local_trylock(lock) \ + ({ \ + int __locked; \ + \ + if (in_nmi() | in_hardirq()) { \ + __locked = 0; \ + } else { \ + migrate_disable(); \ + __locked = spin_trylock(this_cpu_ptr((lock))); \ + if (!__locked) \ + migrate_enable(); \ + } \ + __locked; \ + }) + +#define __local_trylock_irqsave(lock, flags) \ + ({ \ + typecheck(unsigned long, flags); \ + flags = 0; \ + __local_trylock(lock); \ + }) + #endif /* CONFIG_PREEMPT_RT */ diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 919a5cb6368d1..9a1aafb9dcb99 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -657,6 +657,8 @@ do { \ (!in_softirq() || in_irq() || in_nmi())); \ } while (0) +extern void lockdep_assert_in_softirq_func(void); + #else # define might_lock(lock) do { } while (0) # define might_lock_read(lock) do { } while (0) @@ -670,6 +672,7 @@ do { \ # define lockdep_assert_preemption_enabled() do { } while (0) # define lockdep_assert_preemption_disabled() do { } while (0) # define lockdep_assert_in_softirq() do { } while (0) +# define lockdep_assert_in_softirq_func() do { } while (0) #endif #ifdef CONFIG_PROVE_RAW_LOCK_NESTING diff --git a/include/linux/lzo.h b/include/linux/lzo.h index e95c7d1092b28..4d30e3624acd2 100644 --- a/include/linux/lzo.h +++ b/include/linux/lzo.h @@ -24,10 +24,18 @@ int lzo1x_1_compress(const unsigned char *src, size_t src_len, unsigned char *dst, size_t *dst_len, void *wrkmem); +/* Same as above but does not write more than dst_len to dst. */ +int lzo1x_1_compress_safe(const unsigned char *src, size_t src_len, + unsigned char *dst, size_t *dst_len, void *wrkmem); + /* This requires 'wrkmem' of size LZO1X_1_MEM_COMPRESS */ int lzorle1x_1_compress(const unsigned char *src, size_t src_len, unsigned char *dst, size_t *dst_len, void *wrkmem); +/* Same as above but does not write more than dst_len to dst. */ +int lzorle1x_1_compress_safe(const unsigned char *src, size_t src_len, + unsigned char *dst, size_t *dst_len, void *wrkmem); + /* safe decompression with overrun testing */ int lzo1x_decompress_safe(const unsigned char *src, size_t src_len, unsigned char *dst, size_t *dst_len); diff --git a/include/linux/map_benchmark.h b/include/linux/map_benchmark.h index 62674c83bde4e..48e2ff95332f3 100644 --- a/include/linux/map_benchmark.h +++ b/include/linux/map_benchmark.h @@ -27,5 +27,6 @@ struct map_benchmark { __u32 dma_dir; /* DMA data direction */ __u32 dma_trans_ns; /* time for DMA transmission in ns */ __u32 granule; /* how many PAGE_SIZE will do map/unmap once a time */ + __u8 expansion[76]; /* For future use */ }; #endif /* _KERNEL_DMA_BENCHMARK_H */ diff --git a/include/linux/math.h b/include/linux/math.h index dd4152711de7d..ee754ec3dc929 100644 --- a/include/linux/math.h +++ b/include/linux/math.h @@ -34,6 +34,18 @@ */ #define round_down(x, y) ((x) & ~__round_mask(x, y)) +/** + * DIV_ROUND_UP_POW2 - divide and round up + * @n: numerator + * @d: denominator (must be a power of 2) + * + * Divides @n by @d and rounds up to next multiple of @d (which must be a power + * of 2). Avoids integer overflows that may occur with __KERNEL_DIV_ROUND_UP(). + * Performance is roughly equivalent to __KERNEL_DIV_ROUND_UP(). + */ +#define DIV_ROUND_UP_POW2(n, d) \ + ((n) / (d) + !!((n) & ((d) - 1))) + #define DIV_ROUND_UP __KERNEL_DIV_ROUND_UP #define DIV_ROUND_DOWN_ULL(ll, d) \ diff --git a/include/linux/mdio.h b/include/linux/mdio.h index 8fa23bdcedbf9..0bca1a960853f 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -44,10 +44,7 @@ struct mdio_device { unsigned int reset_deassert_delay; }; -static inline struct mdio_device *to_mdio_device(const struct device *dev) -{ - return container_of(dev, struct mdio_device, dev); -} +#define to_mdio_device(__dev) container_of_const(__dev, struct mdio_device, dev) /* struct mdio_driver_common: Common to all MDIO drivers */ struct mdio_driver_common { diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 110e699ef4941..3d400a0f21186 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -731,8 +731,15 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list) __mem_cgroup_uncharge_list(page_list); } -void mem_cgroup_replace_folio(struct folio *old, struct folio *new); +void __mem_cgroup_uncharge_folios(struct folio_batch *folios); +static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) +{ + if (mem_cgroup_disabled()) + return; + __mem_cgroup_uncharge_folios(folios); +} +void mem_cgroup_replace_folio(struct folio *old, struct folio *new); void mem_cgroup_migrate(struct folio *old, struct folio *new); /** @@ -1171,6 +1178,7 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, gfp_t gfp_mask, unsigned long *total_scanned); +extern int mem_cgroup_init(void); #else /* CONFIG_MEMCG */ #define MEM_CGROUP_ID_SHIFT 0 @@ -1297,6 +1305,10 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list) { } +static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) +{ +} + static inline void mem_cgroup_replace_folio(struct folio *old, struct folio *new) { @@ -1619,6 +1631,8 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, { return 0; } + +static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx) @@ -1682,18 +1696,18 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, return folio_lruvec_lock_irq(folio); } -/* Don't lock again iff page's lruvec locked */ -static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio, - struct lruvec *locked_lruvec, unsigned long *flags) +/* Don't lock again iff folio's lruvec locked */ +static inline void folio_lruvec_relock_irqsave(struct folio *folio, + struct lruvec **lruvecp, unsigned long *flags) { - if (locked_lruvec) { - if (folio_matches_lruvec(folio, locked_lruvec)) - return locked_lruvec; + if (*lruvecp) { + if (folio_matches_lruvec(folio, *lruvecp)) + return; - unlock_page_lruvec_irqrestore(locked_lruvec, *flags); + unlock_page_lruvec_irqrestore(*lruvecp, *flags); } - return folio_lruvec_lock_irqsave(folio, flags); + *lruvecp = folio_lruvec_lock_irqsave(folio, flags); } #ifdef CONFIG_CGROUP_WRITEBACK diff --git a/include/linux/memfd.h b/include/linux/memfd.h index e7abf6fa4c522..40cc726a8a0ce 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -6,11 +6,25 @@ #ifdef CONFIG_MEMFD_CREATE extern long memfd_fcntl(struct file *file, unsigned int cmd, unsigned int arg); +unsigned int *memfd_file_seals_ptr(struct file *file); #else static inline long memfd_fcntl(struct file *f, unsigned int c, unsigned int a) { return -EINVAL; } + +static inline unsigned int *memfd_file_seals_ptr(struct file *file) +{ + return NULL; +} #endif +/* Retrieve memfd seals associated with the file, if any. */ +static inline unsigned int memfd_file_seals(struct file *file) +{ + unsigned int *sealsp = memfd_file_seals_ptr(file); + + return sealsp ? *sealsp : 0; +} + #endif /* __LINUX_MEMFD_H */ diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h index 437441cdf78fb..b3cbeb46ddeee 100644 --- a/include/linux/memory-tiers.h +++ b/include/linux/memory-tiers.h @@ -6,6 +6,7 @@ #include #include #include +#include /* * Each tier cover a abstrace distance chunk size of 128 */ @@ -17,7 +18,7 @@ * adistance value (slightly faster) than default DRAM adistance to be part of * the same memory tier. */ -#define MEMTIER_ADISTANCE_DRAM ((4 * MEMTIER_CHUNK_SIZE) + (MEMTIER_CHUNK_SIZE >> 1)) +#define MEMTIER_ADISTANCE_DRAM ((4L * MEMTIER_CHUNK_SIZE) + (MEMTIER_CHUNK_SIZE >> 1)) struct memory_tier; struct memory_dev_type { @@ -30,12 +31,21 @@ struct memory_dev_type { struct kref kref; }; +struct access_coordinate; + #ifdef CONFIG_NUMA extern bool numa_demotion_enabled; +extern struct memory_dev_type *default_dram_type; struct memory_dev_type *alloc_memory_type(int adistance); void put_memory_type(struct memory_dev_type *memtype); void init_node_memory_type(int node, struct memory_dev_type *default_type); void clear_node_memory_type(int node, struct memory_dev_type *memtype); +int register_mt_adistance_algorithm(struct notifier_block *nb); +int unregister_mt_adistance_algorithm(struct notifier_block *nb); +int mt_calc_adistance(int node, int *adist); +int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, + const char *source); +int mt_perf_to_adistance(struct access_coordinate *perf, int *adist); #ifdef CONFIG_MIGRATION int next_demotion_node(int node); void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets); @@ -60,6 +70,7 @@ static inline bool node_is_toptier(int node) #else #define numa_demotion_enabled false +#define default_dram_type NULL /* * CONFIG_NUMA implementation returns non NULL error. */ @@ -97,5 +108,31 @@ static inline bool node_is_toptier(int node) { return true; } + +static inline int register_mt_adistance_algorithm(struct notifier_block *nb) +{ + return 0; +} + +static inline int unregister_mt_adistance_algorithm(struct notifier_block *nb) +{ + return 0; +} + +static inline int mt_calc_adistance(int node, int *adist) +{ + return NOTIFY_DONE; +} + +static inline int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, + const char *source) +{ + return -EIO; +} + +static inline int mt_perf_to_adistance(struct access_coordinate *perf, int *adist) +{ + return -EIO; +} #endif /* CONFIG_NUMA */ #endif /* _LINUX_MEMORY_TIERS_H */ diff --git a/include/linux/minmax.h b/include/linux/minmax.h index 2ec559284a9f6..eaaf5c008e4d0 100644 --- a/include/linux/minmax.h +++ b/include/linux/minmax.h @@ -8,13 +8,10 @@ #include /* - * min()/max()/clamp() macros must accomplish three things: + * min()/max()/clamp() macros must accomplish several things: * * - Avoid multiple evaluations of the arguments (so side-effects like * "x++" happen only once) when non-constant. - * - Retain result as a constant expressions when called with only - * constant expressions (to avoid tripping VLA warnings in stack - * allocation usage). * - Perform signed v unsigned type-checking (to generate compile * errors instead of nasty runtime surprises). * - Unsigned char/short are always promoted to signed int and can be @@ -26,56 +23,79 @@ #define __typecheck(x, y) \ (!!(sizeof((typeof(x) *)1 == (typeof(y) *)1))) -/* is_signed_type() isn't a constexpr for pointer types */ -#define __is_signed(x) \ - __builtin_choose_expr(__is_constexpr(is_signed_type(typeof(x))), \ - is_signed_type(typeof(x)), 0) +/* + * __sign_use for integer expressions: + * bit #0 set if ok for unsigned comparisons + * bit #1 set if ok for signed comparisons + * + * In particular, statically non-negative signed integer expressions + * are ok for both. + * + * NOTE! Unsigned types smaller than 'int' are implicitly converted to 'int' + * in expressions, and are accepted for signed conversions for now. + * This is debatable. + * + * Note that 'x' is the original expression, and 'ux' is the unique variable + * that contains the value. + * + * We use 'ux' for pure type checking, and 'x' for when we need to look at the + * value (but without evaluating it for side effects! + * Careful to only ever evaluate it with sizeof() or __builtin_constant_p() etc). + * + * Pointers end up being checked by the normal C type rules at the actual + * comparison, and these expressions only need to be careful to not cause + * warnings for pointer use. + */ +#define __sign_use(ux) (is_signed_type(typeof(ux)) ? \ + (2 + __is_nonneg(ux)) : (1 + 2 * (sizeof(ux) < 4))) + +/* + * Check whether a signed value is always non-negative. + * + * A cast is needed to avoid any warnings from values that aren't signed + * integer types (in which case the result doesn't matter). + * + * On 64-bit any integer or pointer type can safely be cast to 'long long'. + * But on 32-bit we need to avoid warnings about casting pointers to integers + * of different sizes without truncating 64-bit values so 'long' or 'long long' + * must be used depending on the size of the value. + * + * This does not work for 128-bit signed integers since the cast would truncate + * them, but we do not use s128 types in the kernel (we do use 'u128', + * but they are handled by the !is_signed_type() case). + */ +#if __SIZEOF_POINTER__ == __SIZEOF_LONG_LONG__ +#define __is_nonneg(ux) statically_true((long long)(ux) >= 0) +#else +#define __is_nonneg(ux) statically_true( \ + (typeof(__builtin_choose_expr(sizeof(ux) > 4, 1LL, 1L)))(ux) >= 0) +#endif -/* True for a non-negative signed int constant */ -#define __is_noneg_int(x) \ - (__builtin_choose_expr(__is_constexpr(x) && __is_signed(x), x, -1) >= 0) +#define __types_ok(ux, uy) \ + (__sign_use(ux) & __sign_use(uy)) -#define __types_ok(x, y) \ - (__is_signed(x) == __is_signed(y) || \ - __is_signed((x) + 0) == __is_signed((y) + 0) || \ - __is_noneg_int(x) || __is_noneg_int(y)) +#define __types_ok3(ux, uy, uz) \ + (__sign_use(ux) & __sign_use(uy) & __sign_use(uz)) #define __cmp_op_min < #define __cmp_op_max > #define __cmp(op, x, y) ((x) __cmp_op_##op (y) ? (x) : (y)) -#define __cmp_once(op, x, y, unique_x, unique_y) ({ \ - typeof(x) unique_x = (x); \ - typeof(y) unique_y = (y); \ - static_assert(__types_ok(x, y), \ - #op "(" #x ", " #y ") signedness error, fix types or consider u" #op "() before " #op "_t()"); \ - __cmp(op, unique_x, unique_y); }) - -#define __careful_cmp(op, x, y) \ - __builtin_choose_expr(__is_constexpr((x) - (y)), \ - __cmp(op, x, y), \ - __cmp_once(op, x, y, __UNIQUE_ID(__x), __UNIQUE_ID(__y))) +#define __cmp_once_unique(op, type, x, y, ux, uy) \ + ({ type ux = (x); type uy = (y); __cmp(op, ux, uy); }) -#define __clamp(val, lo, hi) \ - ((val) >= (hi) ? (hi) : ((val) <= (lo) ? (lo) : (val))) +#define __cmp_once(op, type, x, y) \ + __cmp_once_unique(op, type, x, y, __UNIQUE_ID(x_), __UNIQUE_ID(y_)) -#define __clamp_once(val, lo, hi, unique_val, unique_lo, unique_hi) ({ \ - typeof(val) unique_val = (val); \ - typeof(lo) unique_lo = (lo); \ - typeof(hi) unique_hi = (hi); \ - static_assert(__builtin_choose_expr(__is_constexpr((lo) > (hi)), \ - (lo) <= (hi), true), \ - "clamp() low limit " #lo " greater than high limit " #hi); \ - static_assert(__types_ok(val, lo), "clamp() 'lo' signedness error"); \ - static_assert(__types_ok(val, hi), "clamp() 'hi' signedness error"); \ - __clamp(unique_val, unique_lo, unique_hi); }) +#define __careful_cmp_once(op, x, y, ux, uy) ({ \ + __auto_type ux = (x); __auto_type uy = (y); \ + BUILD_BUG_ON_MSG(!__types_ok(ux, uy), \ + #op"("#x", "#y") signedness error"); \ + __cmp(op, ux, uy); }) -#define __careful_clamp(val, lo, hi) ({ \ - __builtin_choose_expr(__is_constexpr((val) - (lo) + (hi)), \ - __clamp(val, lo, hi), \ - __clamp_once(val, lo, hi, __UNIQUE_ID(__val), \ - __UNIQUE_ID(__lo), __UNIQUE_ID(__hi))); }) +#define __careful_cmp(op, x, y) \ + __careful_cmp_once(op, x, y, __UNIQUE_ID(x_), __UNIQUE_ID(y_)) /** * min - return minimum of two values of the same or compatible types @@ -108,13 +128,20 @@ #define umax(x, y) \ __careful_cmp(max, (x) + 0u + 0ul + 0ull, (y) + 0u + 0ul + 0ull) +#define __careful_op3(op, x, y, z, ux, uy, uz) ({ \ + __auto_type ux = (x); __auto_type uy = (y);__auto_type uz = (z);\ + BUILD_BUG_ON_MSG(!__types_ok3(ux, uy, uz), \ + #op"3("#x", "#y", "#z") signedness error"); \ + __cmp(op, ux, __cmp(op, uy, uz)); }) + /** * min3 - return minimum of three values * @x: first value * @y: second value * @z: third value */ -#define min3(x, y, z) min((typeof(x))min(x, y), z) +#define min3(x, y, z) \ + __careful_op3(min, x, y, z, __UNIQUE_ID(x_), __UNIQUE_ID(y_), __UNIQUE_ID(z_)) /** * max3 - return maximum of three values @@ -122,7 +149,24 @@ * @y: second value * @z: third value */ -#define max3(x, y, z) max((typeof(x))max(x, y), z) +#define max3(x, y, z) \ + __careful_op3(max, x, y, z, __UNIQUE_ID(x_), __UNIQUE_ID(y_), __UNIQUE_ID(z_)) + +/** + * min_t - return minimum of two values, using the specified type + * @type: data type to use + * @x: first value + * @y: second value + */ +#define min_t(type, x, y) __cmp_once(min, type, x, y) + +/** + * max_t - return maximum of two values, using the specified type + * @type: data type to use + * @x: first value + * @y: second value + */ +#define max_t(type, x, y) __cmp_once(max, type, x, y) /** * min_not_zero - return the minimum that is _not_ zero, unless both are zero @@ -134,39 +178,57 @@ typeof(y) __y = (y); \ __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); }) +#define __clamp(val, lo, hi) \ + ((val) >= (hi) ? (hi) : ((val) <= (lo) ? (lo) : (val))) + +#define __clamp_once(type, val, lo, hi, uval, ulo, uhi) ({ \ + type uval = (val); \ + type ulo = (lo); \ + type uhi = (hi); \ + BUILD_BUG_ON_MSG(statically_true(ulo > uhi), \ + "clamp() low limit " #lo " greater than high limit " #hi); \ + BUILD_BUG_ON_MSG(!__types_ok3(uval, ulo, uhi), \ + "clamp("#val", "#lo", "#hi") signedness error"); \ + __clamp(uval, ulo, uhi); }) + +#define __careful_clamp(type, val, lo, hi) \ + __clamp_once(type, val, lo, hi, __UNIQUE_ID(v_), __UNIQUE_ID(l_), __UNIQUE_ID(h_)) + /** - * clamp - return a value clamped to a given range with strict typechecking + * clamp - return a value clamped to a given range with typechecking * @val: current value * @lo: lowest allowable value * @hi: highest allowable value * - * This macro does strict typechecking of @lo/@hi to make sure they are of the - * same type as @val. See the unnecessary pointer comparisons. - */ -#define clamp(val, lo, hi) __careful_clamp(val, lo, hi) - -/* - * ..and if you can't take the strict - * types, you can specify one yourself. - * - * Or not use min/max/clamp at all, of course. + * This macro checks @val/@lo/@hi to make sure they have compatible + * signedness. */ +#define clamp(val, lo, hi) __careful_clamp(__auto_type, val, lo, hi) /** - * min_t - return minimum of two values, using the specified type - * @type: data type to use - * @x: first value - * @y: second value + * clamp_t - return a value clamped to a given range using a given type + * @type: the type of variable to use + * @val: current value + * @lo: minimum allowable value + * @hi: maximum allowable value + * + * This macro does no typechecking and uses temporary variables of type + * @type to make all the comparisons. */ -#define min_t(type, x, y) __careful_cmp(min, (type)(x), (type)(y)) +#define clamp_t(type, val, lo, hi) __careful_clamp(type, val, lo, hi) /** - * max_t - return maximum of two values, using the specified type - * @type: data type to use - * @x: first value - * @y: second value + * clamp_val - return a value clamped to a given range using val's type + * @val: current value + * @lo: minimum allowable value + * @hi: maximum allowable value + * + * This macro does no typechecking and uses temporary variables of whatever + * type the input argument @val is. This is useful when @val is an unsigned + * type and @lo and @hi are literals that will otherwise be assigned a signed + * integer type. */ -#define max_t(type, x, y) __careful_cmp(max, (type)(x), (type)(y)) +#define clamp_val(val, lo, hi) __careful_clamp(typeof(val), val, lo, hi) /* * Do not check the array parameter using __must_be_array(). @@ -211,31 +273,6 @@ */ #define max_array(array, len) __minmax_array(max, array, len) -/** - * clamp_t - return a value clamped to a given range using a given type - * @type: the type of variable to use - * @val: current value - * @lo: minimum allowable value - * @hi: maximum allowable value - * - * This macro does no typechecking and uses temporary variables of type - * @type to make all the comparisons. - */ -#define clamp_t(type, val, lo, hi) __careful_clamp((type)(val), (type)(lo), (type)(hi)) - -/** - * clamp_val - return a value clamped to a given range using val's type - * @val: current value - * @lo: minimum allowable value - * @hi: maximum allowable value - * - * This macro does no typechecking and uses temporary variables of whatever - * type the input argument @val is. This is useful when @val is an unsigned - * type and @lo and @hi are literals that will otherwise be assigned a signed - * integer type. - */ -#define clamp_val(val, lo, hi) clamp_t(typeof(val), val, lo, hi) - static inline bool in_range64(u64 val, u64 start, u64 len) { return (val - start) < len; @@ -270,4 +307,13 @@ static inline bool in_range32(u32 val, u32 start, u32 len) #define swap(a, b) \ do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0) +/* + * Use these carefully: no type checking, and uses the arguments + * multiple times. Use for obvious constants only. + */ +#define MIN(a, b) __cmp(min, a, b) +#define MAX(a, b) __cmp(max, a, b) +#define MIN_T(type, a, b) __cmp(min, (type)(a), (type)(b)) +#define MAX_T(type, a, b) __cmp(max, (type)(a), (type)(b)) + #endif /* _LINUX_MINMAX_H */ diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index 27f42f713c891..86f0f2a25a3d6 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -1135,7 +1135,7 @@ int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, struct mlx4_buf *buf); -int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order); +int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, unsigned int order); void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db); int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 69d844b34da0d..c0e0468b25a18 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -390,6 +390,7 @@ struct mlx5_core_rsc_common { enum mlx5_res_type res; refcount_t refcount; struct completion free; + bool invalid; }; struct mlx5_uars_page { @@ -676,6 +677,7 @@ struct mlx5e_resources { struct mlx5_sq_bfreg bfreg; } hw_objs; struct net_device *uplink_netdev; + netdevice_tracker tracker; struct mutex uplink_netdev_lock; struct mlx5_crypto_dek_priv *dek_priv; }; diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index 3fb428ce7d1c7..b6b9a4dfa4fb9 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -40,6 +40,8 @@ #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v) +#define MLX5_FS_MAX_POOL_SIZE BIT(30) + enum mlx5_flow_destination_type { MLX5_FLOW_DESTINATION_TYPE_NONE, MLX5_FLOW_DESTINATION_TYPE_VPORT, diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 9106771bb92f0..4913d364e9774 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1731,7 +1731,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_328[0x2]; u8 relaxed_ordering_read[0x1]; u8 log_max_pd[0x5]; - u8 reserved_at_330[0x6]; + u8 reserved_at_330[0x5]; + u8 pcie_reset_using_hotreset_method[0x1]; u8 pci_sync_for_fw_update_with_driver_unload[0x1]; u8 vnic_env_cnt_steering_fail[0x1]; u8 vport_counter_local_loopback[0x1]; @@ -10824,6 +10825,11 @@ struct mlx5_ifc_mcda_reg_bits { u8 data[][0x20]; }; +enum { + MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE = 0, + MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET = 1, +}; + enum { MLX5_MFRL_REG_RESET_STATE_IDLE = 0, MLX5_MFRL_REG_RESET_STATE_IN_NEGOTIATION = 1, @@ -10851,7 +10857,8 @@ struct mlx5_ifc_mfrl_reg_bits { u8 pci_sync_for_fw_update_start[0x1]; u8 pci_sync_for_fw_update_resp[0x2]; u8 rst_type_sel[0x3]; - u8 reserved_at_28[0x4]; + u8 pci_reset_req_method[0x3]; + u8 reserved_at_2b[0x1]; u8 reset_state[0x4]; u8 reset_type[0x8]; u8 reset_level[0x8]; diff --git a/include/linux/mm.h b/include/linux/mm.h index c2422ab80c295..220d1531a1cf3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -319,7 +319,7 @@ extern unsigned int kobjsize(const void *objp); #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */ #define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */ #define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */ -#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */ +#define VM_MERGEABLE BIT(31) /* KSM may merge identical pages */ #ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS #define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */ @@ -964,6 +964,17 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) return vma->vm_flags & VM_ACCESS_FLAGS; } +static inline bool is_shared_maywrite(vm_flags_t vm_flags) +{ + return (vm_flags & (VM_SHARED | VM_MAYWRITE)) == + (VM_SHARED | VM_MAYWRITE); +} + +static inline bool vma_is_shared_maywrite(struct vm_area_struct *vma) +{ + return is_shared_maywrite(vma->vm_flags); +} + static inline struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) { @@ -2217,6 +2228,61 @@ static inline bool folio_likely_mapped_shared(struct folio *folio) return page_mapcount(folio_page(folio, 0)) > 1; } +/** + * folio_expected_ref_count - calculate the expected folio refcount + * @folio: the folio + * + * Calculate the expected folio refcount, taking references from the pagecache, + * swapcache, PG_private and page table mappings into account. Useful in + * combination with folio_ref_count() to detect unexpected references (e.g., + * GUP or other temporary references). + * + * Does currently not consider references from the LRU cache. If the folio + * was isolated from the LRU (which is the case during migration or split), + * the LRU cache does not apply. + * + * Calling this function on an unmapped folio -- !folio_mapped() -- that is + * locked will return a stable result. + * + * Calling this function on a mapped folio will not result in a stable result, + * because nothing stops additional page table mappings from coming (e.g., + * fork()) or going (e.g., munmap()). + * + * Calling this function without the folio lock will also not result in a + * stable result: for example, the folio might get dropped from the swapcache + * concurrently. + * + * However, even when called without the folio lock or on a mapped folio, + * this function can be used to detect unexpected references early (for example, + * if it makes sense to even lock the folio and unmap it). + * + * The caller must add any reference (e.g., from folio_try_get()) it might be + * holding itself to the result. + * + * Returns the expected folio refcount. + */ +static inline int folio_expected_ref_count(struct folio *folio) +{ + const int order = folio_order(folio); + int ref_count = 0; + + if (WARN_ON_ONCE(folio_test_slab(folio))) + return 0; + + if (folio_test_anon(folio)) { + /* One reference per page from the swapcache. */ + ref_count += folio_test_swapcache(folio) << order; + } else if (!((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS)) { + /* One reference per page from the pagecache. */ + ref_count += !!folio->mapping << order; + /* One reference from PG_private. */ + ref_count += folio_test_private(folio); + } + + /* One reference per page table mapping. */ + return ref_count + folio_mapcount(folio); +} + #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE static inline int arch_make_page_accessible(struct page *page) { @@ -2625,6 +2691,11 @@ static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) return percpu_counter_read_positive(&mm->rss_stat[member]); } +static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member) +{ + return percpu_counter_sum_positive(&mm->rss_stat[member]); +} + void mm_trace_rss_stat(struct mm_struct *mm, int member); static inline void add_mm_counter(struct mm_struct *mm, int member, long value) @@ -4079,34 +4150,57 @@ void mem_dump_obj(void *object); static inline void mem_dump_obj(void *object) {} #endif +static inline bool is_write_sealed(int seals) +{ + return seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE); +} + +/** + * is_readonly_sealed - Checks whether write-sealed but mapped read-only, + * in which case writes should be disallowing moving + * forwards. + * @seals: the seals to check + * @vm_flags: the VMA flags to check + * + * Returns whether readonly sealed, in which case writess should be disallowed + * going forward. + */ +static inline bool is_readonly_sealed(int seals, vm_flags_t vm_flags) +{ + /* + * Since an F_SEAL_[FUTURE_]WRITE sealed memfd can be mapped as + * MAP_SHARED and read-only, take care to not allow mprotect to + * revert protections on such mappings. Do this only for shared + * mappings. For private mappings, don't need to mask + * VM_MAYWRITE as we still want them to be COW-writable. + */ + if (is_write_sealed(seals) && + ((vm_flags & (VM_SHARED | VM_WRITE)) == VM_SHARED)) + return true; + + return false; +} + /** - * seal_check_future_write - Check for F_SEAL_FUTURE_WRITE flag and handle it + * seal_check_write - Check for F_SEAL_WRITE or F_SEAL_FUTURE_WRITE flags and + * handle them. * @seals: the seals to check * @vma: the vma to operate on * - * Check whether F_SEAL_FUTURE_WRITE is set; if so, do proper check/handling on - * the vma flags. Return 0 if check pass, or <0 for errors. + * Check whether F_SEAL_WRITE or F_SEAL_FUTURE_WRITE are set; if so, do proper + * check/handling on the vma flags. Return 0 if check pass, or <0 for errors. */ -static inline int seal_check_future_write(int seals, struct vm_area_struct *vma) -{ - if (seals & F_SEAL_FUTURE_WRITE) { - /* - * New PROT_WRITE and MAP_SHARED mmaps are not allowed when - * "future write" seal active. - */ - if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE)) - return -EPERM; - - /* - * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as - * MAP_SHARED and read-only, take care to not allow mprotect to - * revert protections on such mappings. Do this only for shared - * mappings. For private mappings, don't need to mask - * VM_MAYWRITE as we still want them to be COW-writable. - */ - if (vma->vm_flags & VM_SHARED) - vm_flags_clear(vma, VM_MAYWRITE); - } +static inline int seal_check_write(int seals, struct vm_area_struct *vma) +{ + if (!is_write_sealed(seals)) + return 0; + + /* + * New PROT_WRITE and MAP_SHARED mmaps are not allowed when + * write seals are active. + */ + if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE)) + return -EPERM; return 0; } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 2f0c6b5f939f4..5b56c9bb66982 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1013,7 +1013,7 @@ struct mm_struct { #endif /* CONFIG_LRU_GEN */ ANDROID_KABI_RESERVE(1); - ANDROID_BACKPORT_RESERVE(1); + ANDROID_BACKPORT_USE(1, struct task_dma_buf_info *dmabuf_info); } __randomize_layout; /* diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index 21d28d098eea5..8edc4600482d3 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -300,6 +300,7 @@ struct mmc_card { #define MMC_QUIRK_BROKEN_SD_CACHE (1<<15) /* Disable broken SD cache support */ #define MMC_QUIRK_BROKEN_CACHE_FLUSH (1<<16) /* Don't flush cache until the write has occurred */ #define MMC_QUIRK_BROKEN_SD_POWEROFF_NOTIFY (1<<17) /* Disable broken SD poweroff notify support */ +#define MMC_QUIRK_NO_UHS_DDR50_TUNING (1<<18) /* Disable DDR50 tuning */ bool written_flag; /* Indicates eMMC has been written since power on */ bool reenable_cmdq; /* Re-enable Command Queue */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 3311757c9d985..8d7e93a8437b2 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -37,6 +37,22 @@ #define NR_PAGE_ORDERS (MAX_ORDER + 1) +/* Defines the order for the number of pages that have a migrate type. */ +#ifndef CONFIG_PAGE_BLOCK_ORDER +#define PAGE_BLOCK_ORDER MAX_ORDER +#else +#define PAGE_BLOCK_ORDER CONFIG_PAGE_BLOCK_ORDER +#endif /* CONFIG_PAGE_BLOCK_ORDER */ + +/* + * The MAX_ORDER, which defines the max order of pages to be allocated + * by the buddy allocator, has to be larger or equal to the PAGE_BLOCK_ORDER, + * which defines the order for the number of pages that can have a migrate type + */ +#if (PAGE_BLOCK_ORDER > MAX_ORDER) +#error MAX_ORDER must be >= PAGE_BLOCK_ORDER +#endif + /* * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed * costly to service. That is between allocation orders which should diff --git a/include/linux/module.h b/include/linux/module.h index b6f51eb07586e..bacc10994b77b 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -163,6 +163,8 @@ extern void cleanup_module(void); #define __INITRODATA_OR_MODULE __INITRODATA #endif /*CONFIG_MODULES*/ +struct module_kobject *lookup_or_create_module_kobject(const char *name); + /* Generic info of form tag = "info" */ #define MODULE_INFO(tag, info) __MODULE_INFO(tag, tag, info) @@ -574,6 +576,11 @@ struct module { atomic_t refcnt; #endif +#ifdef CONFIG_MITIGATION_ITS + int its_num_pages; + void **its_page_array; +#endif + #ifdef CONFIG_CONSTRUCTORS /* Constructor functions. */ ctor_fn_t *ctors; diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 962cd41a2cb5a..061e19c94a6bc 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -282,10 +282,9 @@ struct kparam_array #define __moduleparam_const const #endif -/* This is the fundamental function for registering boot/module - parameters. */ +/* This is the fundamental function for registering boot/module parameters. */ #define __module_param_call(prefix, name, ops, arg, perm, level, flags) \ - /* Default value instead of permissions? */ \ + static_assert(sizeof(""prefix) - 1 <= MAX_PARAM_PREFIX_LEN); \ static const char __param_str_##name[] = prefix #name; \ static struct kernel_param __moduleparam_const __param_##name \ __used __section("__param") \ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index ac4b8b828364a..08c3dc752cf83 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3075,7 +3075,8 @@ extern rwlock_t dev_base_lock; /* Device list lock */ #define net_device_entry(lh) list_entry(lh, struct net_device, dev_list) #define for_each_netdev_dump(net, d, ifindex) \ - xa_for_each_start(&(net)->dev_by_index, (ifindex), (d), (ifindex)) + for (; (d = xa_find(&(net)->dev_by_index, &ifindex, \ + ULONG_MAX, XA_PRESENT)); ifindex++) static inline struct net_device *next_net_device(struct net_device *dev) { @@ -3653,6 +3654,17 @@ static inline void netdev_tx_reset_queue(struct netdev_queue *q) #endif } +/** + * netdev_tx_reset_subqueue - reset the BQL stats and state of a netdev queue + * @dev: network device + * @qid: stack index of the queue to reset + */ +static inline void netdev_tx_reset_subqueue(const struct net_device *dev, + u32 qid) +{ + netdev_tx_reset_queue(netdev_get_tx_queue(dev, qid)); +} + /** * netdev_reset_queue - reset the packets and bytes count of a network device * @dev_queue: network device @@ -3662,7 +3674,7 @@ static inline void netdev_tx_reset_queue(struct netdev_queue *q) */ static inline void netdev_reset_queue(struct net_device *dev_queue) { - netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0)); + netdev_tx_reset_subqueue(dev_queue, 0); } /** diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h index 5897f3dbaf7c3..f9f1496a47d4d 100644 --- a/include/linux/netfilter/x_tables.h +++ b/include/linux/netfilter/x_tables.h @@ -144,6 +144,15 @@ struct xt_match { const char name[XT_EXTENSION_MAXNAMELEN]; u_int8_t revision; +#ifndef __GENKSYMS__ + /* + * ANDROID: Indicates that this structure is contained within a compat_xt_match_ext struct, + * which contains the required metadata. + * + * Hide this from GENKSYMS to preserve the original CRC value of this struct. + */ + bool has_compat_metadata; +#endif /* Return true or false: return FALSE and set *hotdrop = 1 to force immediate packet drop. */ @@ -158,7 +167,13 @@ struct xt_match { /* Called when entry of this type deleted. */ void (*destroy)(const struct xt_mtdtor_param *); -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT + /* + * ANDROID: Adding these fields to this structure breaks the KMI, so do not allow them to + * be added within this structure. + * + * Instead, see struct compat_xt_match_ext and its usage below. + */ +#ifdef CONFIG_BROKEN /* Called when userspace align differs from kernel space one */ void (*compat_from_user)(void *dst, const void *src); int (*compat_to_user)(void __user *dst, const void *src); @@ -169,7 +184,13 @@ struct xt_match { const char *table; unsigned int matchsize; unsigned int usersize; -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT + /* + * ANDROID: Adding this field to this structure breaks the KMI, so do not allow it to be + * added within this structure. + * + * Instead, see struct compat_xt_match_ext and its usage below. + */ +#ifdef CONFIG_BROKEN unsigned int compatsize; #endif unsigned int hooks; @@ -184,6 +205,15 @@ struct xt_target { const char name[XT_EXTENSION_MAXNAMELEN]; u_int8_t revision; +#ifndef __GENKSYMS__ + /* + * ANDROID: Indicates that this structure is contained within a compat_xt_target_ext struct, + * which contains the required metadata. + * + * Hide this from GENKSYMS to preserve the original CRC value of this struct. + */ + bool has_compat_metadata; +#endif /* Returns verdict. Argument order changed since 2.6.9, as this must now handle non-linear skbs, using skb_copy_bits and @@ -199,7 +229,13 @@ struct xt_target { /* Called when entry of this type deleted. */ void (*destroy)(const struct xt_tgdtor_param *); -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT + /* + * ANDROID: Adding these fields to this structure breaks the KMI, so do not allow them to + * be added within this structure. + * + * Instead, see struct compat_xt_target_ext and its usage below. + */ +#ifdef CONFIG_BROKEN /* Called when userspace align differs from kernel space one */ void (*compat_from_user)(void *dst, const void *src); int (*compat_to_user)(void __user *dst, const void *src); @@ -210,7 +246,13 @@ struct xt_target { const char *table; unsigned int targetsize; unsigned int usersize; -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT + /* + * ANDROID: Adding this field to this structure breaks the KMI, so do not allow it to be + * added within this structure. + * + * Instead, see struct compat_xt_target_ext and its usage below. + */ +#ifdef CONFIG_BROKEN unsigned int compatsize; #endif unsigned int hooks; @@ -455,6 +497,51 @@ void xt_unregister_template(const struct xt_table *t); #ifdef CONFIG_NETFILTER_XTABLES_COMPAT #include +typedef void (*compat_from_user_fn_t)(void *dst, const void *src); +typedef int (*compat_to_user_fn_t)(void __user *dst, const void *src); + +struct compat_xt_match_ext { + /* Called when userspace align differs from kernel space one */ + compat_from_user_fn_t compat_from_user; + compat_to_user_fn_t compat_to_user; + unsigned int compatsize; + struct xt_match match; +}; + +#define DEFINE_COMPAT_XT_MATCH_GETTER(field, field_type, default_value) \ +static inline field_type get_xt_match_##field(const struct xt_match *match) \ +{ \ + struct compat_xt_match_ext *match_ext; \ + \ + if (match->has_compat_metadata) { \ + match_ext = container_of(match, struct compat_xt_match_ext, match); \ + return match_ext->field; \ + } \ + \ + return default_value; \ +} + +struct compat_xt_target_ext { + /* Called when userspace align differs from kernel space one */ + void (*compat_from_user)(void *dst, const void *src); + int (*compat_to_user)(void __user *dst, const void *src); + unsigned int compatsize; + struct xt_target target; +}; + +#define DEFINE_COMPAT_XT_TARGET_GETTER(field, field_type, default_value) \ +static inline field_type get_xt_target_##field(const struct xt_target *target) \ +{ \ + struct compat_xt_target_ext *target_ext; \ + \ + if (target->has_compat_metadata) { \ + target_ext = container_of(target, struct compat_xt_target_ext, target); \ + return target_ext->field; \ + } \ + \ + return default_value; \ +} + struct compat_xt_entry_match { union { struct { @@ -510,6 +597,14 @@ struct _compat_xt_align { #define COMPAT_XT_ALIGN(s) __ALIGN_KERNEL((s), __alignof__(struct _compat_xt_align)) +DEFINE_COMPAT_XT_MATCH_GETTER(compatsize, unsigned int, 0); +DEFINE_COMPAT_XT_MATCH_GETTER(compat_from_user, compat_from_user_fn_t, NULL); +DEFINE_COMPAT_XT_MATCH_GETTER(compat_to_user, compat_to_user_fn_t, NULL); + +DEFINE_COMPAT_XT_TARGET_GETTER(compatsize, unsigned int, 0); +DEFINE_COMPAT_XT_TARGET_GETTER(compat_from_user, compat_from_user_fn_t, NULL); +DEFINE_COMPAT_XT_TARGET_GETTER(compat_to_user, compat_to_user_fn_t, NULL); + void xt_compat_lock(u_int8_t af); void xt_compat_unlock(u_int8_t af); diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h index 3ddf205b7e2c3..f0dddb8852805 100644 --- a/include/linux/netpoll.h +++ b/include/linux/netpoll.h @@ -32,6 +32,7 @@ struct netpoll { bool ipv6; u16 local_port, remote_port; u8 remote_mac[ETH_ALEN]; + struct sk_buff_head skb_pool; }; struct netpoll_info { diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 86d96e00c2e3d..374b1b208bd89 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -199,6 +199,15 @@ struct nfs_server { char *fscache_uniq; /* Uniquifier (or NULL) */ #endif + /* The following #defines numerically match the NFSv4 equivalents */ +#define NFS_FH_NOEXPIRE_WITH_OPEN (0x1) +#define NFS_FH_VOLATILE_ANY (0x2) +#define NFS_FH_VOL_MIGRATION (0x4) +#define NFS_FH_VOL_RENAME (0x8) +#define NFS_FH_RENAME_UNSAFE (NFS_FH_VOLATILE_ANY | NFS_FH_VOL_RENAME) + u32 fh_expire_type; /* V4 bitmask representing file + handle volatility type for + this filesystem */ u32 pnfs_blksize; /* layout_blksize attr */ #if IS_ENABLED(CONFIG_NFS_V4) u32 attr_bitmask[3];/* V4 bitmask representing the set @@ -222,9 +231,6 @@ struct nfs_server { u32 acl_bitmask; /* V4 bitmask representing the ACEs that are supported on this filesystem */ - u32 fh_expire_type; /* V4 bitmask representing file - handle volatility type for - this filesystem */ struct pnfs_layoutdriver_type *pnfs_curr_ld; /* Active layout driver */ struct rpc_wait_queue roc_rpcwaitq; void *pnfs_ld_data; /* per mount point data */ diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index 1c315f854ea80..6a46069c5a368 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -156,13 +156,13 @@ extern int nfs_wait_on_request(struct nfs_page *); extern void nfs_unlock_request(struct nfs_page *req); extern void nfs_unlock_and_release_request(struct nfs_page *); extern struct nfs_page *nfs_page_group_lock_head(struct nfs_page *req); -extern int nfs_page_group_lock_subrequests(struct nfs_page *head); extern void nfs_join_page_group(struct nfs_page *head, struct nfs_commit_info *cinfo, struct inode *inode); extern int nfs_page_group_lock(struct nfs_page *); extern void nfs_page_group_unlock(struct nfs_page *); extern bool nfs_page_group_sync_on_bit(struct nfs_page *, unsigned int); +extern bool nfs_page_group_sync_on_bit_locked(struct nfs_page *, unsigned int); extern int nfs_page_set_headlock(struct nfs_page *req); extern void nfs_page_clear_headlock(struct nfs_page *req); extern bool nfs_async_iocounter_wait(struct rpc_task *, struct nfs_lock_context *); diff --git a/include/linux/node.h b/include/linux/node.h index 427a5975cf405..dfc004e4bee74 100644 --- a/include/linux/node.h +++ b/include/linux/node.h @@ -20,20 +20,32 @@ #include /** - * struct node_hmem_attrs - heterogeneous memory performance attributes + * struct access_coordinate - generic performance coordinates container * * @read_bandwidth: Read bandwidth in MB/s * @write_bandwidth: Write bandwidth in MB/s * @read_latency: Read latency in nanoseconds * @write_latency: Write latency in nanoseconds */ -struct node_hmem_attrs { +struct access_coordinate { unsigned int read_bandwidth; unsigned int write_bandwidth; unsigned int read_latency; unsigned int write_latency; }; +/* + * ACCESS_COORDINATE_LOCAL correlates to ACCESS CLASS 0 + * - access_coordinate between target node and nearest initiator node + * ACCESS_COORDINATE_CPU correlates to ACCESS CLASS 1 + * - access_coordinate between target node and nearest CPU node + */ +enum access_coordinate_class { + ACCESS_COORDINATE_LOCAL, + ACCESS_COORDINATE_CPU, + ACCESS_COORDINATE_MAX +}; + enum cache_indexing { NODE_CACHE_DIRECT_MAP, NODE_CACHE_INDEXED, @@ -65,8 +77,8 @@ struct node_cache_attrs { #ifdef CONFIG_HMEM_REPORTING void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs); -void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs, - unsigned access); +void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access); #else static inline void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) @@ -74,8 +86,8 @@ static inline void node_add_cache(unsigned int nid, } static inline void node_set_perf_attrs(unsigned int nid, - struct node_hmem_attrs *hmem_attrs, - unsigned access) + struct access_coordinate *coord, + enum access_coordinate_class access) { } #endif @@ -137,7 +149,7 @@ extern void unregister_memory_block_under_nodes(struct memory_block *mem_blk); extern int register_memory_node_under_compute_node(unsigned int mem_nid, unsigned int cpu_nid, - unsigned access); + enum access_coordinate_class access); #else static inline void node_dev_init(void) { diff --git a/include/linux/once.h b/include/linux/once.h index 30346fcdc7995..449a0e34ad5ad 100644 --- a/include/linux/once.h +++ b/include/linux/once.h @@ -46,7 +46,7 @@ void __do_once_sleepable_done(bool *done, struct static_key_true *once_key, #define DO_ONCE(func, ...) \ ({ \ bool ___ret = false; \ - static bool __section(".data..once") ___done = false; \ + static bool __section(".data..do_once") ___done = false; \ static DEFINE_STATIC_KEY_TRUE(___once_key); \ if (static_branch_unlikely(&___once_key)) { \ unsigned long ___flags; \ @@ -64,7 +64,7 @@ void __do_once_sleepable_done(bool *done, struct static_key_true *once_key, #define DO_ONCE_SLEEPABLE(func, ...) \ ({ \ bool ___ret = false; \ - static bool __section(".data..once") ___done = false; \ + static bool __section(".data..do_once") ___done = false; \ static DEFINE_STATIC_KEY_TRUE(___once_key); \ if (static_branch_unlikely(&___once_key)) { \ ___ret = __do_once_sleepable_start(&___done); \ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index a52fe91f78ad5..11cfd3bde50ec 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -581,6 +581,13 @@ PAGEFLAG(Readahead, readahead, PF_NO_COMPOUND) PAGEFLAG_FALSE(HighMem, highmem) #endif +/* Does kmap_local_folio() only allow access to one page of the folio? */ +#ifdef CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP +#define folio_test_partial_kmap(f) true +#else +#define folio_test_partial_kmap(f) folio_test_highmem(f) +#endif + #ifdef CONFIG_SWAP static __always_inline bool folio_test_swapcache(struct folio *folio) { diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h index e83c4c0950417..16b1b29527311 100644 --- a/include/linux/pageblock-flags.h +++ b/include/linux/pageblock-flags.h @@ -28,7 +28,7 @@ enum pageblock_bits { NR_PAGEBLOCK_BITS }; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE @@ -41,14 +41,18 @@ extern unsigned int pageblock_order; * Huge pages are a constant size, but don't exceed the maximum allocation * granularity. */ -#define pageblock_order min_t(unsigned int, HUGETLB_PAGE_ORDER, MAX_ORDER) +#define pageblock_order MIN_T(unsigned int, HUGETLB_PAGE_ORDER, PAGE_BLOCK_ORDER) #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ -#else /* CONFIG_HUGETLB_PAGE */ +#elif defined(CONFIG_TRANSPARENT_HUGEPAGE) + +#define pageblock_order min_t(unsigned int, HPAGE_PMD_ORDER, PAGE_BLOCK_ORDER) + +#else /* CONFIG_TRANSPARENT_HUGEPAGE */ /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ -#define pageblock_order MAX_ORDER +#define pageblock_order PAGE_BLOCK_ORDER #endif /* CONFIG_HUGETLB_PAGE */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 580c173896345..4fda2534b1cd3 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -246,6 +246,8 @@ enum pci_dev_flags { PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 11), /* Device does honor MSI masking despite saying otherwise */ PCI_DEV_FLAGS_HAS_MSI_MASKING = (__force pci_dev_flags_t) (1 << 12), + /* Device requires write to PCI_MSIX_ENTRY_DATA before any MSIX reads */ + PCI_DEV_FLAGS_MSIX_TOUCH_ENTRY_DATA_FIRST = (__force pci_dev_flags_t) (1 << 13), }; enum pci_irq_reroute_variant { @@ -1050,6 +1052,20 @@ static inline struct pci_driver *to_pci_driver(struct device_driver *drv) .vendor = PCI_VENDOR_ID_##vend, .device = (dev), \ .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, 0, 0 +/** + * PCI_VDEVICE_SUB - describe a specific PCI device/subdevice in a short form + * @vend: the vendor name + * @dev: the 16 bit PCI Device ID + * @subvend: the 16 bit PCI Subvendor ID + * @subdev: the 16 bit PCI Subdevice ID + * + * Generate the pci_device_id struct layout for the specific PCI + * device/subdevice. Private data may follow the output. + */ +#define PCI_VDEVICE_SUB(vend, dev, subvend, subdev) \ + .vendor = PCI_VENDOR_ID_##vend, .device = (dev), \ + .subvendor = (subvend), .subdevice = (subdev), 0, 0 + /** * PCI_DEVICE_DATA - macro used to describe a specific PCI device in very short form * @vend: the vendor name (without PCI_VENDOR_ID_ prefix) @@ -2669,7 +2685,7 @@ static inline bool pci_is_thunderbolt_attached(struct pci_dev *pdev) return false; } -#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH) +#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH) || defined(CONFIG_S390) void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type); #endif diff --git a/include/linux/pds/pds_core_if.h b/include/linux/pds/pds_core_if.h index e838a2b90440c..17a87c1a55d7c 100644 --- a/include/linux/pds/pds_core_if.h +++ b/include/linux/pds/pds_core_if.h @@ -79,6 +79,7 @@ enum pds_core_status_code { PDS_RC_EVFID = 31, /* VF ID does not exist */ PDS_RC_BAD_FW = 32, /* FW file is invalid or corrupted */ PDS_RC_ECLIENT = 33, /* No such client id */ + PDS_RC_BAD_PCI = 255, /* Broken PCI when reading status */ }; /** diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h old mode 100644 new mode 100755 diff --git a/include/linux/pgalloc.h b/include/linux/pgalloc.h new file mode 100644 index 0000000000000..9174fa59bbc54 --- /dev/null +++ b/include/linux/pgalloc.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PGALLOC_H +#define _LINUX_PGALLOC_H + +#include +#include + +/* + * {pgd,p4d}_populate_kernel() are defined as macros to allow + * compile-time optimization based on the configured page table levels. + * Without this, linking may fail because callers (e.g., KASAN) may rely + * on calls to these functions being optimized away when passing symbols + * that exist only for certain page table levels. + */ +#define pgd_populate_kernel(addr, pgd, p4d) \ + do { \ + pgd_populate(&init_mm, pgd, p4d); \ + if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PGD_MODIFIED) \ + arch_sync_kernel_mappings(addr, addr); \ + } while (0) + +#define p4d_populate_kernel(addr, p4d, pud) \ + do { \ + p4d_populate(&init_mm, p4d, pud); \ + if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_P4D_MODIFIED) \ + arch_sync_kernel_mappings(addr, addr); \ + } while (0) + +#endif /* _LINUX_PGALLOC_H */ diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h index 359c1807ff1d0..24de4055cd1d1 100644 --- a/include/linux/pgsize_migration.h +++ b/include/linux/pgsize_migration.h @@ -13,9 +13,9 @@ * page size in Android. */ +#include #include #include -#include #if PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) extern void vma_set_pad_pages(struct vm_area_struct *vma, diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index c1b637b709e11..76998c38da6d7 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1679,6 +1679,22 @@ static inline int pmd_protnone(pmd_t pmd) } #endif /* CONFIG_NUMA_BALANCING */ +/* + * Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values + * and let generic vmalloc, ioremap and page table update code know when + * arch_sync_kernel_mappings() needs to be called. + */ +#ifndef ARCH_PAGE_TABLE_SYNC_MASK +#define ARCH_PAGE_TABLE_SYNC_MASK 0 +#endif + +/* + * There is no default implementation for arch_sync_kernel_mappings(). It is + * relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK + * is 0. + */ +void arch_sync_kernel_mappings(unsigned long start, unsigned long end); + #endif /* CONFIG_MMU */ #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP @@ -1799,10 +1815,11 @@ static inline bool arch_has_pfn_modify_check(void) /* * Page Table Modification bits for pgtbl_mod_mask. * - * These are used by the p?d_alloc_track*() set of functions an in the generic - * vmalloc/ioremap code to track at which page-table levels entries have been - * modified. Based on that the code can better decide when vmalloc and ioremap - * mapping changes need to be synchronized to other page-tables in the system. + * These are used by the p?d_alloc_track*() and p*d_populate_kernel() + * functions in the generic vmalloc, ioremap and page table update code + * to track at which page-table levels entries have been modified. + * Based on that the code can better decide when page table changes need + * to be synchronized to other page-tables in the system. */ #define __PGTBL_PGD_MODIFIED 0 #define __PGTBL_P4D_MODIFIED 1 diff --git a/include/linux/phy.h b/include/linux/phy.h index a8e43dc88a540..ac0173a9d3fa8 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -772,10 +772,7 @@ struct phy_device { /* Generic phy_device::dev_flags */ #define PHY_F_NO_IRQ 0x80000000 -static inline struct phy_device *to_phy_device(const struct device *dev) -{ - return container_of(to_mdio_device(dev), struct phy_device, mdio); -} +#define to_phy_device(__dev) container_of_const(to_mdio_device(__dev), struct phy_device, mdio) /** * struct phy_tdr_config - Configuration of a TDR raw test diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 406855d73901a..2c8e86a9c4949 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -93,7 +93,9 @@ extern void pm_runtime_new_link(struct device *dev); extern void pm_runtime_drop_link(struct device_link *link); extern void pm_runtime_release_supplier(struct device_link *link); +int devm_pm_runtime_set_active_enabled(struct device *dev); extern int devm_pm_runtime_enable(struct device *dev); +int devm_pm_runtime_get_noresume(struct device *dev); /** * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. @@ -290,7 +292,9 @@ static inline void __pm_runtime_disable(struct device *dev, bool c) {} static inline void pm_runtime_allow(struct device *dev) {} static inline void pm_runtime_forbid(struct device *dev) {} +static inline int devm_pm_runtime_set_active_enabled(struct device *dev) { return 0; } static inline int devm_pm_runtime_enable(struct device *dev) { return 0; } +static inline int devm_pm_runtime_get_noresume(struct device *dev) { return 0; } static inline void pm_suspend_ignore_children(struct device *dev, bool enable) {} static inline void pm_runtime_get_noresume(struct device *dev) {} diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index ea62201c74c40..703d0c76cc9a0 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -27,6 +27,7 @@ enum { PROC_ENTRY_proc_read_iter = 1U << 1, PROC_ENTRY_proc_compat_ioctl = 1U << 2, + PROC_ENTRY_proc_lseek = 1U << 3, }; struct proc_ops { diff --git a/include/linux/quota.h b/include/linux/quota.h index aed9f4dff8438..b926b0dc29314 100644 --- a/include/linux/quota.h +++ b/include/linux/quota.h @@ -535,7 +535,7 @@ struct quota_info { const struct quota_format_ops *ops[MAXQUOTAS]; /* Operations for each type */ }; -int register_quota_format(struct quota_format_type *fmt); +void register_quota_format(struct quota_format_type *fmt); void unregister_quota_format(struct quota_format_type *fmt); struct quota_module_name { diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 72da69cc5764f..27531a0b3a6e7 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -97,9 +97,9 @@ static inline void __rcu_read_lock(void) static inline void __rcu_read_unlock(void) { - preempt_enable(); if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) rcu_read_unlock_strict(); + preempt_enable(); } static inline int rcu_preempt_depth(void) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 126f6b418f6af..559f758bf2eaa 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -104,7 +104,7 @@ extern int rcu_scheduler_active; void rcu_end_inkernel_boot(void); bool rcu_inkernel_boot_has_ended(void); bool rcu_is_watching(void); -#ifndef CONFIG_PREEMPTION +#ifndef CONFIG_PREEMPT_RCU void rcu_all_qs(void); #endif diff --git a/include/linux/rmap.h b/include/linux/rmap.h index ce92efef05b67..30491e14c3490 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -742,12 +742,7 @@ int folio_mkclean(struct folio *); int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, struct vm_area_struct *vma); -enum rmp_flags { - RMP_LOCKED = 1 << 0, - RMP_USE_SHARED_ZEROPAGE = 1 << 1, -}; - -void remove_migration_ptes(struct folio *src, struct folio *dst, int flags); +void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked); int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma); diff --git a/include/linux/sched.h b/include/linux/sched.h old mode 100644 new mode 100755 index 1299b4497d879..a6181beb77b8f --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -70,9 +70,12 @@ struct seq_file; struct sighand_struct; struct signal_struct; struct task_delay_info; +struct task_dma_buf_info; struct task_group; struct user_event_mm; +#include + /* * Task state bitmask. NOTE! These bits are also * encoded in fs/proc/array.c: get_task_state(). @@ -296,6 +299,11 @@ enum { extern void scheduler_tick(void); +#ifdef CONFIG_SLIM_SCHED +extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer); +extern void stop_shadow_tick_timer(void); +#endif + #define MAX_SCHEDULE_TIMEOUT LONG_MAX extern long schedule_timeout(long timeout); @@ -1516,9 +1524,15 @@ struct task_struct { */ struct callback_head l1d_flush_kill; #endif - ANDROID_KABI_RESERVE(1); +#ifdef CONFIG_SLIM_SCHED + ANDROID_KABI_USE(1, unsigned long sched_prop); + ANDROID_KABI_USE(2, struct sched_ext_entity *scx); + ANDROID_KABI_USE(3, struct task_dma_buf_info *dmabuf_info); +#else + ANDROID_KABI_USE(1, struct task_dma_buf_info *dmabuf_info); ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); +#endif ANDROID_KABI_RESERVE(4); ANDROID_KABI_RESERVE(5); ANDROID_KABI_RESERVE(6); @@ -1921,6 +1935,7 @@ static inline int task_nice(const struct task_struct *p) return PRIO_TO_NICE((p)->static_prio); } + extern int can_nice(const struct task_struct *p, const int nice); extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); @@ -2342,6 +2357,12 @@ enum rseq_event_mask { RSEQ_EVENT_MIGRATE = (1U << RSEQ_EVENT_MIGRATE_BIT), }; +#ifdef CONFIG_MEMBARRIER +# define RSEQ_EVENT_GUARD irq +#else +# define RSEQ_EVENT_GUARD preempt +#endif + static inline void rseq_set_notify_resume(struct task_struct *t) { if (t->rseq) @@ -2360,9 +2381,8 @@ static inline void rseq_handle_notify_resume(struct ksignal *ksig, static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { - preempt_disable(); - __set_bit(RSEQ_EVENT_SIGNAL_BIT, ¤t->rseq_event_mask); - preempt_enable(); + scoped_guard(RSEQ_EVENT_GUARD) + __set_bit(RSEQ_EVENT_SIGNAL_BIT, ¤t->rseq_event_mask); rseq_handle_notify_resume(ksig, regs); } diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h new file mode 100755 index 0000000000000..b737a00c1373c --- /dev/null +++ b/include/linux/sched/ext.h @@ -0,0 +1,647 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#ifndef _LINUX_SCHED_EXT_H +#define _LINUX_SCHED_EXT_H + +#ifdef CONFIG_SCHED_CLASS_EXT + +#include + +enum scx_consts { + SCX_OPS_NAME_LEN = 128, + SCX_EXIT_REASON_LEN = 128, + SCX_EXIT_BT_LEN = 64, + SCX_EXIT_MSG_LEN = 1024, + + SCX_SLICE_DFL = 20 * NSEC_PER_MSEC, + SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */ +}; + +/* + * DSQ (dispatch queue) IDs are 64bit of the format: + * + * Bits: [63] [62 .. 0] + * [ B] [ ID ] + * + * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs + * ID: 63 bit ID + * + * Built-in IDs: + * + * Bits: [63] [62] [61..32] [31 .. 0] + * [ 1] [ L] [ R ] [ V ] + * + * 1: 1 for built-in DSQs. + * L: 1 for LOCAL_ON DSQ IDs, 0 for others + * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. + */ +enum scx_dsq_id_flags { + SCX_DSQ_FLAG_BUILTIN = 1LLU << 63, + SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62, + + SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0, + SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, + SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, + SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, + SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, +}; + +enum scx_exit_type { + SCX_EXIT_NONE, + SCX_EXIT_DONE, + + SCX_EXIT_UNREG = 64, /* BPF unregistration */ + SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ + + SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ + SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ + SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ +}; + +/* + * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is + * being disabled. + */ +struct scx_exit_info { + /* %SCX_EXIT_* - broad category of the exit reason */ + enum scx_exit_type type; + /* textual representation of the above */ + char reason[SCX_EXIT_REASON_LEN]; + /* number of entries in the backtrace */ + u32 bt_len; + /* backtrace if exiting due to an error */ + unsigned long bt[SCX_EXIT_BT_LEN]; + /* extra message */ + char msg[SCX_EXIT_MSG_LEN]; +}; + +/* sched_ext_ops.flags */ +enum scx_ops_flags { + /* + * Keep built-in idle tracking even if ops.update_idle() is implemented. + */ + SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, + + /* + * By default, if there are no other task to run on the CPU, ext core + * keeps running the current task even after its slice expires. If this + * flag is specified, such tasks are passed to ops.enqueue() with + * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. + */ + SCX_OPS_ENQ_LAST = 1LLU << 1, + + /* + * An exiting task may schedule after PF_EXITING is set. In such cases, + * bpf_task_from_pid() may not be able to find the task and if the BPF + * scheduler depends on pid lookup for dispatching, the task will be + * lost leading to various issues including RCU grace period stalls. + * + * To mask this problem, by default, unhashed tasks are automatically + * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't + * depend on pid lookups and wants to handle these tasks directly, the + * following flag can be used. + */ + SCX_OPS_ENQ_EXITING = 1LLU << 2, + + /* + * CPU cgroup knob enable flags + */ + SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16, /* cpu.weight */ + + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | + SCX_OPS_ENQ_LAST | + SCX_OPS_ENQ_EXITING | + SCX_OPS_CGROUP_KNOB_WEIGHT, +}; + +/* argument container for ops.enable() and friends */ +struct scx_enable_args { +}; + +/* argument container for ops->cgroup_init() */ +struct scx_cgroup_init_args { + /* the weight of the cgroup [1..10000] */ + u32 weight; +}; + +enum scx_cpu_preempt_reason { + /* next task is being scheduled by &sched_class_rt */ + SCX_CPU_PREEMPT_RT, + /* next task is being scheduled by &sched_class_dl */ + SCX_CPU_PREEMPT_DL, + /* next task is being scheduled by &sched_class_stop */ + SCX_CPU_PREEMPT_STOP, + /* unknown reason for SCX being preempted */ + SCX_CPU_PREEMPT_UNKNOWN, +}; + +/* + * Argument container for ops->cpu_acquire(). Currently empty, but may be + * expanded in the future. + */ +struct scx_cpu_acquire_args {}; + +/* argument container for ops->cpu_release() */ +struct scx_cpu_release_args { + /* the reason the CPU was preempted */ + enum scx_cpu_preempt_reason reason; + + /* the task that's going to be scheduled on the CPU */ + struct task_struct *task; +}; + +/** + * struct sched_ext_ops - Operation table for BPF scheduler implementation + * + * Userland can implement an arbitrary scheduling policy by implementing and + * loading operations in this table. + */ +struct sched_ext_ops { + /** + * select_cpu - Pick the target CPU for a task which is being woken up + * @p: task being woken up + * @prev_cpu: the cpu @p was on before sleeping + * @wake_flags: SCX_WAKE_* + * + * Decision made here isn't final. @p may be moved to any CPU while it + * is getting dispatched for execution later. However, as @p is not on + * the rq at this point, getting the eventual execution CPU right here + * saves a small bit of overhead down the line. + * + * If an idle CPU is returned, the CPU is kicked and will try to + * dispatch. While an explicit custom mechanism can be added, + * select_cpu() serves as the default way to wake up idle CPUs. + */ + s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); + + /** + * enqueue - Enqueue a task on the BPF scheduler + * @p: task being enqueued + * @enq_flags: %SCX_ENQ_* + * + * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() + * or enqueue on the BPF scheduler. If not directly dispatched, the bpf + * scheduler owns @p and if it fails to dispatch @p, the task will + * stall. + */ + void (*enqueue)(struct task_struct *p, u64 enq_flags); + + /** + * dequeue - Remove a task from the BPF scheduler + * @p: task being dequeued + * @deq_flags: %SCX_DEQ_* + * + * Remove @p from the BPF scheduler. This is usually called to isolate + * the task while updating its scheduling properties (e.g. priority). + * + * The ext core keeps track of whether the BPF side owns a given task or + * not and can gracefully ignore spurious dispatches from BPF side, + * which makes it safe to not implement this method. However, depending + * on the scheduling logic, this can lead to confusing behaviors - e.g. + * scheduling position not being updated across a priority change. + */ + void (*dequeue)(struct task_struct *p, u64 deq_flags); + + /** + * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs + * @cpu: CPU to dispatch tasks for + * @prev: previous task being switched out + * + * Called when a CPU's local dsq is empty. The operation should dispatch + * one or more tasks from the BPF scheduler into the DSQs using + * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using + * scx_bpf_consume(). + * + * The maximum number of times scx_bpf_dispatch() can be called without + * an intervening scx_bpf_consume() is specified by + * ops.dispatch_max_batch. See the comments on top of the two functions + * for more details. + * + * When not %NULL, @prev is an SCX task with its slice depleted. If + * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in + * @prev->scx.flags, it is not enqueued yet and will be enqueued after + * ops.dispatch() returns. To keep executing @prev, return without + * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. + */ + void (*dispatch)(s32 cpu, struct task_struct *prev); + + /** + * runnable - A task is becoming runnable on its associated CPU + * @p: task becoming runnable + * @enq_flags: %SCX_ENQ_* + * + * This and the following three functions can be used to track a task's + * execution state transitions. A task becomes ->runnable() on a CPU, + * and then goes through one or more ->running() and ->stopping() pairs + * as it runs on the CPU, and eventually becomes ->quiescent() when it's + * done running on the CPU. + * + * @p is becoming runnable on the CPU because it's + * + * - waking up (%SCX_ENQ_WAKEUP) + * - being moved from another CPU + * - being restored after temporarily taken off the queue for an + * attribute change. + * + * This and ->enqueue() are related but not coupled. This operation + * notifies @p's state transition and may not be followed by ->enqueue() + * e.g. when @p is being dispatched to a remote CPU. Likewise, a task + * may be ->enqueue()'d without being preceded by this operation e.g. + * after exhausting its slice. + */ + void (*runnable)(struct task_struct *p, u64 enq_flags); + + /** + * running - A task is starting to run on its associated CPU + * @p: task starting to run + * + * See ->runnable() for explanation on the task state notifiers. + */ + void (*running)(struct task_struct *p); + + /** + * stopping - A task is stopping execution + * @p: task stopping to run + * @runnable: is task @p still runnable? + * + * See ->runnable() for explanation on the task state notifiers. If + * !@runnable, ->quiescent() will be invoked after this operation + * returns. + */ + void (*stopping)(struct task_struct *p, bool runnable); + + /** + * quiescent - A task is becoming not runnable on its associated CPU + * @p: task becoming not runnable + * @deq_flags: %SCX_DEQ_* + * + * See ->runnable() for explanation on the task state notifiers. + * + * @p is becoming quiescent on the CPU because it's + * + * - sleeping (%SCX_DEQ_SLEEP) + * - being moved to another CPU + * - being temporarily taken off the queue for an attribute change + * (%SCX_DEQ_SAVE) + * + * This and ->dequeue() are related but not coupled. This operation + * notifies @p's state transition and may not be preceded by ->dequeue() + * e.g. when @p is being dispatched to a remote CPU. + */ + void (*quiescent)(struct task_struct *p, u64 deq_flags); + + /** + * yield - Yield CPU + * @from: yielding task + * @to: optional yield target task + * + * If @to is NULL, @from is yielding the CPU to other runnable tasks. + * The BPF scheduler should ensure that other available tasks are + * dispatched before the yielding task. Return value is ignored in this + * case. + * + * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf + * scheduler can implement the request, return %true; otherwise, %false. + */ + bool (*yield)(struct task_struct *from, struct task_struct *to); + + /** + * core_sched_before - Task ordering for core-sched + * @a: task A + * @b: task B + * + * Used by core-sched to determine the ordering between two tasks. See + * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on + * core-sched. + * + * Both @a and @b are runnable and may or may not currently be queued on + * the BPF scheduler. Should return %true if @a should run before @b. + * %false if there's no required ordering or @b should run before @a. + * + * If not specified, the default is ordering them according to when they + * became runnable. + */ + bool (*core_sched_before)(struct task_struct *a,struct task_struct *b); + + /** + * set_weight - Set task weight + * @p: task to set weight for + * @weight: new eight [1..10000] + * + * Update @p's weight to @weight. + */ + void (*set_weight)(struct task_struct *p, u32 weight); + + /** + * set_cpumask - Set CPU affinity + * @p: task to set CPU affinity for + * @cpumask: cpumask of cpus that @p can run on + * + * Update @p's CPU affinity to @cpumask. + */ + void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask); + + /** + * update_idle - Update the idle state of a CPU + * @cpu: CPU to udpate the idle state for + * @idle: whether entering or exiting the idle state + * + * This operation is called when @rq's CPU goes or leaves the idle + * state. By default, implementing this operation disables the built-in + * idle CPU tracking and the following helpers become unavailable: + * + * - scx_bpf_select_cpu_dfl() + * - scx_bpf_test_and_clear_cpu_idle() + * - scx_bpf_pick_idle_cpu() + * - scx_bpf_any_idle_cpu() + * + * The user also must implement ops.select_cpu() as the default + * implementation relies on scx_bpf_select_cpu_dfl(). + * + * If you keep the built-in idle tracking, specify the + * %SCX_OPS_KEEP_BUILTIN_IDLE flag. + */ + void (*update_idle)(s32 cpu, bool idle); + + /** + * cpu_acquire - A CPU is becoming available to the BPF scheduler + * @cpu: The CPU being acquired by the BPF scheduler. + * @args: Acquire arguments, see the struct definition. + * + * A CPU that was previously released from the BPF scheduler is now once + * again under its control. + */ + void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); + + /** + * cpu_release - A CPU is taken away from the BPF scheduler + * @cpu: The CPU being released by the BPF scheduler. + * @args: Release arguments, see the struct definition. + * + * The specified CPU is no longer under the control of the BPF + * scheduler. This could be because it was preempted by a higher + * priority sched_class, though there may be other reasons as well. The + * caller should consult @args->reason to determine the cause. + */ + void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); + + /** + * cpu_online - A CPU became online + * @cpu: CPU which just came up + * + * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks + * associated with other CPUs beforehand. + */ + void (*cpu_online)(s32 cpu); + + /** + * cpu_offline - A CPU is going offline + * @cpu: CPU which is going offline + * + * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks + * associated with other CPUs afterwards. + */ + void (*cpu_offline)(s32 cpu); + + /** + * prep_enable - Prepare to enable BPF scheduling for a task + * @p: task to prepare BPF scheduling for + * @args: enable arguments, see the struct definition + * + * Either we're loading a BPF scheduler or a new task is being forked. + * Prepare BPF scheduling for @p. This operation may block and can be + * used for allocations. + * + * Return 0 for success, -errno for failure. An error return while + * loading will abort loading of the BPF scheduler. During a fork, will + * abort the specific fork. + */ + s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args); + + /** + * enable - Enable BPF scheduling for a task + * @p: task to enable BPF scheduling for + * @args: enable arguments, see the struct definition + * + * Enable @p for BPF scheduling. @p is now in the cgroup specified for + * the preceding prep_enable() and will start running soon. + */ + void (*enable)(struct task_struct *p, struct scx_enable_args *args); + + /** + * cancel_enable - Cancel prep_enable() + * @p: task being canceled + * @args: enable arguments, see the struct definition + * + * @p was prep_enable()'d but failed before reaching enable(). Undo the + * preparation. + */ + void (*cancel_enable)(struct task_struct *p, + struct scx_enable_args *args); + + /** + * disable - Disable BPF scheduling for a task + * @p: task to disable BPF scheduling for + * + * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. + * Disable BPF scheduling for @p. + */ + void (*disable)(struct task_struct *p); + + /* + * All online ops must come before ops.init(). + */ + + /** + * init - Initialize the BPF scheduler + */ + s32 (*init)(void); + + /** + * exit - Clean up after the BPF scheduler + * @info: Exit info + */ + void (*exit)(struct scx_exit_info *info); + + /** + * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch + */ + u32 dispatch_max_batch; + + /** + * flags - %SCX_OPS_* flags + */ + u64 flags; + + /** + * timeout_ms - The maximum amount of time, in milliseconds, that a + * runnable task should be able to wait before being scheduled. The + * maximum timeout may not exceed the default timeout of 30 seconds. + * + * Defaults to the maximum allowed timeout value of 30 seconds. + */ + u32 timeout_ms; + + /** + * name - BPF scheduler's name + * + * Must be a non-zero valid BPF object name including only isalnum(), + * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the + * BPF scheduler is enabled. + */ + char name[SCX_OPS_NAME_LEN]; +}; + +/* + * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the + * scheduler core and the BPF scheduler. See the documentation for more details. + */ +struct scx_dispatch_q { + raw_spinlock_t lock; + struct list_head fifo; /* processed in dispatching order */ + struct rb_root_cached priq; /* processed in p->scx.dsq_vtime order */ + u32 nr; + u64 id; + struct llist_node free_node; + struct rcu_head rcu; +}; + +/* scx_entity.flags */ +enum scx_ent_flags { + SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ + SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ + SCX_TASK_ENQ_LOCAL = 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */ + + SCX_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ + SCX_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ + + SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ + SCX_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ + + SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ +}; + +/* scx_entity.dsq_flags */ +enum scx_ent_dsq_flags { + SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ +}; + +/* + * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from + * everywhere and the following bits track which kfunc sets are currently + * allowed for %current. This simple per-task tracking works because SCX ops + * nest in a limited way. BPF will likely implement a way to allow and disallow + * kfuncs depending on the calling context which will replace this manual + * mechanism. See scx_kf_allow(). + */ +enum scx_kf_mask { + SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */ + /* all non-sleepables may be nested inside INIT and SLEEPABLE */ + SCX_KF_INIT = 1 << 0, /* running ops.init() */ + SCX_KF_SLEEPABLE = 1 << 1, /* other sleepable init operations */ + /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ + SCX_KF_CPU_RELEASE = 1 << 2, /* ops.cpu_release() */ + /* ops.dequeue (in REST) may be nested inside DISPATCH */ + SCX_KF_DISPATCH = 1 << 3, /* ops.dispatch() */ + SCX_KF_ENQUEUE = 1 << 4, /* ops.enqueue() */ + SCX_KF_REST = 1 << 5, /* other rq-locked operations */ + + __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | + SCX_KF_ENQUEUE | SCX_KF_REST, + __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_REST, +}; + +#define RAVG_HIST_SIZE 5 +struct scx_sched_task_stats { + u64 mark_start; + u64 window_start; + u32 sum; + u32 sum_history[RAVG_HIST_SIZE]; + int cidx; + + u32 demand; + u16 demand_scaled; + void *sdsq; +}; + + + +/* + * The following is embedded in task_struct and contains all fields necessary + * for a task to be scheduled by SCX. + */ +struct sched_ext_entity { + struct scx_dispatch_q *dsq; + struct { + struct list_head fifo; /* dispatch order */ + struct rb_node priq; /* p->scx.dsq_vtime order */ + } dsq_node; + struct list_head watchdog_node; + u32 flags; /* protected by rq lock */ + u32 dsq_flags; /* protected by dsq lock */ + u32 weight; + s32 sticky_cpu; + s32 holding_cpu; + u32 kf_mask; /* see scx_kf_mask above */ + struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ + atomic64_t ops_state; + unsigned long runnable_at; +#ifdef CONFIG_SCHED_CORE + u64 core_sched_at; /* see scx_prio_less() */ +#endif + + /* BPF scheduler modifiable fields */ + + /* + * Runtime budget in nsecs. This is usually set through + * scx_bpf_dispatch() but can also be modified directly by the BPF + * scheduler. Automatically decreased by SCX as the task executes. On + * depletion, a scheduling event is triggered. + * + * This value is cleared to zero if the task is preempted by + * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the + * task ran. Use p->se.sum_exec_runtime instead. + */ + u64 slice; + + /* + * Used to order tasks when dispatching to the vtime-ordered priority + * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime() + * but can also be modified directly by the BPF scheduler. Modifying it + * while a task is queued on a dsq may mangle the ordering and is not + * recommended. + */ + u64 dsq_vtime; + + /* + * If set, reject future sched_setscheduler(2) calls updating the policy + * to %SCHED_EXT with -%EACCES. + * + * If set from ops.prep_enable() and the task's policy is already + * %SCHED_EXT, which can happen while the BPF scheduler is being loaded + * or by inhering the parent's policy during fork, the task's policy is + * rejected and forcefully reverted to %SCHED_NORMAL. The number of such + * events are reported through /sys/kernel/debug/sched_ext::nr_rejected. + */ + bool disallow; /* reject switching into SCX */ + + /* cold fields */ + struct list_head tasks_node; + struct task_struct *task; + struct scx_sched_task_stats sts; +}; + +void sched_ext_free(struct task_struct *p); + +#else /* !CONFIG_SCHED_CLASS_EXT */ + +static inline void sched_ext_free(struct task_struct *p) {} + +#endif /* CONFIG_SCHED_CLASS_EXT */ +#endif /* _LINUX_SCHED_EXT_H */ diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h new file mode 100755 index 0000000000000..a298ee6f64cca --- /dev/null +++ b/include/linux/sched/hmbird.h @@ -0,0 +1,387 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HMBIRD scheduler class: Documentation/scheduler/hmbird.rst + * + * Copyright (c) 2024 OPlus. + * Copyright (c) 2024 Dao Huang + * Copyright (c) 2024 Yuxing Wang + * Copyright (c) 2024 Taiyu Li + */ +#ifndef _LINUX_SCHED_HMBIRD_H +#define _LINUX_SCHED_HMBIRD_H + +#define HMBIRD_TS_IDX 1 +#define HMBIRD_OPS_IDX 14 +#define HMBIRD_RQ_IDX 15 + +#define get_hmbird_ts(p) \ + ((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX])) + +#define get_hmbird_rq(rq) \ + ((struct hmbird_rq *)(rq->android_oem_data1[HMBIRD_RQ_IDX])) + +#define get_hmbird_ops(rq) \ + ((struct hmbird_ops *)(rq->android_oem_data1[HMBIRD_OPS_IDX])) + +#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */ +/* + * Every task has a DEADLINE_LEVEL which stands for + * max schedule latency this task can afford. LEVEL1~5 + * for user-aware tasks, LEVEL6~9 for other tasks. + */ +#define SCHED_PROP_DEADLINE_LEVEL0 (0) +#define SCHED_PROP_DEADLINE_LEVEL1 (1) +#define SCHED_PROP_DEADLINE_LEVEL2 (2) +#define SCHED_PROP_DEADLINE_LEVEL3 (3) +#define SCHED_PROP_DEADLINE_LEVEL4 (4) +#define SCHED_PROP_DEADLINE_LEVEL5 (5) +#define SCHED_PROP_DEADLINE_LEVEL6 (6) +#define SCHED_PROP_DEADLINE_LEVEL7 (7) +#define SCHED_PROP_DEADLINE_LEVEL8 (8) +#define SCHED_PROP_DEADLINE_LEVEL9 (9) +/* + * Distinguish tasks into periodical tasks which requires + * low schedule latency and non-periodical tasks which are + * not sensitive to schedule latency. + */ +#define SCHED_HMBIRD_DSQ_TYPE_PERIOD (0) /* period dsq of hmbird */ +#define SCHED_HMBIRD_DSQ_TYPE_NON_PERIOD (1) /* non period dsq of hmbird */ + +#define TOP_TASK_BITS_MASK (0xFF) +#define TOP_TASK_BITS (8) +#include + +extern atomic_t non_hmbird_task; +extern atomic_t __hmbird_ops_enabled; +#define hmbird_enabled() atomic_read(&__hmbird_ops_enabled) +#define MAX_GLOBAL_DSQS (10) + +enum hmbird_consts { + HMBIRD_SLICE_DFL = 1 * NSEC_PER_MSEC, + HMBIRD_SLICE_ISO = 8 * HMBIRD_SLICE_DFL, + HMBIRD_SLICE_INF = U64_MAX, /* infinite, implies nohz */ +}; + +/* + * DSQ (dispatch queue) IDs are 64bit of the format: + * + * Bits: [63] [62 .. 0] + * [ B] [ ID ] + * + * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs + * ID: 63 bit ID + * + * Built-in IDs: + * + * Bits: [63] [62] [61..32] [31 .. 0] + * [ 1] [ L] [ R ] [ V ] + * + * 1: 1 for built-in DSQs. + * L: 1 for LOCAL_ON DSQ IDs, 0 for others + * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value. + */ +enum hmbird_dsq_id_flags { + HMBIRD_DSQ_FLAG_BUILTIN = 1LLU << 63, + HMBIRD_DSQ_FLAG_LOCAL_ON = 1LLU << 62, + + HMBIRD_DSQ_INVALID = HMBIRD_DSQ_FLAG_BUILTIN | 0, + HMBIRD_DSQ_GLOBAL = HMBIRD_DSQ_FLAG_BUILTIN | 1, + HMBIRD_DSQ_LOCAL = HMBIRD_DSQ_FLAG_BUILTIN | 2, + HMBIRD_DSQ_LOCAL_ON = HMBIRD_DSQ_FLAG_BUILTIN | HMBIRD_DSQ_FLAG_LOCAL_ON, + HMBIRD_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, +}; + +enum hmbird_switch_type { + HMBIRD_SWITCH_PROC, + HMBIRD_SWITCH_ERR_WDT, + HMBIRD_SWITCH_ERR_HB, + HMBIRD_SWITCH_ERR_DSQ, + HMBIRD_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ + HMBIRD_EXIT_ERROR_HEARTBEAT, /* heart beat has stopped */ +}; + +/* + * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the + * scheduler core and the BPF scheduler. See the documentation for more details. + */ +struct hmbird_dispatch_q { + raw_spinlock_t lock; + struct list_head fifo; /* processed in dispatching order */ + struct rb_root_cached priq; + u32 nr; + u64 id; + struct llist_node free_node; + struct rcu_head rcu; + u64 last_consume_at; + bool is_timeout; +}; + +/* hmbird_entity.flags */ +enum hmbird_ent_flags { + HMBIRD_TASK_QUEUED = 1 << 0, /* on hmbird runqueue */ + HMBIRD_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */ + HMBIRD_TASK_ENQ_LOCAL = 1 << 2, /* used by hmbird_select_cpu_dfl, set HMBIRD_ENQ_LOCAL */ + + HMBIRD_TASK_OPS_PREPPED = 1 << 8, /* prepared for BPF scheduler enable */ + HMBIRD_TASK_OPS_ENABLED = 1 << 9, /* task has BPF scheduler enabled */ + + HMBIRD_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */ + HMBIRD_TASK_DEQD_FOR_SLEEP = 1 << 17, /* last dequeue was for SLEEP */ + + HMBIRD_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ +}; + +/* hmbird_entity.dsq_flags */ +enum hmbird_ent_dsq_flags { + HMBIRD_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ +}; + +#define RAVG_HIST_SIZE 5 +struct hmbird_sched_task_stats { + u64 mark_start; + u64 window_start; + u32 sum; + u32 sum_history[RAVG_HIST_SIZE]; + int cidx; + + u32 demand; + u16 demand_scaled; + void *sdsq; +}; + +struct hmbird_sched_rq_stats { + u64 window_start; + u64 latest_clock; + u32 prev_window_size; + u64 task_exec_scale; + u64 prev_runnable_sum; + u64 curr_runnable_sum; + int *sched_ravg_window_ptr; +}; + +/* + * The following is embedded in task_struct and contains all fields necessary + * for a task to be scheduled by HMBIRD. + */ +struct hmbird_entity { + struct hmbird_dispatch_q *dsq; + struct { + struct list_head fifo; /* dispatch order */ + struct rb_node priq; + } dsq_node; + struct list_head watchdog_node; + u32 flags; /* protected by rq lock */ + u32 dsq_flags; /* protected by dsq lock */ + u32 weight; + s32 sticky_cpu; + s32 holding_cpu; + u32 kf_mask; /* see hmbird_kf_mask above */ + struct task_struct *kf_tasks[2]; /* see HMBIRD_CALL_OP_TASK() */ + atomic64_t ops_state; + unsigned long runnable_at; + u64 slice; + u64 dsq_vtime; + bool disallow; /* reject switching into HMBIRD */ + u16 demand_scaled; + + /* cold fields */ + struct list_head tasks_node; + struct task_struct *task; + const struct sched_class *sched_class; + unsigned long sched_prop; + unsigned long top_task_prop; + struct hmbird_sched_task_stats sts; + unsigned long running_at; + int gdsq_idx; + + s32 critical_affinity_cpu; + int dsq_sync_ux; + + int tick_hit_count; /* tick hit boost */ + unsigned long start_jiffies; +}; + + +/* + * All variables use 64bits width, + * Avoid parsing problems caused by automatic alignment(with padding) of structures. + */ + +/* NOTING : Must align to 64bits. */ +#define DESC_STR_LEN (32) +/* Supporti up to three-dimensional arrays. */ +#define PARSE_DIMENS (3) +struct meta_desc_t { + char desc_str[DESC_STR_LEN]; + u64 len; + u64 parse[PARSE_DIMENS]; +}; + +#define MAX_SWITCHS (5) +struct hmbird_switch_t { + u64 switch_at; + u64 is_success; + u64 end_state; + u64 switch_reason; +}; +#define SWITCH_ITEMS (sizeof(struct hmbird_switch_t) / sizeof(u64)) + +#define MAX_EXCEPS (5) +enum excep_id { + NO_CGROUP_L1, + MODULE_UNLOAD, + ALREADY_ENABLED, + ALREADY_DISABLED, + INIT_TASK_FAIL, + ALLOC_RQSCX_FAIL, + DSQ_ID_ERR, + CPU_NO_MASK, + SCAN_ENTITY_NULL, + ITER_RET_NULL, + DEQ_DEQING, + ENQ_EXIST1, + ENQ_EXIST2, + TASK_LINKED1, + TASK_UNLINKED, + TASK_LINKED2, + TASK_UNQUED, + TASK_UNWATCHED, + HMBIRD_OPN, + TASK_WATCHED, + RQ_NO_RUNNING, + EXTRA_FLAGS, + HOLDING_CPU1, + HOLDING_CPU2, + TASK_OPS_PREPPED, + TASK_OPS_UNPREPPED, + HMBIRD_OPS_ERR, + MAX_EXCEP_ID, +}; + +struct snap_misc_t { + u64 hmbird_enabled; + u64 curr_ss; + u64 hmbird_ops_enable_state_var; + u64 non_ext_task; + u64 parctrl_high_ratio; + u64 parctrl_low_ratio; + u64 parctrl_high_ratio_l; + u64 parctrl_low_ratio_l; + u64 isoctrl_high_ratio; + u64 isoctrl_low_ratio; + u64 misfit_ds; + u64 partial_enable; + u64 iso_free_rescue; + u64 isolate_ctrl; + u64 snap_jiffies; + u64 snap_time; +}; +#define SNAP_ITEMS (sizeof(struct snap_misc_t) / sizeof(u64)) + +struct panic_snapshot_t { + /* what time dose the first task of dsq turn in runnable, check starvation */ + struct meta_desc_t runnable_at_meta; + u64 runnable_at[MAX_GLOBAL_DSQS]; + + struct meta_desc_t rq_nr_meta; + u64 rq_nr[NR_CPUS]; + + struct meta_desc_t scxrq_nr_meta; + u64 scxrq_nr[NR_CPUS]; + + struct meta_desc_t snap_misc_meta; + struct snap_misc_t snap_misc; +}; + +/* + * Do not record info in hot paths unless absolutely necessary, + * The impact on performance should be minimized. + */ +struct kernel_info_t { + struct meta_desc_t sw_rec_meta; + struct hmbird_switch_t sw_rec[MAX_SWITCHS]; + + struct meta_desc_t sw_idx_meta; + u64 sw_idx; + + struct meta_desc_t excep_rec_meta; + u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS]; + + struct meta_desc_t excep_idx_meta; + u64 excep_idx[MAX_EXCEP_ID]; + + /* snapshot while panic. */ + struct panic_snapshot_t snap; +}; + +struct ko_info_t {}; + +struct md_meta_t { + u64 self_len; + u64 unit_size; + u64 desc_meta_len; + u64 desc_str_len; + u64 switches; + u64 exceps; + u64 global_dsqs; + u64 parse_dimens; + u64 nr_cpus; + u64 real_cpus; + u64 nr_meta_desc; + u64 dump_real_size; +}; + +struct md_info_t { + struct md_meta_t meta; + struct kernel_info_t kern_dump; + struct ko_info_t ko_dump; +}; + +static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies) +{ + u64 *idx; + + if (!rec) + return; + + idx = &rec->kern_dump.excep_idx[id]; + rec->kern_dump.excep_rec[id][*idx] = jiffies; + *idx = ++(*idx) % MAX_EXCEPS; +} + +static inline void sw_update(struct md_info_t *rec, u64 switch_at, + u64 is_success, u64 end_state, u64 switch_reason) +{ + u64 *idx; + + if (!rec) + return; + + idx = &rec->kern_dump.sw_idx; + rec->kern_dump.sw_rec[*idx].switch_at = switch_at; + rec->kern_dump.sw_rec[*idx].is_success = is_success; + rec->kern_dump.sw_rec[*idx].end_state = end_state; + rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason; + *idx = ++(*idx) % MAX_SWITCHS; +} + +struct hmbird_ops { + bool (*scx_enable)(void); + bool (*check_non_task)(void); + void (*do_sched_yield_before)(long *skip); + void (*window_rollover_run_once)(struct rq *rq); + void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size); + int (*hmbird_get_boost_enable)(void); + unsigned int (*hmbird_get_boost_bottom_freq)(void); + int (*hmbird_get_boost_weight)(void); +}; + +void hmbird_free(struct task_struct *p); + +enum DSQ_SYNC_UX_FLAG { + DSQ_SYNC_UX_NONE = 0, + DSQ_SYNC_STATIC_UX = 1, + DSQ_SYNC_INHERIT_UX = 1 << 1, +}; + +#endif /* _LINUX_SCHED_HMBIRD_H */ diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index a23af225c8983..03d35e3eda3c2 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -61,7 +61,8 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); extern void init_idle(struct task_struct *idle, int cpu); extern int sched_fork(unsigned long clone_flags, struct task_struct *p); -extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); +extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); +extern void sched_cancel_fork(struct task_struct *p); extern void sched_post_fork(struct task_struct *p); extern void sched_dead(struct task_struct *p); diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 72c65bef5505d..edfdfd6e3cc3b 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -286,6 +286,14 @@ void arch_update_thermal_pressure(const struct cpumask *cpus, { } #endif +#ifndef arch_scale_freq_ref +static __always_inline +unsigned int arch_scale_freq_ref(int cpu) +{ + return 0; +} +#endif + static inline int task_node(const struct task_struct *p) { return cpu_to_node(task_cpu(p)); diff --git a/include/linux/shdma-base.h b/include/linux/shdma-base.h index 6dfd05ef5c2d9..03ba4dab2ef73 100644 --- a/include/linux/shdma-base.h +++ b/include/linux/shdma-base.h @@ -96,7 +96,7 @@ struct shdma_ops { int (*desc_setup)(struct shdma_chan *, struct shdma_desc *, dma_addr_t, dma_addr_t, size_t *); int (*set_slave)(struct shdma_chan *, int, dma_addr_t, bool); - void (*setup_xfer)(struct shdma_chan *, int); + int (*setup_xfer)(struct shdma_chan *, int); void (*start_xfer)(struct shdma_chan *, struct shdma_desc *); struct shdma_desc *(*embedded_desc)(void *, int); bool (*chan_irq)(struct shdma_chan *, int); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index bc9a3662d1a33..13fa08cfbb175 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2883,6 +2883,29 @@ static inline void skb_reset_transport_header(struct sk_buff *skb) skb->transport_header = skb->data - skb->head; } +/** + * skb_reset_transport_header_careful - conditionally reset transport header + * @skb: buffer to alter + * + * Hardened version of skb_reset_transport_header(). + * + * Returns: true if the operation was a success. + */ +static inline bool __must_check +skb_reset_transport_header_careful(struct sk_buff *skb) +{ + long offset = skb->data - skb->head; + + if (unlikely(offset != (typeof(skb->transport_header))offset)) + return false; + + if (unlikely(offset == (typeof(skb->transport_header))~0U)) + return false; + + skb->transport_header = offset; + return true; +} + static inline void skb_set_transport_header(struct sk_buff *skb, const int offset) { @@ -3539,7 +3562,13 @@ static inline void *skb_frag_address(const skb_frag_t *frag) */ static inline void *skb_frag_address_safe(const skb_frag_t *frag) { - void *ptr = page_address(skb_frag_page(frag)); + struct page *page = skb_frag_page(frag); + void *ptr; + + if (!page) + return NULL; + + ptr = page_address(page); if (unlikely(!ptr)) return NULL; @@ -4003,6 +4032,8 @@ int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset, struct ahash_request *hash); int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset, struct iov_iter *from, int len); +int skb_copy_datagram_from_iter_full(struct sk_buff *skb, int offset, + struct iov_iter *from, int len); int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm); void skb_free_datagram(struct sock *sk, struct sk_buff *skb); void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len); diff --git a/include/linux/soc/mediatek/gzvm_drv.h b/include/linux/soc/mediatek/gzvm_drv.h index b234edb3b9f31..1a532f5bcac9c 100644 --- a/include/linux/soc/mediatek/gzvm_drv.h +++ b/include/linux/soc/mediatek/gzvm_drv.h @@ -13,6 +13,7 @@ #include #include #include +#include /* GZVM version encode */ #define GZVM_DRV_MAJOR_VERSION 16 @@ -177,6 +178,7 @@ struct gzvm_vm_stat { * page mailbox at the same time * @stat: information for VM memory statistics * @debug_dir: debugfs directory node for VM memory statistics + * @kref: reference counter between vm and vcpu when destroy happened */ struct gzvm { struct gzvm_driver *gzvm_drv; @@ -212,6 +214,7 @@ struct gzvm { struct gzvm_vm_stat stat; struct dentry *debug_dir; + struct kref kref; }; long gzvm_dev_ioctl_check_extension(struct gzvm *gzvm, unsigned long args); @@ -290,6 +293,7 @@ int gzvm_arch_memregion_purpose(struct gzvm *gzvm, int gzvm_arch_set_dtb_config(struct gzvm *gzvm, struct gzvm_dtb_config *args); int gzvm_init_ioeventfd(struct gzvm *gzvm); +void gzvm_vm_ioeventfd_release(struct gzvm *gzvm); int gzvm_ioeventfd(struct gzvm *gzvm, struct gzvm_ioeventfd *args); bool gzvm_ioevent_write(struct gzvm_vcpu *vcpu, __u64 addr, int len, const void *val); @@ -297,5 +301,7 @@ void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt); struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr); void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); +void gzvm_vm_put(struct gzvm *gzvm); +void gzvm_vm_get(struct gzvm *gzvm); #endif /* __GZVM_DRV_H__ */ diff --git a/include/linux/sprintf.h b/include/linux/sprintf.h index 33dcbec719254..9e13b8040b12c 100644 --- a/include/linux/sprintf.h +++ b/include/linux/sprintf.h @@ -4,6 +4,7 @@ #include #include +#include int num_to_str(char *buf, int size, unsigned long long num, unsigned int width); diff --git a/include/linux/string.h b/include/linux/string.h index 5077776e995e0..ce137830a0b99 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -2,6 +2,7 @@ #ifndef _LINUX_STRING_H_ #define _LINUX_STRING_H_ +#include #include /* for inline */ #include /* for size_t */ #include /* for NULL */ diff --git a/include/linux/swap.h b/include/linux/swap.h index 798cbc9b6735f..78b40d1aa3ae3 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -401,6 +401,16 @@ void folio_add_lru_vma(struct folio *, struct vm_area_struct *); void mark_page_accessed(struct page *); void folio_mark_accessed(struct folio *); +static inline bool folio_may_be_lru_cached(struct folio *folio) +{ + /* + * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. + * Holding small numbers of low-order mTHP folios in per-CPU LRU cache + * will be sensible, but nobody has implemented and tested that yet. + */ + return !folio_test_large(folio); +} + extern atomic_t lru_disable_count; static inline bool lru_cache_disabled(void) @@ -429,10 +439,17 @@ extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) #define MEMCG_RECLAIM_PROACTIVE (1 << 2) +#define MIN_SWAPPINESS 0 +#define MAX_SWAPPINESS 200 extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, gfp_t gfp_mask, unsigned int reclaim_options); +extern unsigned long try_to_free_mem_cgroup_pages_with_swappiness(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int *swappiness); extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, gfp_t gfp_mask, bool noswap, pg_data_t *pgdat, diff --git a/include/linux/tpm.h b/include/linux/tpm.h index 5f4998626a988..bf8a4ec8a01c1 100644 --- a/include/linux/tpm.h +++ b/include/linux/tpm.h @@ -181,7 +181,7 @@ enum tpm2_const { enum tpm2_timeouts { TPM2_TIMEOUT_A = 750, - TPM2_TIMEOUT_B = 2000, + TPM2_TIMEOUT_B = 4000, TPM2_TIMEOUT_C = 200, TPM2_TIMEOUT_D = 30, TPM2_DURATION_SHORT = 20, diff --git a/include/linux/trace.h b/include/linux/trace.h index bb7eadcc00752..59e43df84f780 100644 --- a/include/linux/trace.h +++ b/include/linux/trace.h @@ -74,8 +74,8 @@ static inline int unregister_ftrace_export(struct trace_export *export) static inline void trace_printk_init_buffers(void) { } -static inline int trace_array_printk(struct trace_array *tr, unsigned long ip, - const char *fmt, ...) +static inline __printf(3, 4) +int trace_array_printk(struct trace_array *tr, unsigned long ip, const char *fmt, ...) { return 0; } diff --git a/include/linux/trace_seq.h b/include/linux/trace_seq.h index 6be92bf559fe7..0e107710e8637 100644 --- a/include/linux/trace_seq.h +++ b/include/linux/trace_seq.h @@ -77,8 +77,8 @@ extern __printf(2, 3) void trace_seq_printf(struct trace_seq *s, const char *fmt, ...); extern __printf(2, 0) void trace_seq_vprintf(struct trace_seq *s, const char *fmt, va_list args); -extern void -trace_seq_bprintf(struct trace_seq *s, const char *fmt, const u32 *binary); +extern __printf(2, 0) +void trace_seq_bprintf(struct trace_seq *s, const char *fmt, const u32 *binary); extern int trace_print_seq(struct seq_file *m, struct trace_seq *s); extern int trace_seq_to_user(struct trace_seq *s, char __user *ubuf, int cnt); @@ -102,8 +102,8 @@ static inline __printf(2, 3) void trace_seq_printf(struct trace_seq *s, const char *fmt, ...) { } -static inline void -trace_seq_bprintf(struct trace_seq *s, const char *fmt, const u32 *binary) +static inline __printf(2, 0) +void trace_seq_bprintf(struct trace_seq *s, const char *fmt, const u32 *binary) { } diff --git a/include/linux/types.h b/include/linux/types.h index 253168bb3fe15..78d87c751ff58 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -115,8 +115,9 @@ typedef u64 u_int64_t; typedef s64 int64_t; #endif -/* this is a special 64bit data type that is 8-byte aligned */ +/* These are the special 64-bit data types that are 8-byte aligned */ #define aligned_u64 __aligned_u64 +#define aligned_s64 __aligned_s64 #define aligned_be64 __aligned_be64 #define aligned_le64 __aligned_le64 diff --git a/include/linux/usb/cdc_ncm.h b/include/linux/usb/cdc_ncm.h index 2d207cb4837db..4ac082a631738 100644 --- a/include/linux/usb/cdc_ncm.h +++ b/include/linux/usb/cdc_ncm.h @@ -119,6 +119,7 @@ struct cdc_ncm_ctx { u32 timer_interval; u32 max_ndp_size; u8 is_ndp16; + u8 filtering_supported; union { struct usb_cdc_ncm_ndp16 *delayed_ndp16; struct usb_cdc_ncm_ndp32 *delayed_ndp32; diff --git a/include/linux/usb/r8152.h b/include/linux/usb/r8152.h index 33a4c146dc19c..2ca60828f28bb 100644 --- a/include/linux/usb/r8152.h +++ b/include/linux/usb/r8152.h @@ -30,6 +30,7 @@ #define VENDOR_ID_NVIDIA 0x0955 #define VENDOR_ID_TPLINK 0x2357 #define VENDOR_ID_DLINK 0x2001 +#define VENDOR_ID_DELL 0x413c #define VENDOR_ID_ASUS 0x0b05 #if IS_REACHABLE(CONFIG_USB_RTL8152) diff --git a/include/linux/usb/typec_dp.h b/include/linux/usb/typec_dp.h index 8d09c2f0a9b80..c3f08af20295c 100644 --- a/include/linux/usb/typec_dp.h +++ b/include/linux/usb/typec_dp.h @@ -56,6 +56,7 @@ enum { DP_PIN_ASSIGN_D, DP_PIN_ASSIGN_E, DP_PIN_ASSIGN_F, /* Not supported after v1.0b */ + DP_PIN_ASSIGN_MAX, }; /* DisplayPort alt mode specific commands */ diff --git a/include/linux/usb/usbnet.h b/include/linux/usb/usbnet.h index 18059eb752abb..e1d1af82a7753 100644 --- a/include/linux/usb/usbnet.h +++ b/include/linux/usb/usbnet.h @@ -77,6 +77,7 @@ struct usbnet { # define EVENT_LINK_CHANGE 11 # define EVENT_SET_RX_MODE 12 # define EVENT_NO_IP_ALIGN 13 +# define EVENT_LINK_CARRIER_ON 14 /* This one is special, as it indicates that the device is going away * there are cyclic dependencies between tasklet, timer and bh * that must be broken diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h index b5e9f603a75aa..24e9dc62dc8bd 100644 --- a/include/linux/virtio_vsock.h +++ b/include/linux/virtio_vsock.h @@ -12,6 +12,7 @@ struct virtio_vsock_skb_cb { bool reply; bool tap_delivered; + u32 offset; }; #define VIRTIO_VSOCK_SKB_CB(skb) ((struct virtio_vsock_skb_cb *)((skb)->cb)) @@ -46,31 +47,50 @@ static inline void virtio_vsock_skb_clear_tap_delivered(struct sk_buff *skb) VIRTIO_VSOCK_SKB_CB(skb)->tap_delivered = false; } -static inline void virtio_vsock_skb_rx_put(struct sk_buff *skb) +static inline void virtio_vsock_skb_put(struct sk_buff *skb, u32 len) { - u32 len; + DEBUG_NET_WARN_ON_ONCE(skb->len); - len = le32_to_cpu(virtio_vsock_hdr(skb)->len); - - if (len > 0) + if (skb_is_nonlinear(skb)) + skb->len = len; + else skb_put(skb, len); } -static inline struct sk_buff *virtio_vsock_alloc_skb(unsigned int size, gfp_t mask) +static inline struct sk_buff * +__virtio_vsock_alloc_skb_with_frags(unsigned int header_len, + unsigned int data_len, + gfp_t mask) { struct sk_buff *skb; + int err; - if (size < VIRTIO_VSOCK_SKB_HEADROOM) - return NULL; - - skb = alloc_skb(size, mask); + skb = alloc_skb_with_frags(header_len, data_len, + PAGE_ALLOC_COSTLY_ORDER, &err, mask); if (!skb) return NULL; skb_reserve(skb, VIRTIO_VSOCK_SKB_HEADROOM); + skb->data_len = data_len; return skb; } +static inline struct sk_buff * +virtio_vsock_alloc_linear_skb(unsigned int size, gfp_t mask) +{ + return __virtio_vsock_alloc_skb_with_frags(size, 0, mask); +} + +static inline struct sk_buff *virtio_vsock_alloc_skb(unsigned int size, gfp_t mask) +{ + if (size <= SKB_WITH_OVERHEAD(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + return virtio_vsock_alloc_linear_skb(size, mask); + + size -= VIRTIO_VSOCK_SKB_HEADROOM; + return __virtio_vsock_alloc_skb_with_frags(VIRTIO_VSOCK_SKB_HEADROOM, + size, mask); +} + static inline void virtio_vsock_skb_queue_head(struct sk_buff_head *list, struct sk_buff *skb) { @@ -110,10 +130,14 @@ static inline size_t virtio_vsock_skb_len(struct sk_buff *skb) return (size_t)(skb_end_pointer(skb) - skb->head); } -#define VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE (1024 * 4) +/* Dimension the RX SKB so that the entire thing fits exactly into + * a single 4KiB page. This avoids wasting memory due to alloc_skb() + * rounding up to the next page order and also means that we + * don't leave higher-order pages sitting around in the RX queue. + */ +#define VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE SKB_WITH_OVERHEAD(1024 * 4) #define VIRTIO_VSOCK_MAX_BUF_SIZE 0xFFFFFFFFUL -#define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE virtio_transport_max_vsock_pkt_buf_size -extern uint virtio_transport_max_vsock_pkt_buf_size; +#define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE (1024 * 64) enum { VSOCK_VQ_RX = 0, /* for host to guest data */ diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index faa993839c22e..5af85c40d530a 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -148,6 +148,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, #ifdef CONFIG_ZSWAP ZSWPIN, ZSWPOUT, + ZSWPWB, #endif #ifdef CONFIG_X86 DIRECT_MAP_LEVEL2_SPLIT, diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index e13a0c500f56e..1af9199172371 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -175,22 +175,6 @@ extern int remap_vmalloc_range_partial(struct vm_area_struct *vma, extern int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, unsigned long pgoff); -/* - * Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values - * and let generic vmalloc and ioremap code know when arch_sync_kernel_mappings() - * needs to be called. - */ -#ifndef ARCH_PAGE_TABLE_SYNC_MASK -#define ARCH_PAGE_TABLE_SYNC_MASK 0 -#endif - -/* - * There is no default implementation for arch_sync_kernel_mappings(). It is - * relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK - * is 0. - */ -void arch_sync_kernel_mappings(unsigned long start, unsigned long end); - /* * Lowlevel-APIs (not for driver use!) */ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 7725b7579b781..2209c227e8592 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -335,4 +335,64 @@ static inline void clear_and_wake_up_bit(int bit, void *word) wake_up_bit(word, bit); } +/** + * test_and_clear_wake_up_bit - clear a bit if it was set: wake up anyone waiting on that bit + * @bit: the bit of the word being waited on + * @word: the address of memory containing that bit + * + * If the bit is set and can be atomically cleared, any tasks waiting in + * wait_on_bit() or similar will be woken. This call has the same + * complete ordering semantics as test_and_clear_bit(). Any changes to + * memory made before this call are guaranteed to be visible after the + * corresponding wait_on_bit() completes. + * + * Returns %true if the bit was successfully set and the wake up was sent. + */ +static inline bool test_and_clear_wake_up_bit(int bit, unsigned long *word) +{ + if (!test_and_clear_bit(bit, word)) + return false; + /* no extra barrier required */ + wake_up_bit(word, bit); + return true; +} + +/** + * atomic_dec_and_wake_up - decrement an atomic_t and if zero, wake up waiters + * @var: the variable to dec and test + * + * Decrements the atomic variable and if it reaches zero, send a wake_up to any + * processes waiting on the variable. + * + * This function has the same complete ordering semantics as atomic_dec_and_test. + * + * Returns %true is the variable reaches zero and the wake up was sent. + */ + +static inline bool atomic_dec_and_wake_up(atomic_t *var) +{ + if (!atomic_dec_and_test(var)) + return false; + /* No extra barrier required */ + wake_up_var(var); + return true; +} + +/** + * store_release_wake_up - update a variable and send a wake_up + * @var: the address of the variable to be updated and woken + * @val: the value to store in the variable. + * + * Store the given value in the variable send a wake up to any tasks + * waiting on the variable. All necessary barriers are included to ensure + * the task calling wait_var_event() sees the new value and all values + * written to memory before this call. + */ +#define store_release_wake_up(var, val) \ +do { \ + smp_store_release(var, val); \ + smp_mb(); \ + wake_up_var(var); \ +} while (0) + #endif /* _LINUX_WAIT_BIT_H */ diff --git a/include/linux/zstd.h b/include/linux/zstd.h index 113408eef6ece..b2c7cf310c8f6 100644 --- a/include/linux/zstd.h +++ b/include/linux/zstd.h @@ -77,6 +77,30 @@ int zstd_min_clevel(void); */ int zstd_max_clevel(void); +/** + * zstd_default_clevel() - default compression level + * + * Return: Default compression level. + */ +int zstd_default_clevel(void); + +/** + * struct zstd_custom_mem - custom memory allocation + */ +typedef ZSTD_customMem zstd_custom_mem; + +/** + * struct zstd_dict_load_method - Dictionary load method. + * See zstd_lib.h. + */ +typedef ZSTD_dictLoadMethod_e zstd_dict_load_method; + +/** + * struct zstd_dict_content_type - Dictionary context type. + * See zstd_lib.h. + */ +typedef ZSTD_dictContentType_e zstd_dict_content_type; + /* ====== Parameter Selection ====== */ /** @@ -136,6 +160,19 @@ typedef ZSTD_parameters zstd_parameters; zstd_parameters zstd_get_params(int level, unsigned long long estimated_src_size); + +/** + * zstd_get_cparams() - returns zstd_compression_parameters for selected level + * @level: The compression level + * @estimated_src_size: The estimated source size to compress or 0 + * if unknown. + * @dict_size: Dictionary size. + * + * Return: The selected zstd_compression_parameters. + */ +zstd_compression_parameters zstd_get_cparams(int level, + unsigned long long estimated_src_size, size_t dict_size); + /* ====== Single-pass Compression ====== */ typedef ZSTD_CCtx zstd_cctx; @@ -180,6 +217,71 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size); size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity, const void *src, size_t src_size, const zstd_parameters *parameters); +/** + * zstd_create_cctx_advanced() - Create compression context + * @custom_mem: Custom allocator. + * + * Return: NULL on error, pointer to compression context otherwise. + */ +zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem); + +/** + * zstd_free_cctx() - Free compression context + * @cdict: Pointer to compression context. + * + * Return: Always 0. + */ +size_t zstd_free_cctx(zstd_cctx* cctx); + +/** + * struct zstd_cdict - Compression dictionary. + * See zstd_lib.h. + */ +typedef ZSTD_CDict zstd_cdict; + +/** + * zstd_create_cdict_byreference() - Create compression dictionary + * @dict: Pointer to dictionary buffer. + * @dict_size: Size of the dictionary buffer. + * @dict_load_method: Dictionary load method. + * @dict_content_type: Dictionary content type. + * @custom_mem: Memory allocator. + * + * Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be + * free before zstd_cdict is destroyed. + * + * Return: NULL on error, pointer to compression dictionary + * otherwise. + */ +zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size, + zstd_compression_parameters cparams, + zstd_custom_mem custom_mem); + +/** + * zstd_free_cdict() - Free compression dictionary + * @cdict: Pointer to compression dictionary. + * + * Return: Always 0. + */ +size_t zstd_free_cdict(zstd_cdict* cdict); + +/** + * zstd_compress_using_cdict() - compress src into dst using a dictionary + * @cctx: The context. Must have been initialized with zstd_init_cctx(). + * @dst: The buffer to compress src into. + * @dst_capacity: The size of the destination buffer. May be any size, but + * ZSTD_compressBound(srcSize) is guaranteed to be large enough. + * @src: The data to compress. + * @src_size: The size of the data to compress. + * @cdict: The dictionary to be used. + * + * Return: The compressed size or an error, which can be checked using + * zstd_is_error(). + */ +size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst, + size_t dst_capacity, const void *src, size_t src_size, + const zstd_cdict *cdict); + /* ====== Single-pass Decompression ====== */ typedef ZSTD_DCtx zstd_dctx; @@ -220,6 +322,71 @@ zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size); size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity, const void *src, size_t src_size); +/** + * struct zstd_ddict - Decompression dictionary. + * See zstd_lib.h. + */ +typedef ZSTD_DDict zstd_ddict; + +/** + * zstd_create_ddict_byreference() - Create decompression dictionary + * @dict: Pointer to dictionary buffer. + * @dict_size: Size of the dictionary buffer. + * @dict_load_method: Dictionary load method. + * @dict_content_type: Dictionary content type. + * @custom_mem: Memory allocator. + * + * Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be + * free before zstd_ddict is destroyed. + * + * Return: NULL on error, pointer to decompression dictionary + * otherwise. + */ +zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size, + zstd_custom_mem custom_mem); +/** + * zstd_free_ddict() - Free decompression dictionary + * @dict: Pointer to the dictionary. + * + * Return: Always 0. + */ +size_t zstd_free_ddict(zstd_ddict *ddict); + +/** + * zstd_create_dctx_advanced() - Create decompression context + * @custom_mem: Custom allocator. + * + * Return: NULL on error, pointer to decompression context otherwise. + */ +zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem); + +/** + * zstd_free_dctx() -- Free decompression context + * @dctx: Pointer to decompression context. + * Return: Always 0. + */ +size_t zstd_free_dctx(zstd_dctx *dctx); + +/** + * zstd_decompress_using_ddict() - decompress src into dst using a dictionary + * @dctx: The decompression context. + * @dst: The buffer to decompress src into. + * @dst_capacity: The size of the destination buffer. Must be at least as large + * as the decompressed size. If the caller cannot upper bound the + * decompressed size, then it's better to use the streaming API. + * @src: The zstd compressed data to decompress. Multiple concatenated + * frames and skippable frames are allowed. + * @src_size: The exact size of the data to decompress. + * @ddict: The dictionary to be used. + * + * Return: The decompressed size or an error, which can be checked using + * zstd_is_error(). + */ +size_t zstd_decompress_using_ddict(zstd_dctx *dctx, + void *dst, size_t dst_capacity, const void *src, size_t src_size, + const zstd_ddict *ddict); + + /* ====== Streaming Buffers ====== */ /** diff --git a/include/media/v4l2-subdev.h b/include/media/v4l2-subdev.h index ab2a7ef61d420..7bdadfc40c929 100644 --- a/include/media/v4l2-subdev.h +++ b/include/media/v4l2-subdev.h @@ -822,7 +822,9 @@ struct v4l2_subdev_state { * possible configuration from the remote end, likely calling * this operation as close as possible to stream on time. The * operation shall fail if the pad index it has been called on - * is not valid or in case of unrecoverable failures. + * is not valid or in case of unrecoverable failures. The + * config argument has been memset to 0 just before calling + * the op. * * @set_routing: enable or disable data connection routes described in the * subdevice routing table. @@ -1877,19 +1879,23 @@ extern const struct v4l2_subdev_ops v4l2_subdev_call_wrappers; * * Note: only legacy non-MC drivers may need this macro. */ -#define v4l2_subdev_call_state_try(sd, o, f, args...) \ - ({ \ - int __result; \ - static struct lock_class_key __key; \ - const char *name = KBUILD_BASENAME \ - ":" __stringify(__LINE__) ":state->lock"; \ - struct v4l2_subdev_state *state = \ - __v4l2_subdev_state_alloc(sd, name, &__key); \ - v4l2_subdev_lock_state(state); \ - __result = v4l2_subdev_call(sd, o, f, state, ##args); \ - v4l2_subdev_unlock_state(state); \ - __v4l2_subdev_state_free(state); \ - __result; \ +#define v4l2_subdev_call_state_try(sd, o, f, args...) \ + ({ \ + int __result; \ + static struct lock_class_key __key; \ + const char *name = KBUILD_BASENAME \ + ":" __stringify(__LINE__) ":state->lock"; \ + struct v4l2_subdev_state *state = \ + __v4l2_subdev_state_alloc(sd, name, &__key); \ + if (IS_ERR(state)) { \ + __result = PTR_ERR(state); \ + } else { \ + v4l2_subdev_lock_state(state); \ + __result = v4l2_subdev_call(sd, o, f, state, ##args); \ + v4l2_subdev_unlock_state(state); \ + __v4l2_subdev_state_free(state); \ + } \ + __result; \ }) /** diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 5bc5f9ae7d34c..be85afe038cb3 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -36,10 +36,22 @@ struct prefix_info { struct __packed { #if defined(__BIG_ENDIAN_BITFIELD) __u8 onlink : 1, - autoconf : 1, + autoconf : 1, +# ifdef __GENKSYMS__ reserved : 6; +# else + routeraddr : 1, + preferpd : 1, + reserved : 4; +# endif #elif defined(__LITTLE_ENDIAN_BITFIELD) +# ifdef __GENKSYMS__ __u8 reserved : 6, +# else + __u8 reserved : 4, + preferpd : 1, + routeraddr : 1, +# endif autoconf : 1, onlink : 1; #else diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index f8b09a82f62e1..1820b87b8b7ff 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -236,8 +236,8 @@ int __vsock_dgram_recvmsg(struct socket *sock, struct msghdr *msg, int vsock_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags); -#ifdef CONFIG_BPF_SYSCALL extern struct proto vsock_proto; +#ifdef CONFIG_BPF_SYSCALL int vsock_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); void __init vsock_bpf_build_proto(void); #else diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index 8f932ee8a06ce..fb06d44230b98 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -2579,6 +2579,7 @@ struct hci_ev_le_conn_complete { #define LE_EXT_ADV_DIRECT_IND 0x0004 #define LE_EXT_ADV_SCAN_RSP 0x0008 #define LE_EXT_ADV_LEGACY_PDU 0x0010 +#define LE_EXT_ADV_DATA_STATUS_MASK 0x0060 #define LE_EXT_ADV_EVT_TYPE_MASK 0x007f #define ADDR_LE_DEV_PUBLIC 0x00 diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index ede1115bbc98d..00761942e03c8 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -241,6 +242,7 @@ struct adv_info { bool enabled; bool pending; bool periodic; + bool periodic_enabled; __u8 mesh; __u8 instance; __u32 flags; @@ -1211,6 +1213,27 @@ static inline struct hci_conn *hci_conn_hash_lookup_ba(struct hci_dev *hdev, return NULL; } +static inline struct hci_conn *hci_conn_hash_lookup_role(struct hci_dev *hdev, + __u8 type, __u8 role, + bdaddr_t *ba) +{ + struct hci_conn_hash *h = &hdev->conn_hash; + struct hci_conn *c; + + rcu_read_lock(); + + list_for_each_entry_rcu(c, &h->list, list) { + if (c->type == type && c->role == role && !bacmp(&c->dst, ba)) { + rcu_read_unlock(); + return c; + } + } + + rcu_read_unlock(); + + return NULL; +} + static inline struct hci_conn *hci_conn_hash_lookup_le(struct hci_dev *hdev, bdaddr_t *ba, __u8 ba_type) @@ -2354,7 +2377,6 @@ void mgmt_advertising_added(struct sock *sk, struct hci_dev *hdev, u8 instance); void mgmt_advertising_removed(struct sock *sk, struct hci_dev *hdev, u8 instance); -void mgmt_adv_monitor_removed(struct hci_dev *hdev, u16 handle); int mgmt_phy_configuration_changed(struct hci_dev *hdev, struct sock *skip); void mgmt_adv_monitor_device_lost(struct hci_dev *hdev, u16 handle, bdaddr_t *bdaddr, u8 addr_type); diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h index 3cb2d10cac930..e2e588b08fe90 100644 --- a/include/net/bluetooth/hci_sync.h +++ b/include/net/bluetooth/hci_sync.h @@ -72,7 +72,7 @@ int hci_update_class_sync(struct hci_dev *hdev); int hci_update_eir_sync(struct hci_dev *hdev); int hci_update_class_sync(struct hci_dev *hdev); -int hci_update_name_sync(struct hci_dev *hdev); +int hci_update_name_sync(struct hci_dev *hdev, const u8 *name); int hci_write_ssp_mode_sync(struct hci_dev *hdev, u8 mode); int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, diff --git a/include/net/bluetooth/mgmt.h b/include/net/bluetooth/mgmt.h index d382679efd2b1..e083f0fa0113a 100644 --- a/include/net/bluetooth/mgmt.h +++ b/include/net/bluetooth/mgmt.h @@ -774,7 +774,7 @@ struct mgmt_adv_pattern { __u8 ad_type; __u8 offset; __u8 length; - __u8 value[31]; + __u8 value[HCI_MAX_AD_LENGTH]; } __packed; #define MGMT_OP_ADD_ADV_PATTERNS_MONITOR 0x0052 diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h index c5e57c6bd8736..078e16d2512a5 100644 --- a/include/net/bond_3ad.h +++ b/include/net/bond_3ad.h @@ -54,6 +54,8 @@ typedef enum { AD_MUX_DETACHED, /* mux machine */ AD_MUX_WAITING, /* mux machine */ AD_MUX_ATTACHED, /* mux machine */ + AD_MUX_COLLECTING, /* mux machine */ + AD_MUX_DISTRIBUTING, /* mux machine */ AD_MUX_COLLECTING_DISTRIBUTING /* mux machine */ } mux_states_t; @@ -302,6 +304,7 @@ int bond_3ad_lacpdu_recv(const struct sk_buff *skb, struct bonding *bond, struct slave *slave); int bond_3ad_set_carrier(struct bonding *bond); void bond_3ad_update_lacp_rate(struct bonding *bond); +void bond_3ad_update_lacp_active(struct bonding *bond); void bond_3ad_update_ad_actor_settings(struct bonding *bond); int bond_3ad_stats_fill(struct sk_buff *skb, struct bond_3ad_stats *stats); size_t bond_3ad_stats_size(void); diff --git a/include/net/bond_options.h b/include/net/bond_options.h index f631d9f099410..18687ccf06383 100644 --- a/include/net/bond_options.h +++ b/include/net/bond_options.h @@ -76,6 +76,7 @@ enum { BOND_OPT_MISSED_MAX, BOND_OPT_NS_TARGETS, BOND_OPT_PRIO, + BOND_OPT_COUPLED_CONTROL, BOND_OPT_LAST }; diff --git a/include/net/bonding.h b/include/net/bonding.h index 94594026a5c55..8bb5f016969f1 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -148,6 +148,7 @@ struct bond_params { #if IS_ENABLED(CONFIG_IPV6) struct in6_addr ns_targets[BOND_MAX_NS_TARGETS]; #endif + int coupled_control; /* 2 bytes of padding : see ether_addr_equal_64bits() */ u8 ad_actor_system[ETH_ALEN + 2]; @@ -167,6 +168,7 @@ struct slave { u8 backup:1, /* indicates backup slave. Value corresponds with BOND_STATE_ACTIVE and BOND_STATE_BACKUP */ inactive:1, /* indicates inactive slave */ + rx_disabled:1, /* indicates whether slave's Rx is disabled */ should_notify:1, /* indicates whether the state changed */ should_notify_link:1; /* indicates whether the link changed */ u8 duplex; @@ -568,6 +570,14 @@ static inline void bond_set_slave_inactive_flags(struct slave *slave, bond_set_slave_state(slave, BOND_STATE_BACKUP, notify); if (!slave->bond->params.all_slaves_active) slave->inactive = 1; + if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) + slave->rx_disabled = 1; +} + +static inline void bond_set_slave_tx_disabled_flags(struct slave *slave, + bool notify) +{ + bond_set_slave_state(slave, BOND_STATE_BACKUP, notify); } static inline void bond_set_slave_active_flags(struct slave *slave, @@ -575,6 +585,14 @@ static inline void bond_set_slave_active_flags(struct slave *slave, { bond_set_slave_state(slave, BOND_STATE_ACTIVE, notify); slave->inactive = 0; + if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) + slave->rx_disabled = 0; +} + +static inline void bond_set_slave_rx_enabled_flags(struct slave *slave, + bool notify) +{ + slave->rx_disabled = 0; } static inline bool bond_is_slave_inactive(struct slave *slave) @@ -582,6 +600,11 @@ static inline bool bond_is_slave_inactive(struct slave *slave) return slave->inactive; } +static inline bool bond_is_slave_rx_disabled(struct slave *slave) +{ + return slave->rx_disabled; +} + static inline void bond_propose_link_state(struct slave *slave, int state) { slave->link_new_state = state; diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 73dbc130d2159..7a772b4bc691b 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -119,10 +119,6 @@ struct wiphy; * restrictions. * @IEEE80211_CHAN_NO_EHT: EHT operation is not permitted on this channel. * @IEEE80211_CHAN_DFS_CONCURRENT: See %NL80211_RRF_DFS_CONCURRENT - * @IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT: Client connection with VLP AP - * not permitted using this channel - * @IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT: Client connection with AFC AP - * not permitted using this channel */ enum ieee80211_channel_flags { IEEE80211_CHAN_DISABLED = 1<<0, @@ -147,8 +143,6 @@ enum ieee80211_channel_flags { IEEE80211_CHAN_NO_320MHZ = 1<<19, IEEE80211_CHAN_NO_EHT = 1<<20, IEEE80211_CHAN_DFS_CONCURRENT = 1<<21, - IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT = 1<<22, - IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT = 1<<23, }; #define IEEE80211_CHAN_NO_HT40 \ @@ -572,7 +566,7 @@ ieee80211_get_sband_iftype_data(const struct ieee80211_supported_band *sband, { int i; - if (WARN_ON(iftype >= NL80211_IFTYPE_MAX)) + if (WARN_ON(iftype >= NUM_NL80211_IFTYPES)) return NULL; if (iftype == NL80211_IFTYPE_AP_VLAN) @@ -2587,7 +2581,7 @@ struct cfg80211_scan_request { ANDROID_KABI_RESERVE(1); /* keep last */ - struct ieee80211_channel *channels[] __counted_by(n_channels); + struct ieee80211_channel *channels[]; }; static inline void get_random_mask_addr(u8 *buf, const u8 *addr, const u8 *mask) @@ -4849,7 +4843,7 @@ struct cfg80211_ops { * enum wiphy_flags - wiphy capability flags * * @WIPHY_FLAG_SPLIT_SCAN_6GHZ: if set to true, the scan request will be split - * into two, first for legacy bands and second for 6 GHz. + * into two, first for legacy bands and second for UHB. * @WIPHY_FLAG_NETNS_OK: if not set, do not allow changing the netns of this * wiphy at all * @WIPHY_FLAG_PS_ON_BY_DEFAULT: if set to true, powersave will be enabled diff --git a/include/net/checksum.h b/include/net/checksum.h index 1338cb92c8e72..28b101f26636e 100644 --- a/include/net/checksum.h +++ b/include/net/checksum.h @@ -158,7 +158,7 @@ void inet_proto_csum_replace16(__sum16 *sum, struct sk_buff *skb, const __be32 *from, const __be32 *to, bool pseudohdr); void inet_proto_csum_replace_by_diff(__sum16 *sum, struct sk_buff *skb, - __wsum diff, bool pseudohdr); + __wsum diff, bool pseudohdr, bool ipv6); static __always_inline void inet_proto_csum_replace2(__sum16 *sum, struct sk_buff *skb, diff --git a/include/net/cls_cgroup.h b/include/net/cls_cgroup.h index 7e78e7d6f0152..668aeee9b3f66 100644 --- a/include/net/cls_cgroup.h +++ b/include/net/cls_cgroup.h @@ -63,7 +63,7 @@ static inline u32 task_get_classid(const struct sk_buff *skb) * calls by looking at the number of nested bh disable calls because * softirqs always disables bh. */ - if (in_serving_softirq()) { + if (softirq_count()) { struct sock *sk = skb_to_full_sk(skb); /* If there is an sock_cgroup_classid we'll use that. */ diff --git a/include/net/dst.h b/include/net/dst.h index 16b7b99b5f309..1983945dfcdf1 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -464,7 +464,7 @@ INDIRECT_CALLABLE_DECLARE(int ip_output(struct net *, struct sock *, /* Output packet to network from transport. */ static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb) { - return INDIRECT_CALL_INET(skb_dst(skb)->output, + return INDIRECT_CALL_INET(READ_ONCE(skb_dst(skb)->output), ip6_output, ip_output, net, sk, skb); } @@ -474,7 +474,7 @@ INDIRECT_CALLABLE_DECLARE(int ip_local_deliver(struct sk_buff *)); /* Input packet from network to transport. */ static inline int dst_input(struct sk_buff *skb) { - return INDIRECT_CALL_INET(skb_dst(skb)->input, + return INDIRECT_CALL_INET(READ_ONCE(skb_dst(skb)->input), ip6_input, ip_local_deliver, skb); } @@ -569,6 +569,38 @@ static inline void skb_dst_update_pmtu_no_confirm(struct sk_buff *skb, u32 mtu) dst->ops->update_pmtu(dst, NULL, skb, mtu, false); } +static inline struct net_device *dst_dev(const struct dst_entry *dst) +{ + return READ_ONCE(dst->dev); +} + +static inline struct net_device *dst_dev_rcu(const struct dst_entry *dst) +{ + /* In the future, use rcu_dereference(dst->dev) */ + WARN_ON_ONCE(!rcu_read_lock_held()); + return READ_ONCE(dst->dev); +} + +static inline struct net_device *skb_dst_dev(const struct sk_buff *skb) +{ + return dst_dev(skb_dst(skb)); +} + +static inline struct net_device *skb_dst_dev_rcu(const struct sk_buff *skb) +{ + return dst_dev_rcu(skb_dst(skb)); +} + +static inline struct net *skb_dst_dev_net(const struct sk_buff *skb) +{ + return dev_net(skb_dst_dev(skb)); +} + +static inline struct net *skb_dst_dev_net_rcu(const struct sk_buff *skb) +{ + return dev_net_rcu(skb_dst_dev(skb)); +} + struct dst_entry *dst_blackhole_check(struct dst_entry *dst, u32 cookie); void dst_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb, u32 mtu, bool confirm_neigh); diff --git a/include/net/gro.h b/include/net/gro.h index 018343254c90a..9260ed367c919 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -10,6 +10,9 @@ #include #include +/* This should be increased if a protocol with a bigger head is added. */ +#define GRO_MAX_HEAD (MAX_HEADER + 128) + struct napi_gro_cb { union { struct { diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index 006a61ddd36fa..3d36794cb1899 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -489,6 +489,21 @@ struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md, int skb_tunnel_check_pmtu(struct sk_buff *skb, struct dst_entry *encap_dst, int headroom, bool reply); +static inline void ip_tunnel_adj_headroom(struct net_device *dev, + unsigned int headroom) +{ + /* we must cap headroom to some upperlimit, else pskb_expand_head + * will overflow header offsets in skb_headers_offset_update(). + */ + const unsigned int max_allowed = 512; + + if (headroom > max_allowed) + headroom = max_allowed; + + if (headroom > READ_ONCE(dev->needed_headroom)) + WRITE_ONCE(dev->needed_headroom, headroom); +} + int iptunnel_handle_offloads(struct sk_buff *skb, int gso_type_mask); static inline int iptunnel_pull_offloads(struct sk_buff *skb) diff --git a/include/net/lwtunnel.h b/include/net/lwtunnel.h index 53bd2d02a4f0d..09791f5d9b6ec 100644 --- a/include/net/lwtunnel.h +++ b/include/net/lwtunnel.h @@ -138,12 +138,12 @@ int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, static inline void lwtunnel_set_redirect(struct dst_entry *dst) { if (lwtunnel_output_redirect(dst->lwtstate)) { - dst->lwtstate->orig_output = dst->output; - dst->output = lwtunnel_output; + dst->lwtstate->orig_output = READ_ONCE(dst->output); + WRITE_ONCE(dst->output, lwtunnel_output); } if (lwtunnel_input_redirect(dst->lwtstate)) { - dst->lwtstate->orig_input = dst->input; - dst->input = lwtunnel_input; + dst->lwtstate->orig_input = READ_ONCE(dst->input); + WRITE_ONCE(dst->input, lwtunnel_input); } } #else diff --git a/include/net/mac80211.h b/include/net/mac80211.h index 7ce78f59528e4..5053ff51e3ac2 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -4125,6 +4125,8 @@ struct ieee80211_prep_tx_info { * @mgd_complete_tx: Notify the driver that the response frame for a previously * transmitted frame announced with @mgd_prepare_tx was received, the data * is filled similarly to @mgd_prepare_tx though the duration is not used. + * Note that this isn't always called for each mgd_prepare_tx() call, for + * example for SAE the 'confirm' messages can be on the air in any order. * * @mgd_protect_tdls_discover: Protect a TDLS discovery session. After sending * a TDLS discovery-request, we expect a reply to arrive on the AP's diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index c85c8c25a8485..aef1c7dc981ca 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -292,6 +292,7 @@ static inline int check_net(const struct net *net) } void net_drop_ns(void *); +void net_passive_dec(struct net *net); #else @@ -321,8 +322,23 @@ static inline int check_net(const struct net *net) } #define net_drop_ns NULL + +static inline void net_passive_dec(struct net *net) +{ + refcount_dec(&net->passive); +} #endif +static inline void net_passive_inc(struct net *net) +{ + refcount_inc(&net->passive); +} + +/* Returns true if the netns initialization is completed successfully */ +static inline bool net_initialized(const struct net *net) +{ + return READ_ONCE(net->list.next); +} static inline void __netns_tracker_alloc(struct net *net, netns_tracker *tracker, diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index bc99e6db1818b..a83aac5085bc2 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -308,8 +308,19 @@ static inline bool nf_ct_is_expired(const struct nf_conn *ct) /* use after obtaining a reference count */ static inline bool nf_ct_should_gc(const struct nf_conn *ct) { - return nf_ct_is_expired(ct) && nf_ct_is_confirmed(ct) && - !nf_ct_is_dying(ct); + if (!nf_ct_is_confirmed(ct)) + return false; + + /* load ct->timeout after is_confirmed() test. + * Pairs with __nf_conntrack_confirm() which: + * 1. Increases ct->timeout value + * 2. Inserts ct into rcu hlist + * 3. Sets the confirmed bit + * 4. Unlocks the hlist lock + */ + smp_acquire__after_ctrl_dep(); + + return nf_ct_is_expired(ct) && !nf_ct_is_dying(ct); } #define NF_CT_DAY (86400 * HZ) diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h index df7775afb92b9..0097791e1eede 100644 --- a/include/net/netfilter/nf_flow_table.h +++ b/include/net/netfilter/nf_flow_table.h @@ -353,7 +353,7 @@ static inline __be16 __nf_flow_pppoe_proto(const struct sk_buff *skb) static inline bool nf_flow_pppoe_proto(struct sk_buff *skb, __be16 *inner_proto) { - if (!pskb_may_pull(skb, PPPOE_SES_HLEN)) + if (!pskb_may_pull(skb, ETH_HLEN + PPPOE_SES_HLEN)) return false; *inner_proto = __nf_flow_pppoe_proto(skb); diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index c0f4b91e4f5ec..32606d5430605 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -955,8 +955,7 @@ struct nft_expr_ops { const struct nft_expr *expr, bool reset); int (*validate)(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data); + const struct nft_expr *expr); bool (*reduce)(struct nft_regs_track *track, const struct nft_expr *expr); bool (*gc)(struct net *net, diff --git a/include/net/netfilter/nft_fib.h b/include/net/netfilter/nft_fib.h index 167640b843ef8..38cae7113de46 100644 --- a/include/net/netfilter/nft_fib.h +++ b/include/net/netfilter/nft_fib.h @@ -21,9 +21,7 @@ nft_fib_is_loopback(const struct sk_buff *skb, const struct net_device *in) int nft_fib_dump(struct sk_buff *skb, const struct nft_expr *expr, bool reset); int nft_fib_init(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nlattr * const tb[]); -int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, - const struct nft_data **data); - +int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr); void nft_fib4_eval_type(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt); diff --git a/include/net/netfilter/nft_meta.h b/include/net/netfilter/nft_meta.h index ba1238f12a487..d602263590fed 100644 --- a/include/net/netfilter/nft_meta.h +++ b/include/net/netfilter/nft_meta.h @@ -41,8 +41,7 @@ void nft_meta_set_destroy(const struct nft_ctx *ctx, const struct nft_expr *expr); int nft_meta_set_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data); + const struct nft_expr *expr); bool nft_meta_get_reduce(struct nft_regs_track *track, const struct nft_expr *expr); diff --git a/include/net/netfilter/nft_reject.h b/include/net/netfilter/nft_reject.h index 6d9ba62efd750..19060212988a1 100644 --- a/include/net/netfilter/nft_reject.h +++ b/include/net/netfilter/nft_reject.h @@ -15,8 +15,7 @@ struct nft_reject { extern const struct nla_policy nft_reject_policy[]; int nft_reject_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data); + const struct nft_expr *expr); int nft_reject_init(const struct nft_ctx *ctx, const struct nft_expr *expr, diff --git a/include/net/netlink.h b/include/net/netlink.h index 8a7cd1170e1f7..aba2b162a2260 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -128,6 +128,8 @@ * nla_len(nla) length of attribute payload * * Attribute Payload Access for Basic Types: + * nla_get_uint(nla) get payload for a uint attribute + * nla_get_sint(nla) get payload for a sint attribute * nla_get_u8(nla) get payload for a u8 attribute * nla_get_u16(nla) get payload for a u16 attribute * nla_get_u32(nla) get payload for a u32 attribute @@ -183,6 +185,8 @@ enum { NLA_REJECT, NLA_BE16, NLA_BE32, + NLA_SINT, + NLA_UINT, __NLA_TYPE_MAX, }; @@ -229,6 +233,7 @@ enum nla_policy_validation { * nested header (or empty); len field is used if * nested_policy is also used, for the max attr * number in the nested policy. + * NLA_SINT, NLA_UINT, * NLA_U8, NLA_U16, * NLA_U32, NLA_U64, * NLA_S8, NLA_S16, @@ -260,12 +265,14 @@ enum nla_policy_validation { * while an array has the nested attributes at another * level down and the attribute types directly in the * nesting don't matter. + * NLA_UINT, * NLA_U8, * NLA_U16, * NLA_U32, * NLA_U64, * NLA_BE16, * NLA_BE32, + * NLA_SINT, * NLA_S8, * NLA_S16, * NLA_S32, @@ -280,6 +287,7 @@ enum nla_policy_validation { * or NLA_POLICY_FULL_RANGE_SIGNED() macros instead. * Use the NLA_POLICY_MIN(), NLA_POLICY_MAX() and * NLA_POLICY_RANGE() macros. + * NLA_UINT, * NLA_U8, * NLA_U16, * NLA_U32, @@ -288,6 +296,7 @@ enum nla_policy_validation { * to a struct netlink_range_validation that indicates * the min/max values. * Use NLA_POLICY_FULL_RANGE(). + * NLA_SINT, * NLA_S8, * NLA_S16, * NLA_S32, @@ -377,9 +386,11 @@ struct nla_policy { #define __NLA_IS_UINT_TYPE(tp) \ (tp == NLA_U8 || tp == NLA_U16 || tp == NLA_U32 || \ - tp == NLA_U64 || tp == NLA_BE16 || tp == NLA_BE32) + tp == NLA_U64 || tp == NLA_UINT || \ + tp == NLA_BE16 || tp == NLA_BE32) #define __NLA_IS_SINT_TYPE(tp) \ - (tp == NLA_S8 || tp == NLA_S16 || tp == NLA_S32 || tp == NLA_S64) + (tp == NLA_S8 || tp == NLA_S16 || tp == NLA_S32 || tp == NLA_S64 || \ + tp == NLA_SINT) #define __NLA_ENSURE(condition) BUILD_BUG_ON_ZERO(!(condition)) #define NLA_ENSURE_UINT_TYPE(tp) \ @@ -1357,6 +1368,22 @@ static inline int nla_put_u32(struct sk_buff *skb, int attrtype, u32 value) return nla_put(skb, attrtype, sizeof(u32), &tmp); } +/** + * nla_put_uint - Add a variable-size unsigned int to a socket buffer + * @skb: socket buffer to add attribute to + * @attrtype: attribute type + * @value: numeric value + */ +static inline int nla_put_uint(struct sk_buff *skb, int attrtype, u64 value) +{ + u64 tmp64 = value; + u32 tmp32 = value; + + if (tmp64 == tmp32) + return nla_put_u32(skb, attrtype, tmp32); + return nla_put(skb, attrtype, sizeof(u64), &tmp64); +} + /** * nla_put_be32 - Add a __be32 netlink attribute to a socket buffer * @skb: socket buffer to add attribute to @@ -1511,6 +1538,22 @@ static inline int nla_put_s64(struct sk_buff *skb, int attrtype, s64 value, return nla_put_64bit(skb, attrtype, sizeof(s64), &tmp, padattr); } +/** + * nla_put_sint - Add a variable-size signed int to a socket buffer + * @skb: socket buffer to add attribute to + * @attrtype: attribute type + * @value: numeric value + */ +static inline int nla_put_sint(struct sk_buff *skb, int attrtype, s64 value) +{ + s64 tmp64 = value; + s32 tmp32 = value; + + if (tmp64 == tmp32) + return nla_put_s32(skb, attrtype, tmp32); + return nla_put(skb, attrtype, sizeof(s64), &tmp64); +} + /** * nla_put_string - Add a string netlink attribute to a socket buffer * @skb: socket buffer to add attribute to @@ -1667,6 +1710,17 @@ static inline u64 nla_get_u64(const struct nlattr *nla) return tmp; } +/** + * nla_get_uint - return payload of uint attribute + * @nla: uint netlink attribute + */ +static inline u64 nla_get_uint(const struct nlattr *nla) +{ + if (nla_len(nla) == sizeof(u32)) + return nla_get_u32(nla); + return nla_get_u64(nla); +} + /** * nla_get_be64 - return payload of __be64 attribute * @nla: __be64 netlink attribute @@ -1729,6 +1783,17 @@ static inline s64 nla_get_s64(const struct nlattr *nla) return tmp; } +/** + * nla_get_sint - return payload of uint attribute + * @nla: uint netlink attribute + */ +static inline s64 nla_get_sint(const struct nlattr *nla) +{ + if (nla_len(nla) == sizeof(s32)) + return nla_get_s32(nla); + return nla_get_s64(nla); +} + /** * nla_get_flag - return payload of flag attribute * @nla: flag netlink attribute diff --git a/include/net/nfc/nci_core.h b/include/net/nfc/nci_core.h index ea8595651c384..e066bdbc807be 100644 --- a/include/net/nfc/nci_core.h +++ b/include/net/nfc/nci_core.h @@ -52,7 +52,7 @@ enum nci_state { #define NCI_RF_DISC_SELECT_TIMEOUT 5000 #define NCI_RF_DEACTIVATE_TIMEOUT 30000 #define NCI_CMD_TIMEOUT 5000 -#define NCI_DATA_TIMEOUT 700 +#define NCI_DATA_TIMEOUT 3000 struct nci_dev; diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 15960564e0c36..4d72d24b1f33e 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -112,7 +112,6 @@ struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct netlink_ext_ack *extack); void qdisc_put_rtab(struct qdisc_rate_table *tab); void qdisc_put_stab(struct qdisc_size_table *tab); -void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc); bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q, struct net_device *dev, struct netdev_queue *txq, spinlock_t *root_lock, bool validate); @@ -306,4 +305,28 @@ static inline bool tc_qdisc_stats_dump(struct Qdisc *sch, return true; } +static inline void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc) +{ + if (!(qdisc->flags & TCQ_F_WARN_NONWC)) { + pr_warn("%s: %s qdisc %X: is non-work-conserving?\n", + txt, qdisc->ops->id, qdisc->handle >> 16); + qdisc->flags |= TCQ_F_WARN_NONWC; + } +} + +static inline unsigned int qdisc_peek_len(struct Qdisc *sch) +{ + struct sk_buff *skb; + unsigned int len; + + skb = sch->ops->peek(sch); + if (unlikely(skb == NULL)) { + qdisc_warn_nonwc("qdisc_peek_len", sch); + return 0; + } + len = qdisc_pkt_len(skb); + + return len; +} + #endif diff --git a/include/net/rose.h b/include/net/rose.h index 23267b4efcfa3..2b5491bbf39ab 100644 --- a/include/net/rose.h +++ b/include/net/rose.h @@ -8,6 +8,7 @@ #ifndef _ROSE_H #define _ROSE_H +#include #include #include #include @@ -96,7 +97,7 @@ struct rose_neigh { ax25_cb *ax25; struct net_device *dev; unsigned short count; - unsigned short use; + refcount_t use; unsigned int number; char restarted; char dce_mode; @@ -151,6 +152,21 @@ struct rose_sock { #define rose_sk(sk) ((struct rose_sock *)(sk)) +static inline void rose_neigh_hold(struct rose_neigh *rose_neigh) +{ + refcount_inc(&rose_neigh->use); +} + +static inline void rose_neigh_put(struct rose_neigh *rose_neigh) +{ + if (refcount_dec_and_test(&rose_neigh->use)) { + if (rose_neigh->ax25) + ax25_cb_put(rose_neigh->ax25); + kfree(rose_neigh->digipeat); + kfree(rose_neigh); + } +} + /* af_rose.c */ extern ax25_address rose_callsign; extern int sysctl_rose_restart_request_timeout; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 22e2374be1dfd..ff3eb7b86150b 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -1036,6 +1036,21 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) return skb; } +static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool direct) +{ + struct sk_buff *skb; + + skb = __skb_dequeue(&sch->gso_skb); + if (skb) { + sch->q.qlen--; + return skb; + } + if (direct) + return __qdisc_dequeue_head(&sch->q); + else + return sch->dequeue(sch); +} + static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) { struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); diff --git a/include/net/sock.h b/include/net/sock.h index 4e543b9a05e5b..52aa818b23a05 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -355,6 +355,8 @@ struct sk_filter; * @sk_txtime_unused: unused txtime flags * @ns_tracker: tracker for netns reference * @sk_bind2_node: bind node in the bhash2 table + * @sk_owner: reference to the real owner of the socket that calls + * sock_lock_init_class_and_name(). */ struct sock { /* @@ -548,6 +550,10 @@ struct sock { netns_tracker ns_tracker; struct hlist_node sk_bind2_node; +#if IS_ENABLED(CONFIG_PROVE_LOCKING) && IS_ENABLED(CONFIG_MODULES) + struct module *sk_owner; +#endif + ANDROID_KABI_RESERVE(1); ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); @@ -1711,6 +1717,35 @@ static inline void sk_mem_uncharge(struct sock *sk, int size) sk_mem_reclaim(sk); } +#if IS_ENABLED(CONFIG_PROVE_LOCKING) && IS_ENABLED(CONFIG_MODULES) +static inline void sk_owner_set(struct sock *sk, struct module *owner) +{ + __module_get(owner); + sk->sk_owner = owner; +} + +static inline void sk_owner_clear(struct sock *sk) +{ + sk->sk_owner = NULL; +} + +static inline void sk_owner_put(struct sock *sk) +{ + module_put(sk->sk_owner); +} +#else +static inline void sk_owner_set(struct sock *sk, struct module *owner) +{ +} + +static inline void sk_owner_clear(struct sock *sk) +{ +} + +static inline void sk_owner_put(struct sock *sk) +{ +} +#endif /* * Macro so as to not evaluate some arguments when * lockdep is not enabled. @@ -1720,13 +1755,14 @@ static inline void sk_mem_uncharge(struct sock *sk, int size) */ #define sock_lock_init_class_and_name(sk, sname, skey, name, key) \ do { \ + sk_owner_set(sk, THIS_MODULE); \ sk->sk_lock.owned = 0; \ init_waitqueue_head(&sk->sk_lock.wq); \ spin_lock_init(&(sk)->sk_lock.slock); \ debug_check_no_locks_freed((void *)&(sk)->sk_lock, \ - sizeof((sk)->sk_lock)); \ + sizeof((sk)->sk_lock)); \ lockdep_set_class_and_name(&(sk)->sk_lock.slock, \ - (skey), (sname)); \ + (skey), (sname)); \ lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \ } while (0) @@ -1871,6 +1907,7 @@ static inline bool sock_allow_reclassification(const struct sock *csk) struct sock *sk_alloc(struct net *net, int family, gfp_t priority, struct proto *prot, int kern); void sk_free(struct sock *sk); +void sk_net_refcnt_upgrade(struct sock *sk); void sk_destruct(struct sock *sk); struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority); void sk_free_unlock_clone(struct sock *sk); @@ -3065,8 +3102,11 @@ int sock_ioctl_inout(struct sock *sk, unsigned int cmd, int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg); static inline bool sk_is_readable(struct sock *sk) { - if (sk->sk_prot->sock_is_readable) - return sk->sk_prot->sock_is_readable(sk); + const struct proto *prot = READ_ONCE(sk->sk_prot); + + if (prot->sock_is_readable) + return prot->sock_is_readable(sk); + return false; } #endif /* _SOCK_H */ diff --git a/include/net/tc_act/tc_connmark.h b/include/net/tc_act/tc_connmark.h index e8dd77a967480..a5ce83f3eea4b 100644 --- a/include/net/tc_act/tc_connmark.h +++ b/include/net/tc_act/tc_connmark.h @@ -7,6 +7,7 @@ struct tcf_connmark_parms { struct net *net; u16 zone; + int action; struct rcu_head rcu; }; diff --git a/include/net/tc_act/tc_ctinfo.h b/include/net/tc_act/tc_ctinfo.h index f071c1d70a25e..a04bcac7adf4b 100644 --- a/include/net/tc_act/tc_ctinfo.h +++ b/include/net/tc_act/tc_ctinfo.h @@ -18,9 +18,9 @@ struct tcf_ctinfo_params { struct tcf_ctinfo { struct tc_action common; struct tcf_ctinfo_params __rcu *params; - u64 stats_dscp_set; - u64 stats_dscp_error; - u64 stats_cpmark_set; + atomic64_t stats_dscp_set; + atomic64_t stats_dscp_error; + atomic64_t stats_cpmark_set; }; enum { diff --git a/include/net/tls.h b/include/net/tls.h index 2dbbf55e6a57b..b7a9f5f646522 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -485,6 +485,12 @@ tls_offload_rx_resync_async_request_end(struct sock *sk, __be32 seq) ((u64)ntohl(seq) << 32) | RESYNC_REQ); } +static inline void +tls_offload_rx_resync_async_request_cancel(struct tls_offload_resync_async *resync_async) +{ + atomic64_set(&resync_async->req, 0); +} + static inline void tls_offload_rx_resync_set_type(struct sock *sk, enum tls_offload_sync_type type) { diff --git a/include/net/udp.h b/include/net/udp.h index 488a6d2babccf..89eeb187667b0 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -466,6 +466,16 @@ static inline struct sk_buff *udp_rcv_segment(struct sock *sk, { netdev_features_t features = NETIF_F_SG; struct sk_buff *segs; + int drop_count; + + /* + * Segmentation in UDP receive path is only for UDP GRO, drop udp + * fragmentation offload (UFO) packets. + */ + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP) { + drop_count = 1; + goto drop; + } /* Avoid csum recalculation by skb_segment unless userspace explicitly * asks for the final checksum values @@ -489,16 +499,18 @@ static inline struct sk_buff *udp_rcv_segment(struct sock *sk, */ segs = __skb_gso_segment(skb, features, false); if (IS_ERR_OR_NULL(segs)) { - int segs_nr = skb_shinfo(skb)->gso_segs; - - atomic_add(segs_nr, &sk->sk_drops); - SNMP_ADD_STATS(__UDPX_MIB(sk, ipv4), UDP_MIB_INERRORS, segs_nr); - kfree_skb(skb); - return NULL; + drop_count = skb_shinfo(skb)->gso_segs; + goto drop; } consume_skb(skb); return segs; + +drop: + atomic_add(drop_count, &sk->sk_drops); + SNMP_ADD_STATS(__UDPX_MIB(sk, ipv4), UDP_MIB_INERRORS, drop_count); + kfree_skb(skb); + return NULL; } static inline void udp_post_segment_fix_csum(struct sk_buff *skb) diff --git a/include/net/xdp.h b/include/net/xdp.h index b39ac83618a55..3d8989096b5d2 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -113,6 +113,11 @@ static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp) xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC; } +static __always_inline void xdp_buff_clear_frag_pfmemalloc(struct xdp_buff *xdp) +{ + xdp->flags &= ~XDP_FLAGS_FRAGS_PF_MEMALLOC; +} + static __always_inline void xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) { diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 537dd5e58d176..a8f95a23bdad5 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -460,7 +460,8 @@ static inline int xfrm_af2proto(unsigned int family) static inline const struct xfrm_mode *xfrm_ip2inner_mode(struct xfrm_state *x, int ipproto) { - if ((ipproto == IPPROTO_IPIP && x->props.family == AF_INET) || + if ((x->sel.family != AF_UNSPEC) || + (ipproto == IPPROTO_IPIP && x->props.family == AF_INET) || (ipproto == IPPROTO_IPV6 && x->props.family == AF_INET6)) return &x->inner_mode; else diff --git a/include/rdma/uverbs_std_types.h b/include/rdma/uverbs_std_types.h index fe05121169589..555ea3d142a46 100644 --- a/include/rdma/uverbs_std_types.h +++ b/include/rdma/uverbs_std_types.h @@ -34,7 +34,7 @@ static inline void *_uobj_get_obj_read(struct ib_uobject *uobj) { if (IS_ERR(uobj)) - return NULL; + return ERR_CAST(uobj); return uobj->object; } #define uobj_get_obj_read(_object, _type, _id, _attrs) \ diff --git a/include/scsi/scsi_eh.h b/include/scsi/scsi_eh.h index 1ae08e81339fa..15679be90c5c3 100644 --- a/include/scsi/scsi_eh.h +++ b/include/scsi/scsi_eh.h @@ -41,6 +41,12 @@ struct scsi_eh_save { unsigned char cmnd[32]; struct scsi_data_buffer sdb; struct scatterlist sense_sgl; + + /* struct request fields */ +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct bio_crypt_ctx *rq_crypt_ctx; + struct blk_crypto_keyslot *rq_crypt_keyslot; +#endif }; extern void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, diff --git a/include/soc/mscc/ocelot_vcap.h b/include/soc/mscc/ocelot_vcap.h index c601a4598b0da..eb19668a06db1 100644 --- a/include/soc/mscc/ocelot_vcap.h +++ b/include/soc/mscc/ocelot_vcap.h @@ -13,6 +13,7 @@ */ #define OCELOT_VCAP_ES0_TAG_8021Q_RXVLAN(ocelot, port, upstream) ((upstream) << 16 | (port)) #define OCELOT_VCAP_IS1_TAG_8021Q_TXVLAN(ocelot, port) (port) +#define OCELOT_VCAP_IS1_VLAN_RECLASSIFY(ocelot, port) ((ocelot)->num_phys_ports + (port)) #define OCELOT_VCAP_IS2_TAG_8021Q_TXVLAN(ocelot, port) (port) #define OCELOT_VCAP_IS2_MRP_REDIRECT(ocelot, port) ((ocelot)->num_phys_ports + (port)) #define OCELOT_VCAP_IS2_MRP_TRAP(ocelot) ((ocelot)->num_phys_ports * 2) @@ -499,6 +500,7 @@ struct ocelot_vcap_key_vlan { struct ocelot_vcap_u8 pcp; /* PCP (3 bit) */ enum ocelot_vcap_bit dei; /* DEI */ enum ocelot_vcap_bit tagged; /* Tagged/untagged frame */ + enum ocelot_vcap_bit tpid; }; struct ocelot_vcap_key_etype { diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h index 5497dc9c396a5..b58dc869cf77e 100644 --- a/include/sound/hda_codec.h +++ b/include/sound/hda_codec.h @@ -196,6 +196,7 @@ struct hda_codec { /* beep device */ struct hda_beep *beep; unsigned int beep_mode; + bool beep_just_power_on; /* widget capabilities cache */ u32 *wcaps; diff --git a/include/sound/pcm.h b/include/sound/pcm.h index 1846e78e29452..5f6eedac79668 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -1439,6 +1439,8 @@ int snd_pcm_lib_mmap_iomem(struct snd_pcm_substream *substream, struct vm_area_s #define snd_pcm_lib_mmap_iomem NULL #endif +void snd_pcm_runtime_buffer_set_silence(struct snd_pcm_runtime *runtime); + /** * snd_pcm_limit_isa_dma_size - Get the max size fitting with ISA DMA transfer * @dma: DMA number diff --git a/include/sound/ump_convert.h b/include/sound/ump_convert.h index d099ae27f8491..682499b871eac 100644 --- a/include/sound/ump_convert.h +++ b/include/sound/ump_convert.h @@ -19,7 +19,7 @@ struct ump_cvt_to_ump_bank { /* context for converting from MIDI1 byte stream to UMP packet */ struct ump_cvt_to_ump { /* MIDI1 intermediate buffer */ - unsigned char buf[4]; + unsigned char buf[6]; /* up to 6 bytes for SysEx */ int len; int cmd_bytes; diff --git a/include/sound/ump_msg.h b/include/sound/ump_msg.h index 72f60ddfea753..9556b4755a1ed 100644 --- a/include/sound/ump_msg.h +++ b/include/sound/ump_msg.h @@ -604,7 +604,7 @@ struct snd_ump_stream_msg_ep_info { } __packed; /* UMP Stream Message: Device Info Notification (128bit) */ -struct snd_ump_stream_msg_devince_info { +struct snd_ump_stream_msg_device_info { #ifdef __BIG_ENDIAN_BITFIELD /* 0 */ u32 type:4; @@ -754,7 +754,7 @@ struct snd_ump_stream_msg_fb_name { union snd_ump_stream_msg { struct snd_ump_stream_msg_ep_discovery ep_discovery; struct snd_ump_stream_msg_ep_info ep_info; - struct snd_ump_stream_msg_devince_info device_info; + struct snd_ump_stream_msg_device_info device_info; struct snd_ump_stream_msg_stream_cfg stream_cfg; struct snd_ump_stream_msg_fb_discovery fb_discovery; struct snd_ump_stream_msg_fb_info fb_info; diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 3c4d5ef6d4463..f759109caeeac 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -1857,7 +1857,7 @@ TRACE_EVENT(qgroup_update_counters, TRACE_EVENT(qgroup_update_reserve, - TP_PROTO(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup, + TP_PROTO(const struct btrfs_fs_info *fs_info, const struct btrfs_qgroup *qgroup, s64 diff, int type), TP_ARGS(fs_info, qgroup, diff, type), @@ -1883,7 +1883,7 @@ TRACE_EVENT(qgroup_update_reserve, TRACE_EVENT(qgroup_meta_reserve, - TP_PROTO(struct btrfs_root *root, s64 diff, int type), + TP_PROTO(const struct btrfs_root *root, s64 diff, int type), TP_ARGS(root, diff, type), @@ -1906,7 +1906,7 @@ TRACE_EVENT(qgroup_meta_reserve, TRACE_EVENT(qgroup_meta_convert, - TP_PROTO(struct btrfs_root *root, s64 diff), + TP_PROTO(const struct btrfs_root *root, s64 diff), TP_ARGS(root, diff), @@ -1956,7 +1956,7 @@ DECLARE_EVENT_CLASS(btrfs__prelim_ref, TP_PROTO(const struct btrfs_fs_info *fs_info, const struct prelim_ref *oldref, const struct prelim_ref *newref, u64 tree_size), - TP_ARGS(fs_info, newref, oldref, tree_size), + TP_ARGS(fs_info, oldref, newref, tree_size), TP_STRUCT__entry_btrfs( __field( u64, root_id ) diff --git a/include/trace/events/erofs.h b/include/trace/events/erofs.h index e18684b02c3df..0179af103d0f4 100644 --- a/include/trace/events/erofs.h +++ b/include/trace/events/erofs.h @@ -237,24 +237,6 @@ DEFINE_EVENT(erofs__map_blocks_exit, z_erofs_map_blocks_iter_exit, TP_ARGS(inode, map, flags, ret) ); -TRACE_EVENT(erofs_destroy_inode, - TP_PROTO(struct inode *inode), - - TP_ARGS(inode), - - TP_STRUCT__entry( - __field( dev_t, dev ) - __field( erofs_nid_t, nid ) - ), - - TP_fast_assign( - __entry->dev = inode->i_sb->s_dev; - __entry->nid = EROFS_I(inode)->nid; - ), - - TP_printk("dev = (%d,%d), nid = %llu", show_dev_nid(__entry)) -); - #endif /* _TRACE_EROFS_H */ /* This part must be outside protection */ diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index dbea020258972..dd486ef095963 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -175,6 +175,15 @@ TRACE_DEFINE_ENUM(EX_BLOCK_AGE); #define S_ALL_PERM (S_ISUID | S_ISGID | S_ISVTX | \ S_IRWXU | S_IRWXG | S_IRWXO) +#define show_lock_name(lock) \ + __print_symbolic(lock, \ + { LOCK_NAME_CP_RWSEM, "cp_rwsem" }, \ + { LOCK_NAME_NODE_CHANGE, "node_change" }, \ + { LOCK_NAME_NODE_WRITE, "node_write" }, \ + { LOCK_NAME_GC_LOCK, "gc_lock" }, \ + { LOCK_NAME_CP_GLOBAL, "cp_global" }, \ + { LOCK_NAME_IO_RWSEM, "io_rwsem" }) + struct f2fs_sb_info; struct f2fs_io_info; struct extent_info; @@ -2404,6 +2413,127 @@ DEFINE_EVENT(f2fs__rw_end, f2fs_datawrite_end, TP_ARGS(inode, offset, bytes) ); +TRACE_EVENT(f2fs_lock_elapsed_time, + + TP_PROTO(struct f2fs_sb_info *sbi, enum f2fs_lock_name lock_name, + bool is_write, struct task_struct *p, int ioprio, + unsigned long long total_time, + unsigned long long running_time, + unsigned long long runnable_time, + unsigned long long io_sleep_time, + unsigned long long other_time), + + TP_ARGS(sbi, lock_name, is_write, p, ioprio, total_time, running_time, + runnable_time, io_sleep_time, other_time), + + TP_STRUCT__entry( + __field(dev_t, dev) + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __field(int, ioprio_class) + __field(int, ioprio_data) + __field(unsigned int, lock_name) + __field(bool, is_write) + __field(unsigned long long, total_time) + __field(unsigned long long, running_time) + __field(unsigned long long, runnable_time) + __field(unsigned long long, io_sleep_time) + __field(unsigned long long, other_time) + ), + + TP_fast_assign( + __entry->dev = sbi->sb->s_dev; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->ioprio_class = IOPRIO_PRIO_CLASS(ioprio); + __entry->ioprio_data = IOPRIO_PRIO_DATA(ioprio); + __entry->lock_name = lock_name; + __entry->is_write = is_write; + __entry->total_time = total_time; + __entry->running_time = running_time; + __entry->runnable_time = runnable_time; + __entry->io_sleep_time = io_sleep_time; + __entry->other_time = other_time; + ), + + TP_printk("dev = (%d,%d), comm: %s, pid: %d, prio: %d, " + "ioprio_class: %d, ioprio_data: %d, lock_name: %s, " + "lock_type: %s, total: %llu, running: %llu, " + "runnable: %llu, io_sleep: %llu, other: %llu", + show_dev(__entry->dev), + __entry->comm, + __entry->pid, + __entry->prio, + __entry->ioprio_class, + __entry->ioprio_data, + show_lock_name(__entry->lock_name), + __entry->is_write ? "wlock" : "rlock", + __entry->total_time, + __entry->running_time, + __entry->runnable_time, + __entry->io_sleep_time, + __entry->other_time) +); + +DECLARE_EVENT_CLASS(f2fs_priority_update, + + TP_PROTO(struct f2fs_sb_info *sbi, enum f2fs_lock_name lock_name, + bool is_write, struct task_struct *p, int orig_prio, + int new_prio), + + TP_ARGS(sbi, lock_name, is_write, p, orig_prio, new_prio), + + TP_STRUCT__entry( + __field(dev_t, dev) + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned int, lock_name) + __field(bool, is_write) + __field(int, orig_prio) + __field(int, new_prio) + ), + + TP_fast_assign( + __entry->dev = sbi->sb->s_dev; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->lock_name = lock_name; + __entry->is_write = is_write; + __entry->orig_prio = orig_prio; + __entry->new_prio = new_prio; + ), + + TP_printk("dev = (%d,%d), comm: %s, pid: %d, lock_name: %s, " + "lock_type: %s, orig_prio: %d, new_prio: %d", + show_dev(__entry->dev), + __entry->comm, + __entry->pid, + show_lock_name(__entry->lock_name), + __entry->is_write ? "wlock" : "rlock", + __entry->orig_prio, + __entry->new_prio) +); + +DEFINE_EVENT(f2fs_priority_update, f2fs_priority_uplift, + + TP_PROTO(struct f2fs_sb_info *sbi, enum f2fs_lock_name lock_name, + bool is_write, struct task_struct *p, int orig_prio, + int new_prio), + + TP_ARGS(sbi, lock_name, is_write, p, orig_prio, new_prio) +); + +DEFINE_EVENT(f2fs_priority_update, f2fs_priority_restore, + + TP_PROTO(struct f2fs_sb_info *sbi, enum f2fs_lock_name lock_name, + bool is_write, struct task_struct *p, int orig_prio, + int new_prio), + + TP_ARGS(sbi, lock_name, is_write, p, orig_prio, new_prio) +); + #endif /* _TRACE_F2FS_H */ /* This part must be outside protection */ diff --git a/include/trace/events/filelock.h b/include/trace/events/filelock.h index 1646dadd7f37c..3b1c8d93b2654 100644 --- a/include/trace/events/filelock.h +++ b/include/trace/events/filelock.h @@ -27,7 +27,8 @@ { FL_SLEEP, "FL_SLEEP" }, \ { FL_DOWNGRADE_PENDING, "FL_DOWNGRADE_PENDING" }, \ { FL_UNLOCK_PENDING, "FL_UNLOCK_PENDING" }, \ - { FL_OFDLCK, "FL_OFDLCK" }) + { FL_OFDLCK, "FL_OFDLCK" }, \ + { FL_RECLAIM, "FL_RECLAIM"}) #define show_fl_type(val) \ __print_symbolic(val, \ diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index 68f5280a41a4a..896f8de946d09 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -8,6 +8,7 @@ #include #include #include +#include TRACE_EVENT(kmem_cache_alloc, @@ -487,6 +488,30 @@ TRACE_EVENT(rss_stat, __print_symbolic(__entry->member, TRACE_MM_PAGES), __entry->size) ); + +TRACE_EVENT(dmabuf_rss_stat, + + TP_PROTO(size_t rss, ssize_t rss_delta, struct dma_buf *dmabuf), + + TP_ARGS(rss, rss_delta, dmabuf), + + TP_STRUCT__entry( + __field(size_t, rss) + __field(ssize_t, rss_delta) + __field(unsigned long, i_ino) + ), + + TP_fast_assign( + __entry->rss = rss; + __entry->rss_delta = rss_delta; + __entry->i_ino = file_inode(dmabuf->file)->i_ino; + ), + + TP_printk("rss=%zu delta=%zd i_ino=%lu", + __entry->rss, + __entry->rss_delta, + __entry->i_ino) + ); #endif /* _TRACE_KMEM_H */ /* This part must be outside protection */ diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h index e7c7b63894362..743f8f1f42a74 100644 --- a/include/trace/events/rxrpc.h +++ b/include/trace/events/rxrpc.h @@ -278,12 +278,15 @@ EM(rxrpc_call_put_userid, "PUT user-id ") \ EM(rxrpc_call_see_accept, "SEE accept ") \ EM(rxrpc_call_see_activate_client, "SEE act-clnt") \ + EM(rxrpc_call_see_already_released, "SEE alrdy-rl") \ EM(rxrpc_call_see_connect_failed, "SEE con-fail") \ EM(rxrpc_call_see_connected, "SEE connect ") \ EM(rxrpc_call_see_conn_abort, "SEE conn-abt") \ + EM(rxrpc_call_see_discard, "SEE discard ") \ EM(rxrpc_call_see_disconnected, "SEE disconn ") \ EM(rxrpc_call_see_distribute_error, "SEE dist-err") \ EM(rxrpc_call_see_input, "SEE input ") \ + EM(rxrpc_call_see_recvmsg, "SEE recvmsg ") \ EM(rxrpc_call_see_release, "SEE release ") \ EM(rxrpc_call_see_userid_exists, "SEE u-exists") \ EM(rxrpc_call_see_waiting_call, "SEE q-conn ") \ diff --git a/include/trace/events/thp.h b/include/trace/events/thp.h index f50048af5fcc2..c8fe879d5828b 100644 --- a/include/trace/events/thp.h +++ b/include/trace/events/thp.h @@ -8,6 +8,7 @@ #include #include +#ifdef CONFIG_PPC_BOOK3S_64 DECLARE_EVENT_CLASS(hugepage_set, TP_PROTO(unsigned long addr, unsigned long pte), @@ -66,6 +67,7 @@ DEFINE_EVENT(hugepage_update, hugepage_update_pud, TP_PROTO(unsigned long addr, unsigned long pud, unsigned long clr, unsigned long set), TP_ARGS(addr, pud, clr, set) ); +#endif /* CONFIG_PPC_BOOK3S_64 */ DECLARE_EVENT_CLASS(migration_pmd, diff --git a/include/trace/hooks/binder.h b/include/trace/hooks/binder.h old mode 100644 new mode 100755 index 124bf081f0aad..e4d270c402250 --- a/include/trace/hooks/binder.h +++ b/include/trace/hooks/binder.h @@ -93,7 +93,7 @@ DECLARE_HOOK(android_vh_binder_spawn_new_thread, TP_ARGS(thread, proc, force_spawn)); DECLARE_HOOK(android_vh_binder_has_special_work_ilocked, TP_PROTO(struct binder_thread *thread, bool do_proc_work, bool *has_work), - TP_ARGS(thread, do_proc_work, has_work)); + TP_ARGS(thread, do_proc_work, has_work)); DECLARE_HOOK(android_vh_binder_proc_transaction, TP_PROTO(struct task_struct *caller_task, struct task_struct *binder_proc_task, struct task_struct *binder_th_task, int node_debug_id, diff --git a/include/trace/hooks/blk.h b/include/trace/hooks/blk.h index 67453d1c75747..224784aa23bbc 100644 --- a/include/trace/hooks/blk.h +++ b/include/trace/hooks/blk.h @@ -10,8 +10,16 @@ #include +struct bio; +struct blk_mq_hw_ctx; struct block_device; struct gendisk; +struct path; +struct readahead_control; +struct request; +struct request_queue; +struct vfsmount; +typedef __u32 __bitwise blk_opf_t; DECLARE_HOOK(android_vh_bd_link_disk_holder, TP_PROTO(struct block_device *bdev, struct gendisk *disk), @@ -19,37 +27,30 @@ DECLARE_HOOK(android_vh_bd_link_disk_holder, DECLARE_HOOK(android_vh_blk_fill_rwbs, TP_PROTO(char *rwbs, unsigned int opf), TP_ARGS(rwbs, opf)); - -struct path; -struct vfsmount; - DECLARE_HOOK(android_vh_do_new_mount_fc, TP_PROTO(struct path *mountpoint, struct vfsmount *mnt), TP_ARGS(mountpoint, mnt)); - -struct readahead_control; -typedef __u32 __bitwise blk_opf_t; - +DECLARE_RESTRICTED_HOOK(android_rvh_blk_mq_has_request, + TP_PROTO(struct blk_mq_hw_ctx *hctx, struct request *rq, bool *ret), + TP_ARGS(hctx, rq, ret), 1); DECLARE_HOOK(android_vh_f2fs_ra_op_flags, TP_PROTO(blk_opf_t *op_flag, struct readahead_control *rac), TP_ARGS(op_flag, rac)); - -struct blk_mq_hw_ctx; -struct request_queue; - DECLARE_HOOK(android_vh_blk_mq_delay_run_hw_queue, TP_PROTO(int cpu, struct blk_mq_hw_ctx *hctx, unsigned long delay, bool *skip), TP_ARGS(cpu, hctx, delay, skip)); - DECLARE_HOOK(android_vh_blk_mq_kick_requeue_list, TP_PROTO(struct request_queue *q, unsigned long delay, bool *skip), TP_ARGS(q, delay, skip)); - -struct bio; - DECLARE_HOOK(android_vh_check_set_ioprio, TP_PROTO(struct bio *bio), TP_ARGS(bio)); +DECLARE_HOOK(android_vh_loop_skip_queue_work, + TP_PROTO(struct request *req, bool *skip), + TP_ARGS(req, skip)); +DECLARE_HOOK(android_vh_bio_add_page_merge_bypass, + TP_PROTO(struct bio *bio, bool *skip), + TP_ARGS(bio, skip)); #endif /* _TRACE_HOOK_BLK_H */ /* This part must be outside protection */ diff --git a/include/trace/hooks/cgroup.h b/include/trace/hooks/cgroup.h index b824197118f9a..f36c5a4be2bd4 100644 --- a/include/trace/hooks/cgroup.h +++ b/include/trace/hooks/cgroup.h @@ -38,6 +38,14 @@ DECLARE_RESTRICTED_HOOK(android_rvh_cpu_cgroup_attach, DECLARE_RESTRICTED_HOOK(android_rvh_cpu_cgroup_online, TP_PROTO(struct cgroup_subsys_state *css), TP_ARGS(css), 1); + +DECLARE_HOOK(android_vh_cpuset_attach_task, + TP_PROTO(struct cgroup_subsys_state *css, struct task_struct *task), + TP_ARGS(css, task)); + +DECLARE_HOOK(android_vh_cpuset_css_online, + TP_PROTO(struct cgroup_subsys_state *css), + TP_ARGS(css)); #endif #include diff --git a/include/trace/hooks/cpufreq.h b/include/trace/hooks/cpufreq.h index 04be5826b1623..8ffeae59ff681 100644 --- a/include/trace/hooks/cpufreq.h +++ b/include/trace/hooks/cpufreq.h @@ -47,6 +47,10 @@ DECLARE_HOOK(android_vh_cpufreq_online, TP_PROTO(struct cpufreq_policy *policy), TP_ARGS(policy)); +DECLARE_RESTRICTED_HOOK(android_rvh_cpufreq_create_policy, + TP_PROTO(struct cpufreq_policy *policy), + TP_ARGS(policy), 1); + #endif /* _TRACE_HOOK_CPUFREQ_H */ /* This part must be outside protection */ #include diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h old mode 100644 new mode 100755 index 1d3cac7834528..abe188cbd64f2 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -19,6 +19,7 @@ struct rt_mutex_base; struct rw_semaphore; struct task_struct; struct percpu_rw_semaphore; +struct rq; DECLARE_HOOK(android_vh_mutex_wait_start, TP_PROTO(struct mutex *lock), @@ -175,6 +176,30 @@ DECLARE_HOOK(android_vh_read_lazy_flag, DECLARE_HOOK(android_vh_set_tsk_need_resched_lazy, TP_PROTO(struct task_struct *p, struct rq *rq, int *need_lazy), TP_ARGS(p, rq, need_lazy)); + +DECLARE_HOOK(android_vh_resched_curr_lazy, + TP_PROTO(struct rq *rq, bool *skip_preempt), + TP_ARGS(rq, skip_preempt)); + +DECLARE_HOOK(android_vh_restore_curr_resched, + TP_PROTO(unsigned long *flags, int *lazy_flage), + TP_ARGS(flags, lazy_flage)); + +DECLARE_HOOK(android_vh_clear_curr_lazy, + TP_PROTO(struct task_struct *tsk), + TP_ARGS(tsk)); + +DECLARE_HOOK(android_vh_lock_delay_schedule, + TP_PROTO(struct task_struct *prev, unsigned int sched_mode, bool *ext_slice), + TP_ARGS(prev, sched_mode, ext_slice)); + +DECLARE_HOOK(android_vh_lock_task_fork, + TP_PROTO(struct task_struct *p), + TP_ARGS(p)); + +DECLARE_HOOK(android_vh_lock_task_exit, + TP_PROTO(struct task_struct *p), + TP_ARGS(p)); #endif /* _TRACE_HOOK_DTASK_H */ /* This part must be outside protection */ diff --git a/include/trace/hooks/fs.h b/include/trace/hooks/fs.h index d5fcb5357e4f9..0d434ab9786f2 100644 --- a/include/trace/hooks/fs.h +++ b/include/trace/hooks/fs.h @@ -28,6 +28,10 @@ DECLARE_HOOK(android_vh_f2fs_restore_priority, TP_PROTO(struct task_struct *p, int saved_prio), TP_ARGS(p, saved_prio)); +DECLARE_HOOK(android_vh_put_super, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb)); + DECLARE_HOOK(android_vh_ep_create_wakeup_source, TP_PROTO(char *name, int len), TP_ARGS(name, len)); diff --git a/include/trace/hooks/gic.h b/include/trace/hooks/gic.h index da2f3e0020bbe..bc3f874d61da1 100644 --- a/include/trace/hooks/gic.h +++ b/include/trace/hooks/gic.h @@ -9,12 +9,6 @@ #include -struct irq_data; -DECLARE_HOOK(android_vh_gic_set_affinity, - TP_PROTO(struct irq_data *d, const struct cpumask *mask_val, - bool force, u8 *gic_cpu_map, void __iomem *reg), - TP_ARGS(d, mask_val, force, gic_cpu_map, reg)); - DECLARE_RESTRICTED_HOOK(android_rvh_fiq_dump, TP_PROTO(struct pt_regs *regs), TP_ARGS(regs), 1); diff --git a/include/trace/hooks/gic_v3.h b/include/trace/hooks/gic_v3.h index 0f81993df161c..54368d65b319b 100644 --- a/include/trace/hooks/gic_v3.h +++ b/include/trace/hooks/gic_v3.h @@ -14,9 +14,6 @@ */ struct irq_data; struct cpumask; -DECLARE_HOOK(android_vh_gic_v3_affinity_init, - TP_PROTO(int irq, u32 offset, u64 *affinity), - TP_ARGS(irq, offset, affinity)); DECLARE_RESTRICTED_HOOK(android_rvh_gic_v3_set_affinity, TP_PROTO(struct irq_data *d, const struct cpumask *mask_val, u64 *affinity, bool force, void __iomem *base, diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h old mode 100644 new mode 100755 index bbd59dd1ee414..ef3e553d1d4ae --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -55,6 +55,9 @@ DECLARE_RESTRICTED_HOOK(android_rvh_try_alloc_pages_gfp, TP_PROTO(struct page **page, unsigned int order, gfp_t gfp, enum zone_type highest_zoneidx), TP_ARGS(page, order, gfp, highest_zoneidx), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_swap_bio_charge, + TP_PROTO(struct bio *bio), + TP_ARGS(bio), 1); DECLARE_HOOK(android_vh_slab_alloc_node, TP_PROTO(void *object, unsigned long addr, struct kmem_cache *s), TP_ARGS(object, addr, s)); @@ -182,6 +185,18 @@ DECLARE_HOOK(android_vh_tune_mmap_readaround, TP_PROTO(unsigned int ra_pages, pgoff_t pgoff, pgoff_t *start, unsigned int *size, unsigned int *async_size), TP_ARGS(ra_pages, pgoff, start, size, async_size)); +struct slabinfo; +DECLARE_HOOK(android_vh_cache_show, + TP_PROTO(struct seq_file *m, struct slabinfo *sinfo, struct kmem_cache *s), + TP_ARGS(m, sinfo, s)); +DECLARE_HOOK(android_vh_alloc_pages_reclaim_bypass, + TP_PROTO(gfp_t gfp_mask, int order, int alloc_flags, + int migratetype, struct page **page), + TP_ARGS(gfp_mask, order, alloc_flags, migratetype, page)); +DECLARE_HOOK(android_vh_alloc_pages_failure_bypass, + TP_PROTO(gfp_t gfp_mask, int order, int alloc_flags, + int migratetype, struct page **page), + TP_ARGS(gfp_mask, order, alloc_flags, migratetype, page)); DECLARE_HOOK(android_vh_madvise_cold_pageout_skip, TP_PROTO(struct vm_area_struct *vma, struct folio *folio, bool pageout, bool *need_skip), TP_ARGS(vma, folio, pageout, need_skip)); @@ -256,6 +271,17 @@ DECLARE_HOOK(android_vh_try_to_unmap_one, TP_PROTO(struct folio *folio, struct vm_area_struct *vma, unsigned long addr, void *arg, bool ret), TP_ARGS(folio, vma, addr, arg, ret)); +DECLARE_HOOK(android_vh_save_vmalloc_stack, + TP_PROTO(unsigned long flags, struct vm_struct *vm), + TP_ARGS(flags, vm)); +DECLARE_HOOK(android_vh_show_stack_hash, + TP_PROTO(struct seq_file *m, struct vm_struct *v), + TP_ARGS(m, v)); +DECLARE_HOOK(android_vh_drain_all_pages_bypass, + TP_PROTO(gfp_t gfp_mask, unsigned int order, unsigned long alloc_flags, + int migratetype, unsigned long did_some_progress, + bool *bypass), + TP_ARGS(gfp_mask, order, alloc_flags, migratetype, did_some_progress, bypass)); DECLARE_HOOK(android_vh_mm_direct_reclaim_enter, TP_PROTO(unsigned int order), TP_ARGS(order)); @@ -307,10 +333,6 @@ DECLARE_HOOK(android_vh_show_mem, DECLARE_HOOK(android_vh_print_slabinfo_header, TP_PROTO(struct seq_file *m), TP_ARGS(m)); -struct slabinfo; -DECLARE_HOOK(android_vh_cache_show, - TP_PROTO(struct seq_file *m, struct slabinfo *sinfo, struct kmem_cache *s), - TP_ARGS(m, sinfo, s)); DECLARE_HOOK(android_vh_customize_alloc_gfp, TP_PROTO(gfp_t *alloc_gfp, unsigned int order), TP_ARGS(alloc_gfp, order)); @@ -340,14 +362,6 @@ DECLARE_HOOK(android_vh_show_smap_swap_shared, DECLARE_HOOK(android_vh_count_workingset_refault, TP_PROTO(struct folio *folio), TP_ARGS(folio)); -DECLARE_HOOK(android_vh_alloc_pages_reclaim_bypass, - TP_PROTO(gfp_t gfp_mask, int order, int alloc_flags, - int migratetype, struct page **page), - TP_ARGS(gfp_mask, order, alloc_flags, migratetype, page)); -DECLARE_HOOK(android_vh_alloc_pages_failure_bypass, - TP_PROTO(gfp_t gfp_mask, int order, int alloc_flags, - int migratetype, struct page **page), - TP_ARGS(gfp_mask, order, alloc_flags, migratetype, page)); DECLARE_HOOK(android_vh_swapmem_gather_init, TP_PROTO(struct mm_struct *mm), TP_ARGS(mm)); @@ -363,17 +377,6 @@ DECLARE_HOOK(android_vh_oom_swapmem_gather_init, DECLARE_HOOK(android_vh_oom_swapmem_gather_finish, TP_PROTO(struct mm_struct *mm), TP_ARGS(mm)); -DECLARE_HOOK(android_vh_drain_all_pages_bypass, - TP_PROTO(gfp_t gfp_mask, unsigned int order, unsigned long alloc_flags, - int migratetype, unsigned long did_some_progress, - bool *bypass), - TP_ARGS(gfp_mask, order, alloc_flags, migratetype, did_some_progress, bypass)); -DECLARE_HOOK(android_vh_save_vmalloc_stack, - TP_PROTO(unsigned long flags, struct vm_struct *vm), - TP_ARGS(flags, vm)); -DECLARE_HOOK(android_vh_show_stack_hash, - TP_PROTO(struct seq_file *m, struct vm_struct *v), - TP_ARGS(m, v)); DECLARE_HOOK(android_vh_update_page_mapcount, TP_PROTO(struct page *page, bool inc_size, bool compound, int *first_mapping, bool *success), @@ -443,6 +446,18 @@ DECLARE_HOOK(android_vh_do_async_mmap_readahead, DECLARE_HOOK(android_vh_mm_free_page, TP_PROTO(struct page *page), TP_ARGS(page)); +DECLARE_HOOK(android_vh_page_cache_ra_unbounded, + TP_PROTO(struct address_space *mapping, struct folio *folio, u64 *data), + TP_ARGS(mapping, folio, data)); +DECLARE_HOOK(android_vh_force_page_cache_ra, + TP_PROTO(struct address_space *mapping, u64 *data), + TP_ARGS(mapping, data)); +DECLARE_HOOK(android_vh_filemap_fault_folio_locked, + TP_PROTO(struct inode *inode, struct folio *folio, pgoff_t index), + TP_ARGS(inode, folio, index)); +DECLARE_HOOK(android_vh_filemap_read_end, + TP_PROTO(struct inode *inode, struct folio **folios, unsigned int nr), + TP_ARGS(inode, folios, nr)); DECLARE_HOOK(android_vh_cma_debug_show_areas, TP_PROTO(bool *show), @@ -492,6 +507,9 @@ DECLARE_HOOK(android_vh_filemap_map_pages, TP_PROTO(struct file *file, pgoff_t first_pgoff, pgoff_t last_pgoff, vm_fault_t ret), TP_ARGS(file, first_pgoff, last_pgoff, ret)); +DECLARE_HOOK(android_vh_thaw_killed_process, + TP_PROTO(bool *thaw), + TP_ARGS(thaw)); DECLARE_HOOK(android_vh_page_cache_readahead_start, TP_PROTO(struct file *file, pgoff_t pgoff, unsigned int size, bool sync), @@ -499,6 +517,10 @@ DECLARE_HOOK(android_vh_page_cache_readahead_start, DECLARE_HOOK(android_vh_page_cache_readahead_end, TP_PROTO(struct file *file, pgoff_t pgoff), TP_ARGS(file, pgoff)); +DECLARE_HOOK(android_vh_page_cache_ra_order_bypass, + TP_PROTO(struct readahead_control *ractl, struct file_ra_state *ra, + int new_order, gfp_t *gfp, bool *bypass), + TP_ARGS(ractl, ra, new_order, gfp, bypass)); DECLARE_HOOK(android_vh_filemap_fault_start, TP_PROTO(struct file *file, pgoff_t pgoff), TP_ARGS(file, pgoff)); @@ -542,6 +564,9 @@ DECLARE_HOOK(android_vh_do_group_exit, DECLARE_HOOK(android_vh_migration_target_bypass, TP_PROTO(struct page *page, bool *bypass), TP_ARGS(page, bypass)); +DECLARE_HOOK(android_vh_oom_evaluate_task_bypass, + TP_PROTO(struct task_struct *task, struct oom_control *oc, bool *bypass), + TP_ARGS(task, oc, bypass)); DECLARE_HOOK(android_vh_swap_writepage, TP_PROTO(unsigned long *sis_flags, struct page *page), TP_ARGS(sis_flags, page)); @@ -566,9 +591,6 @@ DECLARE_HOOK(android_vh_copy_page_from_user, DECLARE_HOOK(android_vh_page_private_mod, TP_PROTO(struct page *page, unsigned long private), TP_ARGS(page, private)); -DECLARE_HOOK(android_vh_cma_alloc_fail, - TP_PROTO(char *name, unsigned long count, unsigned long req_count), - TP_ARGS(name, count, req_count)); DECLARE_HOOK(android_vh_filemap_map_pages_range, TP_PROTO(struct file *file, pgoff_t orig_start_pgoff, pgoff_t last_pgoff, vm_fault_t ret), @@ -609,9 +631,57 @@ DECLARE_HOOK(android_vh_filemap_folio_mapped, DECLARE_HOOK(android_vh_folio_remove_rmap_ptes, TP_PROTO(struct folio *folio), TP_ARGS(folio)); +DECLARE_HOOK(android_vh_folio_add_lru, + TP_PROTO(struct folio *folio), + TP_ARGS(folio)); DECLARE_HOOK(android_vh_pageset_update, TP_PROTO(unsigned long *high, unsigned long *batch), TP_ARGS(high, batch)); +DECLARE_HOOK(android_vh_mempool_alloc_skip_wait, + TP_PROTO(gfp_t *gfp_flags, bool *skip_wait), + TP_ARGS(gfp_flags, skip_wait)); +DECLARE_HOOK(android_vh_mm_customize_ac, + TP_PROTO(gfp_t gfp, unsigned int order, struct zonelist **zonelist, + struct zoneref **preferred_zoneref, enum zone_type *highest_zoneidx, + unsigned int *alloc_flags), + TP_ARGS(gfp, order, zonelist, preferred_zoneref, highest_zoneidx, alloc_flags)); +DECLARE_HOOK(android_vh_mm_customize_rmqueue, + TP_PROTO(struct zone *zone, unsigned int order, unsigned int *alloc_flags, + int *migratetype), + TP_ARGS(zone, order, alloc_flags, migratetype)); +DECLARE_HOOK(android_vh_mm_customize_suitable_zone, + TP_PROTO(struct zone *zone, gfp_t gfp, int order, enum zone_type highest_zoneidx, + bool *use_this_zone, bool *suitable), + TP_ARGS(zone, gfp, order, highest_zoneidx, use_this_zone, suitable)); +DECLARE_HOOK(android_vh_mm_customize_zone_max_order, + TP_PROTO(struct zone *zone, int *max_order), + TP_ARGS(zone, max_order)); +DECLARE_HOOK(android_vh_mm_customize_zone_pageset, + TP_PROTO(struct zone *zone, int *new_high, int *new_batch), + TP_ARGS(zone, new_high, new_batch)); +DECLARE_HOOK(android_vh_mm_customize_lru_add_dst, + TP_PROTO(struct lruvec *lruvec, struct folio *src, struct folio *dst, bool *added), + TP_ARGS(lruvec, src, dst, added)); +DECLARE_HOOK(android_vh_oom_reaper_delay_bypass, + TP_PROTO(struct task_struct *tsk, bool *bypass), + TP_ARGS(tsk, bypass)); +DECLARE_HOOK(android_vh_filemap_get_folio_end, + TP_PROTO(struct address_space *mapping, struct folio *folio), + TP_ARGS(mapping, folio)); +DECLARE_HOOK(android_vh_nr_pcp_alloc, + TP_PROTO(struct per_cpu_pages *pcp, struct zone *zone, + unsigned long __percpu **pad, unsigned int order, int *batch), + TP_ARGS(pcp, zone, pad, order, batch)); +DECLARE_HOOK(android_vh_pcp_alloc_factor_adjust, + TP_PROTO(struct zone *zone, unsigned long __percpu *pad, + struct per_cpu_pages *pcp, struct page *page, int migratetype, + unsigned int order), + TP_ARGS(zone, pad, pcp, page, migratetype, order)); +DECLARE_RESTRICTED_HOOK(android_rvh_gup_longterm_locked, + TP_PROTO(long rc, long nr_pinned_pages, + unsigned long start, unsigned long nr_pages, + struct page **pages), + TP_ARGS(rc, nr_pinned_pages, start, nr_pages, pages), 5); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/include/trace/hooks/power.h b/include/trace/hooks/power.h index e845d0ef6025b..ec9caa875eb09 100644 --- a/include/trace/hooks/power.h +++ b/include/trace/hooks/power.h @@ -12,6 +12,10 @@ DECLARE_HOOK(android_vh_try_to_freeze_todo, TP_PROTO(unsigned int todo, unsigned int elapsed_msecs, bool wq_busy), TP_ARGS(todo, elapsed_msecs, wq_busy)); +DECLARE_HOOK(android_vh_try_to_freeze_todo_logging, + TP_PROTO(bool *logging_on), + TP_ARGS(logging_on)); + DECLARE_HOOK(android_vh_try_to_freeze_todo_unfrozen, TP_PROTO(struct task_struct *p), TP_ARGS(p)); @@ -33,6 +37,10 @@ DECLARE_HOOK(android_vh_freq_qos_remove_request, TP_PROTO(struct freq_qos_request *req), TP_ARGS(req)); +DECLARE_HOOK(android_vh_fas_gpu_qos_update_tracer, + TP_PROTO(struct freq_qos_request *req, s32 *value), + TP_ARGS(req, value)); + #endif /* _TRACE_HOOK_POWER_H */ /* This part must be outside protection */ #include diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h old mode 100644 new mode 100755 diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h old mode 100644 new mode 100755 index c0b0e9d453a12..7503d5d6a96a6 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -184,6 +184,15 @@ DECLARE_RESTRICTED_HOOK(android_rvh_account_irq, TP_PROTO(struct task_struct *curr, int cpu, s64 delta, bool start), TP_ARGS(curr, cpu, delta, start), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_before_pick_task_fair, + TP_PROTO(struct rq *rq, struct task_struct **p, struct sched_entity **se, + struct task_struct *prev, struct rq_flags *rf), + TP_ARGS(rq, p, se, prev, rf), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_balance_fair, + TP_PROTO(struct rq *rq, struct task_struct *prev, struct rq_flags *rf), + TP_ARGS(rq, prev, rf), 1); + struct sched_entity; DECLARE_RESTRICTED_HOOK(android_rvh_place_entity, TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial, u64 *vruntime), @@ -464,10 +473,6 @@ DECLARE_RESTRICTED_HOOK(android_rvh_update_blocked_fair, struct sched_attr; -DECLARE_HOOK(android_vh_set_sugov_sched_attr, - TP_PROTO(struct sched_attr *attr), - TP_ARGS(attr)); - DECLARE_RESTRICTED_HOOK(android_rvh_set_iowait, TP_PROTO(struct task_struct *p, struct rq *rq, int *should_iowait_boost), TP_ARGS(p, rq, should_iowait_boost), 1); @@ -488,6 +493,19 @@ DECLARE_HOOK(android_vh_set_task_comm, TP_PROTO(struct task_struct *p), TP_ARGS(p)); +DECLARE_HOOK(android_vh_chk_task, + TP_PROTO(struct task_struct **pp, struct rq *rq), + TP_ARGS(pp, rq)); + +DECLARE_HOOK(android_vh_put_task, + TP_PROTO(struct task_struct *p), + TP_ARGS(p)); + +DECLARE_RESTRICTED_HOOK(android_rvh_task_fits_cpu, + TP_PROTO(struct task_struct *tsk, unsigned long util, unsigned long uclamp_min, + unsigned long uclamp_max, int cpu, bool *fits, bool *done), + TP_ARGS(tsk, util, uclamp_min, uclamp_max, cpu, fits, done), 1); + #endif /* _TRACE_HOOK_SCHED_H */ /* This part must be outside protection */ #include diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index 4bdcfc29c2f4f..3ca2201850ae6 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -99,6 +99,12 @@ DECLARE_HOOK(android_vh_use_vm_swappiness, DECLARE_HOOK(android_vh_tune_scan_control, TP_PROTO(bool *skip_swap), TP_ARGS(skip_swap)); +DECLARE_HOOK(android_vh_remove_mapping, + TP_PROTO(struct address_space *mapping, struct folio *folio, bool reclaimed), + TP_ARGS(mapping, folio, reclaimed)); +DECLARE_HOOK(android_vh_remove_mapping_failed, + TP_PROTO(struct address_space *mapping, struct folio *folio, bool reclaimed), + TP_ARGS(mapping, folio, reclaimed)); DECLARE_HOOK(android_vh_handle_trylock_failed_folio, TP_PROTO(struct list_head *folio_list), TP_ARGS(folio_list)); @@ -133,6 +139,17 @@ DECLARE_HOOK(android_vh_direct_reclaim_end, DECLARE_HOOK(android_vh_should_split_folio_to_list, TP_PROTO(struct folio *folio, bool *should_split_to_list), TP_ARGS(folio, should_split_to_list)); +DECLARE_HOOK(android_vh_mm_isolate_priv_lru, + TP_PROTO(unsigned long nr_to_scan, struct lruvec *lruvec, enum lru_list lru, + struct list_head *dst, int reclaim_idx, bool may_unmap, + unsigned long *nr_scanned, unsigned long *nr_taken), + TP_ARGS(nr_to_scan, lruvec, lru, dst, reclaim_idx, may_unmap, nr_scanned, nr_taken)); +DECLARE_HOOK(android_vh_mm_customize_pgdat_balanced, + TP_PROTO(int order, int highest_zoneidx, bool *balanced, bool *customized), + TP_ARGS(order, highest_zoneidx, balanced, customized)); +DECLARE_HOOK(android_vh_mm_customize_file_is_tiny, + TP_PROTO(unsigned int may_swap, int order, int highest_zoneidx, bool *file_is_tiny), + TP_ARGS(may_swap, order, highest_zoneidx, file_is_tiny)); #endif /* _TRACE_HOOK_VMSCAN_H */ /* This part must be outside protection */ #include diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 1682262eff30f..e7e5659b56252 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1150,6 +1150,7 @@ enum bpf_perf_event_type { #define BPF_F_BEFORE (1U << 3) #define BPF_F_AFTER (1U << 4) #define BPF_F_ID (1U << 5) +#define BPF_F_PREORDER (1U << 6) #define BPF_F_LINK BPF_F_LINK /* 1 << 13 */ /* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the @@ -1922,6 +1923,7 @@ union bpf_attr { * for updates resulting in a null checksum the value is set to * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates * the checksum is to be computed against a pseudo-header. + * Flag **BPF_F_IPV6** should be set for IPv6 packets. * * This helper works in combination with **bpf_csum_diff**\ (), * which does not update the checksum in-place, but offers more @@ -5929,6 +5931,7 @@ enum { BPF_F_PSEUDO_HDR = (1ULL << 4), BPF_F_MARK_MANGLED_0 = (1ULL << 5), BPF_F_MARK_ENFORCE = (1ULL << 6), + BPF_F_IPV6 = (1ULL << 7), }; /* BPF_FUNC_skb_set_tunnel_key and BPF_FUNC_skb_get_tunnel_key flags. */ diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index 606b52e88ce33..3d1987e1bb2dd 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -31,6 +31,7 @@ enum idxd_scmd_stat { IDXD_SCMD_WQ_IRQ_ERR = 0x80100000, IDXD_SCMD_WQ_USER_NO_IOMMU = 0x80110000, IDXD_SCMD_DEV_EVL_ERR = 0x80120000, + IDXD_SCMD_WQ_NO_DRV_NAME = 0x80200000, }; #define IDXD_SCMD_SOFTERR_MASK 0x80000000 diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index ce3117df9cec2..6750911da4f06 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -950,6 +950,7 @@ enum { IFLA_BOND_AD_LACP_ACTIVE, IFLA_BOND_MISSED_MAX, IFLA_BOND_NS_IP6_TARGET, + IFLA_BOND_COUPLED_CONTROL, __IFLA_BOND_MAX, }; diff --git a/include/uapi/linux/in6.h b/include/uapi/linux/in6.h index ff8d21f9e95b7..5a47339ef7d76 100644 --- a/include/uapi/linux/in6.h +++ b/include/uapi/linux/in6.h @@ -152,7 +152,6 @@ struct in6_flowlabel_req { /* * IPV6 socket options */ -#if __UAPI_DEF_IPV6_OPTIONS #define IPV6_ADDRFORM 1 #define IPV6_2292PKTINFO 2 #define IPV6_2292HOPOPTS 3 @@ -169,8 +168,10 @@ struct in6_flowlabel_req { #define IPV6_MULTICAST_IF 17 #define IPV6_MULTICAST_HOPS 18 #define IPV6_MULTICAST_LOOP 19 +#if __UAPI_DEF_IPV6_OPTIONS #define IPV6_ADD_MEMBERSHIP 20 #define IPV6_DROP_MEMBERSHIP 21 +#endif #define IPV6_ROUTER_ALERT 22 #define IPV6_MTU_DISCOVER 23 #define IPV6_MTU 24 @@ -203,7 +204,6 @@ struct in6_flowlabel_req { #define IPV6_IPSEC_POLICY 34 #define IPV6_XFRM_POLICY 35 #define IPV6_HDRINCL 36 -#endif /* * Multicast: diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 8e61f8b7c2ced..368645040e5c8 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -466,6 +466,7 @@ struct io_cqring_offsets { #define IORING_ENTER_SQ_WAIT (1U << 2) #define IORING_ENTER_EXT_ARG (1U << 3) #define IORING_ENTER_REGISTERED_RING (1U << 4) +#define IORING_ENTER_NO_IOWAIT (1U << 7) /* * Passed in for io_uring_setup(2). Copied back with updated info on success @@ -500,6 +501,7 @@ struct io_uring_params { #define IORING_FEAT_CQE_SKIP (1U << 11) #define IORING_FEAT_LINKED_FILE (1U << 12) #define IORING_FEAT_REG_REG_RING (1U << 13) +#define IORING_FEAT_NO_IOWAIT (1U << 17) /* * io_uring_register(2) opcodes and arguments diff --git a/include/uapi/linux/mptcp.h b/include/uapi/linux/mptcp.h index ee9c49f949a2c..00d6221216733 100644 --- a/include/uapi/linux/mptcp.h +++ b/include/uapi/linux/mptcp.h @@ -81,6 +81,8 @@ enum { #define MPTCP_PM_ADDR_ATTR_MAX (__MPTCP_PM_ADDR_ATTR_MAX - 1) +#define MPTCP_PM_EV_FLAG_DENY_JOIN_ID0 _BITUL(0) + #define MPTCP_PM_ADDR_FLAG_SIGNAL (1 << 0) #define MPTCP_PM_ADDR_FLAG_SUBFLOW (1 << 1) #define MPTCP_PM_ADDR_FLAG_BACKUP (1 << 2) @@ -132,13 +134,13 @@ struct mptcp_info { /* * MPTCP_EVENT_CREATED: token, family, saddr4 | saddr6, daddr4 | daddr6, - * sport, dport + * sport, dport, server-side, [flags] * A new MPTCP connection has been created. It is the good time to allocate * memory and send ADD_ADDR if needed. Depending on the traffic-patterns * it can take a long time until the MPTCP_EVENT_ESTABLISHED is sent. * * MPTCP_EVENT_ESTABLISHED: token, family, saddr4 | saddr6, daddr4 | daddr6, - * sport, dport + * sport, dport, server-side, [flags] * A MPTCP connection is established (can start new subflows). * * MPTCP_EVENT_CLOSED: token diff --git a/include/uapi/linux/netlink.h b/include/uapi/linux/netlink.h index e2ae82e3f9f71..f87aaf28a6491 100644 --- a/include/uapi/linux/netlink.h +++ b/include/uapi/linux/netlink.h @@ -298,6 +298,8 @@ struct nla_bitfield32 { * entry has attributes again, the policy for those inner ones * and the corresponding maxtype may be specified. * @NL_ATTR_TYPE_BITFIELD32: &struct nla_bitfield32 attribute + * @NL_ATTR_TYPE_SINT: 32-bit or 64-bit signed attribute, aligned to 4B + * @NL_ATTR_TYPE_UINT: 32-bit or 64-bit unsigned attribute, aligned to 4B */ enum netlink_attribute_type { NL_ATTR_TYPE_INVALID, @@ -322,6 +324,9 @@ enum netlink_attribute_type { NL_ATTR_TYPE_NESTED_ARRAY, NL_ATTR_TYPE_BITFIELD32, + + NL_ATTR_TYPE_SINT, + NL_ATTR_TYPE_UINT, }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 056c6694d1f11..8315d0f7e4672 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -11,7 +11,7 @@ * Copyright 2008 Jouni Malinen * Copyright 2008 Colin McCabe * Copyright 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2023 Intel Corporation * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -4254,10 +4254,6 @@ enum nl80211_wmm_rule { * allowed for peer-to-peer or adhoc communication under the control * of a DFS master which operates on the same channel (FCC-594280 D01 * Section B.3). Should be used together with %NL80211_RRF_DFS only. - * @NL80211_FREQUENCY_ATTR_NO_6GHZ_VLP_CLIENT: Client connection to VLP AP - * not allowed using this channel - * @NL80211_FREQUENCY_ATTR_NO_6GHZ_AFC_CLIENT: Client connection to AFC AP - * not allowed using this channel * @NL80211_FREQUENCY_ATTR_MAX: highest frequency attribute number * currently defined * @__NL80211_FREQUENCY_ATTR_AFTER_LAST: internal use @@ -4298,8 +4294,6 @@ enum nl80211_frequency_attr { NL80211_FREQUENCY_ATTR_NO_EHT, NL80211_FREQUENCY_ATTR_PSD, NL80211_FREQUENCY_ATTR_DFS_CONCURRENT, - NL80211_FREQUENCY_ATTR_NO_6GHZ_VLP_CLIENT, - NL80211_FREQUENCY_ATTR_NO_6GHZ_AFC_CLIENT, /* keep last */ __NL80211_FREQUENCY_ATTR_AFTER_LAST, @@ -4312,10 +4306,6 @@ enum nl80211_frequency_attr { #define NL80211_FREQUENCY_ATTR_NO_IR NL80211_FREQUENCY_ATTR_NO_IR #define NL80211_FREQUENCY_ATTR_GO_CONCURRENT \ NL80211_FREQUENCY_ATTR_IR_CONCURRENT -#define NL80211_FREQUENCY_ATTR_NO_UHB_VLP_CLIENT \ - NL80211_FREQUENCY_ATTR_NO_6GHZ_VLP_CLIENT -#define NL80211_FREQUENCY_ATTR_NO_UHB_AFC_CLIENT \ - NL80211_FREQUENCY_ATTR_NO_6GHZ_AFC_CLIENT /** * enum nl80211_bitrate_attr - bitrate attributes @@ -4513,8 +4503,6 @@ enum nl80211_sched_scan_match_attr { peer-to-peer or adhoc communication under the control of a DFS master which operates on the same channel (FCC-594280 D01 Section B.3). Should be used together with %NL80211_RRF_DFS only. - * @NL80211_RRF_NO_6GHZ_VLP_CLIENT: Client connection to VLP AP not allowed - * @NL80211_RRF_NO_6GHZ_AFC_CLIENT: Client connection to AFC AP not allowed */ enum nl80211_reg_rule_flags { NL80211_RRF_NO_OFDM = 1<<0, @@ -4537,8 +4525,6 @@ enum nl80211_reg_rule_flags { NL80211_RRF_NO_EHT = 1<<19, NL80211_RRF_PSD = 1<<20, NL80211_RRF_DFS_CONCURRENT = 1<<21, - NL80211_RRF_NO_6GHZ_VLP_CLIENT = 1<<22, - NL80211_RRF_NO_6GHZ_AFC_CLIENT = 1<<23, }; #define NL80211_RRF_PASSIVE_SCAN NL80211_RRF_NO_IR @@ -4547,8 +4533,6 @@ enum nl80211_reg_rule_flags { #define NL80211_RRF_NO_HT40 (NL80211_RRF_NO_HT40MINUS |\ NL80211_RRF_NO_HT40PLUS) #define NL80211_RRF_GO_CONCURRENT NL80211_RRF_IR_CONCURRENT -#define NL80211_RRF_NO_UHB_VLP_CLIENT NL80211_RRF_NO_6GHZ_VLP_CLIENT -#define NL80211_RRF_NO_UHB_AFC_CLIENT NL80211_RRF_NO_6GHZ_AFC_CLIENT /* For backport compatibility with older userspace */ #define NL80211_RRF_NO_IR_ALL (NL80211_RRF_NO_IR | __NL80211_RRF_NO_IBSS) @@ -5096,17 +5080,11 @@ enum nl80211_bss_use_for { * BSS isn't possible * @NL80211_BSS_CANNOT_USE_NSTR_NONPRIMARY: NSTR nonprimary links aren't * supported by the device, and this BSS entry represents one. - * @NL80211_BSS_CANNOT_USE_6GHZ_PWR_MISMATCH: STA is not supporting - * the AP power type (SP, VLP, AP) that the AP uses. */ enum nl80211_bss_cannot_use_reasons { NL80211_BSS_CANNOT_USE_NSTR_NONPRIMARY = 1 << 0, - NL80211_BSS_CANNOT_USE_6GHZ_PWR_MISMATCH = 1 << 1, }; -#define NL80211_BSS_CANNOT_USE_UHB_PWR_MISMATCH \ - NL80211_BSS_CANNOT_USE_6GHZ_PWR_MISMATCH - /** * enum nl80211_bss - netlink attributes for a BSS * diff --git a/include/uapi/linux/pfrut.h b/include/uapi/linux/pfrut.h index 42fa15f8310d6..b77d5c210c262 100644 --- a/include/uapi/linux/pfrut.h +++ b/include/uapi/linux/pfrut.h @@ -89,6 +89,7 @@ struct pfru_payload_hdr { __u32 hw_ver; __u32 rt_ver; __u8 platform_id[16]; + __u32 svn_ver; }; enum pfru_dsm_status { diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h old mode 100644 new mode 100755 index 3bac0a8ceab26..359a14cc76a40 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -118,6 +118,7 @@ struct clone_args { /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 #define SCHED_DEADLINE 6 +#define SCHED_EXT 7 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 diff --git a/include/uapi/linux/types.h b/include/uapi/linux/types.h index 6375a06840520..48b933938877d 100644 --- a/include/uapi/linux/types.h +++ b/include/uapi/linux/types.h @@ -53,6 +53,7 @@ typedef __u32 __bitwise __wsum; * No conversions are necessary between 32-bit user-space and a 64-bit kernel. */ #define __aligned_u64 __u64 __attribute__((aligned(8))) +#define __aligned_s64 __s64 __attribute__((aligned(8))) #define __aligned_be64 __be64 __attribute__((aligned(8))) #define __aligned_le64 __le64 __attribute__((aligned(8))) diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index ca2df6b3e2a10..adfabdba5eed9 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -364,8 +364,10 @@ struct uffdio_move { * in swap-cache. Also, to avoid livelock when multiple threads try * to move same src folio. It's a KMI workaround and cannot be relied * upon by userspace. + * + * 61: confirm if anon_vma lock has been removed from MOVE ioctl. */ -#define UFFDIO_MOVE_MODE_CONFIRM_FIXED ((__u64)1<<63) +#define UFFDIO_MOVE_MODE_CONFIRM_FIXED ((__u64)1<<61) __u64 mode; /* * "move" is written by the ioctl and must be at the end: the diff --git a/include/uapi/linux/vm_sockets.h b/include/uapi/linux/vm_sockets.h index ed07181d4eff9..e05280e415228 100644 --- a/include/uapi/linux/vm_sockets.h +++ b/include/uapi/linux/vm_sockets.h @@ -17,6 +17,10 @@ #ifndef _UAPI_VM_SOCKETS_H #define _UAPI_VM_SOCKETS_H +#ifndef __KERNEL__ +#include /* for struct sockaddr and sa_family_t */ +#endif + #include #include diff --git a/include/ufs/ufs_quirks.h b/include/ufs/ufs_quirks.h index 41ff44dfa1db3..83563247c36cb 100644 --- a/include/ufs/ufs_quirks.h +++ b/include/ufs/ufs_quirks.h @@ -107,4 +107,13 @@ struct ufs_dev_quirk { */ #define UFS_DEVICE_QUIRK_DELAY_AFTER_LPM (1 << 11) +/* + * Some ufs devices may need more time to be in hibern8 before exiting. + * Enable this quirk to give it an additional 100us. + */ +#define UFS_DEVICE_QUIRK_PA_HIBER8TIME (1 << 12) + +/* Some UFS 4 devices do not support the qTimestamp attribute */ +#define UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT (1 << 13) + #endif /* UFS_QUIRKS_H_ */ diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index cde9ad6489b2c..12b2458be1834 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1507,5 +1507,6 @@ int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask); int ufshcd_write_ee_control(struct ufs_hba *hba); int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, const u16 *other_mask, u16 set, u16 clr); +void ufshcd_force_error_recovery(struct ufs_hba *hba); #endif /* End of Header */ diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h index 21001c09b1db0..08249f719bfe3 100644 --- a/include/ufs/ufshci.h +++ b/include/ufs/ufshci.h @@ -182,6 +182,7 @@ static inline u32 ufshci_version(u32 major, u32 minor) #define UTP_TASK_REQ_COMPL 0x200 #define UIC_COMMAND_COMPL 0x400 #define DEVICE_FATAL_ERROR 0x800 +#define UTP_ERROR 0x1000 #define CONTROLLER_FATAL_ERROR 0x10000 #define SYSTEM_BUS_FATAL_ERROR 0x20000 #define CRYPTO_ENGINE_FATAL_ERROR 0x40000 @@ -201,7 +202,8 @@ static inline u32 ufshci_version(u32 major, u32 minor) CONTROLLER_FATAL_ERROR |\ SYSTEM_BUS_FATAL_ERROR |\ CRYPTO_ENGINE_FATAL_ERROR |\ - UIC_LINK_LOST) + UIC_LINK_LOST |\ + UTP_ERROR) /* HCS - Host Controller Status 30h */ #define DEVICE_PRESENT 0x1 diff --git a/init/Kconfig b/init/Kconfig index fbbc4a970b004..fbb3b41e66ccb 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1030,6 +1030,10 @@ config RT_GROUP_SCHED realtime bandwidth for them. See Documentation/scheduler/sched-rt-group.rst for more information. +config EXT_GROUP_SCHED + bool + depends on SCHED_CLASS_EXT && CGROUP_SCHED + default y endif #CGROUP_SCHED config SCHED_MM_CID @@ -1375,8 +1379,27 @@ config BOOT_CONFIG_EMBED_FILE This bootconfig will be used if there is no initrd or no other bootconfig in the initrd. +config CMDLINE_LOG_WRAP_IDEAL_LEN + int "Length to try to wrap the cmdline when logged at boot" + default 1021 + range 0 1021 + help + At boot time, the kernel command line is logged to the console. + The log message will start with the prefix "Kernel command line: ". + The log message will attempt to be wrapped (split into multiple log + messages) at spaces based on CMDLINE_LOG_WRAP_IDEAL_LEN characters. + If wrapping happens, each log message will start with the prefix and + all but the last message will end with " \". Messages may exceed the + ideal length if a place to wrap isn't found before the specified + number of characters. + + A value of 0 disables wrapping, though be warned that the maximum + length of a log message (1021 characters) may cause the cmdline to + be truncated. + config INITRAMFS_PRESERVE_MTIME bool "Preserve cpio archive mtimes in initramfs" + depends on BLK_DEV_INITRD default y help Each entry in an initramfs cpio archive carries an mtime value. When @@ -2019,4 +2042,6 @@ config ARCH_HAS_SYSCALL_WRAPPER if 64BIT source "init/Kconfig.gki" + endif +source "init/Kconfig.gki-debug" diff --git a/init/Kconfig.gki-debug b/init/Kconfig.gki-debug new file mode 100644 index 0000000000000..3e0ea7989eaf8 --- /dev/null +++ b/init/Kconfig.gki-debug @@ -0,0 +1,9 @@ +config GKI_HIDDEN_RCUTORTURE + bool + select TASKS_RUDE_RCU + +config GKI_HACKS_FOR_DEBUG_CONFIG + bool "GKI Dummy config options for debug configurations" + select TRACE_PREEMPT_TOGGLE + select TRACE_IRQFLAGS + select GKI_HIDDEN_RCUTORTURE diff --git a/init/init_task.c b/init/init_task.c old mode 100644 new mode 100755 index 31ceb0e469f78..e01803ae4e1c8 --- a/init/init_task.c +++ b/init/init_task.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -101,6 +102,9 @@ struct task_struct init_task #endif #ifdef CONFIG_CGROUP_SCHED .sched_task_group = &root_task_group, +#endif +#ifdef CONFIG_SLIM_SCHED + .scx = NULL, #endif .ptraced = LIST_HEAD_INIT(init_task.ptraced), .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), @@ -214,6 +218,7 @@ struct task_struct init_task .android_vendor_data1 = {0, }, .android_oem_data1 = {0, }, #endif + .dmabuf_info = NULL, }; EXPORT_SYMBOL(init_task); diff --git a/init/main.c b/init/main.c index c787e94cc8982..230f0a0bdd92f 100644 --- a/init/main.c +++ b/init/main.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -530,6 +531,12 @@ static int __init unknown_bootoption(char *param, char *val, const char *unused, void *arg) { size_t len = strlen(param); + /* + * Well-known bootloader identifiers: + * 1. LILO/Grub pass "BOOT_IMAGE=..."; + * 2. kexec/kdump (kexec-tools) pass "kexec". + */ + const char *bootloader[] = { "BOOT_IMAGE=", "kexec", NULL }; /* Handle params aliased to sysctls */ if (sysctl_is_alias(param)) @@ -537,6 +544,12 @@ static int __init unknown_bootoption(char *param, char *val, repair_env_string(param, val); + /* Handle bootloader identifier */ + for (int i = 0; bootloader[i]; i++) { + if (strstarts(param, bootloader[i])) + return 0; + } + /* Handle obsolete-style parameters */ if (obsolete_checksetup(param)) return 0; @@ -872,6 +885,101 @@ static void __init print_unknown_bootoptions(void) memblock_free(unknown_options, len); } +#define KERNEL_CMDLINE_PREFIX "Kernel command line: " +#define KERNEL_CMDLINE_PREFIX_LEN (sizeof(KERNEL_CMDLINE_PREFIX) - 1) +#define KERNEL_CMDLINE_CONTINUATION " \\" +#define KERNEL_CMDLINE_CONTINUATION_LEN (sizeof(KERNEL_CMDLINE_CONTINUATION) - 1) + +#define MIN_CMDLINE_LOG_WRAP_IDEAL_LEN (KERNEL_CMDLINE_PREFIX_LEN + \ + KERNEL_CMDLINE_CONTINUATION_LEN) +#define CMDLINE_LOG_WRAP_IDEAL_LEN (CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN > \ + MIN_CMDLINE_LOG_WRAP_IDEAL_LEN ? \ + CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN : \ + MIN_CMDLINE_LOG_WRAP_IDEAL_LEN) + +#define IDEAL_CMDLINE_LEN (CMDLINE_LOG_WRAP_IDEAL_LEN - KERNEL_CMDLINE_PREFIX_LEN) +#define IDEAL_CMDLINE_SPLIT_LEN (IDEAL_CMDLINE_LEN - KERNEL_CMDLINE_CONTINUATION_LEN) + +/** + * print_kernel_cmdline() - Print the kernel cmdline with wrapping. + * @cmdline: The cmdline to print. + * + * Print the kernel command line, trying to wrap based on the Kconfig knob + * CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN. + * + * Wrapping is based on spaces, ignoring quotes. All lines are prefixed + * with "Kernel command line: " and lines that are not the last line have + * a " \" suffix added to them. The prefix and suffix count towards the + * line length for wrapping purposes. The ideal length will be exceeded + * if no appropriate place to wrap is found. + * + * Example output if CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN is 40: + * Kernel command line: loglevel=7 \ + * Kernel command line: init=/sbin/init \ + * Kernel command line: root=PARTUUID=8c3efc1a-768b-6642-8d0c-89eb782f19f0/PARTNROFF=1 \ + * Kernel command line: rootwait ro \ + * Kernel command line: my_quoted_arg="The \ + * Kernel command line: quick brown fox \ + * Kernel command line: jumps over the \ + * Kernel command line: lazy dog." + */ +static void __init print_kernel_cmdline(const char *cmdline) +{ + size_t len; + + /* Config option of 0 or anything longer than the max disables wrapping */ + if (CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN == 0 || + IDEAL_CMDLINE_LEN >= COMMAND_LINE_SIZE - 1) { + pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline); + return; + } + + len = strlen(cmdline); + while (len > IDEAL_CMDLINE_LEN) { + const char *first_space; + const char *prev_cutoff; + const char *cutoff; + int to_print; + size_t used; + + /* Find the last ' ' that wouldn't make the line too long */ + prev_cutoff = NULL; + cutoff = cmdline; + while (true) { + cutoff = strchr(cutoff + 1, ' '); + if (!cutoff || cutoff - cmdline > IDEAL_CMDLINE_SPLIT_LEN) + break; + prev_cutoff = cutoff; + } + if (prev_cutoff) + cutoff = prev_cutoff; + else if (!cutoff) + break; + + /* Find the beginning and end of the string of spaces */ + first_space = cutoff; + while (first_space > cmdline && first_space[-1] == ' ') + first_space--; + to_print = first_space - cmdline; + while (*cutoff == ' ') + cutoff++; + used = cutoff - cmdline; + + /* If the whole string is used, break and do the final printout */ + if (len == used) + break; + + if (to_print) + pr_notice("%s%.*s%s\n", KERNEL_CMDLINE_PREFIX, + to_print, cmdline, KERNEL_CMDLINE_CONTINUATION); + + len -= used; + cmdline += used; + } + if (len) + pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline); +} + asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector void start_kernel(void) { @@ -904,7 +1012,7 @@ void start_kernel(void) smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ boot_cpu_hotplug_init(); - pr_notice("Kernel command line: %s\n", saved_command_line); + print_kernel_cmdline(saved_command_line); /* parameters may set static keys */ jump_label_init(); parse_early_param(); @@ -1062,6 +1170,7 @@ void start_kernel(void) proc_root_init(); nsfs_init(); cpuset_init(); + mem_cgroup_init(); cgroup_init(); taskstats_init_early(); delayacct_init(); diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 976e9500f6518..a26cf840e623d 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -81,11 +81,11 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *f) seq_printf(m, "SqMask:\t0x%x\n", sq_mask); seq_printf(m, "SqHead:\t%u\n", sq_head); seq_printf(m, "SqTail:\t%u\n", sq_tail); - seq_printf(m, "CachedSqHead:\t%u\n", ctx->cached_sq_head); + seq_printf(m, "CachedSqHead:\t%u\n", data_race(ctx->cached_sq_head)); seq_printf(m, "CqMask:\t0x%x\n", cq_mask); seq_printf(m, "CqHead:\t%u\n", cq_head); seq_printf(m, "CqTail:\t%u\n", cq_tail); - seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail); + seq_printf(m, "CachedCqTail:\t%u\n", data_race(ctx->cached_cq_tail)); seq_printf(m, "SQEs:\t%u\n", sq_tail - sq_head); sq_entries = min(sq_tail - sq_head, ctx->sq_entries); for (i = 0; i < sq_entries; i++) { diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 6e86e6188dbee..ff74d41d9e53c 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -62,7 +62,7 @@ void io_free_file_tables(struct io_file_table *table) static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, u32 slot_index) - __must_hold(&req->ctx->uring_lock) + __must_hold(&ctx->uring_lock) { struct io_fixed_file *file_slot; int ret; diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 93f3e4d1df853..1c4ef4e4eb52b 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -1206,8 +1206,10 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) atomic_set(&wq->worker_refs, 1); init_completion(&wq->worker_done); ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node); - if (ret) + if (ret) { + put_task_struct(wq->task); goto err; + } return wq; err: diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 12af8ce7ad35a..a79d5d734b50e 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -121,6 +121,7 @@ #define IO_COMPL_BATCH 32 #define IO_REQ_ALLOC_BATCH 8 +#define IO_LOCAL_TW_DEFAULT_MAX 20 enum { IO_CHECK_CQ_OVERFLOW_BIT, @@ -423,24 +424,6 @@ static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req) return req->link; } -static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) -{ - if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT))) - return NULL; - return __io_prep_linked_timeout(req); -} - -static noinline void __io_arm_ltimeout(struct io_kiocb *req) -{ - io_queue_linked_timeout(__io_prep_linked_timeout(req)); -} - -static inline void io_arm_ltimeout(struct io_kiocb *req) -{ - if (unlikely(req->flags & REQ_F_ARM_LTIMEOUT)) - __io_arm_ltimeout(req); -} - static void io_prep_async_work(struct io_kiocb *req) { const struct io_issue_def *def = &io_issue_defs[req->opcode]; @@ -494,7 +477,6 @@ static void io_prep_async_link(struct io_kiocb *req) static void io_queue_iowq(struct io_kiocb *req) { - struct io_kiocb *link = io_prep_linked_timeout(req); struct io_uring_task *tctx = req->task->io_uring; BUG_ON(!tctx); @@ -519,8 +501,6 @@ static void io_queue_iowq(struct io_kiocb *req) trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work)); io_wq_enqueue(tctx->io_wq, &req->work); - if (link) - io_queue_linked_timeout(link); } static __cold void io_queue_deferred(struct io_ring_ctx *ctx) @@ -723,6 +703,7 @@ static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx) * to care for a non-real case. */ if (need_resched()) { + ctx->cqe_sentinel = ctx->cqe_cached; io_cq_unlock_post(ctx); mutex_unlock(&ctx->uring_lock); cond_resched(); @@ -1387,12 +1368,20 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) node = node->next; io_req_normal_work_add(req); } + node = llist_del_all(&ctx->retry_llist); + while (node) { + struct io_kiocb *req = container_of(node, struct io_kiocb, + io_task_work.node); + + node = node->next; + io_req_normal_work_add(req); + } } static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, int min_events) { - if (llist_empty(&ctx->work_llist)) + if (!io_local_work_pending(ctx)) return false; if (events < min_events) return true; @@ -1401,8 +1390,29 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, return false; } +static int __io_run_local_work_loop(struct llist_node **node, + struct io_tw_state *ts, + int events) +{ + int ret = 0; + + while (*node) { + struct llist_node *next = (*node)->next; + struct io_kiocb *req = container_of(*node, struct io_kiocb, + io_task_work.node); + INDIRECT_CALL_2(req->io_task_work.func, + io_poll_task_func, io_req_rw_complete, + req, ts); + *node = next; + if (++ret >= events) + break; + } + + return ret; +} + static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts, - int min_events) + int min_events, int max_events) { struct llist_node *node; unsigned int loops = 0; @@ -1413,26 +1423,23 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts, if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); again: + min_events -= ret; + ret = __io_run_local_work_loop(&ctx->retry_llist.first, ts, max_events); + if (ctx->retry_llist.first) + goto retry_done; + /* * llists are in reverse order, flip it back the right way before * running the pending items. */ - node = llist_reverse_order(io_llist_xchg(&ctx->work_llist, NULL)); - while (node) { - struct llist_node *next = node->next; - struct io_kiocb *req = container_of(node, struct io_kiocb, - io_task_work.node); - prefetch(container_of(next, struct io_kiocb, io_task_work.node)); - INDIRECT_CALL_2(req->io_task_work.func, - io_poll_task_func, io_req_rw_complete, - req, ts); - ret++; - node = next; - } + node = llist_reverse_order(llist_del_all(&ctx->work_llist)); + ret += __io_run_local_work_loop(&node, ts, max_events - ret); + ctx->retry_llist.first = node; loops++; if (io_run_local_work_continue(ctx, ret, min_events)) goto again; +retry_done: if (ts->locked) { io_submit_flush_completions(ctx); if (io_run_local_work_continue(ctx, ret, min_events)) @@ -1449,23 +1456,24 @@ static inline int io_run_local_work_locked(struct io_ring_ctx *ctx, struct io_tw_state ts = { .locked = true, }; int ret; - if (llist_empty(&ctx->work_llist)) + if (!io_local_work_pending(ctx)) return 0; - ret = __io_run_local_work(ctx, &ts, min_events); + ret = __io_run_local_work(ctx, &ts, min_events, max(IO_LOCAL_TW_DEFAULT_MAX, min_events)); /* shouldn't happen! */ if (WARN_ON_ONCE(!ts.locked)) mutex_lock(&ctx->uring_lock); return ret; } -static int io_run_local_work(struct io_ring_ctx *ctx, int min_events) +static int io_run_local_work(struct io_ring_ctx *ctx, int min_events, + int max_events) { struct io_tw_state ts = {}; int ret; ts.locked = mutex_trylock(&ctx->uring_lock); - ret = __io_run_local_work(ctx, &ts, min_events); + ret = __io_run_local_work(ctx, &ts, min_events, max_events); if (ts.locked) mutex_unlock(&ctx->uring_lock); @@ -1480,9 +1488,10 @@ static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts) void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts) { - io_tw_lock(req->ctx, ts); - /* req->task == current here, checking PF_EXITING is safe */ - if (unlikely(req->task->flags & PF_EXITING)) + struct io_ring_ctx *ctx = req->ctx; + + io_tw_lock(ctx, ts); + if (unlikely(io_should_terminate_tw(ctx))) io_req_defer_failed(req, -EFAULT); else if (req->flags & REQ_F_FORCE_ASYNC) io_queue_iowq(req); @@ -1864,7 +1873,7 @@ static __cold void io_drain_req(struct io_kiocb *req) spin_unlock(&ctx->completion_lock); io_prep_async_link(req); - de = kmalloc(sizeof(*de), GFP_KERNEL); + de = kmalloc(sizeof(*de), GFP_KERNEL_ACCOUNT); if (!de) { ret = -ENOMEM; io_req_defer_failed(req, ret); @@ -1899,17 +1908,22 @@ static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def, return !!req->file; } -static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) +#define REQ_ISSUE_SLOW_FLAGS (REQ_F_CREDS | REQ_F_ARM_LTIMEOUT) + +static inline int __io_issue_sqe(struct io_kiocb *req, + unsigned int issue_flags, + const struct io_issue_def *def) { - const struct io_issue_def *def = &io_issue_defs[req->opcode]; const struct cred *creds = NULL; + struct io_kiocb *link = NULL; int ret; - if (unlikely(!io_assign_file(req, def, issue_flags))) - return -EBADF; - - if (unlikely((req->flags & REQ_F_CREDS) && req->creds != current_cred())) - creds = override_creds(req->creds); + if (unlikely(req->flags & REQ_ISSUE_SLOW_FLAGS)) { + if ((req->flags & REQ_F_CREDS) && req->creds != current_cred()) + creds = override_creds(req->creds); + if (req->flags & REQ_F_ARM_LTIMEOUT) + link = __io_prep_linked_timeout(req); + } if (!def->audit_skip) audit_uring_entry(req->opcode); @@ -1919,8 +1933,25 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) if (!def->audit_skip) audit_uring_exit(!ret, ret); - if (creds) - revert_creds(creds); + if (unlikely(creds || link)) { + if (creds) + revert_creds(creds); + if (link) + io_queue_linked_timeout(link); + } + + return ret; +} + +static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) +{ + const struct io_issue_def *def = &io_issue_defs[req->opcode]; + int ret; + + if (unlikely(!io_assign_file(req, def, issue_flags))) + return -EBADF; + + ret = __io_issue_sqe(req, issue_flags, def); if (ret == IOU_OK) { if (issue_flags & IO_URING_F_COMPLETE_DEFER) @@ -1943,9 +1974,24 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts) { + const unsigned int issue_flags = IO_URING_F_NONBLOCK | + IO_URING_F_MULTISHOT | + IO_URING_F_COMPLETE_DEFER; + int ret; + io_tw_lock(req->ctx, ts); - return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT| - IO_URING_F_COMPLETE_DEFER); + + WARN_ON_ONCE(!req->file); + if (WARN_ON_ONCE(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EFAULT; + + ret = __io_issue_sqe(req, issue_flags, &io_issue_defs[req->opcode]); + + WARN_ON_ONCE(ret == IOU_OK); + + if (ret == IOU_ISSUE_SKIP_COMPLETE) + ret = 0; + return ret; } struct io_wq_work *io_wq_free_work(struct io_wq_work *work) @@ -1975,8 +2021,6 @@ void io_wq_submit_work(struct io_wq_work *work) else req_ref_get(req); - io_arm_ltimeout(req); - /* either cancelled or io-wq is dying, so don't touch tctx->iowq */ if (work->flags & IO_WQ_WORK_CANCEL) { fail: @@ -1992,7 +2036,7 @@ void io_wq_submit_work(struct io_wq_work *work) if (req->flags & REQ_F_FORCE_ASYNC) { bool opcode_poll = def->pollin || def->pollout; - if (opcode_poll && file_can_poll(req->file)) { + if (opcode_poll && io_file_can_poll(req)) { needs_poll = true; issue_flags |= IO_URING_F_NONBLOCK; } @@ -2072,15 +2116,11 @@ struct file *io_file_get_normal(struct io_kiocb *req, int fd) static void io_queue_async(struct io_kiocb *req, int ret) __must_hold(&req->ctx->uring_lock) { - struct io_kiocb *linked_timeout; - if (ret != -EAGAIN || (req->flags & REQ_F_NOWAIT)) { io_req_defer_failed(req, ret); return; } - linked_timeout = io_prep_linked_timeout(req); - switch (io_arm_poll_handler(req, 0)) { case IO_APOLL_READY: io_kbuf_recycle(req, 0); @@ -2093,9 +2133,6 @@ static void io_queue_async(struct io_kiocb *req, int ret) case IO_APOLL_OK: break; } - - if (linked_timeout) - io_queue_linked_timeout(linked_timeout); } static inline void io_queue_sqe(struct io_kiocb *req) @@ -2109,9 +2146,7 @@ static inline void io_queue_sqe(struct io_kiocb *req) * We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts */ - if (likely(!ret)) - io_arm_ltimeout(req); - else + if (unlikely(ret)) io_queue_async(req, ret); } @@ -2519,7 +2554,7 @@ struct io_wait_queue { static inline bool io_has_work(struct io_ring_ctx *ctx) { return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || - !llist_empty(&ctx->work_llist); + io_local_work_pending(ctx); } static inline bool io_should_wake(struct io_wait_queue *iowq) @@ -2551,9 +2586,9 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, int io_run_task_work_sig(struct io_ring_ctx *ctx) { - if (!llist_empty(&ctx->work_llist)) { + if (io_local_work_pending(ctx)) { __set_current_state(TASK_RUNNING); - if (io_run_local_work(ctx, INT_MAX) > 0) + if (io_run_local_work(ctx, INT_MAX, IO_LOCAL_TW_DEFAULT_MAX) > 0) return 0; } if (io_run_task_work() > 0) @@ -2574,13 +2609,14 @@ static bool current_pending_io(void) /* when returns >0, the caller should retry */ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, - struct io_wait_queue *iowq) + struct io_wait_queue *iowq, + bool iowait) { int ret; if (unlikely(READ_ONCE(ctx->check_cq))) return 1; - if (unlikely(!llist_empty(&ctx->work_llist))) + if (unlikely(io_local_work_pending(ctx))) return 1; if (unlikely(task_work_pending(current))) return 1; @@ -2594,7 +2630,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, * can take into account that the task is waiting for IO - turns out * to be important for low QD IO. */ - if (current_pending_io()) + if (iowait && current_pending_io()) current->in_iowait = 1; ret = 0; if (iowq->timeout == KTIME_MAX) @@ -2611,7 +2647,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, */ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, const sigset_t __user *sig, size_t sigsz, - struct __kernel_timespec __user *uts) + struct __kernel_timespec __user *uts, bool iowait) { struct io_wait_queue iowq; struct io_rings *rings = ctx->rings; @@ -2619,8 +2655,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, if (!io_allowed_run_tw(ctx)) return -EEXIST; - if (!llist_empty(&ctx->work_llist)) - io_run_local_work(ctx, min_events); + if (io_local_work_pending(ctx)) + io_run_local_work(ctx, min_events, + max(IO_LOCAL_TW_DEFAULT_MAX, min_events)); io_run_task_work(); io_cqring_overflow_flush(ctx); /* if user messes with these they will just get an early return */ @@ -2669,7 +2706,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, TASK_INTERRUPTIBLE); } - ret = io_cqring_wait_schedule(ctx, &iowq); + ret = io_cqring_wait_schedule(ctx, &iowq, iowait); __set_current_state(TASK_RUNNING); atomic_set(&ctx->cq_wait_nr, 0); @@ -2678,8 +2715,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, * If we got woken because of task_work being processed, run it * now rather than let the caller do another wait loop. */ - if (!llist_empty(&ctx->work_llist)) - io_run_local_work(ctx, nr_wait); + if (io_local_work_pending(ctx)) + io_run_local_work(ctx, nr_wait, nr_wait); io_run_task_work(); /* @@ -3467,7 +3504,7 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && io_allowed_defer_tw_run(ctx)) - ret |= io_run_local_work(ctx, INT_MAX) > 0; + ret |= io_run_local_work(ctx, INT_MAX, INT_MAX) > 0; ret |= io_cancel_defer_files(ctx, task, cancel_all); mutex_lock(&ctx->uring_lock); ret |= io_poll_remove_all(ctx, task, cancel_all); @@ -3542,7 +3579,7 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) io_run_task_work(); io_uring_drop_tctx_refs(current); xa_for_each(&tctx->xa, index, node) { - if (!llist_empty(&node->ctx->work_llist)) { + if (io_local_work_pending(node->ctx)) { WARN_ON_ONCE(node->ctx->submitter_task && node->ctx->submitter_task != current); goto end_wait; @@ -3736,10 +3773,12 @@ static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t a static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz, struct __kernel_timespec __user **ts, - const sigset_t __user **sig) + const sigset_t __user **sig, bool *iowait) { struct io_uring_getevents_arg arg; + *iowait = !(flags & IORING_ENTER_NO_IOWAIT); + /* * If EXT_ARG isn't set, then we have no timespec and the argp pointer * is just a pointer to the sigset_t. @@ -3776,7 +3815,8 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG | - IORING_ENTER_REGISTERED_RING))) + IORING_ENTER_REGISTERED_RING | + IORING_ENTER_NO_IOWAIT))) return -EINVAL; /* @@ -3871,13 +3911,15 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, } else { const sigset_t __user *sig; struct __kernel_timespec __user *ts; + bool iowait; - ret2 = io_get_ext_arg(flags, argp, &argsz, &ts, &sig); + ret2 = io_get_ext_arg(flags, argp, &argsz, &ts, &sig, + &iowait); if (likely(!ret2)) { min_complete = min(min_complete, ctx->cq_entries); ret2 = io_cqring_wait(ctx, min_complete, sig, - argsz, ts); + argsz, ts, iowait); } } @@ -4159,7 +4201,8 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED | IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS | IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP | - IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING; + IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING | + IORING_FEAT_NO_IOWAIT; if (copy_to_user(params, p, sizeof(*p))) { ret = -EFAULT; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 0ffec66deee7f..8900093a09acb 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include "io-wq.h" @@ -312,9 +313,14 @@ static inline int io_run_task_work(void) return 0; } +static inline bool io_local_work_pending(struct io_ring_ctx *ctx) +{ + return !llist_empty(&ctx->work_llist) || !llist_empty(&ctx->retry_llist); +} + static inline bool io_task_work_pending(struct io_ring_ctx *ctx) { - return task_work_pending(current) || !llist_empty(&ctx->work_llist); + return task_work_pending(current) || io_local_work_pending(ctx); } static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts) @@ -393,6 +399,19 @@ static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) ctx->submitter_task == current); } +/* + * Terminate the request if either of these conditions are true: + * + * 1) It's being executed by the original task, but that task is marked + * with PF_EXITING as it's exiting. + * 2) PF_KTHREAD is set, in which case the invoker of the task_work is + * our fallback task_work. + */ +static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx) +{ + return (current->flags & (PF_KTHREAD | PF_EXITING)) || percpu_ref_is_dying(&ctx->refs); +} + static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) { io_req_set_res(req, res, 0); @@ -410,4 +429,15 @@ static inline size_t uring_sqe_size(struct io_ring_ctx *ctx) return 2 * sizeof(struct io_uring_sqe); return sizeof(struct io_uring_sqe); } + +static inline bool io_file_can_poll(struct io_kiocb *req) +{ + if (req->flags & REQ_F_CAN_POLL) + return true; + if (file_can_poll(req->file)) { + req->flags |= REQ_F_CAN_POLL; + return true; + } + return false; +} #endif diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 8c6611fe4f463..b2c381634393d 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -148,8 +148,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, req->buf_list = bl; req->buf_index = buf->bid; - if (issue_flags & IO_URING_F_UNLOCKED || - (req->file && !file_can_poll(req->file))) { + if (issue_flags & IO_URING_F_UNLOCKED || !io_file_can_poll(req)) { /* * If we came in unlocked, we have no choice but to consume the * buffer here, otherwise nothing ensures that the buffer won't @@ -561,7 +560,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) io_destroy_bl(ctx, bl); } - free_bl = bl = kzalloc(sizeof(*bl), GFP_KERNEL); + free_bl = bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT); if (!bl) return -ENOMEM; diff --git a/io_uring/net.c b/io_uring/net.c index 4948a67bbac48..e7f8a79e049c9 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -351,6 +351,13 @@ static int io_setup_async_addr(struct io_kiocb *req, return -EAGAIN; } +static void io_net_kbuf_recyle(struct io_kiocb *req) +{ + req->flags |= REQ_F_PARTIAL_IO; + if (req->flags & REQ_F_BUFFER_RING) + io_kbuf_recycle_ring(req); +} + int io_sendmsg_prep_async(struct io_kiocb *req) { int ret; @@ -442,7 +449,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_controllen = 0; kmsg->msg.msg_control = NULL; sr->done_io += ret; - req->flags |= REQ_F_PARTIAL_IO; + io_net_kbuf_recyle(req); return io_setup_async_msg(req, kmsg, issue_flags); } if (ret == -ERESTARTSYS) @@ -521,7 +528,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) sr->len -= ret; sr->buf += ret; sr->done_io += ret; - req->flags |= REQ_F_PARTIAL_IO; + io_net_kbuf_recyle(req); return io_setup_async_addr(req, &__address, issue_flags); } if (ret == -ERESTARTSYS) @@ -891,7 +898,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) } if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; - req->flags |= REQ_F_PARTIAL_IO; + io_net_kbuf_recyle(req); return io_setup_async_msg(req, kmsg, issue_flags); } if (ret == -ERESTARTSYS) @@ -991,7 +998,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) sr->len -= ret; sr->buf += ret; sr->done_io += ret; - req->flags |= REQ_F_PARTIAL_IO; + io_net_kbuf_recyle(req); return -EAGAIN; } if (ret == -ERESTARTSYS) @@ -1235,7 +1242,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) zc->len -= ret; zc->buf += ret; zc->done_io += ret; - req->flags |= REQ_F_PARTIAL_IO; + io_net_kbuf_recyle(req); return io_setup_async_addr(req, &__address, issue_flags); } if (ret == -ERESTARTSYS) @@ -1306,7 +1313,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; - req->flags |= REQ_F_PARTIAL_IO; + io_net_kbuf_recyle(req); return io_setup_async_msg(req, kmsg, issue_flags); } if (ret == -ERESTARTSYS) @@ -1537,9 +1544,11 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) io = &__io; } - if (unlikely(req->flags & REQ_F_FAIL)) { - ret = -ECONNRESET; - goto out; + if (connect->in_progress) { + struct poll_table_struct pt = { ._key = EPOLLERR }; + + if (vfs_poll(req->file, &pt) & EPOLLERR) + goto get_sock_err; } file_flags = force_nonblock ? O_NONBLOCK : 0; @@ -1571,8 +1580,10 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) * which means the previous result is good. For both of these, * grab the sock_error() and use that for the completion. */ - if (ret == -EBADFD || ret == -EISCONN) + if (ret == -EBADFD || ret == -EISCONN) { +get_sock_err: ret = sock_error(sock_from_file(req->file)->sk); + } } if (ret == -ERESTARTSYS) ret = -EINTR; diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 3b9c6489b8b6d..2d0a7db940fdb 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -202,6 +202,7 @@ const struct io_issue_def io_issue_defs[] = { }, [IORING_OP_FALLOCATE] = { .needs_file = 1, + .hash_reg_file = 1, .prep = io_fallocate_prep, .issue = io_fallocate, }, diff --git a/io_uring/poll.c b/io_uring/poll.c index cf8e86bc96deb..b6c8acd8625e3 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -258,8 +258,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts) { int v; - /* req->task == current here, checking PF_EXITING is safe */ - if (unlikely(req->task->flags & PF_EXITING)) + if (unlikely(io_should_terminate_tw(req->ctx))) return -ECANCELED; do { @@ -308,8 +307,6 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts) return IOU_POLL_REISSUE; } } - if (unlikely(req->cqe.res & EPOLLERR)) - req_set_fail(req); if (req->apoll_events & EPOLLONESHOT) return IOU_POLL_DONE; @@ -717,7 +714,7 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) if (!def->pollin && !def->pollout) return IO_APOLL_ABORTED; - if (!file_can_poll(req->file)) + if (!io_file_can_poll(req)) return IO_APOLL_ABORTED; if (!(req->flags & REQ_F_APOLL_MULTISHOT)) mask |= EPOLLONESHOT; diff --git a/io_uring/rw.c b/io_uring/rw.c index 75b001febb4d2..4ff3442ac2eee 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -28,9 +28,19 @@ struct io_rw { rwf_t flags; }; -static inline bool io_file_supports_nowait(struct io_kiocb *req) +static bool io_file_supports_nowait(struct io_kiocb *req, __poll_t mask) { - return req->flags & REQ_F_SUPPORT_NOWAIT; + /* If FMODE_NOWAIT is set for a file, we're golden */ + if (req->flags & REQ_F_SUPPORT_NOWAIT) + return true; + /* No FMODE_NOWAIT, if we can poll, check the status */ + if (io_file_can_poll(req)) { + struct poll_table_struct pt = { ._key = mask }; + + return vfs_poll(req->file, &pt) & mask; + } + /* No FMODE_NOWAIT support, and file isn't pollable. Tough luck. */ + return false; } #ifdef CONFIG_COMPAT @@ -629,7 +639,7 @@ static bool io_rw_should_retry(struct io_kiocb *req) * just use poll if we can, and don't attempt if the fs doesn't * support callback based unlocks */ - if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) + if (io_file_can_poll(req) || !(req->file->f_mode & FMODE_BUF_RASYNC)) return false; wait->wait.func = io_async_buf_func; @@ -685,8 +695,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) * supports async. Otherwise it's impossible to use O_NONBLOCK files * reliably. If not, or it IOCB_NOWAIT is set, don't retry. */ - if ((kiocb->ki_flags & IOCB_NOWAIT) || - ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) + if (kiocb->ki_flags & IOCB_NOWAIT || + ((file->f_flags & O_NONBLOCK && !(req->flags & REQ_F_SUPPORT_NOWAIT)))) req->flags |= REQ_F_NOWAIT; if (ctx->flags & IORING_SETUP_IOPOLL) { @@ -752,7 +762,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags) if (force_nonblock) { /* If the file doesn't support async, just async punt */ - if (unlikely(!io_file_supports_nowait(req))) { + if (unlikely(!io_file_supports_nowait(req, EPOLLIN))) { ret = io_setup_async_rw(req, iovec, s, true); return ret ?: -EAGAIN; } @@ -783,7 +793,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags) if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { req->flags &= ~REQ_F_REISSUE; /* if we can poll, just do that */ - if (req->opcode == IORING_OP_READ && file_can_poll(req->file)) + if (req->opcode == IORING_OP_READ && io_file_can_poll(req)) return -EAGAIN; /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) @@ -927,7 +937,7 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags) if (force_nonblock) { /* If the file doesn't support async, just async punt */ - if (unlikely(!io_file_supports_nowait(req))) + if (unlikely(!io_file_supports_nowait(req, EPOLLOUT))) goto copy_iov; /* File path supports NOWAIT for non-direct_IO only for block devices. */ diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 277e22d55c617..be2a0f6c209b2 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -307,7 +307,7 @@ static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *t int ret = -ENOENT; if (prev) { - if (!(req->task->flags & PF_EXITING)) { + if (!io_should_terminate_tw(req->ctx)) { struct io_cancel_data cd = { .ctx = req->ctx, .data = prev->cqe.user_data, diff --git a/ipc/shm.c b/ipc/shm.c index 576a543b7cfff..c4845689b7f68 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -430,8 +430,11 @@ static int shm_try_destroy_orphaned(int id, void *p, void *data) void shm_destroy_orphaned(struct ipc_namespace *ns) { down_write(&shm_ids(ns).rwsem); - if (shm_ids(ns).in_use) + if (shm_ids(ns).in_use) { + rcu_read_lock(); idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns); + rcu_read_unlock(); + } up_write(&shm_ids(ns).rwsem); } diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt old mode 100644 new mode 100755 index c2f1fd95a8214..cf22ef6107b81 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -133,4 +133,29 @@ config SCHED_CORE which is the likely usage by Linux distributions, there should be no measurable impact on performance. - +config SLIM_SCHED + bool "slim sched" + default n +config SCHED_CLASS_EXT + bool "Extensible Scheduling Class" + depends on BPF_SYSCALL && BPF_JIT + help + This option enables a new scheduler class sched_ext (SCX), which + allows scheduling policies to be implemented as BPF programs to + achieve the following: + + - Ease of experimentation and exploration: Enabling rapid + iteration of new scheduling policies. + - Customization: Building application-specific schedulers which + implement policies that are not applicable to general-purpose + schedulers. + - Rapid scheduler deployments: Non-disruptive swap outs of + scheduling policies in production environments. + + sched_ext leverages BPF’s struct_ops feature to define a structure + which exports function callbacks and flags to BPF programs that + wish to implement scheduling policies. The struct_ops structure + exported by sched_ext is struct sched_ext_ops, and is conceptually + similar to struct sched_class. + + See Documentation/scheduler/sched-ext.rst for more details. diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index d60bc65b58753..20f1f00f6e3ad 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -44,6 +44,11 @@ endif obj-$(CONFIG_BPF_PRELOAD) += preload/ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o + +ifeq ($(CONFIG_DMA_SHARED_BUFFER),y) +obj-$(CONFIG_BPF_SYSCALL) += dmabuf_iter.o +endif + $(obj)/relo_core.o: $(srctree)/tools/lib/bpf/relo_core.c FORCE $(call if_changed_rule,cc_o_c) ifeq ($(CONFIG_FUSE_BPF),y) diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c index 3dabdd137d102..2d6e1c98d8adc 100644 --- a/kernel/bpf/bpf_lru_list.c +++ b/kernel/bpf/bpf_lru_list.c @@ -337,12 +337,12 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, list) { __bpf_lru_node_move_to_free(l, node, local_free_list(loc_l), BPF_LRU_LOCAL_LIST_T_FREE); - if (++nfree == LOCAL_FREE_TARGET) + if (++nfree == lru->target_free) break; } - if (nfree < LOCAL_FREE_TARGET) - __bpf_lru_list_shrink(lru, l, LOCAL_FREE_TARGET - nfree, + if (nfree < lru->target_free) + __bpf_lru_list_shrink(lru, l, lru->target_free - nfree, local_free_list(loc_l), BPF_LRU_LOCAL_LIST_T_FREE); @@ -577,6 +577,9 @@ static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf, list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); buf += elem_size; } + + lru->target_free = clamp((nr_elems / num_possible_cpus()) / 2, + 1, LOCAL_FREE_TARGET); } static void bpf_percpu_lru_populate(struct bpf_lru *lru, void *buf, diff --git a/kernel/bpf/bpf_lru_list.h b/kernel/bpf/bpf_lru_list.h index cbd8d3720c2bb..fe2661a58ea94 100644 --- a/kernel/bpf/bpf_lru_list.h +++ b/kernel/bpf/bpf_lru_list.h @@ -58,6 +58,7 @@ struct bpf_lru { del_from_htab_func del_from_htab; void *del_arg; unsigned int hash_offset; + unsigned int target_free; unsigned int nr_scans; bool percpu; }; diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index fdc3e8705a3cb..54ffca20b9dd2 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -352,18 +352,17 @@ const struct bpf_link_ops bpf_struct_ops_link_lops = { int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, - void *image, void *image_end) + void *stub_func, void *image, void *image_end) { - u32 flags; + u32 flags = BPF_TRAMP_F_INDIRECT; tlinks[BPF_TRAMP_FENTRY].links[0] = link; tlinks[BPF_TRAMP_FENTRY].nr_links = 1; - /* BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops, - * and it must be used alone. - */ - flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0; + + if (model->ret_size > 0) + flags |= BPF_TRAMP_F_RET_FENTRY_RET; return arch_prepare_bpf_trampoline(NULL, image, image_end, - model, flags, tlinks, NULL); + model, flags, tlinks, stub_func); } static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, @@ -497,11 +496,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[i], + *(void **)(st_ops->cfi_stubs + moff), image, image_end); if (err < 0) goto reset_unlock; - *(void **)(kdata + moff) = image; + *(void **)(kdata + moff) = image + cfi_get_offset(); image += err; /* put prog_id to udata */ diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h index 5678a9ddf8178..3618769d853d0 100644 --- a/kernel/bpf/bpf_struct_ops_types.h +++ b/kernel/bpf/bpf_struct_ops_types.h @@ -9,4 +9,8 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) #include BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) #endif +#ifdef CONFIG_SCHED_CLASS_EXT +#include +BPF_STRUCT_OPS_TYPE(sched_ext_ops) +#endif #endif diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index cf2eb0895d403..684fb450ad086 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -369,7 +369,7 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl) /* count number of elements in the list. * it's slow but the list cannot be long */ -static u32 prog_list_length(struct hlist_head *head) +static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt) { struct bpf_prog_list *pl; u32 cnt = 0; @@ -377,6 +377,8 @@ static u32 prog_list_length(struct hlist_head *head) hlist_for_each_entry(pl, head, node) { if (!prog_list_prog(pl)) continue; + if (preorder_cnt && (pl->flags & BPF_F_PREORDER)) + (*preorder_cnt)++; cnt++; } return cnt; @@ -400,7 +402,7 @@ static bool hierarchy_allows_attach(struct cgroup *cgrp, if (flags & BPF_F_ALLOW_MULTI) return true; - cnt = prog_list_length(&p->bpf.progs[atype]); + cnt = prog_list_length(&p->bpf.progs[atype], NULL); WARN_ON_ONCE(cnt > 1); if (cnt == 1) return !!(flags & BPF_F_ALLOW_OVERRIDE); @@ -423,12 +425,12 @@ static int compute_effective_progs(struct cgroup *cgrp, struct bpf_prog_array *progs; struct bpf_prog_list *pl; struct cgroup *p = cgrp; - int cnt = 0; + int i, j, cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart; /* count number of effective programs by walking parents */ do { if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) - cnt += prog_list_length(&p->bpf.progs[atype]); + cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt); p = cgroup_parent(p); } while (p); @@ -439,20 +441,34 @@ static int compute_effective_progs(struct cgroup *cgrp, /* populate the array with effective progs */ cnt = 0; p = cgrp; + fstart = preorder_cnt; + bstart = preorder_cnt - 1; do { if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) continue; + init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { if (!prog_list_prog(pl)) continue; - item = &progs->items[cnt]; + if (pl->flags & BPF_F_PREORDER) { + item = &progs->items[bstart]; + bstart--; + } else { + item = &progs->items[fstart]; + fstart++; + } item->prog = prog_list_prog(pl); bpf_cgroup_storages_assign(item->cgroup_storage, pl->storage); cnt++; } + + /* reverse pre-ordering progs at this cgroup level */ + for (i = bstart + 1, j = init_bstart; i < j; i++, j--) + swap(progs->items[i], progs->items[j]); + } while ((p = cgroup_parent(p))); *array = progs; @@ -663,7 +679,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, */ return -EPERM; - if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS) + if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS) return -E2BIG; pl = find_attach_entry(progs, prog, link, replace_prog, @@ -698,6 +714,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, pl->prog = prog; pl->link = link; + pl->flags = flags; bpf_cgroup_storages_assign(pl->storage, storage); cgrp->bpf.flags[atype] = saved_flags; @@ -1073,7 +1090,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, lockdep_is_held(&cgroup_mutex)); total_cnt += bpf_prog_array_length(effective); } else { - total_cnt += prog_list_length(&cgrp->bpf.progs[atype]); + total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); } } @@ -1105,7 +1122,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, u32 id; progs = &cgrp->bpf.progs[atype]; - cnt = min_t(int, prog_list_length(progs), total_cnt); + cnt = min_t(int, prog_list_length(progs, NULL), total_cnt); i = 0; hlist_for_each_entry(pl, progs, node) { prog = prog_list_prog(pl); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 9e567850bf892..0a1c67534415d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -121,6 +121,9 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag #endif INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); +#ifdef CONFIG_FINEIBT + INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode); +#endif mutex_init(&fp->aux->used_maps_mutex); mutex_init(&fp->aux->dst_mutex); @@ -691,6 +694,23 @@ void bpf_prog_kallsyms_add(struct bpf_prog *fp) fp->aux->ksym.prog = true; bpf_ksym_add(&fp->aux->ksym); + +#ifdef CONFIG_FINEIBT + /* + * When FineIBT, code in the __cfi_foo() symbols can get executed + * and hence unwinder needs help. + */ + if (cfi_mode != CFI_FINEIBT) + return; + + snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN, + "__cfi_%s", fp->aux->ksym.name); + + fp->aux->ksym_prefix.start = (unsigned long) fp->bpf_func - 16; + fp->aux->ksym_prefix.end = (unsigned long) fp->bpf_func; + + bpf_ksym_add(&fp->aux->ksym_prefix); +#endif } void bpf_prog_kallsyms_del(struct bpf_prog *fp) @@ -699,6 +719,11 @@ void bpf_prog_kallsyms_del(struct bpf_prog *fp) return; bpf_ksym_del(&fp->aux->ksym); +#ifdef CONFIG_FINEIBT + if (cfi_mode != CFI_FINEIBT) + return; + bpf_ksym_del(&fp->aux->ksym_prefix); +#endif } static struct bpf_ksym *bpf_ksym_find(unsigned long addr) @@ -2258,8 +2283,8 @@ static unsigned int __bpf_prog_ret0_warn(const void *ctx, } #endif -bool bpf_prog_map_compatible(struct bpf_map *map, - const struct bpf_prog *fp) +static bool __bpf_prog_map_compatible(struct bpf_map *map, + const struct bpf_prog *fp) { enum bpf_prog_type prog_type = resolve_prog_type(fp); bool ret; @@ -2267,14 +2292,6 @@ bool bpf_prog_map_compatible(struct bpf_map *map, if (fp->kprobe_override) return false; - /* XDP programs inserted into maps are not guaranteed to run on - * a particular netdev (and can run outside driver context entirely - * in the case of devmap and cpumap). Until device checks - * are implemented, prohibit adding dev-bound programs to program maps. - */ - if (bpf_prog_is_dev_bound(fp->aux)) - return false; - spin_lock(&map->owner.lock); if (!map->owner.type) { /* There's no owner yet where we could check for @@ -2294,6 +2311,19 @@ bool bpf_prog_map_compatible(struct bpf_map *map, return ret; } +bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp) +{ + /* XDP programs inserted into maps are not guaranteed to run on + * a particular netdev (and can run outside driver context entirely + * in the case of devmap and cpumap). Until device checks + * are implemented, prohibit adding dev-bound programs to program maps. + */ + if (bpf_prog_is_dev_bound(fp->aux)) + return false; + + return __bpf_prog_map_compatible(map, fp); +} + static int bpf_check_tail_call(const struct bpf_prog *fp) { struct bpf_prog_aux *aux = fp->aux; @@ -2306,7 +2336,7 @@ static int bpf_check_tail_call(const struct bpf_prog *fp) if (!map_type_contains_progs(map)) continue; - if (!bpf_prog_map_compatible(map, fp)) { + if (!__bpf_prog_map_compatible(map, fp)) { ret = -EINVAL; goto out; } @@ -2344,7 +2374,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ - bool jit_needed = false; + bool jit_needed = fp->jit_requested; if (fp->bpf_func) goto finalize; diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index 6983af8e093c4..09cb2a71e8507 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -98,6 +98,12 @@ __bpf_kfunc void bpf_cpumask_release(struct bpf_cpumask *cpumask) migrate_enable(); } +__bpf_kfunc void bpf_cpumask_release_dtor(void *cpumask) +{ + bpf_cpumask_release(cpumask); +} +CFI_NOSEAL(bpf_cpumask_release_dtor); + /** * bpf_cpumask_first() - Get the index of the first nonzero bit in the cpumask. * @cpumask: The cpumask being queried. @@ -443,7 +449,7 @@ static const struct btf_kfunc_id_set cpumask_kfunc_set = { BTF_ID_LIST(cpumask_dtor_ids) BTF_ID(struct, bpf_cpumask) -BTF_ID(func, bpf_cpumask_release) +BTF_ID(func, bpf_cpumask_release_dtor) static int __init cpumask_kfunc_init(void) { diff --git a/kernel/bpf/dmabuf_iter.c b/kernel/bpf/dmabuf_iter.c new file mode 100644 index 0000000000000..491bab0ad4cfe --- /dev/null +++ b/kernel/bpf/dmabuf_iter.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Google LLC */ +#include +#include +#include +#include +#include + +struct dmabuf_iter_priv { + /* + * If this pointer is non-NULL, the buffer's refcount is elevated to + * prevent destruction between stop/start. If reading is not resumed and + * start is never called again, then dmabuf_iter_seq_fini drops the + * reference when the iterator is released. + */ + struct dma_buf *dmabuf; +}; + +static void *dmabuf_iter_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct dmabuf_iter_priv *p = seq->private; + + if (*pos) { + struct dma_buf *dmabuf = p->dmabuf; + + if (!dmabuf) + return NULL; + + /* + * Always resume from where we stopped, regardless of the value + * of pos. + */ + p->dmabuf = NULL; + return dmabuf; + } + + return dma_buf_iter_begin(); +} + +static void *dmabuf_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct dma_buf *dmabuf = v; + + ++*pos; + + return dma_buf_iter_next(dmabuf); +} + +struct bpf_iter__dmabuf { + __bpf_md_ptr(struct bpf_iter_meta *, meta); + __bpf_md_ptr(struct dma_buf *, dmabuf); +}; + +static int __dmabuf_seq_show(struct seq_file *seq, void *v, bool in_stop) +{ + struct bpf_iter_meta meta = { + .seq = seq, + }; + struct bpf_iter__dmabuf ctx = { + .meta = &meta, + .dmabuf = v, + }; + struct bpf_prog *prog = bpf_iter_get_info(&meta, in_stop); + + if (prog) + return bpf_iter_run_prog(prog, &ctx); + + return 0; +} + +static int dmabuf_iter_seq_show(struct seq_file *seq, void *v) +{ + return __dmabuf_seq_show(seq, v, false); +} + +static void dmabuf_iter_seq_stop(struct seq_file *seq, void *v) +{ + struct dma_buf *dmabuf = v; + + if (dmabuf) { + struct dmabuf_iter_priv *p = seq->private; + + p->dmabuf = dmabuf; + } +} + +static const struct seq_operations dmabuf_iter_seq_ops = { + .start = dmabuf_iter_seq_start, + .next = dmabuf_iter_seq_next, + .stop = dmabuf_iter_seq_stop, + .show = dmabuf_iter_seq_show, +}; + +static void bpf_iter_dmabuf_show_fdinfo(const struct bpf_iter_aux_info *aux, + struct seq_file *seq) +{ + seq_puts(seq, "dmabuf iter\n"); +} + +static int dmabuf_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux) +{ + struct dmabuf_iter_priv *p = (struct dmabuf_iter_priv *)priv; + + p->dmabuf = NULL; + return 0; +} + +static void dmabuf_iter_seq_fini(void *priv) +{ + struct dmabuf_iter_priv *p = (struct dmabuf_iter_priv *)priv; + + if (p->dmabuf) + dma_buf_put(p->dmabuf); +} + +static const struct bpf_iter_seq_info dmabuf_iter_seq_info = { + .seq_ops = &dmabuf_iter_seq_ops, + .init_seq_private = dmabuf_iter_seq_init, + .fini_seq_private = dmabuf_iter_seq_fini, + .seq_priv_size = sizeof(struct dmabuf_iter_priv), +}; + +static struct bpf_iter_reg bpf_dmabuf_reg_info = { + .target = "dmabuf", + .feature = BPF_ITER_RESCHED, + .show_fdinfo = bpf_iter_dmabuf_show_fdinfo, + .ctx_arg_info_size = 1, + .ctx_arg_info = { + { offsetof(struct bpf_iter__dmabuf, dmabuf), + PTR_TO_BTF_ID_OR_NULL }, + }, + .seq_info = &dmabuf_iter_seq_info, +}; + +DEFINE_BPF_ITER_FUNC(dmabuf, struct bpf_iter_meta *meta, struct dma_buf *dmabuf) +BTF_ID_LIST_SINGLE(bpf_dmabuf_btf_id, struct, dma_buf) + +static int __init dmabuf_iter_init(void) +{ + bpf_dmabuf_reg_info.ctx_arg_info[0].btf_id = bpf_dmabuf_btf_id[0]; + return bpf_iter_reg_target(&bpf_dmabuf_reg_info); +} + +late_initcall(dmabuf_iter_init); + +struct bpf_iter_dmabuf { + /* + * opaque iterator state; having __u64 here allows to preserve correct + * alignment requirements in vmlinux.h, generated from BTF + */ + __u64 __opaque[1]; +} __aligned(8); + +/* Non-opaque version of bpf_iter_dmabuf */ +struct bpf_iter_dmabuf_kern { + struct dma_buf *dmabuf; +} __aligned(8); + +__bpf_kfunc int bpf_iter_dmabuf_new(struct bpf_iter_dmabuf *it) +{ + struct bpf_iter_dmabuf_kern *kit = (void *)it; + + BUILD_BUG_ON(sizeof(*kit) > sizeof(*it)); + BUILD_BUG_ON(__alignof__(*kit) != __alignof__(*it)); + + kit->dmabuf = NULL; + return 0; +} + +__bpf_kfunc struct dma_buf *bpf_iter_dmabuf_next(struct bpf_iter_dmabuf *it) +{ + struct bpf_iter_dmabuf_kern *kit = (void *)it; + + if (kit->dmabuf) + kit->dmabuf = dma_buf_iter_next(kit->dmabuf); + else + kit->dmabuf = dma_buf_iter_begin(); + + return kit->dmabuf; +} + +__bpf_kfunc void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) +{ + struct bpf_iter_dmabuf_kern *kit = (void *)it; + + if (kit->dmabuf) + dma_buf_put(kit->dmabuf); +} diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index fc34f72702cc4..8a3eadf17f785 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -2212,7 +2212,7 @@ static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_ b = &htab->buckets[i]; rcu_read_lock(); head = &b->head; - hlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) { + hlist_nulls_for_each_entry_safe(elem, n, head, hash_node) { key = elem->key; if (is_percpu) { /* current cpu value for percpu map */ diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index df7cb3856f57b..e704488264ab9 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -128,7 +128,8 @@ const struct bpf_func_proto bpf_map_peek_elem_proto = { BPF_CALL_3(bpf_map_lookup_percpu_elem, struct bpf_map *, map, void *, key, u32, cpu) { - WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && + !rcu_read_lock_bh_held()); return (unsigned long) map->ops->map_lookup_percpu_elem(map, key, cpu); } @@ -538,7 +539,8 @@ const struct bpf_func_proto bpf_strtol_proto = { .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_LONG, + .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_ALIGNED, + .arg4_size = sizeof(s64), }; BPF_CALL_4(bpf_strtoul, const char *, buf, size_t, buf_len, u64, flags, @@ -567,7 +569,8 @@ const struct bpf_func_proto bpf_strtoul_proto = { .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_LONG, + .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_ALIGNED, + .arg4_size = sizeof(u64), }; BPF_CALL_3(bpf_strncmp, const char *, s1, u32, s1_sz, const char *, s2) @@ -882,6 +885,13 @@ int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args, if (fmt[i] == 'p') { sizeof_cur_arg = sizeof(long); + if (fmt[i + 1] == 0 || isspace(fmt[i + 1]) || + ispunct(fmt[i + 1])) { + if (tmp_buf) + cur_arg = raw_args[num_spec]; + goto nocopy_fmt; + } + if ((fmt[i + 1] == 'k' || fmt[i + 1] == 'u') && fmt[i + 2] == 's') { fmt_ptype = fmt[i + 1]; @@ -889,11 +899,9 @@ int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args, goto fmt_str; } - if (fmt[i + 1] == 0 || isspace(fmt[i + 1]) || - ispunct(fmt[i + 1]) || fmt[i + 1] == 'K' || + if (fmt[i + 1] == 'K' || fmt[i + 1] == 'x' || fmt[i + 1] == 's' || fmt[i + 1] == 'S') { - /* just kernel pointers */ if (tmp_buf) cur_arg = raw_args[num_spec]; i++; @@ -1196,8 +1204,11 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u goto out; } - /* allocate hrtimer via map_kmalloc to use memcg accounting */ - cb = bpf_map_kmalloc_node(map, size, GFP_ATOMIC, map->numa_node); + /* Allocate via bpf_map_kmalloc_node() for memcg accounting. Until + * kmalloc_nolock() is available, avoid locking issues by using + * __GFP_HIGH (GFP_ATOMIC & ~__GFP_RECLAIM). + */ + cb = bpf_map_kmalloc_node(map, size, __GFP_HIGH, map->numa_node); if (!cb) { ret = -ENOMEM; goto out; @@ -2210,6 +2221,12 @@ __bpf_kfunc void bpf_task_release(struct task_struct *p) put_task_struct_rcu_user(p); } +__bpf_kfunc void bpf_task_release_dtor(void *p) +{ + put_task_struct_rcu_user(p); +} +CFI_NOSEAL(bpf_task_release_dtor); + #ifdef CONFIG_CGROUPS /** * bpf_cgroup_acquire - Acquire a reference to a cgroup. A cgroup acquired by @@ -2234,6 +2251,12 @@ __bpf_kfunc void bpf_cgroup_release(struct cgroup *cgrp) cgroup_put(cgrp); } +__bpf_kfunc void bpf_cgroup_release_dtor(void *cgrp) +{ + cgroup_put(cgrp); +} +CFI_NOSEAL(bpf_cgroup_release_dtor); + /** * bpf_cgroup_ancestor - Perform a lookup on an entry in a cgroup's ancestor * array. A cgroup returned by this kfunc which is not subsequently stored in a @@ -2265,7 +2288,7 @@ __bpf_kfunc struct cgroup *bpf_cgroup_from_id(u64 cgid) { struct cgroup *cgrp; - cgrp = cgroup_get_from_id(cgid); + cgrp = __cgroup_get_from_id(cgid); if (IS_ERR(cgrp)) return NULL; return cgrp; @@ -2564,10 +2587,10 @@ static const struct btf_kfunc_id_set generic_kfunc_set = { BTF_ID_LIST(generic_dtor_ids) BTF_ID(struct, task_struct) -BTF_ID(func, bpf_task_release) +BTF_ID(func, bpf_task_release_dtor) #ifdef CONFIG_CGROUPS BTF_ID(struct, cgroup) -BTF_ID(func, bpf_cgroup_release) +BTF_ID(func, bpf_cgroup_release_dtor) #endif BTF_SET8_START(common_btf_ids) @@ -2585,6 +2608,11 @@ BTF_ID_FLAGS(func, bpf_dynptr_is_null) BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly) BTF_ID_FLAGS(func, bpf_dynptr_size) BTF_ID_FLAGS(func, bpf_dynptr_clone) +#ifdef CONFIG_DMA_SHARED_BUFFER +BTF_ID_FLAGS(func, bpf_iter_dmabuf_new, KF_ITER_NEW | KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_iter_dmabuf_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_iter_dmabuf_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#endif BTF_SET8_END(common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = { diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 99d0625b6c828..9a9630adcba4f 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -607,7 +607,7 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) return 0; } -static void bpf_free_inode(struct inode *inode) +static void bpf_destroy_inode(struct inode *inode) { enum bpf_type type; @@ -622,7 +622,7 @@ static const struct super_operations bpf_super_ops = { .statfs = simple_statfs, .drop_inode = generic_delete_inode, .show_options = bpf_show_options, - .free_inode = bpf_free_inode, + .destroy_inode = bpf_destroy_inode, }; enum { diff --git a/kernel/bpf/preload/Kconfig b/kernel/bpf/preload/Kconfig index c9d45c9d6918d..f9b11d01c3b50 100644 --- a/kernel/bpf/preload/Kconfig +++ b/kernel/bpf/preload/Kconfig @@ -10,7 +10,6 @@ menuconfig BPF_PRELOAD # The dependency on !COMPILE_TEST prevents it from being enabled # in allmodconfig or allyesconfig configurations depends on !COMPILE_TEST - select USERMODE_DRIVER help This builds kernel module with several embedded BPF programs that are pinned into BPF FS mount point as human readable files that are diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index f779adee0fb2b..c9a6de426b698 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -215,6 +215,8 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr) static void bpf_ringbuf_free(struct bpf_ringbuf *rb) { + irq_work_sync(&rb->work); + /* copy pages pointer and nr_pages to local variable, as we are going * to unmap rb itself with vunmap() below */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 08b1d59f6f075..27f2d2aced4b4 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3876,7 +3876,8 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, #define BPF_F_ATTACH_MASK_BASE \ (BPF_F_ALLOW_OVERRIDE | \ BPF_F_ALLOW_MULTI | \ - BPF_F_REPLACE) + BPF_F_REPLACE | \ + BPF_F_PREORDER) #define BPF_F_ATTACH_MASK_MPROG \ (BPF_F_REPLACE | \ @@ -4418,6 +4419,8 @@ static int bpf_prog_get_info_by_fd(struct file *file, info.recursion_misses = stats.misses; info.verified_insns = prog->aux->verified_insns; + if (prog->aux->btf) + info.btf_id = btf_obj_id(prog->aux->btf); if (!bpf_capable()) { info.jited_prog_len = 0; @@ -4564,8 +4567,6 @@ static int bpf_prog_get_info_by_fd(struct file *file, } } - if (prog->aux->btf) - info.btf_id = btf_obj_id(prog->aux->btf); info.attach_btf_id = prog->aux->attach_btf_id; if (attach_btf) info.attach_btf_obj_id = btf_obj_id(attach_btf); @@ -5679,7 +5680,8 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .arg1_type = ARG_PTR_TO_MEM, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_LONG, + .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_ALIGNED, + .arg4_size = sizeof(u64), }; static const struct bpf_func_proto * diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e97aeda3a86b5..b6024fc903490 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -460,10 +460,6 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut * BPF_TRAMP_F_SHARE_IPMODIFY is set, we can generate the * trampoline again, and retry register. */ - /* reset fops->func and fops->trampoline for re-register */ - tr->fops->func = NULL; - tr->fops->trampoline = 0; - /* reset im->image memory attr for arch_prepare_bpf_trampoline */ set_memory_nx((long)im->image, 1); set_memory_rw((long)im->image, 1); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ab17e5b9278d4..0a2d9c2703dec 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -460,7 +460,8 @@ static bool reg_not_null(const struct bpf_reg_state *reg) type == PTR_TO_MAP_KEY || type == PTR_TO_SOCK_COMMON || (type == PTR_TO_BTF_ID && is_trusted_reg(reg)) || - type == PTR_TO_MEM; + type == PTR_TO_MEM || + type == CONST_PTR_TO_MAP; } static bool type_is_ptr_alloc_obj(u32 type) @@ -7535,6 +7536,10 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno, verbose(env, "verifier bug. Two map pointers in a timer helper\n"); return -EFAULT; } + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n"); + return -EOPNOTSUPP; + } meta->map_uid = reg->map_uid; meta->map_ptr = map; return 0; @@ -7824,7 +7829,7 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, struct bpf_verifier_state *cur) { struct bpf_func_state *fold, *fcur; - int i, fr; + int i, fr, num_slots; reset_idmap_scratch(env); for (fr = old->curframe; fr >= 0; fr--) { @@ -7837,7 +7842,9 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, &fcur->regs[i], &env->idmap_scratch); - for (i = 0; i < fold->allocated_stack / BPF_REG_SIZE; i++) { + num_slots = min(fold->allocated_stack / BPF_REG_SIZE, + fcur->allocated_stack / BPF_REG_SIZE); + for (i = 0; i < num_slots; i++) { if (!is_spilled_reg(&fold->stack[i]) || !is_spilled_reg(&fcur->stack[i])) continue; @@ -8015,16 +8022,6 @@ static bool arg_type_is_dynptr(enum bpf_arg_type type) return base_type(type) == ARG_PTR_TO_DYNPTR; } -static int int_ptr_type_to_size(enum bpf_arg_type type) -{ - if (type == ARG_PTR_TO_INT) - return sizeof(u32); - else if (type == ARG_PTR_TO_LONG) - return sizeof(u64); - - return -EINVAL; -} - static int resolve_map_arg_type(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type) @@ -8097,16 +8094,6 @@ static const struct bpf_reg_types mem_types = { }, }; -static const struct bpf_reg_types int_ptr_types = { - .types = { - PTR_TO_STACK, - PTR_TO_PACKET, - PTR_TO_PACKET_META, - PTR_TO_MAP_KEY, - PTR_TO_MAP_VALUE, - }, -}; - static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE, @@ -8161,8 +8148,6 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, [ARG_PTR_TO_MEM] = &mem_types, [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, - [ARG_PTR_TO_INT] = &int_ptr_types, - [ARG_PTR_TO_LONG] = &int_ptr_types, [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, [ARG_PTR_TO_FUNC] = &func_ptr_types, [ARG_PTR_TO_STACK] = &stack_ptr_types, @@ -8669,9 +8654,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, */ meta->raw_mode = arg_type & MEM_UNINIT; if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, regno, - fn->arg_size[arg], false, - meta); + err = check_helper_mem_access(env, regno, fn->arg_size[arg], false, meta); + if (err) + return err; + if (arg_type & MEM_ALIGNED) + err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); } break; case ARG_CONST_SIZE: @@ -8696,17 +8683,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; break; - case ARG_PTR_TO_INT: - case ARG_PTR_TO_LONG: - { - int size = int_ptr_type_to_size(arg_type); - - err = check_helper_mem_access(env, regno, size, false, meta); - if (err) - return err; - err = check_ptr_alignment(env, reg, 0, size, true); - break; - } case ARG_PTR_TO_CONST_STR: { struct bpf_map *map = reg->map_ptr; @@ -13734,7 +13710,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) } else { /* all other ALU ops: and, sub, xor, add, ... */ if (BPF_SRC(insn->code) == BPF_X) { - if (insn->imm != 0 || insn->off > 1 || + if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) || (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { verbose(env, "BPF_ALU uses reserved fields\n"); return -EINVAL; @@ -13744,7 +13720,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (err) return err; } else { - if (insn->src_reg != BPF_REG_0 || insn->off > 1 || + if (insn->src_reg != BPF_REG_0 || (insn->off != 0 && insn->off != 1) || (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { verbose(env, "BPF_ALU uses reserved fields\n"); return -EINVAL; @@ -15997,12 +15973,16 @@ static void clean_verifier_state(struct bpf_verifier_env *env, static void clean_live_states(struct bpf_verifier_env *env, int insn, struct bpf_verifier_state *cur) { + struct bpf_verifier_state *loop_entry; struct bpf_verifier_state_list *sl; sl = *explored_state(env, insn); while (sl) { if (sl->state.branches) goto next; + loop_entry = get_loop_entry(&sl->state); + if (loop_entry && loop_entry->branches) + goto next; if (sl->state.insn_idx != insn || !same_callsites(&sl->state, cur)) goto next; diff --git a/kernel/cfi.c b/kernel/cfi.c index 08caad7767176..d2c5861624fa7 100644 --- a/kernel/cfi.c +++ b/kernel/cfi.c @@ -5,6 +5,8 @@ * Copyright (C) 2022 Google LLC */ +#include +#include #include enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, @@ -25,6 +27,19 @@ enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, return BUG_TRAP_TYPE_BUG; } +/* + * Declare two non-existent functions with types that match bpf_func_t and + * bpf_callback_t pointers, and use DEFINE_CFI_TYPE to define type hash + * variables for each function type. The cfi_bpf_* variables are used by + * arch-specific BPF JIT implementations to ensure indirectly callable JIT + * code has matching CFI type hashes. + */ +extern typeof(*(bpf_func_t)0) __bpf_prog_runX; +DEFINE_CFI_TYPE(cfi_bpf_hash, __bpf_prog_runX); + +extern typeof(*(bpf_callback_t)0) __bpf_callback_fn; +DEFINE_CFI_TYPE(cfi_bpf_subprog_hash, __bpf_callback_fn); + #ifdef CONFIG_ARCH_USES_CFI_TRAPS static inline unsigned long trap_address(s32 *p) { diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 9cb68c924a2e0..21d1e56b8d4f9 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -93,7 +93,7 @@ DEFINE_MUTEX(cgroup_mutex); DEFINE_SPINLOCK(css_set_lock); -#ifdef CONFIG_PROVE_RCU +#if (defined CONFIG_PROVE_RCU || defined CONFIG_LOCKDEP) EXPORT_SYMBOL_GPL(cgroup_mutex); EXPORT_SYMBOL_GPL(css_set_lock); #endif @@ -127,8 +127,31 @@ EXPORT_SYMBOL_GPL(cgroup_threadgroup_rwsem); * of concurrent destructions. Use a separate workqueue so that cgroup * destruction work items don't end up filling up max_active of system_wq * which may lead to deadlock. + * + * A cgroup destruction should enqueue work sequentially to: + * cgroup_offline_wq: use for css offline work + * cgroup_release_wq: use for css release work + * cgroup_free_wq: use for free work + * + * Rationale for using separate workqueues: + * The cgroup root free work may depend on completion of other css offline + * operations. If all tasks were enqueued to a single workqueue, this could + * create a deadlock scenario where: + * - Free work waits for other css offline work to complete. + * - But other css offline work is queued after free work in the same queue. + * + * Example deadlock scenario with single workqueue (cgroup_destroy_wq): + * 1. umount net_prio + * 2. net_prio root destruction enqueues work to cgroup_destroy_wq (CPUx) + * 3. perf_event CSS A offline enqueues work to same cgroup_destroy_wq (CPUx) + * 4. net_prio cgroup_destroy_root->cgroup_lock_and_drain_offline. + * 5. net_prio root destruction blocks waiting for perf_event CSS A offline, + * which can never complete as it's behind in the same queue and + * workqueue's max_active is 1. */ -static struct workqueue_struct *cgroup_destroy_wq; +static struct workqueue_struct *cgroup_offline_wq; +static struct workqueue_struct *cgroup_release_wq; +static struct workqueue_struct *cgroup_free_wq; /* generate an array of cgroup subsystem pointers */ #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys, @@ -3687,6 +3710,27 @@ static int cgroup_stat_show(struct seq_file *seq, void *v) return 0; } +static int cgroup_core_local_stat_show(struct seq_file *seq, void *v) +{ + struct cgroup *cgrp = seq_css(seq)->cgroup; + unsigned int sequence; + u64 freeze_time; + + do { + sequence = read_seqcount_begin(&cgrp->kmi_ext_info->freezer.freeze_seq); + freeze_time = cgrp->kmi_ext_info->freezer.frozen_nsec; + /* Add in current freezer interval if the cgroup is freezing. */ + if (test_bit(CGRP_FREEZE, &cgrp->flags)) + freeze_time += (ktime_get_ns() - + cgrp->kmi_ext_info->freezer.freeze_start_nsec); + } while (read_seqcount_retry(&cgrp->kmi_ext_info->freezer.freeze_seq, sequence)); + + do_div(freeze_time, NSEC_PER_USEC); + seq_printf(seq, "frozen_usec %llu\n", freeze_time); + + return 0; +} + #ifdef CONFIG_CGROUP_SCHED /** * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem @@ -4488,6 +4532,7 @@ int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) cft->flags |= __CFTYPE_ONLY_ON_DFL; return cgroup_add_cftypes(ss, cfts); } +EXPORT_SYMBOL_GPL(cgroup_add_dfl_cftypes); /** * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies @@ -5301,6 +5346,11 @@ static struct cftype cgroup_base_files[] = { .name = "cgroup.stat", .seq_show = cgroup_stat_show, }, + { + .name = "cgroup.stat.local", + .flags = CFTYPE_NOT_ON_ROOT, + .seq_show = cgroup_core_local_stat_show, + }, { .name = "cgroup.freeze", .flags = CFTYPE_NOT_ON_ROOT, @@ -5491,7 +5541,7 @@ static void css_release_work_fn(struct work_struct *work) cgroup_unlock(); INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn); - queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork); + queue_rcu_work(cgroup_free_wq, &css->destroy_rwork); } static void css_release(struct percpu_ref *ref) @@ -5500,7 +5550,7 @@ static void css_release(struct percpu_ref *ref) container_of(ref, struct cgroup_subsys_state, refcnt); INIT_WORK(&css->destroy_work, css_release_work_fn); - queue_work(cgroup_destroy_wq, &css->destroy_work); + queue_work(cgroup_release_wq, &css->destroy_work); } static void init_and_link_css(struct cgroup_subsys_state *css, @@ -5622,7 +5672,7 @@ static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, err_free_css: list_del_rcu(&css->rstat_css_node); INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn); - queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork); + queue_rcu_work(cgroup_free_wq, &css->destroy_rwork); return ERR_PTR(err); } @@ -5676,11 +5726,16 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, goto out_psi_free; } + cgrp->kmi_ext_info = kzalloc(sizeof(*cgrp->kmi_ext_info), GFP_KERNEL); + if (!cgrp->kmi_ext_info) + goto out_psi_free; + /* * New cgroup inherits effective freeze counter, and * if the parent has to be frozen, the child has too. */ cgrp->freezer.e_freeze = parent->freezer.e_freeze; + seqcount_init(&cgrp->kmi_ext_info->freezer.freeze_seq); if (cgrp->freezer.e_freeze) { /* * Set the CGRP_FREEZE flag, so when a process will be @@ -5689,6 +5744,7 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, * consider it frozen immediately. */ set_bit(CGRP_FREEZE, &cgrp->flags); + cgrp->kmi_ext_info->freezer.freeze_start_nsec = ktime_get_ns(); set_bit(CGRP_FROZEN, &cgrp->flags); } @@ -5857,7 +5913,7 @@ static void css_killed_ref_fn(struct percpu_ref *ref) if (atomic_dec_and_test(&css->online_cnt)) { INIT_WORK(&css->destroy_work, css_killed_work_fn); - queue_work(cgroup_destroy_wq, &css->destroy_work); + queue_work(cgroup_offline_wq, &css->destroy_work); } } @@ -5964,6 +6020,7 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) spin_lock_irq(&css_set_lock); list_for_each_entry(link, &cgrp->cset_links, cset_link) link->cset->dead = true; + kfree(cgrp->kmi_ext_info); spin_unlock_irq(&css_set_lock); /* initiate massacre of all css's */ @@ -6229,8 +6286,14 @@ static int __init cgroup_wq_init(void) * We would prefer to do this in cgroup_init() above, but that * is called before init_workqueues(): so leave this until after. */ - cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); - BUG_ON(!cgroup_destroy_wq); + cgroup_offline_wq = alloc_workqueue("cgroup_offline", 0, 1); + BUG_ON(!cgroup_offline_wq); + + cgroup_release_wq = alloc_workqueue("cgroup_release", 0, 1); + BUG_ON(!cgroup_release_wq); + + cgroup_free_wq = alloc_workqueue("cgroup_free", 0, 1); + BUG_ON(!cgroup_free_wq); return 0; } core_initcall(cgroup_wq_init); @@ -6247,15 +6310,15 @@ void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) } /* - * cgroup_get_from_id : get the cgroup associated with cgroup id + * __cgroup_get_from_id : get the cgroup associated with cgroup id * @id: cgroup id * On success return the cgrp or ERR_PTR on failure - * Only cgroups within current task's cgroup NS are valid. + * There are no cgroup NS restrictions. */ -struct cgroup *cgroup_get_from_id(u64 id) +struct cgroup *__cgroup_get_from_id(u64 id) { struct kernfs_node *kn; - struct cgroup *cgrp, *root_cgrp; + struct cgroup *cgrp; kn = kernfs_find_and_get_node_by_id(cgrp_dfl_root.kf_root, id); if (!kn) @@ -6277,6 +6340,22 @@ struct cgroup *cgroup_get_from_id(u64 id) if (!cgrp) return ERR_PTR(-ENOENT); + return cgrp; +} + +/* + * cgroup_get_from_id : get the cgroup associated with cgroup id + * @id: cgroup id + * On success return the cgrp or ERR_PTR on failure + * Only cgroups within current task's cgroup NS are valid. + */ +struct cgroup *cgroup_get_from_id(u64 id) +{ + struct cgroup *cgrp, *root_cgrp; + + cgrp = __cgroup_get_from_id(id); + if (IS_ERR(cgrp)) + return cgrp; root_cgrp = current_cgns_cgroup_dfl(); if (!cgroup_is_descendant(cgrp, root_cgrp)) { diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 5b478f94ec31a..d01324adae527 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -440,7 +440,7 @@ static inline void check_insane_mems_config(nodemask_t *nodes) { if (!cpusets_insane_config() && movable_only_nodes(nodes)) { - static_branch_enable(&cpusets_insane_config_key); + static_branch_enable_cpuslocked(&cpusets_insane_config_key); pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n" "Cpuset allocations might fail even with a lot of memory available.\n", nodemask_pr_args(nodes)); @@ -1250,9 +1250,11 @@ static void update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) if (top_cs) { /* - * Percpu kthreads in top_cpuset are ignored + * PF_NO_SETAFFINITY tasks are ignored. + * All per cpu kthreads should have PF_NO_SETAFFINITY + * flag set, see kthread_set_per_cpu(). */ - if (kthread_is_per_cpu(task)) + if (task->flags & PF_NO_SETAFFINITY) continue; cpumask_andnot(new_cpus, possible_mask, cs->subparts_cpus); } else { @@ -2679,6 +2681,8 @@ static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to); cpuset_update_task_spread_flags(cs, task); + + trace_android_vh_cpuset_attach_task(&cs->css, task); } static void cpuset_attach(struct cgroup_taskset *tset) @@ -3352,6 +3356,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css) out_unlock: mutex_unlock(&cpuset_mutex); cpus_read_unlock(); + trace_android_vh_cpuset_css_online(css); return 0; } diff --git a/kernel/cgroup/freezer.c b/kernel/cgroup/freezer.c index d0224d3c0a323..397a2e604d2f5 100644 --- a/kernel/cgroup/freezer.c +++ b/kernel/cgroup/freezer.c @@ -179,7 +179,7 @@ static void cgroup_freeze_task(struct task_struct *task, bool freeze) /* * Freeze or unfreeze all tasks in the given cgroup. */ -static void cgroup_do_freeze(struct cgroup *cgrp, bool freeze) +static void cgroup_do_freeze(struct cgroup *cgrp, bool freeze, u64 ts_nsec) { struct css_task_iter it; struct task_struct *task; @@ -187,10 +187,16 @@ static void cgroup_do_freeze(struct cgroup *cgrp, bool freeze) lockdep_assert_held(&cgroup_mutex); spin_lock_irq(&css_set_lock); - if (freeze) + write_seqcount_begin(&cgrp->kmi_ext_info->freezer.freeze_seq); + if (freeze) { set_bit(CGRP_FREEZE, &cgrp->flags); - else + cgrp->kmi_ext_info->freezer.freeze_start_nsec = ts_nsec; + } else { clear_bit(CGRP_FREEZE, &cgrp->flags); + cgrp->kmi_ext_info->freezer.frozen_nsec += (ts_nsec - + cgrp->kmi_ext_info->freezer.freeze_start_nsec); + } + write_seqcount_end(&cgrp->kmi_ext_info->freezer.freeze_seq); spin_unlock_irq(&css_set_lock); if (freeze) @@ -267,6 +273,7 @@ void cgroup_freeze(struct cgroup *cgrp, bool freeze) struct cgroup_subsys_state *css; struct cgroup *dsct; bool applied = false; + u64 ts_nsec; lockdep_assert_held(&cgroup_mutex); @@ -277,6 +284,7 @@ void cgroup_freeze(struct cgroup *cgrp, bool freeze) return; cgrp->freezer.freeze = freeze; + ts_nsec = ktime_get_ns(); /* * Propagate changes downwards the cgroup tree. @@ -308,7 +316,7 @@ void cgroup_freeze(struct cgroup *cgrp, bool freeze) /* * Do change actual state: freeze or unfreeze. */ - cgroup_do_freeze(dsct, freeze); + cgroup_do_freeze(dsct, freeze, ts_nsec); applied = true; } diff --git a/kernel/cgroup/legacy_freezer.c b/kernel/cgroup/legacy_freezer.c index 75609996f0da9..28d87289ae9ea 100644 --- a/kernel/cgroup/legacy_freezer.c +++ b/kernel/cgroup/legacy_freezer.c @@ -66,15 +66,9 @@ static struct freezer *parent_freezer(struct freezer *freezer) bool cgroup_freezing(struct task_struct *task) { bool ret; - unsigned int state; rcu_read_lock(); - /* Check if the cgroup is still FREEZING, but not FROZEN. The extra - * !FROZEN check is required, because the FREEZING bit is not cleared - * when the state FROZEN is reached. - */ - state = task_freezer(task)->state; - ret = (state & CGROUP_FREEZING) && !(state & CGROUP_FROZEN); + ret = task_freezer(task)->state & CGROUP_FREEZING; rcu_read_unlock(); return ret; @@ -189,13 +183,12 @@ static void freezer_attach(struct cgroup_taskset *tset) if (!(freezer->state & CGROUP_FREEZING)) { __thaw_task(task); } else { - freeze_task(task); - /* clear FROZEN and propagate upwards */ while (freezer && (freezer->state & CGROUP_FROZEN)) { freezer->state &= ~CGROUP_FROZEN; freezer = parent_freezer(freezer); } + freeze_task(task); } } diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c index cc18b8e321e38..73747af869a1b 100644 --- a/kernel/cgroup/rstat.c +++ b/kernel/cgroup/rstat.c @@ -249,13 +249,11 @@ static void cgroup_rstat_flush_locked(struct cgroup *cgrp) rcu_read_unlock(); } - /* play nice and yield if necessary */ - if (need_resched() || spin_needbreak(&cgroup_rstat_lock)) { - spin_unlock_irq(&cgroup_rstat_lock); - if (!cond_resched()) - cpu_relax(); - spin_lock_irq(&cgroup_rstat_lock); - } + /* play nice and avoid disabling interrupts for a long time */ + spin_unlock_irq(&cgroup_rstat_lock); + if (!cond_resched()) + cpu_relax(); + spin_lock_irq(&cgroup_rstat_lock); } } diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index d934e8aa0baa9..d09c801cae40f 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "debug.h" @@ -601,7 +602,9 @@ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) if (rc == -ENOMEM) { pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n"); global_disable = true; - } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { + } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && + is_swiotlb_active(entry->dev))) { err_printk(entry->dev, entry, "cacheline tracking EEXIST, overlapping mappings aren't supported\n"); } diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index 3783f55b9fcf4..b3bd8e0413051 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -102,8 +102,8 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, #ifdef CONFIG_DMA_DIRECT_REMAP addr = dma_common_contiguous_remap(page, pool_size, - pgprot_dmacoherent(PAGE_KERNEL), - __builtin_return_address(0)); + pgprot_decrypted(pgprot_dmacoherent(PAGE_KERNEL)), + __builtin_return_address(0)); if (!addr) goto free_page; #else diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c index 1273be84392cf..ee01cfcc35064 100644 --- a/kernel/events/callchain.c +++ b/kernel/events/callchain.c @@ -184,6 +184,10 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, struct perf_callchain_entry_ctx ctx; int rctx; + /* crosstask is not supported for user stacks */ + if (crosstask && user && !kernel) + return NULL; + entry = get_callchain_entry(&rctx); if (!entry) return NULL; @@ -200,18 +204,15 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, perf_callchain_kernel(&ctx, regs); } - if (user) { + if (user && !crosstask) { if (!user_mode(regs)) { - if (current->mm) - regs = task_pt_regs(current); - else + if (current->flags & (PF_KTHREAD | PF_USER_WORKER)) regs = NULL; + else + regs = task_pt_regs(current); } if (regs) { - if (crosstask) - goto exit_put; - if (add_mark) perf_callchain_store_context(&ctx, PERF_CONTEXT_USER); @@ -219,7 +220,6 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, } } -exit_put: put_callchain_entry(rctx); return entry; diff --git a/kernel/events/core.c b/kernel/events/core.c index 0f7964bbc786f..08911840eecff 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -172,6 +172,19 @@ static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, raw_spin_unlock(&cpuctx->ctx.lock); } +typedef struct { + struct perf_cpu_context *cpuctx; + struct perf_event_context *ctx; +} class_perf_ctx_lock_t; + +static inline void class_perf_ctx_lock_destructor(class_perf_ctx_lock_t *_T) +{ perf_ctx_unlock(_T->cpuctx, _T->ctx); } + +static inline class_perf_ctx_lock_t +class_perf_ctx_lock_constructor(struct perf_cpu_context *cpuctx, + struct perf_event_context *ctx) +{ perf_ctx_lock(cpuctx, ctx); return (class_perf_ctx_lock_t){ cpuctx, ctx }; } + #define TASK_TOMBSTONE ((void *)-1L) static bool is_kernel_event(struct perf_event *event) @@ -861,13 +874,19 @@ static void perf_cgroup_switch(struct task_struct *task) if (READ_ONCE(cpuctx->cgrp) == NULL) return; - WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); - cgrp = perf_cgroup_from_task(task, NULL); if (READ_ONCE(cpuctx->cgrp) == cgrp) return; - perf_ctx_lock(cpuctx, cpuctx->task_ctx); + guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx); + /* + * Re-check, could've raced vs perf_remove_from_context(). + */ + if (READ_ONCE(cpuctx->cgrp) == NULL) + return; + + WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); + perf_ctx_disable(&cpuctx->ctx, true); ctx_sched_out(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP); @@ -885,7 +904,6 @@ static void perf_cgroup_switch(struct task_struct *task) ctx_sched_in(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP); perf_ctx_enable(&cpuctx->ctx, true); - perf_ctx_unlock(cpuctx, cpuctx->task_ctx); } static int perf_cgroup_ensure_storage(struct perf_event *event, @@ -2061,8 +2079,9 @@ perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event) } static void put_event(struct perf_event *event); -static void event_sched_out(struct perf_event *event, - struct perf_event_context *ctx); +static void __event_disable(struct perf_event *event, + struct perf_event_context *ctx, + enum perf_event_state state); static void perf_put_aux_event(struct perf_event *event) { @@ -2095,8 +2114,7 @@ static void perf_put_aux_event(struct perf_event *event) * state so that we don't try to schedule it again. Note * that perf_event_enable() will clear the ERROR status. */ - event_sched_out(iter, ctx); - perf_event_set_state(event, PERF_EVENT_STATE_ERROR); + __event_disable(iter, ctx, PERF_EVENT_STATE_ERROR); } } @@ -2150,18 +2168,6 @@ static inline struct list_head *get_event_list(struct perf_event *event) &event->pmu_ctx->flexible_active; } -/* - * Events that have PERF_EV_CAP_SIBLING require being part of a group and - * cannot exist on their own, schedule them out and move them into the ERROR - * state. Also see _perf_event_enable(), it will not be able to recover - * this ERROR state. - */ -static inline void perf_remove_sibling_event(struct perf_event *event) -{ - event_sched_out(event, event->ctx); - perf_event_set_state(event, PERF_EVENT_STATE_ERROR); -} - static void perf_group_detach(struct perf_event *event) { struct perf_event *leader = event->group_leader; @@ -2197,8 +2203,15 @@ static void perf_group_detach(struct perf_event *event) */ list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { + /* + * Events that have PERF_EV_CAP_SIBLING require being part of + * a group and cannot exist on their own, schedule them out + * and move them into the ERROR state. Also see + * _perf_event_enable(), it will not be able to recover this + * ERROR state. + */ if (sibling->event_caps & PERF_EV_CAP_SIBLING) - perf_remove_sibling_event(sibling); + __event_disable(sibling, ctx, PERF_EVENT_STATE_ERROR); sibling->group_leader = sibling; list_del_init(&sibling->sibling_list); @@ -2432,6 +2445,15 @@ static void perf_remove_from_context(struct perf_event *event, unsigned long fla event_function_call(event, __perf_remove_from_context, (void *)flags); } +static void __event_disable(struct perf_event *event, + struct perf_event_context *ctx, + enum perf_event_state state) +{ + event_sched_out(event, ctx); + perf_cgroup_event_disable(event, ctx); + perf_event_set_state(event, state); +} + /* * Cross CPU call to disable a performance event */ @@ -2450,13 +2472,18 @@ static void __perf_event_disable(struct perf_event *event, perf_pmu_disable(event->pmu_ctx->pmu); + /* + * When disabling a group leader, the whole group becomes ineligible + * to run, so schedule out the full group. + */ if (event == event->group_leader) group_sched_out(event, ctx); - else - event_sched_out(event, ctx); - perf_event_set_state(event, PERF_EVENT_STATE_OFF); - perf_cgroup_event_disable(event, ctx); + /* + * But only mark the leader OFF; the siblings will remain + * INACTIVE. + */ + __event_disable(event, ctx, PERF_EVENT_STATE_OFF); perf_pmu_enable(event->pmu_ctx->pmu); } @@ -5893,6 +5920,9 @@ static int perf_event_set_output(struct perf_event *event, static int perf_event_set_filter(struct perf_event *event, void __user *arg); static int perf_copy_attr(struct perf_event_attr __user *uattr, struct perf_event_attr *attr); +static int __perf_event_set_bpf_prog(struct perf_event *event, + struct bpf_prog *prog, + u64 bpf_cookie); static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) { @@ -5961,7 +5991,7 @@ static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned lon if (IS_ERR(prog)) return PTR_ERR(prog); - err = perf_event_set_bpf_prog(event, prog, 0); + err = __perf_event_set_bpf_prog(event, prog, 0); if (err) { bpf_prog_put(prog); return err; @@ -6450,11 +6480,21 @@ static void perf_mmap_close(struct vm_area_struct *vma) ring_buffer_put(rb); /* could be last */ } +static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr) +{ + /* + * Forbid splitting perf mappings to prevent refcount leaks due to + * the resulting non-matching offsets and sizes. See open()/close(). + */ + return -EINVAL; +} + static const struct vm_operations_struct perf_mmap_vmops = { .open = perf_mmap_open, .close = perf_mmap_close, /* non mergeable */ .fault = perf_mmap_fault, .page_mkwrite = perf_mmap_fault, + .may_split = perf_mmap_may_split, }; static int perf_mmap(struct file *file, struct vm_area_struct *vma) @@ -6542,9 +6582,7 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) goto unlock; } - atomic_set(&rb->aux_mmap_count, 1); user_extra = nr_pages; - goto accounting; } @@ -6646,8 +6684,10 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) } else { ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, event->attr.aux_watermark, flags); - if (!ret) + if (!ret) { + atomic_set(&rb->aux_mmap_count, 1); rb->aux_mmap_locked = extra; + } } unlock: @@ -6657,11 +6697,15 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) atomic_inc(&event->mmap_count); } else if (rb) { + /* AUX allocation failed */ atomic_dec(&rb->mmap_count); } aux_unlock: mutex_unlock(&event->mmap_mutex); + if (ret) + return ret; + /* * Since pinned accounting is per vm we cannot allow fork() to copy our * vma. @@ -6897,7 +6941,7 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, if (user_mode(regs)) { regs_user->abi = perf_reg_abi(current); regs_user->regs = regs; - } else if (!(current->flags & PF_KTHREAD)) { + } else if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) { perf_get_regs_user(regs_user, regs); } else { regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; @@ -6940,6 +6984,10 @@ perf_sample_ustack_size(u16 stack_size, u16 header_size, if (!regs) return 0; + /* No mm, no stack, no dump. */ + if (!current->mm) + return 0; + /* * Check if we fit in with the requested stack size into the: * - TASK_SIZE @@ -7519,7 +7567,7 @@ static u64 perf_virt_to_phys(u64 virt) * Try IRQ-safe get_user_page_fast_only first. * If failed, leave phys_addr as 0. */ - if (current->mm != NULL) { + if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) { struct page *p; pagefault_disable(); @@ -7631,12 +7679,16 @@ struct perf_callchain_entry * perf_callchain(struct perf_event *event, struct pt_regs *regs) { bool kernel = !event->attr.exclude_callchain_kernel; - bool user = !event->attr.exclude_callchain_user; + bool user = !event->attr.exclude_callchain_user && + !(current->flags & (PF_KTHREAD | PF_USER_WORKER)); /* Disallow cross-task user callchains. */ bool crosstask = event->ctx->task && event->ctx->task != current; const u32 max_stack = event->attr.sample_max_stack; struct perf_callchain_entry *callchain; + if (!current->mm) + user = false; + if (!kernel && !user) return &__empty_callchain; @@ -9508,14 +9560,14 @@ __perf_event_account_interrupt(struct perf_event *event, int throttle) hwc->interrupts = 1; } else { hwc->interrupts++; - if (unlikely(throttle && - hwc->interrupts > max_samples_per_tick)) { - __this_cpu_inc(perf_throttled_count); - tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); - hwc->interrupts = MAX_INTERRUPTS; - perf_log_throttle(event, 0); - ret = 1; - } + } + + if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) { + __this_cpu_inc(perf_throttled_count); + tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); + hwc->interrupts = MAX_INTERRUPTS; + perf_log_throttle(event, 0); + ret = 1; } if (event->attr.freq) { @@ -10391,7 +10443,7 @@ static int perf_uprobe_event_init(struct perf_event *event) if (event->attr.type != perf_uprobe.type) return -ENOENT; - if (!perfmon_capable()) + if (!capable(CAP_SYS_ADMIN)) return -EACCES; /* @@ -10538,8 +10590,9 @@ static inline bool perf_event_is_tracing(struct perf_event *event) return false; } -int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, - u64 bpf_cookie) +static int __perf_event_set_bpf_prog(struct perf_event *event, + struct bpf_prog *prog, + u64 bpf_cookie) { bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp; @@ -10577,6 +10630,20 @@ int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, return perf_event_attach_bpf_prog(event, prog, bpf_cookie); } +int perf_event_set_bpf_prog(struct perf_event *event, + struct bpf_prog *prog, + u64 bpf_cookie) +{ + struct perf_event_context *ctx; + int ret; + + ctx = perf_event_ctx_lock(event); + ret = __perf_event_set_bpf_prog(event, prog, bpf_cookie); + perf_event_ctx_unlock(event, ctx); + + return ret; +} + void perf_event_free_bpf_prog(struct perf_event *event) { if (!perf_event_is_tracing(event)) { @@ -10596,7 +10663,15 @@ static void perf_event_free_filter(struct perf_event *event) { } -int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, +static int __perf_event_set_bpf_prog(struct perf_event *event, + struct bpf_prog *prog, + u64 bpf_cookie) +{ + return -ENOENT; +} + +int perf_event_set_bpf_prog(struct perf_event *event, + struct bpf_prog *prog, u64 bpf_cookie) { return -ENOENT; diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c index 6c2cb4e4f48da..8f3f624419aa9 100644 --- a/kernel/events/hw_breakpoint.c +++ b/kernel/events/hw_breakpoint.c @@ -950,9 +950,10 @@ static int hw_breakpoint_event_init(struct perf_event *bp) return -ENOENT; /* - * no branch sampling for breakpoint events + * Check if breakpoint type is supported before proceeding. + * Also, no branch sampling for breakpoint events. */ - if (has_branch_stack(bp)) + if (!hw_breakpoint_slots_cached(find_slot_idx(bp->attr.bp_type)) || has_branch_stack(bp)) return -EOPNOTSUPP; err = register_perf_hw_breakpoint(bp); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index c63123f4eda07..3b662823cfb09 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -2257,6 +2257,13 @@ static void handle_swbp(struct pt_regs *regs) handler_chain(uprobe, regs); + /* + * If user decided to take execution elsewhere, it makes little sense + * to execute the original instruction, so let's skip it. + */ + if (instruction_pointer(regs) != bp_vaddr) + goto out; + if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) goto out; diff --git a/kernel/exit.c b/kernel/exit.c old mode 100644 new mode 100755 index c38b863d73399..c41cac5a2663f --- a/kernel/exit.c +++ b/kernel/exit.c @@ -835,6 +835,7 @@ void __noreturn do_exit(long code) trace_android_vh_exit_check(current); + trace_android_vh_lock_task_exit(tsk); /* sync mm's RSS info before statistics gathering */ if (tsk->mm) sync_mm_rss(tsk->mm); @@ -864,6 +865,15 @@ void __noreturn do_exit(long code) tsk->exit_code = code; taskstats_exit(tsk, group_dead); + /* + * Since sampling can touch ->mm, make sure to stop everything before we + * tear it down. + * + * Also flushes inherited counters to the parent - before the parent + * gets woken up by child-exit notifications. + */ + perf_event_exit_task(tsk); + exit_mm(); if (group_dead) @@ -880,14 +890,6 @@ void __noreturn do_exit(long code) exit_task_work(tsk); exit_thread(tsk); - /* - * Flush inherited counters to the parent - before the parent - * gets woken up by child-exit notifications. - * - * because of cgroup mode, must be called before cgroup_exit() - */ - perf_event_exit_task(tsk); - sched_autogroup_exit_task(tsk); cgroup_exit(tsk); diff --git a/kernel/fork.c b/kernel/fork.c old mode 100644 new mode 100755 index 6a88a88b70f38..28cd36bc6336e --- a/kernel/fork.c +++ b/kernel/fork.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -101,6 +102,7 @@ #include #include #include +#include #include #include @@ -115,6 +117,7 @@ #undef CREATE_TRACE_POINTS #include +#include /* * Minimum number of threads to boot the kernel */ @@ -525,10 +528,6 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) vma_numab_state_init(new); dup_anon_vma_name(orig, new); - /* track_pfn_copy() will later take care of copying internal state. */ - if (unlikely(new->vm_flags & VM_PFNMAP)) - untrack_pfn_clear(new); - return new; } @@ -726,6 +725,11 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, tmp = vm_area_dup(mpnt); if (!tmp) goto fail_nomem; + + /* track_pfn_copy() will later take care of copying internal state. */ + if (unlikely(tmp->vm_flags & VM_PFNMAP)) + untrack_pfn_clear(tmp); + retval = vma_dup_policy(mpnt, tmp); if (retval) goto fail_nomem_policy; @@ -749,7 +753,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, get_file(file); i_mmap_lock_write(mapping); - if (tmp->vm_flags & VM_SHARED) + if (vma_is_shared_maywrite(tmp)) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ @@ -999,6 +1003,9 @@ void __put_task_struct(struct task_struct *tsk) WARN_ON(refcount_read(&tsk->usage)); WARN_ON(tsk == current); + sched_ext_free(tsk); + trace_android_vh_put_task(tsk); + put_dmabuf_info(tsk->dmabuf_info); io_uring_free(tsk); cgroup_free(tsk); task_numa_free(tsk, true); @@ -1170,6 +1177,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) setup_thread_stack(tsk, orig); clear_user_return_notifier(tsk); clear_tsk_need_resched(tsk); + trace_android_vh_clear_curr_lazy(tsk); set_task_stack_end_magic(tsk); clear_syscall_work_syscall_user_dispatch(tsk); @@ -1194,6 +1202,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->task_frag.page = NULL; tsk->wake_q.next = NULL; tsk->worker_private = NULL; + tsk->dmabuf_info = NULL; kcov_task_init(tsk); kmsan_task_create(tsk); @@ -1382,6 +1391,7 @@ static inline void __mmput(struct mm_struct *mm) exit_mmap(mm); mm_put_huge_zero_page(mm); set_mm_exe_file(mm, NULL); + put_dmabuf_info(mm->dmabuf_info); if (!list_empty(&mm->mmlist)) { spin_lock(&mmlist_lock); list_del(&mm->mmlist); @@ -1716,6 +1726,7 @@ static struct mm_struct *dup_mm(struct task_struct *tsk, goto fail_nomem; memcpy(mm, oldmm, sizeof(*mm)); + mm->dmabuf_info = NULL; if (!mm_init(mm, tsk, mm->user_ns)) goto fail_nomem; @@ -1834,7 +1845,7 @@ static int copy_files(unsigned long clone_flags, struct task_struct *tsk, return 0; } -static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) +static int copy_sighand(u64 clone_flags, struct task_struct *tsk) { struct sighand_struct *sig; @@ -2515,7 +2526,7 @@ __latent_entropy struct task_struct *copy_process( retval = perf_event_init_task(p, clone_flags); if (retval) - goto bad_fork_cleanup_policy; + goto bad_fork_sched_cancel_fork; retval = audit_alloc(p); if (retval) goto bad_fork_cleanup_perf; @@ -2628,6 +2639,12 @@ __latent_entropy struct task_struct *copy_process( p->rethooks.first = NULL; #endif + retval = copy_dmabuf_info(clone_flags, p); + if (retval) { + pr_err("failed to copy dmabuf accounting info, err %d\n", retval); + goto bad_fork_put_pidfd; + } + /* * Ensure that the cgroup subsystem policies allow the new process to be * forked. It should be noted that the new process's css_set can be changed @@ -2636,7 +2653,7 @@ __latent_entropy struct task_struct *copy_process( */ retval = cgroup_can_fork(p, args); if (retval) - goto bad_fork_put_pidfd; + goto bad_fork_cleanup_dmabuf; /* * Now that the cgroups are pinned, re-clone the parent cgroup and put @@ -2647,7 +2664,9 @@ __latent_entropy struct task_struct *copy_process( * cgroup specific, it unconditionally needs to place the task on a * runqueue. */ - sched_cgroup_fork(p, args); + retval = sched_cgroup_fork(p, args); + if (retval) + goto bad_fork_cancel_cgroup; /* * From this point on we must avoid any synchronous user-space @@ -2693,13 +2712,13 @@ __latent_entropy struct task_struct *copy_process( /* Don't start children in a dying pid namespace */ if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { retval = -ENOMEM; - goto bad_fork_cancel_cgroup; + goto bad_fork_core_free; } /* Let kill terminate clone/fork in the middle */ if (fatal_signal_pending(current)) { retval = -EINTR; - goto bad_fork_cancel_cgroup; + goto bad_fork_core_free; } /* No more failure paths after this point. */ @@ -2772,15 +2791,19 @@ __latent_entropy struct task_struct *copy_process( uprobe_copy_process(p, clone_flags); user_events_fork(p, clone_flags); + trace_android_vh_lock_task_fork(p); copy_oom_score_adj(clone_flags, p); return p; -bad_fork_cancel_cgroup: +bad_fork_core_free: sched_core_free(p); spin_unlock(¤t->sighand->siglock); write_unlock_irq(&tasklist_lock); +bad_fork_cancel_cgroup: cgroup_cancel_fork(p, args); +bad_fork_cleanup_dmabuf: + put_dmabuf_info(p->dmabuf_info); bad_fork_put_pidfd: if (clone_flags & CLONE_PIDFD) { fput(pidfile); @@ -2818,6 +2841,8 @@ __latent_entropy struct task_struct *copy_process( audit_free(p); bad_fork_cleanup_perf: perf_event_free_task(p); +bad_fork_sched_cancel_fork: + sched_cancel_fork(p); bad_fork_cleanup_policy: lockdep_free_task(p); #ifdef CONFIG_NUMA @@ -3568,6 +3593,17 @@ int unshare_files(void) old = task->files; task_lock(task); task->files = copy; + + /* + * This is a new partial sharing relationship for task, since we have a new + * files_struct (but the MM is still used). Since partial sharing is not + * supported for dmabuf accounting, we need to remove the accounting info + * from the task. Leave the mm->dmabuf_info so any existing accounting can + * be unaccounted properly. The fixup for this new files_struct happens + * externally with appropriate locking. + */ + put_dmabuf_info(task->dmabuf_info); + task->dmabuf_info = NULL; task_unlock(task); put_files_struct(old); return 0; diff --git a/kernel/freezer.c b/kernel/freezer.c index a576b00be9e47..0b181124b6d3f 100644 --- a/kernel/freezer.c +++ b/kernel/freezer.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #undef CREATE_TRACE_POINT @@ -43,7 +44,7 @@ bool freezing_slow_path(struct task_struct *p) if (p->flags & (PF_NOFREEZE | PF_SUSPEND_TASK)) return false; - if (test_tsk_thread_flag(p, TIF_MEMDIE)) + if (tsk_is_oom_victim(p)) return false; if (pm_nosig_freezing || cgroup_freezing(p)) @@ -200,18 +201,26 @@ static int __restore_freezer_state(struct task_struct *p, void *arg) void __thaw_task(struct task_struct *p) { - unsigned long flags; - - spin_lock_irqsave(&freezer_lock, flags); - if (WARN_ON_ONCE(freezing(p))) - goto unlock; + guard(spinlock_irqsave)(&freezer_lock); + if (frozen(p) && !task_call_func(p, __restore_freezer_state, NULL)) + wake_up_state(p, TASK_FROZEN); +} - if (!frozen(p) || task_call_func(p, __restore_freezer_state, NULL)) - goto unlock; +/* + * thaw_process - Thaw a frozen process + * @p: the process to be thawed + * + * Iterate over all threads of @p and call __thaw_task() on each. + */ +void thaw_process(struct task_struct *p) +{ + struct task_struct *t; - wake_up_state(p, TASK_FROZEN); -unlock: - spin_unlock_irqrestore(&freezer_lock, flags); + rcu_read_lock(); + for_each_thread(p, t) { + __thaw_task(t); + } + rcu_read_unlock(); } /** diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index cba8b1a6a4cc2..7e43839ca7b05 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -223,18 +223,20 @@ static inline void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, struct futex_hash_bucket *hb) { - q->key = *key; + struct task_struct *task; + q->key = *key; __futex_unqueue(q); WARN_ON(!q->rt_waiter); q->rt_waiter = NULL; q->lock_ptr = &hb->lock; + task = READ_ONCE(q->task); /* Signal locked state to the waiter */ futex_requeue_pi_complete(q, 1); - wake_up_state(q->task, TASK_NORMAL); + wake_up_state(task, TASK_NORMAL); } /** diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index e4970139ed5b5..a662afd8b1d2a 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -41,6 +41,56 @@ SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, return 0; } +static inline void __user *futex_task_robust_list(struct task_struct *p, bool compat) +{ +#ifdef CONFIG_COMPAT + if (compat) + return p->compat_robust_list; +#endif + return p->robust_list; +} + +static void __user *futex_get_robust_list_common(int pid, bool compat) +{ + struct task_struct *p = current; + void __user *head; + int ret; + + scoped_guard(rcu) { + if (pid) { + p = find_task_by_vpid(pid); + if (!p) + return (void __user *)ERR_PTR(-ESRCH); + } + get_task_struct(p); + } + + /* + * Hold exec_update_lock to serialize with concurrent exec() + * so ptrace_may_access() is checked against stable credentials + */ + ret = down_read_killable(&p->signal->exec_update_lock); + if (ret) + goto err_put; + + ret = -EPERM; + if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) + goto err_unlock; + + head = futex_task_robust_list(p, compat); + + up_read(&p->signal->exec_update_lock); + put_task_struct(p); + + return head; + +err_unlock: + up_read(&p->signal->exec_update_lock); +err_put: + put_task_struct(p); + return (void __user *)ERR_PTR(ret); +} + /** * sys_get_robust_list() - Get the robust-futex list head of a task * @pid: pid of the process [zero for current task] @@ -51,36 +101,14 @@ SYSCALL_DEFINE3(get_robust_list, int, pid, struct robust_list_head __user * __user *, head_ptr, size_t __user *, len_ptr) { - struct robust_list_head __user *head; - unsigned long ret; - struct task_struct *p; - - rcu_read_lock(); - - ret = -ESRCH; - if (!pid) - p = current; - else { - p = find_task_by_vpid(pid); - if (!p) - goto err_unlock; - } - - ret = -EPERM; - if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) - goto err_unlock; + struct robust_list_head __user *head = futex_get_robust_list_common(pid, false); - head = p->robust_list; - rcu_read_unlock(); + if (IS_ERR(head)) + return PTR_ERR(head); if (put_user(sizeof(*head), len_ptr)) return -EFAULT; return put_user(head, head_ptr); - -err_unlock: - rcu_read_unlock(); - - return ret; } long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, @@ -324,36 +352,14 @@ COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid, compat_uptr_t __user *, head_ptr, compat_size_t __user *, len_ptr) { - struct compat_robust_list_head __user *head; - unsigned long ret; - struct task_struct *p; - - rcu_read_lock(); - - ret = -ESRCH; - if (!pid) - p = current; - else { - p = find_task_by_vpid(pid); - if (!p) - goto err_unlock; - } - - ret = -EPERM; - if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) - goto err_unlock; + struct compat_robust_list_head __user *head = futex_get_robust_list_common(pid, true); - head = p->compat_robust_list; - rcu_read_unlock(); + if (IS_ERR(head)) + return PTR_ERR(head); if (put_user(sizeof(*head), len_ptr)) return -EFAULT; return put_user(ptr_to_compat(head), head_ptr); - -err_unlock: - rcu_read_unlock(); - - return ret; } #endif /* CONFIG_COMPAT */ diff --git a/kernel/gcov/gcc_4_7.c b/kernel/gcov/gcc_4_7.c index fd75b4a484d76..bbccbae331d72 100644 --- a/kernel/gcov/gcc_4_7.c +++ b/kernel/gcov/gcc_4_7.c @@ -18,7 +18,9 @@ #include #include "gcov.h" -#if (__GNUC__ >= 14) +#if (__GNUC__ >= 15) +#define GCOV_COUNTERS 10 +#elif (__GNUC__ >= 14) #define GCOV_COUNTERS 9 #elif (__GNUC__ >= 10) #define GCOV_COUNTERS 8 diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 6df0f0c839578..12adc178635e0 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -224,6 +224,27 @@ __irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff, return IRQ_STARTUP_ABORT; return IRQ_STARTUP_MANAGED; } + +void irq_startup_managed(struct irq_desc *desc) +{ + struct irq_data *d = irq_desc_get_irq_data(desc); + + /* + * Clear managed-shutdown flag, so we don't repeat managed-startup for + * multiple hotplugs, and cause imbalanced disable depth. + */ + irqd_clr_managed_shutdown(d); + + /* + * Only start it up when the disable depth is 1, so that a disable, + * hotunplug, hotplug sequence does not end up enabling it during + * hotplug unconditionally. + */ + desc->depth--; + if (!desc->depth) + irq_startup(desc, IRQ_RESEND, IRQ_START_COND); +} + #else static __always_inline int __irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff, diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index eb86283901565..4a1c63d0901ca 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -211,13 +211,6 @@ static void irq_restore_affinity_of_irq(struct irq_desc *desc, unsigned int cpu) !irq_data_get_irq_chip(data) || !cpumask_test_cpu(cpu, affinity)) return; - /* - * Don't restore suspended interrupts here when a system comes back - * from S3. They are reenabled via resume_device_irqs(). - */ - if (desc->istate & IRQS_SUSPENDED) - return; - if (irqd_is_managed_and_shutdown(data)) irq_startup(desc, IRQ_RESEND, IRQ_START_COND); diff --git a/kernel/kcsan/kcsan_test.c b/kernel/kcsan/kcsan_test.c index 0ddbdab5903df..9d8c95defdd67 100644 --- a/kernel/kcsan/kcsan_test.c +++ b/kernel/kcsan/kcsan_test.c @@ -530,7 +530,7 @@ static void test_barrier_nothreads(struct kunit *test) struct kcsan_scoped_access *reorder_access = NULL; #endif arch_spinlock_t arch_spinlock = __ARCH_SPIN_LOCK_UNLOCKED; - atomic_t dummy; + atomic_t dummy = ATOMIC_INIT(0); KCSAN_TEST_REQUIRES(test, reorder_access != NULL); KCSAN_TEST_REQUIRES(test, IS_ENABLED(CONFIG_SMP)); diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index b7246b7171b73..88024cb22a9db 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1132,7 +1132,7 @@ static int __crash_shrink_memory(struct resource *old_res, old_res->start = 0; old_res->end = 0; } else { - crashk_res.end = ram_res->start - 1; + old_res->end = ram_res->start - 1; } crash_free_reserved_phys_range(ram_res->start, ram_res->end); diff --git a/kernel/locking/locking_main.h b/kernel/locking/locking_main.h new file mode 120000 index 0000000000000..7b0aab795f945 --- /dev/null +++ b/kernel/locking/locking_main.h @@ -0,0 +1 @@ +../../../../vendor/oplus/kernel/synchronize/oplus_locking.h \ No newline at end of file diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c old mode 100644 new mode 100755 diff --git a/kernel/locking/oplus_locking.c b/kernel/locking/oplus_locking.c new file mode 120000 index 0000000000000..bf5d91689ec8c --- /dev/null +++ b/kernel/locking/oplus_locking.c @@ -0,0 +1 @@ +../../../../vendor/oplus/kernel/synchronize/oplus_locking.c \ No newline at end of file diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c old mode 100644 new mode 100755 index fab181e95e8c4..b49b437e57bad --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -10,7 +10,6 @@ #include #include #include - #include /* diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c old mode 100644 new mode 100755 diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c old mode 100644 new mode 100755 index 0639e49836d35..3f433da37dcd7 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 + // SPDX-License-Identifier: GPL-2.0 /* kernel/rwsem.c: R/W semaphores, public implementation * * Written by David Howells (dhowells@redhat.com). diff --git a/kernel/locking/sa_common_struct.h b/kernel/locking/sa_common_struct.h new file mode 120000 index 0000000000000..995033021b284 --- /dev/null +++ b/kernel/locking/sa_common_struct.h @@ -0,0 +1 @@ +../../../../vendor/oplus/kernel/cpu/sched/sched_assist/sa_common_struct.h \ No newline at end of file diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index 8475a0794f8c5..438c6086d540e 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -413,3 +413,11 @@ notrace int in_lock_functions(unsigned long addr) && addr < (unsigned long)__lock_text_end; } EXPORT_SYMBOL(in_lock_functions); + +#if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_PREEMPT_RT) +void notrace lockdep_assert_in_softirq_func(void) +{ + lockdep_assert_in_softirq(); +} +EXPORT_SYMBOL(lockdep_assert_in_softirq_func); +#endif diff --git a/kernel/module/main.c b/kernel/module/main.c index ea1b4e1163898..a1f9ba300bd82 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -705,14 +705,16 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user, struct module *mod; char name[MODULE_NAME_LEN]; char buf[MODULE_FLAGS_BUF_SIZE]; - int ret, forced = 0; + int ret, len, forced = 0; if (!capable(CAP_SYS_MODULE) || modules_disabled) return -EPERM; - if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0) - return -EFAULT; - name[MODULE_NAME_LEN-1] = '\0'; + len = strncpy_from_user(name, name_user, MODULE_NAME_LEN); + if (len == 0 || len == MODULE_NAME_LEN) + return -ENOENT; + if (len < 0) + return len; audit_log_kern_module(name); diff --git a/kernel/padata.c b/kernel/padata.c index 071d8cad80787..9260ab0b39eb5 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -290,7 +290,11 @@ static struct padata_priv *padata_find_next(struct parallel_data *pd, if (remove_object) { list_del_init(&padata->list); ++pd->processed; - pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false); + /* When sequence wraps around, reset to the first CPU. */ + if (unlikely(pd->processed == 0)) + pd->cpu = cpumask_first(pd->cpumask.pcpu); + else + pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false); } spin_unlock(&reorder->lock); @@ -358,7 +362,8 @@ static void padata_reorder(struct parallel_data *pd) * To avoid UAF issue, add pd ref here, and put pd ref after reorder_work finish. */ padata_get_pd(pd); - queue_work(pinst->serial_wq, &pd->reorder_work); + if (!queue_work(pinst->serial_wq, &pd->reorder_work)) + padata_put_pd(pd); } } diff --git a/kernel/params.c b/kernel/params.c index 2d4a0564697e8..e39ac5420cd6d 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -759,7 +759,7 @@ void destroy_params(const struct kernel_param *params, unsigned num) params[i].ops->free(params[i].arg); } -static struct module_kobject * __init locate_module_kobject(const char *name) +struct module_kobject __modinit * lookup_or_create_module_kobject(const char *name) { struct module_kobject *mk; struct kobject *kobj; @@ -801,7 +801,7 @@ static void __init kernel_add_sysfs_param(const char *name, struct module_kobject *mk; int err; - mk = locate_module_kobject(name); + mk = lookup_or_create_module_kobject(name); if (!mk) return; @@ -872,7 +872,7 @@ static void __init version_sysfs_builtin(void) int err; for (vattr = __start___modver; vattr < __stop___modver; vattr++) { - mk = locate_module_kobject(vattr->module_name); + mk = lookup_or_create_module_kobject(vattr->module_name); if (mk) { err = sysfs_create_file(&mk->kobj, &vattr->mattr.attr); WARN_ON_ONCE(err); @@ -945,7 +945,9 @@ struct kset *module_kset; static void module_kobj_release(struct kobject *kobj) { struct module_kobject *mk = to_module_kobject(kobj); - complete(mk->kobj_completion); + + if (mk->kobj_completion) + complete(mk->kobj_completion); } const struct kobj_type module_ktype = { diff --git a/kernel/pid.c b/kernel/pid.c index 0c5e5455d71a8..8f7d1ef0d739a 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -478,7 +478,7 @@ pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns) struct upid *upid; pid_t nr = 0; - if (pid && ns->level <= pid->level) { + if (pid && ns && ns->level <= pid->level) { upid = &pid->numbers[ns->level]; if (upid->ns == ns) nr = upid->nr; @@ -580,6 +580,7 @@ struct task_struct *pidfd_get_task(int pidfd, unsigned int *flags) *flags = f_flags; return task; } +EXPORT_SYMBOL_GPL(pidfd_get_task); /** * pidfd_create() - Create a new pid file descriptor. diff --git a/kernel/power/console.c b/kernel/power/console.c index fcdf0e14a47d4..19c48aa5355d2 100644 --- a/kernel/power/console.c +++ b/kernel/power/console.c @@ -16,6 +16,7 @@ #define SUSPEND_CONSOLE (MAX_NR_CONSOLES-1) static int orig_fgconsole, orig_kmsg; +static bool vt_switch_done; static DEFINE_MUTEX(vt_switch_mutex); @@ -136,17 +137,21 @@ void pm_prepare_console(void) if (orig_fgconsole < 0) return; + vt_switch_done = true; + orig_kmsg = vt_kmsg_redirect(SUSPEND_CONSOLE); return; } void pm_restore_console(void) { - if (!pm_vt_switch()) + if (!pm_vt_switch() && !vt_switch_done) return; if (orig_fgconsole >= 0) { vt_move_to_console(orig_fgconsole, 0); vt_kmsg_redirect(orig_kmsg); } + + vt_switch_done = false; } diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index c2fc58938dee5..56e505ca9ddab 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -80,6 +80,11 @@ void hibernate_release(void) atomic_inc(&hibernate_atomic); } +bool hibernation_in_progress(void) +{ + return !atomic_read(&hibernate_atomic); +} + bool hibernation_available(void) { return nohibernate == 0 && @@ -744,7 +749,9 @@ int hibernate(void) if (error) goto Restore; - ksys_sync_helper(); + error = pm_sleep_fs_sync(); + if (error) + goto Exit; error = freeze_processes(); if (error) diff --git a/kernel/power/main.c b/kernel/power/main.c index f6425ae3e8b05..968740d84b750 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include "power.h" @@ -95,6 +97,61 @@ void ksys_sync_helper(void) } EXPORT_SYMBOL_GPL(ksys_sync_helper); +#if defined(CONFIG_SUSPEND) || defined(CONFIG_HIBERNATION) +/* Wakeup events handling resolution while syncing file systems in jiffies */ +#define PM_FS_SYNC_WAKEUP_RESOLUTION 5 + +static atomic_t pm_fs_sync_count = ATOMIC_INIT(0); +static struct workqueue_struct *pm_fs_sync_wq; +static DECLARE_WAIT_QUEUE_HEAD(pm_fs_sync_wait); + +static bool pm_fs_sync_completed(void) +{ + return atomic_read(&pm_fs_sync_count) == 0; +} + +static void pm_fs_sync_work_fn(struct work_struct *work) +{ + ksys_sync_helper(); + + if (atomic_dec_and_test(&pm_fs_sync_count)) + wake_up(&pm_fs_sync_wait); +} +static DECLARE_WORK(pm_fs_sync_work, pm_fs_sync_work_fn); + +/** + * pm_sleep_fs_sync() - Sync file systems in an interruptible way + * + * Return: 0 on successful file system sync, or -EBUSY if the file system sync + * was aborted. + */ +int pm_sleep_fs_sync(void) +{ + pm_wakeup_clear(0); + + /* + * Take back-to-back sleeps into account by queuing a subsequent fs sync + * only if the previous fs sync is running or is not queued. Multiple fs + * syncs increase the likelihood of saving the latest files immediately + * before sleep. + */ + if (!work_pending(&pm_fs_sync_work)) { + atomic_inc(&pm_fs_sync_count); + queue_work(pm_fs_sync_wq, &pm_fs_sync_work); + } + + while (!pm_fs_sync_completed()) { + if (pm_wakeup_pending()) + return -EBUSY; + + wait_event_timeout(pm_fs_sync_wait, pm_fs_sync_completed(), + PM_FS_SYNC_WAKEUP_RESOLUTION); + } + + return 0; +} +#endif /* CONFIG_SUSPEND || CONFIG_HIBERNATION */ + /* Routines for PM-transition notifications */ static BLOCKING_NOTIFIER_HEAD(pm_chain_head); @@ -239,10 +296,10 @@ static ssize_t mem_sleep_store(struct kobject *kobj, struct kobj_attribute *attr power_attr(mem_sleep); /* - * sync_on_suspend: invoke ksys_sync_helper() before suspend. + * sync_on_suspend: Sync file systems before suspend. * - * show() returns whether ksys_sync_helper() is invoked before suspend. - * store() accepts 0 or 1. 0 disables ksys_sync_helper() and 1 enables it. + * show() returns whether file systems sync before suspend is enabled. + * store() accepts 0 or 1. 0 disables file systems sync and 1 enables it. */ bool sync_on_suspend_enabled = !IS_ENABLED(CONFIG_SUSPEND_SKIP_SYNC); @@ -585,7 +642,8 @@ bool pm_debug_messages_on __read_mostly; bool pm_debug_messages_should_print(void) { - return pm_debug_messages_on && pm_suspend_target_state != PM_SUSPEND_ON; + return pm_debug_messages_on && (hibernation_in_progress() || + pm_suspend_target_state != PM_SUSPEND_ON); } EXPORT_SYMBOL_GPL(pm_debug_messages_should_print); @@ -980,16 +1038,26 @@ static const struct attribute_group *attr_groups[] = { struct workqueue_struct *pm_wq; EXPORT_SYMBOL_GPL(pm_wq); -static int __init pm_start_workqueue(void) +static int __init pm_start_workqueues(void) { pm_wq = alloc_workqueue("pm", WQ_FREEZABLE, 0); + if (!pm_wq) + return -ENOMEM; - return pm_wq ? 0 : -ENOMEM; +#if defined(CONFIG_SUSPEND) || defined(CONFIG_HIBERNATION) + pm_fs_sync_wq = alloc_ordered_workqueue("pm_fs_sync", 0); + if (!pm_fs_sync_wq) { + destroy_workqueue(pm_wq); + return -ENOMEM; + } +#endif + + return 0; } static int __init pm_init(void) { - int error = pm_start_workqueue(); + int error = pm_start_workqueues(); if (error) return error; hibernate_image_size_init(); diff --git a/kernel/power/power.h b/kernel/power/power.h index a98f95e309a33..331b3681693a5 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -17,6 +17,10 @@ struct swsusp_info { unsigned long size; } __aligned(PAGE_SIZE); +#if defined(CONFIG_SUSPEND) || defined(CONFIG_HIBERNATION) +extern int pm_sleep_fs_sync(void); +#endif + #ifdef CONFIG_HIBERNATION /* kernel/power/snapshot.c */ extern void __init hibernate_reserved_size_init(void); @@ -66,10 +70,14 @@ extern void enable_restore_image_protection(void); static inline void enable_restore_image_protection(void) {} #endif /* CONFIG_STRICT_KERNEL_RWX */ +extern bool hibernation_in_progress(void); + #else /* !CONFIG_HIBERNATION */ static inline void hibernate_reserved_size_init(void) {} static inline void hibernate_image_size_init(void) {} + +static inline bool hibernation_in_progress(void) { return false; } #endif /* !CONFIG_HIBERNATION */ #define power_attr(_name) \ diff --git a/kernel/power/process.c b/kernel/power/process.c index 7e517bb0ff28c..31f3413d9bed1 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -39,6 +39,7 @@ static int try_to_freeze_tasks(bool user_only) unsigned int elapsed_msecs; bool wakeup = false; int sleep_usecs = USEC_PER_MSEC; + bool todo_logging_on = false; pr_info("Freezing %s\n", what); @@ -99,7 +100,8 @@ static int try_to_freeze_tasks(bool user_only) if (wq_busy) show_freezable_workqueues(); - if (pm_debug_messages_on) { + trace_android_vh_try_to_freeze_todo_logging(&todo_logging_on); + if (pm_debug_messages_on || todo_logging_on) { read_lock(&tasklist_lock); for_each_process_thread(g, p) { if (p != current && freezing(p) && !frozen(p)) { diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 8972ebab0461e..3324d469dcf22 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -599,7 +599,11 @@ static int enter_state(suspend_state_t state) if (sync_on_suspend_enabled) { trace_suspend_resume(TPS("sync_filesystems"), 0, true); - ksys_sync_helper(); + + error = pm_sleep_fs_sync(); + if (error) + goto Unlock; + trace_suspend_resume(TPS("sync_filesystems"), 0, false); } diff --git a/kernel/power/user.c b/kernel/power/user.c index 3a4e70366f354..8ce24be24e092 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -278,7 +278,9 @@ static long snapshot_ioctl(struct file *filp, unsigned int cmd, if (data->frozen) break; - ksys_sync_helper(); + error = pm_sleep_fs_sync(); + if (error) + break; error = freeze_processes(); if (error) diff --git a/kernel/power/wakelock.c b/kernel/power/wakelock.c index 52571dcad768b..4e941999a53ba 100644 --- a/kernel/power/wakelock.c +++ b/kernel/power/wakelock.c @@ -49,6 +49,9 @@ ssize_t pm_show_wakelocks(char *buf, bool show_active) len += sysfs_emit_at(buf, len, "%s ", wl->name); } + if (len > 0) + --len; + len += sysfs_emit_at(buf, len, "\n"); mutex_unlock(&wakelocks_lock); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index e0ae6a2f16744..f66ce94b3c72c 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1128,6 +1128,17 @@ static unsigned int __init add_to_rb(struct printk_ringbuffer *rb, static char setup_text_buf[PRINTKRB_RECORD_MAX] __initdata; +static void print_log_buf_usage_stats(void) +{ + unsigned int descs_count = log_buf_len >> PRB_AVGBITS; + size_t meta_data_size; + + meta_data_size = descs_count * (sizeof(struct prb_desc) + sizeof(struct printk_info)); + + pr_info("log buffer data + meta data: %u + %zu = %zu bytes\n", + log_buf_len, meta_data_size, log_buf_len + meta_data_size); +} + void __init setup_log_buf(int early) { struct printk_info *new_infos; @@ -1157,20 +1168,25 @@ void __init setup_log_buf(int early) if (!early && !new_log_buf_len) log_buf_add_cpu(); - if (!new_log_buf_len) + if (!new_log_buf_len) { + /* Show the memory stats only once. */ + if (!early) + goto out; + return; + } new_descs_count = new_log_buf_len >> PRB_AVGBITS; if (new_descs_count == 0) { pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len); - return; + goto out; } new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN); if (unlikely(!new_log_buf)) { pr_err("log_buf_len: %lu text bytes not available\n", new_log_buf_len); - return; + goto out; } new_descs_size = new_descs_count * sizeof(struct prb_desc); @@ -1233,7 +1249,7 @@ void __init setup_log_buf(int early) prb_next_seq(&printk_rb_static) - seq); } - pr_info("log_buf_len: %u bytes\n", log_buf_len); + print_log_buf_usage_stats(); pr_info("early log buf free: %u(%u%%)\n", free, (free * 100) / __LOG_BUF_LEN); return; @@ -1242,6 +1258,8 @@ void __init setup_log_buf(int early) memblock_free(new_descs, new_descs_size); err_free_log_buf: memblock_free(new_log_buf, new_log_buf_len); +out: + print_log_buf_usage_stats(); } static bool __read_mostly ignore_loglevel; @@ -3145,7 +3163,12 @@ void console_unblank(void) */ cookie = console_srcu_read_lock(); for_each_console_srcu(c) { - if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank) { + short flags = console_srcu_read_flags(c); + + if (flags & CON_SUSPENDED) + continue; + + if ((flags & CON_ENABLED) && c->unblank) { found_unblank = true; break; } @@ -3182,7 +3205,12 @@ void console_unblank(void) cookie = console_srcu_read_lock(); for_each_console_srcu(c) { - if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank) + short flags = console_srcu_read_flags(c); + + if (flags & CON_SUSPENDED) + continue; + + if ((flags & CON_ENABLED) && c->unblank) c->unblank(); } console_srcu_read_unlock(cookie); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index d6c7a3969cfd8..8be451d32fe1a 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -754,6 +754,10 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp) return 0; } +#ifndef arch_irq_stat_cpu +#define arch_irq_stat_cpu(cpu) 0 +#endif + /* * Returns positive if the specified CPU has passed through a quiescent state * by virtue of being in or having passed through an dynticks idle state since @@ -889,9 +893,9 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) rsrp->cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu); rsrp->cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); rsrp->cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); - rsrp->nr_hardirqs = kstat_cpu_irqs_sum(rdp->cpu); - rsrp->nr_softirqs = kstat_cpu_softirqs_sum(rdp->cpu); - rsrp->nr_csw = nr_context_switches_cpu(rdp->cpu); + rsrp->nr_hardirqs = kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu); + rsrp->nr_softirqs = kstat_cpu_softirqs_sum(cpu); + rsrp->nr_csw = nr_context_switches_cpu(cpu); rsrp->jiffies = jiffies; rsrp->gp_seq = rdp->gp_seq; } @@ -2695,6 +2699,10 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) /* Misaligned rcu_head! */ WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); + /* Avoid NULL dereference if callback is NULL. */ + if (WARN_ON_ONCE(!func)) + return; + if (debug_rcu_head_queue(head)) { /* * Probable double call_rcu(), so leak the callback. @@ -4424,6 +4432,8 @@ int rcutree_prepare_cpu(unsigned int cpu) rdp->rcu_iw_gp_seq = rdp->gp_seq - 1; trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); raw_spin_unlock_irqrestore_rcu_node(rnp, flags); + + rcu_preempt_deferred_qs_init(rdp); rcu_spawn_one_boost_kthread(rnp); rcu_spawn_cpu_nocb_kthread(cpu); WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 9eb43b501ff5c..71403d22a8465 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -169,12 +169,23 @@ struct rcu_snap_record { u64 cputime_irq; /* Accumulated cputime of hard irqs */ u64 cputime_softirq;/* Accumulated cputime of soft irqs */ u64 cputime_system; /* Accumulated cputime of kernel tasks */ - unsigned long nr_hardirqs; /* Accumulated number of hard irqs */ + u64 nr_hardirqs; /* Accumulated number of hard irqs */ unsigned int nr_softirqs; /* Accumulated number of soft irqs */ unsigned long long nr_csw; /* Accumulated number of task switches */ unsigned long jiffies; /* Track jiffies value */ }; +/* + * An IRQ work (deferred_qs_iw) is used by RCU to get the scheduler's attention. + * to report quiescent states at the soonest possible time. + * The request can be in one of the following states: + * - DEFER_QS_IDLE: An IRQ work is yet to be scheduled. + * - DEFER_QS_PENDING: An IRQ work was scheduled but either not yet run, or it + * ran and we still haven't reported a quiescent state. + */ +#define DEFER_QS_IDLE 0 +#define DEFER_QS_PENDING 1 + /* Per-CPU data for read-copy update. */ struct rcu_data { /* 1) quiescent-state and grace-period handling : */ @@ -192,7 +203,7 @@ struct rcu_data { /* during and after the last grace */ /* period it is aware of. */ struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ - bool defer_qs_iw_pending; /* Scheduler attention pending? */ + int defer_qs_iw_pending; /* Scheduler attention pending? */ struct work_struct strict_work; /* Schedule readers for strict GPs. */ /* 2) batch handling */ @@ -452,6 +463,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp); static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); static void rcu_flavor_sched_clock_irq(int user); static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck); +static void rcu_preempt_deferred_qs_init(struct rcu_data *rdp); static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); static bool rcu_is_callbacks_kthread(struct rcu_data *rdp); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 41021080ad258..8707f155afb6d 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -474,13 +474,16 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) struct rcu_node *rnp; union rcu_special special; + rdp = this_cpu_ptr(&rcu_data); + if (rdp->defer_qs_iw_pending == DEFER_QS_PENDING) + rdp->defer_qs_iw_pending = DEFER_QS_IDLE; + /* * If RCU core is waiting for this CPU to exit its critical section, * report the fact that it has exited. Because irqs are disabled, * t->rcu_read_unlock_special cannot change. */ special = t->rcu_read_unlock_special; - rdp = this_cpu_ptr(&rcu_data); if (!special.s && !rdp->cpu_no_qs.b.exp) { local_irq_restore(flags); return; @@ -612,10 +615,29 @@ notrace void rcu_preempt_deferred_qs(struct task_struct *t) */ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) { + unsigned long flags; struct rcu_data *rdp; rdp = container_of(iwp, struct rcu_data, defer_qs_iw); - rdp->defer_qs_iw_pending = false; + local_irq_save(flags); + + /* + * If the IRQ work handler happens to run in the middle of RCU read-side + * critical section, it could be ineffective in getting the scheduler's + * attention to report a deferred quiescent state (the whole point of the + * IRQ work). For this reason, requeue the IRQ work. + * + * Basically, we want to avoid following situation: + * 1. rcu_read_unlock() queues IRQ work (state -> DEFER_QS_PENDING) + * 2. CPU enters new rcu_read_lock() + * 3. IRQ work runs but cannot report QS due to rcu_preempt_depth() > 0 + * 4. rcu_read_unlock() does not re-queue work (state still PENDING) + * 5. Deferred QS reporting does not happen. + */ + if (rcu_preempt_depth() > 0) + WRITE_ONCE(rdp->defer_qs_iw_pending, DEFER_QS_IDLE); + + local_irq_restore(flags); } /* @@ -661,17 +683,11 @@ static void rcu_read_unlock_special(struct task_struct *t) set_tsk_need_resched(current); set_preempt_need_resched(); if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled && - expboost && !rdp->defer_qs_iw_pending && cpu_online(rdp->cpu)) { + expboost && rdp->defer_qs_iw_pending != DEFER_QS_PENDING && + cpu_online(rdp->cpu)) { // Get scheduler to re-evaluate and call hooks. // If !IRQ_WORK, FQS scan will eventually IPI. - if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) && - IS_ENABLED(CONFIG_PREEMPT_RT)) - rdp->defer_qs_iw = IRQ_WORK_INIT_HARD( - rcu_preempt_deferred_qs_handler); - else - init_irq_work(&rdp->defer_qs_iw, - rcu_preempt_deferred_qs_handler); - rdp->defer_qs_iw_pending = true; + rdp->defer_qs_iw_pending = DEFER_QS_PENDING; irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); } } @@ -810,6 +826,10 @@ dump_blkd_tasks(struct rcu_node *rnp, int ncheck) } } +static void rcu_preempt_deferred_qs_init(struct rcu_data *rdp) +{ + rdp->defer_qs_iw = IRQ_WORK_INIT_HARD(rcu_preempt_deferred_qs_handler); +} #else /* #ifdef CONFIG_PREEMPT_RCU */ /* @@ -821,8 +841,17 @@ void rcu_read_unlock_strict(void) { struct rcu_data *rdp; - if (irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) + if (irqs_disabled() || in_atomic_preempt_off() || !rcu_state.gp_kthread) return; + + /* + * rcu_report_qs_rdp() can only be invoked with a stable rdp and + * from the local CPU. + * + * The in_atomic_preempt_off() check ensures that we come here holding + * the last preempt_count (which will get dropped once we return to + * __rcu_read_unlock(). + */ rdp = this_cpu_ptr(&rcu_data); rdp->cpu_no_qs.b.norm = false; rcu_report_qs_rdp(rdp); @@ -963,13 +992,16 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) */ static void rcu_flavor_sched_clock_irq(int user) { - if (user || rcu_is_cpu_rrupt_from_idle()) { + if (user || rcu_is_cpu_rrupt_from_idle() || + (IS_ENABLED(CONFIG_PREEMPT_COUNT) && + (preempt_count() == HARDIRQ_OFFSET))) { /* * Get here if this CPU took its interrupt from user - * mode or from the idle loop, and if this is not a - * nested interrupt. In this case, the CPU is in - * a quiescent state, so note it. + * mode, from the idle loop without this being a nested + * interrupt, or while not holding the task preempt count + * (with PREEMPT_COUNT=y). In this case, the CPU is in a + * quiescent state, so note it. * * No memory barrier is required here because rcu_qs() * references only CPU-local variables that other CPUs @@ -997,6 +1029,8 @@ dump_blkd_tasks(struct rcu_node *rnp, int ncheck) WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks)); } +static void rcu_preempt_deferred_qs_init(struct rcu_data *rdp) { } + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 11a1fac3a5898..aab91040b83b1 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -452,8 +452,8 @@ static void print_cpu_stat_info(int cpu) rsr.cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); pr_err("\t hardirqs softirqs csw/system\n"); - pr_err("\t number: %8ld %10d %12lld\n", - kstat_cpu_irqs_sum(cpu) - rsrp->nr_hardirqs, + pr_err("\t number: %8lld %10d %12lld\n", + kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu) - rsrp->nr_hardirqs, kstat_cpu_softirqs_sum(cpu) - rsrp->nr_softirqs, nr_context_switches_cpu(cpu) - rsrp->nr_csw); pr_err("\tcputime: %8lld %10lld %12lld ==> %d(ms)\n", diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 2490b4a86d754..95d530ff55f53 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -49,6 +49,8 @@ #include "rcu.h" +EXPORT_TRACEPOINT_SYMBOL_GPL(rcu_stall_warning); + #ifdef MODULE_PARAM_PREFIX #undef MODULE_PARAM_PREFIX #endif @@ -595,7 +597,8 @@ EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read); do { } while (0) #endif -#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) +#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) \ + || IS_ENABLED(CONFIG_GKI_HIDDEN_RCUTORTURE) /* Get rcutorture access to sched_setaffinity(). */ long rcutorture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask) { diff --git a/kernel/resource.c b/kernel/resource.c index 635e858db0fe8..6506839f8a811 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -1222,8 +1222,9 @@ static int __request_region_locked(struct resource *res, struct resource *parent * become unavailable to other users. Conflicts are * not expected. Warn to aid debugging if encountered. */ - if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) { - pr_warn("Unaddressable device %s %pR conflicts with %pR", + if (parent == &iomem_resource && + conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) { + pr_warn("Unaddressable device %s %pR conflicts with %pR\n", conflict->name, conflict, res); } if (conflict != parent) { diff --git a/kernel/rseq.c b/kernel/rseq.c index 9de6e35fe6791..810005f927d7c 100644 --- a/kernel/rseq.c +++ b/kernel/rseq.c @@ -149,6 +149,29 @@ static int rseq_reset_rseq_cpu_node_id(struct task_struct *t) return 0; } +/* + * Get the user-space pointer value stored in the 'rseq_cs' field. + */ +static int rseq_get_rseq_cs_ptr_val(struct rseq __user *rseq, u64 *rseq_cs) +{ + if (!rseq_cs) + return -EFAULT; + +#ifdef CONFIG_64BIT + if (get_user(*rseq_cs, &rseq->rseq_cs)) + return -EFAULT; +#else + if (copy_from_user(rseq_cs, &rseq->rseq_cs, sizeof(*rseq_cs))) + return -EFAULT; +#endif + + return 0; +} + +/* + * If the rseq_cs field of 'struct rseq' contains a valid pointer to + * user-space, copy 'struct rseq_cs' from user-space and validate its fields. + */ static int rseq_get_rseq_cs(struct task_struct *t, struct rseq_cs *rseq_cs) { struct rseq_cs __user *urseq_cs; @@ -157,17 +180,16 @@ static int rseq_get_rseq_cs(struct task_struct *t, struct rseq_cs *rseq_cs) u32 sig; int ret; -#ifdef CONFIG_64BIT - if (get_user(ptr, &t->rseq->rseq_cs)) - return -EFAULT; -#else - if (copy_from_user(&ptr, &t->rseq->rseq_cs, sizeof(ptr))) - return -EFAULT; -#endif + ret = rseq_get_rseq_cs_ptr_val(t->rseq, &ptr); + if (ret) + return ret; + + /* If the rseq_cs pointer is NULL, return a cleared struct rseq_cs. */ if (!ptr) { memset(rseq_cs, 0, sizeof(*rseq_cs)); return 0; } + /* Check that the pointer value fits in the user-space process space. */ if (ptr >= TASK_SIZE) return -EINVAL; urseq_cs = (struct rseq_cs __user *)(unsigned long)ptr; @@ -233,17 +255,17 @@ static int rseq_need_restart(struct task_struct *t, u32 cs_flags) /* * Load and clear event mask atomically with respect to - * scheduler preemption. + * scheduler preemption and membarrier IPIs. */ - preempt_disable(); - event_mask = t->rseq_event_mask; - t->rseq_event_mask = 0; - preempt_enable(); + scoped_guard(RSEQ_EVENT_GUARD) { + event_mask = t->rseq_event_mask; + t->rseq_event_mask = 0; + } return !!event_mask; } -static int clear_rseq_cs(struct task_struct *t) +static int clear_rseq_cs(struct rseq __user *rseq) { /* * The rseq_cs field is set to NULL on preemption or signal @@ -254,9 +276,9 @@ static int clear_rseq_cs(struct task_struct *t) * Set rseq_cs to NULL. */ #ifdef CONFIG_64BIT - return put_user(0UL, &t->rseq->rseq_cs); + return put_user(0UL, &rseq->rseq_cs); #else - if (clear_user(&t->rseq->rseq_cs, sizeof(t->rseq->rseq_cs))) + if (clear_user(&rseq->rseq_cs, sizeof(rseq->rseq_cs))) return -EFAULT; return 0; #endif @@ -288,11 +310,11 @@ static int rseq_ip_fixup(struct pt_regs *regs) * Clear the rseq_cs pointer and return. */ if (!in_rseq_cs(ip, &rseq_cs)) - return clear_rseq_cs(t); + return clear_rseq_cs(t->rseq); ret = rseq_need_restart(t, rseq_cs.flags); if (ret <= 0) return ret; - ret = clear_rseq_cs(t); + ret = clear_rseq_cs(t->rseq); if (ret) return ret; trace_rseq_ip_fixup(ip, rseq_cs.start_ip, rseq_cs.post_commit_offset, @@ -366,6 +388,7 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32, sig) { int ret; + u64 rseq_cs; if (flags & RSEQ_FLAG_UNREGISTER) { if (flags & ~RSEQ_FLAG_UNREGISTER) @@ -420,6 +443,19 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, return -EINVAL; if (!access_ok(rseq, rseq_len)) return -EFAULT; + + /* + * If the rseq_cs pointer is non-NULL on registration, clear it to + * avoid a potential segfault on return to user-space. The proper thing + * to do would have been to fail the registration but this would break + * older libcs that reuse the rseq area for new threads without + * clearing the fields. + */ + if (rseq_get_rseq_cs_ptr_val(rseq, &rseq_cs)) + return -EFAULT; + if (rseq_cs && clear_rseq_cs(rseq)) + return -EFAULT; + current->rseq = rseq; current->rseq_len = rseq_len; current->rseq_sig = sig; diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c old mode 100644 new mode 100755 index d9dc9ab3773f2..005025f55beaa --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include @@ -52,3 +54,6 @@ #include "cputime.c" #include "deadline.c" +#ifdef CONFIG_SCHED_CLASS_EXT +# include "ext.c" +#endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c old mode 100644 new mode 100755 index e3e17a54c71fa..b49273faf0fdb --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -170,6 +170,7 @@ __read_mostly int scheduler_running; DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); + /* kernel prio, less is more */ static inline int __task_prio(const struct task_struct *p) { @@ -182,7 +183,12 @@ static inline int __task_prio(const struct task_struct *p) if (p->sched_class == &idle_sched_class) return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ - return MAX_RT_PRIO + MAX_NICE; /* 120, squash fair */ +#ifdef CONFIG_SCHED_CLASS_EXT + if (p->sched_class == &ext_sched_class) + return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ +#endif + + return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ } /* @@ -211,6 +217,11 @@ static inline bool prio_less(const struct task_struct *a, if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ return cfs_prio_less(a, b, in_fi); +#ifdef CONFIG_SCHED_CLASS_EXT + if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ + return scx_prio_less(a, b, in_fi); +#endif + return false; } @@ -1269,11 +1280,14 @@ bool sched_can_stop_tick(struct rq *rq) return true; /* - * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; - * if there's more than one we need the tick for involuntary - * preemption. + * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks + * left. For CFS, if there's more than one we need the tick for + * involuntary preemption. For SCX, ask. */ - if (rq->nr_running > 1) + if (!scx_switched_all() && rq->nr_running > 1) + return false; + + if (scx_enabled() && !scx_can_stop_tick(rq)) return false; /* @@ -2208,6 +2222,40 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) } EXPORT_SYMBOL_GPL(deactivate_task); +struct sched_change_guard +sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags) +{ + struct sched_change_guard cg = { + .rq = rq, + .p = p, + .queued = task_on_rq_queued(p), + .running = task_current(rq, p), + }; + + if (cg.queued) { + /* + * __kthread_bind() may call this on blocked tasks without + * holding rq->lock through __do_set_cpus_allowed(). Assert @rq + * locked iff @p is queued. + */ + lockdep_assert_rq_held(rq); + dequeue_task(rq, p, flags); + } + if (cg.running) + put_prev_task(rq, p); + + return cg; +} + +void sched_change_guard_fini(struct sched_change_guard *cg, int flags) +{ + if (cg->queued) + enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK); + if (cg->running) + set_next_task(cg->rq, cg->p); + cg->done = true; +} + static inline int __normal_prio(int policy, int rt_prio, int nice) { int prio; @@ -2272,9 +2320,9 @@ inline int task_curr(const struct task_struct *p) * this means any call to check_class_changed() must be followed by a call to * balance_callback(). */ -static inline void check_class_changed(struct rq *rq, struct task_struct *p, - const struct sched_class *prev_class, - int oldprio) +void check_class_changed(struct rq *rq, struct task_struct *p, + const struct sched_class *prev_class, + int oldprio) { if (prev_class != p->sched_class) { if (prev_class->switched_from) @@ -2837,7 +2885,6 @@ static void __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) { struct rq *rq = task_rq(p); - bool queued, running; /* * This here violates the locking rules for affinity, since we're only @@ -2856,26 +2903,9 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) else lockdep_assert_held(&p->pi_lock); - queued = task_on_rq_queued(p); - running = task_current(rq, p); - - if (queued) { - /* - * Because __kthread_bind() calls this on blocked tasks without - * holding rq->lock. - */ - lockdep_assert_rq_held(rq); - dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); + SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { + p->sched_class->set_cpus_allowed(p, ctx); } - if (running) - put_prev_task(rq, p); - - p->sched_class->set_cpus_allowed(p, ctx); - - if (queued) - enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); - if (running) - set_next_task(rq, p); } /* @@ -4073,6 +4103,11 @@ bool cpus_share_cache(int this_cpu, int that_cpu) static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) { +#ifdef CONFIG_SMP + if (p->sched_class == &stop_sched_class) + return false; +#endif + /* * Do not complicate things with the async wake_list while the CPU is * in hotplug state. @@ -4861,6 +4896,9 @@ late_initcall(sched_core_sysctl_init); */ int sched_fork(unsigned long clone_flags, struct task_struct *p) { + + int ret; + trace_android_rvh_sched_fork(p); __sched_fork(clone_flags, p); @@ -4900,12 +4938,20 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) p->sched_reset_on_fork = 0; } - if (dl_prio(p->prio)) - return -EAGAIN; - else if (rt_prio(p->prio)) + scx_pre_fork(p); + + if (dl_prio(p->prio)) { + ret = -EAGAIN; + goto out_cancel; +#ifdef CONFIG_SCHED_CLASS_EXT + } else if (task_on_scx(p)) { + p->sched_class = &ext_sched_class; +#endif + } else if (rt_prio(p->prio)) { p->sched_class = &rt_sched_class; - else + } else { p->sched_class = &fair_sched_class; + } init_entity_runnable_average(&p->se); trace_android_rvh_finish_prio_fork(p); @@ -4924,9 +4970,13 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) RB_CLEAR_NODE(&p->pushable_dl_tasks); #endif return 0; + +out_cancel: + scx_cancel_fork(p); + return ret; } -void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) +int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) { unsigned long flags; @@ -4953,11 +5003,20 @@ void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) if (p->sched_class->task_fork) p->sched_class->task_fork(p); raw_spin_unlock_irqrestore(&p->pi_lock, flags); + + return scx_fork(p); +} + +void sched_cancel_fork(struct task_struct *p) +{ + scx_cancel_fork(p); } void sched_post_fork(struct task_struct *p) { uclamp_post_fork(p); + + scx_post_fork(p); } unsigned long to_ratio(u64 period, u64 runtime) @@ -5822,17 +5881,20 @@ void scheduler_tick(void) if (sched_feat(LATENCY_WARN) && resched_latency) resched_latency_warn(cpu, resched_latency); + scx_notify_sched_tick(); perf_event_task_tick(); if (curr->flags & PF_WQ_WORKER) wq_worker_tick(curr); #ifdef CONFIG_SMP - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq); + if (!scx_switched_all()) { + rq->idle_balance = idle_cpu(cpu); + trigger_load_balance(rq); + } #endif - trace_android_vh_scheduler_tick(rq); + } #ifdef CONFIG_NO_HZ_FULL @@ -6134,7 +6196,7 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, * We can terminate the balance pass as soon as we know there is * a runnable task of @class priority or higher. */ - for_class_range(class, prev->sched_class, &idle_sched_class) { + for_balance_class_range(class, prev->sched_class, &idle_sched_class) { if (class->balance(rq, prev, rf)) break; } @@ -6152,6 +6214,9 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) const struct sched_class *class; struct task_struct *p; + if (scx_enabled()) + goto restart; + /* * Optimization: we know that if all tasks are in the fair class we can * call that function directly, but only if the @prev task wasn't of a @@ -6165,6 +6230,8 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) if (unlikely(p == RETRY_TASK)) goto restart; + trace_android_vh_chk_task(&p, rq); + /* Assume the next prioritized class is idle_sched_class */ if (!p) { put_prev_task(rq, prev); @@ -6177,10 +6244,12 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) restart: put_prev_task_balance(rq, prev, rf); - for_each_class(class) { + for_each_active_class(class) { p = class->pick_next_task(rq); - if (p) + if (p) { + scx_notify_pick_next_task(rq, p, class); return p; + } } BUG(); /* The idle class should always have a runnable task. */ @@ -6210,7 +6279,7 @@ static inline struct task_struct *pick_task(struct rq *rq) const struct sched_class *class; struct task_struct *p; - for_each_class(class) { + for_each_active_class(class) { p = class->pick_task(rq); if (p) return p; @@ -6741,6 +6810,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) struct rq_flags rf; struct rq *rq; int cpu; + bool skip_schedule = false; cpu = smp_processor_id(); rq = cpu_rq(cpu); @@ -6748,6 +6818,11 @@ static void __sched notrace __schedule(unsigned int sched_mode) schedule_debug(prev, !!sched_mode); + trace_android_vh_lock_delay_schedule(prev, sched_mode, &skip_schedule); + + if (skip_schedule) + return; + if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) hrtick_clear(rq); @@ -6820,6 +6895,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) next = pick_next_task(rq, prev, &rf); clear_tsk_need_resched(prev); clear_preempt_need_resched(); + trace_android_vh_clear_curr_lazy(prev); #ifdef CONFIG_SCHED_DEBUG rq->last_seen_need_resched_ns = 0; #endif @@ -7182,10 +7258,20 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag } EXPORT_SYMBOL(default_wake_function); -static void __setscheduler_prio(struct task_struct *p, int prio) +void __setscheduler_prio(struct task_struct *p, int prio) { - if (dl_prio(prio)) + /* + * After switching all rt and fair class to ext, + * stop class is switched to ext class too. Just + * skip it. */ + if (p->sched_class == &stop_sched_class) + ;/* do nothing */ + else if (dl_prio(prio)) p->sched_class = &dl_sched_class; +#ifdef CONFIG_SCHED_CLASS_EXT + else if (task_on_scx(p)) + p->sched_class = &ext_sched_class; +#endif else if (rt_prio(prio)) p->sched_class = &rt_sched_class; else @@ -7225,7 +7311,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) */ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) { - int prio, oldprio, queued, running, queue_flag = + int prio, oldprio, queue_flag = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; const struct sched_class *prev_class; struct rq_flags rf; @@ -7288,51 +7374,41 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) queue_flag &= ~DEQUEUE_MOVE; prev_class = p->sched_class; - queued = task_on_rq_queued(p); - running = task_current(rq, p); - if (queued) - dequeue_task(rq, p, queue_flag); - if (running) - put_prev_task(rq, p); - - /* - * Boosting condition are: - * 1. -rt task is running and holds mutex A - * --> -dl task blocks on mutex A - * - * 2. -dl task is running and holds mutex A - * --> -dl task blocks on mutex A and could preempt the - * running task - */ - if (dl_prio(prio)) { - if (!dl_prio(p->normal_prio) || - (pi_task && dl_prio(pi_task->prio) && - dl_entity_preempt(&pi_task->dl, &p->dl))) { - p->dl.pi_se = pi_task->dl.pi_se; - queue_flag |= ENQUEUE_REPLENISH; + SCHED_CHANGE_BLOCK(rq, p, queue_flag) { + /* + * Boosting condition are: + * 1. -rt task is running and holds mutex A + * --> -dl task blocks on mutex A + * + * 2. -dl task is running and holds mutex A + * --> -dl task blocks on mutex A and could preempt the + * running task + */ + if (dl_prio(prio)) { + if (!dl_prio(p->normal_prio) || + (pi_task && dl_prio(pi_task->prio) && + dl_entity_preempt(&pi_task->dl, &p->dl))) { + p->dl.pi_se = pi_task->dl.pi_se; + queue_flag |= ENQUEUE_REPLENISH; + } else { + p->dl.pi_se = &p->dl; + } + } else if (rt_prio(prio)) { + if (dl_prio(oldprio)) + p->dl.pi_se = &p->dl; + if (oldprio < prio) + queue_flag |= ENQUEUE_HEAD; } else { - p->dl.pi_se = &p->dl; + if (dl_prio(oldprio)) + p->dl.pi_se = &p->dl; + else if (rt_prio(oldprio)) + p->rt.timeout = 0; + else if (!task_has_idle_policy(p)) + reweight_task(p, prio - MAX_RT_PRIO); } - } else if (rt_prio(prio)) { - if (dl_prio(oldprio)) - p->dl.pi_se = &p->dl; - if (oldprio < prio) - queue_flag |= ENQUEUE_HEAD; - } else { - if (dl_prio(oldprio)) - p->dl.pi_se = &p->dl; - else if (rt_prio(oldprio)) - p->rt.timeout = 0; - else if (!task_has_idle_policy(p)) - reweight_task(p, prio - MAX_RT_PRIO); - } - - __setscheduler_prio(p, prio); - if (queued) - enqueue_task(rq, p, queue_flag); - if (running) - set_next_task(rq, p); + __setscheduler_prio(p, prio); + } check_class_changed(rq, p, prev_class, oldprio); out_unlock: @@ -7354,7 +7430,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio) void set_user_nice(struct task_struct *p, long nice) { - bool queued, running, allowed = false; + bool allowed = false; int old_prio; struct rq_flags rf; struct rq *rq; @@ -7383,22 +7459,13 @@ void set_user_nice(struct task_struct *p, long nice) p->static_prio = NICE_TO_PRIO(nice); goto out_unlock; } - queued = task_on_rq_queued(p); - running = task_current(rq, p); - if (queued) - dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); - if (running) - put_prev_task(rq, p); - - p->static_prio = NICE_TO_PRIO(nice); - set_load_weight(p, true); - old_prio = p->prio; - p->prio = effective_prio(p); - if (queued) - enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); - if (running) - set_next_task(rq, p); + SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { + p->static_prio = NICE_TO_PRIO(nice); + set_load_weight(p, true); + old_prio = p->prio; + p->prio = effective_prio(p); + } /* * If the task increased its priority or is running and @@ -7815,7 +7882,7 @@ static int __sched_setscheduler(struct task_struct *p, bool user, bool pi) { int oldpolicy = -1, policy = attr->sched_policy; - int retval, oldprio, newprio, queued, running; + int retval, oldprio, newprio; const struct sched_class *prev_class; struct balance_callback *head; struct rq_flags rf; @@ -7824,6 +7891,7 @@ static int __sched_setscheduler(struct task_struct *p, struct rq *rq; bool cpuset_locked = false; + /* The pi code expects interrupts enabled */ BUG_ON(pi && in_interrupt()); recheck: @@ -7888,6 +7956,7 @@ static int __sched_setscheduler(struct task_struct *p, * To be able to change p->policy safely, the appropriate * runqueue lock must be held. */ + rq = task_rq_lock(p, &rf); update_rq_clock(rq); @@ -7899,6 +7968,10 @@ static int __sched_setscheduler(struct task_struct *p, goto unlock; } + retval = scx_check_setscheduler(p, policy); + if (retval) + goto unlock; + /* * If not changing anything there's no need to proceed further, * but store a possible modification of reset_on_fork. @@ -7955,6 +8028,7 @@ static int __sched_setscheduler(struct task_struct *p, if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { policy = oldpolicy = -1; task_rq_unlock(rq, p, &rf); + if (cpuset_locked) cpuset_unlock(); goto recheck; @@ -7987,23 +8061,16 @@ static int __sched_setscheduler(struct task_struct *p, queue_flags &= ~DEQUEUE_MOVE; } - queued = task_on_rq_queued(p); - running = task_current(rq, p); - if (queued) - dequeue_task(rq, p, queue_flags); - if (running) - put_prev_task(rq, p); - - prev_class = p->sched_class; + SCHED_CHANGE_BLOCK(rq, p, queue_flags) { + prev_class = p->sched_class; - if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { - __setscheduler_params(p, attr); - __setscheduler_prio(p, newprio); - trace_android_rvh_setscheduler(p); - } - __setscheduler_uclamp(p, attr); + if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { + __setscheduler_params(p, attr); + __setscheduler_prio(p, newprio); + trace_android_rvh_setscheduler(p); + } + __setscheduler_uclamp(p, attr); - if (queued) { /* * We enqueue to tail when the priority of a task is * increased (user space view). @@ -8011,10 +8078,7 @@ static int __sched_setscheduler(struct task_struct *p, if (oldprio < p->prio) queue_flags |= ENQUEUE_HEAD; - enqueue_task(rq, p, queue_flags); } - if (running) - set_next_task(rq, p); check_class_changed(rq, p, prev_class, oldprio); @@ -8037,6 +8101,7 @@ static int __sched_setscheduler(struct task_struct *p, unlock: task_rq_unlock(rq, p, &rf); + if (cpuset_locked) cpuset_unlock(); return retval; @@ -9258,6 +9323,7 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) case SCHED_NORMAL: case SCHED_BATCH: case SCHED_IDLE: + case SCHED_EXT: ret = 0; break; } @@ -9285,6 +9351,7 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) case SCHED_NORMAL: case SCHED_BATCH: case SCHED_IDLE: + case SCHED_EXT: ret = 0; } return ret; @@ -9585,25 +9652,15 @@ int migrate_task_to(struct task_struct *p, int target_cpu) */ void sched_setnuma(struct task_struct *p, int nid) { - bool queued, running; struct rq_flags rf; struct rq *rq; rq = task_rq_lock(p, &rf); - queued = task_on_rq_queued(p); - running = task_current(rq, p); - if (queued) - dequeue_task(rq, p, DEQUEUE_SAVE); - if (running) - put_prev_task(rq, p); - - p->numa_preferred_nid = nid; + SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) { + p->numa_preferred_nid = nid; + } - if (queued) - enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); - if (running) - set_next_task(rq, p); task_rq_unlock(rq, p, &rf); } #endif /* CONFIG_NUMA_BALANCING */ @@ -10169,11 +10226,15 @@ void __init sched_init(void) int i; /* Make sure the linker didn't screw up */ - BUG_ON(&idle_sched_class != &fair_sched_class + 1 || - &fair_sched_class != &rt_sched_class + 1 || - &rt_sched_class != &dl_sched_class + 1); #ifdef CONFIG_SMP - BUG_ON(&dl_sched_class != &stop_sched_class + 1); + BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); +#endif + BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); + BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); + BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); +#ifdef CONFIG_SCHED_CLASS_EXT + BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); + BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); #endif wait_bit_init(); @@ -10345,6 +10406,7 @@ void __init sched_init(void) balance_push_set(smp_processor_id(), false); #endif init_sched_fair_class(); + init_sched_ext_class(); psi_init(); @@ -10869,8 +10931,10 @@ static void cpu_cgroup_attach(struct cgroup_taskset *tset) struct task_struct *task; struct cgroup_subsys_state *css; - cgroup_taskset_for_each(task, css, tset) + cgroup_taskset_for_each(task, css, tset) { + sched_move_task(task); + } trace_android_rvh_cpu_cgroup_attach(tset); } @@ -11550,6 +11614,7 @@ static struct cftype cpu_legacy_files[] = { .write_u64 = cpu_uclamp_ls_write_u64, }, #endif + { } /* Terminate */ }; @@ -11598,6 +11663,7 @@ static int cpu_local_stat_show(struct seq_file *sf, } #ifdef CONFIG_FAIR_GROUP_SCHED + static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) { diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 2aaf0c38cf392..0429ea4d831ff 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -128,6 +128,32 @@ static void sugov_deferred_update(struct sugov_policy *sg_policy) } } +/** + * get_capacity_ref_freq - get the reference frequency that has been used to + * correlate frequency and compute capacity for a given cpufreq policy. We use + * the CPU managing it for the arch_scale_freq_ref() call in the function. + * @policy: the cpufreq policy of the CPU in question. + * + * Return: the reference CPU frequency to compute a capacity. + */ +static __always_inline +unsigned long get_capacity_ref_freq(struct cpufreq_policy *policy) +{ + unsigned int freq = arch_scale_freq_ref(policy->cpu); + + if (freq) + return freq; + + if (arch_scale_freq_invariant()) + return policy->cpuinfo.max_freq; + + /* + * Apply a 25% margin so that we select a higher frequency than + * the current one before the CPU is fully busy: + */ + return policy->cur + (policy->cur >> 2); +} + /** * get_next_freq - Compute a new frequency for a given cpufreq policy. * @sg_policy: schedutil policy object to compute the new frequency for. @@ -154,10 +180,10 @@ static unsigned int get_next_freq(struct sugov_policy *sg_policy, unsigned long util, unsigned long max) { struct cpufreq_policy *policy = sg_policy->policy; - unsigned int freq = arch_scale_freq_invariant() ? - policy->cpuinfo.max_freq : policy->cur; + unsigned int freq; unsigned long next_freq = 0; + freq = get_capacity_ref_freq(policy); util = map_util_perf(util); trace_android_vh_map_util_freq(util, freq, max, &next_freq, policy, &sg_policy->need_freq_update); @@ -620,7 +646,6 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy) if (policy->fast_switch_enabled) return 0; - trace_android_vh_set_sugov_sched_attr(&attr); kthread_init_work(&sg_policy->work, sugov_work); kthread_init_worker(&sg_policy->worker); thread = kthread_create(kthread_worker_fn, &sg_policy->worker, diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index ca41ecd36ec2d..e108b66062410 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2207,6 +2207,25 @@ static int find_later_rq(struct task_struct *task) return -1; } +static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) +{ + struct task_struct *p; + + if (!has_pushable_dl_tasks(rq)) + return NULL; + + p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); + + WARN_ON_ONCE(rq->cpu != task_cpu(p)); + WARN_ON_ONCE(task_current(rq, p)); + WARN_ON_ONCE(p->nr_cpus_allowed <= 1); + + WARN_ON_ONCE(!task_on_rq_queued(p)); + WARN_ON_ONCE(!dl_task(p)); + + return p; +} + /* Locks the rq it finds */ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) { @@ -2234,12 +2253,37 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) /* Retry if something changed. */ if (double_lock_balance(rq, later_rq)) { - if (unlikely(task_rq(task) != rq || + /* + * double_lock_balance had to release rq->lock, in the + * meantime, task may no longer be fit to be migrated. + * Check the following to ensure that the task is + * still suitable for migration: + * 1. It is possible the task was scheduled, + * migrate_disabled was set and then got preempted, + * so we must check the task migration disable + * flag. + * 2. The CPU picked is in the task's affinity. + * 3. For throttled task (dl_task_offline_migration), + * check the following: + * - the task is not on the rq anymore (it was + * migrated) + * - the task is not on CPU anymore + * - the task is still a dl task + * - the task is not queued on the rq anymore + * 4. For the non-throttled task (push_dl_task), the + * check to ensure that this task is still at the + * head of the pushable tasks list is enough. + */ + if (unlikely(is_migration_disabled(task) || !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) || - task_on_cpu(rq, task) || - !dl_task(task) || - is_migration_disabled(task) || - !task_on_rq_queued(task))) { + (task->dl.dl_throttled && + (task_rq(task) != rq || + task_on_cpu(rq, task) || + !dl_task(task) || + !task_on_rq_queued(task))) || + (!task->dl.dl_throttled && + task != pick_next_pushable_dl_task(rq)))) { + double_unlock_balance(rq, later_rq); later_rq = NULL; break; @@ -2262,25 +2306,6 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) return later_rq; } -static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) -{ - struct task_struct *p; - - if (!has_pushable_dl_tasks(rq)) - return NULL; - - p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); - - WARN_ON_ONCE(rq->cpu != task_cpu(p)); - WARN_ON_ONCE(task_current(rq, p)); - WARN_ON_ONCE(p->nr_cpus_allowed <= 1); - - WARN_ON_ONCE(!task_on_rq_queued(p)); - WARN_ON_ONCE(!dl_task(p)); - - return p; -} - /* * See if the non running -deadline tasks on this rq * can be sent to some other CPU where they can preempt diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c old mode 100644 new mode 100755 index 2c1248e335730..a6c99df9edf94 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -376,6 +376,9 @@ static __init int sched_init_debug(void) debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); +#ifdef CONFIG_SCHED_CLASS_EXT + debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops); +#endif return 0; } late_initcall(sched_init_debug); @@ -1003,6 +1006,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, SEQ_printf(m, "---------------------------------------------------------" "----------\n"); +#ifdef CONFIG_SLIM_SCHED + SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop); +#endif #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) @@ -1098,6 +1104,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, } else if (fair_policy(p->policy)) { P(se.slice); } +#ifdef CONFIG_SCHED_CLASS_EXT + __PS("ext.enabled", p->sched_class == &ext_sched_class); +#endif #undef PN_SCHEDSTAT #undef P_SCHEDSTAT diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c new file mode 100755 index 0000000000000..4845d441df2b0 --- /dev/null +++ b/kernel/sched/ext.c @@ -0,0 +1,3978 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) + +enum scx_internal_consts { + SCX_NR_ONLINE_OPS = SCX_OP_IDX(init), + SCX_DSP_DFL_MAX_BATCH = 32, + SCX_DSP_MAX_LOOPS = 32, + SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, +}; + +enum scx_ops_enable_state { + SCX_OPS_PREPPING, + SCX_OPS_ENABLING, + SCX_OPS_ENABLED, + SCX_OPS_DISABLING, + SCX_OPS_DISABLED, +}; + +static inline struct task_group *css_tg(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct task_group, css) : NULL; +} + +/* + * sched_ext_entity->ops_state + * + * Used to track the task ownership between the SCX core and the BPF scheduler. + * State transitions look as follows: + * + * NONE -> QUEUEING -> QUEUED -> DISPATCHING + * ^ | | + * | v v + * \-------------------------------/ + * + * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call + * sites for explanations on the conditions being waited upon and why they are + * safe. Transitions out of them into NONE or QUEUED must store_release and the + * waiters should load_acquire. + * + * Tracking scx_ops_state enables sched_ext core to reliably determine whether + * any given task can be dispatched by the BPF scheduler at all times and thus + * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler + * to try to dispatch any task anytime regardless of its state as the SCX core + * can safely reject invalid dispatches. + */ +enum scx_ops_state { + SCX_OPSS_NONE, /* owned by the SCX core */ + SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ + SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ + SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ + + /* + * QSEQ brands each QUEUED instance so that, when dispatch races + * dequeue/requeue, the dispatcher can tell whether it still has a claim + * on the task being dispatched. + */ + SCX_OPSS_QSEQ_SHIFT = 2, + SCX_OPSS_STATE_MASK = (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1, + SCX_OPSS_QSEQ_MASK = ~SCX_OPSS_STATE_MASK, +}; + +/* + * During exit, a task may schedule after losing its PIDs. When disabling the + * BPF scheduler, we need to be able to iterate tasks in every state to + * guarantee system safety. Maintain a dedicated task list which contains every + * task between its fork and eventual free. + */ +static DEFINE_SPINLOCK(scx_tasks_lock); +static LIST_HEAD(scx_tasks); + +/* ops enable/disable */ +static struct kthread_worker *scx_ops_helper; +static DEFINE_MUTEX(scx_ops_enable_mutex); +DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); +DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); +static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); +static bool scx_switch_all_req; +static bool scx_switching_all; +DEFINE_STATIC_KEY_FALSE(__scx_switched_all); + +static struct sched_ext_ops scx_ops; +static bool scx_warned_zero_slice; + +static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); +static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); +DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); +static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); + +struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] = + { [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT }; + +static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE); +static struct scx_exit_info scx_exit_info; + +static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0); + +/* + * The maximum amount of time in jiffies that a task may be runnable without + * being scheduled on a CPU. If this timeout is exceeded, it will trigger + * scx_ops_error(). + */ +unsigned long scx_watchdog_timeout; + +/* + * The last time the delayed work was run. This delayed work relies on + * ksoftirqd being able to run to service timer interrupts, so it's possible + * that this work itself could get wedged. To account for this, we check that + * it's not stalled in the timer tick, and trigger an error if it is. + */ +unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; + +static struct delayed_work scx_watchdog_work; + +/* idle tracking */ +#ifdef CONFIG_SMP +#ifdef CONFIG_CPUMASK_OFFSTACK +#define CL_ALIGNED_IF_ONSTACK +#else +#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp +#endif + +static struct { + cpumask_var_t cpu; + cpumask_var_t smt; +} idle_masks CL_ALIGNED_IF_ONSTACK; + +static bool __cacheline_aligned_in_smp scx_has_idle_cpus; +#endif /* CONFIG_SMP */ + +/* for %SCX_KICK_WAIT */ +static u64 __percpu *scx_kick_cpus_pnt_seqs; + +/* + * Direct dispatch marker. + * + * Non-NULL values are used for direct dispatch from enqueue path. A valid + * pointer points to the task currently being enqueued. An ERR_PTR value is used + * to indicate that direct dispatch has already happened. + */ +static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); + +/* dispatch queues */ +static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; + +static LLIST_HEAD(dsqs_to_free); + +/* dispatch buf */ +struct scx_dsp_buf_ent { + struct task_struct *task; + u64 qseq; + u64 dsq_id; + u64 enq_flags; +}; + +static u32 scx_dsp_max_batch; +static struct scx_dsp_buf_ent __percpu *scx_dsp_buf; + +struct scx_dsp_ctx { + struct rq *rq; + struct rq_flags *rf; + u32 buf_cursor; + u32 nr_tasks; +}; + +static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx); + +void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, + u64 enq_flags); +void scx_bpf_kick_cpu(s32 cpu, u64 flags); + +struct scx_task_iter { + struct sched_ext_entity cursor; + struct task_struct *locked; + struct rq *rq; + struct rq_flags rf; +}; + +#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) + +/* if the highest set bit is N, return a mask with bits [N+1, 31] set */ +static u32 higher_bits(u32 flags) +{ + return ~((1 << fls(flags)) - 1); +} + +/* return the mask with only the highest bit set */ +static u32 highest_bit(u32 flags) +{ + int bit = fls(flags); + return bit ? 1 << (bit - 1) : 0; +} + +/* + * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX + * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate + * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check + * whether it's running from an allowed context. + * + * @mask is constant, always inline to cull the mask calculations. + */ +static __always_inline void scx_kf_allow(u32 mask) +{ + /* nesting is allowed only in increasing scx_kf_mask order */ + WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask, + "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n", + current->scx->kf_mask, mask); + current->scx->kf_mask |= mask; +} + +static void scx_kf_disallow(u32 mask) +{ + current->scx->kf_mask &= ~mask; +} + +#define SCX_CALL_OP(mask, op, args...) \ +do { \ + if (mask) { \ + scx_kf_allow(mask); \ + scx_ops.op(args); \ + scx_kf_disallow(mask); \ + } else { \ + scx_ops.op(args); \ + } \ +} while (0) + +#define SCX_CALL_OP_RET(mask, op, args...) \ +({ \ + __typeof__(scx_ops.op(args)) __ret; \ + if (mask) { \ + scx_kf_allow(mask); \ + __ret = scx_ops.op(args); \ + scx_kf_disallow(mask); \ + } else { \ + __ret = scx_ops.op(args); \ + } \ + __ret; \ +}) + +/* + * Some kfuncs are allowed only on the tasks that are subjects of the + * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such + * restrictions, the following SCX_CALL_OP_*() variants should be used when + * invoking scx_ops operations that take task arguments. These can only be used + * for non-nesting operations due to the way the tasks are tracked. + * + * kfuncs which can only operate on such tasks can in turn use + * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on + * the specific task. + */ +#define SCX_CALL_OP_TASK(mask, op, task, args...) \ +do { \ + BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ + current->scx->kf_tasks[0] = task; \ + SCX_CALL_OP(mask, op, task, ##args); \ + current->scx->kf_tasks[0] = NULL; \ +} while (0) + +#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ +({ \ + __typeof__(scx_ops.op(task, ##args)) __ret; \ + BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ + current->scx->kf_tasks[0] = task; \ + __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ + current->scx->kf_tasks[0] = NULL; \ + __ret; \ +}) + +#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ +({ \ + __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ + BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL); \ + current->scx->kf_tasks[0] = task0; \ + current->scx->kf_tasks[1] = task1; \ + __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ + current->scx->kf_tasks[0] = NULL; \ + current->scx->kf_tasks[1] = NULL; \ + __ret; \ +}) + +//#include "./slim_walt.c" + +/* @mask is constant, always inline to cull unnecessary branches */ +static __always_inline bool scx_kf_allowed(u32 mask) +{ + if (unlikely(!(current->scx->kf_mask & mask))) { + scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", + mask, current->scx->kf_mask); + return false; + } + + if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) && + in_interrupt())) { + scx_ops_error("sleepable kfunc called from non-sleepable context"); + return false; + } + + /* + * Enforce nesting boundaries. e.g. A kfunc which can be called from + * DISPATCH must not be called if we're running DEQUEUE which is nested + * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE + * boundary thanks to the above in_interrupt() check. + */ + if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && + (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { + scx_ops_error("cpu_release kfunc called from a nested operation"); + return false; + } + + if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && + (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) { + scx_ops_error("dispatch kfunc called from a nested operation"); + return false; + } + + return true; +} + +/* see SCX_CALL_OP_TASK() */ +static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, + struct task_struct *p) +{ + if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED)) + return false; + + if (unlikely((p != current->scx->kf_tasks[0] && + p != current->scx->kf_tasks[1]))) { + scx_ops_error("called on a task not being operated on"); + return false; + } + + return true; +} + +/** + * scx_task_iter_init - Initialize a task iterator + * @iter: iterator to init + * + * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized, + * @iter must eventually be exited with scx_task_iter_exit(). + * + * scx_tasks_lock may be released between this and the first next() call or + * between any two next() calls. If scx_tasks_lock is released between two + * next() calls, the caller is responsible for ensuring that the task being + * iterated remains accessible either through RCU read lock or obtaining a + * reference count. + * + * All tasks which existed when the iteration started are guaranteed to be + * visited as long as they still exist. + */ +static void scx_task_iter_init(struct scx_task_iter *iter) +{ + lockdep_assert_held(&scx_tasks_lock); + + iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; + list_add(&iter->cursor.tasks_node, &scx_tasks); + iter->locked = NULL; +} + +/** + * scx_task_iter_exit - Exit a task iterator + * @iter: iterator to exit + * + * Exit a previously initialized @iter. Must be called with scx_tasks_lock held. + * If the iterator holds a task's rq lock, that rq lock is released. See + * scx_task_iter_init() for details. + */ +static void scx_task_iter_exit(struct scx_task_iter *iter) +{ + struct list_head *cursor = &iter->cursor.tasks_node; + + lockdep_assert_held(&scx_tasks_lock); + + if (iter->locked) { + task_rq_unlock(iter->rq, iter->locked, &iter->rf); + iter->locked = NULL; + } + + if (list_empty(cursor)) + return; + + list_del_init(cursor); +} + +/** + * scx_task_iter_next - Next task + * @iter: iterator to walk + * + * Visit the next task. See scx_task_iter_init() for details. + */ +static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) +{ + struct list_head *cursor = &iter->cursor.tasks_node; + struct sched_ext_entity *pos; + + lockdep_assert_held(&scx_tasks_lock); + + list_for_each_entry(pos, cursor, tasks_node) { + if (&pos->tasks_node == &scx_tasks) + return NULL; + if (!(pos->flags & SCX_TASK_CURSOR)) { + list_move(cursor, &pos->tasks_node); + return pos->task; + } + } + + /* can't happen, should always terminate at scx_tasks above */ + BUG(); +} + +/** + * scx_task_iter_next_filtered - Next non-idle task + * @iter: iterator to walk + * + * Visit the next non-idle task. See scx_task_iter_init() for details. + */ +static struct task_struct * +scx_task_iter_next_filtered(struct scx_task_iter *iter) +{ + struct task_struct *p; + + while ((p = scx_task_iter_next(iter))) { + if (!is_idle_task(p)) + return p; + } + return NULL; +} + +/** + * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked + * @iter: iterator to walk + * + * Visit the next non-idle task with its rq lock held. See scx_task_iter_init() + * for details. + */ +static struct task_struct * +scx_task_iter_next_filtered_locked(struct scx_task_iter *iter) +{ + struct task_struct *p; + + if (iter->locked) { + task_rq_unlock(iter->rq, iter->locked, &iter->rf); + iter->locked = NULL; + } + + p = scx_task_iter_next_filtered(iter); + if (!p) + return NULL; + + iter->rq = task_rq_lock(p, &iter->rf); + iter->locked = p; + return p; +} + +static enum scx_ops_enable_state scx_ops_enable_state(void) +{ + return atomic_read(&scx_ops_enable_state_var); +} + +static enum scx_ops_enable_state +scx_ops_set_enable_state(enum scx_ops_enable_state to) +{ + return atomic_xchg(&scx_ops_enable_state_var, to); +} + +static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, + enum scx_ops_enable_state from) +{ + int from_v = from; + + return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); +} + +static bool scx_ops_disabling(void) +{ + return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING); +} + +/** + * wait_ops_state - Busy-wait the specified ops state to end + * @p: target task + * @opss: state to wait the end of + * + * Busy-wait for @p to transition out of @opss. This can only be used when the + * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also + * has load_acquire semantics to ensure that the caller can see the updates made + * in the enqueueing and dispatching paths. + */ +static void wait_ops_state(struct task_struct *p, u64 opss) +{ + do { + cpu_relax(); + } while (atomic64_read_acquire(&p->scx->ops_state) == opss); +} + +/** + * ops_cpu_valid - Verify a cpu number + * @cpu: cpu number which came from a BPF ops + * + * @cpu is a cpu number which came from the BPF scheduler and can be any value. + * Verify that it is in range and one of the possible cpus. + */ +static bool ops_cpu_valid(s32 cpu) +{ + return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); +} + +/** + * ops_sanitize_err - Sanitize a -errno value + * @ops_name: operation to blame on failure + * @err: -errno value to sanitize + * + * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return + * -%EPROTO. This is necessary because returning a rogue -errno up the chain can + * cause misbehaviors. For an example, a large negative return from + * ops.prep_enable() triggers an oops when passed up the call chain because the + * value fails IS_ERR() test after being encoded with ERR_PTR() and then is + * handled as a pointer. + */ +static int ops_sanitize_err(const char *ops_name, s32 err) +{ + if (err < 0 && err >= -MAX_ERRNO) + return err; + + scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); + return -EPROTO; +} + +/** + * touch_core_sched - Update timestamp used for core-sched task ordering + * @rq: rq to read clock from, must be locked + * @p: task to update the timestamp for + * + * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to + * implement global or local-DSQ FIFO ordering for core-sched. Should be called + * when a task becomes runnable and its turn on the CPU ends (e.g. slice + * exhaustion). + */ +static void touch_core_sched(struct rq *rq, struct task_struct *p) +{ +#ifdef CONFIG_SCHED_CORE + /* + * It's okay to update the timestamp spuriously. Use + * sched_core_disabled() which is cheaper than enabled(). + */ + if (!sched_core_disabled()) + p->scx->core_sched_at = rq_clock_task(rq); +#endif +} + +/** + * touch_core_sched_dispatch - Update core-sched timestamp on dispatch + * @rq: rq to read clock from, must be locked + * @p: task being dispatched + * + * If the BPF scheduler implements custom core-sched ordering via + * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO + * ordering within each local DSQ. This function is called from dispatch paths + * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect. + */ +static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + assert_clock_updated(rq); + +#ifdef CONFIG_SCHED_CORE + if (SCX_HAS_OP(core_sched_before)) + touch_core_sched(rq, p); +#endif +} + +static void update_curr_scx(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + u64 now = rq_clock_task(rq); + u64 delta_exec; + + if (time_before_eq64(now, curr->se.exec_start)) + return; + + delta_exec = now - curr->se.exec_start; + curr->se.exec_start = now; + curr->se.sum_exec_runtime += delta_exec; + account_group_exec_runtime(curr, delta_exec); + cgroup_account_cputime(curr, delta_exec); + + if (curr->scx->slice != SCX_SLICE_INF) { + curr->scx->slice -= min(curr->scx->slice, delta_exec); + if (!curr->scx->slice) + touch_core_sched(rq, curr); + } +} + +static bool scx_dsq_priq_less(struct rb_node *node_a, + const struct rb_node *node_b) +{ + const struct sched_ext_entity *a = + container_of(node_a, struct sched_ext_entity, dsq_node.priq); + const struct sched_ext_entity *b = + container_of(node_b, struct sched_ext_entity, dsq_node.priq); + + return time_before64(a->dsq_vtime, b->dsq_vtime); +} + +static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) +{ + bool is_local = dsq->id == SCX_DSQ_LOCAL; + + WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo)); + WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || + !RB_EMPTY_NODE(&p->scx->dsq_node.priq)); + + if (!is_local) { + raw_spin_lock(&dsq->lock); + if (unlikely(dsq->id == SCX_DSQ_INVALID)) { + scx_ops_error("attempting to dispatch to a destroyed dsq"); + /* fall back to the global dsq */ + raw_spin_unlock(&dsq->lock); + dsq = &scx_dsq_global; + raw_spin_lock(&dsq->lock); + } + } + + if (enq_flags & SCX_ENQ_DSQ_PRIQ) { + p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; + rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq, + scx_dsq_priq_less); + } else { + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) + list_add(&p->scx->dsq_node.fifo, &dsq->fifo); + else + list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo); + } + dsq->nr++; + p->scx->dsq = dsq; + + /* + * We're transitioning out of QUEUEING or DISPATCHING. store_release to + * match waiters' load_acquire. + */ + if (enq_flags & SCX_ENQ_CLEAR_OPSS) + atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); + + if (is_local) { + struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq); + struct rq *rq = scx->rq; + bool preempt = false; + + if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && + rq->curr->sched_class == &ext_sched_class) { + rq->curr->scx->slice = 0; + preempt = true; + } + + if (preempt || sched_class_above(&ext_sched_class, + rq->curr->sched_class)) + resched_curr(rq); + } else { + raw_spin_unlock(&dsq->lock); + } +} + +static void task_unlink_from_dsq(struct task_struct *p, + struct scx_dispatch_q *dsq) +{ + if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { + rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq); + RB_CLEAR_NODE(&p->scx->dsq_node.priq); + p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; + } else { + list_del_init(&p->scx->dsq_node.fifo); + } + +} + +static bool task_linked_on_dsq(struct task_struct *p) +{ + return !list_empty(&p->scx->dsq_node.fifo) || + !RB_EMPTY_NODE(&p->scx->dsq_node.priq); +} + +static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p) +{ + struct scx_dispatch_q *dsq = p->scx->dsq; + bool is_local = dsq == &scx_rq->local_dsq; + + if (!dsq) { + WARN_ON_ONCE(task_linked_on_dsq(p)); + /* + * When dispatching directly from the BPF scheduler to a local + * DSQ, the task isn't associated with any DSQ but + * @p->scx->holding_cpu may be set under the protection of + * %SCX_OPSS_DISPATCHING. + */ + if (p->scx->holding_cpu >= 0) + p->scx->holding_cpu = -1; + return; + } + + if (!is_local) + raw_spin_lock(&dsq->lock); + + /* + * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node + * can't change underneath us. + */ + if (p->scx->holding_cpu < 0) { + /* @p must still be on @dsq, dequeue */ + WARN_ON_ONCE(!task_linked_on_dsq(p)); + task_unlink_from_dsq(p, dsq); + dsq->nr--; + } else { + /* + * We're racing against dispatch_to_local_dsq() which already + * removed @p from @dsq and set @p->scx->holding_cpu. Clear the + * holding_cpu which tells dispatch_to_local_dsq() that it lost + * the race. + */ + WARN_ON_ONCE(task_linked_on_dsq(p)); + p->scx->holding_cpu = -1; + } + p->scx->dsq = NULL; + + if (!is_local) + raw_spin_unlock(&dsq->lock); +} + +static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) +{ + lockdep_assert(rcu_read_lock_any_held()); + + return &scx_dsq_global; +} + +static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, + struct task_struct *p) +{ + struct scx_dispatch_q *dsq; + + if (dsq_id == SCX_DSQ_LOCAL) + return &rq->scx->local_dsq; + + dsq = find_non_local_dsq(dsq_id); + if (unlikely(!dsq)) { + scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", + dsq_id, p->comm, p->pid); + return &scx_dsq_global; + } + + return dsq; +} + +static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, + u64 dsq_id, u64 enq_flags) +{ + struct scx_dispatch_q *dsq; + + /* @p must match the task which is being enqueued */ + if (unlikely(p != ddsp_task)) { + if (IS_ERR(ddsp_task)) + scx_ops_error("%s[%d] already direct-dispatched", + p->comm, p->pid); + else + scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]", + ddsp_task->comm, ddsp_task->pid, + p->comm, p->pid); + return; + } + + /* + * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because + * dispatching to the local DSQ of a different CPU requires unlocking + * the current rq which isn't allowed in the enqueue path. Use + * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL. + */ + if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) { + scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch"); + return; + } + + touch_core_sched_dispatch(task_rq(p), p); + + dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p); + dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + + /* + * Mark that dispatch already happened by spoiling direct_dispatch_task + * with a non-NULL value which can never match a valid task pointer. + */ + __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); +} + +static bool test_rq_online(struct rq *rq) +{ +#ifdef CONFIG_SMP + return rq->online; +#else + return true; +#endif +} + +static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) +{ + struct task_struct **ddsp_taskp; + u64 qseq; + + WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); + + if (p->scx->flags & SCX_TASK_ENQ_LOCAL) { + enq_flags |= SCX_ENQ_LOCAL; + p->scx->flags &= ~SCX_TASK_ENQ_LOCAL; + } + + /* rq migration */ + if (sticky_cpu == cpu_of(rq)) + goto local_norefill; + + /* + * If !rq->online, we already told the BPF scheduler that the CPU is + * offline. We're just trying to on/offline the CPU. Don't bother the + * BPF scheduler. + */ + if (unlikely(!test_rq_online(rq))) + goto local; + + /* see %SCX_OPS_ENQ_EXITING */ + if (!static_branch_unlikely(&scx_ops_enq_exiting) && + unlikely(p->flags & PF_EXITING)) + goto local; + + /* see %SCX_OPS_ENQ_LAST */ + if (!static_branch_unlikely(&scx_ops_enq_last) && + (enq_flags & SCX_ENQ_LAST)) + goto local; + + if (!SCX_HAS_OP(enqueue)) { + if (enq_flags & SCX_ENQ_LOCAL) + goto local; + else + goto global; + } + + /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ + qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; + + WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); + atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq); + + ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); + WARN_ON_ONCE(*ddsp_taskp); + *ddsp_taskp = p; + + SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); + + /* + * If not directly dispatched, QUEUEING isn't clear yet and dispatch or + * dequeue may be waiting. The store_release matches their load_acquire. + */ + if (*ddsp_taskp == p) + atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq); + *ddsp_taskp = NULL; + return; + +local: + /* + * For task-ordering, slice refill must be treated as implying the end + * of the current slice. Otherwise, the longer @p stays on the CPU, the + * higher priority it becomes from scx_prio_less()'s POV. + */ + touch_core_sched(rq, p); + p->scx->slice = SCX_SLICE_DFL; +local_norefill: + dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags); + return; + +global: + touch_core_sched(rq, p); /* see the comment in local: */ + p->scx->slice = SCX_SLICE_DFL; + dispatch_enqueue(&scx_dsq_global, p, enq_flags); +} + +static bool watchdog_task_watched(const struct task_struct *p) +{ + return !list_empty(&p->scx->watchdog_node); +} + +static void watchdog_watch_task(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + if (p->scx->flags & SCX_TASK_WATCHDOG_RESET) + p->scx->runnable_at = jiffies; + p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET; + list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list); +} + +static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout) +{ + list_del_init(&p->scx->watchdog_node); + if (reset_timeout) + p->scx->flags |= SCX_TASK_WATCHDOG_RESET; +} + +static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) +{ + int sticky_cpu = p->scx->sticky_cpu; + + enq_flags |= rq->scx->extra_enq_flags; + + if (sticky_cpu >= 0) + p->scx->sticky_cpu = -1; + + /* + * Restoring a running task will be immediately followed by + * set_next_task_scx() which expects the task to not be on the BPF + * scheduler as tasks can only start running through local DSQs. Force + * direct-dispatch into the local DSQ by setting the sticky_cpu. + */ + if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) + sticky_cpu = cpu_of(rq); + + if (p->scx->flags & SCX_TASK_QUEUED) { + WARN_ON_ONCE(!watchdog_task_watched(p)); + return; + } + + watchdog_watch_task(rq, p); + p->scx->flags |= SCX_TASK_QUEUED; + rq->scx->nr_running++; + add_nr_running(rq, 1); + + if (SCX_HAS_OP(runnable)) + SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); + + if (enq_flags & SCX_ENQ_WAKEUP) + touch_core_sched(rq, p); + + do_enqueue_task(rq, p, enq_flags, sticky_cpu); +} + +static void ops_dequeue(struct task_struct *p, u64 deq_flags) +{ + u64 opss; + + watchdog_unwatch_task(p, false); + + /* acquire ensures that we see the preceding updates on QUEUED */ + opss = atomic64_read_acquire(&p->scx->ops_state); + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_NONE: + break; + case SCX_OPSS_QUEUEING: + /* + * QUEUEING is started and finished while holding @p's rq lock. + * As we're holding the rq lock now, we shouldn't see QUEUEING. + */ + BUG(); + case SCX_OPSS_QUEUED: + if (SCX_HAS_OP(dequeue)) + SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); + + if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss, + SCX_OPSS_NONE)) + break; + fallthrough; + case SCX_OPSS_DISPATCHING: + /* + * If @p is being dispatched from the BPF scheduler to a DSQ, + * wait for the transfer to complete so that @p doesn't get + * added to its DSQ after dequeueing is complete. + * + * As we're waiting on DISPATCHING with the rq locked, the + * dispatching side shouldn't try to lock the rq while + * DISPATCHING is set. See dispatch_to_local_dsq(). + * + * DISPATCHING shouldn't have qseq set and control can reach + * here with NONE @opss from the above QUEUED case block. + * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. + */ + wait_ops_state(p, SCX_OPSS_DISPATCHING); + BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); + break; + } +} + +static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) +{ + struct scx_rq *scx_rq = rq->scx; + + if (!(p->scx->flags & SCX_TASK_QUEUED)) { + WARN_ON_ONCE(watchdog_task_watched(p)); + return; + } + + ops_dequeue(p, deq_flags); + + /* + * A currently running task which is going off @rq first gets dequeued + * and then stops running. As we want running <-> stopping transitions + * to be contained within runnable <-> quiescent transitions, trigger + * ->stopping() early here instead of in put_prev_task_scx(). + * + * @p may go through multiple stopping <-> running transitions between + * here and put_prev_task_scx() if task attribute changes occur while + * balance_scx() leaves @rq unlocked. However, they don't contain any + * information meaningful to the BPF scheduler and can be suppressed by + * skipping the callbacks if the task is !QUEUED. + */ + if (SCX_HAS_OP(stopping) && task_current(rq, p)) { + update_curr_scx(rq); + SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); + } + + //if(task_current(rq, p)) + // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); + + if (SCX_HAS_OP(quiescent)) + SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); + + if (deq_flags & SCX_DEQ_SLEEP) + p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP; + else + p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP; + + p->scx->flags &= ~SCX_TASK_QUEUED; + BUG_ON(!scx_rq->nr_running); + scx_rq->nr_running--; + sub_nr_running(rq, 1); + + dispatch_dequeue(scx_rq, p); +} + +static void yield_task_scx(struct rq *rq) +{ + struct task_struct *p = rq->curr; + + if (SCX_HAS_OP(yield)) + SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); + else + p->scx->slice = 0; +} + +static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) +{ + struct task_struct *from = rq->curr; + + if (SCX_HAS_OP(yield)) + return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); + else + return false; +} + +#ifdef CONFIG_SMP +/** + * move_task_to_local_dsq - Move a task from a different rq to a local DSQ + * @rq: rq to move the task into, currently locked + * @p: task to move + * @enq_flags: %SCX_ENQ_* + * + * Move @p which is currently on a different rq to @rq's local DSQ. The caller + * must: + * + * 1. Start with exclusive access to @p either through its DSQ lock or + * %SCX_OPSS_DISPATCHING flag. + * + * 2. Set @p->scx->holding_cpu to raw_smp_processor_id(). + * + * 3. Remember task_rq(@p). Release the exclusive access so that we don't + * deadlock with dequeue. + * + * 4. Lock @rq and the task_rq from #3. + * + * 5. Call this function. + * + * Returns %true if @p was successfully moved. %false after racing dequeue and + * losing. + */ +static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p, + u64 enq_flags) +{ + struct rq *task_rq; + + lockdep_assert_rq_held(rq); + + /* + * If dequeue got to @p while we were trying to lock both rq's, it'd + * have cleared @p->scx->holding_cpu to -1. While other cpus may have + * updated it to different values afterwards, as this operation can't be + * preempted or recurse, @p->scx->holding_cpu can never become + * raw_smp_processor_id() again before we're done. Thus, we can tell + * whether we lost to dequeue by testing whether @p->scx->holding_cpu is + * still raw_smp_processor_id(). + * + * See dispatch_dequeue() for the counterpart. + */ + if (unlikely(p->scx->holding_cpu != raw_smp_processor_id())) + return false; + + /* @p->rq couldn't have changed if we're still the holding cpu */ + task_rq = task_rq(p); + lockdep_assert_rq_held(task_rq); + + WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)); + deactivate_task(task_rq, p, 0); + set_task_cpu(p, cpu_of(rq)); + p->scx->sticky_cpu = cpu_of(rq); + + /* + * We want to pass scx-specific enq_flags but activate_task() will + * truncate the upper 32 bit. As we own @rq, we can pass them through + * @rq->scx->extra_enq_flags instead. + */ + WARN_ON_ONCE(rq->scx->extra_enq_flags); + rq->scx->extra_enq_flags = enq_flags; + activate_task(rq, p, 0); + rq->scx->extra_enq_flags = 0; + + return true; +} + +/** + * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked + * @rq: current rq which is locked + * @rf: rq_flags to use when unlocking @rq + * @src_rq: rq to move task from + * @dst_rq: rq to move task to + * + * We're holding @rq lock and trying to dispatch a task from @src_rq to + * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether + * @rq stays locked isn't important as long as the state is restored after + * dispatch_to_local_dsq_unlock(). + */ +static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf, + struct rq *src_rq, struct rq *dst_rq) +{ + rq_unpin_lock(rq, rf); + + if (src_rq == dst_rq) { + raw_spin_rq_unlock(rq); + raw_spin_rq_lock(dst_rq); + } else if (rq == src_rq) { + double_lock_balance(rq, dst_rq); + rq_repin_lock(rq, rf); + } else if (rq == dst_rq) { + double_lock_balance(rq, src_rq); + rq_repin_lock(rq, rf); + } else { + raw_spin_rq_unlock(rq); + double_rq_lock(src_rq, dst_rq); + } +} + +/** + * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock() + * @rq: current rq which is locked + * @rf: rq_flags to use when unlocking @rq + * @src_rq: rq to move task from + * @dst_rq: rq to move task to + * + * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return. + */ +static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf, + struct rq *src_rq, struct rq *dst_rq) +{ + if (src_rq == dst_rq) { + raw_spin_rq_unlock(dst_rq); + raw_spin_rq_lock(rq); + rq_repin_lock(rq, rf); + } else if (rq == src_rq) { + double_unlock_balance(rq, dst_rq); + } else if (rq == dst_rq) { + double_unlock_balance(rq, src_rq); + } else { + double_rq_unlock(src_rq, dst_rq); + raw_spin_rq_lock(rq); + rq_repin_lock(rq, rf); + } +} +#endif /* CONFIG_SMP */ + + +static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq) +{ + return likely(test_rq_online(rq)) && !is_migration_disabled(p) && + cpumask_test_cpu(cpu_of(rq), p->cpus_ptr); +} + +static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf, + struct scx_dispatch_q *dsq) +{ + struct scx_rq *scx_rq = rq->scx; + struct sched_ext_entity *entity; + struct task_struct *p; + struct rb_node *rb_node; + struct rq *task_rq; + bool moved = false; +retry: + if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq)) + return false; + + raw_spin_lock(&dsq->lock); + + list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) { + p = entity->task; + task_rq = task_rq(p); + if (rq == task_rq) + goto this_rq; + if (task_can_run_on_rq(p, rq)) + goto remote_rq; + } + + for (rb_node = rb_first_cached(&dsq->priq); rb_node; + rb_node = rb_next(rb_node)) { + entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); + p = entity->task; + task_rq = task_rq(p); + if (rq == task_rq) + goto this_rq; + if (task_can_run_on_rq(p, rq)) + goto remote_rq; + } + + raw_spin_unlock(&dsq->lock); + return false; + +this_rq: + /* @dsq is locked and @p is on this rq */ + WARN_ON_ONCE(p->scx->holding_cpu >= 0); + task_unlink_from_dsq(p, dsq); + list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo); + dsq->nr--; + scx_rq->local_dsq.nr++; + p->scx->dsq = &scx_rq->local_dsq; + raw_spin_unlock(&dsq->lock); + return true; + +remote_rq: +#ifdef CONFIG_SMP + /* + * @dsq is locked and @p is on a remote rq. @p is currently protected by + * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab + * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the + * rq lock or fail, do a little dancing from our side. See + * move_task_to_local_dsq(). + */ + WARN_ON_ONCE(p->scx->holding_cpu >= 0); + task_unlink_from_dsq(p, dsq); + dsq->nr--; + p->scx->holding_cpu = raw_smp_processor_id(); + raw_spin_unlock(&dsq->lock); + + rq_unpin_lock(rq, rf); + double_lock_balance(rq, task_rq); + rq_repin_lock(rq, rf); + + moved = move_task_to_local_dsq(rq, p, 0); + + double_unlock_balance(rq, task_rq); +#endif /* CONFIG_SMP */ + if (likely(moved)) + return true; + goto retry; +} + +enum dispatch_to_local_dsq_ret { + DTL_DISPATCHED, /* successfully dispatched */ + DTL_LOST, /* lost race to dequeue */ + DTL_NOT_LOCAL, /* destination is not a local DSQ */ + DTL_INVALID, /* invalid local dsq_id */ +}; + +/** + * dispatch_to_local_dsq - Dispatch a task to a local dsq + * @rq: current rq which is locked + * @rf: rq_flags to use when unlocking @rq + * @dsq_id: destination dsq ID + * @p: task to dispatch + * @enq_flags: %SCX_ENQ_* + * + * We're holding @rq lock and want to dispatch @p to the local DSQ identified by + * @dsq_id. This function performs all the synchronization dancing needed + * because local DSQs are protected with rq locks. + * + * The caller must have exclusive ownership of @p (e.g. through + * %SCX_OPSS_DISPATCHING). + */ +static enum dispatch_to_local_dsq_ret +dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id, + struct task_struct *p, u64 enq_flags) +{ + struct rq *src_rq = task_rq(p); + struct rq *dst_rq; + + /* + * We're synchronized against dequeue through DISPATCHING. As @p can't + * be dequeued, its task_rq and cpus_allowed are stable too. + */ + if (dsq_id == SCX_DSQ_LOCAL) { + dst_rq = rq; + } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { + s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; + + if (!ops_cpu_valid(cpu)) { + scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]", + cpu, p->comm, p->pid); + return DTL_INVALID; + } + dst_rq = cpu_rq(cpu); + } else { + return DTL_NOT_LOCAL; + } + + /* if dispatching to @rq that @p is already on, no lock dancing needed */ + if (rq == src_rq && rq == dst_rq) { + dispatch_enqueue(&dst_rq->scx->local_dsq, p, + enq_flags | SCX_ENQ_CLEAR_OPSS); + return DTL_DISPATCHED; + } + +#ifdef CONFIG_SMP + if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { + struct rq *locked_dst_rq = dst_rq; + bool dsp; + + /* + * @p is on a possibly remote @src_rq which we need to lock to + * move the task. If dequeue is in progress, it'd be locking + * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq + * lock while holding DISPATCHING. + * + * As DISPATCHING guarantees that @p is wholly ours, we can + * pretend that we're moving from a DSQ and use the same + * mechanism - mark the task under transfer with holding_cpu, + * release DISPATCHING and then follow the same protocol. + */ + p->scx->holding_cpu = raw_smp_processor_id(); + + /* store_release ensures that dequeue sees the above */ + atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE); + + dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq); + + /* + * We don't require the BPF scheduler to avoid dispatching to + * offline CPUs mostly for convenience but also because CPUs can + * go offline between scx_bpf_dispatch() calls and here. If @p + * is destined to an offline CPU, queue it on its current CPU + * instead, which should always be safe. As this is an allowed + * behavior, don't trigger an ops error. + */ + if (unlikely(!test_rq_online(dst_rq))) + dst_rq = src_rq; + + if (src_rq == dst_rq) { + /* + * As @p is staying on the same rq, there's no need to + * go through the full deactivate/activate cycle. + * Optimize by abbreviating the operations in + * move_task_to_local_dsq(). + */ + dsp = p->scx->holding_cpu == raw_smp_processor_id(); + if (likely(dsp)) { + p->scx->holding_cpu = -1; + dispatch_enqueue(&dst_rq->scx->local_dsq, p, + enq_flags); + } + } else { + dsp = move_task_to_local_dsq(dst_rq, p, enq_flags); + } + + /* if the destination CPU is idle, wake it up */ + if (dsp && p->sched_class > dst_rq->curr->sched_class) + resched_curr(dst_rq); + + dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq); + + return dsp ? DTL_DISPATCHED : DTL_LOST; + } +#endif /* CONFIG_SMP */ + + scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", + cpu_of(dst_rq), p->comm, p->pid); + return DTL_INVALID; +} + +/** + * finish_dispatch - Asynchronously finish dispatching a task + * @rq: current rq which is locked + * @rf: rq_flags to use when unlocking @rq + * @p: task to finish dispatching + * @qseq_at_dispatch: qseq when @p started getting dispatched + * @dsq_id: destination DSQ ID + * @enq_flags: %SCX_ENQ_* + * + * Dispatching to local DSQs may need to wait for queueing to complete or + * require rq lock dancing. As we don't wanna do either while inside + * ops.dispatch() to avoid locking order inversion, we split dispatching into + * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the + * task and its qseq. Once ops.dispatch() returns, this function is called to + * finish up. + * + * There is no guarantee that @p is still valid for dispatching or even that it + * was valid in the first place. Make sure that the task is still owned by the + * BPF scheduler and claim the ownership before dispatching. + */ +static void finish_dispatch(struct rq *rq, struct rq_flags *rf, + struct task_struct *p, u64 qseq_at_dispatch, + u64 dsq_id, u64 enq_flags) +{ + struct scx_dispatch_q *dsq; + u64 opss; + + touch_core_sched_dispatch(rq, p); +retry: + /* + * No need for _acquire here. @p is accessed only after a successful + * try_cmpxchg to DISPATCHING. + */ + opss = atomic64_read(&p->scx->ops_state); + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_DISPATCHING: + case SCX_OPSS_NONE: + /* someone else already got to it */ + return; + case SCX_OPSS_QUEUED: + /* + * If qseq doesn't match, @p has gone through at least one + * dispatch/dequeue and re-enqueue cycle between + * scx_bpf_dispatch() and here and we have no claim on it. + */ + if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) + return; + + /* + * While we know @p is accessible, we don't yet have a claim on + * it - the BPF scheduler is allowed to dispatch tasks + * spuriously and there can be a racing dequeue attempt. Let's + * claim @p by atomically transitioning it from QUEUED to + * DISPATCHING. + */ + if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss, + SCX_OPSS_DISPATCHING))) + break; + goto retry; + case SCX_OPSS_QUEUEING: + /* + * do_enqueue_task() is in the process of transferring the task + * to the BPF scheduler while holding @p's rq lock. As we aren't + * holding any kernel or BPF resource that the enqueue path may + * depend upon, it's safe to wait. + */ + wait_ops_state(p, opss); + goto retry; + } + + BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED)); + + switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) { + case DTL_DISPATCHED: + break; + case DTL_LOST: + break; + case DTL_INVALID: + dsq_id = SCX_DSQ_GLOBAL; + fallthrough; + case DTL_NOT_LOCAL: + dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()), + dsq_id, p); + dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + break; + } +} + +static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf) +{ + struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); + u32 u; + + for (u = 0; u < dspc->buf_cursor; u++) { + struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u]; + + finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id, + ent->enq_flags); + } + + dspc->nr_tasks += dspc->buf_cursor; + dspc->buf_cursor = 0; +} + +static int balance_one(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf, bool local) +{ + struct scx_rq *scx_rq = rq->scx; + struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); + bool prev_on_scx = prev->sched_class == &ext_sched_class; + int nr_loops = SCX_DSP_MAX_LOOPS; + + lockdep_assert_rq_held(rq); + + if (static_branch_unlikely(&scx_ops_cpu_preempt) && + unlikely(rq->scx->cpu_released)) { + /* + * If the previous sched_class for the current CPU was not SCX, + * notify the BPF scheduler that it again has control of the + * core. This callback complements ->cpu_release(), which is + * emitted in scx_notify_pick_next_task(). + */ + if (SCX_HAS_OP(cpu_acquire)) + SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq), + NULL); + rq->scx->cpu_released = false; + } + + if (prev_on_scx) { + WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP)); + update_curr_scx(rq); + + /* + * If @prev is runnable & has slice left, it has priority and + * fetching more just increases latency for the fetched tasks. + * Tell put_prev_task_scx() to put @prev on local_dsq. If the + * BPF scheduler wants to handle this explicitly, it should + * implement ->cpu_released(). + * + * See scx_ops_disable_workfn() for the explanation on the + * disabling() test. + * + * When balancing a remote CPU for core-sched, there won't be a + * following put_prev_task_scx() call and we don't own + * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the + * same conditions later and pick @rq->curr accordingly. + */ + if ((prev->scx->flags & SCX_TASK_QUEUED) && + prev->scx->slice && !scx_ops_disabling()) { + if (local) + prev->scx->flags |= SCX_TASK_BAL_KEEP; + return 1; + } + } + + /* if there already are tasks to run, nothing to do */ + if (scx_rq->local_dsq.nr) + return 1; + + if (consume_dispatch_q(rq, rf, &scx_dsq_global)) + return 1; + + if (!SCX_HAS_OP(dispatch)) + return 0; + + dspc->rq = rq; + dspc->rf = rf; + + /* + * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, + * the local DSQ might still end up empty after a successful + * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() + * produced some tasks, retry. The BPF scheduler may depend on this + * looping behavior to simplify its implementation. + */ + do { + dspc->nr_tasks = 0; + + SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), + prev_on_scx ? prev : NULL); + + flush_dispatch_buf(rq, rf); + + if (scx_rq->local_dsq.nr) + return 1; + if (consume_dispatch_q(rq, rf, &scx_dsq_global)) + return 1; + + /* + * ops.dispatch() can trap us in this loop by repeatedly + * dispatching ineligible tasks. Break out once in a while to + * allow the watchdog to run. As IRQ can't be enabled in + * balance(), we want to complete this scheduling cycle and then + * start a new one. IOW, we want to call resched_curr() on the + * next, most likely idle, task, not the current one. Use + * scx_bpf_kick_cpu() for deferred kicking. + */ + if (unlikely(!--nr_loops)) { + scx_bpf_kick_cpu(cpu_of(rq), 0); + break; + } + } while (dspc->nr_tasks); + + return 0; +} + +static int balance_scx(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) +{ + int ret; + + ret = balance_one(rq, prev, rf, true); +#ifdef CONFIG_SCHED_SMT + /* + * When core-sched is enabled, this ops.balance() call will be followed + * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx() + * on the SMT siblings. Balance the siblings too. + */ + if (sched_core_enabled(rq)) { + const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); + int scpu; + + for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { + struct rq *srq = cpu_rq(scpu); + struct rq_flags srf; + struct task_struct *sprev = srq->curr; + + /* + * While core-scheduling, rq lock is shared among + * siblings but the debug annotations and rq clock + * aren't. Do pinning dance to transfer the ownership. + */ + WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); + rq_unpin_lock(rq, rf); + rq_pin_lock(srq, &srf); + + update_rq_clock(srq); + balance_one(srq, sprev, &srf, false); + + rq_unpin_lock(srq, &srf); + rq_repin_lock(rq, rf); + } + } +#endif + return ret; +} + +static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) +{ + if (p->scx->flags & SCX_TASK_QUEUED) { + /* + * Core-sched might decide to execute @p before it is + * dispatched. Call ops_dequeue() to notify the BPF scheduler. + */ + ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); + dispatch_dequeue(rq->scx, p); + } + + p->se.exec_start = rq_clock_task(rq); + + /* see dequeue_task_scx() on why we skip when !QUEUED */ + if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED)) + SCX_CALL_OP_TASK(SCX_KF_REST, running, p); + + //if(p->scx->flags & SCX_TASK_QUEUED) + // scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock); + + watchdog_unwatch_task(p, true); + + /* + * @p is getting newly scheduled or got kicked after someone updated its + * slice. Refresh whether tick can be stopped. See can_stop_tick_scx(). + */ + if ((p->scx->slice == SCX_SLICE_INF) != + (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) { + if (p->scx->slice == SCX_SLICE_INF) + rq->scx->flags |= SCX_RQ_CAN_STOP_TICK; + else + rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK; + + sched_update_tick_dependency(rq); + } +} + +static void put_prev_task_scx(struct rq *rq, struct task_struct *p) +{ +#ifndef CONFIG_SMP + /* + * UP workaround. + * + * Because SCX may transfer tasks across CPUs during dispatch, dispatch + * is performed from its balance operation which isn't called in UP. + * Let's work around by calling it from the operations which come right + * after. + * + * 1. If the prev task is on SCX, pick_next_task() calls + * .put_prev_task() right after. As .put_prev_task() is also called + * from other places, we need to distinguish the calls which can be + * done by looking at the previous task's state - if still queued or + * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task(). + * This case is handled here. + * + * 2. If the prev task is not on SCX, the first following call into SCX + * will be .pick_next_task(), which is covered by calling + * balance_scx() from pick_next_task_scx(). + * + * Note that we can't merge the first case into the second as + * balance_scx() must be called before the previous SCX task goes + * through put_prev_task_scx(). + * + * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf. + * Pass in %NULL. + */ + if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP)) + balance_scx(rq, p, NULL); +#endif + + update_curr_scx(rq); + + /* see dequeue_task_scx() on why we skip when !QUEUED */ + if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED)) + SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); + + //if(p->scx->flags & SCX_TASK_QUEUED) + // scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock); + + + /* + * If we're being called from put_prev_task_balance(), balance_scx() may + * have decided that @p should keep running. + */ + if (p->scx->flags & SCX_TASK_BAL_KEEP) { + p->scx->flags &= ~SCX_TASK_BAL_KEEP; + watchdog_watch_task(rq, p); + dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); + return; + } + + if (p->scx->flags & SCX_TASK_QUEUED) { + watchdog_watch_task(rq, p); + + /* + * If @p has slice left and balance_scx() didn't tag it for + * keeping, @p is getting preempted by a higher priority + * scheduler class or core-sched forcing a different task. Leave + * it at the head of the local DSQ. + */ + if (p->scx->slice && !scx_ops_disabling()) { + dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD); + return; + } + + /* + * If we're in the pick_next_task path, balance_scx() should + * have already populated the local DSQ if there are any other + * available tasks. If empty, tell ops.enqueue() that @p is the + * only one available for this cpu. ops.enqueue() should put it + * on the local DSQ so that the subsequent pick_next_task_scx() + * can find the task unless it wants to trigger a separate + * follow-up scheduling event. + */ + if (list_empty(&rq->scx->local_dsq.fifo)) + do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1); + else + do_enqueue_task(rq, p, 0, -1); + } +} + +static struct task_struct *first_local_task(struct rq *rq) +{ + struct rb_node *rb_node; + struct sched_ext_entity *entity; + + if (!list_empty(&rq->scx->local_dsq.fifo)) { + entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo); + return entity->task; + } + + rb_node = rb_first_cached(&rq->scx->local_dsq.priq); + if (rb_node) { + entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq); + return entity->task; + } + + return NULL; +} + +static struct task_struct *pick_next_task_scx(struct rq *rq) +{ + struct task_struct *p; + +#ifndef CONFIG_SMP + /* UP workaround - see the comment at the head of put_prev_task_scx() */ + if (unlikely(rq->curr->sched_class != &ext_sched_class)) + balance_scx(rq, rq->curr, NULL); +#endif + + p = first_local_task(rq); + if (!p) + return NULL; + + if (unlikely(!p->scx->slice)) { + if (!scx_ops_disabling() && !scx_warned_zero_slice) { + printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n", + p->comm, p->pid); + scx_warned_zero_slice = true; + } + p->scx->slice = SCX_SLICE_DFL; + } + + set_next_task_scx(rq, p, true); + + return p; +} + +#ifdef CONFIG_SCHED_CORE +/** + * scx_prio_less - Task ordering for core-sched + * @a: task A + * @b: task B + * + * Core-sched is implemented as an additional scheduling layer on top of the + * usual sched_class'es and needs to find out the expected task ordering. For + * SCX, core-sched calls this function to interrogate the task ordering. + * + * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used + * to implement the default task ordering. The older the timestamp, the higher + * prority the task - the global FIFO ordering matching the default scheduling + * behavior. + * + * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to + * implement FIFO ordering within each local DSQ. See pick_task_scx(). + */ +bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, + bool in_fi) +{ + /* + * The const qualifiers are dropped from task_struct pointers when + * calling ops.core_sched_before(). Accesses are controlled by the + * verifier. + */ + if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling()) + return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, + (struct task_struct *)a, + (struct task_struct *)b); + else + return time_after64(a->scx->core_sched_at, b->scx->core_sched_at); +} + +/** + * pick_task_scx - Pick a candidate task for core-sched + * @rq: rq to pick the candidate task from + * + * Core-sched calls this function on each SMT sibling to determine the next + * tasks to run on the SMT siblings. balance_one() has been called on all + * siblings and put_prev_task_scx() has been called only for the current CPU. + * + * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look + * at the first task in the local dsq. @rq->curr has to be considered explicitly + * to mimic %SCX_TASK_BAL_KEEP. + */ +static struct task_struct *pick_task_scx(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + struct task_struct *first = first_local_task(rq); + + if (curr->scx->flags & SCX_TASK_QUEUED) { + /* is curr the only runnable task? */ + if (!first) + return curr; + + /* + * Does curr trump first? We can always go by core_sched_at for + * this comparison as it represents global FIFO ordering when + * the default core-sched ordering is used and local-DSQ FIFO + * ordering otherwise. + * + * We can have a task with an earlier timestamp on the DSQ. For + * example, when a current task is preempted by a sibling + * picking a different cookie, the task would be requeued at the + * head of the local DSQ with an earlier timestamp than the + * core-sched picked next task. Besides, the BPF scheduler may + * dispatch any tasks to the local DSQ anytime. + */ + if (curr->scx->slice && time_before64(curr->scx->core_sched_at, + first->scx->core_sched_at)) + return curr; + } + + return first; /* this may be %NULL */ +} +#endif /* CONFIG_SCHED_CORE */ + +static enum scx_cpu_preempt_reason +preempt_reason_from_class(const struct sched_class *class) +{ +#ifdef CONFIG_SMP + if (class == &stop_sched_class) + return SCX_CPU_PREEMPT_STOP; +#endif + if (class == &dl_sched_class) + return SCX_CPU_PREEMPT_DL; + if (class == &rt_sched_class) + return SCX_CPU_PREEMPT_RT; + return SCX_CPU_PREEMPT_UNKNOWN; +} + +void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task, + const struct sched_class *active) +{ + lockdep_assert_rq_held(rq); + + /* + * The callback is conceptually meant to convey that the CPU is no + * longer under the control of SCX. Therefore, don't invoke the + * callback if the CPU is is staying on SCX, or going idle (in which + * case the SCX scheduler has actively decided not to schedule any + * tasks on the CPU). + */ + if (likely(active >= &ext_sched_class)) + return; + + /* + * At this point we know that SCX was preempted by a higher priority + * sched_class, so invoke the ->cpu_release() callback if we have not + * done so already. We only send the callback once between SCX being + * preempted, and it regaining control of the CPU. + * + * ->cpu_release() complements ->cpu_acquire(), which is emitted the + * next time that balance_scx() is invoked. + */ + if (!rq->scx->cpu_released) { + if (SCX_HAS_OP(cpu_release)) { + struct scx_cpu_release_args args = { + .reason = preempt_reason_from_class(active), + .task = task, + }; + + SCX_CALL_OP(SCX_KF_CPU_RELEASE, + cpu_release, cpu_of(rq), &args); + } + rq->scx->cpu_released = true; + } +} + +#ifdef CONFIG_SMP + +static bool test_and_clear_cpu_idle(int cpu) +{ + if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) { + if (cpumask_empty(idle_masks.cpu)) + scx_has_idle_cpus = false; + return true; + } else { + return false; + } +} + +static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) +{ + int cpu; + + do { + cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); + if (cpu < nr_cpu_ids) { + const struct cpumask *sbm = topology_sibling_cpumask(cpu); + + /* + * If offline, @cpu is not its own sibling and we can + * get caught in an infinite loop as @cpu is never + * cleared from idle_masks.smt. Clear @cpu directly in + * such cases. + */ + if (likely(cpumask_test_cpu(cpu, sbm))) + cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm); + else + cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu)); + } else { + cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); + if (cpu >= nr_cpu_ids) + return -EBUSY; + } + } while (!test_and_clear_cpu_idle(cpu)); + + return cpu; +} + +static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags) +{ + s32 cpu; + + if (!static_branch_likely(&scx_builtin_idle_enabled)) { + scx_ops_error("built-in idle tracking is disabled"); + return prev_cpu; + } + + /* + * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the + * local DSQ of the waker. + */ + if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 && + scx_has_idle_cpus && !(current->flags & PF_EXITING)) { + cpu = smp_processor_id(); + if (cpumask_test_cpu(cpu, p->cpus_ptr)) { + p->scx->flags |= SCX_TASK_ENQ_LOCAL; + return cpu; + } + } + + /* if the previous CPU is idle, dispatch directly to it */ + if (test_and_clear_cpu_idle(prev_cpu)) { + p->scx->flags |= SCX_TASK_ENQ_LOCAL; + return prev_cpu; + } + + if (p->nr_cpus_allowed == 1) + return prev_cpu; + + cpu = scx_pick_idle_cpu(p->cpus_ptr); + if (cpu >= 0) { + p->scx->flags |= SCX_TASK_ENQ_LOCAL; + return cpu; + } + + return prev_cpu; +} + +static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) +{ + if (SCX_HAS_OP(select_cpu)) { + s32 cpu; + + cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu, + wake_flags); + if (ops_cpu_valid(cpu)) { + return cpu; + } else { + scx_ops_error("select_cpu returned invalid cpu %d", cpu); + return prev_cpu; + } + } else { + return scx_select_cpu_dfl(p, prev_cpu, wake_flags); + } +} + +static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx) +{ + set_cpus_allowed_common(p, ctx); + + /* + * The effective cpumask is stored in @p->cpus_ptr which may temporarily + * differ from the configured one in @p->cpus_mask. Always tell the bpf + * scheduler the effective one. + * + * Fine-grained memory write control is enforced by BPF making the const + * designation pointless. Cast it away when calling the operation. + */ + if (SCX_HAS_OP(set_cpumask)) + SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, + (struct cpumask *)p->cpus_ptr); +} + +static void reset_idle_masks(void) +{ + /* consider all cpus idle, should converge to the actual state quickly */ + cpumask_setall(idle_masks.cpu); + cpumask_setall(idle_masks.smt); + scx_has_idle_cpus = true; +} + +void __scx_update_idle(struct rq *rq, bool idle) +{ + int cpu = cpu_of(rq); + struct cpumask *sib_mask = topology_sibling_cpumask(cpu); + + if (SCX_HAS_OP(update_idle)) { + SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); + if (!static_branch_unlikely(&scx_builtin_idle_enabled)) + return; + } + + if (idle) { + cpumask_set_cpu(cpu, idle_masks.cpu); + if (!scx_has_idle_cpus) + scx_has_idle_cpus = true; + + /* + * idle_masks.smt handling is racy but that's fine as it's only + * for optimization and self-correcting. + */ + for_each_cpu(cpu, sib_mask) { + if (!cpumask_test_cpu(cpu, idle_masks.cpu)) + return; + } + cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask); + } else { + cpumask_clear_cpu(cpu, idle_masks.cpu); + if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu)) + scx_has_idle_cpus = false; + + cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask); + } +} + +#else /* !CONFIG_SMP */ + +static bool test_and_clear_cpu_idle(int cpu) { return false; } +static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; } +static void reset_idle_masks(void) {} + +#endif /* CONFIG_SMP */ + +static bool check_rq_for_timeouts(struct rq *rq) +{ + struct sched_ext_entity *entity; + struct task_struct *p; + struct rq_flags rf; + bool timed_out = false; + + rq_lock_irqsave(rq, &rf); + list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) { + unsigned long last_runnable; + + p = entity->task; + last_runnable = p->scx->runnable_at; + + if (unlikely(time_after(jiffies, + last_runnable + scx_watchdog_timeout))) { + u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); + + scx_ops_error_type(SCX_EXIT_ERROR_STALL, + "%s[%d] failed to run for %u.%03us", + p->comm, p->pid, + dur_ms / 1000, dur_ms % 1000); + timed_out = true; + break; + } + } + rq_unlock_irqrestore(rq, &rf); + + return timed_out; +} + +static void scx_watchdog_workfn(struct work_struct *work) +{ + int cpu; + + scx_watchdog_timestamp = jiffies; + + for_each_online_cpu(cpu) { + if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) + break; + + cond_resched(); + } + queue_delayed_work(system_unbound_wq, to_delayed_work(work), + scx_watchdog_timeout / 2); +} + +static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) +{ + update_curr_scx(rq); + //scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock); + /* + * While disabling, always resched and refresh core-sched timestamp as + * we can't trust the slice management or ops.core_sched_before(). + */ + if (scx_ops_disabling()) { + curr->scx->slice = 0; + touch_core_sched(rq, curr); + } + + if (!curr->scx->slice) + resched_curr(rq); +} + +#define SCX_ENABLE_ARGS_INIT_CGROUP(tg) + +static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg) +{ + int ret; + + WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED); + + p->scx->disallow = false; + + if (SCX_HAS_OP(prep_enable)) { + struct scx_enable_args args = { + SCX_ENABLE_ARGS_INIT_CGROUP(tg) + }; + + ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args); + if (unlikely(ret)) { + ret = ops_sanitize_err("prep_enable", ret); + return ret; + } + } + //scx_sched_init_task(p); + + if (p->scx->disallow) { + struct rq *rq; + struct rq_flags rf; + + rq = task_rq_lock(p, &rf); + + /* + * We're either in fork or load path and @p->policy will be + * applied right after. Reverting @p->policy here and rejecting + * %SCHED_EXT transitions from scx_check_setscheduler() + * guarantees that if ops.prep_enable() sets @p->disallow, @p + * can never be in SCX. + */ + if (p->policy == SCHED_EXT) { + p->policy = SCHED_NORMAL; + atomic64_inc(&scx_nr_rejected); + } + + task_rq_unlock(rq, p, &rf); + } + + p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET); + return 0; +} + +static void scx_ops_enable_task(struct task_struct *p) +{ + lockdep_assert_rq_held(task_rq(p)); + WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED)); + + if (SCX_HAS_OP(enable)) { + struct scx_enable_args args = { + SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) + }; + SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args); + } + p->scx->flags &= ~SCX_TASK_OPS_PREPPED; + p->scx->flags |= SCX_TASK_OPS_ENABLED; +} + +static void scx_ops_disable_task(struct task_struct *p) +{ + lockdep_assert_rq_held(task_rq(p)); + + if (p->scx->flags & SCX_TASK_OPS_PREPPED) { + if (SCX_HAS_OP(cancel_enable)) { + struct scx_enable_args args = { + SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p)) + }; + SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args); + } + p->scx->flags &= ~SCX_TASK_OPS_PREPPED; + } else if (p->scx->flags & SCX_TASK_OPS_ENABLED) { + if (SCX_HAS_OP(disable)) + SCX_CALL_OP(SCX_KF_REST, disable, p); + p->scx->flags &= ~SCX_TASK_OPS_ENABLED; + } +} + +static void set_task_scx_weight(struct task_struct *p) +{ + u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; + + p->scx->weight = sched_weight_to_cgroup(weight); +} + +/** + * refresh_scx_weight - Refresh a task's ext weight + * @p: task to refresh ext weight for + * + * @p->scx->weight carries the task's static priority in cgroup weight scale to + * enable easy access from the BPF scheduler. To keep it synchronized with the + * current task priority, this function should be called when a new task is + * created, priority is changed for a task on sched_ext, and a task is switched + * to sched_ext from other classes. + */ +static void refresh_scx_weight(struct task_struct *p) +{ + lockdep_assert_rq_held(task_rq(p)); + set_task_scx_weight(p); + if (SCX_HAS_OP(set_weight)) + SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight); +} + +void scx_pre_fork(struct task_struct *p) +{ + p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL); + if (!p->scx) { + goto lock; + } + + p->scx->dsq = NULL; + INIT_LIST_HEAD(&p->scx->dsq_node.fifo); + RB_CLEAR_NODE(&p->scx->dsq_node.priq); + INIT_LIST_HEAD(&p->scx->watchdog_node); + p->scx->flags = 0; + p->scx->weight = 0; + p->scx->sticky_cpu = -1; + p->scx->holding_cpu = -1; + p->scx->kf_mask = 0; + atomic64_set(&p->scx->ops_state, 0); + p->scx->runnable_at = INITIAL_JIFFIES; + p->scx->slice = SCX_SLICE_DFL; + p->scx->task = p; + p->sched_prop = 0; + + /* + * BPF scheduler enable/disable paths want to be able to iterate and + * update all tasks which can become complex when racing forks. As + * enable/disable are very cold paths, let's use a percpu_rwsem to + * exclude forks. + */ +lock: + percpu_down_read(&scx_fork_rwsem); +} + +int scx_fork(struct task_struct *p) +{ + percpu_rwsem_assert_held(&scx_fork_rwsem); + + if (scx_enabled()) + return scx_ops_prepare_task(p, task_group(p)); + else + return 0; +} + +void scx_post_fork(struct task_struct *p) +{ + if (scx_enabled()) { + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + /* + * Set the weight manually before calling ops.enable() so that + * the scheduler doesn't see a stale value if they inspect the + * task struct. We'll invoke ops.set_weight() afterwards, as it + * would be odd to receive a callback on the task before we + * tell the scheduler that it's been fully enabled. + */ + set_task_scx_weight(p); + scx_ops_enable_task(p); + refresh_scx_weight(p); + task_rq_unlock(rq, p, &rf); + } + + spin_lock_irq(&scx_tasks_lock); + list_add_tail(&p->scx->tasks_node, &scx_tasks); + spin_unlock_irq(&scx_tasks_lock); + + percpu_up_read(&scx_fork_rwsem); +} + +void scx_cancel_fork(struct task_struct *p) +{ + if (scx_enabled()) + scx_ops_disable_task(p); + percpu_up_read(&scx_fork_rwsem); +} + +void sched_ext_free(struct task_struct *p) +{ + unsigned long flags; + + spin_lock_irqsave(&scx_tasks_lock, flags); + list_del_init(&p->scx->tasks_node); + spin_unlock_irqrestore(&scx_tasks_lock, flags); + + /* + * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED -> + * ENABLED transitions can't race us. Disable ops for @p. + */ + if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) { + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + scx_ops_disable_task(p); + task_rq_unlock(rq, p, &rf); + } +} + +static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) +{ +} + +static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} +static void switched_to_scx(struct rq *rq, struct task_struct *p) {} + +int scx_check_setscheduler(struct task_struct *p, int policy) +{ + lockdep_assert_rq_held(task_rq(p)); + + /* if disallow, reject transitioning into SCX */ + if (scx_enabled() && READ_ONCE(p->scx->disallow) && + p->policy != policy && policy == SCHED_EXT) + return -EACCES; + + return 0; +} + +#ifdef CONFIG_NO_HZ_FULL +bool scx_can_stop_tick(struct rq *rq) +{ + struct task_struct *p = rq->curr; + + if (scx_ops_disabling()) + return false; + + if (p->sched_class != &ext_sched_class) + return true; + + /* + * @rq can dispatch from different DSQs, so we can't tell whether it + * needs the tick or not by looking at nr_running. Allow stopping ticks + * iff the BPF scheduler indicated so. See set_next_task_scx(). + */ + return rq->scx->flags & SCX_RQ_CAN_STOP_TICK; +} +#endif + +static inline void scx_cgroup_lock(void) {} +static inline void scx_cgroup_unlock(void) {} + +/* + * Omitted operations: + * + * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the + * task isn't tied to the CPU at that point. Preemption is implemented by + * resetting the victim task's slice to 0 and triggering reschedule on the + * target CPU. + * + * - migrate_task_rq: Unncessary as task to cpu mapping is transient. + * + * - task_fork/dead: We need fork/dead notifications for all tasks regardless of + * their current sched_class. Call them directly from sched core instead. + * + * - task_woken, switched_from: Unnecessary. + */ +DEFINE_SCHED_CLASS(ext) = { + .enqueue_task = enqueue_task_scx, + .dequeue_task = dequeue_task_scx, + .yield_task = yield_task_scx, + .yield_to_task = yield_to_task_scx, + + .check_preempt_curr = check_preempt_curr_scx, + + .pick_next_task = pick_next_task_scx, + + .put_prev_task = put_prev_task_scx, + .set_next_task = set_next_task_scx, + +#ifdef CONFIG_SMP + .balance = balance_scx, + .select_task_rq = select_task_rq_scx, + .set_cpus_allowed = set_cpus_allowed_scx, +#endif + +#ifdef CONFIG_SCHED_CORE + .pick_task = pick_task_scx, +#endif + + .task_tick = task_tick_scx, + + .switched_to = switched_to_scx, + .prio_changed = prio_changed_scx, + + .update_curr = update_curr_scx, + +#ifdef CONFIG_UCLAMP_TASK + .uclamp_enabled = 0, +#endif +}; + +static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) +{ + memset(dsq, 0, sizeof(*dsq)); + + raw_spin_lock_init(&dsq->lock); + INIT_LIST_HEAD(&dsq->fifo); + dsq->id = dsq_id; +} + +static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) +{ + struct scx_dispatch_q *dsq; + + if (dsq_id & SCX_DSQ_FLAG_BUILTIN) + return ERR_PTR(-EINVAL); + + dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node); + if (!dsq) + return ERR_PTR(-ENOMEM); + + init_dsq(dsq, dsq_id); + + return dsq; +} + +static void free_dsq_irq_workfn(struct irq_work *irq_work) +{ + struct llist_node *to_free = llist_del_all(&dsqs_to_free); + struct scx_dispatch_q *dsq, *tmp_dsq; + + llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) + kfree_rcu(dsq, rcu); +} + +static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); + +static void destroy_dsq(u64 dsq_id) +{ +} + +static void scx_cgroup_exit(void) {} +static int scx_cgroup_init(void) { return 0; } +static void scx_cgroup_config_knobs(void) {} + +/* + * Used by sched_fork() and __setscheduler_prio() to pick the matching + * sched_class. dl/rt are already handled. + */ +bool task_on_scx(struct task_struct *p) +{ + if (!scx_enabled() || scx_ops_disabling()) + return false; + if (READ_ONCE(scx_switching_all)) + return true; + return p->policy == SCHED_EXT; +} + +static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags) +{ + if (enq_flags & SCX_ENQ_LAST) + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); + else + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); +} + +static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {} + +static void scx_ops_disable_workfn(struct kthread_work *work) +{ + struct scx_exit_info *ei = &scx_exit_info; + struct scx_task_iter sti; + struct task_struct *p; + const char *reason; + int i, cpu, type; + + type = atomic_read(&scx_exit_type); + while (true) { + /* + * NONE indicates that a new scx_ops has been registered since + * disable was scheduled - don't kill the new ops. DONE + * indicates that the ops has already been disabled. + */ + if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE) + return; + if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE)) + break; + } + + cancel_delayed_work_sync(&scx_watchdog_work); + + switch (type) { + case SCX_EXIT_UNREG: + reason = "BPF scheduler unregistered"; + break; + case SCX_EXIT_SYSRQ: + reason = "disabled by sysrq-S"; + break; + case SCX_EXIT_ERROR: + reason = "runtime error"; + break; + case SCX_EXIT_ERROR_BPF: + reason = "scx_bpf_error"; + break; + case SCX_EXIT_ERROR_STALL: + reason = "runnable task stall"; + break; + default: + reason = ""; + } + + ei->type = type; + strlcpy(ei->reason, reason, sizeof(ei->reason)); + + switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { + case SCX_OPS_DISABLED: + pr_warn("sched_ext: ops error detected without ops (%s)\n", + scx_exit_info.msg); + WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != + SCX_OPS_DISABLING); + return; + case SCX_OPS_PREPPING: + goto forward_progress_guaranteed; + case SCX_OPS_DISABLING: + /* shouldn't happen but handle it like ENABLING if it does */ + WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); + fallthrough; + case SCX_OPS_ENABLING: + case SCX_OPS_ENABLED: + break; + } + + /* + * DISABLING is set and ops was either ENABLING or ENABLED indicating + * that the ops and static branches are set. + * + * We must guarantee that all runnable tasks make forward progress + * without trusting the BPF scheduler. We can't grab any mutexes or + * rwsems as they might be held by tasks that the BPF scheduler is + * forgetting to run, which unfortunately also excludes toggling the + * static branches. + * + * Let's work around by overriding a couple ops and modifying behaviors + * based on the DISABLING state and then cycling the tasks through + * dequeue/enqueue to force global FIFO scheduling. + * + * a. ops.enqueue() and .dispatch() are overridden for simple global + * FIFO scheduling. + * + * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value + * can't be trusted. Whenever a tick triggers, the running task is + * rotated to the tail of the queue with core_sched_at touched. + * + * c. pick_next_task() suppresses zero slice warning. + * + * d. scx_prio_less() reverts to the default core_sched_at order. + */ + scx_ops.enqueue = scx_ops_fallback_enqueue; + scx_ops.dispatch = scx_ops_fallback_dispatch; + + spin_lock_irq(&scx_tasks_lock); + scx_task_iter_init(&sti); + while ((p = scx_task_iter_next_filtered_locked(&sti))) { + if (READ_ONCE(p->__state) != TASK_DEAD) { + SCHED_CHANGE_BLOCK(task_rq(p), p, + DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* cycling deq/enq is enough, see above */ + } + } + } + scx_task_iter_exit(&sti); + spin_unlock_irq(&scx_tasks_lock); + + /* kick all CPUs to restore ticks */ + for_each_possible_cpu(cpu) + resched_cpu(cpu); + +forward_progress_guaranteed: + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_ops_enable_mutex); + + static_branch_disable(&__scx_switched_all); + WRITE_ONCE(scx_switching_all, false); + + /* avoid racing against fork and cgroup changes */ + cpus_read_lock(); + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + spin_lock_irq(&scx_tasks_lock); + scx_task_iter_init(&sti); + while ((p = scx_task_iter_next_filtered_locked(&sti))) { + const struct sched_class *old_class = p->sched_class; + struct rq *rq = task_rq(p); + bool alive = READ_ONCE(p->__state) != TASK_DEAD; + + update_rq_clock(rq); + + SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | + DEQUEUE_NOCLOCK) { + p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL); + + __setscheduler_prio(p, p->prio); + } + + if (alive) + check_class_changed(task_rq(p), p, old_class, p->prio); + + scx_ops_disable_task(p); + } + scx_task_iter_exit(&sti); + spin_unlock_irq(&scx_tasks_lock); + + /* no task is on scx, turn off all the switches and flush in-progress calls */ + static_branch_disable_cpuslocked(&__scx_ops_enabled); + for (i = 0; i < SCX_NR_ONLINE_OPS; i++) + static_branch_disable_cpuslocked(&scx_has_op[i]); + static_branch_disable_cpuslocked(&scx_ops_enq_last); + static_branch_disable_cpuslocked(&scx_ops_enq_exiting); + static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); + static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); + synchronize_rcu(); + + scx_cgroup_exit(); + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + cpus_read_unlock(); + + if (ei->type >= SCX_EXIT_ERROR) { + printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name); + + if (ei->msg[0] == '\0') + printk(KERN_ERR "sched_ext: %s\n", ei->reason); + else + printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg); + + stack_trace_print(ei->bt, ei->bt_len, 2); + } + + if (scx_ops.exit) + SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); + + memset(&scx_ops, 0, sizeof(scx_ops)); + + free_percpu(scx_dsp_buf); + scx_dsp_buf = NULL; + scx_dsp_max_batch = 0; + + mutex_unlock(&scx_ops_enable_mutex); + + WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != + SCX_OPS_DISABLING); + + scx_cgroup_config_knobs(); +} + +static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); + +static void schedule_scx_ops_disable_work(void) +{ + struct kthread_worker *helper = READ_ONCE(scx_ops_helper); + + /* + * We may be called spuriously before the first bpf_sched_ext_reg(). If + * scx_ops_helper isn't set up yet, there's nothing to do. + */ + if (helper) + kthread_queue_work(helper, &scx_ops_disable_work); +} + +static void scx_ops_disable(enum scx_exit_type type) +{ + int none = SCX_EXIT_NONE; + + if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)) + type = SCX_EXIT_ERROR; + + atomic_try_cmpxchg(&scx_exit_type, &none, type); + + schedule_scx_ops_disable_work(); +} + +static void scx_ops_error_irq_workfn(struct irq_work *irq_work) +{ + schedule_scx_ops_disable_work(); +} + +static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); + +__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, + const char *fmt, ...) +{ + struct scx_exit_info *ei = &scx_exit_info; + int none = SCX_EXIT_NONE; + va_list args; + + if (!atomic_try_cmpxchg(&scx_exit_type, &none, type)) + return; + + ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1); + + va_start(args, fmt); + vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args); + va_end(args); + + irq_work_queue(&scx_ops_error_irq_work); +} + +static struct kthread_worker *scx_create_rt_helper(const char *name) +{ + struct kthread_worker *helper; + + helper = kthread_create_worker(0, name); + if (helper) + sched_set_fifo(helper->task); + return helper; +} + +static int scx_ops_enable(struct sched_ext_ops *ops) +{ + struct scx_task_iter sti; + struct task_struct *p; + int i, ret; + int tcnt = 0; + unsigned long long start = 0; + unsigned long long total_start = sched_clock(); + + mutex_lock(&scx_ops_enable_mutex); + + if (!scx_ops_helper) { + WRITE_ONCE(scx_ops_helper, + scx_create_rt_helper("sched_ext_ops_helper")); + if (!scx_ops_helper) { + ret = -ENOMEM; + goto err_unlock; + } + } + + if (scx_ops_enable_state() != SCX_OPS_DISABLED) { + ret = -EBUSY; + goto err_unlock; + } + + //slim_walt_enable(true); + + /* + * Set scx_ops, transition to PREPPING and clear exit info to arm the + * disable path. Failure triggers full disabling from here on. + */ + scx_ops = *ops; + + WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != + SCX_OPS_DISABLED); + + memset(&scx_exit_info, 0, sizeof(scx_exit_info)); + atomic_set(&scx_exit_type, SCX_EXIT_NONE); + scx_warned_zero_slice = false; + + atomic64_set(&scx_nr_rejected, 0); + + /* + * Keep CPUs stable during enable so that the BPF scheduler can track + * online CPUs by watching ->on/offline_cpu() after ->init(). + */ + cpus_read_lock(); + + scx_switch_all_req = true; + if (scx_ops.init) { + ret = SCX_CALL_OP_RET(SCX_KF_INIT, init); + if (ret) { + ret = ops_sanitize_err("init", ret); + goto err_disable; + } + + /* + * Exit early if ops.init() triggered scx_bpf_error(). Not + * strictly necessary as we'll fail transitioning into ENABLING + * later but that'd be after calling ops.prep_enable() on all + * tasks and with -EBUSY which isn't very intuitive. Let's exit + * early with success so that the condition is notified through + * ops.exit() like other scx_bpf_error() invocations. + */ + if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE) + goto err_disable; + } + + WARN_ON_ONCE(scx_dsp_buf); + scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; + scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch, + __alignof__(scx_dsp_buf[0])); + if (!scx_dsp_buf) { + ret = -ENOMEM; + goto err_disable; + } + + scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; + if (ops->timeout_ms) + scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); + + scx_watchdog_timestamp = jiffies; + queue_delayed_work(system_unbound_wq, &scx_watchdog_work, + scx_watchdog_timeout / 2); + + /* + * Lock out forks, cgroup on/offlining and moves before opening the + * floodgate so that they don't wander into the operations prematurely. + */ + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + for (i = 0; i < SCX_NR_ONLINE_OPS; i++) + if (((void (**)(void))ops)[i]) + static_branch_enable_cpuslocked(&scx_has_op[i]); + + if (ops->flags & SCX_OPS_ENQ_LAST) + static_branch_enable_cpuslocked(&scx_ops_enq_last); + + if (ops->flags & SCX_OPS_ENQ_EXITING) + static_branch_enable_cpuslocked(&scx_ops_enq_exiting); + if (scx_ops.cpu_acquire || scx_ops.cpu_release) + static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); + + if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { + reset_idle_masks(); + static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); + } else { + static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); + } + + /* + * All cgroups should be initialized before letting in tasks. cgroup + * on/offlining and task migrations are already locked out. + */ + ret = scx_cgroup_init(); + if (ret) + goto err_disable_unlock; + + static_branch_enable_cpuslocked(&__scx_ops_enabled); + + /* + * Enable ops for every task. Fork is excluded by scx_fork_rwsem + * preventing new tasks from being added. No need to exclude tasks + * leaving as sched_ext_free() can handle both prepped and enabled + * tasks. Prep all tasks first and then enable them with preemption + * disabled. + */ + spin_lock_irq(&scx_tasks_lock); + + scx_task_iter_init(&sti); + while ((p = scx_task_iter_next_filtered(&sti))) { + get_task_struct(p); + spin_unlock_irq(&scx_tasks_lock); + + ret = scx_ops_prepare_task(p, task_group(p)); + if (ret) { + put_task_struct(p); + spin_lock_irq(&scx_tasks_lock); + scx_task_iter_exit(&sti); + spin_unlock_irq(&scx_tasks_lock); + pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n", + ret, p->comm, p->pid); + goto err_disable_unlock; + } + + put_task_struct(p); + spin_lock_irq(&scx_tasks_lock); + } + scx_task_iter_exit(&sti); + + /* + * All tasks are prepped but are still ops-disabled. Ensure that + * %current can't be scheduled out and switch everyone. + * preempt_disable() is necessary because we can't guarantee that + * %current won't be starved if scheduled out while switching. + */ + preempt_disable(); + + /* + * From here on, the disable path must assume that tasks have ops + * enabled and need to be recovered. + */ + if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { + preempt_enable(); + spin_unlock_irq(&scx_tasks_lock); + ret = -EBUSY; + goto err_disable_unlock; + } + + /* + * We're fully committed and can't fail. The PREPPED -> ENABLED + * transitions here are synchronized against sched_ext_free() through + * scx_tasks_lock. + */ + WRITE_ONCE(scx_switching_all, scx_switch_all_req); + start = sched_clock(); + + scx_task_iter_init(&sti); + while ((p = scx_task_iter_next_filtered_locked(&sti))) { + tcnt++; + if (READ_ONCE(p->__state) != TASK_DEAD) { + const struct sched_class *old_class = p->sched_class; + struct rq *rq = task_rq(p); + + update_rq_clock(rq); + + SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE | + DEQUEUE_NOCLOCK) { + scx_ops_enable_task(p); + __setscheduler_prio(p, p->prio); + } + + check_class_changed(task_rq(p), p, old_class, p->prio); + } else { + scx_ops_disable_task(p); + } + } + printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id()); + scx_task_iter_exit(&sti); + + spin_unlock_irq(&scx_tasks_lock); + preempt_enable(); + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { + ret = -EBUSY; + goto err_disable; + } + + if (scx_switch_all_req) + static_branch_enable_cpuslocked(&__scx_switched_all); + + cpus_read_unlock(); + mutex_unlock(&scx_ops_enable_mutex); + + scx_cgroup_config_knobs(); + printk("\n total duration = %llu , current cpu = %d\n", sched_clock() - total_start, smp_processor_id()); + + return 0; + +err_unlock: + mutex_unlock(&scx_ops_enable_mutex); + return ret; + +err_disable_unlock: + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); +err_disable: + cpus_read_unlock(); + mutex_unlock(&scx_ops_enable_mutex); + /* must be fully disabled before returning */ + scx_ops_disable(SCX_EXIT_ERROR); + kthread_flush_work(&scx_ops_disable_work); + return ret; +} + +#ifdef CONFIG_SCHED_DEBUG +static const char *scx_ops_enable_state_str[] = { + [SCX_OPS_PREPPING] = "prepping", + [SCX_OPS_ENABLING] = "enabling", + [SCX_OPS_ENABLED] = "enabled", + [SCX_OPS_DISABLING] = "disabling", + [SCX_OPS_DISABLED] = "disabled", +}; + +static int scx_debug_show(struct seq_file *m, void *v) +{ + mutex_lock(&scx_ops_enable_mutex); + seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name); + seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled()); + seq_printf(m, "%-30s: %d\n", "switching_all", + READ_ONCE(scx_switching_all)); + seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all()); + seq_printf(m, "%-30s: %s\n", "enable_state", + scx_ops_enable_state_str[scx_ops_enable_state()]); + seq_printf(m, "%-30s: %llu\n", "nr_rejected", + atomic64_read(&scx_nr_rejected)); + mutex_unlock(&scx_ops_enable_mutex); + return 0; +} + +static int scx_debug_open(struct inode *inode, struct file *file) +{ + return single_open(file, scx_debug_show, NULL); +} + +const struct file_operations sched_ext_fops = { + .open = scx_debug_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + +/******************************************************************************** + * bpf_struct_ops plumbing. + */ +#include +#include +#include + +extern struct btf *btf_vmlinux; +static const struct btf_type *task_struct_type; + +static bool bpf_scx_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) + return false; + if (type != BPF_READ) + return false; + if (off % size != 0) + return false; + + return btf_ctx_access(off, size, type, prog, info); +} + +static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, + const struct bpf_reg_state *reg, + int off, int size) +{ + return 0; +} + +static const struct bpf_func_proto * +bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_task_storage_get: + return &bpf_task_storage_get_proto; + case BPF_FUNC_task_storage_delete: + return &bpf_task_storage_delete_proto; + default: + return bpf_base_func_proto(func_id); + } +} + +const struct bpf_verifier_ops bpf_scx_verifier_ops = { + .get_func_proto = bpf_scx_get_func_proto, + .is_valid_access = bpf_scx_is_valid_access, + .btf_struct_access = bpf_scx_btf_struct_access, +}; + +static int bpf_scx_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + const struct sched_ext_ops *uops = udata; + struct sched_ext_ops *ops = kdata; + u32 moff = __btf_member_bit_offset(t, member) / 8; + int ret; + + switch (moff) { + case offsetof(struct sched_ext_ops, dispatch_max_batch): + if (*(u32 *)(udata + moff) > INT_MAX) + return -E2BIG; + ops->dispatch_max_batch = *(u32 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, flags): + if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) + return -EINVAL; + ops->flags = *(u64 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, name): + ret = bpf_obj_name_cpy(ops->name, uops->name, + sizeof(ops->name)); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + return 1; + case offsetof(struct sched_ext_ops, timeout_ms): + if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT) + return -E2BIG; + ops->timeout_ms = *(u32 *)(udata + moff); + return 1; + } + + return 0; +} + +static int bpf_scx_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + u32 moff = __btf_member_bit_offset(t, member) / 8; + + switch (moff) { + case offsetof(struct sched_ext_ops, prep_enable): + case offsetof(struct sched_ext_ops, init): + case offsetof(struct sched_ext_ops, exit): + break; + default: + /* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */ + /* if (prog->aux->sleepable) + return -EINVAL; */ + break; + } + + return 0; +} + +static int bpf_scx_reg(void *kdata) +{ + return scx_ops_enable(kdata); +} + +static void bpf_scx_unreg(void *kdata) +{ + scx_ops_disable(SCX_EXIT_UNREG); + kthread_flush_work(&scx_ops_disable_work); +} + +static int bpf_scx_init(struct btf *btf) +{ + u32 type_id; + + type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); + if (type_id < 0) + return -EINVAL; + task_struct_type = btf_type_by_id(btf, type_id); + + return 0; +} + +/* "extern" to avoid sparse warning, only used in this file */ +extern struct bpf_struct_ops bpf_sched_ext_ops; + +struct bpf_struct_ops bpf_sched_ext_ops = { + .verifier_ops = &bpf_scx_verifier_ops, + .reg = bpf_scx_reg, + .unreg = bpf_scx_unreg, + .check_member = bpf_scx_check_member, + .init_member = bpf_scx_init_member, + .init = bpf_scx_init, + .name = "sched_ext_ops", +}; + +static void sysrq_handle_sched_ext_reset(u8 key) +{ + if (scx_ops_helper) + scx_ops_disable(SCX_EXIT_SYSRQ); + else + pr_info("sched_ext: BPF scheduler not yet used\n"); +} + +static const struct sysrq_key_op sysrq_sched_ext_reset_op = { + .handler = sysrq_handle_sched_ext_reset, + .help_msg = "reset-sched-ext(S)", + .action_msg = "Disable sched_ext and revert all tasks to CFS", + .enable_mask = SYSRQ_ENABLE_RTNICE, +}; + +static void kick_cpus_irq_workfn(struct irq_work *irq_work) +{ + struct rq *this_rq = this_rq(); + u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); + int this_cpu = cpu_of(this_rq); + int cpu; + + for_each_cpu(cpu, this_rq->scx->cpus_to_kick) { + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + raw_spin_rq_lock_irqsave(rq, flags); + + if (cpu_online(cpu) || cpu == this_cpu) { + if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) && + rq->curr->sched_class == &ext_sched_class) + rq->curr->scx->slice = 0; + pseqs[cpu] = rq->scx->pnt_seq; + resched_curr(rq); + } else { + cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait); + } + + raw_spin_rq_unlock_irqrestore(rq, flags); + } + + for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait, + cpumask_of(this_cpu)) { + /* + * Pairs with smp_store_release() issued by this CPU in + * scx_notify_pick_next_task() on the resched path. + * + * We busy-wait here to guarantee that no other task can be + * scheduled on our core before the target CPU has entered the + * resched path. + */ + while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu]) + cpu_relax(); + } + + cpumask_clear(this_rq->scx->cpus_to_kick); + cpumask_clear(this_rq->scx->cpus_to_preempt); + cpumask_clear(this_rq->scx->cpus_to_wait); +} + +void __init init_sched_ext_class(void) +{ + int cpu; + u32 v; + + /* + * The following is to prevent the compiler from optimizing out the enum + * definitions so that BPF scheduler implementations can use them + * through the generated vmlinux.h. + */ + WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | + SCX_TG_ONLINE | SCX_KICK_PREEMPT); + + init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); +#ifdef CONFIG_SMP + BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); + BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); +#endif + scx_kick_cpus_pnt_seqs = + __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * + num_possible_cpus(), + __alignof__(scx_kick_cpus_pnt_seqs[0])); + BUG_ON(!scx_kick_cpus_pnt_seqs); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL); + if (rq->scx) + rq->scx->rq = rq; + else + pr_err("fatal error : alloc rq->scx failed!!!\n"); + + init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL); + INIT_LIST_HEAD(&rq->scx->watchdog_list); + + BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL)); + BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL)); + BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL)); + init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn); + } + + register_sysrq_key('S', &sysrq_sched_ext_reset_op); + INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); + scx_cgroup_config_knobs(); +} + + +/******************************************************************************** + * Helpers that can be called from the BPF scheduler. + */ +#include + +/* Disables missing prototype warnings for kfuncs */ +__diag_push(); +__diag_ignore_all("-Wmissing-prototypes", + "Global functions as their definitions will be in vmlinux BTF"); + +/** + * scx_bpf_switch_all - Switch all tasks into SCX + * @into_scx: switch direction + * + * If @into_scx is %true, all existing and future non-dl/rt tasks are switched + * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on + * SCX. The actual switching is asynchronous. Can be called from ops.init(). + */ +void scx_bpf_switch_all(void) +{ + if (!scx_kf_allowed(SCX_KF_INIT)) + return; + + scx_switch_all_req = true; +} + +BTF_SET8_START(scx_kfunc_ids_init) +BTF_ID_FLAGS(func, scx_bpf_switch_all) +BTF_SET8_END(scx_kfunc_ids_init) + +static const struct btf_kfunc_id_set scx_kfunc_set_init = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_init, +}; + +/** + * scx_bpf_create_dsq - Create a custom DSQ + * @dsq_id: DSQ to create + * @node: NUMA node to allocate from + * + * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(), + * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move(). + */ +s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) +{ + if (!scx_kf_allowed(SCX_KF_INIT)) + return -EINVAL; + + if (unlikely(node >= (int)nr_node_ids || + (node < 0 && node != NUMA_NO_NODE))) + return -EINVAL; + return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); +} + +BTF_SET8_START(scx_kfunc_ids_sleepable) +BTF_ID_FLAGS(func, scx_bpf_create_dsq) +BTF_SET8_END(scx_kfunc_ids_sleepable) + +static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_sleepable, +}; + +static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) +{ + if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) + return false; + + lockdep_assert_irqs_disabled(); + + if (unlikely(!p)) { + scx_ops_error("called with NULL task"); + return false; + } + + if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { + scx_ops_error("invalid enq_flags 0x%llx", enq_flags); + return false; + } + + return true; +} + +static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) +{ + struct task_struct *ddsp_task; + int idx; + + ddsp_task = __this_cpu_read(direct_dispatch_task); + if (ddsp_task) { + direct_dispatch(ddsp_task, p, dsq_id, enq_flags); + return; + } + + idx = __this_cpu_read(scx_dsp_ctx.buf_cursor); + if (unlikely(idx >= scx_dsp_max_batch)) { + scx_ops_error("dispatch buffer overflow"); + return; + } + + this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){ + .task = p, + .qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK, + .dsq_id = dsq_id, + .enq_flags = enq_flags, + }; + __this_cpu_inc(scx_dsp_ctx.buf_cursor); +} + +/** + * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ + * @p: task_struct to dispatch + * @dsq_id: DSQ to dispatch to + * @slice: duration @p can run for in nsecs + * @enq_flags: SCX_ENQ_* + * + * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe + * to call this function spuriously. Can be called from ops.enqueue() and + * ops.dispatch(). + * + * When called from ops.enqueue(), it's for direct dispatch and @p must match + * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the + * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be + * on the target CPU in the first place. + * + * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id + * and this function can be called upto ops.dispatch_max_batch times to dispatch + * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the + * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. + * + * This function doesn't have any locking restrictions and may be called under + * BPF locks (in the future when BPF introduces more flexible locking). + * + * @p is allowed to run for @slice. The scheduling path is triggered on slice + * exhaustion. If zero, the current residual slice is maintained. If + * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with + * scx_bpf_kick_cpu() to trigger scheduling. + */ +void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, + u64 enq_flags) +{ + if (!scx_dispatch_preamble(p, enq_flags)) + return; + + if (slice) + p->scx->slice = slice; + else + p->scx->slice = p->scx->slice ?: 1; + + scx_dispatch_commit(p, dsq_id, enq_flags); +} + +/** + * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ + * @p: task_struct to dispatch + * @dsq_id: DSQ to dispatch to + * @slice: duration @p can run for in nsecs + * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ + * @enq_flags: SCX_ENQ_* + * + * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. + * Tasks queued into the priority queue are ordered by @vtime and always + * consumed after the tasks in the FIFO queue. All other aspects are identical + * to scx_bpf_dispatch(). + * + * @vtime ordering is according to time_before64() which considers wrapping. A + * numerically larger vtime may indicate an earlier position in the ordering and + * vice-versa. + */ +void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, + u64 vtime, u64 enq_flags) +{ + if (!scx_dispatch_preamble(p, enq_flags)) + return; + + if (slice) + p->scx->slice = slice; + else + p->scx->slice = p->scx->slice ?: 1; + + p->scx->dsq_vtime = vtime; + + scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); +} + +BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch) +BTF_ID_FLAGS(func, scx_bpf_dispatch) +BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime) +BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch) + +static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_enqueue_dispatch, +}; + +/** + * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots + * + * Can only be called from ops.dispatch(). + */ +u32 scx_bpf_dispatch_nr_slots(void) +{ + if (!scx_kf_allowed(SCX_KF_DISPATCH)) + return 0; + + return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor); +} + +/** + * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ + * @dsq_id: DSQ to consume + * + * Consume a task from the non-local DSQ identified by @dsq_id and transfer it + * to the current CPU's local DSQ for execution. Can only be called from + * ops.dispatch(). + * + * This function flushes the in-flight dispatches from scx_bpf_dispatch() before + * trying to consume the specified DSQ. It may also grab rq locks and thus can't + * be called under any BPF locks. + * + * Returns %true if a task has been consumed, %false if there isn't any task to + * consume. + */ +bool scx_bpf_consume(u64 dsq_id) +{ + struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx); + struct scx_dispatch_q *dsq; + + if (!scx_kf_allowed(SCX_KF_DISPATCH)) + return false; + + flush_dispatch_buf(dspc->rq, dspc->rf); + + dsq = find_non_local_dsq(dsq_id); + if (unlikely(!dsq)) { + scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); + return false; + } + + if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) { + /* + * A successfully consumed task can be dequeued before it starts + * running while the CPU is trying to migrate other dispatched + * tasks. Bump nr_tasks to tell balance_scx() to retry on empty + * local DSQ. + */ + dspc->nr_tasks++; + return true; + } else { + return false; + } +} + +BTF_SET8_START(scx_kfunc_ids_dispatch) +BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) +BTF_ID_FLAGS(func, scx_bpf_consume) +BTF_SET8_END(scx_kfunc_ids_dispatch) + +static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_dispatch, +}; + +/** + * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ + * + * Iterate over all of the tasks currently enqueued on the local DSQ of the + * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of + * processed tasks. Can only be called from ops.cpu_release(). + */ +u32 scx_bpf_reenqueue_local(void) +{ + u32 nr_enqueued, i; + struct rq *rq; + struct scx_rq *scx_rq; + + if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) + return 0; + + rq = cpu_rq(smp_processor_id()); + lockdep_assert_rq_held(rq); + scx_rq = rq->scx; + + /* + * Get the number of tasks on the local DSQ before iterating over it to + * pull off tasks. The enqueue callback below can signal that it wants + * the task to stay on the local DSQ, and we want to prevent the BPF + * scheduler from causing us to loop indefinitely. + */ + nr_enqueued = scx_rq->local_dsq.nr; + for (i = 0; i < nr_enqueued; i++) { + struct task_struct *p; + + p = first_local_task(rq); + WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE); + WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED)); + WARN_ON_ONCE(p->scx->holding_cpu != -1); + dispatch_dequeue(scx_rq, p); + do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + } + + return nr_enqueued; +} + +BTF_SET8_START(scx_kfunc_ids_cpu_release) +BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) +BTF_SET8_END(scx_kfunc_ids_cpu_release) + +static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_cpu_release, +}; + +/** + * scx_bpf_kick_cpu - Trigger reschedule on a CPU + * @cpu: cpu to kick + * @flags: SCX_KICK_* flags + * + * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or + * trigger rescheduling on a busy CPU. This can be called from any online + * scx_ops operation and the actual kicking is performed asynchronously through + * an irq work. + */ +void scx_bpf_kick_cpu(s32 cpu, u64 flags) +{ + struct rq *rq; + + if (!ops_cpu_valid(cpu)) { + scx_ops_error("invalid cpu %d", cpu); + return; + } + + preempt_disable(); + rq = this_rq(); + + /* + * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting + * rq locks. We can probably be smarter and avoid bouncing if called + * from ops which don't hold a rq lock. + */ + cpumask_set_cpu(cpu, rq->scx->cpus_to_kick); + if (flags & SCX_KICK_PREEMPT) + cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt); + if (flags & SCX_KICK_WAIT) + cpumask_set_cpu(cpu, rq->scx->cpus_to_wait); + + irq_work_queue(&rq->scx->kick_cpus_irq_work); + preempt_enable(); +} + +/** + * scx_bpf_dsq_nr_queued - Return the number of queued tasks + * @dsq_id: id of the DSQ + * + * Return the number of tasks in the DSQ matching @dsq_id. If not found, + * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops + * operations. + */ +s32 scx_bpf_dsq_nr_queued(u64 dsq_id) +{ + struct scx_dispatch_q *dsq; + + lockdep_assert(rcu_read_lock_any_held()); + + if (dsq_id == SCX_DSQ_LOCAL) { + return this_rq()->scx->local_dsq.nr; + } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { + s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; + + if (ops_cpu_valid(cpu)) + return cpu_rq(cpu)->scx->local_dsq.nr; + } else { + dsq = find_non_local_dsq(dsq_id); + if (dsq) + return dsq->nr; + } + return -ENOENT; +} + +/** + * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state + * @cpu: cpu to test and clear idle for + * + * Returns %true if @cpu was idle and its idle state was successfully cleared. + * %false otherwise. + * + * Unavailable if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. + */ +bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) +{ + if (!static_branch_likely(&scx_builtin_idle_enabled)) { + scx_ops_error("built-in idle tracking is disabled"); + return false; + } + + if (ops_cpu_valid(cpu)) + return test_and_clear_cpu_idle(cpu); + else + return false; +} + +/** + * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu + * @cpus_allowed: Allowed cpumask + * + * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked + * idle cpu number on success. -%EBUSY if no matching cpu was found. + * + * Unavailable if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. + */ +s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed) +{ + if (!static_branch_likely(&scx_builtin_idle_enabled)) { + scx_ops_error("built-in idle tracking is disabled"); + return -EBUSY; + } + + return scx_pick_idle_cpu(cpus_allowed); +} + +/** + * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking + * per-CPU cpumask. + * + * Returns NULL if idle tracking is not enabled, or running on a UP kernel. + */ +const struct cpumask *scx_bpf_get_idle_cpumask(void) +{ + if (!static_branch_likely(&scx_builtin_idle_enabled)) { + scx_ops_error("built-in idle tracking is disabled"); + return cpu_none_mask; + } + +#ifdef CONFIG_SMP + return idle_masks.cpu; +#else + return cpu_none_mask; +#endif +} + +/** + * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, + * per-physical-core cpumask. Can be used to determine if an entire physical + * core is free. + * + * Returns NULL if idle tracking is not enabled, or running on a UP kernel. + */ +const struct cpumask *scx_bpf_get_idle_smtmask(void) +{ + if (!static_branch_likely(&scx_builtin_idle_enabled)) { + scx_ops_error("built-in idle tracking is disabled"); + return cpu_none_mask; + } + +#ifdef CONFIG_SMP + return idle_masks.smt; +#else + return cpu_none_mask; +#endif +} + +/** + * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to + * either the percpu, or SMT idle-tracking cpumask. + */ +void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) +{ + /* + * Empty function body because we aren't actually acquiring or + * releasing a reference to a global idle cpumask, which is read-only + * in the caller and is never released. The acquire / release semantics + * here are just used to make the cpumask is a trusted pointer in the + * caller. + */ +} + +struct scx_bpf_error_bstr_bufs { + u64 data[MAX_BPRINTF_VARARGS]; + char msg[SCX_EXIT_MSG_LEN]; +}; + +static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs); + +/** + * scx_bpf_error_bstr - Indicate fatal error + * @fmt: error message format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * + * Indicate that the BPF scheduler encountered a fatal error and initiate ops + * disabling. + */ +void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) +{ + struct scx_bpf_error_bstr_bufs *bufs; + unsigned long flags; + struct bpf_bprintf_data args = {}; + int ret; + + local_irq_save(flags); + bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs); + + if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || + (data__sz && !data)) { + scx_ops_error("invalid data=%p and data__sz=%u", + (void *)data, data__sz); + goto out_restore; + } + + ret = copy_from_kernel_nofault(bufs->data, data, data__sz); + if (ret) { + scx_ops_error("failed to read data fields (%d)", ret); + goto out_restore; + } + + ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args); + if (ret < 0) { + scx_ops_error("failed to format prepration (%d)", ret); + goto out_restore; + } + + ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt, + args.bin_args); + bpf_bprintf_cleanup(&args); + if (ret < 0) { + scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format", + fmt, data, data__sz); + goto out_restore; + } + + scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg); +out_restore: + local_irq_restore(flags); +} + +/** + * scx_bpf_destroy_dsq - Destroy a custom DSQ + * @dsq_id: DSQ to destroy + * + * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with + * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is + * empty and no further tasks are dispatched to it. Ignored if called on a DSQ + * which doesn't exist. Can be called from any online scx_ops operations. + */ +void scx_bpf_destroy_dsq(u64 dsq_id) +{ + destroy_dsq(dsq_id); +} + +/** + * scx_bpf_task_running - Is task currently running? + * @p: task of interest + */ +bool scx_bpf_task_running(const struct task_struct *p) +{ + return task_rq(p)->curr == p; +} + +/** + * scx_bpf_task_cpu - CPU a task is currently associated with + * @p: task of interest + */ +s32 scx_bpf_task_cpu(const struct task_struct *p) +{ + return task_cpu(p); +} + +/** + * scx_bpf_task_cgroup - Return the sched cgroup of a task + * @p: task of interest + * + * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with + * from the scheduler's POV. SCX operations should use this function to + * determine @p's current cgroup as, unlike following @p->cgroups, + * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all + * rq-locked operations. Can be called on the parameter tasks of rq-locked + * operations. The restriction guarantees that @p's rq is locked by the caller. + */ +struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) +{ + struct task_group *tg = p->sched_task_group; + struct cgroup *cgrp = &cgrp_dfl_root.cgrp; + + if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) + goto out; + + /* + * A task_group may either be a cgroup or an autogroup. In the latter + * case, @tg->css.cgroup is %NULL. A task_group can't become the other + * kind once created. + */ + if (tg && tg->css.cgroup) + cgrp = tg->css.cgroup; + else + cgrp = &cgrp_dfl_root.cgrp; +out: + cgroup_get(cgrp); + return cgrp; +} + +BTF_SET8_START(scx_kfunc_ids_any) +BTF_ID_FLAGS(func, scx_bpf_kick_cpu) +BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) +BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) +BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) +BTF_ID_FLAGS(func, scx_bpf_task_running) +BTF_ID_FLAGS(func, scx_bpf_task_cpu) +BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE) +BTF_SET8_END(scx_kfunc_ids_any) + +static const struct btf_kfunc_id_set scx_kfunc_set_any = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_any, +}; + +__diag_pop(); + +/* + * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set() + * needs most of the system to be up. + */ +static int __init register_ext_kfuncs(void) +{ + int ret; + + /* + * Some kfuncs are context-sensitive and can only be called from + * specific SCX ops. They are grouped into BTF sets accordingly. + * Unfortunately, BPF currently doesn't have a way of enforcing such + * restrictions. Eventually, the verifier should be able to enforce + * them. For now, register them the same and make each kfunc explicitly + * check using scx_kf_allowed(). + */ + if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_init)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_sleepable)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_enqueue_dispatch)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_dispatch)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_cpu_release)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_any))) { + pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret); + return ret; + } + + return 0; +} +__initcall(register_ext_kfuncs); diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h new file mode 100755 index 0000000000000..fba8ed845f99f --- /dev/null +++ b/kernel/sched/ext.h @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ + + +/* + * Tag marking a kernel function as a kfunc. This is meant to minimize the + * amount of copy-paste that kfunc authors have to include for correctness so + * as to avoid issues such as the compiler inlining or eliding either a static + * kfunc, or a global kfunc in an LTO build. + */ +#define __bpf_kfunc __used noinline + +enum scx_wake_flags { + /* expose select WF_* flags as enums */ + SCX_WAKE_EXEC = WF_EXEC, + SCX_WAKE_FORK = WF_FORK, + SCX_WAKE_TTWU = WF_TTWU, + SCX_WAKE_SYNC = WF_SYNC, +}; + +enum scx_enq_flags { + /* expose select ENQUEUE_* flags as enums */ + SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, + SCX_ENQ_HEAD = ENQUEUE_HEAD, + + /* high 32bits are SCX specific */ + + /* + * Set the following to trigger preemption when calling + * scx_bpf_dispatch() with a local dsq as the target. The slice of the + * current task is cleared to zero and the CPU is kicked into the + * scheduling path. Implies %SCX_ENQ_HEAD. + */ + SCX_ENQ_PREEMPT = 1LLU << 32, + + /* + * The task being enqueued was previously enqueued on the current CPU's + * %SCX_DSQ_LOCAL, but was removed from it in a call to the + * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was + * invoked in a ->cpu_release() callback, and the task is again + * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the + * task will not be scheduled on the CPU until at least the next invocation + * of the ->cpu_acquire() callback. + */ + SCX_ENQ_REENQ = 1LLU << 40, + + /* + * The task being enqueued is the only task available for the cpu. By + * default, ext core keeps executing such tasks but when + * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with + * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set. + * + * If the BPF scheduler wants to continue executing the task, + * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately. + * If the task gets queued on a different dsq or the BPF side, the BPF + * scheduler is responsible for triggering a follow-up scheduling event. + * Otherwise, Execution may stall. + */ + SCX_ENQ_LAST = 1LLU << 41, + + /* + * A hint indicating that it's advisable to enqueue the task on the + * local dsq of the currently selected CPU. Currently used by + * select_cpu_dfl() and together with %SCX_ENQ_LAST. + */ + SCX_ENQ_LOCAL = 1LLU << 42, + + /* high 8 bits are internal */ + __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, + + SCX_ENQ_CLEAR_OPSS = 1LLU << 56, + SCX_ENQ_DSQ_PRIQ = 1LLU << 57, +}; + +enum scx_deq_flags { + /* expose select DEQUEUE_* flags as enums */ + SCX_DEQ_SLEEP = DEQUEUE_SLEEP, + + /* high 32bits are SCX specific */ + + /* + * The generic core-sched layer decided to execute the task even though + * it hasn't been dispatched yet. Dequeue from the BPF side. + */ + SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, +}; + +enum scx_tg_flags { + SCX_TG_ONLINE = 1U << 0, + SCX_TG_INITED = 1U << 1, +}; + +enum scx_kick_flags { + SCX_KICK_PREEMPT = 1LLU << 0, /* force scheduling on the CPU */ + SCX_KICK_WAIT = 1LLU << 1, /* wait for the CPU to be rescheduled */ +}; + +#ifdef CONFIG_SCHED_CLASS_EXT + +extern const struct sched_class ext_sched_class; +extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops; +extern const struct file_operations sched_ext_fops; +extern unsigned long scx_watchdog_timeout; +extern unsigned long scx_watchdog_timestamp; + +DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); +DECLARE_STATIC_KEY_FALSE(__scx_switched_all); +#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) +#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) + +DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); + +bool task_on_scx(struct task_struct *p); +void scx_pre_fork(struct task_struct *p); +int scx_fork(struct task_struct *p); +void scx_post_fork(struct task_struct *p); +void scx_cancel_fork(struct task_struct *p); +int scx_check_setscheduler(struct task_struct *p, int policy); +bool scx_can_stop_tick(struct rq *rq); +void init_sched_ext_class(void); + +__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type, + const char *fmt, ...); +#define scx_ops_error(fmt, args...) \ + scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args) + +void __scx_notify_pick_next_task(struct rq *rq, + struct task_struct *p, + const struct sched_class *active); + +static inline void scx_notify_pick_next_task(struct rq *rq, + struct task_struct *p, + const struct sched_class *active) +{ + if (!scx_enabled()) + return; +#ifdef CONFIG_SMP + /* + * Pairs with the smp_load_acquire() issued by a CPU in + * kick_cpus_irq_workfn() who is waiting for this CPU to perform a + * resched. + */ + smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1); +#endif + if (!static_branch_unlikely(&scx_ops_cpu_preempt)) + return; + __scx_notify_pick_next_task(rq, p, active); +} + +//extern void scx_scheduler_tick(void); +static inline void scx_notify_sched_tick(void) +{ + unsigned long last_check; + + if (!scx_enabled()) + return; + //scx_scheduler_tick(); + + last_check = scx_watchdog_timestamp; + if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) { + u32 dur_ms = jiffies_to_msecs(jiffies - last_check); + + scx_ops_error_type(SCX_EXIT_ERROR_STALL, + "watchdog failed to check in for %u.%03us", + dur_ms / 1000, dur_ms % 1000); + } +} + +static inline const struct sched_class *next_active_class(const struct sched_class *class) +{ + class++; + if (scx_switched_all() && class == &fair_sched_class) + class++; + if (!scx_enabled() && class == &ext_sched_class) + class++; + return class; +} + +#define for_active_class_range(class, _from, _to) \ + for (class = (_from); class != (_to); class = next_active_class(class)) + +#define for_each_active_class(class) \ + for_active_class_range(class, __sched_class_highest, __sched_class_lowest) + +/* + * SCX requires a balance() call before every pick_next_task() call including + * when waking up from idle. + */ +#define for_balance_class_range(class, prev_class, end_class) \ + for_active_class_range(class, (prev_class) > &ext_sched_class ? \ + &ext_sched_class : (prev_class), (end_class)) + +#ifdef CONFIG_SCHED_CORE +bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, + bool in_fi); +#endif + +#else /* CONFIG_SCHED_CLASS_EXT */ + +#define scx_enabled() false +#define scx_switched_all() false + +static inline void scx_pre_fork(struct task_struct *p) {} +static inline int scx_fork(struct task_struct *p) { return 0; } +static inline void scx_post_fork(struct task_struct *p) {} +static inline void scx_cancel_fork(struct task_struct *p) {} +static inline int scx_check_setscheduler(struct task_struct *p, + int policy) { return 0; } +static inline bool scx_can_stop_tick(struct rq *rq) { return true; } +static inline void init_sched_ext_class(void) {} +static inline void scx_notify_pick_next_task(struct rq *rq, + const struct task_struct *p, + const struct sched_class *active) {} +static inline void scx_notify_sched_tick(void) {} + +#define for_each_active_class for_each_class +#define for_balance_class_range for_class_range + +#endif /* CONFIG_SCHED_CLASS_EXT */ + +#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) +void __scx_update_idle(struct rq *rq, bool idle); + +static inline void scx_update_idle(struct rq *rq, bool idle) +{ + if (scx_enabled()) + __scx_update_idle(rq, idle); +} +#else +static inline void scx_update_idle(struct rq *rq, bool idle) {} +#endif + +#ifdef CONFIG_CGROUP_SCHED +#ifdef CONFIG_EXT_GROUP_SCHED +int scx_tg_online(struct task_group *tg); +void scx_tg_offline(struct task_group *tg); +int scx_cgroup_can_attach(struct cgroup_taskset *tset); +void scx_move_task(struct task_struct *p); +void scx_cgroup_finish_attach(void); +void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); +void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); +#else /* CONFIG_EXT_GROUP_SCHED */ +static inline int scx_tg_online(struct task_group *tg) { return 0; } +static inline void scx_tg_offline(struct task_group *tg) {} +static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } +static inline void scx_move_task(struct task_struct *p) {} +static inline void scx_cgroup_finish_attach(void) {} +static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} +static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} +#endif /* CONFIG_EXT_GROUP_SCHED */ +#endif /* CONFIG_CGROUP_SCHED */ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 35ffd5b504b2b..5507d333ebf35 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -58,6 +58,7 @@ #include "autogroup.h" #include +#include EXPORT_TRACEPOINT_SYMBOL_GPL(sched_stat_runtime); @@ -93,11 +94,11 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; /* * Minimal preemption granularity for CPU-bound tasks: * - * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) + * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) */ -unsigned int sysctl_sched_base_slice = 750000ULL; +unsigned int sysctl_sched_base_slice = 700000ULL; EXPORT_SYMBOL_GPL(sysctl_sched_base_slice); -static unsigned int normalized_sysctl_sched_base_slice = 750000ULL; +static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; /* * After fork, child runs first. If set to 0 (default) then @@ -919,7 +920,7 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) * * Which allows tree pruning through eligibility. */ -static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) +static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq) { struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; struct sched_entity *se = __pick_first_entity(cfs_rq); @@ -981,9 +982,29 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) if (!best || (curr && entity_before(curr, best))) best = curr; + /* EEVDF search sched entity fail, so picking leftmost */ + if (!best) { + best = __pick_first_entity(cfs_rq); + printk_deferred("EEVDF scheduling fail, picking leftmost\n"); + } + return best; } +static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) +{ + struct sched_entity *se = __pick_eevdf(cfs_rq); + if (!se) { + struct sched_entity *left = __pick_first_entity(cfs_rq); + if (left) { + pr_err("EEVDF scheduling fail, picking leftmost\n"); + return left; + } + } + + return se; +} + #ifdef CONFIG_SCHED_DEBUG struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) { @@ -1046,6 +1067,11 @@ static void update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) * The task has consumed its request, reschedule. */ if (cfs_rq->nr_running > 1) { + trace_android_vh_resched_curr_lazy(rq_of(cfs_rq), &skip_preempt); + + if (skip_preempt) + return; + resched_curr(rq_of(cfs_rq)); clear_buddies(cfs_rq, se); } @@ -4820,7 +4846,7 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) return cfs_rq->avg.load_avg; } -static int newidle_balance(struct rq *this_rq, struct rq_flags *rf); +static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf); static inline unsigned long task_util(struct task_struct *p) { @@ -5097,6 +5123,13 @@ static inline int task_fits_cpu(struct task_struct *p, int cpu) unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); unsigned long util = task_util_est(p); + bool fits = true, done = false; + + trace_android_rvh_task_fits_cpu(p, util, uclamp_min, uclamp_max, cpu, &fits, &done); + + if (done) + return fits; + /* * Return true only if the cpu fully fits the task requirements, which * include the utilization but also the performance hints. @@ -5167,7 +5200,7 @@ attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} -static inline int newidle_balance(struct rq *rq, struct rq_flags *rf) +static inline int sched_balance_newidle(struct rq *rq, struct rq_flags *rf) { return 0; } @@ -5539,6 +5572,13 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) * validating it and just reschedule. */ if (queued) { + bool skip_preempt = false; + + trace_android_vh_resched_curr_lazy(rq_of(cfs_rq), &skip_preempt); + + if (skip_preempt) + return; + resched_curr(rq_of(cfs_rq)); return; } @@ -8313,10 +8353,11 @@ static void task_dead_fair(struct task_struct *p) static int balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) { + trace_android_rvh_balance_fair(rq, prev, rf); if (rq->nr_running) return 1; - return newidle_balance(rq, rf) != 0; + return sched_balance_newidle(rq, rf) != 0; } #endif /* CONFIG_SMP */ @@ -8420,6 +8461,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ return; preempt: + trace_android_vh_resched_curr_lazy(rq_of(cfs_rq), &ignore); + + if (ignore) + return; + resched_curr(rq); } @@ -8465,6 +8511,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf int new_tasks; bool repick = false; + trace_android_rvh_before_pick_task_fair(rq, &p, &se, prev, rf); again: if (!sched_fair_runnable(rq)) goto idle; @@ -8473,6 +8520,9 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf if (!prev || prev->sched_class != &fair_sched_class) goto simple; + if (p) + goto put_prev_set_next; + /* * Because of the set_next_buddy() in dequeue_task_fair() it is rather * likely that a next task is from the same cgroup as the current. @@ -8518,6 +8568,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf p = task_of(se); trace_android_rvh_replace_next_task_fair(rq, &p, &se, &repick, false, prev); +put_prev_set_next: /* * Since we haven't yet done put_prev_entity and if the selected task * is a different task than we started out with, try and touch the @@ -8581,21 +8632,21 @@ done: __maybe_unused; return p; idle: - if (!rf) - return NULL; - - new_tasks = newidle_balance(rq, rf); + if (rf) { + new_tasks = sched_balance_newidle(rq, rf); - /* - * Because newidle_balance() releases (and re-acquires) rq->lock, it is - * possible for any higher priority task to appear. In that case we - * must re-start the pick_next_entity() loop. - */ - if (new_tasks < 0) - return RETRY_TASK; + /* + * Because sched_balance_newidle() releases (and re-acquires) + * rq->lock, it is possible for any higher priority task to + * appear. In that case we must re-start the pick_next_entity() + * loop. + */ + if (new_tasks < 0) + return RETRY_TASK; - if (new_tasks > 0) - goto again; + if (new_tasks > 0) + goto again; + } /* * rq is about to be idle, check if we need to update the @@ -11310,7 +11361,7 @@ static int need_active_balance(struct lb_env *env) return 0; } -static int active_load_balance_cpu_stop(void *data); +int active_load_balance_cpu_stop(void *data); static int should_we_balance(struct lb_env *env) { @@ -11632,7 +11683,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, ld_moved = 0; /* - * newidle_balance() disregards balance intervals, so we could + * sched_balance_newidle() disregards balance intervals, so we could * repeatedly reach this code, which would lead to balance_interval * skyrocketing in a short amount of time. Skip the balance_interval * increase logic to avoid that. @@ -11692,7 +11743,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance) * least 1 task to be running on each physical CPU where possible, and * avoids physical / logical imbalances. */ -static int active_load_balance_cpu_stop(void *data) +int active_load_balance_cpu_stop(void *data) { struct rq *busiest_rq = data; int busiest_cpu = cpu_of(busiest_rq); @@ -11770,6 +11821,7 @@ static int active_load_balance_cpu_stop(void *data) return 0; } +EXPORT_SYMBOL_GPL(active_load_balance_cpu_stop); static DEFINE_SPINLOCK(balancing); @@ -11788,8 +11840,14 @@ static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost) /* * Track max cost of a domain to make sure to not delay the * next wakeup on the CPU. + * + * sched_balance_newidle() bumps the cost whenever newidle + * balance fails, and we don't want things to grow out of + * control. Use the sysctl_sched_migration_cost as the upper + * limit, plus a litle extra to avoid off by ones. */ - sd->max_newidle_lb_cost = cost; + sd->max_newidle_lb_cost = + min(cost, sysctl_sched_migration_cost + 200); sd->last_decay_max_lb_cost = jiffies; } else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) { /* @@ -12406,7 +12464,7 @@ static inline void nohz_newidle_balance(struct rq *this_rq) { } #endif /* CONFIG_NO_HZ_COMMON */ /* - * newidle_balance is called by schedule() if this_cpu is about to become + * sched_balance_newidle is called by schedule() if this_cpu is about to become * idle. Attempts to pull tasks from other CPUs. * * Returns: @@ -12414,7 +12472,7 @@ static inline void nohz_newidle_balance(struct rq *this_rq) { } * 0 - failed, no new tasks * > 0 - success, new (fair) tasks present */ -static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) +static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) { unsigned long next_balance = jiffies + HZ; int this_cpu = this_rq->cpu; @@ -12493,10 +12551,17 @@ static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) t1 = sched_clock_cpu(this_cpu); domain_cost = t1 - t0; - update_newidle_cost(sd, domain_cost); - curr_cost += domain_cost; t0 = t1; + + /* + * Failing newidle means it is not effective; + * bump the cost so we end up doing less of it. + */ + if (!pulled_task) + domain_cost = (3 * sd->max_newidle_lb_cost) / 2; + + update_newidle_cost(sd, domain_cost); } /* diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c old mode 100644 new mode 100755 index 5007b25c5bc65..b33cefeb4188a --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -408,11 +408,13 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) { + scx_update_idle(rq, false); } static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) { update_idle_core(rq); + scx_update_idle(rq, true); schedstat_inc(rq->sched_goidle); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h old mode 100644 new mode 100755 index 581fc21cf34af..e09b29a9d2bbe --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -186,9 +187,19 @@ static inline int idle_policy(int policy) { return policy == SCHED_IDLE; } + +static inline int normal_policy(int policy) +{ +#ifdef CONFIG_SCHED_CLASS_EXT + if (policy == SCHED_EXT) + return true; +#endif + return policy == SCHED_NORMAL; +} + static inline int fair_policy(int policy) { - return policy == SCHED_NORMAL || policy == SCHED_BATCH; + return normal_policy(policy) || policy == SCHED_BATCH; } static inline int rt_policy(int policy) @@ -236,6 +247,24 @@ static inline void update_avg(u64 *avg, u64 sample) #define shr_bound(val, shift) \ (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) +/* + * cgroup weight knobs should use the common MIN, DFL and MAX values which are + * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it + * maps pretty well onto the shares value used by scheduler and the round-trip + * conversions preserve the original value over the entire range. + */ +static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) +{ + return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); +} + +static inline unsigned long sched_weight_to_cgroup(unsigned long weight) +{ + return clamp_t(unsigned long, + DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), + CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); +} + /* * !! For sched_setattr_nocheck() (kernel) only !! * @@ -502,6 +531,11 @@ extern void set_task_rq_fair(struct sched_entity *se, static inline void set_task_rq_fair(struct sched_entity *se, struct cfs_rq *prev, struct cfs_rq *next) { } #endif /* CONFIG_SMP */ +#else /* CONFIG_FAIR_GROUP_SCHED */ +static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) +{ + return 0; +} #endif /* CONFIG_FAIR_GROUP_SCHED */ #else /* CONFIG_CGROUP_SCHED */ @@ -666,6 +700,29 @@ struct cfs_rq { #endif /* CONFIG_FAIR_GROUP_SCHED */ }; +#ifdef CONFIG_SCHED_CLASS_EXT +/* scx_rq->flags, protected by the rq lock */ +enum scx_rq_flags { + SCX_RQ_CAN_STOP_TICK = 1 << 0, +}; + +struct scx_rq { + struct scx_dispatch_q local_dsq; + struct list_head watchdog_list; + u64 ops_qseq; + u64 extra_enq_flags; /* see move_task_to_local_dsq() */ + u32 nr_running; + u32 flags; + bool cpu_released; + cpumask_var_t cpus_to_kick; + cpumask_var_t cpus_to_preempt; + cpumask_var_t cpus_to_wait; + u64 pnt_seq; + struct irq_work kick_cpus_irq_work; + struct rq *rq; +}; +#endif /* CONFIG_SCHED_CLASS_EXT */ + static inline int rt_bandwidth_enabled(void) { return sysctl_sched_rt_runtime >= 0; @@ -1201,7 +1258,11 @@ struct rq { ANDROID_OEM_DATA_ARRAY(1, 16); +#ifdef CONFIG_SLIM_SCHED + ANDROID_KABI_USE(1, struct scx_rq *scx); +#else ANDROID_KABI_RESERVE(1); +#endif ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); ANDROID_KABI_RESERVE(4); @@ -2279,6 +2340,11 @@ struct affinity_context { unsigned int flags; }; +enum rq_onoff_reason { + RQ_ONOFF_HOTPLUG, /* CPU is going on/offline */ + RQ_ONOFF_TOPOLOGY, /* sched domain topology update */ +}; + struct sched_class { #ifdef CONFIG_UCLAMP_TASK @@ -2485,6 +2551,7 @@ extern void init_sched_rt_class(void); extern void init_sched_fair_class(void); extern void reweight_task(struct task_struct *p, int prio); +extern void __setscheduler_prio(struct task_struct *p, int prio); extern void resched_curr(struct rq *rq); extern void resched_cpu(int cpu); @@ -2565,6 +2632,53 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) extern void activate_task(struct rq *rq, struct task_struct *p, int flags); extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); +struct sched_change_guard { + struct task_struct *p; + struct rq *rq; + bool queued; + bool running; + bool done; +}; + +extern struct sched_change_guard +sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags); + +extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags); + +/** + * SCHED_CHANGE_BLOCK - Nested block for task attribute updates + * @__rq: Runqueue the target task belongs to + * @__p: Target task + * @__flags: DEQUEUE/ENQUEUE_* flags + * + * A task may need to be dequeued and put_prev_task'd for attribute updates and + * set_next_task'd and re-enqueued afterwards. This helper defines a nested + * block which automatically handles these preparation and cleanup operations. + * + * SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) { + * update_attribute(p); + * ... + * } + * + * If @__flags is a variable, the variable may be updated in the block body and + * the updated value will be used when re-enqueueing @p. + * + * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling + * update_rq_clock() beforehand. Otherwise, the rq clock is automatically + * updated iff the task needs to be dequeued and re-enqueued. Only the former + * case guarantees that the rq clock is up-to-date inside and after the block. + */ +#define SCHED_CHANGE_BLOCK(__rq, __p, __flags) \ + for (struct sched_change_guard __cg = \ + sched_change_guard_init(__rq, __p, __flags); \ + !__cg.done; sched_change_guard_fini(&__cg, __flags)) + +extern void check_class_changing(struct rq *rq, struct task_struct *p, + const struct sched_class *prev_class); +extern void check_class_changed(struct rq *rq, struct task_struct *p, + const struct sched_class *prev_class, + int oldprio); + extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); #ifdef CONFIG_PREEMPT_RT @@ -3506,11 +3620,9 @@ static inline int __mm_cid_get(struct rq *rq, struct mm_struct *mm) static inline int mm_cid_get(struct rq *rq, struct mm_struct *mm) { struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid; - struct cpumask *cpumask; int cid; lockdep_assert_rq_held(rq); - cpumask = mm_cidmask(mm); cid = __this_cpu_read(pcpu_cid->cid); if (mm_cid_is_valid(cid)) { mm_cid_snapshot_time(rq, mm); @@ -3596,4 +3708,6 @@ static inline bool cpu_busy_with_softirqs(int cpu) } #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */ +#include "ext.h" + #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index eb36532d6eb48..9e604fd5c8e7d 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -212,6 +212,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) DEFINE_STATIC_KEY_FALSE(sched_energy_present); +EXPORT_SYMBOL_GPL(sched_energy_present); static unsigned int sysctl_sched_energy_aware = 1; static DEFINE_MUTEX(sched_energy_mutex); static bool sched_energy_update; @@ -2132,6 +2133,8 @@ int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node) goto unlock; hop_masks = bsearch(&k, k.masks, sched_domains_numa_levels, sizeof(k.masks[0]), hop_cmp); + if (!hop_masks) + goto unlock; hop = hop_masks - k.masks; ret = hop ? diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c old mode 100644 new mode 100755 index 788b1dfd27bca..4820e5c6de8a0 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -119,8 +119,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_remove_entity_load_avg); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_blocked_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_rt_rq_load_avg); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_uclamp_validate); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_sugov_sched_attr); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_iowait); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_prio_inheritance); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_prio_restore); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_task_comm); + +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_before_pick_task_fair); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_balance_fair); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_chk_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_put_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_task_fits_cpu); + diff --git a/kernel/seccomp.c b/kernel/seccomp.c index 255999ba9190a..1d26a0e978f26 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -1099,7 +1099,7 @@ static void seccomp_handle_addfd(struct seccomp_kaddfd *addfd, struct seccomp_kn static bool should_sleep_killable(struct seccomp_filter *match, struct seccomp_knotif *n) { - return match->wait_killable_recv && n->state == SECCOMP_NOTIFY_SENT; + return match->wait_killable_recv && n->state >= SECCOMP_NOTIFY_SENT; } static int seccomp_do_user_notification(int this_syscall, @@ -1146,13 +1146,11 @@ static int seccomp_do_user_notification(int this_syscall, if (err != 0) { /* - * Check to see if the notifcation got picked up and - * whether we should switch to wait killable. + * Check to see whether we should switch to wait + * killable. Only return the interrupted error if not. */ - if (!wait_killable && should_sleep_killable(match, &n)) - continue; - - goto interrupted; + if (!(!wait_killable && should_sleep_killable(match, &n))) + goto interrupted; } addfd = list_first_entry_or_null(&n.addfd, diff --git a/kernel/smp.c b/kernel/smp.c index 3eeffeaf5450c..13b376c7296a5 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -862,16 +862,15 @@ static void smp_call_function_many_cond(const struct cpumask *mask, * @mask: The set of cpus to run on (only runs on online subset). * @func: The function to run. This must be fast and non-blocking. * @info: An arbitrary pointer to pass to the function. - * @wait: Bitmask that controls the operation. If %SCF_WAIT is set, wait - * (atomically) until function has completed on other CPUs. If - * %SCF_RUN_LOCAL is set, the function will also be run locally - * if the local CPU is set in the @cpumask. - * - * If @wait is true, then returns once @func has returned. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. * * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. Preemption * must be disabled when calling this function. + * + * @func is not called on the local CPU even if @mask contains it. Consider + * using on_each_cpu_cond_mask() instead if this is not desirable. */ void smp_call_function_many(const struct cpumask *mask, smp_call_func_t func, void *info, bool wait) diff --git a/kernel/softirq.c b/kernel/softirq.c index d26b1c663f8d9..28207ff4f110a 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -148,6 +148,18 @@ static DEFINE_PER_CPU(struct softirq_ctrl, softirq_ctrl) = { .lock = INIT_LOCAL_LOCK(softirq_ctrl.lock), }; +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static struct lock_class_key bh_lock_key; +struct lockdep_map bh_lock_map = { + .name = "local_bh", + .key = &bh_lock_key, + .wait_type_outer = LD_WAIT_FREE, + .wait_type_inner = LD_WAIT_CONFIG, /* PREEMPT_RT makes BH preemptible. */ + .lock_type = LD_LOCK_PERCPU, +}; +EXPORT_SYMBOL_GPL(bh_lock_map); +#endif + /** * local_bh_blocked() - Check for idle whether BH processing is blocked * @@ -170,6 +182,8 @@ void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) WARN_ON_ONCE(in_hardirq()); + lock_map_acquire_read(&bh_lock_map); + /* First entry of a task into a BH disabled section? */ if (!current->softirq_disable_cnt) { if (preemptible()) { @@ -233,6 +247,8 @@ void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) WARN_ON_ONCE(in_hardirq()); lockdep_assert_irqs_enabled(); + lock_map_release(&bh_lock_map); + local_irq_save(flags); curcnt = __this_cpu_read(softirq_ctrl.cnt); @@ -283,6 +299,8 @@ static inline void ksoftirqd_run_begin(void) /* Counterpart to ksoftirqd_run_begin() */ static inline void ksoftirqd_run_end(void) { + /* pairs with the lock_map_acquire_read() in ksoftirqd_run_begin() */ + lock_map_release(&bh_lock_map); __local_bh_enable(SOFTIRQ_OFFSET, true); WARN_ON_ONCE(in_interrupt()); local_irq_enable(); diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index 7b65bb0b4a668..0c3d387d3db73 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -82,18 +82,15 @@ static void cpu_stop_signal_done(struct cpu_stop_done *done) } static void __cpu_stop_queue_work(struct cpu_stopper *stopper, - struct cpu_stop_work *work, - struct wake_q_head *wakeq) + struct cpu_stop_work *work) { list_add_tail(&work->list, &stopper->works); - wake_q_add(wakeq, stopper->thread); } /* queue @work to @stopper. if offline, @work is completed immediately */ static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work) { struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); - DEFINE_WAKE_Q(wakeq); unsigned long flags; bool enabled; @@ -101,12 +98,13 @@ static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work) raw_spin_lock_irqsave(&stopper->lock, flags); enabled = stopper->enabled; if (enabled) - __cpu_stop_queue_work(stopper, work, &wakeq); + __cpu_stop_queue_work(stopper, work); else if (work->done) cpu_stop_signal_done(work->done); raw_spin_unlock_irqrestore(&stopper->lock, flags); - wake_up_q(&wakeq); + if (enabled) + wake_up_process(stopper->thread); preempt_enable(); return enabled; @@ -264,7 +262,6 @@ static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1, { struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1); struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2); - DEFINE_WAKE_Q(wakeq); int err; retry: @@ -300,8 +297,8 @@ static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1, } err = 0; - __cpu_stop_queue_work(stopper1, work1, &wakeq); - __cpu_stop_queue_work(stopper2, work2, &wakeq); + __cpu_stop_queue_work(stopper1, work1); + __cpu_stop_queue_work(stopper2, work2); unlock: raw_spin_unlock(&stopper2->lock); @@ -316,7 +313,10 @@ static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1, goto retry; } - wake_up_q(&wakeq); + if (!err) { + wake_up_process(stopper1->thread); + wake_up_process(stopper2->thread); + } preempt_enable(); return err; diff --git a/kernel/sys.c b/kernel/sys.c index 74bed736f6a85..c6b0141e41354 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1691,6 +1691,7 @@ SYSCALL_DEFINE4(prlimit64, pid_t, pid, unsigned int, resource, struct rlimit old, new; struct task_struct *tsk; unsigned int checkflags = 0; + bool need_tasklist; int ret; if (old_rlim) @@ -1717,8 +1718,25 @@ SYSCALL_DEFINE4(prlimit64, pid_t, pid, unsigned int, resource, get_task_struct(tsk); rcu_read_unlock(); - ret = do_prlimit(tsk, resource, new_rlim ? &new : NULL, - old_rlim ? &old : NULL); + need_tasklist = !same_thread_group(tsk, current); + if (need_tasklist) { + /* + * Ensure we can't race with group exit or de_thread(), + * so tsk->group_leader can't be freed or changed until + * read_unlock(tasklist_lock) below. + */ + read_lock(&tasklist_lock); + if (!pid_alive(tsk)) + ret = -ESRCH; + } + + if (!ret) { + ret = do_prlimit(tsk, resource, new_rlim ? &new : NULL, + old_rlim ? &old : NULL); + } + + if (need_tasklist) + read_unlock(&tasklist_lock); if (!ret && old_rlim) { rlim_to_rlim64(&old, &old64); diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 3130f24daf597..353829883e66d 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -288,7 +288,7 @@ static void clocksource_verify_choose_cpus(void) { int cpu, i, n = verify_n_cpus; - if (n < 0) { + if (n < 0 || n >= num_online_cpus()) { /* Check all of the CPUs. */ cpumask_copy(&cpus_chosen, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &cpus_chosen); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 9036b4b53c4a4..2221b77fccfe8 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -114,7 +114,7 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = .clockid = CLOCK_TAI, .get_time = &ktime_get_clocktai, }, - } + }, }; DEFINE_PER_CPU(call_single_data_t, hrtimer_base_csd) = @@ -205,11 +205,9 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, * * Called with cpu_base->lock of target cpu held. */ -static bool -hrtimer_suitable_target(struct hrtimer *timer, - struct hrtimer_clock_base *new_base, - struct hrtimer_cpu_base *new_cpu_base, - struct hrtimer_cpu_base *this_cpu_base) +static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base, + struct hrtimer_cpu_base *new_cpu_base, + struct hrtimer_cpu_base *this_cpu_base) { ktime_t expires; @@ -234,9 +232,7 @@ hrtimer_suitable_target(struct hrtimer *timer, return expires >= new_base->cpu_base->expires_next; } -static inline -struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, - int pinned) +static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, int pinned) { if (!hrtimer_base_is_online(base)) { int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER)); @@ -304,8 +300,7 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, } WRITE_ONCE(timer->base, new_base); } else { - if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, - this_cpu_base)) { + if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) { new_cpu_base = this_cpu_base; goto again; } @@ -679,17 +674,12 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) /* * Is the high resolution mode active ? */ -static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) +static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) { return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ? cpu_base->hres_active : 0; } -static inline int hrtimer_hres_active(void) -{ - return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases)); -} - static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer, ktime_t expires_next) @@ -713,7 +703,7 @@ static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, * set. So we'd effectively block all timers until the T2 event * fires. */ - if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) + if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) return; tick_program_event(expires_next, 1); @@ -821,13 +811,13 @@ static void retrigger_next_event(void *arg) * of the next expiring timer is enough. The return from the SMP * function call will take care of the reprogramming in case the * CPU was in a NOHZ idle sleep. + * + * In periodic low resolution mode, the next softirq expiration + * must also be updated. */ - if (!__hrtimer_hres_active(base) && !tick_nohz_active) - return; - raw_spin_lock(&base->lock); hrtimer_update_base(base); - if (__hrtimer_hres_active(base)) + if (hrtimer_hres_active(base)) hrtimer_force_reprogram(base, 0); else hrtimer_update_next_event(base); @@ -984,7 +974,7 @@ void clock_was_set(unsigned int bases) cpumask_var_t mask; int cpu; - if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active) + if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active) goto out_timerfd; if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { @@ -1302,9 +1292,19 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, first = enqueue_hrtimer(timer, new_base, mode); if (!force_local) { + /* + * If the current CPU base is online, then the timer is + * never queued on a remote CPU if it would be the first + * expiring timer there. + */ if (hrtimer_base_is_online(this_cpu_base)) return first; + /* + * Timer was enqueued remote because the current base is + * already offline. If the timer is the first to expire, + * kick the remote CPU to reprogram the clock event. + */ if (first) { struct hrtimer_cpu_base *new_cpu_base = new_base->cpu_base; call_single_data_t *csd = per_cpu_ptr(&hrtimer_base_csd, new_cpu_base->cpu); @@ -1553,7 +1553,7 @@ u64 hrtimer_get_next_event(void) raw_spin_lock_irqsave(&cpu_base->lock, flags); - if (!__hrtimer_hres_active(cpu_base)) + if (!hrtimer_hres_active(cpu_base)) expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); raw_spin_unlock_irqrestore(&cpu_base->lock, flags); @@ -1576,7 +1576,7 @@ u64 hrtimer_next_event_without(const struct hrtimer *exclude) raw_spin_lock_irqsave(&cpu_base->lock, flags); - if (__hrtimer_hres_active(cpu_base)) { + if (hrtimer_hres_active(cpu_base)) { unsigned int active; if (!cpu_base->softirq_activated) { @@ -1937,25 +1937,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) tick_program_event(expires_next, 1); pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta)); } - -/* called with interrupts disabled */ -static inline void __hrtimer_peek_ahead_timers(void) -{ - struct tick_device *td; - - if (!hrtimer_hres_active()) - return; - - td = this_cpu_ptr(&tick_cpu_device); - if (td && td->evtdev) - hrtimer_interrupt(td->evtdev); -} - -#else /* CONFIG_HIGH_RES_TIMERS */ - -static inline void __hrtimer_peek_ahead_timers(void) { } - -#endif /* !CONFIG_HIGH_RES_TIMERS */ +#endif /* !CONFIG_HIGH_RES_TIMERS */ /* * Called from run_local_timers in hardirq context every jiffy @@ -1966,7 +1948,7 @@ void hrtimer_run_queues(void) unsigned long flags; ktime_t now; - if (__hrtimer_hres_active(cpu_base)) + if (hrtimer_hres_active(cpu_base)) return; /* @@ -2313,11 +2295,6 @@ int hrtimers_cpu_dying(unsigned int dying_cpu) &new_base->clock_base[i]); } - /* - * The migration might have changed the first expiring softirq - * timer on this CPU. Update it. - */ - __hrtimer_get_next_event(new_base, HRTIMER_ACTIVE_SOFT); /* Tell the other CPU to retrigger the next event */ smp_call_function_single(ncpu, retrigger_next_event, NULL, 0); diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index b924f0f096fa4..7534069f60333 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -118,6 +118,7 @@ static int posix_timer_add(struct k_itimer *timer) return id; } spin_unlock(&hash_lock); + cond_resched(); } /* POSIX return code when no timer ID could be allocated */ return -EAGAIN; diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c old mode 100644 new mode 100755 index 254624e4bd431..998c2eefe1730 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -1124,7 +1124,6 @@ void tick_nohz_idle_stop_tick(void) ktime_t expires; trace_android_vh_tick_nohz_idle_stop_tick(NULL); - /* * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the * tick timer expiration time is known already. diff --git a/kernel/time/timer.c b/kernel/time/timer.c index e8148f77e31b6..2cf3566e41883 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1416,10 +1416,11 @@ static int __try_to_del_timer_sync(struct timer_list *timer, bool shutdown) base = lock_timer_base(timer, &flags); - if (base->running_timer != timer) + if (base->running_timer != timer) { ret = detach_if_pending(timer, base, true); - if (shutdown) - timer->function = NULL; + if (shutdown) + timer->function = NULL; + } raw_spin_unlock_irqrestore(&base->lock, flags); diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index ed7d6ad694fba..20a5e6962b696 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c @@ -46,7 +46,7 @@ static void print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer, int idx, u64 now) { - SEQ_printf(m, " #%d: <%pK>, %ps", idx, taddr, timer->function); + SEQ_printf(m, " #%d: <%p>, %ps", idx, taddr, timer->function); SEQ_printf(m, ", S:%02x", timer->state); SEQ_printf(m, "\n"); SEQ_printf(m, " # expires at %Lu-%Lu nsecs [in %Ld to %Ld nsecs]\n", @@ -98,7 +98,7 @@ print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, static void print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now) { - SEQ_printf(m, " .base: %pK\n", base); + SEQ_printf(m, " .base: %p\n", base); SEQ_printf(m, " .index: %d\n", base->index); SEQ_printf(m, " .resolution: %u nsecs\n", hrtimer_resolution); diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 287de43442bd0..99032db30c5bb 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1222,7 +1222,8 @@ static const struct bpf_func_proto bpf_get_func_arg_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_LONG, + .arg3_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_ALIGNED, + .arg3_size = sizeof(u64), }; BPF_CALL_2(get_func_ret, void *, ctx, u64 *, value) @@ -1238,7 +1239,8 @@ static const struct bpf_func_proto bpf_get_func_ret_proto = { .func = get_func_ret, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_LONG, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_ALIGNED, + .arg2_size = sizeof(u64), }; BPF_CALL_1(get_func_arg_cnt, void *, ctx) @@ -1832,7 +1834,7 @@ static struct pt_regs *get_bpf_raw_tp_regs(void) struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs); int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level); - if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) { + if (nest_level > ARRAY_SIZE(tp_regs->regs)) { this_cpu_dec(bpf_raw_tp_nest_level); return ERR_PTR(-EBUSY); } @@ -2723,18 +2725,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, struct bpf_run_ctx *old_run_ctx; int err; + /* + * graph tracer framework ensures we won't migrate, so there is no need + * to use migrate_disable for bpf_prog_run again. The check here just for + * __this_cpu_inc_return. + */ + cant_sleep(); + if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { err = 0; goto out; } - migrate_disable(); rcu_read_lock(); old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); err = bpf_prog_run(link->link.prog, regs); bpf_reset_run_ctx(old_run_ctx); rcu_read_unlock(); - migrate_enable(); out: __this_cpu_dec(bpf_prog_active); diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 650493ed76cd4..44e31eebb7aaa 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -4058,13 +4058,17 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag, } else { iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); } + } else { + if (hash) + iter->hash = alloc_and_copy_ftrace_hash(hash->size_bits, hash); + else + iter->hash = EMPTY_HASH; + } - if (!iter->hash) { - trace_parser_put(&iter->parser); - goto out_unlock; - } - } else - iter->hash = hash; + if (!iter->hash) { + trace_parser_put(&iter->parser); + goto out_unlock; + } ret = 0; @@ -5366,6 +5370,17 @@ static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long } } +static void reset_direct(struct ftrace_ops *ops, unsigned long addr) +{ + struct ftrace_hash *hash = ops->func_hash->filter_hash; + + remove_direct_functions_hash(hash, addr); + + /* cleanup for possible another register call */ + ops->func = NULL; + ops->trampoline = 0; +} + /** * register_ftrace_direct - Call a custom trampoline directly * for multiple functions registered in @ops @@ -5461,6 +5476,8 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) ops->direct_call = addr; err = register_ftrace_function_nolock(ops); + if (err) + reset_direct(ops, addr); out_unlock: mutex_unlock(&direct_mutex); @@ -5493,7 +5510,6 @@ EXPORT_SYMBOL_GPL(register_ftrace_direct); int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr, bool free_filters) { - struct ftrace_hash *hash = ops->func_hash->filter_hash; int err; if (check_direct_multi(ops)) @@ -5503,13 +5519,9 @@ int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr, mutex_lock(&direct_mutex); err = unregister_ftrace_function(ops); - remove_direct_functions_hash(hash, addr); + reset_direct(ops, addr); mutex_unlock(&direct_mutex); - /* cleanup for possible another register call */ - ops->func = NULL; - ops->trampoline = 0; - if (free_filters) ftrace_free_filter(ops); return err; @@ -5922,9 +5934,6 @@ int ftrace_regex_release(struct inode *inode, struct file *file) ftrace_hash_move_and_update_ops(iter->ops, orig_hash, iter->hash, filter_hash); mutex_unlock(&ftrace_lock); - } else { - /* For read only, the hash is the ops hash */ - iter->hash = NULL; } mutex_unlock(&iter->ops->func_hash->regex_lock); @@ -6772,9 +6781,10 @@ void ftrace_release_mod(struct module *mod) mutex_lock(&ftrace_lock); - if (ftrace_disabled) - goto out_unlock; - + /* + * To avoid the UAF problem after the module is unloaded, the + * 'mod_map' resource needs to be released unconditionally. + */ list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) { if (mod_map->mod == mod) { list_del_rcu(&mod_map->list); @@ -6783,6 +6793,9 @@ void ftrace_release_mod(struct module *mod) } } + if (ftrace_disabled) + goto out_unlock; + /* * Each module has its own ftrace_pages, remove * them from the list. @@ -6868,6 +6881,8 @@ void ftrace_module_enable(struct module *mod) if (!within_module(rec->ip, mod)) break; + cond_resched(); + /* Weak functions should still be ignored */ if (!test_for_valid_rec(rec)) { /* Clear all other flags. Should not be enabled anyway */ diff --git a/kernel/trace/preemptirq_delay_test.c b/kernel/trace/preemptirq_delay_test.c index cb0871fbdb07f..acb0c971a4082 100644 --- a/kernel/trace/preemptirq_delay_test.c +++ b/kernel/trace/preemptirq_delay_test.c @@ -34,8 +34,6 @@ MODULE_PARM_DESC(cpu_affinity, "Cpu num test is running on"); static struct completion done; -#define MIN(x, y) ((x) < (y) ? (x) : (y)) - static void busy_wait(ulong time) { u64 start, end; @@ -119,12 +117,15 @@ static int preemptirq_delay_run(void *data) { int i; int s = MIN(burst_size, NR_TEST_FUNCS); - struct cpumask cpu_mask; + cpumask_var_t cpu_mask; + + if (!alloc_cpumask_var(&cpu_mask, GFP_KERNEL)) + return -ENOMEM; if (cpu_affinity > -1) { - cpumask_clear(&cpu_mask); - cpumask_set_cpu(cpu_affinity, &cpu_mask); - if (set_cpus_allowed_ptr(current, &cpu_mask)) + cpumask_clear(cpu_mask); + cpumask_set_cpu(cpu_affinity, cpu_mask); + if (set_cpus_allowed_ptr(current, cpu_mask)) pr_err("cpu_affinity:%d, failed\n", cpu_affinity); } @@ -141,6 +142,8 @@ static int preemptirq_delay_run(void *data) __set_current_state(TASK_RUNNING); + free_cpumask_var(cpu_mask); + return 0; } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 7007ed1e25128..6d6ee2bb69844 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -767,7 +767,10 @@ int trace_pid_write(struct trace_pid_list *filtered_pids, /* copy the current bits to the new max */ ret = trace_pid_list_first(filtered_pids, &pid); while (!ret) { - trace_pid_list_set(pid_list, pid); + ret = trace_pid_list_set(pid_list, pid); + if (ret < 0) + goto out; + ret = trace_pid_list_next(filtered_pids, pid + 1, &pid); nr_pids++; } @@ -804,6 +807,7 @@ int trace_pid_write(struct trace_pid_list *filtered_pids, trace_parser_clear(&parser); ret = 0; } + out: trace_parser_put(&parser); if (ret < 0) { @@ -1677,7 +1681,7 @@ int trace_get_user(struct trace_parser *parser, const char __user *ubuf, ret = get_user(ch, ubuf++); if (ret) - goto out; + goto fail; read++; cnt--; @@ -1691,7 +1695,7 @@ int trace_get_user(struct trace_parser *parser, const char __user *ubuf, while (cnt && isspace(ch)) { ret = get_user(ch, ubuf++); if (ret) - goto out; + goto fail; read++; cnt--; } @@ -1701,8 +1705,7 @@ int trace_get_user(struct trace_parser *parser, const char __user *ubuf, /* only spaces were written */ if (isspace(ch) || !ch) { *ppos += read; - ret = read; - goto out; + return read; } } @@ -1712,11 +1715,12 @@ int trace_get_user(struct trace_parser *parser, const char __user *ubuf, parser->buffer[parser->idx++] = ch; else { ret = -EINVAL; - goto out; + goto fail; } + ret = get_user(ch, ubuf++); if (ret) - goto out; + goto fail; read++; cnt--; } @@ -1732,13 +1736,13 @@ int trace_get_user(struct trace_parser *parser, const char __user *ubuf, parser->buffer[parser->idx] = 0; } else { ret = -EINVAL; - goto out; + goto fail; } *ppos += read; - ret = read; - -out: + return read; +fail: + trace_parser_fail(parser); return ret; } @@ -2227,10 +2231,10 @@ int __init register_tracer(struct tracer *type) mutex_unlock(&trace_types_lock); if (ret || !default_bootup_tracer) - goto out_unlock; + return ret; if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) - goto out_unlock; + return 0; printk(KERN_INFO "Starting tracer '%s'\n", type->name); /* Do we want this tracer to start on bootup? */ @@ -2242,8 +2246,7 @@ int __init register_tracer(struct tracer *type) /* disable other selftests, since this will break it. */ disable_tracing_selftest("running a tracer"); - out_unlock: - return ret; + return 0; } static void tracing_reset_cpu(struct array_buffer *buf, int cpu) @@ -3503,10 +3506,9 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) } EXPORT_SYMBOL_GPL(trace_vbprintk); -__printf(3, 0) -static int -__trace_array_vprintk(struct trace_buffer *buffer, - unsigned long ip, const char *fmt, va_list args) +static __printf(3, 0) +int __trace_array_vprintk(struct trace_buffer *buffer, + unsigned long ip, const char *fmt, va_list args) { struct trace_event_call *call = &event_print; struct ring_buffer_event *event; @@ -3559,7 +3561,6 @@ __trace_array_vprintk(struct trace_buffer *buffer, return len; } -__printf(3, 0) int trace_array_vprintk(struct trace_array *tr, unsigned long ip, const char *fmt, va_list args) { @@ -3589,7 +3590,6 @@ int trace_array_vprintk(struct trace_array *tr, * Note, trace_array_init_printk() must be called on @tr before this * can be used. */ -__printf(3, 0) int trace_array_printk(struct trace_array *tr, unsigned long ip, const char *fmt, ...) { @@ -3634,7 +3634,6 @@ int trace_array_init_printk(struct trace_array *tr) } EXPORT_SYMBOL_GPL(trace_array_init_printk); -__printf(3, 4) int trace_array_printk_buf(struct trace_buffer *buffer, unsigned long ip, const char *fmt, ...) { @@ -3650,7 +3649,6 @@ int trace_array_printk_buf(struct trace_buffer *buffer, return ret; } -__printf(2, 0) int trace_vprintk(unsigned long ip, const char *fmt, va_list args) { return trace_array_vprintk(&global_trace, ip, fmt, args); @@ -7152,13 +7150,14 @@ static ssize_t tracing_splice_read_pipe(struct file *filp, /* Copy the data into the page, so we can start over. */ ret = trace_seq_to_buffer(&iter->seq, page_address(spd.pages[i]), - trace_seq_used(&iter->seq)); + min((size_t)trace_seq_used(&iter->seq), + (size_t)PAGE_SIZE)); if (ret < 0) { __free_page(spd.pages[i]); break; } spd.partial[i].offset = 0; - spd.partial[i].len = trace_seq_used(&iter->seq); + spd.partial[i].len = ret; trace_seq_init(&iter->seq); } @@ -8863,11 +8862,10 @@ ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, out_reg: ret = tracing_alloc_snapshot_instance(tr); if (ret < 0) - goto out; + return ret; ret = register_ftrace_function_probe(glob, tr, ops, count); - out: return ret < 0 ? ret : 0; } @@ -10332,10 +10330,10 @@ static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_m ret = print_trace_line(&iter); if (ret != TRACE_TYPE_NO_CONSUME) trace_consume(&iter); + + trace_printk_seq(&iter.seq); } touch_nmi_watchdog(); - - trace_printk_seq(&iter.seq); } if (!cnt) @@ -10582,7 +10580,7 @@ __init static int tracer_alloc_buffers(void) BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE); if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) - goto out; + return -ENOMEM; if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) goto out_free_buffer_mask; @@ -10695,7 +10693,6 @@ __init static int tracer_alloc_buffers(void) free_cpumask_var(global_trace.tracing_cpumask); out_free_buffer_mask: free_cpumask_var(tracing_buffer_mask); -out: return ret; } diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 17e54ddc56d54..bcff90d6dd257 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -806,13 +806,15 @@ static inline void __init disable_tracing_selftest(const char *reason) extern void *head_page(struct trace_array_cpu *data); extern unsigned long long ns2usecs(u64 nsec); -extern int -trace_vbprintk(unsigned long ip, const char *fmt, va_list args); -extern int -trace_vprintk(unsigned long ip, const char *fmt, va_list args); -extern int -trace_array_vprintk(struct trace_array *tr, - unsigned long ip, const char *fmt, va_list args); + +__printf(2, 0) +int trace_vbprintk(unsigned long ip, const char *fmt, va_list args); +__printf(2, 0) +int trace_vprintk(unsigned long ip, const char *fmt, va_list args); +__printf(3, 0) +int trace_array_vprintk(struct trace_array *tr, + unsigned long ip, const char *fmt, va_list args); +__printf(3, 4) int trace_array_printk_buf(struct trace_buffer *buffer, unsigned long ip, const char *fmt, ...); void trace_printk_seq(struct trace_seq *s); @@ -1157,6 +1159,7 @@ bool ftrace_event_is_function(struct trace_event_call *call); */ struct trace_parser { bool cont; + bool fail; char *buffer; unsigned idx; unsigned size; @@ -1164,7 +1167,7 @@ struct trace_parser { static inline bool trace_parser_loaded(struct trace_parser *parser) { - return (parser->idx != 0); + return !parser->fail && parser->idx != 0; } static inline bool trace_parser_cont(struct trace_parser *parser) @@ -1178,6 +1181,11 @@ static inline void trace_parser_clear(struct trace_parser *parser) parser->idx = 0; } +static inline void trace_parser_fail(struct trace_parser *parser) +{ + parser->fail = true; +} + extern int trace_parser_get_init(struct trace_parser *parser, int size); extern void trace_parser_put(struct trace_parser *parser); extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf, @@ -1646,6 +1654,9 @@ extern int event_enable_register_trigger(char *glob, extern void event_enable_unregister_trigger(char *glob, struct event_trigger_data *test, struct trace_event_file *file); +extern struct event_trigger_data * +trigger_data_alloc(struct event_command *cmd_ops, char *cmd, char *param, + void *private_data); extern void trigger_data_free(struct event_trigger_data *data); extern int event_trigger_init(struct event_trigger_data *data); extern int trace_event_trigger_enable_disable(struct trace_event_file *file, @@ -1672,11 +1683,6 @@ extern bool event_trigger_check_remove(const char *glob); extern bool event_trigger_empty_param(const char *param); extern int event_trigger_separate_filter(char *param_and_filter, char **param, char **filter, bool param_required); -extern struct event_trigger_data * -event_trigger_alloc(struct event_command *cmd_ops, - char *cmd, - char *param, - void *private_data); extern int event_trigger_parse_num(char *trigger, struct event_trigger_data *trigger_data); extern int event_trigger_set_filter(struct event_command *cmd_ops, @@ -2058,7 +2064,7 @@ static inline bool is_good_system_name(const char *name) static inline void sanitize_event_name(char *name) { while (*name++ != '\0') - if (*name == ':' || *name == '.') + if (*name == ':' || *name == '.' || *name == '*') *name = '_'; } diff --git a/kernel/trace/trace_dynevent.c b/kernel/trace/trace_dynevent.c index 4376887e0d8aa..76737492e750e 100644 --- a/kernel/trace/trace_dynevent.c +++ b/kernel/trace/trace_dynevent.c @@ -16,7 +16,7 @@ #include "trace_output.h" /* for trace_event_sem */ #include "trace_dynevent.h" -static DEFINE_MUTEX(dyn_event_ops_mutex); +DEFINE_MUTEX(dyn_event_ops_mutex); static LIST_HEAD(dyn_event_ops_list); bool trace_event_dyn_try_get_ref(struct trace_event_call *dyn_call) @@ -125,6 +125,20 @@ int dyn_event_release(const char *raw_command, struct dyn_event_operations *type return ret; } +/* + * Locked version of event creation. The event creation must be protected by + * dyn_event_ops_mutex because of protecting trace_probe_log. + */ +int dyn_event_create(const char *raw_command, struct dyn_event_operations *type) +{ + int ret; + + mutex_lock(&dyn_event_ops_mutex); + ret = type->create(raw_command); + mutex_unlock(&dyn_event_ops_mutex); + return ret; +} + static int create_dyn_event(const char *raw_command) { struct dyn_event_operations *ops; @@ -225,6 +239,10 @@ static int dyn_event_open(struct inode *inode, struct file *file) { int ret; + ret = security_locked_down(LOCKDOWN_TRACEFS); + if (ret) + return ret; + ret = tracing_check_open_get_tr(NULL); if (ret) return ret; diff --git a/kernel/trace/trace_dynevent.h b/kernel/trace/trace_dynevent.h index 936477a111d3e..beee3f8d75444 100644 --- a/kernel/trace/trace_dynevent.h +++ b/kernel/trace/trace_dynevent.h @@ -100,6 +100,7 @@ void *dyn_event_seq_next(struct seq_file *m, void *v, loff_t *pos); void dyn_event_seq_stop(struct seq_file *m, void *v); int dyn_events_release_all(struct dyn_event_operations *type); int dyn_event_release(const char *raw_command, struct dyn_event_operations *type); +int dyn_event_create(const char *raw_command, struct dyn_event_operations *type); /* * for_each_dyn_event - iterate over the dyn_event list diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 5f0bd608bf909..7d6c06323539d 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2821,7 +2821,10 @@ __register_event(struct trace_event_call *call, struct module *mod) if (ret < 0) return ret; + down_write(&trace_event_sem); list_add(&call->list, &ftrace_events); + up_write(&trace_event_sem); + if (call->flags & TRACE_EVENT_FL_DYNAMIC) atomic_set(&call->refcnt, 0); else @@ -3399,6 +3402,8 @@ __trace_add_event_dirs(struct trace_array *tr) struct trace_event_call *call; int ret; + lockdep_assert_held(&trace_event_sem); + list_for_each_entry(call, &ftrace_events, list) { ret = __trace_add_new_event(call, tr); if (ret < 0) diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 604d63380a90b..9fc93e3c3fe43 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -3258,14 +3258,16 @@ static struct field_var *create_field_var(struct hist_trigger_data *hist_data, var = create_var(hist_data, file, field_name, val->size, val->type); if (IS_ERR(var)) { hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); - kfree(val); + destroy_hist_field(val, 0); ret = PTR_ERR(var); goto err; } field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); if (!field_var) { - kfree(val); + destroy_hist_field(val, 0); + kfree_const(var->type); + kfree(var->var.name); kfree(var); ret = -ENOMEM; goto err; @@ -5257,17 +5259,94 @@ hist_trigger_actions(struct hist_trigger_data *hist_data, } } +/* + * The hist_pad structure is used to save information to create + * a histogram from the histogram trigger. It's too big to store + * on the stack, so when the histogram trigger is initialized + * a percpu array of 4 hist_pad structures is allocated. + * This will cover every context from normal, softirq, irq and NMI + * in the very unlikely event that a tigger happens at each of + * these contexts and interrupts a currently active trigger. + */ +struct hist_pad { + unsigned long entries[HIST_STACKTRACE_DEPTH]; + u64 var_ref_vals[TRACING_MAP_VARS_MAX]; + char compound_key[HIST_KEY_SIZE_MAX]; +}; + +static struct hist_pad __percpu *hist_pads; +static DEFINE_PER_CPU(int, hist_pad_cnt); +static refcount_t hist_pad_ref; + +/* One hist_pad for every context (normal, softirq, irq, NMI) */ +#define MAX_HIST_CNT 4 + +static int alloc_hist_pad(void) +{ + lockdep_assert_held(&event_mutex); + + if (refcount_read(&hist_pad_ref)) { + refcount_inc(&hist_pad_ref); + return 0; + } + + hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT, + __alignof__(struct hist_pad)); + if (!hist_pads) + return -ENOMEM; + + refcount_set(&hist_pad_ref, 1); + return 0; +} + +static void free_hist_pad(void) +{ + lockdep_assert_held(&event_mutex); + + if (!refcount_dec_and_test(&hist_pad_ref)) + return; + + free_percpu(hist_pads); + hist_pads = NULL; +} + +static struct hist_pad *get_hist_pad(void) +{ + struct hist_pad *hist_pad; + int cnt; + + if (WARN_ON_ONCE(!hist_pads)) + return NULL; + + preempt_disable(); + + hist_pad = per_cpu_ptr(hist_pads, smp_processor_id()); + + if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) { + preempt_enable(); + return NULL; + } + + cnt = this_cpu_inc_return(hist_pad_cnt) - 1; + + return &hist_pad[cnt]; +} + +static void put_hist_pad(void) +{ + this_cpu_dec(hist_pad_cnt); + preempt_enable(); +} + static void event_hist_trigger(struct event_trigger_data *data, struct trace_buffer *buffer, void *rec, struct ring_buffer_event *rbe) { struct hist_trigger_data *hist_data = data->private_data; bool use_compound_key = (hist_data->n_keys > 1); - unsigned long entries[HIST_STACKTRACE_DEPTH]; - u64 var_ref_vals[TRACING_MAP_VARS_MAX]; - char compound_key[HIST_KEY_SIZE_MAX]; struct tracing_map_elt *elt = NULL; struct hist_field *key_field; + struct hist_pad *hist_pad; u64 field_contents; void *key = NULL; unsigned int i; @@ -5275,12 +5354,18 @@ static void event_hist_trigger(struct event_trigger_data *data, if (unlikely(!rbe)) return; - memset(compound_key, 0, hist_data->key_size); + hist_pad = get_hist_pad(); + if (!hist_pad) + return; + + memset(hist_pad->compound_key, 0, hist_data->key_size); for_each_hist_key_field(i, hist_data) { key_field = hist_data->fields[i]; if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { + unsigned long *entries = hist_pad->entries; + memset(entries, 0, HIST_STACKTRACE_SIZE); if (key_field->field) { unsigned long *stack, n_entries; @@ -5304,24 +5389,29 @@ static void event_hist_trigger(struct event_trigger_data *data, } if (use_compound_key) - add_to_key(compound_key, key, key_field, rec); + add_to_key(hist_pad->compound_key, key, key_field, rec); } if (use_compound_key) - key = compound_key; + key = hist_pad->compound_key; if (hist_data->n_var_refs && - !resolve_var_refs(hist_data, key, var_ref_vals, false)) - return; + !resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, false)) + goto out; elt = tracing_map_insert(hist_data->map, key); if (!elt) - return; + goto out; - hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals); + hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, hist_pad->var_ref_vals); - if (resolve_var_refs(hist_data, key, var_ref_vals, true)) - hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals); + if (resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, true)) { + hist_trigger_actions(hist_data, elt, buffer, rec, rbe, + key, hist_pad->var_ref_vals); + } + + out: + put_hist_pad(); } static void hist_trigger_stacktrace_print(struct seq_file *m, @@ -6081,6 +6171,9 @@ static int event_hist_trigger_init(struct event_trigger_data *data) { struct hist_trigger_data *hist_data = data->private_data; + if (alloc_hist_pad() < 0) + return -ENOMEM; + if (!data->ref && hist_data->attrs->name) save_named_trigger(hist_data->attrs->name, data); @@ -6125,6 +6218,7 @@ static void event_hist_trigger_free(struct event_trigger_data *data) destroy_hist_data(hist_data); } + free_hist_pad(); } static struct event_trigger_ops event_hist_trigger_ops = { @@ -6140,9 +6234,7 @@ static int event_hist_trigger_named_init(struct event_trigger_data *data) save_named_trigger(data->named_data->name, data); - event_hist_trigger_init(data->named_data); - - return 0; + return event_hist_trigger_init(data->named_data); } static void event_hist_trigger_named_free(struct event_trigger_data *data) @@ -6629,7 +6721,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, return PTR_ERR(hist_data); } - trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data); + trigger_data = trigger_data_alloc(cmd_ops, cmd, param, hist_data); if (!trigger_data) { ret = -ENOMEM; goto out_free; diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 76abc9a45f971..fe079ff82ef1b 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -809,7 +809,7 @@ int event_trigger_separate_filter(char *param_and_filter, char **param, } /** - * event_trigger_alloc - allocate and init event_trigger_data for a trigger + * trigger_data_alloc - allocate and init event_trigger_data for a trigger * @cmd_ops: The event_command operations for the trigger * @cmd: The cmd string * @param: The param string @@ -820,14 +820,14 @@ int event_trigger_separate_filter(char *param_and_filter, char **param, * trigger_ops to assign to the event_trigger_data. @private_data can * also be passed in and associated with the event_trigger_data. * - * Use event_trigger_free() to free an event_trigger_data object. + * Use trigger_data_free() to free an event_trigger_data object. * * Return: The trigger_data object success, NULL otherwise */ -struct event_trigger_data *event_trigger_alloc(struct event_command *cmd_ops, - char *cmd, - char *param, - void *private_data) +struct event_trigger_data *trigger_data_alloc(struct event_command *cmd_ops, + char *cmd, + char *param, + void *private_data) { struct event_trigger_data *trigger_data; struct event_trigger_ops *trigger_ops; @@ -994,13 +994,13 @@ event_trigger_parse(struct event_command *cmd_ops, return ret; ret = -ENOMEM; - trigger_data = event_trigger_alloc(cmd_ops, cmd, param, file); + trigger_data = trigger_data_alloc(cmd_ops, cmd, param, file); if (!trigger_data) goto out; if (remove) { event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); - kfree(trigger_data); + trigger_data_free(trigger_data); ret = 0; goto out; } @@ -1027,7 +1027,7 @@ event_trigger_parse(struct event_command *cmd_ops, out_free: event_trigger_reset_filter(cmd_ops, trigger_data); - kfree(trigger_data); + trigger_data_free(trigger_data); goto out; } @@ -1554,7 +1554,7 @@ stacktrace_trigger(struct event_trigger_data *data, struct trace_event_file *file = data->private_data; if (file) - __trace_stack(file->tr, tracing_gen_ctx(), STACK_SKIP); + __trace_stack(file->tr, tracing_gen_ctx_dec(), STACK_SKIP); else trace_dump_stack(STACK_SKIP); } @@ -1787,7 +1787,7 @@ int event_enable_trigger_parse(struct event_command *cmd_ops, enable_data->enable = enable; enable_data->file = event_enable_file; - trigger_data = event_trigger_alloc(cmd_ops, cmd, param, enable_data); + trigger_data = trigger_data_alloc(cmd_ops, cmd, param, enable_data); if (!trigger_data) { kfree(enable_data); goto out; diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c index 93620a7835876..a233262f858c8 100644 --- a/kernel/trace/trace_fprobe.c +++ b/kernel/trace/trace_fprobe.c @@ -342,12 +342,14 @@ static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip, void *entry_data) { struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); + unsigned int flags = trace_probe_load_flag(&tf->tp); int ret = 0; - if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE)) + if (flags & TP_FLAG_TRACE) fentry_trace_func(tf, entry_ip, regs); + #ifdef CONFIG_PERF_EVENTS - if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE)) + if (flags & TP_FLAG_PROFILE) ret = fentry_perf_func(tf, entry_ip, regs); #endif return ret; @@ -359,11 +361,12 @@ static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip, void *entry_data) { struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); + unsigned int flags = trace_probe_load_flag(&tf->tp); - if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE)) + if (flags & TP_FLAG_TRACE) fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data); #ifdef CONFIG_PERF_EVENTS - if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE)) + if (flags & TP_FLAG_PROFILE) fexit_perf_func(tf, entry_ip, ret_ip, regs, entry_data); #endif } diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 69e92c7359fb9..44ae51d1dc45d 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -561,11 +561,7 @@ ftrace_traceoff(unsigned long ip, unsigned long parent_ip, static __always_inline void trace_stack(struct trace_array *tr) { - unsigned int trace_ctx; - - trace_ctx = tracing_gen_ctx(); - - __trace_stack(tr, trace_ctx, FTRACE_STACK_SKIP); + __trace_stack(tr, tracing_gen_ctx_dec(), FTRACE_STACK_SKIP); } static void diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 721a9f000ba3c..68302d9133fbe 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -1004,7 +1004,7 @@ static int create_or_delete_trace_kprobe(const char *raw_command) if (raw_command[0] == '-') return dyn_event_release(raw_command, &trace_kprobe_ops); - ret = trace_kprobe_create(raw_command); + ret = dyn_event_create(raw_command, &trace_kprobe_ops); return ret == -ECANCELED ? -EINVAL : ret; } @@ -1705,14 +1705,15 @@ static int kprobe_register(struct trace_event_call *event, static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) { struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); + unsigned int flags = trace_probe_load_flag(&tk->tp); int ret = 0; raw_cpu_inc(*tk->nhit); - if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) + if (flags & TP_FLAG_TRACE) kprobe_trace_func(tk, regs); #ifdef CONFIG_PERF_EVENTS - if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) + if (flags & TP_FLAG_PROFILE) ret = kprobe_perf_func(tk, regs); #endif return ret; @@ -1724,6 +1725,7 @@ kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) { struct kretprobe *rp = get_kretprobe(ri); struct trace_kprobe *tk; + unsigned int flags; /* * There is a small chance that get_kretprobe(ri) returns NULL when @@ -1736,10 +1738,11 @@ kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) tk = container_of(rp, struct trace_kprobe, rp); raw_cpu_inc(*tk->nhit); - if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) + flags = trace_probe_load_flag(&tk->tp); + if (flags & TP_FLAG_TRACE) kretprobe_trace_func(tk, ri, regs); #ifdef CONFIG_PERF_EVENTS - if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) + if (flags & TP_FLAG_PROFILE) kretprobe_perf_func(tk, ri, regs); #endif return 0; /* We don't tweak kernel, so just return 0 */ diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 5bd781359d38b..4d12d02965a4b 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -665,8 +665,8 @@ __timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, u entry = ring_buffer_event_data(event); - memcpy(&entry->caller, fstack->calls, size); entry->size = fstack->nr_entries; + memcpy(&entry->caller, fstack->calls, size); if (!call_filter_check_discard(call, entry, buffer, event)) trace_buffer_unlock_commit_nostack(buffer, event); diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index db575094c4982..1b9c876da2260 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c @@ -946,11 +946,12 @@ enum print_line_t print_event_fields(struct trace_iterator *iter, struct trace_event_call *call; struct list_head *head; + lockdep_assert_held_read(&trace_event_sem); + /* ftrace defined events have separate call structures */ if (event->type <= __TRACE_LAST_TYPE) { bool found = false; - down_read(&trace_event_sem); list_for_each_entry(call, &ftrace_events, list) { if (call->event.type == event->type) { found = true; @@ -960,7 +961,6 @@ enum print_line_t print_event_fields(struct trace_iterator *iter, if (call->event.type > __TRACE_LAST_TYPE) break; } - up_read(&trace_event_sem); if (!found) { trace_seq_printf(&iter->seq, "UNKNOWN TYPE %d\n", event->type); goto out; diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 606190239c877..187b1fc403c13 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -153,9 +153,12 @@ static const struct fetch_type *find_fetch_type(const char *type, unsigned long } static struct trace_probe_log trace_probe_log; +extern struct mutex dyn_event_ops_mutex; void trace_probe_log_init(const char *subsystem, int argc, const char **argv) { + lockdep_assert_held(&dyn_event_ops_mutex); + trace_probe_log.subsystem = subsystem; trace_probe_log.argc = argc; trace_probe_log.argv = argv; @@ -164,11 +167,15 @@ void trace_probe_log_init(const char *subsystem, int argc, const char **argv) void trace_probe_log_clear(void) { + lockdep_assert_held(&dyn_event_ops_mutex); + memset(&trace_probe_log, 0, sizeof(trace_probe_log)); } void trace_probe_log_set_index(int index) { + lockdep_assert_held(&dyn_event_ops_mutex); + trace_probe_log.index = index; } @@ -177,6 +184,8 @@ void __trace_probe_log_err(int offset, int err_type) char *command, *p; int i, len = 0, pos = 0; + lockdep_assert_held(&dyn_event_ops_mutex); + if (!trace_probe_log.argv) return; @@ -647,7 +656,7 @@ static int parse_btf_arg(char *varname, ret = query_btf_context(ctx); if (ret < 0 || ctx->nr_params == 0) { trace_probe_log_err(ctx->offset, NO_BTF_ENTRY); - return PTR_ERR(params); + return -ENOENT; } } params = ctx->params; diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 48afed3c3f88e..c71fa9c2f3815 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -268,16 +268,21 @@ struct event_file_link { struct list_head list; }; +static inline unsigned int trace_probe_load_flag(struct trace_probe *tp) +{ + return smp_load_acquire(&tp->event->flags); +} + static inline bool trace_probe_test_flag(struct trace_probe *tp, unsigned int flag) { - return !!(tp->event->flags & flag); + return !!(trace_probe_load_flag(tp) & flag); } static inline void trace_probe_set_flag(struct trace_probe *tp, unsigned int flag) { - tp->event->flags |= flag; + smp_store_release(&tp->event->flags, tp->event->flags | flag); } static inline void trace_probe_clear_flag(struct trace_probe *tp, diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 79f8da7e3cd49..03d79a9ebd8c8 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -730,7 +730,7 @@ static int create_or_delete_trace_uprobe(const char *raw_command) if (raw_command[0] == '-') return dyn_event_release(raw_command, &trace_uprobe_ops); - ret = trace_uprobe_create(raw_command); + ret = dyn_event_create(raw_command, &trace_uprobe_ops); return ret == -ECANCELED ? -EINVAL : ret; } @@ -1514,6 +1514,7 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) struct trace_uprobe *tu; struct uprobe_dispatch_data udd; struct uprobe_cpu_buffer *ucb = NULL; + unsigned int flags; int ret = 0; tu = container_of(con, struct trace_uprobe, consumer); @@ -1527,11 +1528,12 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) if (WARN_ON_ONCE(!uprobe_cpu_buffer)) return 0; - if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) + flags = trace_probe_load_flag(&tu->tp); + if (flags & TP_FLAG_TRACE) ret |= uprobe_trace_func(tu, regs, &ucb); #ifdef CONFIG_PERF_EVENTS - if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) + if (flags & TP_FLAG_PROFILE) ret |= uprobe_perf_func(tu, regs, &ucb); #endif uprobe_buffer_put(ucb); @@ -1544,6 +1546,7 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, struct trace_uprobe *tu; struct uprobe_dispatch_data udd; struct uprobe_cpu_buffer *ucb = NULL; + unsigned int flags; tu = container_of(con, struct trace_uprobe, consumer); @@ -1555,11 +1558,12 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, if (WARN_ON_ONCE(!uprobe_cpu_buffer)) return 0; - if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) + flags = trace_probe_load_flag(&tu->tp); + if (flags & TP_FLAG_TRACE) uretprobe_trace_func(tu, func, regs, &ucb); #ifdef CONFIG_PERF_EVENTS - if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) + if (flags & TP_FLAG_PROFILE) uretprobe_perf_func(tu, func, regs, &ucb); #endif uprobe_buffer_put(ucb); diff --git a/kernel/ucount.c b/kernel/ucount.c index 3456018730b6c..a7fd89693bd2a 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -213,7 +213,7 @@ void put_ucounts(struct ucounts *ucounts) } } -static inline bool atomic_long_inc_below(atomic_long_t *v, int u) +static inline bool atomic_long_inc_below(atomic_long_t *v, long u) { long c, old; c = atomic_long_read(v); diff --git a/kernel/vhost_task.c b/kernel/vhost_task.c index da35e5b7f0473..c1bb2fc3cbe6b 100644 --- a/kernel/vhost_task.c +++ b/kernel/vhost_task.c @@ -89,6 +89,7 @@ void vhost_task_stop(struct vhost_task *vtsk) * freeing it below. */ wait_for_completion(&vtsk->exited); + put_task_struct(vtsk->task); kfree(vtsk); } EXPORT_SYMBOL_GPL(vhost_task_stop); @@ -133,7 +134,7 @@ struct vhost_task *vhost_task_create(bool (*fn)(void *), void *arg, return NULL; } - vtsk->task = tsk; + vtsk->task = get_task_struct(tsk); return vtsk; } EXPORT_SYMBOL_GPL(vhost_task_create); diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 71b4060858fe8..a4117b3848caf 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -46,6 +46,7 @@ int __read_mostly watchdog_user_enabled = 1; static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT; static int __read_mostly watchdog_softlockup_user_enabled = 1; int __read_mostly watchdog_thresh = 10; +static int __read_mostly watchdog_thresh_next; static int __read_mostly watchdog_hardlockup_available; struct cpumask watchdog_cpumask __read_mostly; @@ -727,12 +728,20 @@ int lockup_detector_offline_cpu(unsigned int cpu) return 0; } -static void __lockup_detector_reconfigure(void) +static void __lockup_detector_reconfigure(bool thresh_changed) { cpus_read_lock(); watchdog_hardlockup_stop(); softlockup_stop_all(); + /* + * To prevent watchdog_timer_fn from using the old interval and + * the new watchdog_thresh at the same time, which could lead to + * false softlockup reports, it is necessary to update the + * watchdog_thresh after the softlockup is completed. + */ + if (thresh_changed) + watchdog_thresh = READ_ONCE(watchdog_thresh_next); set_sample_period(); lockup_detector_update_enable(); if (watchdog_enabled && watchdog_thresh) @@ -750,7 +759,7 @@ static void __lockup_detector_reconfigure(void) void lockup_detector_reconfigure(void) { mutex_lock(&watchdog_mutex); - __lockup_detector_reconfigure(); + __lockup_detector_reconfigure(false); mutex_unlock(&watchdog_mutex); } @@ -770,27 +779,29 @@ static __init void lockup_detector_setup(void) return; mutex_lock(&watchdog_mutex); - __lockup_detector_reconfigure(); + __lockup_detector_reconfigure(false); softlockup_initialized = true; mutex_unlock(&watchdog_mutex); } #else /* CONFIG_SOFTLOCKUP_DETECTOR */ -static void __lockup_detector_reconfigure(void) +static void __lockup_detector_reconfigure(bool thresh_changed) { cpus_read_lock(); watchdog_hardlockup_stop(); + if (thresh_changed) + watchdog_thresh = READ_ONCE(watchdog_thresh_next); lockup_detector_update_enable(); watchdog_hardlockup_start(); cpus_read_unlock(); } void lockup_detector_reconfigure(void) { - __lockup_detector_reconfigure(); + __lockup_detector_reconfigure(false); } static inline void lockup_detector_setup(void) { - __lockup_detector_reconfigure(); + __lockup_detector_reconfigure(false); } #endif /* !CONFIG_SOFTLOCKUP_DETECTOR */ @@ -826,11 +837,11 @@ void lockup_detector_soft_poweroff(void) #ifdef CONFIG_SYSCTL /* Propagate any changes to the watchdog infrastructure */ -static void proc_watchdog_update(void) +static void proc_watchdog_update(bool thresh_changed) { /* Remove impossible cpus to keep sysctl output clean. */ cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask); - __lockup_detector_reconfigure(); + __lockup_detector_reconfigure(thresh_changed); } /* @@ -863,7 +874,7 @@ static int proc_watchdog_common(int which, struct ctl_table *table, int write, old = READ_ONCE(*param); err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); if (!err && old != READ_ONCE(*param)) - proc_watchdog_update(); + proc_watchdog_update(false); } mutex_unlock(&watchdog_mutex); return err; @@ -912,11 +923,13 @@ int proc_watchdog_thresh(struct ctl_table *table, int write, mutex_lock(&watchdog_mutex); - old = READ_ONCE(watchdog_thresh); + watchdog_thresh_next = READ_ONCE(watchdog_thresh); + + old = watchdog_thresh_next; err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); - if (!err && write && old != READ_ONCE(watchdog_thresh)) - proc_watchdog_update(); + if (!err && write && old != READ_ONCE(watchdog_thresh_next)) + proc_watchdog_update(true); mutex_unlock(&watchdog_mutex); return err; @@ -937,7 +950,7 @@ int proc_watchdog_cpumask(struct ctl_table *table, int write, err = proc_do_large_bitmap(table, write, buffer, lenp, ppos); if (!err && write) - proc_watchdog_update(); + proc_watchdog_update(false); mutex_unlock(&watchdog_mutex); return err; @@ -957,7 +970,7 @@ static struct ctl_table watchdog_sysctls[] = { }, { .procname = "watchdog_thresh", - .data = &watchdog_thresh, + .data = &watchdog_thresh_next, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_watchdog_thresh, diff --git a/lib/Kconfig b/lib/Kconfig index ee365b7402f19..43d69669465af 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -751,6 +751,7 @@ config GENERIC_LIB_DEVMEM_IS_ALLOWED config PLDMFW bool + select CRC32 default n config ASN1_ENCODER diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8120c444784bf..8edbcc49bf34d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2767,6 +2767,15 @@ config FORTIFY_KUNIT_TEST by the str*() and mem*() family of functions. For testing runtime traps of FORTIFY_SOURCE, see LKDTM's "FORTIFY_*" tests. +config LONGEST_SYM_KUNIT_TEST + tristate "Test the longest symbol possible" if !KUNIT_ALL_TESTS + depends on KUNIT && KPROBES + default KUNIT_ALL_TESTS + help + Tests the longest symbol possible + + If unsure, say N. + config HW_BREAKPOINT_KUNIT_TEST bool "Test hw_breakpoint constraints accounting" if !KUNIT_ALL_TESTS depends on HAVE_HW_BREAKPOINT diff --git a/lib/Makefile b/lib/Makefile index 740109b6e2c89..b9d2577fbbe19 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -402,6 +402,8 @@ obj-$(CONFIG_FORTIFY_KUNIT_TEST) += fortify_kunit.o obj-$(CONFIG_STRCAT_KUNIT_TEST) += strcat_kunit.o obj-$(CONFIG_STRSCPY_KUNIT_TEST) += strscpy_kunit.o obj-$(CONFIG_SIPHASH_KUNIT_TEST) += siphash_kunit.o +obj-$(CONFIG_LONGEST_SYM_KUNIT_TEST) += longest_symbol_kunit.o +CFLAGS_longest_symbol_kunit.o += $(call cc-disable-warning, missing-prototypes) obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o diff --git a/lib/btree.c b/lib/btree.c index 49420cae3a833..bb81d3393ac5c 100644 --- a/lib/btree.c +++ b/lib/btree.c @@ -43,7 +43,6 @@ #include #include -#define MAX(a, b) ((a) > (b) ? (a) : (b)) #define NODESIZE MAX(L1_CACHE_BYTES, 128) struct btree_geo { diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 8d1446c2be719..b1b81998e101e 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -30,6 +30,10 @@ obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519-generic.o libcurve25519-generic-y := curve25519-fiat32.o libcurve25519-generic-$(CONFIG_ARCH_SUPPORTS_INT128) := curve25519-hacl64.o libcurve25519-generic-y += curve25519-generic.o +# clang versions prior to 18 may blow out the stack with KASAN +ifeq ($(CONFIG_CC_IS_CLANG)_$(call clang-min-version, 180000),y_) +KASAN_SANITIZE_curve25519-hacl64.o := n +endif obj-$(CONFIG_CRYPTO_LIB_CURVE25519) += libcurve25519.o libcurve25519-y += curve25519.o diff --git a/lib/decompress_unlzma.c b/lib/decompress_unlzma.c index 20a858031f12b..9d34d35908daa 100644 --- a/lib/decompress_unlzma.c +++ b/lib/decompress_unlzma.c @@ -37,7 +37,9 @@ #include +#ifndef MIN #define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif static long long INIT read_int(unsigned char *ptr, int size) { diff --git a/lib/dynamic_queue_limits.c b/lib/dynamic_queue_limits.c index fde0aa2441480..a75a9ca46b594 100644 --- a/lib/dynamic_queue_limits.c +++ b/lib/dynamic_queue_limits.c @@ -116,7 +116,7 @@ EXPORT_SYMBOL(dql_completed); void dql_reset(struct dql *dql) { /* Reset all dynamic values */ - dql->limit = 0; + dql->limit = dql->min_limit; dql->num_queued = 0; dql->num_completed = 0; dql->last_obj_cnt = 0; diff --git a/lib/genalloc.c b/lib/genalloc.c index 4fa5635bf81bd..841f297838333 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -899,8 +899,11 @@ struct gen_pool *of_gen_pool_get(struct device_node *np, if (!name) name = of_node_full_name(np_pool); } - if (pdev) + if (pdev) { pool = gen_pool_get(&pdev->dev, name); + put_device(&pdev->dev); + } + of_node_put(np_pool); return pool; diff --git a/lib/group_cpus.c b/lib/group_cpus.c index ee272c4cefcc1..18d43a406114b 100644 --- a/lib/group_cpus.c +++ b/lib/group_cpus.c @@ -352,6 +352,9 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps) int ret = -ENOMEM; struct cpumask *masks = NULL; + if (numgrps == 0) + return NULL; + if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) return NULL; @@ -426,8 +429,12 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps) #else /* CONFIG_SMP */ struct cpumask *group_cpus_evenly(unsigned int numgrps) { - struct cpumask *masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL); + struct cpumask *masks; + if (numgrps == 0) + return NULL; + + masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL); if (!masks) return NULL; diff --git a/lib/kunit/static_stub.c b/lib/kunit/static_stub.c index 92b2cccd5e763..484fd85251b41 100644 --- a/lib/kunit/static_stub.c +++ b/lib/kunit/static_stub.c @@ -96,7 +96,7 @@ void __kunit_activate_static_stub(struct kunit *test, /* If the replacement address is NULL, deactivate the stub. */ if (!replacement_addr) { - kunit_deactivate_static_stub(test, replacement_addr); + kunit_deactivate_static_stub(test, real_fn_addr); return; } diff --git a/lib/longest_symbol_kunit.c b/lib/longest_symbol_kunit.c new file mode 100644 index 0000000000000..2fea82a6d34e5 --- /dev/null +++ b/lib/longest_symbol_kunit.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test the longest symbol length. Execute with: + * ./tools/testing/kunit/kunit.py run longest-symbol + * --arch=x86_64 --kconfig_add CONFIG_KPROBES=y --kconfig_add CONFIG_MODULES=y + * --kconfig_add CONFIG_RETPOLINE=n --kconfig_add CONFIG_CFI_CLANG=n + * --kconfig_add CONFIG_MITIGATION_RETPOLINE=n + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#define DI(name) s##name##name +#define DDI(name) DI(n##name##name) +#define DDDI(name) DDI(n##name##name) +#define DDDDI(name) DDDI(n##name##name) +#define DDDDDI(name) DDDDI(n##name##name) + +/*Generate a symbol whose name length is 511 */ +#define LONGEST_SYM_NAME DDDDDI(g1h2i3j4k5l6m7n) + +#define RETURN_LONGEST_SYM 0xAAAAA + +noinline int LONGEST_SYM_NAME(void); +noinline int LONGEST_SYM_NAME(void) +{ + return RETURN_LONGEST_SYM; +} + +_Static_assert(sizeof(__stringify(LONGEST_SYM_NAME)) == KSYM_NAME_LEN, +"Incorrect symbol length found. Expected KSYM_NAME_LEN: " +__stringify(KSYM_NAME_LEN) ", but found: " +__stringify(sizeof(LONGEST_SYM_NAME))); + +static void test_longest_symbol(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, RETURN_LONGEST_SYM, LONGEST_SYM_NAME()); +}; + +static void test_longest_symbol_kallsyms(struct kunit *test) +{ + unsigned long (*kallsyms_lookup_name)(const char *name); + static int (*longest_sym)(void); + + struct kprobe kp = { + .symbol_name = "kallsyms_lookup_name", + }; + + if (register_kprobe(&kp) < 0) { + pr_info("%s: kprobe not registered", __func__); + KUNIT_FAIL(test, "test_longest_symbol kallsyms: kprobe not registered\n"); + return; + } + + kunit_warn(test, "test_longest_symbol kallsyms: kprobe registered\n"); + kallsyms_lookup_name = (unsigned long (*)(const char *name))kp.addr; + unregister_kprobe(&kp); + + longest_sym = + (void *) kallsyms_lookup_name(__stringify(LONGEST_SYM_NAME)); + KUNIT_EXPECT_EQ(test, RETURN_LONGEST_SYM, longest_sym()); +}; + +static struct kunit_case longest_symbol_test_cases[] = { + KUNIT_CASE(test_longest_symbol), + KUNIT_CASE(test_longest_symbol_kallsyms), + {} +}; + +static struct kunit_suite longest_symbol_test_suite = { + .name = "longest-symbol", + .test_cases = longest_symbol_test_cases, +}; +kunit_test_suite(longest_symbol_test_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Test the longest symbol length"); +MODULE_AUTHOR("Sergio González Collado"); diff --git a/lib/lz4/lz4hc_compress.c b/lib/lz4/lz4hc_compress.c index e7ac8694b797d..bc45594ad2a80 100644 --- a/lib/lz4/lz4hc_compress.c +++ b/lib/lz4/lz4hc_compress.c @@ -621,6 +621,7 @@ void LZ4_resetStreamHC(LZ4_streamHC_t *LZ4_streamHCPtr, int compressionLevel) LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned int)compressionLevel; } +EXPORT_SYMBOL(LZ4_resetStreamHC); int LZ4_loadDictHC(LZ4_streamHC_t *LZ4_streamHCPtr, const char *dictionary, diff --git a/lib/lzo/Makefile b/lib/lzo/Makefile index 2f58fafbbdddc..fc7b2b7ef4b20 100644 --- a/lib/lzo/Makefile +++ b/lib/lzo/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -lzo_compress-objs := lzo1x_compress.o +lzo_compress-objs := lzo1x_compress.o lzo1x_compress_safe.o lzo_decompress-objs := lzo1x_decompress_safe.o obj-$(CONFIG_LZO_COMPRESS) += lzo_compress.o diff --git a/lib/lzo/lzo1x_compress.c b/lib/lzo/lzo1x_compress.c index 9d31e7126606a..f00dff9b9d4e1 100644 --- a/lib/lzo/lzo1x_compress.c +++ b/lib/lzo/lzo1x_compress.c @@ -18,11 +18,22 @@ #include #include "lzodefs.h" -static noinline size_t -lzo1x_1_do_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - size_t ti, void *wrkmem, signed char *state_offset, - const unsigned char bitstream_version) +#undef LZO_UNSAFE + +#ifndef LZO_SAFE +#define LZO_UNSAFE 1 +#define LZO_SAFE(name) name +#define HAVE_OP(x) 1 +#endif + +#define NEED_OP(x) if (!HAVE_OP(x)) goto output_overrun + +static noinline int +LZO_SAFE(lzo1x_1_do_compress)(const unsigned char *in, size_t in_len, + unsigned char **out, unsigned char *op_end, + size_t *tp, void *wrkmem, + signed char *state_offset, + const unsigned char bitstream_version) { const unsigned char *ip; unsigned char *op; @@ -30,8 +41,9 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, const unsigned char * const ip_end = in + in_len - 20; const unsigned char *ii; lzo_dict_t * const dict = (lzo_dict_t *) wrkmem; + size_t ti = *tp; - op = out; + op = *out; ip = in; ii = ip; ip += ti < 4 ? 4 - ti : 0; @@ -116,25 +128,32 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, if (t != 0) { if (t <= 3) { op[*state_offset] |= t; + NEED_OP(4); COPY4(op, ii); op += t; } else if (t <= 16) { + NEED_OP(17); *op++ = (t - 3); COPY8(op, ii); COPY8(op + 8, ii + 8); op += t; } else { if (t <= 18) { + NEED_OP(1); *op++ = (t - 3); } else { size_t tt = t - 18; + NEED_OP(1); *op++ = 0; while (unlikely(tt > 255)) { tt -= 255; + NEED_OP(1); *op++ = 0; } + NEED_OP(1); *op++ = tt; } + NEED_OP(t); do { COPY8(op, ii); COPY8(op + 8, ii + 8); @@ -151,6 +170,7 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, if (unlikely(run_length)) { ip += run_length; run_length -= MIN_ZERO_RUN_LENGTH; + NEED_OP(4); put_unaligned_le32((run_length << 21) | 0xfffc18 | (run_length & 0x7), op); op += 4; @@ -243,10 +263,12 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, ip += m_len; if (m_len <= M2_MAX_LEN && m_off <= M2_MAX_OFFSET) { m_off -= 1; + NEED_OP(2); *op++ = (((m_len - 1) << 5) | ((m_off & 7) << 2)); *op++ = (m_off >> 3); } else if (m_off <= M3_MAX_OFFSET) { m_off -= 1; + NEED_OP(1); if (m_len <= M3_MAX_LEN) *op++ = (M3_MARKER | (m_len - 2)); else { @@ -254,14 +276,18 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, *op++ = M3_MARKER | 0; while (unlikely(m_len > 255)) { m_len -= 255; + NEED_OP(1); *op++ = 0; } + NEED_OP(1); *op++ = (m_len); } + NEED_OP(2); *op++ = (m_off << 2); *op++ = (m_off >> 6); } else { m_off -= 0x4000; + NEED_OP(1); if (m_len <= M4_MAX_LEN) *op++ = (M4_MARKER | ((m_off >> 11) & 8) | (m_len - 2)); @@ -282,11 +308,14 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, m_len -= M4_MAX_LEN; *op++ = (M4_MARKER | ((m_off >> 11) & 8)); while (unlikely(m_len > 255)) { + NEED_OP(1); m_len -= 255; *op++ = 0; } + NEED_OP(1); *op++ = (m_len); } + NEED_OP(2); *op++ = (m_off << 2); *op++ = (m_off >> 6); } @@ -295,14 +324,20 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, ii = ip; goto next; } - *out_len = op - out; - return in_end - (ii - ti); + *out = op; + *tp = in_end - (ii - ti); + return LZO_E_OK; + +output_overrun: + return LZO_E_OUTPUT_OVERRUN; } -static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - void *wrkmem, const unsigned char bitstream_version) +static int LZO_SAFE(lzogeneric1x_1_compress)( + const unsigned char *in, size_t in_len, + unsigned char *out, size_t *out_len, + void *wrkmem, const unsigned char bitstream_version) { + unsigned char * const op_end = out + *out_len; const unsigned char *ip = in; unsigned char *op = out; unsigned char *data_start; @@ -326,14 +361,18 @@ static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, while (l > 20) { size_t ll = min_t(size_t, l, m4_max_offset + 1); uintptr_t ll_end = (uintptr_t) ip + ll; + int err; + if ((ll_end + ((t + ll) >> 5)) <= ll_end) break; BUILD_BUG_ON(D_SIZE * sizeof(lzo_dict_t) > LZO1X_1_MEM_COMPRESS); memset(wrkmem, 0, D_SIZE * sizeof(lzo_dict_t)); - t = lzo1x_1_do_compress(ip, ll, op, out_len, t, wrkmem, - &state_offset, bitstream_version); + err = LZO_SAFE(lzo1x_1_do_compress)( + ip, ll, &op, op_end, &t, wrkmem, + &state_offset, bitstream_version); + if (err != LZO_E_OK) + return err; ip += ll; - op += *out_len; l -= ll; } t += l; @@ -342,20 +381,26 @@ static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, const unsigned char *ii = in + in_len - t; if (op == data_start && t <= 238) { + NEED_OP(1); *op++ = (17 + t); } else if (t <= 3) { op[state_offset] |= t; } else if (t <= 18) { + NEED_OP(1); *op++ = (t - 3); } else { size_t tt = t - 18; + NEED_OP(1); *op++ = 0; while (tt > 255) { tt -= 255; + NEED_OP(1); *op++ = 0; } + NEED_OP(1); *op++ = tt; } + NEED_OP(t); if (t >= 16) do { COPY8(op, ii); COPY8(op + 8, ii + 8); @@ -368,31 +413,38 @@ static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, } while (--t > 0); } + NEED_OP(3); *op++ = M4_MARKER | 1; *op++ = 0; *op++ = 0; *out_len = op - out; return LZO_E_OK; + +output_overrun: + return LZO_E_OUTPUT_OVERRUN; } -int lzo1x_1_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - void *wrkmem) +int LZO_SAFE(lzo1x_1_compress)(const unsigned char *in, size_t in_len, + unsigned char *out, size_t *out_len, + void *wrkmem) { - return lzogeneric1x_1_compress(in, in_len, out, out_len, wrkmem, 0); + return LZO_SAFE(lzogeneric1x_1_compress)( + in, in_len, out, out_len, wrkmem, 0); } -int lzorle1x_1_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - void *wrkmem) +int LZO_SAFE(lzorle1x_1_compress)(const unsigned char *in, size_t in_len, + unsigned char *out, size_t *out_len, + void *wrkmem) { - return lzogeneric1x_1_compress(in, in_len, out, out_len, - wrkmem, LZO_VERSION); + return LZO_SAFE(lzogeneric1x_1_compress)( + in, in_len, out, out_len, wrkmem, LZO_VERSION); } -EXPORT_SYMBOL_GPL(lzo1x_1_compress); -EXPORT_SYMBOL_GPL(lzorle1x_1_compress); +EXPORT_SYMBOL_GPL(LZO_SAFE(lzo1x_1_compress)); +EXPORT_SYMBOL_GPL(LZO_SAFE(lzorle1x_1_compress)); +#ifndef LZO_UNSAFE MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("LZO1X-1 Compressor"); +#endif diff --git a/lib/lzo/lzo1x_compress_safe.c b/lib/lzo/lzo1x_compress_safe.c new file mode 100644 index 0000000000000..371c9f8494928 --- /dev/null +++ b/lib/lzo/lzo1x_compress_safe.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * LZO1X Compressor from LZO + * + * Copyright (C) 1996-2012 Markus F.X.J. Oberhumer + * + * The full LZO package can be found at: + * http://www.oberhumer.com/opensource/lzo/ + * + * Changed for Linux kernel use by: + * Nitin Gupta + * Richard Purdie + */ + +#define LZO_SAFE(name) name##_safe +#define HAVE_OP(x) ((size_t)(op_end - op) >= (size_t)(x)) + +#include "lzo1x_compress.c" diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 3991813338d15..eb5a0979c0965 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -64,6 +64,8 @@ #define CREATE_TRACE_POINTS #include +#define TP_FCT tracepoint_string(__func__) + #define MA_ROOT_PARENT 1 /* @@ -3002,7 +3004,7 @@ static inline int mas_rebalance(struct ma_state *mas, MA_STATE(l_mas, mas->tree, mas->index, mas->last); MA_STATE(r_mas, mas->tree, mas->index, mas->last); - trace_ma_op(__func__, mas); + trace_ma_op(TP_FCT, mas); /* * Rebalancing occurs if a node is insufficient. Data is rebalanced @@ -3377,7 +3379,7 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); - trace_ma_op(__func__, mas); + trace_ma_op(TP_FCT, mas); mas->depth = mas_mt_height(mas); /* Allocation failures will happen early. */ mas_node_count(mas, 1 + mas->depth * 2); @@ -3610,7 +3612,7 @@ static bool mas_is_span_wr(struct ma_wr_state *wr_mas) return false; } - trace_ma_write(__func__, wr_mas->mas, wr_mas->r_max, entry); + trace_ma_write(TP_FCT, wr_mas->mas, wr_mas->r_max, entry); return true; } @@ -3857,7 +3859,7 @@ static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas) * of data may happen. */ mas = wr_mas->mas; - trace_ma_op(__func__, mas); + trace_ma_op(TP_FCT, mas); if (unlikely(!mas->index && mas->last == ULONG_MAX)) return mas_new_root(mas, wr_mas->entry); @@ -4008,7 +4010,7 @@ static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas, } else { memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); } - trace_ma_write(__func__, mas, 0, wr_mas->entry); + trace_ma_write(TP_FCT, mas, 0, wr_mas->entry); mas_update_gap(mas); return true; } @@ -4054,7 +4056,7 @@ static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) return false; } - trace_ma_write(__func__, mas, 0, wr_mas->entry); + trace_ma_write(TP_FCT, mas, 0, wr_mas->entry); /* * Only update gap when the new entry is empty or there is an empty * entry in the original two ranges. @@ -4190,7 +4192,7 @@ static inline bool mas_wr_append(struct ma_wr_state *wr_mas, if (!wr_mas->content || !wr_mas->entry) mas_update_gap(mas); - trace_ma_write(__func__, mas, new_end, wr_mas->entry); + trace_ma_write(TP_FCT, mas, new_end, wr_mas->entry); return true; } @@ -4204,7 +4206,7 @@ static void mas_wr_bnode(struct ma_wr_state *wr_mas) { struct maple_big_node b_node; - trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry); + trace_ma_write(TP_FCT, wr_mas->mas, 0, wr_mas->entry); memset(&b_node, 0, sizeof(struct maple_big_node)); mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end); mas_commit_b_node(wr_mas, &b_node, wr_mas->node_end); @@ -5282,6 +5284,7 @@ static void mt_destroy_walk(struct maple_enode *enode, struct maple_tree *mt, struct maple_enode *start; if (mte_is_leaf(enode)) { + mte_set_node_dead(enode); node->type = mte_node_type(enode); goto free_leaf; } @@ -5406,7 +5409,7 @@ void *mas_store(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); - trace_ma_write(__func__, mas, 0, entry); + trace_ma_write(TP_FCT, mas, 0, entry); #ifdef CONFIG_DEBUG_MAPLE_TREE if (MAS_WARN_ON(mas, mas->index > mas->last)) pr_err("Error %lX > %lX %p\n", mas->index, mas->last, entry); @@ -5444,7 +5447,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) MA_WR_STATE(wr_mas, mas, entry); mas_wr_store_setup(&wr_mas); - trace_ma_write(__func__, mas, 0, entry); + trace_ma_write(TP_FCT, mas, 0, entry); retry: mas_wr_store_entry(&wr_mas); if (unlikely(mas_nomem(mas, gfp))) @@ -5468,7 +5471,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry) MA_WR_STATE(wr_mas, mas, entry); mas_wr_store_setup(&wr_mas); - trace_ma_write(__func__, mas, 0, entry); + trace_ma_write(TP_FCT, mas, 0, entry); mas_wr_store_entry(&wr_mas); MAS_WR_BUG_ON(&wr_mas, mas_is_err(mas)); mas_destroy(mas); @@ -5509,7 +5512,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) /* At this point, we are at the leaf node that needs to be altered. */ /* Exact fit, no nodes needed. */ if (wr_mas.r_min == mas->index && wr_mas.r_max == mas->last) - return 0; + goto set_flag; mas_wr_end_piv(&wr_mas); node_size = mas_wr_new_end(&wr_mas); @@ -5518,10 +5521,10 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) if (node_size == wr_mas.node_end) { /* reuse node */ if (!mt_in_rcu(mas->tree)) - return 0; + goto set_flag; /* shifting boundary */ if (wr_mas.offset_end - mas->offset == 1) - return 0; + goto set_flag; } if (node_size >= mt_slots[wr_mas.type]) { @@ -5540,10 +5543,13 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) /* node store, slot store needs one node */ ask_now: + mas->mas_flags &= ~MA_STATE_PREALLOC; mas_node_count_gfp(mas, request, gfp); - mas->mas_flags |= MA_STATE_PREALLOC; - if (likely(!mas_is_err(mas))) + if (likely(!mas_is_err(mas))) { +set_flag: + mas->mas_flags |= MA_STATE_PREALLOC; return 0; + } mas_set_alloc_req(mas, 0); ret = xa_err(mas->node); @@ -6253,7 +6259,7 @@ void *mtree_load(struct maple_tree *mt, unsigned long index) MA_STATE(mas, mt, index, index); void *entry; - trace_ma_read(__func__, &mas); + trace_ma_read(TP_FCT, &mas); rcu_read_lock(); retry: entry = mas_start(&mas); @@ -6296,7 +6302,7 @@ int mtree_store_range(struct maple_tree *mt, unsigned long index, MA_STATE(mas, mt, index, last); MA_WR_STATE(wr_mas, &mas, entry); - trace_ma_write(__func__, &mas, 0, entry); + trace_ma_write(TP_FCT, &mas, 0, entry); if (WARN_ON_ONCE(xa_is_advanced(entry))) return -EINVAL; @@ -6478,7 +6484,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index) void *entry = NULL; MA_STATE(mas, mt, index, index); - trace_ma_op(__func__, &mas); + trace_ma_op(TP_FCT, &mas); mtree_lock(mt); entry = mas_erase(&mas); @@ -6816,7 +6822,7 @@ void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) unsigned long copy = *index; #endif - trace_ma_read(__func__, &mas); + trace_ma_read(TP_FCT, &mas); if ((*index) > max) return NULL; diff --git a/lib/nlattr.c b/lib/nlattr.c index ba698a097fc81..0319e811bb10a 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -138,6 +138,7 @@ void nla_get_range_unsigned(const struct nla_policy *pt, range->max = U32_MAX; break; case NLA_U64: + case NLA_UINT: case NLA_MSECS: range->max = U64_MAX; break; @@ -187,6 +188,9 @@ static int nla_validate_range_unsigned(const struct nla_policy *pt, case NLA_U64: value = nla_get_u64(nla); break; + case NLA_UINT: + value = nla_get_uint(nla); + break; case NLA_MSECS: value = nla_get_u64(nla); break; @@ -252,6 +256,7 @@ void nla_get_range_signed(const struct nla_policy *pt, range->max = S32_MAX; break; case NLA_S64: + case NLA_SINT: range->min = S64_MIN; range->max = S64_MAX; break; @@ -299,6 +304,9 @@ static int nla_validate_int_range_signed(const struct nla_policy *pt, case NLA_S64: value = nla_get_s64(nla); break; + case NLA_SINT: + value = nla_get_sint(nla); + break; default: return -EINVAL; } @@ -324,6 +332,7 @@ static int nla_validate_int_range(const struct nla_policy *pt, case NLA_U16: case NLA_U32: case NLA_U64: + case NLA_UINT: case NLA_MSECS: case NLA_BINARY: case NLA_BE16: @@ -333,6 +342,7 @@ static int nla_validate_int_range(const struct nla_policy *pt, case NLA_S16: case NLA_S32: case NLA_S64: + case NLA_SINT: return nla_validate_int_range_signed(pt, nla, extack); default: WARN_ON(1); @@ -359,6 +369,9 @@ static int nla_validate_mask(const struct nla_policy *pt, case NLA_U64: value = nla_get_u64(nla); break; + case NLA_UINT: + value = nla_get_uint(nla); + break; case NLA_BE16: value = ntohs(nla_get_be16(nla)); break; @@ -437,6 +450,15 @@ static int validate_nla(const struct nlattr *nla, int maxtype, goto out_err; break; + case NLA_SINT: + case NLA_UINT: + if (attrlen != sizeof(u32) && attrlen != sizeof(u64)) { + NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt, + "invalid attribute length"); + return -EINVAL; + } + break; + case NLA_BITFIELD32: if (attrlen != sizeof(struct nla_bitfield32)) goto out_err; diff --git a/lib/plist.c b/lib/plist.c index 0d86ed7a76ac6..7587068629151 100644 --- a/lib/plist.c +++ b/lib/plist.c @@ -102,6 +102,7 @@ void plist_add(struct plist_node *node, struct plist_head *head) plist_check_head(head); } +EXPORT_SYMBOL_GPL(plist_add); /** * plist_del - Remove a @node from plist. @@ -131,6 +132,7 @@ void plist_del(struct plist_node *node, struct plist_head *head) plist_check_head(head); } +EXPORT_SYMBOL_GPL(plist_del); /** * plist_requeue - Requeue @node at end of same-prio entries. diff --git a/lib/test_objagg.c b/lib/test_objagg.c index c0c957c506354..c0f7bb53db8d5 100644 --- a/lib/test_objagg.c +++ b/lib/test_objagg.c @@ -899,8 +899,10 @@ static int check_expect_hints_stats(struct objagg_hints *objagg_hints, int err; stats = objagg_hints_stats_get(objagg_hints); - if (IS_ERR(stats)) + if (IS_ERR(stats)) { + *errmsg = "objagg_hints_stats_get() failed."; return PTR_ERR(stats); + } err = __check_expect_stats(stats, expect_stats, errmsg); objagg_stats_put(stats); return err; diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 2aa408441cd3e..f4ab2750cfc12 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -1079,7 +1079,7 @@ char *resource_string(char *buf, char *end, struct resource *res, #define FLAG_BUF_SIZE (2 * sizeof(res->flags)) #define DECODED_BUF_SIZE sizeof("[mem - 64bit pref window disabled]") #define RAW_BUF_SIZE sizeof("[mem - flags 0x]") - char sym[max(2*RSRC_BUF_SIZE + DECODED_BUF_SIZE, + char sym[MAX(2*RSRC_BUF_SIZE + DECODED_BUF_SIZE, 2*RSRC_BUF_SIZE + FLAG_BUF_SIZE + RAW_BUF_SIZE)]; char *p = sym, *pend = sym + sizeof(sym); diff --git a/lib/zstd/compress/zstd_compress.c b/lib/zstd/compress/zstd_compress.c index f620cafca633b..16bb995bc6c4c 100644 --- a/lib/zstd/compress/zstd_compress.c +++ b/lib/zstd/compress/zstd_compress.c @@ -4810,6 +4810,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced2( dictLoadMethod, cctxParams.cParams, cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, customMem); + if (!cdict) + return NULL; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, diff --git a/lib/zstd/zstd_compress_module.c b/lib/zstd/zstd_compress_module.c index 04e1b5c01d9b6..bd8784449b319 100644 --- a/lib/zstd/zstd_compress_module.c +++ b/lib/zstd/zstd_compress_module.c @@ -66,6 +66,12 @@ int zstd_max_clevel(void) } EXPORT_SYMBOL(zstd_max_clevel); +int zstd_default_clevel(void) +{ + return ZSTD_defaultCLevel(); +} +EXPORT_SYMBOL(zstd_default_clevel); + size_t zstd_compress_bound(size_t src_size) { return ZSTD_compressBound(src_size); @@ -79,6 +85,13 @@ zstd_parameters zstd_get_params(int level, } EXPORT_SYMBOL(zstd_get_params); +zstd_compression_parameters zstd_get_cparams(int level, + unsigned long long estimated_src_size, size_t dict_size) +{ + return ZSTD_getCParams(level, estimated_src_size, dict_size); +} +EXPORT_SYMBOL(zstd_get_cparams); + size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams) { return ZSTD_estimateCCtxSize_usingCParams(*cparams); @@ -93,6 +106,33 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size) } EXPORT_SYMBOL(zstd_init_cctx); +zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem) +{ + return ZSTD_createCCtx_advanced(custom_mem); +} +EXPORT_SYMBOL(zstd_create_cctx_advanced); + +size_t zstd_free_cctx(zstd_cctx *cctx) +{ + return ZSTD_freeCCtx(cctx); +} +EXPORT_SYMBOL(zstd_free_cctx); + +zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size, + zstd_compression_parameters cparams, + zstd_custom_mem custom_mem) +{ + return ZSTD_createCDict_advanced(dict, dict_size, ZSTD_dlm_byRef, + ZSTD_dct_auto, cparams, custom_mem); +} +EXPORT_SYMBOL(zstd_create_cdict_byreference); + +size_t zstd_free_cdict(zstd_cdict *cdict) +{ + return ZSTD_freeCDict(cdict); +} +EXPORT_SYMBOL(zstd_free_cdict); + size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity, const void *src, size_t src_size, const zstd_parameters *parameters) { @@ -101,6 +141,15 @@ size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity, } EXPORT_SYMBOL(zstd_compress_cctx); +size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst, + size_t dst_capacity, const void *src, size_t src_size, + const ZSTD_CDict *cdict) +{ + return ZSTD_compress_usingCDict(cctx, dst, dst_capacity, + src, src_size, cdict); +} +EXPORT_SYMBOL(zstd_compress_using_cdict); + size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams) { return ZSTD_estimateCStreamSize_usingCParams(*cparams); diff --git a/lib/zstd/zstd_decompress_module.c b/lib/zstd/zstd_decompress_module.c index f4ed952ed4852..469fc3059be09 100644 --- a/lib/zstd/zstd_decompress_module.c +++ b/lib/zstd/zstd_decompress_module.c @@ -44,6 +44,33 @@ size_t zstd_dctx_workspace_bound(void) } EXPORT_SYMBOL(zstd_dctx_workspace_bound); +zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem) +{ + return ZSTD_createDCtx_advanced(custom_mem); +} +EXPORT_SYMBOL(zstd_create_dctx_advanced); + +size_t zstd_free_dctx(zstd_dctx *dctx) +{ + return ZSTD_freeDCtx(dctx); +} +EXPORT_SYMBOL(zstd_free_dctx); + +zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size, + zstd_custom_mem custom_mem) +{ + return ZSTD_createDDict_advanced(dict, dict_size, ZSTD_dlm_byRef, + ZSTD_dct_auto, custom_mem); + +} +EXPORT_SYMBOL(zstd_create_ddict_byreference); + +size_t zstd_free_ddict(zstd_ddict *ddict) +{ + return ZSTD_freeDDict(ddict); +} +EXPORT_SYMBOL(zstd_free_ddict); + zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size) { if (workspace == NULL) @@ -59,6 +86,15 @@ size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity, } EXPORT_SYMBOL(zstd_decompress_dctx); +size_t zstd_decompress_using_ddict(zstd_dctx *dctx, + void *dst, size_t dst_capacity, const void* src, size_t src_size, + const zstd_ddict* ddict) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dst_capacity, src, + src_size, ddict); +} +EXPORT_SYMBOL(zstd_decompress_using_ddict); + size_t zstd_dstream_workspace_bound(size_t max_window_size) { return ZSTD_estimateDStreamSize(max_window_size); diff --git a/mm/Kconfig b/mm/Kconfig index 1d88bab99ff1f..80808301ecc71 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -280,7 +280,7 @@ config SLAB config SLUB_TINY bool "Configure SLUB for minimal memory footprint" - depends on SLUB && EXPERT + depends on SLUB && EXPERT && !COMPILE_TEST select SLAB_MERGE_DEFAULT help Configures the SLUB allocator in a way to achieve minimal memory @@ -994,6 +994,40 @@ config CMA_AREAS If unsure, leave the default value "7" in UMA and "19" in NUMA. +# +# Select this config option from the architecture Kconfig, if available, to set +# the max page order for physically contiguous allocations. +# +config ARCH_FORCE_MAX_ORDER + int + +# +# When ARCH_FORCE_MAX_ORDER is not defined, +# the default page block order is MAX_PAGE_ORDER (10) as per +# include/linux/mmzone.h. +# +config PAGE_BLOCK_ORDER + int "Page Block Order" + range 1 10 if ARCH_FORCE_MAX_ORDER = 0 || ARCH_FORCE_MAX_ORDER = "" + default 10 if ARCH_FORCE_MAX_ORDER = 0 || ARCH_FORCE_MAX_ORDER = "" + range 1 ARCH_FORCE_MAX_ORDER if ARCH_FORCE_MAX_ORDER != 0 + default ARCH_FORCE_MAX_ORDER if ARCH_FORCE_MAX_ORDER != 0 + help + The page block order refers to the power of two number of pages that + are physically contiguous and can have a migrate type associated to + them. The maximum size of the page block order is limited by + ARCH_FORCE_MAX_ORDER. + + This config allows overriding the default page block order when the + page block order is required to be smaller than ARCH_FORCE_MAX_ORDER + or MAX_ORDER. + + Reducing pageblock order can negatively impact THP generation + success rate. If your workloads uses THP heavily, please use this + option with caution. + + Don't change if unsure. + config MEM_SOFT_DIRTY bool "Track memory changes" depends on CHECKPOINT_RESTORE && HAVE_ARCH_SOFT_DIRTY && PROC_FS diff --git a/mm/cma.c b/mm/cma.c index 56c1b0fdd5577..88d3bc28d962c 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -485,7 +485,6 @@ struct page *__cma_alloc(struct cma *cma, unsigned long count, goto out; trace_android_vh_cma_alloc_set_max_retries(&max_retries); - trace_cma_alloc_start(cma->name, count, align); mask = cma_bitmap_aligned_mask(cma, align); offset = cma_bitmap_aligned_offset(cma, align); @@ -495,6 +494,7 @@ struct page *__cma_alloc(struct cma *cma, unsigned long count, if (bitmap_count > bitmap_maxno) goto out; + trace_cma_alloc_start(cma->name, count, align); trace_android_vh_cma_alloc_retry(cma->name, &max_retries); for (;;) { spin_lock_irq(&cma->lock); @@ -569,7 +569,6 @@ struct page *__cma_alloc(struct cma *cma, unsigned long count, } if (ret && !(gfp_mask & __GFP_NOWARN)) { - trace_android_vh_cma_alloc_fail(cma->name, cma->count, count); pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n", __func__, cma->name, count, ret); cma_debug_show_areas(cma); diff --git a/mm/damon/core.c b/mm/damon/core.c index 98ae790dd04ef..f230cb56621da 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1076,6 +1076,10 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) if (!quota->ms && !quota->sz) return; + /* First charge window */ + if (!quota->total_charged_sz && !quota->charged_from) + quota->charged_from = jiffies; + /* New charge window starts */ if (time_after_eq(jiffies, quota->charged_from + msecs_to_jiffies(quota->reset_interval))) { diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 3de2916a65c38..b4032538b22cf 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -203,6 +203,9 @@ static int damon_lru_sort_apply_parameters(void) unsigned int hot_thres, cold_thres; int err = 0; + if (!damon_lru_sort_mon_attrs.sample_interval) + return -EINVAL; + err = damon_set_attrs(ctx, &damon_lru_sort_mon_attrs); if (err) return err; diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5e6dc312072cd..df09a01131a90 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -38,7 +38,6 @@ static void damon_pa_mkold(unsigned long paddr) .rmap_one = __damon_pa_mkold, .anon_lock = folio_lock_anon_vma_read, }; - bool need_lock; if (!folio) return; @@ -48,14 +47,11 @@ static void damon_pa_mkold(unsigned long paddr) goto out; } - need_lock = !folio_test_anon(folio) || folio_test_ksm(folio); - if (need_lock && !folio_trylock(folio)) + if (!folio_trylock(folio)) goto out; rmap_walk(folio, &rwc); - - if (need_lock) - folio_unlock(folio); + folio_unlock(folio); out: folio_put(folio); @@ -120,7 +116,6 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz) .rmap_one = __damon_pa_young, .anon_lock = folio_lock_anon_vma_read, }; - bool need_lock; if (!folio) return false; @@ -133,14 +128,11 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz) goto out; } - need_lock = !folio_test_anon(folio) || folio_test_ksm(folio); - if (need_lock && !folio_trylock(folio)) + if (!folio_trylock(folio)) goto out; rmap_walk(folio, &rwc); - - if (need_lock) - folio_unlock(folio); + folio_unlock(folio); out: *folio_sz = folio_size(folio); diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 66e190f0374ac..586daa2cefe4f 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -167,6 +167,9 @@ static int damon_reclaim_apply_parameters(void) struct damos_filter *filter; int err = 0; + if (!damon_reclaim_mon_attrs.aggr_interval) + return -EINVAL; + err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); if (err) return err; diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 604b0f93d53fe..f998146f748be 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -421,6 +421,7 @@ static ssize_t memcg_path_store(struct kobject *kobj, return -ENOMEM; strscpy(path, buf, count + 1); + kfree(filter->memcg_path); filter->memcg_path = path; return count; } diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 485559c9904fd..f52a8a1485d68 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1055,14 +1055,18 @@ static ssize_t state_show(struct kobject *kobj, struct kobj_attribute *attr, { struct damon_sysfs_kdamond *kdamond = container_of(kobj, struct damon_sysfs_kdamond, kobj); - struct damon_ctx *ctx = kdamond->damon_ctx; - bool running; + struct damon_ctx *ctx; + bool running = false; - if (!ctx) - running = false; - else + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; + + ctx = kdamond->damon_ctx; + if (ctx) running = damon_sysfs_ctx_running(ctx); + mutex_unlock(&damon_sysfs_lock); + return sysfs_emit(buf, "%s\n", running ? damon_sysfs_cmd_strs[DAMON_SYSFS_CMD_ON] : damon_sysfs_cmd_strs[DAMON_SYSFS_CMD_OFF]); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index b61919f646f69..96f7e378fde60 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -324,10 +324,8 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, } pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); - if (!pte) { - walk->action = ACTION_AGAIN; + if (!pte) return 0; - } if (!pte_present(ptep_get(pte))) goto out; damon_ptep_mkold(pte, walk->vma, addr); @@ -479,10 +477,8 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); - if (!pte) { - walk->action = ACTION_AGAIN; + if (!pte) return 0; - } ptent = ptep_get(pte); if (!pte_present(ptent)) goto out; diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 68af76ca8bc99..0a5454fae3169 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -1047,29 +1047,34 @@ static void __init destroy_args(struct pgtable_debug_args *args) /* Free page table entries */ if (args->start_ptep) { + pmd_clear(args->pmdp); pte_free(args->mm, args->start_ptep); mm_dec_nr_ptes(args->mm); } if (args->start_pmdp) { + pud_clear(args->pudp); pmd_free(args->mm, args->start_pmdp); mm_dec_nr_pmds(args->mm); } if (args->start_pudp) { + p4d_clear(args->p4dp); pud_free(args->mm, args->start_pudp); mm_dec_nr_puds(args->mm); } - if (args->start_p4dp) + if (args->start_p4dp) { + pgd_clear(args->pgdp); p4d_free(args->mm, args->start_p4dp); + } /* Free vma and mm struct */ if (args->vma) vm_area_free(args->vma); if (args->mm) - mmdrop(args->mm); + mmput(args->mm); } static struct page * __init diff --git a/mm/filemap.c b/mm/filemap.c index 0c22dbd701f9b..3f16c4889c8c6 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2057,6 +2057,7 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, if (!folio) return ERR_PTR(-ENOENT); + trace_android_vh_filemap_get_folio_end(mapping, folio); return folio; } EXPORT_SYMBOL(__filemap_get_folio); @@ -2819,6 +2820,8 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, break; } } + trace_android_vh_filemap_read_end(inode, fbatch.folios, + folio_batch_count(&fbatch)); put_folios: for (i = 0; i < folio_batch_count(&fbatch); i++) folio_put(fbatch.folios[i]); @@ -3495,6 +3498,7 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) } vmf->page = folio_file_page(folio, index); + trace_android_vh_filemap_fault_folio_locked(inode, folio, index); return ret | VM_FAULT_LOCKED; page_not_uptodate: @@ -3695,7 +3699,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, struct vm_area_struct *vma = vmf->vma; struct file *file = vma->vm_file; struct address_space *mapping = file->f_mapping; - pgoff_t last_pgoff = start_pgoff; + pgoff_t file_end, last_pgoff = start_pgoff; unsigned long addr; XA_STATE(xas, &mapping->i_pages, start_pgoff); struct folio *folio; @@ -3703,6 +3707,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned int nr_pages = 0, mmap_miss = 0, mmap_miss_saved; pgoff_t first_pgoff = 0; pgoff_t orig_start_pgoff = start_pgoff; + bool can_map_large; rcu_read_lock(); folio = next_uptodate_folio(&xas, mapping, end_pgoff); @@ -3711,7 +3716,20 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, first_pgoff = xas.xa_index; orig_start_pgoff = xas.xa_index; - if (filemap_map_pmd(vmf, folio, start_pgoff)) { + file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE) - 1; + end_pgoff = min(end_pgoff, file_end); + + /* + * Do not allow to map with PTEs beyond i_size and with PMD + * across i_size to preserve SIGBUS semantics. + * + * Make an exception for shmem/tmpfs that for long time + * intentionally mapped with PMDs across i_size. + */ + can_map_large = shmem_mapping(mapping) || + file_end >= folio_next_index(folio); + + if (can_map_large && filemap_map_pmd(vmf, folio, start_pgoff)) { ret = VM_FAULT_NOPAGE; goto out; } @@ -3723,6 +3741,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, folio_put(folio); goto out; } + do { unsigned long end; @@ -3811,7 +3830,7 @@ int generic_file_mmap(struct file *file, struct vm_area_struct *vma) */ int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) { - if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + if (vma_is_shared_maywrite(vma)) return -EINVAL; return generic_file_mmap(file, vma); } diff --git a/mm/gup.c b/mm/gup.c index 5ce9ff4a8b561..86e2c67b8817e 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1946,14 +1946,14 @@ struct page *get_dump_page(unsigned long addr) /* * Returns the number of collected pages. Return value is always >= 0. */ -static void collect_longterm_unpinnable_pages( +static unsigned long collect_longterm_unpinnable_pages( struct list_head *movable_page_list, unsigned long nr_pages, struct page **pages) { + unsigned long i, collected = 0; struct folio *prev_folio = NULL; - bool drain_allow = true; - unsigned long i; + int drained = 0; for (i = 0; i < nr_pages; i++) { struct folio *folio = page_folio(pages[i]); @@ -1965,6 +1965,8 @@ static void collect_longterm_unpinnable_pages( if (folio_is_longterm_pinnable(folio)) continue; + collected++; + if (folio_is_device_coherent(folio)) continue; @@ -1973,9 +1975,17 @@ static void collect_longterm_unpinnable_pages( continue; } - if (!folio_test_lru(folio) && drain_allow) { + if (drained == 0 && folio_may_be_lru_cached(folio) && + folio_ref_count(folio) != + folio_expected_ref_count(folio) + 1) { + lru_add_drain(); + drained = 1; + } + if (drained == 1 && folio_may_be_lru_cached(folio) && + folio_ref_count(folio) != + folio_expected_ref_count(folio) + 1) { lru_add_drain_all(); - drain_allow = false; + drained = 2; } if (!folio_isolate_lru(folio)) @@ -1986,6 +1996,8 @@ static void collect_longterm_unpinnable_pages( NR_ISOLATED_ANON + folio_is_file_lru(folio), folio_nr_pages(folio)); } + + return collected; } /* @@ -2078,10 +2090,12 @@ static int migrate_longterm_unpinnable_pages( static long check_and_migrate_movable_pages(unsigned long nr_pages, struct page **pages) { + unsigned long collected; LIST_HEAD(movable_page_list); - collect_longterm_unpinnable_pages(&movable_page_list, nr_pages, pages); - if (list_empty(&movable_page_list)) + collected = collect_longterm_unpinnable_pages(&movable_page_list, + nr_pages, pages); + if (!collected) return 0; return migrate_longterm_unpinnable_pages(&movable_page_list, nr_pages, @@ -2127,6 +2141,10 @@ static long __gup_longterm_locked(struct mm_struct *mm, rc = check_and_migrate_movable_pages(nr_pinned_pages, pages); } while (rc == -EAGAIN); memalloc_pin_restore(flags); + + if (rc) + trace_android_rvh_gup_longterm_locked(rc, nr_pinned_pages, start, nr_pages, pages); + return rc ? rc : nr_pinned_pages; } diff --git a/mm/hmm.c b/mm/hmm.c index 277ddcab4947d..a36e1b4046dbd 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -173,6 +173,7 @@ static inline unsigned long hmm_pfn_flags_order(unsigned long order) return order << HMM_PFN_ORDER_SHIFT; } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd) { @@ -183,7 +184,6 @@ static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range, hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT); } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, unsigned long end, unsigned long hmm_pfns[], pmd_t pmd) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 6ac6febf6fad8..6e7629472cd43 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2320,7 +2320,6 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm pmd_t _dst_pmd, src_pmdval; struct page *src_page; struct folio *src_folio; - struct anon_vma *src_anon_vma; spinlock_t *src_ptl, *dst_ptl; pgtable_t src_pgtable; struct mmu_notifier_range range; @@ -2369,23 +2368,9 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm src_addr + HPAGE_PMD_SIZE); mmu_notifier_invalidate_range_start(&range); - if (src_folio) { + if (src_folio) folio_lock(src_folio); - /* - * split_huge_page walks the anon_vma chain without the page - * lock. Serialize against it with the anon_vma lock, the page - * lock is not enough. - */ - src_anon_vma = folio_get_anon_vma(src_folio); - if (!src_anon_vma) { - err = -EAGAIN; - goto unlock_folio; - } - anon_vma_lock_write(src_anon_vma); - } else - src_anon_vma = NULL; - dst_ptl = pmd_lockptr(mm, dst_pmd); double_pt_lock(src_ptl, dst_ptl); if (unlikely(!pmd_same(*src_pmd, src_pmdval) || @@ -2430,11 +2415,6 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); unlock_ptls: double_pt_unlock(src_ptl, dst_ptl); - if (src_anon_vma) { - anon_vma_unlock_write(src_anon_vma); - put_anon_vma(src_anon_vma); - } -unlock_folio: /* unblock rmap walks */ if (src_folio) folio_unlock(src_folio); @@ -2886,11 +2866,13 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, static void unmap_folio(struct folio *folio) { - enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD | - TTU_SYNC; + enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC; VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); + if (folio_test_pmd_mappable(folio)) + ttu_flags |= TTU_SPLIT_HUGE_PMD; + /* * Anon pages need migration entries to preserve them, but file * pages can simply be left unmapped, then faulted back on demand. @@ -2977,7 +2959,7 @@ bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); } -static void remap_page(struct folio *folio, unsigned long nr, int flags) +static void remap_page(struct folio *folio, unsigned long nr) { int i = 0; @@ -2985,7 +2967,7 @@ static void remap_page(struct folio *folio, unsigned long nr, int flags) if (!folio_test_anon(folio)) return; for (;;) { - remove_migration_ptes(folio, folio, RMP_LOCKED | flags); + remove_migration_ptes(folio, folio, true); i += folio_nr_pages(folio); if (i >= nr) break; @@ -3119,6 +3101,12 @@ static void reset_src_folio(struct folio *src) static bool lru_add_dst(struct lruvec *lruvec, struct folio *src, struct folio *dst) { + bool added = false; + + trace_android_vh_mm_customize_lru_add_dst(lruvec, src, dst, &added); + if (added) + return true; + if (folio_can_split(src)) return false; @@ -3338,7 +3326,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, if (nr_dropped) shmem_uncharge(head->mapping->host, nr_dropped); - remap_page(folio, nr, PageAnon(head) ? RMP_USE_SHARED_ZEROPAGE : 0); + remap_page(folio, nr); for (i = 0; i < nr; i++) { struct page *subpage = folio_dst_page(folio, i); @@ -3555,7 +3543,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) folio_ref_unfreeze(folio, 1 + extra_pins); remap: free_dst_pages(folio); - remap_page(folio, folio_nr_pages(folio), 0); + remap_page(folio, folio_nr_pages(folio)); } out_unlock: diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5ef48c97f35d8..576abe1c330ef 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -96,7 +96,7 @@ static void hugetlb_vma_lock_free(struct vm_area_struct *vma); static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma); static void hugetlb_unshare_pmds(struct vm_area_struct *vma, - unsigned long start, unsigned long end); + unsigned long start, unsigned long end, bool take_locks); static struct resv_map *vma_resv_map(struct vm_area_struct *vma); static inline bool subpool_is_free(struct hugepage_subpool *spool) @@ -3298,6 +3298,9 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h) return; } + if (!h->max_huge_pages) + return; + /* do node specific alloc */ for_each_online_node(i) { if (h->max_huge_pages_node[i] > 0) { @@ -4903,26 +4906,40 @@ static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) { if (addr & ~(huge_page_mask(hstate_vma(vma)))) return -EINVAL; + return 0; +} +void hugetlb_split(struct vm_area_struct *vma, unsigned long addr) +{ /* * PMD sharing is only possible for PUD_SIZE-aligned address ranges * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this * split, unshare PMDs in the PUD_SIZE interval surrounding addr now. + * This function is called in the middle of a VMA split operation, with + * MM, VMA and rmap all write-locked to prevent concurrent page table + * walks (except hardware and gup_fast()). */ + vma_assert_write_locked(vma); + i_mmap_assert_write_locked(vma->vm_file->f_mapping); + if (addr & ~PUD_MASK) { - /* - * hugetlb_vm_op_split is called right before we attempt to - * split the VMA. We will need to unshare PMDs in the old and - * new VMAs, so let's unshare before we split. - */ unsigned long floor = addr & PUD_MASK; unsigned long ceil = floor + PUD_SIZE; - if (floor >= vma->vm_start && ceil <= vma->vm_end) - hugetlb_unshare_pmds(vma, floor, ceil); + if (floor >= vma->vm_start && ceil <= vma->vm_end) { + /* + * Locking: + * Use take_locks=false here. + * The file rmap lock is already held. + * The hugetlb VMA lock can't be taken when we already + * hold the file rmap lock, and we don't need it because + * its purpose is to synchronize against concurrent page + * table walks, which are not possible thanks to the + * locks held by our caller. + */ + hugetlb_unshare_pmds(vma, floor, ceil, /* take_locks = */ false); + } } - - return 0; } static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma) @@ -6680,6 +6697,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma, psize); } spin_unlock(ptl); + + cond_resched(); } /* * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare @@ -7063,6 +7082,13 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, return 0; pud_clear(pud); + /* + * Once our caller drops the rmap lock, some other process might be + * using this page table as a normal, non-hugetlb page table. + * Wait for pending gup_fast() in other threads to finish before letting + * that happen. + */ + tlb_remove_table_sync_one(); ptdesc_pmd_pts_dec(virt_to_ptdesc(ptep)); mm_dec_nr_pmds(mm); return 1; @@ -7299,9 +7325,16 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re } } +/* + * If @take_locks is false, the caller must ensure that no concurrent page table + * access can happen (except for gup_fast() and hardware page walks). + * If @take_locks is true, we take the hugetlb VMA lock (to lock out things like + * concurrent page fault handling) and the file rmap lock. + */ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, unsigned long start, - unsigned long end) + unsigned long end, + bool take_locks) { struct hstate *h = hstate_vma(vma); unsigned long sz = huge_page_size(h); @@ -7325,8 +7358,12 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, start, end); mmu_notifier_invalidate_range_start(&range); - hugetlb_vma_lock_write(vma); - i_mmap_lock_write(vma->vm_file->f_mapping); + if (take_locks) { + hugetlb_vma_lock_write(vma); + i_mmap_lock_write(vma->vm_file->f_mapping); + } else { + i_mmap_assert_write_locked(vma->vm_file->f_mapping); + } for (address = start; address < end; address += PUD_SIZE) { ptep = hugetlb_walk(vma, address, sz); if (!ptep) @@ -7336,8 +7373,10 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, spin_unlock(ptl); } flush_hugetlb_tlb_range(vma, start, end); - i_mmap_unlock_write(vma->vm_file->f_mapping); - hugetlb_vma_unlock_write(vma); + if (take_locks) { + i_mmap_unlock_write(vma->vm_file->f_mapping); + hugetlb_vma_unlock_write(vma); + } /* * No need to call mmu_notifier_arch_invalidate_secondary_tlbs(), see * Documentation/mm/mmu_notifier.rst. @@ -7352,7 +7391,8 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE), - ALIGN_DOWN(vma->vm_end, PUD_SIZE)); + ALIGN_DOWN(vma->vm_end, PUD_SIZE), + /* take_locks = */ true); } #ifdef CONFIG_CMA diff --git a/mm/kasan/init.c b/mm/kasan/init.c index 89895f38f7224..afecc04b486a6 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -13,9 +13,9 @@ #include #include #include +#include #include -#include #include "kasan.h" @@ -197,7 +197,7 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, pud_t *pud; pmd_t *pmd; - p4d_populate(&init_mm, p4d, + p4d_populate_kernel(addr, p4d, lm_alias(kasan_early_shadow_pud)); pud = pud_offset(p4d, addr); pud_populate(&init_mm, pud, @@ -218,7 +218,7 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, } else { p = early_alloc(PAGE_SIZE, NUMA_NO_NODE); pud_init(p); - p4d_populate(&init_mm, p4d, p); + p4d_populate_kernel(addr, p4d, p); } } zero_pud_populate(p4d, addr, next); @@ -257,10 +257,10 @@ int __ref kasan_populate_early_shadow(const void *shadow_start, * puds,pmds, so pgd_populate(), pud_populate() * is noops. */ - pgd_populate(&init_mm, pgd, + pgd_populate_kernel(addr, pgd, lm_alias(kasan_early_shadow_p4d)); p4d = p4d_offset(pgd, addr); - p4d_populate(&init_mm, p4d, + p4d_populate_kernel(addr, p4d, lm_alias(kasan_early_shadow_pud)); pud = pud_offset(p4d, addr); pud_populate(&init_mm, pud, @@ -279,7 +279,7 @@ int __ref kasan_populate_early_shadow(const void *shadow_start, if (!p) return -ENOMEM; } else { - pgd_populate(&init_mm, pgd, + pgd_populate_kernel(addr, pgd, early_alloc(PAGE_SIZE, NUMA_NO_NODE)); } } diff --git a/mm/kasan/kasan_test.c b/mm/kasan/kasan_test.c index 94d6debde748f..95322e284922b 100644 --- a/mm/kasan/kasan_test.c +++ b/mm/kasan/kasan_test.c @@ -1054,6 +1054,7 @@ static void kasan_strings(struct kunit *test) ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + OPTIMIZER_HIDE_VAR(ptr); src = kmalloc(KASAN_GRANULE_SIZE, GFP_KERNEL | __GFP_ZERO); strscpy(src, "f0cacc1a0000000", KASAN_GRANULE_SIZE); diff --git a/mm/kasan/report.c b/mm/kasan/report.c index ecced40e51032..7301773b5208a 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -385,17 +385,10 @@ static void print_address_description(void *addr, u8 tag, } if (is_vmalloc_addr(addr)) { - struct vm_struct *va = find_vm_area(addr); - - if (va) { - pr_err("The buggy address belongs to the virtual mapping at\n" - " [%px, %px) created by:\n" - " %pS\n", - va->addr, va->addr + va->size, va->caller); - pr_err("\n"); - - page = vmalloc_to_page(addr); - } + pr_err("The buggy address belongs to a"); + if (!vmalloc_dump_obj(addr)) + pr_cont(" vmalloc virtual mapping\n"); + page = vmalloc_to_page(addr); } if (page) { @@ -538,7 +531,7 @@ void kasan_report_invalid_free(void *ptr, unsigned long ip, enum kasan_report_ty start_report(&flags, true); - memset(&info, 0, sizeof(info)); + __memset(&info, 0, sizeof(info)); info.type = type; info.access_addr = ptr; info.access_size = 0; @@ -576,7 +569,7 @@ bool kasan_report(const void *addr, size_t size, bool is_write, start_report(&irq_flags, true); - memset(&info, 0, sizeof(info)); + __memset(&info, 0, sizeof(info)); info.type = KASAN_REPORT_ACCESS; info.access_addr = addr; info.access_size = size; diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index dd772f9d0f080..d687f09a7ae37 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -324,7 +324,7 @@ static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr, if (!page) return -ENOMEM; - memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE); + __memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE); pte = pfn_pte(PFN_DOWN(__pa(page)), PAGE_KERNEL); spin_lock(&init_mm.page_table_lock); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index c717d3f799619..67ea833bbeb52 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1137,11 +1137,11 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, if (result != SCAN_SUCCEED) goto out_up_write; /* check if the pmd is still valid */ + vma_start_write(vma); result = check_pmd_still_valid(mm, address, pmd); if (result != SCAN_SUCCEED) goto out_up_write; - vma_start_write(vma); anon_vma_lock_write(vma->anon_vma); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, address, @@ -1377,8 +1377,8 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm, */ if (cc->is_khugepaged && (pte_young(pteval) || folio_test_young(folio) || - folio_test_referenced(folio) || mmu_notifier_test_young(vma->vm_mm, - address))) + folio_test_referenced(folio) || + mmu_notifier_test_young(vma->vm_mm, _address))) referenced++; } if (!writable) { @@ -2386,7 +2386,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, VM_BUG_ON(khugepaged_scan.address < hstart || khugepaged_scan.address + HPAGE_PMD_SIZE > hend); - if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { + if (IS_ENABLED(CONFIG_SHMEM) && !vma_is_anonymous(vma)) { struct file *file = get_file(vma->vm_file); pgoff_t pgoff = linear_page_index(vma, khugepaged_scan.address); @@ -2733,7 +2733,7 @@ int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, mmap_assert_locked(mm); memset(cc->node_load, 0, sizeof(cc->node_load)); nodes_clear(cc->alloc_nmask); - if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { + if (IS_ENABLED(CONFIG_SHMEM) && !vma_is_anonymous(vma)) { struct file *file = get_file(vma->vm_file); pgoff_t pgoff = linear_page_index(vma, addr); diff --git a/mm/kmemleak.c b/mm/kmemleak.c index f86d4e04d95e1..e2e41de55c02b 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -452,6 +452,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp) { unsigned long flags; struct kmemleak_object *object; + bool warn = false; /* try the slab allocator first */ if (object_cache) { @@ -469,8 +470,10 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp) else if (mem_pool_free_count) object = &mem_pool[--mem_pool_free_count]; else - pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n"); + warn = true; raw_spin_unlock_irqrestore(&kmemleak_lock, flags); + if (warn) + pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n"); return object; } @@ -2006,6 +2009,7 @@ static const struct file_operations kmemleak_fops = { static void __kmemleak_do_cleanup(void) { struct kmemleak_object *object, *tmp; + unsigned int cnt = 0; /* * Kmemleak has already been disabled, no need for RCU list traversal @@ -2014,6 +2018,10 @@ static void __kmemleak_do_cleanup(void) list_for_each_entry_safe(object, tmp, &object_list, object_list) { __remove_object(object); __delete_object(object); + + /* Call cond_resched() once per 64 iterations to avoid soft lockup */ + if (!(++cnt & 0x3f)) + cond_resched(); } } diff --git a/mm/kmsan/core.c b/mm/kmsan/core.c index 38a3bff23e8d0..38155d6982152 100644 --- a/mm/kmsan/core.c +++ b/mm/kmsan/core.c @@ -262,7 +262,8 @@ void kmsan_internal_set_shadow_origin(void *addr, size_t size, int b, u32 origin, bool checked) { u64 address = (u64)addr; - u32 *shadow_start, *origin_start; + void *shadow_start; + u32 *aligned_shadow, *origin_start; size_t pad = 0; KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(addr, size)); @@ -281,9 +282,12 @@ void kmsan_internal_set_shadow_origin(void *addr, size_t size, int b, } __memset(shadow_start, b, size); - if (!IS_ALIGNED(address, KMSAN_ORIGIN_SIZE)) { + if (IS_ALIGNED(address, KMSAN_ORIGIN_SIZE)) { + aligned_shadow = shadow_start; + } else { pad = address % KMSAN_ORIGIN_SIZE; address -= pad; + aligned_shadow = shadow_start - pad; size += pad; } size = ALIGN(size, KMSAN_ORIGIN_SIZE); @@ -297,7 +301,7 @@ void kmsan_internal_set_shadow_origin(void *addr, size_t size, int b, * corresponding shadow slot is zero. */ for (int i = 0; i < size / KMSAN_ORIGIN_SIZE; i++) { - if (origin || !shadow_start[i]) + if (origin || !aligned_shadow[i]) origin_start[i] = origin; } } diff --git a/mm/kmsan/kmsan_test.c b/mm/kmsan/kmsan_test.c index 312989aa2865c..e98c89d3a73aa 100644 --- a/mm/kmsan/kmsan_test.c +++ b/mm/kmsan/kmsan_test.c @@ -523,6 +523,21 @@ DEFINE_TEST_MEMSETXX(16) DEFINE_TEST_MEMSETXX(32) DEFINE_TEST_MEMSETXX(64) +/* Test case: ensure that KMSAN does not access shadow memory out of bounds. */ +static void test_memset_on_guarded_buffer(struct kunit *test) +{ + void *buf = vmalloc(PAGE_SIZE); + + kunit_info(test, + "memset() on ends of guarded buffer should not crash\n"); + + for (size_t size = 0; size <= 128; size++) { + memset(buf, 0xff, size); + memset(buf + PAGE_SIZE - size, 0xff, size); + } + vfree(buf); +} + static noinline void fibonacci(int *array, int size, int start) { if (start < 2 || (start == size)) @@ -602,6 +617,7 @@ static struct kunit_case kmsan_test_cases[] = { KUNIT_CASE(test_memset16), KUNIT_CASE(test_memset32), KUNIT_CASE(test_memset64), + KUNIT_CASE(test_memset_on_guarded_buffer), KUNIT_CASE(test_long_origin_chain), KUNIT_CASE(test_stackdepot_roundtrip), {}, diff --git a/mm/list_lru.c b/mm/list_lru.c index a05e5bef3b400..86f63c92fd889 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -566,6 +566,9 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, lru->shrinker_id = shrinker->id; else lru->shrinker_id = -1; + + if (mem_cgroup_kmem_disabled()) + memcg_aware = false; #endif lru->node = kcalloc(nr_node_ids, sizeof(*lru->node), GFP_KERNEL); diff --git a/mm/madvise.c b/mm/madvise.c index b36a6a32a1e3c..26ac589aba590 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -48,30 +48,18 @@ struct madvise_walk_private { void *private; }; -/* - * Any behaviour which results in changes to the vma->vm_flags needs to - * take mmap_lock for writing. Others, which simply traverse vmas, need - * to only take it for reading. - */ -static int madvise_need_mmap_write(int behavior) -{ - switch (behavior) { - case MADV_REMOVE: - case MADV_WILLNEED: - case MADV_DONTNEED: - case MADV_DONTNEED_LOCKED: - case MADV_COLD: - case MADV_PAGEOUT: - case MADV_FREE: - case MADV_POPULATE_READ: - case MADV_POPULATE_WRITE: - case MADV_COLLAPSE: - return 0; - default: - /* be safe, default to 1. list exceptions explicitly */ - return 1; - } -} +enum madvise_lock_mode { + MADVISE_NO_LOCK, + MADVISE_MMAP_READ_LOCK, + MADVISE_MMAP_WRITE_LOCK, + MADVISE_VMA_READ_LOCK, +}; + +struct madvise_behavior { + int behavior; + struct mmu_gather *tlb; + enum madvise_lock_mode lock_mode; +}; #ifdef CONFIG_ANON_VMA_NAME struct anon_vma_name *anon_vma_name_alloc(const char *name) @@ -941,8 +929,9 @@ static bool madvise_dontneed_free_valid_vma(struct vm_area_struct *vma, static long madvise_dontneed_free(struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, unsigned long end, - int behavior) + struct madvise_behavior *madv_behavior) { + int behavior = madv_behavior->behavior; struct mm_struct *mm = vma->vm_mm; *prev = vma; @@ -1069,7 +1058,7 @@ static long madvise_remove(struct vm_area_struct *vma, return -EINVAL; } - if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE)) + if (!vma_is_shared_maywrite(vma)) return -EACCES; offset = (loff_t)(start - vma->vm_start) @@ -1102,8 +1091,10 @@ static long madvise_remove(struct vm_area_struct *vma, static int madvise_vma_behavior(struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, unsigned long end, - unsigned long behavior) + void *behavior_arg) { + struct madvise_behavior *arg = behavior_arg; + int behavior = arg->behavior; int error; struct anon_vma_name *anon_name; unsigned long new_flags = vma->vm_flags; @@ -1123,7 +1114,7 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, case MADV_FREE: case MADV_DONTNEED: case MADV_DONTNEED_LOCKED: - return madvise_dontneed_free(vma, prev, start, end, behavior); + return madvise_dontneed_free(vma, prev, start, end, arg); case MADV_POPULATE_READ: case MADV_POPULATE_WRITE: return madvise_populate(vma, prev, start, end, behavior); @@ -1298,6 +1289,44 @@ static bool process_madvise_behavior_valid(int behavior) } } +/* + * Try to acquire a VMA read lock if possible. + * + * We only support this lock over a single VMA, which the input range must + * span either partially or fully. + * + * This function always returns with an appropriate lock held. If a VMA read + * lock could be acquired, we return the locked VMA. + * + * If a VMA read lock could not be acquired, we return NULL and expect caller to + * fallback to mmap lock behaviour. + */ +static struct vm_area_struct *try_vma_read_lock(struct mm_struct *mm, + struct madvise_behavior *madv_behavior, + unsigned long start, unsigned long end) +{ + struct vm_area_struct *vma; + + vma = lock_vma_under_rcu(mm, start); + if (!vma) + goto take_mmap_read_lock; + /* + * Must span only a single VMA; uffd and remote processes are + * unsupported. + */ + if (end > vma->vm_end || current->mm != mm || + userfaultfd_armed(vma)) { + vma_end_read(vma); + goto take_mmap_read_lock; + } + return vma; + +take_mmap_read_lock: + mmap_read_lock(mm); + madv_behavior->lock_mode = MADVISE_MMAP_READ_LOCK; + return NULL; +} + /* * Walk the vmas in range [start,end), and call the visit function on each one. * The visit function will get start and end parameters that cover the overlap @@ -1308,15 +1337,30 @@ static bool process_madvise_behavior_valid(int behavior) */ static int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, - unsigned long end, unsigned long arg, + unsigned long end, struct madvise_behavior *madv_behavior, + void *arg, int (*visit)(struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, - unsigned long end, unsigned long arg)) + unsigned long end, void *arg)) { struct vm_area_struct *vma; struct vm_area_struct *prev; unsigned long tmp; int unmapped_error = 0; + int error; + + /* + * If VMA read lock is supported, apply madvise to a single VMA + * tentatively, avoiding walking VMAs. + */ + if (madv_behavior && madv_behavior->lock_mode == MADVISE_VMA_READ_LOCK) { + vma = try_vma_read_lock(mm, madv_behavior, start, end); + if (vma) { + error = visit(vma, &prev, start, end, arg); + vma_end_read(vma); + return error; + } + } /* * If the interval [start,end) covers some unmapped address @@ -1328,8 +1372,6 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, prev = vma; for (;;) { - int error; - /* Still start < end. */ if (!vma) return -ENOMEM; @@ -1369,7 +1411,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, static int madvise_vma_anon_name(struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, unsigned long end, - unsigned long anon_name) + void *anon_name) { int error; @@ -1379,7 +1421,7 @@ static int madvise_vma_anon_name(struct vm_area_struct *vma, trace_android_vh_update_vma_flags(vma); error = madvise_update_vma(vma, prev, start, end, vma->vm_flags, - (struct anon_vma_name *)anon_name); + anon_name); /* * madvise() returns EAGAIN if kernel resources, such as @@ -1411,10 +1453,118 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, if (end == start) return 0; - return madvise_walk_vmas(mm, start, end, (unsigned long)anon_name, + return madvise_walk_vmas(mm, start, end, NULL, anon_name, madvise_vma_anon_name); } #endif /* CONFIG_ANON_VMA_NAME */ + +#ifdef CONFIG_MEMORY_FAILURE +static bool is_memory_failure(int behavior) +{ + switch (behavior) { + case MADV_HWPOISON: + case MADV_SOFT_OFFLINE: + return true; + default: + return false; + } +} +#else +static bool is_memory_failure(int behavior) +{ + return false; +} +#endif + +/* + * Any behaviour which results in changes to the vma->vm_flags needs to + * take mmap_lock for writing. Others, which simply traverse vmas, need + * to only take it for reading. + */ +static enum madvise_lock_mode get_lock_mode(struct madvise_behavior *madv_behavior) +{ + int behavior = madv_behavior->behavior; + + if (is_memory_failure(behavior)) + return MADVISE_NO_LOCK; + + switch (behavior) { + case MADV_REMOVE: + case MADV_WILLNEED: + case MADV_COLD: + case MADV_PAGEOUT: + case MADV_FREE: + case MADV_POPULATE_READ: + case MADV_POPULATE_WRITE: + case MADV_COLLAPSE: + return MADVISE_MMAP_READ_LOCK; + case MADV_DONTNEED: + case MADV_DONTNEED_LOCKED: + return MADVISE_VMA_READ_LOCK; + default: + return MADVISE_MMAP_WRITE_LOCK; + } +} + +static int madvise_lock(struct mm_struct *mm, + struct madvise_behavior *madv_behavior) +{ + enum madvise_lock_mode lock_mode = get_lock_mode(madv_behavior); + + switch (lock_mode) { + case MADVISE_NO_LOCK: + break; + case MADVISE_MMAP_WRITE_LOCK: + if (mmap_write_lock_killable(mm)) + return -EINTR; + break; + case MADVISE_MMAP_READ_LOCK: + mmap_read_lock(mm); + break; + case MADVISE_VMA_READ_LOCK: + /* We will acquire the lock per-VMA in madvise_walk_vmas(). */ + break; + } + + madv_behavior->lock_mode = lock_mode; + return 0; +} + +static void madvise_unlock(struct mm_struct *mm, + struct madvise_behavior *madv_behavior) +{ + switch (madv_behavior->lock_mode) { + case MADVISE_NO_LOCK: + return; + case MADVISE_MMAP_WRITE_LOCK: + mmap_write_unlock(mm); + break; + case MADVISE_MMAP_READ_LOCK: + mmap_read_unlock(mm); + break; + case MADVISE_VMA_READ_LOCK: + /* We will drop the lock per-VMA in madvise_walk_vmas(). */ + break; + } + + madv_behavior->lock_mode = MADVISE_NO_LOCK; +} + +/* + * untagged_addr_remote() assumes mmap_lock is already held. On + * architectures like x86 and RISC-V, tagging is tricky because each + * mm may have a different tagging mask. However, we might only hold + * the per-VMA lock (currently only local processes are supported), + * so untagged_addr is used to avoid the mmap_lock assertion for + * local processes. + */ +static inline unsigned long get_untagged_addr(struct mm_struct *mm, + unsigned long start) +{ + return current->mm == mm ? untagged_addr(start) : + untagged_addr_remote(mm, start); +} + /* * The madvise(2) system call. * @@ -1491,9 +1641,9 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh { unsigned long end; int error; - int write; size_t len; struct blk_plug plug; + struct madvise_behavior madv_behavior = {.behavior = behavior}; if (!madvise_behavior_valid(behavior)) return -EINVAL; @@ -1513,31 +1663,29 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh if (end == start) return 0; + error = madvise_lock(mm, &madv_behavior); + if (error) + return error; + #ifdef CONFIG_MEMORY_FAILURE - if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE) - return madvise_inject_error(behavior, start, start + len_in); -#endif + if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE) { + int ret = madvise_inject_error(behavior, start, start + len_in); - write = madvise_need_mmap_write(behavior); - if (write) { - if (mmap_write_lock_killable(mm)) - return -EINTR; - } else { - mmap_read_lock(mm); + madvise_unlock(mm, &madv_behavior); + + return ret; } +#endif - start = untagged_addr_remote(mm, start); + start = get_untagged_addr(mm, start); end = start + len; blk_start_plug(&plug); - error = madvise_walk_vmas(mm, start, end, behavior, - madvise_vma_behavior); + error = madvise_walk_vmas(mm, start, end, &madv_behavior, + &madv_behavior, madvise_vma_behavior); blk_finish_plug(&plug); - if (write) - mmap_write_unlock(mm); - else - mmap_read_unlock(mm); + madvise_unlock(mm, &madv_behavior); return error; } @@ -1605,7 +1753,7 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, * Require CAP_SYS_NICE for influencing process performance. Note that * only non-destructive hints are currently supported. */ - if (!capable(CAP_SYS_NICE)) { + if (mm != current->mm && !capable(CAP_SYS_NICE)) { ret = -EPERM; goto release_mm; } diff --git a/mm/memblock.c b/mm/memblock.c index 942a96536dc65..c89465846d32c 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -465,7 +465,14 @@ static int __init_memblock memblock_double_array(struct memblock_type *type, min(new_area_start, memblock.current_limit), new_alloc_size, PAGE_SIZE); - new_array = addr ? __va(addr) : NULL; + if (addr) { + /* The memory may not have been accepted, yet. */ + accept_memory(addr, addr + new_alloc_size); + + new_array = __va(addr); + } else { + new_array = NULL; + } } if (!addr) { pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", @@ -2440,11 +2447,14 @@ static void __init memmap_init_reserved_pages(void) struct memblock_region *region; phys_addr_t start, end; int nid; + unsigned long max_reserved; /* * set nid on all reserved pages and also treat struct * pages for the NOMAP regions as PageReserved */ +repeat: + max_reserved = memblock.reserved.max; for_each_mem_region(region) { nid = memblock_get_region_node(region); start = region->base; @@ -2453,8 +2463,15 @@ static void __init memmap_init_reserved_pages(void) if (memblock_is_nomap(region)) reserve_bootmem_region(start, end, nid); - memblock_set_node(start, end, &memblock.reserved, nid); + memblock_set_node(start, region->size, &memblock.reserved, nid); } + /* + * 'max' is changed means memblock.reserved has been doubled its + * array, which may result a new reserved region before current + * 'start'. Now we should repeat the procedure to set its node id. + */ + if (max_reserved != memblock.reserved.max) + goto repeat; /* initialize struct pages for the reserved regions */ for_each_reserved_mem_region(region) { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2be7af3de20d4..7d208e0bfb7fd 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +97,9 @@ static bool cgroup_memory_nokmem __ro_after_init; /* BPF memory accounting disabled? */ static bool cgroup_memory_nobpf __ro_after_init; +static struct kmem_cache *memcg_cachep; +static struct kmem_cache *memcg_pn_cachep; + #ifdef CONFIG_CGROUP_WRITEBACK static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); #endif @@ -603,10 +608,13 @@ static const unsigned int memcg_vm_event_stat[] = { #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) ZSWPIN, ZSWPOUT, + ZSWPWB, #endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE THP_FAULT_ALLOC, THP_COLLAPSE_ALLOC, + THP_SWPOUT, + THP_SWPOUT_FALLBACK, #endif }; @@ -1314,7 +1322,6 @@ void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, { struct mem_cgroup *iter; int ret = 0; - int i = 0; BUG_ON(mem_cgroup_is_root(memcg)); @@ -1324,10 +1331,9 @@ void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it); while (!ret && (task = css_task_iter_next(&it))) { - /* Avoid potential softlockup warning */ - if ((++i & 1023) == 0) - cond_resched(); ret = fn(task, arg); + /* Avoid potential softlockup warning */ + cond_resched(); } css_task_iter_end(&it); if (ret) { @@ -2477,9 +2483,10 @@ static unsigned long reclaim_high(struct mem_cgroup *memcg, memcg_memory_event(memcg, MEMCG_HIGH); psi_memstall_enter(&pflags); - nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages, + nr_reclaimed += try_to_free_mem_cgroup_pages_with_swappiness(memcg, nr_pages, gfp_mask, - MEMCG_RECLAIM_MAY_SWAP); + MEMCG_RECLAIM_MAY_SWAP, + NULL); psi_memstall_leave(&pflags); } while ((memcg = parent_mem_cgroup(memcg)) && !mem_cgroup_is_root(memcg)); @@ -2769,8 +2776,8 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, raised_max_event = true; psi_memstall_enter(&pflags); - nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, - gfp_mask, reclaim_options); + nr_reclaimed = try_to_free_mem_cgroup_pages_with_swappiness(mem_over_limit, nr_pages, + gfp_mask, reclaim_options, NULL); psi_memstall_leave(&pflags); if (mem_cgroup_margin(mem_over_limit) >= nr_pages) @@ -5386,7 +5393,8 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) { struct mem_cgroup_per_node *pn; - pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, node); + pn = kmem_cache_alloc_node(memcg_pn_cachep, GFP_KERNEL | __GFP_ZERO, + node); if (!pn) return 1; @@ -5442,7 +5450,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) int __maybe_unused i; long error = -ENOMEM; - memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL); + memcg = kmem_cache_zalloc(memcg_cachep, GFP_KERNEL); if (!memcg) return ERR_PTR(error); @@ -6685,8 +6693,8 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, continue; } - reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high, - GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP); + reclaimed = try_to_free_mem_cgroup_pages_with_swappiness(memcg, nr_pages - high, + GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL); if (!reclaimed && !nr_retries--) break; @@ -6734,8 +6742,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, } if (nr_reclaims) { - if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, - GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP)) + if (!try_to_free_mem_cgroup_pages_with_swappiness(memcg, nr_pages - max, + GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL)) nr_reclaims--; continue; } @@ -6860,19 +6868,50 @@ static ssize_t memory_oom_group_write(struct kernfs_open_file *of, return nbytes; } +enum { + MEMORY_RECLAIM_SWAPPINESS = 0, + MEMORY_RECLAIM_NULL, +}; + +static const match_table_t tokens = { + { MEMORY_RECLAIM_SWAPPINESS, "swappiness=%d"}, + { MEMORY_RECLAIM_NULL, NULL }, +}; + static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned int nr_retries = MAX_RECLAIM_RETRIES; unsigned long nr_to_reclaim, nr_reclaimed = 0; + int swappiness = -1; unsigned int reclaim_options; - int err; + char *old_buf, *start; + substring_t args[MAX_OPT_ARGS]; buf = strstrip(buf); - err = page_counter_memparse(buf, "", &nr_to_reclaim); - if (err) - return err; + + old_buf = buf; + nr_to_reclaim = memparse(buf, &buf) / PAGE_SIZE; + if (buf == old_buf) + return -EINVAL; + + buf = strstrip(buf); + + while ((start = strsep(&buf, " ")) != NULL) { + if (!strlen(start)) + continue; + switch (match_token(start, tokens, args)) { + case MEMORY_RECLAIM_SWAPPINESS: + if (match_int(&args[0], &swappiness)) + return -EINVAL; + if (swappiness < MIN_SWAPPINESS || swappiness > MAX_SWAPPINESS) + return -EINVAL; + break; + default: + return -EINVAL; + } + } reclaim_options = MEMCG_RECLAIM_MAY_SWAP | MEMCG_RECLAIM_PROACTIVE; while (nr_reclaimed < nr_to_reclaim) { @@ -6891,8 +6930,10 @@ static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf, if (!nr_retries) lru_add_drain_all(); - reclaimed = try_to_free_mem_cgroup_pages(memcg, - batch_size, GFP_KERNEL, reclaim_options); + reclaimed = try_to_free_mem_cgroup_pages_with_swappiness(memcg, + batch_size, GFP_KERNEL, + reclaim_options, + swappiness == -1 ? NULL : &swappiness); if (!reclaimed && !nr_retries--) return -EAGAIN; @@ -7395,6 +7436,18 @@ void __mem_cgroup_uncharge_list(struct list_head *page_list) uncharge_batch(&ug); } +void __mem_cgroup_uncharge_folios(struct folio_batch *folios) +{ + struct uncharge_gather ug; + unsigned int i; + + uncharge_gather_clear(&ug); + for (i = 0; i < folios->nr; i++) + uncharge_folio(folios->folios[i], &ug); + if (ug.memcg) + uncharge_batch(&ug); +} + /** * mem_cgroup_replace_folio - Charge a folio's replacement. * @old: Currently circulating folio. @@ -7583,15 +7636,16 @@ static int __init cgroup_memory(char *s) __setup("cgroup.memory=", cgroup_memory); /* - * subsys_initcall() for memory controller. + * Memory controller init before cgroup_init() initialize root_mem_cgroup. * * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this * context because of lock dependencies (cgroup_lock -> cpu hotplug) but * basically everything that doesn't depend on a specific mem_cgroup structure * should be initialized from here. */ -static int __init mem_cgroup_init(void) +int __init mem_cgroup_init(void) { + unsigned int memcg_size; int cpu, node; /* @@ -7609,6 +7663,13 @@ static int __init mem_cgroup_init(void) INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, drain_local_stock); + memcg_size = struct_size_t(struct mem_cgroup, nodeinfo, nr_node_ids); + memcg_cachep = kmem_cache_create("mem_cgroup", memcg_size, 0, + SLAB_PANIC | SLAB_HWCACHE_ALIGN, NULL); + + memcg_pn_cachep = KMEM_CACHE(mem_cgroup_per_node, + SLAB_PANIC | SLAB_HWCACHE_ALIGN); + for_each_node(node) { struct mem_cgroup_tree_per_node *rtpn; @@ -7622,7 +7683,6 @@ static int __init mem_cgroup_init(void) return 0; } -subsys_initcall(mem_cgroup_init); #ifdef CONFIG_SWAP static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) diff --git a/mm/memfd.c b/mm/memfd.c index 2dba2cb6f0d0f..187265dc68f5e 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -134,7 +134,7 @@ static int memfd_wait_for_pins(struct address_space *mapping) return error; } -static unsigned int *memfd_file_seals_ptr(struct file *file) +unsigned int *memfd_file_seals_ptr(struct file *file) { if (shmem_file(file)) return &SHMEM_I(file_inode(file))->seals; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index ae4448d1ed35f..6e69e8dce6097 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -835,9 +835,17 @@ static int hwpoison_hugetlb_range(pte_t *ptep, unsigned long hmask, #define hwpoison_hugetlb_range NULL #endif +static int hwpoison_test_walk(unsigned long start, unsigned long end, + struct mm_walk *walk) +{ + /* We also want to consider pages mapped into VM_PFNMAP. */ + return 0; +} + static const struct mm_walk_ops hwpoison_walk_ops = { .pmd_entry = hwpoison_pte_range, .hugetlb_entry = hwpoison_hugetlb_range, + .test_walk = hwpoison_test_walk, .walk_lock = PGWALK_RDLOCK, }; @@ -2527,10 +2535,9 @@ int unpoison_memory(unsigned long pfn) static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); - if (!pfn_valid(pfn)) - return -ENXIO; - - p = pfn_to_page(pfn); + p = pfn_to_online_page(pfn); + if (!p) + return -EIO; folio = page_folio(p); mutex_lock(&mf_mutex); diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 37a4f59d9585b..93cb8eed4bc23 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "internal.h" @@ -36,7 +37,7 @@ struct node_memory_type_map { static DEFINE_MUTEX(memory_tier_lock); static LIST_HEAD(memory_tiers); static struct node_memory_type_map node_memory_types[MAX_NUMNODES]; -static struct memory_dev_type *default_dram_type; +struct memory_dev_type *default_dram_type; static struct bus_type memory_tier_subsys = { .name = "memory_tiering", @@ -105,6 +106,13 @@ static int top_tier_adistance; static struct demotion_nodes *node_demotion __read_mostly; #endif /* CONFIG_MIGRATION */ +static BLOCKING_NOTIFIER_HEAD(mt_adistance_algorithms); + +static bool default_dram_perf_error; +static struct access_coordinate default_dram_perf; +static int default_dram_perf_ref_nid = NUMA_NO_NODE; +static const char *default_dram_perf_ref_source; + static inline struct memory_tier *to_memory_tier(struct device *device) { return container_of(device, struct memory_tier, dev); @@ -592,6 +600,158 @@ void clear_node_memory_type(int node, struct memory_dev_type *memtype) } EXPORT_SYMBOL_GPL(clear_node_memory_type); +static void dump_hmem_attrs(struct access_coordinate *coord, const char *prefix) +{ + pr_info( +"%sread_latency: %u, write_latency: %u, read_bandwidth: %u, write_bandwidth: %u\n", + prefix, coord->read_latency, coord->write_latency, + coord->read_bandwidth, coord->write_bandwidth); +} + +int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, + const char *source) +{ + int rc = 0; + + mutex_lock(&memory_tier_lock); + if (default_dram_perf_error) { + rc = -EIO; + goto out; + } + + if (perf->read_latency + perf->write_latency == 0 || + perf->read_bandwidth + perf->write_bandwidth == 0) { + rc = -EINVAL; + goto out; + } + + if (default_dram_perf_ref_nid == NUMA_NO_NODE) { + default_dram_perf = *perf; + default_dram_perf_ref_nid = nid; + default_dram_perf_ref_source = kstrdup(source, GFP_KERNEL); + goto out; + } + + /* + * The performance of all default DRAM nodes is expected to be + * same (that is, the variation is less than 10%). And it + * will be used as base to calculate the abstract distance of + * other memory nodes. + */ + if (abs(perf->read_latency - default_dram_perf.read_latency) * 10 > + default_dram_perf.read_latency || + abs(perf->write_latency - default_dram_perf.write_latency) * 10 > + default_dram_perf.write_latency || + abs(perf->read_bandwidth - default_dram_perf.read_bandwidth) * 10 > + default_dram_perf.read_bandwidth || + abs(perf->write_bandwidth - default_dram_perf.write_bandwidth) * 10 > + default_dram_perf.write_bandwidth) { + pr_info( +"memory-tiers: the performance of DRAM node %d mismatches that of the reference\n" +"DRAM node %d.\n", nid, default_dram_perf_ref_nid); + pr_info(" performance of reference DRAM node %d from %s:\n", + default_dram_perf_ref_nid, default_dram_perf_ref_source); + dump_hmem_attrs(&default_dram_perf, " "); + pr_info(" performance of DRAM node %d from %s:\n", nid, source); + dump_hmem_attrs(perf, " "); + pr_info( +" disable default DRAM node performance based abstract distance algorithm.\n"); + default_dram_perf_error = true; + rc = -EINVAL; + } + +out: + mutex_unlock(&memory_tier_lock); + return rc; +} + +int mt_perf_to_adistance(struct access_coordinate *perf, int *adist) +{ + if (default_dram_perf_error) + return -EIO; + + if (default_dram_perf_ref_nid == NUMA_NO_NODE) + return -ENOENT; + + if (perf->read_latency + perf->write_latency == 0 || + perf->read_bandwidth + perf->write_bandwidth == 0) + return -EINVAL; + + mutex_lock(&memory_tier_lock); + /* + * The abstract distance of a memory node is in direct proportion to + * its memory latency (read + write) and inversely proportional to its + * memory bandwidth (read + write). The abstract distance, memory + * latency, and memory bandwidth of the default DRAM nodes are used as + * the base. + */ + *adist = MEMTIER_ADISTANCE_DRAM * + (perf->read_latency + perf->write_latency) / + (default_dram_perf.read_latency + default_dram_perf.write_latency) * + (default_dram_perf.read_bandwidth + default_dram_perf.write_bandwidth) / + (perf->read_bandwidth + perf->write_bandwidth); + mutex_unlock(&memory_tier_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(mt_perf_to_adistance); + +/** + * register_mt_adistance_algorithm() - Register memory tiering abstract distance algorithm + * @nb: The notifier block which describe the algorithm + * + * Return: 0 on success, errno on error. + * + * Every memory tiering abstract distance algorithm provider needs to + * register the algorithm with register_mt_adistance_algorithm(). To + * calculate the abstract distance for a specified memory node, the + * notifier function will be called unless some high priority + * algorithm has provided result. The prototype of the notifier + * function is as follows, + * + * int (*algorithm_notifier)(struct notifier_block *nb, + * unsigned long nid, void *data); + * + * Where "nid" specifies the memory node, "data" is the pointer to the + * returned abstract distance (that is, "int *adist"). If the + * algorithm provides the result, NOTIFY_STOP should be returned. + * Otherwise, return_value & %NOTIFY_STOP_MASK == 0 to allow the next + * algorithm in the chain to provide the result. + */ +int register_mt_adistance_algorithm(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&mt_adistance_algorithms, nb); +} +EXPORT_SYMBOL_GPL(register_mt_adistance_algorithm); + +/** + * unregister_mt_adistance_algorithm() - Unregister memory tiering abstract distance algorithm + * @nb: the notifier block which describe the algorithm + * + * Return: 0 on success, errno on error. + */ +int unregister_mt_adistance_algorithm(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&mt_adistance_algorithms, nb); +} +EXPORT_SYMBOL_GPL(unregister_mt_adistance_algorithm); + +/** + * mt_calc_adistance() - Calculate abstract distance with registered algorithms + * @node: the node to calculate abstract distance for + * @adist: the returned abstract distance + * + * Return: if return_value & %NOTIFY_STOP_MASK != 0, then some + * abstract distance algorithm provides the result, and return it via + * @adist. Otherwise, no algorithm can provide the result and @adist + * will be kept as it is. + */ +int mt_calc_adistance(int node, int *adist) +{ + return blocking_notifier_call_chain(&mt_adistance_algorithms, node, adist); +} +EXPORT_SYMBOL_GPL(mt_calc_adistance); + static int __meminit memtier_hotplug_callback(struct notifier_block *self, unsigned long action, void *_arg) { diff --git a/mm/memory.c b/mm/memory.c index 2646e93c90048..51646a051ea7e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -68,6 +68,9 @@ #include #include #include +#ifndef __GENKSYMS__ +#include +#endif #include #include #include @@ -5033,6 +5036,8 @@ static bool vmf_pte_changed(struct vm_fault *vmf) vm_fault_t finish_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; + bool needs_fallback = false; + struct folio *folio; struct page *page; vm_fault_t ret; @@ -5042,6 +5047,8 @@ vm_fault_t finish_fault(struct vm_fault *vmf) else page = vmf->page; + folio = page_folio(page); + /* * check even for read faults because we might have lost our CoWed * page @@ -5052,8 +5059,25 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } + if (!needs_fallback && vma->vm_file) { + struct address_space *mapping = vma->vm_file->f_mapping; + pgoff_t file_end; + + file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); + + /* + * Do not allow to map with PTEs beyond i_size and with PMD + * across i_size to preserve SIGBUS semantics. + * + * Make an exception for shmem/tmpfs that for long time + * intentionally mapped with PMDs across i_size. + */ + needs_fallback = !shmem_mapping(mapping) && + file_end < folio_next_index(folio); + } + if (pmd_none(*vmf->pmd)) { - if (PageTransCompound(page)) { + if (!needs_fallback && PageTransCompound(page)) { ret = do_set_pmd(vmf, page); if (ret != VM_FAULT_FALLBACK) return ret; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index d76009c0d4830..3eac1db21dae1 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1737,8 +1737,12 @@ static void do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) if (PageHWPoison(page)) { if (WARN_ON(folio_test_lru(folio))) folio_isolate_lru(folio); - if (folio_mapped(folio)) + if (folio_mapped(folio)) { + folio_lock(folio); try_to_unmap(folio, TTU_IGNORE_MLOCK); + folio_unlock(folio); + } + continue; } diff --git a/mm/mempool.c b/mm/mempool.c index 734bcf5afbb78..50b5799a0c794 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -19,6 +19,8 @@ #include #include #include "slab.h" +#undef CREATE_TRACE_POINTS +#include #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON) static void poison_error(mempool_t *pool, void *element, size_t size, @@ -64,10 +66,20 @@ static void check_element(mempool_t *pool, void *element) } else if (pool->free == mempool_free_pages) { /* Mempools backed by page allocator */ int order = (int)(long)pool->pool_data; - void *addr = kmap_atomic((struct page *)element); - __check_element(pool, addr, 1UL << (PAGE_SHIFT + order)); - kunmap_atomic(addr); +#ifdef CONFIG_HIGHMEM + for (int i = 0; i < (1 << order); i++) { + struct page *page = (struct page *)element; + void *addr = kmap_local_page(page + i); + + __check_element(pool, addr, PAGE_SIZE); + kunmap_local(addr); + } +#else + void *addr = page_address((struct page *)element); + + __check_element(pool, addr, PAGE_SIZE << order); +#endif } } @@ -89,10 +101,20 @@ static void poison_element(mempool_t *pool, void *element) } else if (pool->alloc == mempool_alloc_pages) { /* Mempools backed by page allocator */ int order = (int)(long)pool->pool_data; - void *addr = kmap_atomic((struct page *)element); - __poison_element(addr, 1UL << (PAGE_SHIFT + order)); - kunmap_atomic(addr); +#ifdef CONFIG_HIGHMEM + for (int i = 0; i < (1 << order); i++) { + struct page *page = (struct page *)element; + void *addr = kmap_local_page(page + i); + + __poison_element(addr, PAGE_SIZE); + kunmap_local(addr); + } +#else + void *addr = page_address((struct page *)element); + + __poison_element(addr, PAGE_SIZE << order); +#endif } } #else /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */ @@ -383,6 +405,7 @@ void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) unsigned long flags; wait_queue_entry_t wait; gfp_t gfp_temp; + bool skip_wait = false; VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO); might_alloc(gfp_mask); @@ -428,6 +451,11 @@ void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) spin_unlock_irqrestore(&pool->lock, flags); return NULL; } + trace_android_vh_mempool_alloc_skip_wait(&gfp_temp, &skip_wait); + if (skip_wait) { + spin_unlock_irqrestore(&pool->lock, flags); + goto repeat_alloc; + } /* Let's wait for someone else to return an element to @pool */ init_wait(&wait); diff --git a/mm/migrate.c b/mm/migrate.c index b7cd1f7e6bc2a..e9e46b3579b4f 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -54,6 +54,9 @@ #include #include +#undef CREATE_TRACE_POINTS +#include +#include #undef CREATE_TRACE_POINTS #include @@ -182,57 +185,13 @@ void putback_movable_pages(struct list_head *l) } EXPORT_SYMBOL_GPL(putback_movable_pages); -static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, - struct folio *folio, - unsigned long idx) -{ - struct page *page = folio_page(folio, idx); - bool contains_data; - pte_t newpte; - void *addr; - - VM_BUG_ON_PAGE(PageCompound(page), page); - VM_BUG_ON_PAGE(!PageAnon(page), page); - VM_BUG_ON_PAGE(!PageLocked(page), page); - VM_BUG_ON_PAGE(pte_present(*pvmw->pte), page); - - if (folio_test_mlocked(folio) || (pvmw->vma->vm_flags & VM_LOCKED) || - mm_forbids_zeropage(pvmw->vma->vm_mm)) - return false; - - /* - * The pmd entry mapping the old thp was flushed and the pte mapping - * this subpage has been non present. If the subpage is only zero-filled - * then map it to the shared zeropage. - */ - addr = kmap_local_page(page); - contains_data = memchr_inv(addr, 0, PAGE_SIZE); - kunmap_local(addr); - - if (contains_data) - return false; - - newpte = pte_mkspecial(pfn_pte(my_zero_pfn(pvmw->address), - pvmw->vma->vm_page_prot)); - set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); - - dec_mm_counter(pvmw->vma->vm_mm, mm_counter(folio)); - return true; -} - -struct rmap_walk_arg { - struct folio *folio; - bool map_unused_to_zeropage; -}; - /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct folio *dst, struct vm_area_struct *vma, unsigned long addr, void *arg) { - struct rmap_walk_arg *rmap_walk_arg = arg; - struct folio *src = rmap_walk_arg->folio; + struct folio *src = arg; DEFINE_FOLIO_VMA_WALK(pvmw, src, vma, addr, PVMW_SYNC | PVMW_MIGRATION); while (page_vma_mapped_walk(&pvmw)) { @@ -272,9 +231,6 @@ static bool remove_migration_pte(struct folio *dst, continue; } #endif - if (rmap_walk_arg->map_unused_to_zeropage && - try_to_map_unused_to_zeropage(&pvmw, folio, idx)) - continue; folio_get(folio); pte = mk_pte(page, READ_ONCE(vma->vm_page_prot)); @@ -350,21 +306,14 @@ static bool remove_migration_pte(struct folio *dst, * Get rid of all migration entries and replace them by * references to the indicated page. */ -void remove_migration_ptes(struct folio *src, struct folio *dst, int flags) +void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked) { - struct rmap_walk_arg rmap_walk_arg = { - .folio = src, - .map_unused_to_zeropage = flags & RMP_USE_SHARED_ZEROPAGE, - }; - struct rmap_walk_control rwc = { .rmap_one = remove_migration_pte, - .arg = &rmap_walk_arg, + .arg = src, }; - VM_BUG_ON_FOLIO((flags & RMP_USE_SHARED_ZEROPAGE) && (src != dst), src); - - if (flags & RMP_LOCKED) + if (locked) rmap_walk_locked(dst, &rwc); else rmap_walk(dst, &rwc); @@ -988,7 +937,7 @@ static int writeout(struct address_space *mapping, struct folio *folio) * At this point we know that the migration attempt cannot * be successful. */ - remove_migration_ptes(folio, folio, 0); + remove_migration_ptes(folio, folio, false); rc = mapping->a_ops->writepage(&folio->page, &wbc); @@ -1151,7 +1100,7 @@ static void migrate_folio_undo_src(struct folio *src, struct list_head *ret) { if (page_was_mapped) - remove_migration_ptes(src, src, 0); + remove_migration_ptes(src, src, false); /* Drop an anon_vma reference if we took one */ if (anon_vma) put_anon_vma(anon_vma); @@ -1390,7 +1339,7 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, lru_add_drain(); if (old_page_state & PAGE_WAS_MAPPED) - remove_migration_ptes(src, dst, 0); + remove_migration_ptes(src, dst, false); out_unlock_both: folio_unlock(dst); @@ -1529,7 +1478,7 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, if (page_was_mapped) remove_migration_ptes(src, - rc == MIGRATEPAGE_SUCCESS ? dst : src, 0); + rc == MIGRATEPAGE_SUCCESS ? dst : src, false); unlock_put_anon: folio_unlock(dst); @@ -1615,6 +1564,7 @@ struct migrate_pages_stats { int nr_thp_succeeded; /* THP migrated successfully */ int nr_thp_failed; /* THP failed to be migrated */ int nr_thp_split; /* THP split before migrating */ + int nr_split; /* Large folio (include THP) split before migrating */ }; /* @@ -1734,6 +1684,7 @@ static int migrate_pages_batch(struct list_head *from, int nr_retry_pages = 0; int pass = 0; bool is_thp = false; + bool is_large = false; struct folio *folio, *folio2, *dst = NULL, *dst2; int rc, rc_saved = 0, nr_pages; LIST_HEAD(unmap_folios); @@ -1749,7 +1700,8 @@ static int migrate_pages_batch(struct list_head *from, nr_retry_pages = 0; list_for_each_entry_safe(folio, folio2, from, lru) { - is_thp = folio_test_large(folio) && folio_test_pmd_mappable(folio); + is_large = folio_test_large(folio); + is_thp = is_large && folio_test_pmd_mappable(folio); nr_pages = folio_nr_pages(folio); cond_resched(); @@ -1798,6 +1750,7 @@ static int migrate_pages_batch(struct list_head *from, stats->nr_thp_failed++; if (!try_split_folio(folio, split_folios, reason)) { stats->nr_thp_split++; + stats->nr_split++; continue; } stats->nr_failed_pages += nr_pages; @@ -1826,11 +1779,12 @@ static int migrate_pages_batch(struct list_head *from, nr_failed++; stats->nr_thp_failed += is_thp; /* Large folio NUMA faulting doesn't split to retry. */ - if (folio_test_large(folio) && !nosplit) { + if (is_large && !nosplit) { int ret = try_split_folio(folio, split_folios, reason); if (!ret) { stats->nr_thp_split += is_thp; + stats->nr_split += is_large; break; } else if (reason == MR_LONGTERM_PIN && ret == -EAGAIN) { @@ -1976,6 +1930,7 @@ static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, stats->nr_succeeded += astats.nr_succeeded; stats->nr_thp_succeeded += astats.nr_thp_succeeded; stats->nr_thp_split += astats.nr_thp_split; + stats->nr_split += astats.nr_split; if (rc < 0) { stats->nr_failed_pages += astats.nr_failed_pages; stats->nr_thp_failed += astats.nr_thp_failed; @@ -1983,7 +1938,11 @@ static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, return rc; } stats->nr_thp_failed += astats.nr_thp_split; - nr_failed += astats.nr_thp_split; + /* + * Do not count rc, as pages will be retried below. + * Count nr_split only, since it includes nr_thp_split. + */ + nr_failed += astats.nr_split; /* * Fall back to migrate all failed folios one by one synchronously. All * failed folios except split THPs will be retried, so their failure diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 38815d3dd72db..e5328815406a0 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -422,7 +422,7 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns, continue; folio = page_folio(page); - remove_migration_ptes(folio, folio, 0); + remove_migration_ptes(folio, folio, false); src_pfns[i] = 0; folio_unlock(folio); @@ -818,42 +818,40 @@ void migrate_device_finalize(unsigned long *src_pfns, unsigned long i; for (i = 0; i < npages; i++) { - struct folio *dst, *src; + struct folio *dst = NULL, *src = NULL; struct page *newpage = migrate_pfn_to_page(dst_pfns[i]); struct page *page = migrate_pfn_to_page(src_pfns[i]); + if (newpage) + dst = page_folio(newpage); + if (!page) { - if (newpage) { - unlock_page(newpage); - put_page(newpage); + if (dst) { + folio_unlock(dst); + folio_put(dst); } continue; } - if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE) || !newpage) { - if (newpage) { - unlock_page(newpage); - put_page(newpage); + src = page_folio(page); + + if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE) || !dst) { + if (dst) { + folio_unlock(dst); + folio_put(dst); } - newpage = page; + dst = src; } - src = page_folio(page); - dst = page_folio(newpage); - remove_migration_ptes(src, dst, 0); + if (!folio_is_zone_device(dst)) + folio_add_lru(dst); + remove_migration_ptes(src, dst, false); folio_unlock(src); + folio_put(src); - if (is_zone_device_page(page)) - put_page(page); - else - putback_lru_page(page); - - if (newpage != page) { - unlock_page(newpage); - if (is_zone_device_page(newpage)) - put_page(newpage); - else - putback_lru_page(newpage); + if (dst != src) { + folio_unlock(dst); + folio_put(dst); } } } diff --git a/mm/mlock.c b/mm/mlock.c index 36898c50f537f..4aca8609e94f7 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -257,7 +257,7 @@ void mlock_folio(struct folio *folio) folio_get(folio); if (!folio_batch_add(fbatch, mlock_lru(folio)) || - folio_test_large(folio) || lru_cache_disabled()) + !folio_may_be_lru_cached(folio) || lru_cache_disabled()) mlock_folio_batch(fbatch); local_unlock(&mlock_fbatch.lock); } @@ -280,7 +280,7 @@ void mlock_new_folio(struct folio *folio) folio_get(folio); if (!folio_batch_add(fbatch, mlock_new(folio)) || - folio_test_large(folio) || lru_cache_disabled()) + !folio_may_be_lru_cached(folio) || lru_cache_disabled()) mlock_folio_batch(fbatch); local_unlock(&mlock_fbatch.lock); } @@ -301,7 +301,7 @@ void munlock_folio(struct folio *folio) */ folio_get(folio); if (!folio_batch_add(fbatch, folio) || - folio_test_large(folio) || lru_cache_disabled()) + !folio_may_be_lru_cached(folio) || lru_cache_disabled()) mlock_folio_batch(fbatch); local_unlock(&mlock_fbatch.lock); } diff --git a/mm/mm_init.c b/mm/mm_init.c index dad55dbde265f..a6b6c8b396ab7 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1558,7 +1558,7 @@ static inline void setup_usemap(struct zone *zone) {} /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ void __init set_pageblock_order(void) { - unsigned int order = MAX_ORDER; + unsigned int order = PAGE_BLOCK_ORDER; /* Check that pageblock_nr_pages has not already been setup */ if (pageblock_order) @@ -2642,7 +2642,7 @@ void *__init alloc_large_system_hash(const char *tablename, panic("Failed to allocate %s hash table\n", tablename); pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", - tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, + tablename, 1UL << log2qty, get_order(size), size, virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); if (_hash_shift) diff --git a/mm/mmap.c b/mm/mmap.c index 4c74fb3d7a94b..b03226df862b5 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -49,6 +49,8 @@ #include #include #include +#include +#include #include #include @@ -113,7 +115,7 @@ void vma_set_page_prot(struct vm_area_struct *vma) static void __remove_shared_vm_struct(struct vm_area_struct *vma, struct file *file, struct address_space *mapping) { - if (vma->vm_flags & VM_SHARED) + if (vma_is_shared_maywrite(vma)) mapping_unmap_writable(mapping); flush_dcache_mmap_lock(mapping); @@ -144,8 +146,12 @@ static void remove_vma(struct vm_area_struct *vma, bool unreachable) { might_sleep(); vma_close(vma); - if (vma->vm_file) + if (vma->vm_file) { + if (is_dma_buf_file(vma->vm_file)) + dma_buf_unaccount_task(vma->vm_file->private_data, + vma->vm_mm->dmabuf_info); fput(vma->vm_file); + } mpol_put(vma_policy(vma)); if (unreachable) __vm_area_free(vma); @@ -389,7 +395,7 @@ static unsigned long count_vma_pages_range(struct mm_struct *mm, static void __vma_link_file(struct vm_area_struct *vma, struct address_space *mapping) { - if (vma->vm_flags & VM_SHARED) + if (vma_is_shared_maywrite(vma)) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); @@ -1304,6 +1310,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (file) { struct inode *inode = file_inode(file); + unsigned int seals = memfd_file_seals(file); unsigned long flags_mask; if (!file_mmap_ok(file, inode, pgoff, len)) @@ -1344,6 +1351,8 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_SHARED | VM_MAYSHARE; if (!(file->f_mode & FMODE_WRITE)) vm_flags &= ~(VM_MAYWRITE | VM_SHARED); + else if (is_readonly_sealed(seals, vm_flags)) + vm_flags &= ~VM_MAYWRITE; fallthrough; case MAP_PRIVATE: if (!(file->f_mode & FMODE_READ)) @@ -2417,8 +2426,16 @@ int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (err) goto out_free_mpol; - if (new->vm_file) + if (new->vm_file) { get_file(new->vm_file); + if (is_dma_buf_file(new->vm_file)) { + int acct_err = dma_buf_account_task(new->vm_file->private_data, + new->vm_mm->dmabuf_info); + + if (acct_err) + pr_err("failed to account dmabuf, err %d\n", acct_err); + } + } if (new->vm_ops && new->vm_ops->open) new->vm_ops->open(new); @@ -2429,7 +2446,13 @@ int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, init_vma_prep(&vp, vma); vp.insert = new; vma_prepare(&vp); + /* + * Get rid of huge pages and shared page tables straddling the split + * boundary. + */ vma_adjust_trans_huge(vma, vma->vm_start, addr, 0); + if (is_vm_hugetlb_page(vma)) + hugetlb_split(vma, addr); if (new_below) { vma->vm_start = addr; @@ -2703,14 +2726,14 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, return do_vmi_munmap(&vmi, mm, start, len, uf, false); } -unsigned long mmap_region(struct file *file, unsigned long addr, +static unsigned long __mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, struct list_head *uf) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; struct vm_area_struct *next, *prev, *merge; - pgoff_t pglen = len >> PAGE_SHIFT; + pgoff_t pglen = PHYS_PFN(len); unsigned long charged = 0; unsigned long end = addr + len; unsigned long merge_start = addr, merge_end = end; @@ -2810,25 +2833,26 @@ unsigned long mmap_region(struct file *file, unsigned long addr, vma->vm_page_prot = vm_get_page_prot(vm_flags); vma->vm_pgoff = pgoff; - if (file) { - if (vm_flags & VM_SHARED) { - error = mapping_map_writable(file->f_mapping); - if (error) - goto free_vma; - } + if (vma_iter_prealloc(&vmi, vma)) { + error = -ENOMEM; + goto free_vma; + } + if (file) { vma->vm_file = get_file(file); error = mmap_file(file, vma); if (error) - goto unmap_and_free_vma; + goto unmap_and_free_file_vma; + /* Drivers cannot alter the address of the VMA. */ + WARN_ON_ONCE(addr != vma->vm_start); /* - * Expansion is handled above, merging is handled below. - * Drivers should not alter the address of the VMA. + * Drivers should not permit writability when previously it was + * disallowed. */ - error = -EINVAL; - if (WARN_ON((addr != vma->vm_start))) - goto close_and_free_vma; + VM_WARN_ON_ONCE(vm_flags != vma->vm_flags && + !(vm_flags & VM_MAYWRITE) && + (vma->vm_flags & VM_MAYWRITE)); vma_iter_config(&vmi, addr, end); /* @@ -2840,6 +2864,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, vma->vm_end, vma->vm_flags, NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); + if (merge) { /* * ->mmap() can change vma->vm_file and fput @@ -2853,7 +2878,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, vma = merge; /* Update vm_flags to pick up the change. */ vm_flags = vma->vm_flags; - goto unmap_writable; + goto file_expanded; } } @@ -2861,24 +2886,15 @@ unsigned long mmap_region(struct file *file, unsigned long addr, } else if (vm_flags & VM_SHARED) { error = shmem_zero_setup(vma); if (error) - goto free_vma; + goto free_iter_vma; } else { vma_set_anonymous(vma); } - if (map_deny_write_exec(vma->vm_flags, vma->vm_flags)) { - error = -EACCES; - goto close_and_free_vma; - } - - /* Allow architectures to sanity-check the vm_flags */ - error = -EINVAL; - if (!arch_validate_flags(vma->vm_flags)) - goto close_and_free_vma; - - error = -ENOMEM; - if (vma_iter_prealloc(&vmi, vma)) - goto close_and_free_vma; +#ifdef CONFIG_SPARC64 + /* TODO: Fix SPARC ADI! */ + WARN_ON_ONCE(!arch_validate_flags(vm_flags)); +#endif /* Lock the VMA since it is modified after insertion into VMA tree */ vma_start_write(vma); @@ -2886,7 +2902,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, mm->map_count++; if (vma->vm_file) { i_mmap_lock_write(vma->vm_file->f_mapping); - if (vma->vm_flags & VM_SHARED) + if (vma_is_shared_maywrite(vma)) mapping_allow_writable(vma->vm_file->f_mapping); flush_dcache_mmap_lock(vma->vm_file->f_mapping); @@ -2901,10 +2917,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, */ khugepaged_enter_vma(vma, vma->vm_flags); - /* Once vma denies write, undo our temporary denial count */ -unmap_writable: - if (file && vm_flags & VM_SHARED) - mapping_unmap_writable(file->f_mapping); +file_expanded: file = vma->vm_file; ksm_add_vma(vma); expanded: @@ -2936,33 +2949,60 @@ unsigned long mmap_region(struct file *file, unsigned long addr, trace_android_vh_mmap_region(vma, addr); - validate_mm(mm); return addr; -close_and_free_vma: - vma_close(vma); - - if (file || vma->vm_file) { -unmap_and_free_vma: - fput(vma->vm_file); - vma->vm_file = NULL; +unmap_and_free_file_vma: + fput(vma->vm_file); + vma->vm_file = NULL; - vma_iter_set(&vmi, vma->vm_end); - /* Undo any partial mapping done by a device driver. */ - unmap_region(mm, &vmi.mas, vma, prev, next, vma->vm_start, - vma->vm_end, vma->vm_end, true); - } - if (file && (vm_flags & VM_SHARED)) - mapping_unmap_writable(file->f_mapping); + vma_iter_set(&vmi, vma->vm_end); + /* Undo any partial mapping done by a device driver. */ + unmap_region(mm, &vmi.mas, vma, prev, next, vma->vm_start, + vma->vm_end, vma->vm_end, true); +free_iter_vma: + vma_iter_free(&vmi); free_vma: vm_area_free(vma); unacct_error: if (charged) vm_unacct_memory(charged); - validate_mm(mm); return error; } +unsigned long mmap_region(struct file *file, unsigned long addr, + unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + struct list_head *uf) +{ + unsigned long ret; + bool writable_file_mapping = false; + + /* Check to see if MDWE is applicable. */ + if (map_deny_write_exec(vm_flags, vm_flags)) + return -EACCES; + + /* Allow architectures to sanity-check the vm_flags. */ + if (!arch_validate_flags(vm_flags)) + return -EINVAL; + + /* Map writable and ensure this isn't a sealed memfd. */ + if (file && is_shared_maywrite(vm_flags)) { + int error = mapping_map_writable(file->f_mapping); + + if (error) + return error; + writable_file_mapping = true; + } + + ret = __mmap_region(file, addr, len, vm_flags, pgoff, uf); + + /* Clear our write mapping regardless of error. */ + if (writable_file_mapping) + mapping_unmap_writable(file->f_mapping); + + validate_mm(current->mm); + return ret; +} + static int __vm_munmap(unsigned long start, size_t len, bool unlock) { int ret; diff --git a/mm/oom_kill.c b/mm/oom_kill.c index d4615c68cdc1d..12cdde44fee43 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -313,6 +313,7 @@ static int oom_evaluate_task(struct task_struct *task, void *arg) { struct oom_control *oc = arg; long points; + bool bypass = false; if (oom_unkillable_task(task)) goto next; @@ -342,6 +343,10 @@ static int oom_evaluate_task(struct task_struct *task, void *arg) goto select; } + trace_android_vh_oom_evaluate_task_bypass(task, oc, &bypass); + if (bypass) + goto next; + points = oom_badness(task, oc->totalpages); if (points == LONG_MIN || points < oc->chosen_points) goto next; @@ -521,7 +526,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) { struct vm_area_struct *vma; bool ret = true; - VMA_ITERATOR(vmi, mm, 0); + MA_STATE(mas, &mm->mm_mt, ULONG_MAX, ULONG_MAX); /* * Tell all users of get_user/copy_from_user etc... that the content @@ -532,7 +537,14 @@ bool __oom_reap_task_mm(struct mm_struct *mm) set_bit(MMF_UNSTABLE, &mm->flags); trace_android_vh_oom_swapmem_gather_init(mm); - for_each_vma(vmi, vma) { + + /* + * It might start racing with the dying task and compete for shared + * resources - e.g. page table lock contention has been observed. + * Reduce those races by reaping the oom victim from the other end + * of the address space. + */ + while ((vma = mas_find_rev(&mas, 0)) != NULL) { if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP)) continue; @@ -702,6 +714,8 @@ static void wake_oom_reaper(struct timer_list *timer) #define OOM_REAPER_DELAY (2*HZ) static void queue_oom_reaper(struct task_struct *tsk) { + bool bypass = false; + /* mm is already queued? */ if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags)) return; @@ -709,6 +723,9 @@ static void queue_oom_reaper(struct task_struct *tsk) get_task_struct(tsk); timer_setup(&tsk->oom_reaper_timer, wake_oom_reaper, 0); tsk->oom_reaper_timer.expires = jiffies + OOM_REAPER_DELAY; + trace_android_vh_oom_reaper_delay_bypass(tsk, &bypass); + if (bypass) + tsk->oom_reaper_timer.expires = jiffies; add_timer(&tsk->oom_reaper_timer); } @@ -792,12 +809,12 @@ static void mark_oom_victim(struct task_struct *tsk) __mark_oom_victim(tsk); /* - * Make sure that the task is woken up from uninterruptible sleep - * if it is frozen because OOM killer wouldn't be able to free - * any memory and livelock. freezing_slow_path will tell the freezer - * that TIF_MEMDIE tasks should be ignored. + * Make sure that the process is woken up from uninterruptible sleep + * if it is frozen because OOM killer wouldn't be able to free any + * memory and livelock. The freezer will thaw the tasks that are OOM + * victims regardless of the PM freezing and cgroup freezing states. */ - __thaw_task(tsk); + thaw_process(tsk); atomic_inc(&oom_victims); cred = get_task_cred(tsk); trace_mark_victim(tsk, cred->uid.val); @@ -1285,12 +1302,17 @@ SYSCALL_DEFINE2(process_mrelease, int, pidfd, unsigned int, flags) void add_to_oom_reaper(struct task_struct *p) { + bool thaw = false; + p = find_lock_task_mm(p); if (!p) return; if (task_will_free_mem(p)) { __mark_oom_victim(p); + trace_android_vh_thaw_killed_process(&thaw); + if (thaw) + thaw_process(p); queue_oom_reaper(p); } task_unlock(p); diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 2b93a86c9d66a..e96337e75d3da 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -542,8 +542,8 @@ static int dirty_ratio_handler(struct ctl_table *table, int write, void *buffer, ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); if (ret == 0 && write && vm_dirty_ratio != old_ratio) { - writeback_set_ratelimit(); vm_dirty_bytes = 0; + writeback_set_ratelimit(); } return ret; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c old mode 100644 new mode 100755 index 068ea2d706e35..b8f2be67576ee --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -57,7 +57,6 @@ #include #include #include - #include #include "internal.h" #include "shuffle.h" @@ -236,6 +235,12 @@ EXPORT_PER_CPU_SYMBOL(_numa_mem_); static DEFINE_MUTEX(pcpu_drain_mutex); +struct per_cpu_zone_extra_data { + unsigned long __percpu *pad; +}; + +static struct per_cpu_zone_extra_data zones_extra_data[__MAX_NR_ZONES]; + #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY volatile unsigned long latent_entropy __latent_entropy; EXPORT_SYMBOL(latent_entropy); @@ -345,7 +350,6 @@ EXPORT_SYMBOL(nr_online_nodes); static bool page_contains_unaccepted(struct page *page, unsigned int order); static void accept_page(struct page *page, unsigned int order); static bool cond_accept_memory(struct zone *zone, unsigned int order); -static inline bool has_unaccepted_memory(void); static bool __free_unaccepted(struct page *page); int page_group_by_mobility_disabled __read_mostly; @@ -870,7 +874,10 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, static int zone_max_order(struct zone *zone) { - return zone->order && zone_idx(zone) == ZONE_NOMERGE ? zone->order : MAX_ORDER; + int max_order = zone->order && zone_idx(zone) == ZONE_NOMERGE ? zone->order : MAX_ORDER; + + trace_android_vh_mm_customize_zone_max_order(zone, &max_order); + return max_order; } /* @@ -2604,6 +2611,8 @@ static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, int pindex; bool free_high; + trace_android_vh_pcp_alloc_factor_adjust(zone, zones_extra_data[zone_idx(zone)].pad, + pcp, page, migratetype, order); __count_vm_events(PGFREE, 1 << order); pindex = order_to_pindex(migratetype, order); list_add(&page->pcp_list, &pcp->lists[pindex]); @@ -2975,6 +2984,9 @@ struct page *___rmqueue_pcplist(struct zone *zone, unsigned int order, trace_android_vh_rmqueue_bulk_bypass(order, pcp, migratetype, list); if (!list_empty(list)) goto get_list; + + trace_android_vh_nr_pcp_alloc(pcp, zone, + &(zones_extra_data[zone_idx(zone)].pad), order, &batch); /* * Scale batch relative to order if batch implies * free pages can be stored on the PCP. Batch can @@ -3088,6 +3100,8 @@ struct page *rmqueue(struct zone *preferred_zone, */ WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); + trace_android_vh_mm_customize_rmqueue(zone, order, &alloc_flags, &migratetype); + if (likely(pcp_allowed_order(order))) { page = rmqueue_pcplist(preferred_zone, zone, order, migratetype, alloc_flags); @@ -3547,12 +3561,25 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, z = ac->preferred_zoneref; for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, ac->nodemask) { + bool use_this_zone = false; + bool suitable = true; struct page *page; unsigned long mark; if (!zone_is_suitable(zone, order)) continue; + trace_android_vh_mm_customize_suitable_zone(zone, gfp_mask, order, ac->highest_zoneidx, + &use_this_zone, &suitable); + if (!suitable) + continue; + + if (use_this_zone) + goto try_this_zone; + + /* + * This hook is deprecated by trace_android_vh_mm_customize_suitable_zone. + */ trace_android_vh_should_skip_zone(zone, gfp_mask, order, ac->migratetype, &should_skip_zone); if (should_skip_zone) @@ -4230,7 +4257,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) if (!(gfp_mask & __GFP_NOMEMALLOC)) { alloc_flags |= ALLOC_NON_BLOCK; - if (order > 0) + if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE)) alloc_flags |= ALLOC_HIGHATOMIC; } @@ -4551,6 +4578,15 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, retry: retry_loop_count++; + + /* + * Deal with possible cpuset update races or zonelist updates to avoid + * infinite retries. + */ + if (check_retry_cpuset(cpuset_mems_cookie, ac) || + check_retry_zonelist(zonelist_iter_cookie)) + goto restart; + /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ if (alloc_flags & ALLOC_KSWAPD) wake_all_kswapds(order, gfp_mask, ac); @@ -4985,6 +5021,9 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, &alloc_gfp, &alloc_flags)) return NULL; + trace_android_vh_mm_customize_ac(gfp, order, &ac.zonelist, &ac.preferred_zoneref, + &ac.highest_zoneidx, &alloc_flags); + trace_android_rvh_try_alloc_pages_gfp(&page, order, gfp, gfp_zone(gfp)); if (page) goto out; @@ -5945,6 +5984,8 @@ static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) zone->pageset_batch == new_batch) return; + trace_android_vh_mm_customize_zone_pageset(zone, &new_high, &new_batch); + zone->pageset_high = new_high; zone->pageset_batch = new_batch; @@ -7201,9 +7242,6 @@ bool has_managed_dma(void) #ifdef CONFIG_UNACCEPTED_MEMORY -/* Counts number of zones with unaccepted pages. */ -static DEFINE_STATIC_KEY_FALSE(zones_with_unaccepted_pages); - static bool lazy_accept = true; static int __init accept_memory_parse(char *p) @@ -7239,7 +7277,6 @@ static bool try_to_accept_memory_one(struct zone *zone) { unsigned long flags; struct page *page; - bool last; spin_lock_irqsave(&zone->lock, flags); page = list_first_entry_or_null(&zone->unaccepted_pages, @@ -7250,7 +7287,6 @@ static bool try_to_accept_memory_one(struct zone *zone) } list_del(&page->lru); - last = list_empty(&zone->unaccepted_pages); account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); __mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES); @@ -7260,9 +7296,6 @@ static bool try_to_accept_memory_one(struct zone *zone) __free_pages_ok(page, MAX_ORDER, FPI_TO_TAIL); - if (last) - static_branch_dec(&zones_with_unaccepted_pages); - return true; } @@ -7271,9 +7304,6 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order) long to_accept, wmark; bool ret = false; - if (!has_unaccepted_memory()) - return false; - if (list_empty(&zone->unaccepted_pages)) return false; @@ -7303,30 +7333,20 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order) return ret; } -static inline bool has_unaccepted_memory(void) -{ - return static_branch_unlikely(&zones_with_unaccepted_pages); -} - static bool __free_unaccepted(struct page *page) { struct zone *zone = page_zone(page); unsigned long flags; - bool first = false; if (!lazy_accept) return false; spin_lock_irqsave(&zone->lock, flags); - first = list_empty(&zone->unaccepted_pages); list_add_tail(&page->lru, &zone->unaccepted_pages); account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); __mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES); spin_unlock_irqrestore(&zone->lock, flags); - if (first) - static_branch_inc(&zones_with_unaccepted_pages); - return true; } @@ -7346,11 +7366,6 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order) return false; } -static inline bool has_unaccepted_memory(void) -{ - return false; -} - static bool __free_unaccepted(struct page *page) { BUILD_BUG(); diff --git a/mm/page_idle.c b/mm/page_idle.c index 4e51c5ecd63ec..b0f15d0ab675b 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -97,19 +97,15 @@ static void page_idle_clear_pte_refs(struct folio *folio) .rmap_one = page_idle_clear_pte_refs_one, .anon_lock = folio_lock_anon_vma_read, }; - bool need_lock; if (!folio_mapped(folio) || !folio_raw_mapping(folio)) return; - need_lock = !folio_test_anon(folio) || folio_test_ksm(folio); - if (need_lock && !folio_trylock(folio)) + if (!folio_trylock(folio)) return; rmap_walk(folio, &rwc); - - if (need_lock) - folio_unlock(folio); + folio_unlock(folio); } static ssize_t page_idle_bitmap_read(struct file *file, struct kobject *kobj, diff --git a/mm/page_io.c b/mm/page_io.c index a3feadd1ba9e5..ce37b02276269 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -211,8 +211,10 @@ int swap_writepage(struct page *page, struct writeback_control *wbc) static inline void count_swpout_vm_event(struct folio *folio) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (unlikely(folio_test_pmd_mappable(folio))) + if (unlikely(folio_test_pmd_mappable(folio))) { + count_memcg_folio_events(folio, THP_SWPOUT, 1); count_vm_event(THP_SWPOUT); + } count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); #endif count_vm_events(PSWPOUT, folio_nr_pages(folio)); @@ -282,9 +284,6 @@ static void sio_write_complete(struct kiocb *iocb, long ret) set_page_dirty(page); ClearPageReclaim(page); } - } else { - for (p = 0; p < sio->pages; p++) - count_swpout_vm_event(page_folio(sio->bvec[p].bv_page)); } for (p = 0; p < sio->pages; p++) @@ -300,6 +299,7 @@ static void swap_writepage_fs(struct page *page, struct writeback_control *wbc) struct file *swap_file = sis->swap_file; loff_t pos = page_file_offset(page); + count_swpout_vm_event(page_folio(page)); set_page_writeback(page); unlock_page(page); if (wbc->swap_plug) @@ -502,6 +502,7 @@ static void swap_readpage_bdev_sync(struct folio *folio, get_task_struct(current); count_vm_events(PSWPIN, folio_nr_pages(folio)); submit_bio_wait(&bio); + trace_android_rvh_swap_bio_charge(&bio); __end_swap_bio_read(&bio); put_task_struct(current); } diff --git a/mm/page_size_compat.c b/mm/page_size_compat.c index 4973bd793520a..aa62a5329be88 100644 --- a/mm/page_size_compat.c +++ b/mm/page_size_compat.c @@ -345,6 +345,7 @@ int __fixup_swap_header(struct file *swap_file, struct address_space *mapping) } #if IS_ENABLED(CONFIG_PERF_EVENTS) + static int __init init_sysctl_perf_event_mlock(void) { if (!static_branch_unlikely(&page_shift_compat_enabled)) diff --git a/mm/percpu.c b/mm/percpu.c index 3bc9332e3cdbc..1b77dd9d31d81 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1734,7 +1734,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, gfp = current_gfp_context(gfp); /* whitelisted flags that can be passed to the backing allocators */ pcpu_gfp = gfp & (GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); - is_atomic = (gfp & GFP_KERNEL) != GFP_KERNEL; + is_atomic = !gfpflags_allow_blocking(gfp); do_warn = !(gfp & __GFP_NOWARN); /* @@ -2231,7 +2231,12 @@ static void pcpu_balance_workfn(struct work_struct *work) * to grow other chunks. This then gives pcpu_reclaim_populated() time * to move fully free chunks to the active list to be freed if * appropriate. + * + * Enforce GFP_NOIO allocations because we have pcpu_alloc users + * constrained to GFP_NOIO/NOFS contexts and they could form lock + * dependency through pcpu_alloc_mutex */ + unsigned int flags = memalloc_noio_save(); mutex_lock(&pcpu_alloc_mutex); spin_lock_irq(&pcpu_lock); @@ -2242,6 +2247,7 @@ static void pcpu_balance_workfn(struct work_struct *work) spin_unlock_irq(&pcpu_lock); mutex_unlock(&pcpu_alloc_mutex); + memalloc_noio_restore(flags); } /** @@ -3158,7 +3164,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, #endif /* BUILD_EMBED_FIRST_CHUNK */ #ifdef BUILD_PAGE_FIRST_CHUNK -#include +#include #ifndef P4D_TABLE_SIZE #define P4D_TABLE_SIZE PAGE_SIZE @@ -3186,7 +3192,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr) p4d = memblock_alloc(P4D_TABLE_SIZE, P4D_TABLE_SIZE); if (!p4d) goto err_alloc; - pgd_populate(&init_mm, pgd, p4d); + pgd_populate_kernel(addr, pgd, p4d); } p4d = p4d_offset(pgd, addr); @@ -3194,7 +3200,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr) pud = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); if (!pud) goto err_alloc; - p4d_populate(&init_mm, p4d, pud); + p4d_populate_kernel(addr, p4d, pud); } pud = pud_offset(p4d, addr); diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index 3b020924a7a97..5142b80512214 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -154,6 +154,7 @@ static __always_inline bool str_has_suffix(const char *str, const char *suffix) return !strncmp(str + str_len - suffix_len, suffix, suffix_len); } +#ifdef CONFIG_PER_VMA_LOCK /* * The dynamic linker, or interpreter, operates within the process context * of the binary that necessitated dynamic linking. @@ -167,8 +168,6 @@ static __always_inline bool str_has_suffix(const char *str, const char *suffix) * VMAs of the current task. * * Returns true if in linker context, otherwise false. - * - * Caller must hold mmap lock in read mode. */ static inline bool linker_ctx(void) { @@ -180,14 +179,37 @@ static inline bool linker_ctx(void) if (!regs) return false; - vma = find_vma(mm, instruction_pointer(regs)); + vma = lock_vma_under_rcu(mm, instruction_pointer(regs)); + + /* + * lock_vma_under_rcu() is a try-lock that can fail if the + * VMA is already locked for modification. + * + * Fallback to finding the vma under mmap read lock. + */ + if (!vma) { + mmap_read_lock(mm); + + vma = find_vma(mm, instruction_pointer(regs)); + + /* Current execution context, the VMA must be present */ + BUG_ON(!vma); - /* Current execution context, the VMA must be present */ - BUG_ON(!vma); + /* + * We cannot use vma_start_read() as it may fail due to + * false locked (see comment in vma_start_read()). We + * can avoid that by directly locking vm_lock under + * mmap_lock, which guarantees that nobody can lock the + * vma for write (vma_start_write()) under us. + */ + down_read(&vma->vm_lock->lock); + + mmap_read_unlock(mm); + } file = vma->vm_file; if (!file) - return false; + goto out; if ((vma->vm_flags & VM_EXEC)) { char buf[64]; @@ -205,13 +227,21 @@ static inline bool linker_ctx(void) * * Check the base name (linker64). */ - if (!strcmp(kbasename(path), "linker64")) + if (!strcmp(kbasename(path), "linker64")) { + vma_end_read(vma); return true; + } } - +out: + vma_end_read(vma); return false; } +#else /* CONFIG_PER_VMA_LOCK */ + +static inline bool linker_ctx(void) { return false; } + +#endif /* CONFIG_PER_VMA_LOCK */ /* * Saves the number of padding pages for an ELF segment mapping * in vm_flags. @@ -271,22 +301,10 @@ static const struct vm_operations_struct pad_vma_ops = { }; /* - * Returns a new VMA representing the padding in @vma; - * returns NULL if no padding in @vma or allocation failed. + * Initialize @pad VMA fields with information from the original @vma. */ -static struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) +static void init_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad) { - struct vm_area_struct *pad; - - if (!is_pgsize_migration_enabled() || !(vma->vm_flags & VM_PAD_MASK)) - return NULL; - - pad = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL); - if (!pad) { - pr_warn("Page size migration: Failed to allocate padding VMA"); - return NULL; - } - memcpy(pad, vma, sizeof(struct vm_area_struct)); /* Remove file */ @@ -306,48 +324,34 @@ static struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) __vm_flags_mod(pad, 0, VM_READ|VM_WRITE|VM_EXEC); /* Remove padding bits */ __vm_flags_mod(pad, 0, VM_PAD_MASK); - - return pad; } /* - * Calls the show_pad_vma_fn on the @pad VMA, and frees the copies of @vma - * and @pad. + * Calls the show_pad_vma_fn on the @pad VMA. */ void show_map_pad_vma(struct vm_area_struct *vma, struct seq_file *m, void *func, bool smaps) { - struct vm_area_struct *pad = get_pad_vma(vma); - if (!pad) + if (!is_pgsize_migration_enabled() || !(vma->vm_flags & VM_PAD_MASK)) return; - /* - * This cannot happen. If @pad vma was allocated the corresponding - * @vma should have the VM_PAD_MASK bit(s) set. - */ - BUG_ON(!(vma->vm_flags & VM_PAD_MASK)); + struct vm_area_struct pad; - /* - * This cannot happen. @pad is a section of the original VMA. - * Therefore @vma cannot be null if @pad is not null. - */ - BUG_ON(!vma); + init_pad_vma(vma, &pad); /* The pad VMA should be anonymous. */ - BUG_ON(pad->vm_file); + BUG_ON(pad.vm_file); /* The pad VMA should be PROT_NONE. */ - BUG_ON(pad->vm_flags & (VM_READ|VM_WRITE|VM_EXEC)); + BUG_ON(pad.vm_flags & (VM_READ|VM_WRITE|VM_EXEC)); /* The pad VMA itself cannot have padding; infinite recursion */ - BUG_ON(pad->vm_flags & VM_PAD_MASK); + BUG_ON(pad.vm_flags & VM_PAD_MASK); if (smaps) - ((show_pad_smaps_fn)func)(m, pad); + ((show_pad_smaps_fn)func)(m, &pad); else - ((show_pad_maps_fn)func)(m, pad); - - kfree(pad); + ((show_pad_maps_fn)func)(m, &pad); } /* diff --git a/mm/ptdump.c b/mm/ptdump.c index 03c1bdae4a436..e46df2c24d858 100644 --- a/mm/ptdump.c +++ b/mm/ptdump.c @@ -152,6 +152,7 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) { const struct ptdump_range *range = st->range; + get_online_mems(); mmap_write_lock(mm); while (range->start != range->end) { walk_page_range_novma(mm, range->start, range->end, @@ -159,6 +160,7 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) range++; } mmap_write_unlock(mm); + put_online_mems(); /* Flush out the last page */ st->note_page(st, 0, -1, 0); diff --git a/mm/readahead.c b/mm/readahead.c index 7efce4b584458..f8527ddbadae3 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -251,6 +251,10 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, read_pages(ractl); ractl->_index++; i = ractl->_index + ractl->_nr_pages - index - 1; +#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA + trace_android_vh_page_cache_ra_unbounded(mapping, folio, + &ractl->android_oem_data1); +#endif continue; } @@ -270,6 +274,10 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, folio_set_readahead(folio); ractl->_workingset |= folio_test_workingset(folio); ractl->_nr_pages++; +#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA + trace_android_vh_page_cache_ra_unbounded(mapping, folio, + &ractl->android_oem_data1); +#endif } /* @@ -332,6 +340,9 @@ void force_page_cache_ra(struct readahead_control *ractl, index = readahead_index(ractl); max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages); nr_to_read = min_t(unsigned long, nr_to_read, max_pages); +#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA + trace_android_vh_force_page_cache_ra(mapping, &ractl->android_oem_data1); +#endif while (nr_to_read) { unsigned long this_chunk = (2 * 1024 * 1024) / PAGE_SIZE; @@ -504,10 +515,16 @@ void page_cache_ra_order(struct readahead_control *ractl, unsigned int nofs; int err = 0; gfp_t gfp = readahead_gfp_mask(mapping); + bool bypass = false; if (!mapping_large_folio_support(mapping) || ra->size < 4) goto fallback; + trace_android_vh_page_cache_ra_order_bypass(ractl, ra, new_order, &gfp, + &bypass); + if (bypass) + goto fallback; + limit = min(limit, index + ra->size - 1); if (new_order < MAX_PAGECACHE_ORDER) { diff --git a/mm/rmap.c b/mm/rmap.c old mode 100644 new mode 100755 index 12a6f9d9af85b..2d7be8626ae31 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -87,6 +87,9 @@ #include #include +#undef CREATE_TRACE_POINTS +#include + #include "internal.h" static struct kmem_cache *anon_vma_cachep; @@ -495,17 +498,15 @@ void __init anon_vma_init(void) * if there is a mapcount, we can dereference the anon_vma after observing * those. * - * NOTE: the caller should normally hold folio lock when calling this. If - * not, the caller needs to double check the anon_vma didn't change after - * taking the anon_vma lock for either read or write (UFFDIO_MOVE can modify it - * concurrently without folio lock protection). See folio_lock_anon_vma_read() - * which has already covered that, and comment above remap_pages(). + * NOTE: the caller should hold folio lock when calling this. */ struct anon_vma *folio_get_anon_vma(struct folio *folio) { struct anon_vma *anon_vma = NULL; unsigned long anon_mapping; + VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); + rcu_read_lock(); anon_mapping = (unsigned long)READ_ONCE(folio->mapping); if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) @@ -552,7 +553,8 @@ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, struct anon_vma *root_anon_vma; unsigned long anon_mapping; -retry: + VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); + rcu_read_lock(); anon_mapping = (unsigned long)READ_ONCE(folio->mapping); if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) @@ -563,17 +565,6 @@ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); root_anon_vma = READ_ONCE(anon_vma->root); if (down_read_trylock(&root_anon_vma->rwsem)) { - /* - * folio_move_anon_rmap() might have changed the anon_vma as we - * might not hold the folio lock here. - */ - if (unlikely((unsigned long)READ_ONCE(folio->mapping) != - anon_mapping)) { - up_read(&root_anon_vma->rwsem); - rcu_read_unlock(); - goto retry; - } - /* * If the folio is still mapped, then this anon_vma is still * its anon_vma, and holding the mutex ensures that it will @@ -608,18 +599,6 @@ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, rcu_read_unlock(); anon_vma_lock_read(anon_vma); - /* - * folio_move_anon_rmap() might have changed the anon_vma as we might - * not hold the folio lock here. - */ - if (unlikely((unsigned long)READ_ONCE(folio->mapping) != - anon_mapping)) { - anon_vma_unlock_read(anon_vma); - put_anon_vma(anon_vma); - anon_vma = NULL; - goto retry; - } - if (atomic_dec_and_test(&anon_vma->refcount)) { /* * Oops, we held the last refcount, release the lock @@ -1000,7 +979,7 @@ int folio_referenced(struct folio *folio, int is_locked, if (!folio_raw_mapping(folio)) return 0; - if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) { + if (!is_locked) { we_locked = folio_trylock(folio); if (!we_locked) return 1; @@ -2667,6 +2646,12 @@ static void rmap_walk_anon(struct folio *folio, pgoff_t pgoff_start, pgoff_end; struct anon_vma_chain *avc; + /* + * The folio lock ensures that folio->mapping can't be changed under us + * to an anon_vma with different root. + */ + VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); + if (locked) { anon_vma = folio_anon_vma(folio); /* anon_vma disappear under us? */ diff --git a/mm/secretmem.c b/mm/secretmem.c index 399552814fd0f..f64ea1cde2f92 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -84,13 +84,13 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) __folio_mark_uptodate(folio); err = filemap_add_folio(mapping, folio, offset, gfp); if (unlikely(err)) { - folio_put(folio); /* * If a split of large page was required, it * already happened when we marked the page invalid * which guarantees that this call won't fail */ set_direct_map_default_noflush(page); + folio_put(folio); if (err == -EEXIST) goto retry; @@ -195,19 +195,10 @@ static struct file *secretmem_file_create(unsigned long flags) struct file *file; struct inode *inode; const char *anon_name = "[secretmem]"; - const struct qstr qname = QSTR_INIT(anon_name, strlen(anon_name)); - int err; - inode = alloc_anon_inode(secretmem_mnt->mnt_sb); + inode = anon_inode_make_secure_inode(secretmem_mnt->mnt_sb, anon_name, NULL); if (IS_ERR(inode)) return ERR_CAST(inode); - - err = security_inode_init_security_anon(inode, &qname, NULL); - if (err) { - file = ERR_PTR(err); - goto err_free_inode; - } - file = alloc_file_pseudo(inode, secretmem_mnt, "secretmem", O_RDWR, &secretmem_fops); if (IS_ERR(file)) diff --git a/mm/shmem.c b/mm/shmem.c index 099c26f8073e5..4226225d43f25 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -135,8 +135,7 @@ struct shmem_options { #define SHMEM_SEEN_INODES 2 #define SHMEM_SEEN_HUGE 4 #define SHMEM_SEEN_INUMS 8 -#define SHMEM_SEEN_NOSWAP 16 -#define SHMEM_SEEN_QUOTA 32 +#define SHMEM_SEEN_QUOTA 16 }; #ifdef CONFIG_TMPFS @@ -2415,7 +2414,7 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma) struct shmem_inode_info *info = SHMEM_I(inode); int ret; - ret = seal_check_future_write(info->seals, vma); + ret = seal_check_write(info->seals, vma); if (ret) return ret; @@ -4024,7 +4023,6 @@ static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) "Turning off swap in unprivileged tmpfs mounts unsupported"); } ctx->noswap = true; - ctx->seen |= SHMEM_SEEN_NOSWAP; break; case Opt_quota: if (fc->user_ns != &init_user_ns) @@ -4174,14 +4172,15 @@ static int shmem_reconfigure(struct fs_context *fc) err = "Current inum too high to switch to 32-bit inums"; goto out; } - if ((ctx->seen & SHMEM_SEEN_NOSWAP) && ctx->noswap && !sbinfo->noswap) { + + /* + * "noswap" doesn't use fsparam_flag_no, i.e. there's no "swap" + * counterpart for (re-)enabling swap. + */ + if (ctx->noswap && !sbinfo->noswap) { err = "Cannot disable swap on remount"; goto out; } - if (!(ctx->seen & SHMEM_SEEN_NOSWAP) && !ctx->noswap && sbinfo->noswap) { - err = "Cannot enable swap on remount if it was disabled on first mount"; - goto out; - } if (ctx->seen & SHMEM_SEEN_QUOTA && !sb_any_quota_loaded(fc->root->d_sb)) { @@ -4643,11 +4642,7 @@ void __init shmem_init(void) shmem_init_inodecache(); #ifdef CONFIG_TMPFS_QUOTA - error = register_quota_format(&shmem_quota_format); - if (error < 0) { - pr_err("Could not register quota format\n"); - goto out3; - } + register_quota_format(&shmem_quota_format); #endif error = register_filesystem(&shmem_fs_type); @@ -4676,7 +4671,6 @@ void __init shmem_init(void) out2: #ifdef CONFIG_TMPFS_QUOTA unregister_quota_format(&shmem_quota_format); -out3: #endif shmem_destroy_inodecache(); shm_mnt = ERR_PTR(error); diff --git a/mm/show_mem.c b/mm/show_mem.c index 1bda780cb5df1..511ca5a4cdfd8 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -14,6 +14,7 @@ #include #include #include +#include #undef CREATE_TRACE_POINTS #include diff --git a/mm/slab_common.c b/mm/slab_common.c index 59ae933a0fe51..9ebe84dba06be 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -723,7 +723,6 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags, unsigned long caller) return NULL; index = fls(size - 1); } - trace_android_vh_kmalloc_slab(index, flags, &s); if (s) return s; diff --git a/mm/slub.c b/mm/slub.c index 904e081d6a912..f97e39dfd6cdc 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -807,19 +807,19 @@ unsigned long get_each_object_track(struct kmem_cache *s, EXPORT_SYMBOL_NS_GPL(get_each_object_track, MINIDUMP); #ifdef CONFIG_STACKDEPOT -static noinline depot_stack_handle_t set_track_prepare(void) +static noinline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) { depot_stack_handle_t handle; unsigned long entries[TRACK_ADDRS_COUNT]; unsigned int nr_entries; nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3); - handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT); + handle = stack_depot_save(entries, nr_entries, gfp_flags); return handle; } #else -static inline depot_stack_handle_t set_track_prepare(void) +static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) { return 0; } @@ -842,9 +842,9 @@ static void set_track_update(struct kmem_cache *s, void *object, } static __always_inline void set_track(struct kmem_cache *s, void *object, - enum track_item alloc, unsigned long addr) + enum track_item alloc, unsigned long addr, gfp_t gfp_flags) { - depot_stack_handle_t handle = set_track_prepare(); + depot_stack_handle_t handle = set_track_prepare(gfp_flags); set_track_update(s, object, alloc, addr, handle); } @@ -1031,8 +1031,13 @@ static void object_err(struct kmem_cache *s, struct slab *slab, if (slab_add_kunit_errors()) return; - slab_bug(s, reason); - print_trailer(s, slab, object); + slab_bug(s, "%s", reason); + if (!object || !check_valid_pointer(s, slab, object)) { + print_slab_info(slab); + pr_err("Invalid pointer 0x%p\n", object); + } else { + print_trailer(s, slab, object); + } add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); WARN_ON(1); @@ -1788,9 +1793,9 @@ static inline bool free_debug_processing(struct kmem_cache *s, static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {} static inline int check_object(struct kmem_cache *s, struct slab *slab, void *object, u8 val) { return 1; } -static inline depot_stack_handle_t set_track_prepare(void) { return 0; } +static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) { return 0; } static inline void set_track(struct kmem_cache *s, void *object, - enum track_item alloc, unsigned long addr) {} + enum track_item alloc, unsigned long addr, gfp_t gfp_flags) {} static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) {} static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, @@ -3280,8 +3285,26 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, pc.slab = &slab; pc.orig_size = orig_size; freelist = get_partial(s, node, &pc); - if (freelist) - goto check_new_slab; + if (freelist) { + if (kmem_cache_debug(s)) { + /* + * For debug caches here we had to go through + * alloc_single_from_partial() so just store the + * tracking info and return the object. + * + * Due to disabled preemption we need to disallow + * blocking. The flags are further adjusted by + * gfp_nested_mask() in stack_depot itself. + */ + if (s->flags & SLAB_STORE_USER) + set_track(s, freelist, TRACK_ALLOC, addr, + gfpflags & ~(__GFP_DIRECT_RECLAIM)); + + return freelist; + } + + goto retry_load_slab; + } slub_put_cpu_ptr(s->cpu_slab); slab = new_slab(s, gfpflags, node); @@ -3301,7 +3324,8 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, goto new_objects; if (s->flags & SLAB_STORE_USER) - set_track(s, freelist, TRACK_ALLOC, addr); + set_track(s, freelist, TRACK_ALLOC, addr, + gfpflags & ~(__GFP_DIRECT_RECLAIM)); return freelist; } @@ -3317,20 +3341,6 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, inc_slabs_node(s, slab_nid(slab), slab->objects); -check_new_slab: - - if (kmem_cache_debug(s)) { - /* - * For debug caches here we had to go through - * alloc_single_from_partial() so just store the tracking info - * and return the object - */ - if (s->flags & SLAB_STORE_USER) - set_track(s, freelist, TRACK_ALLOC, addr); - - return freelist; - } - if (unlikely(!pfmemalloc_match(slab, gfpflags))) { /* * For !pfmemalloc_match() case we don't load freelist so that @@ -3605,8 +3615,12 @@ static noinline void free_to_partial_list( unsigned long flags; depot_stack_handle_t handle = 0; + /* + * We cannot use GFP_NOWAIT as there are callsites where waking up + * kswapd could deadlock + */ if (s->flags & SLAB_STORE_USER) - handle = set_track_prepare(); + handle = set_track_prepare(__GFP_NOWARN); spin_lock_irqsave(&n->list_lock, flags); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index a2cbe44c48e10..589d6a262b6da 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -27,9 +27,9 @@ #include #include #include +#include #include -#include /* * Allocate a block of memory to be used to back the virtual memory map @@ -225,7 +225,7 @@ p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node) if (!p) return NULL; pud_init(p); - p4d_populate(&init_mm, p4d, p); + p4d_populate_kernel(addr, p4d, p); } return p4d; } @@ -237,7 +237,7 @@ pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node); if (!p) return NULL; - pgd_populate(&init_mm, pgd, p); + pgd_populate_kernel(addr, pgd, p); } return pgd; } diff --git a/mm/swap.c b/mm/swap.c index 216b4e015d2ad..ad7cd231491ed 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -77,26 +77,33 @@ static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = { .lock = INIT_LOCAL_LOCK(lock), }; +static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp, + unsigned long *flagsp) +{ + if (folio_test_lru(folio)) { + folio_lruvec_relock_irqsave(folio, lruvecp, flagsp); + lruvec_del_folio(*lruvecp, folio); + __folio_clear_lru_flags(folio); + } +} + /* * This path almost never happens for VM activity - pages are normally freed * in batches. But it gets used by networking - and for compound pages. */ -static void __page_cache_release(struct folio *folio) +static void page_cache_release(struct folio *folio) { - if (folio_test_lru(folio)) { - struct lruvec *lruvec; - unsigned long flags; + struct lruvec *lruvec = NULL; + unsigned long flags; - lruvec = folio_lruvec_lock_irqsave(folio, &flags); - lruvec_del_folio(lruvec, folio); - __folio_clear_lru_flags(folio); + __page_cache_release(folio, &lruvec, &flags); + if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); - } } static void __folio_put_small(struct folio *folio) { - __page_cache_release(folio); + page_cache_release(folio); mem_cgroup_uncharge(folio); free_unref_page(&folio->page, 0); } @@ -110,7 +117,7 @@ static void __folio_put_large(struct folio *folio) * be called for hugetlb (it has a separate hugetlb_cgroup.) */ if (!folio_test_hugetlb(folio)) - __page_cache_release(folio); + page_cache_release(folio); destroy_large_folio(folio); } @@ -133,22 +140,25 @@ EXPORT_SYMBOL(__folio_put); */ void put_pages_list(struct list_head *pages) { + struct folio_batch fbatch; struct folio *folio, *next; + folio_batch_init(&fbatch); list_for_each_entry_safe(folio, next, pages, lru) { - if (!folio_put_testzero(folio)) { - list_del(&folio->lru); + if (!folio_put_testzero(folio)) continue; - } if (folio_test_large(folio)) { - list_del(&folio->lru); __folio_put_large(folio); continue; } /* LRU flag must be clear because it's passed using the lru */ + if (folio_batch_add(&fbatch, folio) > 0) + continue; + free_unref_folios(&fbatch); } - free_unref_page_list(pages); + if (fbatch.nr) + free_unref_folios(&fbatch); INIT_LIST_HEAD(pages); } EXPORT_SYMBOL(put_pages_list); @@ -208,7 +218,7 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) if (move_fn != lru_add_fn && !folio_test_clear_lru(folio)) continue; - lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags); + folio_lruvec_relock_irqsave(folio, &lruvec, &flags); move_fn(lruvec, folio); folio_set_lru(folio); @@ -222,8 +232,8 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) static void folio_batch_add_and_move(struct folio_batch *fbatch, struct folio *folio, move_fn_t move_fn) { - if (folio_batch_add(fbatch, folio) && !folio_test_large(folio) && - !lru_cache_disabled()) + if (folio_batch_add(fbatch, folio) && + folio_may_be_lru_cached(folio) && !lru_cache_disabled()) return; folio_batch_move_lru(fbatch, move_fn); } @@ -501,6 +511,7 @@ void folio_add_lru(struct folio *folio) folio_test_unevictable(folio), folio); VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + trace_android_vh_folio_add_lru(folio); /* see the comment in lru_gen_add_folio() */ if (lru_gen_enabled() && !folio_test_unevictable(folio) && lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) { @@ -726,6 +737,7 @@ void folio_deactivate(struct folio *folio) local_unlock(&cpu_fbatches.lock); } } +EXPORT_SYMBOL_GPL(folio_deactivate); /** * folio_mark_lazyfree - make an anon folio lazyfree @@ -973,12 +985,11 @@ EXPORT_SYMBOL_GPL(lru_cache_disable); */ void folios_put_refs(struct folio_batch *folios, unsigned int *refs) { - int i; - LIST_HEAD(pages_to_free); + int i, j; struct lruvec *lruvec = NULL; unsigned long flags = 0; - for (i = 0; i < folios->nr; i++) { + for (i = 0, j = 0; i < folios->nr; i++) { struct folio *folio = folios->folios[i]; unsigned int nr_refs = refs ? refs[i] : 1; @@ -1010,21 +1021,23 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs) continue; } - if (folio_test_lru(folio)) { - lruvec = folio_lruvec_relock_irqsave(folio, lruvec, - &flags); - lruvec_del_folio(lruvec, folio); - __folio_clear_lru_flags(folio); - } + folio_unqueue_deferred_split(folio); + __page_cache_release(folio, &lruvec, &flags); - list_add(&folio->lru, &pages_to_free); + if (j != i) + folios->folios[j] = folio; + j++; } if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); + if (!j) { + folio_batch_reinit(folios); + return; + } - mem_cgroup_uncharge_list(&pages_to_free); - free_unref_page_list(&pages_to_free); - folio_batch_reinit(folios); + folios->nr = j; + mem_cgroup_uncharge_folios(folios); + free_unref_folios(folios); } EXPORT_SYMBOL(folios_put_refs); diff --git a/mm/swapfile.c b/mm/swapfile.c index 57c9051a23b0c..5a8c7cd45dfcf 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1531,6 +1531,7 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry) percpu_ref_put(&si->users); return NULL; } +EXPORT_SYMBOL_GPL(get_swap_device); static unsigned char __swap_entry_free(struct swap_info_struct *p, swp_entry_t entry) diff --git a/mm/truncate.c b/mm/truncate.c index 299fd6690a610..32d3d54fa88e1 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -200,6 +200,31 @@ int truncate_inode_folio(struct address_space *mapping, struct folio *folio) return 0; } +static int try_folio_split_or_unmap(struct folio *folio) +{ + enum ttu_flags ttu_flags = + TTU_SYNC | + TTU_SPLIT_HUGE_PMD | + TTU_IGNORE_MLOCK; + int ret; + + ret = split_folio(folio); + + /* + * If the split fails, unmap the folio, so it will be refaulted + * with PTEs to respect SIGBUS semantics. + * + * Make an exception for shmem/tmpfs that for long time + * intentionally mapped with PMDs across i_size. + */ + if (ret && !shmem_mapping(folio->mapping)) { + try_to_unmap(folio, ttu_flags); + WARN_ON(folio_mapped(folio)); + } + + return ret; +} + /* * Handle partial folios. The folio may be entirely within the * range if a split has raced with us. If not, we zero the part of the @@ -245,7 +270,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end) folio_invalidate(folio, offset, length); if (!folio_test_large(folio)) return true; - err = split_folio(folio); + err = try_folio_split_or_unmap(folio); if (!err) return true; if (err > 0) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 9a67c63ec874b..222b9b45275f4 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1020,17 +1020,59 @@ void double_pt_unlock(spinlock_t *ptl1, } -static int move_present_pte(struct mm_struct *mm, - struct vm_area_struct *dst_vma, - struct vm_area_struct *src_vma, - unsigned long dst_addr, unsigned long src_addr, - pte_t *dst_pte, pte_t *src_pte, - pte_t orig_dst_pte, pte_t orig_src_pte, - spinlock_t *dst_ptl, spinlock_t *src_ptl, - struct folio *src_folio) +/* + * Checks if the two ptes and the corresponding folio are eligible for batched + * move. If so, then returns pointer to the locked folio. Otherwise, returns NULL. + * + * NOTE: folio's reference is not required as the whole operation is within + * PTL's critical section. + */ +static struct folio *check_ptes_for_batched_move(struct vm_area_struct *src_vma, + unsigned long src_addr, + pte_t *src_pte, pte_t *dst_pte) +{ + pte_t orig_dst_pte, orig_src_pte; + struct folio *folio; + + orig_dst_pte = ptep_get(dst_pte); + if (!pte_none(orig_dst_pte)) + return NULL; + + orig_src_pte = ptep_get(src_pte); + if (!pte_present(orig_src_pte) || is_zero_pfn(pte_pfn(orig_src_pte))) + return NULL; + + folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); + if (!folio || !folio_trylock(folio)) + return NULL; + if (!PageAnonExclusive(&folio->page) || folio_test_large(folio)) { + folio_unlock(folio); + return NULL; + } + return folio; +} + +/* + * Moves src folios to dst in a batch as long as they are not large, and can + * successfully take the lock via folio_trylock(). + */ +static long move_present_ptes(struct mm_struct *mm, + struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, + pte_t orig_dst_pte, pte_t orig_src_pte, + spinlock_t *dst_ptl, spinlock_t *src_ptl, + struct folio **first_src_folio, unsigned long len) { int err = 0; + struct folio *src_folio = *first_src_folio; + unsigned long src_start = src_addr; + unsigned long src_end; + len = pmd_addr_end(dst_addr, dst_addr + len) - dst_addr; + src_end = pmd_addr_end(src_addr, src_addr + len); + flush_cache_range(src_vma, src_addr, src_end); double_pt_lock(dst_ptl, src_ptl); if (!pte_same(ptep_get(src_pte), orig_src_pte) || @@ -1044,31 +1086,56 @@ static int move_present_pte(struct mm_struct *mm, err = -EBUSY; goto out; } + /* It's safe to drop the reference now as the page-table is holding one. */ + folio_put(*first_src_folio); + *first_src_folio = NULL; + arch_enter_lazy_mmu_mode(); + + while (true) { + orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); + /* Folio got pinned from under us. Put it back and fail the move. */ + if (folio_maybe_dma_pinned(src_folio)) { + set_pte_at(mm, src_addr, src_pte, orig_src_pte); + err = -EBUSY; + break; + } - orig_src_pte = ptep_clear_flush(src_vma, src_addr, src_pte); - /* Folio got pinned from under us. Put it back and fail the move. */ - if (folio_maybe_dma_pinned(src_folio)) { - set_pte_at(mm, src_addr, src_pte, orig_src_pte); - err = -EBUSY; - goto out; - } - - folio_move_anon_rmap(src_folio, dst_vma); - WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + folio_move_anon_rmap(src_folio, dst_vma); + WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); - orig_dst_pte = mk_pte(&src_folio->page, dst_vma->vm_page_prot); - /* Set soft dirty bit so userspace can notice the pte was moved */ + orig_dst_pte = mk_pte(&src_folio->page, dst_vma->vm_page_prot); + /* Set soft dirty bit so userspace can notice the pte was moved */ #ifdef CONFIG_MEM_SOFT_DIRTY - orig_dst_pte = pte_mksoft_dirty(orig_dst_pte); + orig_dst_pte = pte_mksoft_dirty(orig_dst_pte); #endif - if (pte_dirty(orig_src_pte)) - orig_dst_pte = pte_mkdirty(orig_dst_pte); - orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma); + if (pte_dirty(orig_src_pte)) + orig_dst_pte = pte_mkdirty(orig_dst_pte); + orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma); + set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); + + src_addr += PAGE_SIZE; + if (src_addr == src_end) + break; + dst_addr += PAGE_SIZE; + dst_pte++; + src_pte++; + + folio_unlock(src_folio); + src_folio = check_ptes_for_batched_move(src_vma, src_addr, + src_pte, dst_pte); + if (!src_folio) + break; + } - set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); + arch_leave_lazy_mmu_mode(); + if (src_addr > src_start) + flush_tlb_range(src_vma, src_start, src_addr); + + if (src_folio) + folio_unlock(src_folio); out: double_pt_unlock(dst_ptl, src_ptl); - return err; + return src_addr > src_start ? src_addr - src_start : err; } static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, @@ -1076,8 +1143,18 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst_pte, pte_t orig_src_pte, spinlock_t *dst_ptl, spinlock_t *src_ptl, - struct folio *src_folio) + struct folio *src_folio, + struct swap_info_struct *si, swp_entry_t entry) { + /* + * Check if the folio still belongs to the target swap entry after + * acquiring the lock. Folio can be freed in the swap cache while + * not locked. + */ + if (src_folio && unlikely(!folio_test_swapcache(src_folio) || + entry.val != src_folio->swap.val)) + return -EAGAIN; + double_pt_lock(dst_ptl, src_ptl); if (!pte_same(ptep_get(src_pte), orig_src_pte) || @@ -1094,6 +1171,25 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, if (src_folio) { folio_move_anon_rmap(src_folio, dst_vma); src_folio->index = linear_page_index(dst_vma, dst_addr); + } else { + /* + * Check if the swap entry is cached after acquiring the src_pte + * lock. Otherwise, we might miss a newly loaded swap cache folio. + * + * Check swap_map directly to minimize overhead, READ_ONCE is sufficient. + * We are trying to catch newly added swap cache, the only possible case is + * when a folio is swapped in and out again staying in swap cache, using the + * same entry before the PTE check above. The PTL is acquired and released + * twice, each time after updating the swap_map's flag. So holding + * the PTL here ensures we see the updated value. False positive is possible, + * e.g. SWP_SYNCHRONOUS_IO swapin may set the flag without touching the + * cache, or during the tiny synchronization window between swap cache and + * swap_map, but it will be gone very quickly, worst result is retry jitters. + */ + if (READ_ONCE(si->swap_map[swp_offset(entry)]) & SWAP_HAS_CACHE) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } } orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); @@ -1103,7 +1199,7 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); double_pt_unlock(dst_ptl, src_ptl); - return 0; + return PAGE_SIZE; } static int move_zeropage_pte(struct mm_struct *mm, @@ -1129,20 +1225,20 @@ static int move_zeropage_pte(struct mm_struct *mm, set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); - return 0; + return PAGE_SIZE; } /* - * The mmap_lock for reading is held by the caller. Just move the page - * from src_pmd to dst_pmd if possible, and return true if succeeded - * in moving the page. + * The mmap_lock for reading is held by the caller. Just move the page(s) + * from src_pmd to dst_pmd if possible, and return number of bytes moved. + * On failure, an error code is returned. */ -static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, - struct vm_area_struct *dst_vma, - struct vm_area_struct *src_vma, - unsigned long dst_addr, unsigned long src_addr, - __u64 mode) +static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, + struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + unsigned long len, __u64 mode) { swp_entry_t entry; struct swap_info_struct *si = NULL; @@ -1153,20 +1249,18 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pte_t *dst_pte = NULL; struct folio *src_folio = NULL; - struct anon_vma *src_anon_vma = NULL; struct mmu_notifier_range range; - int err = 0; + long ret = 0; - flush_cache_range(src_vma, src_addr, src_addr + PAGE_SIZE); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - src_addr, src_addr + PAGE_SIZE); + src_addr, src_addr + len); mmu_notifier_invalidate_range_start(&range); retry: dst_pte = pte_offset_map_nolock(mm, dst_pmd, dst_addr, &dst_ptl); /* Retry if a huge pmd materialized from under us */ if (unlikely(!dst_pte)) { - err = -EAGAIN; + ret = -EAGAIN; goto out; } @@ -1179,14 +1273,14 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, * transparent huge pages under us. */ if (unlikely(!src_pte)) { - err = -EAGAIN; + ret = -EAGAIN; goto out; } /* Sanity checks before the operation */ if (WARN_ON_ONCE(pmd_none(*dst_pmd)) || WARN_ON_ONCE(pmd_none(*src_pmd)) || WARN_ON_ONCE(pmd_trans_huge(*dst_pmd)) || WARN_ON_ONCE(pmd_trans_huge(*src_pmd))) { - err = -EINVAL; + ret = -EINVAL; goto out; } @@ -1194,7 +1288,7 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, orig_dst_pte = ptep_get(dst_pte); spin_unlock(dst_ptl); if (!pte_none(orig_dst_pte)) { - err = -EEXIST; + ret = -EEXIST; goto out; } @@ -1203,21 +1297,21 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, spin_unlock(src_ptl); if (pte_none(orig_src_pte)) { if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) - err = -ENOENT; + ret = -ENOENT; else /* nothing to do to move a hole */ - err = 0; + ret = PAGE_SIZE; goto out; } /* If PTE changed after we locked the folio them start over */ if (src_folio && unlikely(!pte_same(src_folio_pte, orig_src_pte))) { - err = -EAGAIN; + ret = -EAGAIN; goto out; } if (pte_present(orig_src_pte)) { if (is_zero_pfn(pte_pfn(orig_src_pte))) { - err = move_zeropage_pte(mm, dst_vma, src_vma, + ret = move_zeropage_pte(mm, dst_vma, src_vma, dst_addr, src_addr, dst_pte, src_pte, orig_dst_pte, orig_src_pte, dst_ptl, src_ptl); @@ -1225,9 +1319,9 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, } /* - * Pin and lock both source folio and anon_vma. Since we are in - * RCU read section, we can't block, so on contention have to - * unmap the ptes, obtain the lock and retry. + * Pin and lock source folio. Since we are in RCU read section, + * we can't block, so on contention have to unmap the ptes, + * obtain the lock and retry. */ if (!src_folio) { struct folio *folio; @@ -1240,14 +1334,14 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, spin_lock(src_ptl); if (!pte_same(orig_src_pte, ptep_get(src_pte))) { spin_unlock(src_ptl); - err = -EAGAIN; + ret = -EAGAIN; goto out; } folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); if (!folio || !PageAnonExclusive(&folio->page)) { spin_unlock(src_ptl); - err = -EBUSY; + ret = -EBUSY; goto out; } @@ -1261,7 +1355,7 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, */ if (!locked && folio_test_large(folio)) { spin_unlock(src_ptl); - err = -EAGAIN; + ret = -EAGAIN; goto out; } @@ -1280,7 +1374,7 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, } if (WARN_ON_ONCE(!folio_test_anon(src_folio))) { - err = -EBUSY; + ret = -EBUSY; goto out; } } @@ -1291,8 +1385,8 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pte_unmap(src_pte); pte_unmap(dst_pte); src_pte = dst_pte = NULL; - err = split_folio(src_folio); - if (err < 0) + ret = split_folio(src_folio); + if (ret < 0) goto out; /* have to reacquire the folio after it got split */ folio_unlock(src_folio); @@ -1301,32 +1395,11 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, goto retry; } - if (!src_anon_vma) { - /* - * folio_referenced walks the anon_vma chain - * without the folio lock. Serialize against it with - * the anon_vma lock, the folio lock is not enough. - */ - src_anon_vma = folio_get_anon_vma(src_folio); - if (!src_anon_vma) { - /* page was unmapped from under us */ - err = -EAGAIN; - goto out; - } - if (!anon_vma_trylock_write(src_anon_vma)) { - pte_unmap(src_pte); - pte_unmap(dst_pte); - src_pte = dst_pte = NULL; - /* now we can block and wait */ - anon_vma_lock_write(src_anon_vma); - goto retry; - } - } - - err = move_present_pte(mm, dst_vma, src_vma, - dst_addr, src_addr, dst_pte, src_pte, - orig_dst_pte, orig_src_pte, - dst_ptl, src_ptl, src_folio); + ret = move_present_ptes(mm, dst_vma, src_vma, + dst_addr, src_addr, dst_pte, src_pte, + orig_dst_pte, orig_src_pte, + dst_ptl, src_ptl, &src_folio, + len); } else { struct folio *folio = NULL; @@ -1337,20 +1410,20 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pte_unmap(dst_pte); src_pte = dst_pte = NULL; migration_entry_wait(mm, src_pmd, src_addr); - err = -EAGAIN; + ret = -EAGAIN; } else - err = -EFAULT; + ret = -EFAULT; goto out; } if (!pte_swp_exclusive(orig_src_pte)) { - err = -EBUSY; + ret = -EBUSY; goto out; } si = get_swap_device(entry); if (unlikely(!si)) { - err = -EAGAIN; + ret = -EAGAIN; goto out; } /* @@ -1370,7 +1443,7 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, swp_offset(entry)); if (!IS_ERR_OR_NULL(folio)) { if (folio_test_large(folio)) { - err = -EBUSY; + ret = -EBUSY; folio_put(folio); goto out; } @@ -1387,28 +1460,30 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, goto retry; } } - err = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, - orig_dst_pte, orig_src_pte, dst_ptl, src_ptl, src_folio); + ret = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, + orig_dst_pte, orig_src_pte, + dst_ptl, src_ptl, src_folio, si, entry); } out: - if (src_anon_vma) { - anon_vma_unlock_write(src_anon_vma); - put_anon_vma(src_anon_vma); - } if (src_folio) { folio_unlock(src_folio); folio_put(src_folio); } - if (dst_pte) - pte_unmap(dst_pte); + /* + * Unmap in reverse order (LIFO) to maintain proper kmap_local + * index ordering when CONFIG_HIGHPTE is enabled. We mapped dst_pte + * first, then src_pte, so we must unmap src_pte first, then dst_pte. + */ if (src_pte) pte_unmap(src_pte); + if (dst_pte) + pte_unmap(dst_pte); mmu_notifier_invalidate_range_end(&range); if (si) put_swap_device(si); - return err; + return ret; } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1656,22 +1731,13 @@ static void uffd_move_unlock(struct vm_area_struct *dst_vma, * virtual regions without knowing if there are transparent hugepage * in the regions or not, but preventing the risk of having to split * the hugepmd during the remap. - * - * If there's any rmap walk that is taking the anon_vma locks without - * first obtaining the folio lock (the only current instance is - * folio_referenced), they will have to verify if the folio->mapping - * has changed after taking the anon_vma lock. If it changed they - * should release the lock and retry obtaining a new anon_vma, because - * it means the anon_vma was changed by move_pages() before the lock - * could be obtained. This is the only additional complexity added to - * the rmap code to provide this anonymous page remapping functionality. */ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long src_start, unsigned long len, __u64 mode) { struct mm_struct *mm = ctx->mm; struct vm_area_struct *src_vma, *dst_vma; - unsigned long src_addr, dst_addr; + unsigned long src_addr, dst_addr, src_end; pmd_t *src_pmd, *dst_pmd; long err = -EINVAL; ssize_t moved = 0; @@ -1716,8 +1782,8 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, if (err) goto out_unlock; - for (src_addr = src_start, dst_addr = dst_start; - src_addr < src_start + len;) { + for (src_addr = src_start, dst_addr = dst_start, src_end = src_start + len; + src_addr < src_end;) { spinlock_t *ptl; pmd_t dst_pmdval; unsigned long step_size; @@ -1768,13 +1834,16 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, /* Check if we can move the pmd without splitting it. */ if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) || !pmd_none(dst_pmdval)) { - struct folio *folio = pfn_folio(pmd_pfn(*src_pmd)); - - if (!folio || (!is_huge_zero_page(&folio->page) && - !PageAnonExclusive(&folio->page))) { - spin_unlock(ptl); - err = -EBUSY; - break; + /* Can be a migration entry */ + if (pmd_present(*src_pmd)) { + struct folio *folio = pfn_folio(pmd_pfn(*src_pmd)); + + if (!is_huge_zero_page(&folio->page) && + !PageAnonExclusive(&folio->page)) { + spin_unlock(ptl); + err = -EBUSY; + break; + } } spin_unlock(ptl); @@ -1788,6 +1857,8 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, dst_addr, src_addr); step_size = HPAGE_PMD_SIZE; } else { + long ret; + if (pmd_none(*src_pmd)) { if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) { err = -ENOENT; @@ -1804,10 +1875,13 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, break; } - err = move_pages_pte(mm, dst_pmd, src_pmd, - dst_vma, src_vma, - dst_addr, src_addr, mode); - step_size = PAGE_SIZE; + ret = move_pages_ptes(mm, dst_pmd, src_pmd, + dst_vma, src_vma, dst_addr, + src_addr, src_end - src_addr, mode); + if (ret < 0) + err = ret; + else + step_size = ret; } cond_resched(); diff --git a/mm/vmalloc.c b/mm/vmalloc.c old mode 100644 new mode 100755 index f5288b6936be8..3af41e32ce50e --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -468,6 +468,7 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, pgtbl_mod_mask *mask) { + int err = 0; pte_t *pte; /* @@ -481,18 +482,25 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, do { struct page *page = pages[*nr]; - if (WARN_ON(!pte_none(ptep_get(pte)))) - return -EBUSY; - if (WARN_ON(!page)) - return -ENOMEM; - if (WARN_ON(!pfn_valid(page_to_pfn(page)))) - return -EINVAL; + if (WARN_ON(!pte_none(ptep_get(pte)))) { + err = -EBUSY; + break; + } + if (WARN_ON(!page)) { + err = -ENOMEM; + break; + } + if (WARN_ON(!pfn_valid(page_to_pfn(page)))) { + err = -EINVAL; + break; + } set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); (*nr)++; } while (pte++, addr += PAGE_SIZE, addr != end); *mask |= PGTBL_PTE_MODIFIED; - return 0; + + return err; } static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, diff --git a/mm/vmscan.c b/mm/vmscan.c index 4c62b503788b7..367549bafdf1b 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -104,6 +104,11 @@ struct scan_control { unsigned long anon_cost; unsigned long file_cost; +#ifdef CONFIG_MEMCG + /* Swappiness value for proactive reclaim. Always use sc_swappiness()! */ + int *proactive_swappiness; +#endif + /* Can active folios be deactivated as part of reclaim? */ #define DEACTIVATE_ANON 1 #define DEACTIVATE_FILE 2 @@ -196,7 +201,7 @@ struct scan_control { #endif /* - * From 0 .. 200. Higher means more swappy. + * From 0 .. MAX_SWAPPINESS. Higher means more swappy. */ int vm_swappiness = 60; @@ -479,6 +484,13 @@ static bool writeback_throttling_sane(struct scan_control *sc) #endif return false; } + +static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) +{ + if (sc->proactive && sc->proactive_swappiness) + return *sc->proactive_swappiness; + return mem_cgroup_swappiness(memcg); +} #else static int prealloc_memcg_shrinker(struct shrinker *shrinker) { @@ -515,6 +527,11 @@ static bool writeback_throttling_sane(struct scan_control *sc) { return true; } + +static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) +{ + return READ_ONCE(vm_swappiness); +} #endif static void set_task_reclaim_state(struct task_struct *task, @@ -1412,6 +1429,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, BUG_ON(!folio_test_locked(folio)); BUG_ON(mapping != folio_mapping(folio)); + trace_android_vh_remove_mapping(mapping, folio, reclaimed); if (!folio_test_swapcache(folio)) spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); @@ -1504,6 +1522,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, xa_unlock_irq(&mapping->i_pages); if (!folio_test_swapcache(folio)) spin_unlock(&mapping->host->i_lock); + trace_android_vh_remove_mapping_failed(mapping, folio, reclaimed); return 0; } @@ -1983,6 +2002,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (nr_pages >= HPAGE_PMD_NR) { count_vm_event(THP_SWPOUT_FALLBACK); + count_memcg_folio_events(folio, THP_SWPOUT_FALLBACK, 1); } count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK); #endif @@ -2390,6 +2410,12 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan, unsigned long skipped = 0; unsigned long scan, total_scan, nr_pages; LIST_HEAD(folios_skipped); + unsigned long nr_scanned_before = *nr_scanned; + + trace_android_vh_mm_isolate_priv_lru(nr_to_scan, lruvec, lru, dst, sc->reclaim_idx, + sc->may_unmap, nr_scanned, &nr_taken); + if (*nr_scanned != nr_scanned_before) + return nr_taken; total_scan = 0; scan = 0; @@ -2647,7 +2673,7 @@ unsigned long shrink_inactive_list(unsigned long nr_to_scan, enum lru_list lru) { LIST_HEAD(folio_list); - unsigned long nr_scanned; + unsigned long nr_scanned = 0; unsigned int nr_reclaimed = 0; unsigned long nr_taken; struct reclaim_stat stat; @@ -2770,7 +2796,7 @@ static void shrink_active_list(unsigned long nr_to_scan, enum lru_list lru) { unsigned long nr_taken; - unsigned long nr_scanned; + unsigned long nr_scanned = 0; unsigned long vm_flags; LIST_HEAD(l_hold); /* The folios which were snipped off */ LIST_HEAD(l_active); @@ -3014,6 +3040,15 @@ static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) return inactive * inactive_ratio < active; } +static void customize_sc_file_is_tiny(struct scan_control *sc) +{ + bool file_is_tiny = sc->file_is_tiny; + + trace_android_vh_mm_customize_file_is_tiny(sc->may_swap, sc->order, + sc->reclaim_idx, &file_is_tiny); + sc->file_is_tiny = file_is_tiny; +} + enum scan_balance { SCAN_EQUAL, SCAN_FRACT, @@ -3131,6 +3166,8 @@ static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) !(sc->may_deactivate & DEACTIVATE_ANON) && anon >> sc->priority; } + + customize_sc_file_is_tiny(sc); } /* @@ -3146,7 +3183,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, struct pglist_data *pgdat = lruvec_pgdat(lruvec); struct mem_cgroup *memcg = lruvec_memcg(lruvec); unsigned long anon_cost, file_cost, total_cost; - int swappiness = mem_cgroup_swappiness(memcg); + int swappiness = sc_swappiness(sc, memcg); u64 fraction[ANON_AND_FILE]; u64 denominator = 0; /* gcc */ enum scan_balance scan_balance; @@ -3229,7 +3266,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, ap = swappiness * (total_cost + 1); ap /= anon_cost + 1; - fp = (200 - swappiness) * (total_cost + 1); + fp = (MAX_SWAPPINESS - swappiness) * (total_cost + 1); fp /= file_cost + 1; fraction[0] = ap; @@ -3436,7 +3473,7 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) return 0; - swappiness = mem_cgroup_swappiness(memcg); + swappiness = sc_swappiness(sc, memcg); trace_android_vh_tune_swappiness(&swappiness); return swappiness; @@ -5268,7 +5305,7 @@ static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx { int type, tier; struct ctrl_pos sp, pv; - int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; + int gain[ANON_AND_FILE] = { swappiness, MAX_SWAPPINESS - swappiness }; /* * Compare the first tier of anon with that of file to determine which @@ -5312,7 +5349,7 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw type = LRU_GEN_ANON; else if (swappiness == 1) type = LRU_GEN_FILE; - else if (swappiness == 200) + else if (swappiness == MAX_SWAPPINESS) type = LRU_GEN_ANON; else type = get_type_to_scan(lruvec, swappiness, &tier); @@ -6265,9 +6302,9 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, lruvec = get_lruvec(memcg, nid); - if (swappiness < 0) + if (swappiness < MIN_SWAPPINESS) swappiness = get_swappiness(lruvec, sc); - else if (swappiness > 200) + else if (swappiness > MAX_SWAPPINESS) goto done; switch (cmd) { @@ -6695,6 +6732,7 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) cond_resched(); trace_android_vh_shrink_node_memcgs(memcg, &skip); + trace_android_vh_should_memcg_bypass(memcg, sc->priority, &skip); if (skip) continue; @@ -7379,15 +7417,17 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, return sc.nr_reclaimed; } -unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - unsigned long nr_pages, - gfp_t gfp_mask, - unsigned int reclaim_options) +unsigned long try_to_free_mem_cgroup_pages_with_swappiness(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int *swappiness) { unsigned long nr_reclaimed; unsigned int noreclaim_flag; struct scan_control sc = { .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), + .proactive_swappiness = swappiness, .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), .reclaim_idx = MAX_NR_ZONES - 1, @@ -7417,6 +7457,15 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, return nr_reclaimed; } + +unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options) +{ + return try_to_free_mem_cgroup_pages_with_swappiness( + memcg, nr_pages, gfp_mask, reclaim_options, NULL); +} EXPORT_SYMBOL_GPL(try_to_free_mem_cgroup_pages); #endif @@ -7478,8 +7527,15 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) { int i; unsigned long mark = -1; + bool customized = false; + bool balanced = false; struct zone *zone; + trace_android_vh_mm_customize_pgdat_balanced(order, highest_zoneidx, + &balanced, &customized); + if (customized) + return balanced; + /* * Check watermarks bottom-up as lower zones are more likely to * meet watermarks. diff --git a/mm/vmstat.c b/mm/vmstat.c index bfea0422afe4c..e625291540427 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1401,6 +1401,7 @@ const char * const vmstat_text[] = { #ifdef CONFIG_ZSWAP "zswpin", "zswpout", + "zswpwb", #endif #ifdef CONFIG_X86 "direct_map_level2_splits", diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 4ef553e56dd57..e151f33b9dfd9 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -120,8 +120,6 @@ #define ISOLATED_BITS 5 #define MAGIC_VAL_BITS 8 -#define MAX(a, b) ((a) >= (b) ? (a) : (b)) - #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL)) /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */ @@ -989,6 +987,9 @@ static struct zspage *alloc_zspage(struct zs_pool *pool, if (!zspage) return NULL; + if (!IS_ENABLED(CONFIG_COMPACTION)) + gfp &= ~__GFP_MOVABLE; + zspage->magic = ZSPAGE_MAGIC; migrate_lock_init(zspage); diff --git a/mm/zswap.c b/mm/zswap.c index 8ad7c97fac191..ee9d64e02224b 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -680,6 +680,10 @@ static int zswap_reclaim_entry(struct zswap_pool *pool) goto put_unlock; } + if (entry->objcg) + count_objcg_event(entry->objcg, ZSWPWB); + + count_vm_event(ZSWPWB); /* * Writeback started successfully, the page now belongs to the * swapcache. Drop the entry from zswap - unless invalidate already diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index b477ba37a6991..4245c3dd8316c 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -194,6 +194,8 @@ int register_vlan_dev(struct net_device *dev, struct netlink_ext_ack *extack) vlan_group_set_device(grp, vlan->vlan_proto, vlan_id, dev); grp->nr_vlan_devs++; + netdev_update_features(dev); + return 0; out_unregister_netdev: diff --git a/net/9p/trans_fd.c b/net/9p/trans_fd.c index 0432bfa73888b..e22512cdf8df5 100644 --- a/net/9p/trans_fd.c +++ b/net/9p/trans_fd.c @@ -719,10 +719,10 @@ static int p9_fd_cancelled(struct p9_client *client, struct p9_req_t *req) p9_debug(P9_DEBUG_TRANS, "client %p req %p\n", client, req); spin_lock(&m->req_lock); - /* Ignore cancelled request if message has been received - * before lock. - */ - if (req->status == REQ_STATUS_RCVD) { + /* Ignore cancelled request if status changed since the request was + * processed in p9_client_flush() + */ + if (req->status != REQ_STATUS_SENT) { spin_unlock(&m->req_lock); return 0; } diff --git a/net/Kconfig b/net/Kconfig index d532ec33f1fed..d94b3cf303f49 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -289,15 +289,21 @@ config MAX_SKB_FRAGS If unsure, say 17. config RPS - bool + bool "Receive packet steering" depends on SMP && SYSFS default y + help + Software receive side packet steering (RPS) distributes the + load of received packet processing across multiple CPUs. config RFS_ACCEL - bool + bool "Hardware acceleration of RFS" depends on RPS select CPU_RMAP default y + help + Allowing drivers for multiqueue hardware with flow filter tables to + accelerate RFS. config SOCK_RX_QUEUE_MAPPING bool @@ -333,6 +339,7 @@ config NET_RX_BUSY_POLL config BQL bool + prompt "Enable Byte Queue Limits" depends on SYSFS select DQL default y @@ -349,7 +356,7 @@ config BPF_STREAM_PARSER BPF_MAP_TYPE_SOCKMAP. config NET_FLOW_LIMIT - bool + bool "Net flow limit" depends on RPS default y help diff --git a/net/appletalk/aarp.c b/net/appletalk/aarp.c index c7236daa24152..0d7c14a496681 100644 --- a/net/appletalk/aarp.c +++ b/net/appletalk/aarp.c @@ -35,6 +35,7 @@ #include #include #include +#include int sysctl_aarp_expiry_time = AARP_EXPIRY_TIME; int sysctl_aarp_tick_time = AARP_TICK_TIME; @@ -44,6 +45,7 @@ int sysctl_aarp_resolve_time = AARP_RESOLVE_TIME; /* Lists of aarp entries */ /** * struct aarp_entry - AARP entry + * @refcnt: Reference count * @last_sent: Last time we xmitted the aarp request * @packet_queue: Queue of frames wait for resolution * @status: Used for proxy AARP @@ -55,6 +57,7 @@ int sysctl_aarp_resolve_time = AARP_RESOLVE_TIME; * @next: Next entry in chain */ struct aarp_entry { + refcount_t refcnt; /* These first two are only used for unresolved entries */ unsigned long last_sent; struct sk_buff_head packet_queue; @@ -79,6 +82,17 @@ static DEFINE_RWLOCK(aarp_lock); /* Used to walk the list and purge/kick entries. */ static struct timer_list aarp_timer; +static inline void aarp_entry_get(struct aarp_entry *a) +{ + refcount_inc(&a->refcnt); +} + +static inline void aarp_entry_put(struct aarp_entry *a) +{ + if (refcount_dec_and_test(&a->refcnt)) + kfree(a); +} + /* * Delete an aarp queue * @@ -87,7 +101,7 @@ static struct timer_list aarp_timer; static void __aarp_expire(struct aarp_entry *a) { skb_queue_purge(&a->packet_queue); - kfree(a); + aarp_entry_put(a); } /* @@ -380,9 +394,11 @@ static void aarp_purge(void) static struct aarp_entry *aarp_alloc(void) { struct aarp_entry *a = kmalloc(sizeof(*a), GFP_ATOMIC); + if (!a) + return NULL; - if (a) - skb_queue_head_init(&a->packet_queue); + refcount_set(&a->refcnt, 1); + skb_queue_head_init(&a->packet_queue); return a; } @@ -508,6 +524,7 @@ int aarp_proxy_probe_network(struct atalk_iface *atif, struct atalk_addr *sa) entry->dev = atif->dev; write_lock_bh(&aarp_lock); + aarp_entry_get(entry); hash = sa->s_node % (AARP_HASH_SIZE - 1); entry->next = proxies[hash]; @@ -533,6 +550,7 @@ int aarp_proxy_probe_network(struct atalk_iface *atif, struct atalk_addr *sa) retval = 1; } + aarp_entry_put(entry); write_unlock_bh(&aarp_lock); out: return retval; diff --git a/net/appletalk/ddp.c b/net/appletalk/ddp.c index b070a89912000..febb1617e1a6a 100644 --- a/net/appletalk/ddp.c +++ b/net/appletalk/ddp.c @@ -563,6 +563,7 @@ static int atrtr_create(struct rtentry *r, struct net_device *devhint) /* Fill in the routing entry */ rt->target = ta->sat_addr; + dev_put(rt->dev); /* Release old device */ dev_hold(devhint); rt->dev = devhint; rt->flags = r->rt_flags; diff --git a/net/atm/clip.c b/net/atm/clip.c index 294cb9efe3d38..53d62361ae460 100644 --- a/net/atm/clip.c +++ b/net/atm/clip.c @@ -45,7 +45,8 @@ #include static struct net_device *clip_devs; -static struct atm_vcc *atmarpd; +static struct atm_vcc __rcu *atmarpd; +static DEFINE_MUTEX(atmarpd_lock); static struct timer_list idle_timer; static const struct neigh_ops clip_neigh_ops; @@ -53,24 +54,35 @@ static int to_atmarpd(enum atmarp_ctrl_type type, int itf, __be32 ip) { struct sock *sk; struct atmarp_ctrl *ctrl; + struct atm_vcc *vcc; struct sk_buff *skb; + int err = 0; pr_debug("(%d)\n", type); - if (!atmarpd) - return -EUNATCH; + + rcu_read_lock(); + vcc = rcu_dereference(atmarpd); + if (!vcc) { + err = -EUNATCH; + goto unlock; + } skb = alloc_skb(sizeof(struct atmarp_ctrl), GFP_ATOMIC); - if (!skb) - return -ENOMEM; + if (!skb) { + err = -ENOMEM; + goto unlock; + } ctrl = skb_put(skb, sizeof(struct atmarp_ctrl)); ctrl->type = type; ctrl->itf_num = itf; ctrl->ip = ip; - atm_force_charge(atmarpd, skb->truesize); + atm_force_charge(vcc, skb->truesize); - sk = sk_atm(atmarpd); + sk = sk_atm(vcc); skb_queue_tail(&sk->sk_receive_queue, skb); sk->sk_data_ready(sk); - return 0; +unlock: + rcu_read_unlock(); + return err; } static void link_vcc(struct clip_vcc *clip_vcc, struct atmarp_entry *entry) @@ -193,12 +205,6 @@ static void clip_push(struct atm_vcc *vcc, struct sk_buff *skb) pr_debug("\n"); - if (!clip_devs) { - atm_return(vcc, skb->truesize); - kfree_skb(skb); - return; - } - if (!skb) { pr_debug("removing VCC %p\n", clip_vcc); if (clip_vcc->entry) @@ -208,6 +214,11 @@ static void clip_push(struct atm_vcc *vcc, struct sk_buff *skb) return; } atm_return(vcc, skb->truesize); + if (!clip_devs) { + kfree_skb(skb); + return; + } + skb->dev = clip_vcc->entry ? clip_vcc->entry->neigh->dev : clip_devs; /* clip_vcc->entry == NULL if we don't have an IP address yet */ if (!skb->dev) { @@ -418,6 +429,8 @@ static int clip_mkip(struct atm_vcc *vcc, int timeout) if (!vcc->push) return -EBADFD; + if (vcc->user_back) + return -EINVAL; clip_vcc = kmalloc(sizeof(struct clip_vcc), GFP_KERNEL); if (!clip_vcc) return -ENOMEM; @@ -608,17 +621,27 @@ static void atmarpd_close(struct atm_vcc *vcc) { pr_debug("\n"); - rtnl_lock(); - atmarpd = NULL; + mutex_lock(&atmarpd_lock); + RCU_INIT_POINTER(atmarpd, NULL); + mutex_unlock(&atmarpd_lock); + + synchronize_rcu(); skb_queue_purge(&sk_atm(vcc)->sk_receive_queue); - rtnl_unlock(); pr_debug("(done)\n"); module_put(THIS_MODULE); } +static int atmarpd_send(struct atm_vcc *vcc, struct sk_buff *skb) +{ + atm_return_tx(vcc, skb); + dev_kfree_skb_any(skb); + return 0; +} + static const struct atmdev_ops atmarpd_dev_ops = { - .close = atmarpd_close + .close = atmarpd_close, + .send = atmarpd_send }; @@ -632,15 +655,18 @@ static struct atm_dev atmarpd_dev = { static int atm_init_atmarp(struct atm_vcc *vcc) { - rtnl_lock(); + if (vcc->push == clip_push) + return -EINVAL; + + mutex_lock(&atmarpd_lock); if (atmarpd) { - rtnl_unlock(); + mutex_unlock(&atmarpd_lock); return -EADDRINUSE; } mod_timer(&idle_timer, jiffies + CLIP_CHECK_INTERVAL * HZ); - atmarpd = vcc; + rcu_assign_pointer(atmarpd, vcc); set_bit(ATM_VF_META, &vcc->flags); set_bit(ATM_VF_READY, &vcc->flags); /* allow replies and avoid getting closed if signaling dies */ @@ -649,13 +675,14 @@ static int atm_init_atmarp(struct atm_vcc *vcc) vcc->push = NULL; vcc->pop = NULL; /* crash */ vcc->push_oam = NULL; /* crash */ - rtnl_unlock(); + mutex_unlock(&atmarpd_lock); return 0; } static int clip_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) { struct atm_vcc *vcc = ATM_SD(sock); + struct sock *sk = sock->sk; int err = 0; switch (cmd) { @@ -676,14 +703,18 @@ static int clip_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) err = clip_create(arg); break; case ATMARPD_CTRL: + lock_sock(sk); err = atm_init_atmarp(vcc); if (!err) { sock->state = SS_CONNECTED; __module_get(THIS_MODULE); } + release_sock(sk); break; case ATMARP_MKIP: + lock_sock(sk); err = clip_mkip(vcc, arg); + release_sock(sk); break; case ATMARP_SETENTRY: err = clip_setentry(vcc, (__force __be32)arg); diff --git a/net/atm/common.c b/net/atm/common.c index f7019df41c3e5..48bb3f66a3f2a 100644 --- a/net/atm/common.c +++ b/net/atm/common.c @@ -635,17 +635,27 @@ int vcc_sendmsg(struct socket *sock, struct msghdr *m, size_t size) skb->dev = NULL; /* for paths shared with net_device interfaces */ if (!copy_from_iter_full(skb_put(skb, size), size, &m->msg_iter)) { - kfree_skb(skb); error = -EFAULT; - goto out; + goto free_skb; } if (eff != size) memset(skb->data + size, 0, eff-size); + + if (vcc->dev->ops->pre_send) { + error = vcc->dev->ops->pre_send(vcc, skb); + if (error) + goto free_skb; + } + error = vcc->dev->ops->send(vcc, skb); error = error ? error : size; out: release_sock(sk); return error; +free_skb: + atm_return_tx(vcc, skb); + kfree_skb(skb); + goto out; } __poll_t vcc_poll(struct file *file, struct socket *sock, poll_table *wait) diff --git a/net/atm/lec.c b/net/atm/lec.c index ac3cfc1ae5102..b7fa48a9b7205 100644 --- a/net/atm/lec.c +++ b/net/atm/lec.c @@ -124,6 +124,7 @@ static unsigned char bus_mac[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; /* Device structures */ static struct net_device *dev_lec[MAX_LEC_ITF]; +static DEFINE_MUTEX(lec_mutex); #if IS_ENABLED(CONFIG_BRIDGE) static void lec_handle_bridge(struct sk_buff *skb, struct net_device *dev) @@ -685,6 +686,7 @@ static int lec_vcc_attach(struct atm_vcc *vcc, void __user *arg) int bytes_left; struct atmlec_ioc ioc_data; + lockdep_assert_held(&lec_mutex); /* Lecd must be up in this case */ bytes_left = copy_from_user(&ioc_data, arg, sizeof(struct atmlec_ioc)); if (bytes_left != 0) @@ -710,6 +712,7 @@ static int lec_vcc_attach(struct atm_vcc *vcc, void __user *arg) static int lec_mcast_attach(struct atm_vcc *vcc, int arg) { + lockdep_assert_held(&lec_mutex); if (arg < 0 || arg >= MAX_LEC_ITF) return -EINVAL; arg = array_index_nospec(arg, MAX_LEC_ITF); @@ -725,6 +728,7 @@ static int lecd_attach(struct atm_vcc *vcc, int arg) int i; struct lec_priv *priv; + lockdep_assert_held(&lec_mutex); if (arg < 0) arg = 0; if (arg >= MAX_LEC_ITF) @@ -742,6 +746,7 @@ static int lecd_attach(struct atm_vcc *vcc, int arg) snprintf(dev_lec[i]->name, IFNAMSIZ, "lec%d", i); if (register_netdev(dev_lec[i])) { free_netdev(dev_lec[i]); + dev_lec[i] = NULL; return -EINVAL; } @@ -904,7 +909,6 @@ static void *lec_itf_walk(struct lec_state *state, loff_t *l) v = (dev && netdev_priv(dev)) ? lec_priv_walk(state, l, netdev_priv(dev)) : NULL; if (!v && dev) { - dev_put(dev); /* Partial state reset for the next time we get called */ dev = NULL; } @@ -928,6 +932,7 @@ static void *lec_seq_start(struct seq_file *seq, loff_t *pos) { struct lec_state *state = seq->private; + mutex_lock(&lec_mutex); state->itf = 0; state->dev = NULL; state->locked = NULL; @@ -945,8 +950,9 @@ static void lec_seq_stop(struct seq_file *seq, void *v) if (state->dev) { spin_unlock_irqrestore(&state->locked->lec_arp_lock, state->flags); - dev_put(state->dev); + state->dev = NULL; } + mutex_unlock(&lec_mutex); } static void *lec_seq_next(struct seq_file *seq, void *v, loff_t *pos) @@ -1003,6 +1009,7 @@ static int lane_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) return -ENOIOCTLCMD; } + mutex_lock(&lec_mutex); switch (cmd) { case ATMLEC_CTRL: err = lecd_attach(vcc, (int)arg); @@ -1017,6 +1024,7 @@ static int lane_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) break; } + mutex_unlock(&lec_mutex); return err; } diff --git a/net/atm/raw.c b/net/atm/raw.c index 2b5f78a7ec3e4..1e6511ec842cb 100644 --- a/net/atm/raw.c +++ b/net/atm/raw.c @@ -36,7 +36,7 @@ static void atm_pop_raw(struct atm_vcc *vcc, struct sk_buff *skb) pr_debug("(%d) %d -= %d\n", vcc->vci, sk_wmem_alloc_get(sk), ATM_SKB(skb)->acct_truesize); - WARN_ON(refcount_sub_and_test(ATM_SKB(skb)->acct_truesize, &sk->sk_wmem_alloc)); + atm_return_tx(vcc, skb); dev_kfree_skb_any(skb); sk->sk_write_space(sk); } diff --git a/net/atm/resources.c b/net/atm/resources.c index 995d29e7fb138..7c6fdedbcf4e5 100644 --- a/net/atm/resources.c +++ b/net/atm/resources.c @@ -112,7 +112,9 @@ struct atm_dev *atm_dev_register(const char *type, struct device *parent, if (atm_proc_dev_register(dev) < 0) { pr_err("atm_proc_dev_register failed for dev %s\n", type); - goto out_fail; + mutex_unlock(&atm_dev_mutex); + kfree(dev); + return NULL; } if (atm_register_sysfs(dev, parent) < 0) { @@ -128,7 +130,7 @@ struct atm_dev *atm_dev_register(const char *type, struct device *parent, return dev; out_fail: - kfree(dev); + put_device(&dev->class_dev); dev = NULL; goto out; } @@ -146,11 +148,10 @@ void atm_dev_deregister(struct atm_dev *dev) */ mutex_lock(&atm_dev_mutex); list_del(&dev->dev_list); - mutex_unlock(&atm_dev_mutex); - atm_dev_release_vccs(dev); atm_unregister_sysfs(dev); atm_proc_dev_deregister(dev); + mutex_unlock(&atm_dev_mutex); atm_dev_put(dev); } diff --git a/net/ax25/ax25_in.c b/net/ax25/ax25_in.c index 1cac25aca6378..f2d66af863595 100644 --- a/net/ax25/ax25_in.c +++ b/net/ax25/ax25_in.c @@ -433,6 +433,10 @@ static int ax25_rcv(struct sk_buff *skb, struct net_device *dev, int ax25_kiss_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *ptype, struct net_device *orig_dev) { + skb = skb_share_check(skb, GFP_ATOMIC); + if (!skb) + return NET_RX_DROP; + skb_orphan(skb); if (!net_eq(dev_net(dev), &init_net)) { diff --git a/net/batman-adv/network-coding.c b/net/batman-adv/network-coding.c index 71ebd0284f95d..0adc783fb83ca 100644 --- a/net/batman-adv/network-coding.c +++ b/net/batman-adv/network-coding.c @@ -1687,7 +1687,12 @@ batadv_nc_skb_decode_packet(struct batadv_priv *bat_priv, struct sk_buff *skb, coding_len = ntohs(coded_packet_tmp.coded_len); - if (coding_len > skb->len) + /* ensure dst buffer is large enough (payload only) */ + if (coding_len + h_size > skb->len) + return NULL; + + /* ensure src buffer is large enough (payload only) */ + if (coding_len + h_size > nc_packet->skb->len) return NULL; /* Here the magic is reversed: diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index 13b752c169bed..e65d4754c94f4 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -52,6 +52,11 @@ static bool enable_6lowpan; static struct l2cap_chan *listen_chan; static DEFINE_MUTEX(set_lock); +enum { + LOWPAN_PEER_CLOSING, + LOWPAN_PEER_MAXBITS +}; + struct lowpan_peer { struct list_head list; struct rcu_head rcu; @@ -60,6 +65,8 @@ struct lowpan_peer { /* peer addresses in various formats */ unsigned char lladdr[ETH_ALEN]; struct in6_addr peer_addr; + + DECLARE_BITMAP(flags, LOWPAN_PEER_MAXBITS); }; struct lowpan_btle_dev { @@ -288,6 +295,7 @@ static int recv_pkt(struct sk_buff *skb, struct net_device *dev, local_skb->pkt_type = PACKET_HOST; local_skb->dev = dev; + skb_reset_mac_header(local_skb); skb_set_transport_header(local_skb, sizeof(struct ipv6hdr)); if (give_skb_to_upper(local_skb, dev) != NET_RX_SUCCESS) { @@ -955,10 +963,11 @@ static struct l2cap_chan *bt_6lowpan_listen(void) } static int get_l2cap_conn(char *buf, bdaddr_t *addr, u8 *addr_type, - struct l2cap_conn **conn) + struct l2cap_conn **conn, bool disconnect) { struct hci_conn *hcon; struct hci_dev *hdev; + int le_addr_type; int n; n = sscanf(buf, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx %hhu", @@ -969,13 +978,32 @@ static int get_l2cap_conn(char *buf, bdaddr_t *addr, u8 *addr_type, if (n < 7) return -EINVAL; + if (disconnect) { + /* The "disconnect" debugfs command has used different address + * type constants than "connect" since 2015. Let's retain that + * for now even though it's obviously buggy... + */ + *addr_type += 1; + } + + switch (*addr_type) { + case BDADDR_LE_PUBLIC: + le_addr_type = ADDR_LE_DEV_PUBLIC; + break; + case BDADDR_LE_RANDOM: + le_addr_type = ADDR_LE_DEV_RANDOM; + break; + default: + return -EINVAL; + } + /* The LE_PUBLIC address type is ignored because of BDADDR_ANY */ hdev = hci_get_route(addr, BDADDR_ANY, BDADDR_LE_PUBLIC); if (!hdev) return -ENOENT; hci_dev_lock(hdev); - hcon = hci_conn_hash_lookup_le(hdev, addr, *addr_type); + hcon = hci_conn_hash_lookup_le(hdev, addr, le_addr_type); hci_dev_unlock(hdev); hci_dev_put(hdev); @@ -992,41 +1020,52 @@ static int get_l2cap_conn(char *buf, bdaddr_t *addr, u8 *addr_type, static void disconnect_all_peers(void) { struct lowpan_btle_dev *entry; - struct lowpan_peer *peer, *tmp_peer, *new_peer; - struct list_head peers; - - INIT_LIST_HEAD(&peers); + struct lowpan_peer *peer; + int nchans; - /* We make a separate list of peers as the close_cb() will - * modify the device peers list so it is better not to mess - * with the same list at the same time. + /* l2cap_chan_close() cannot be called from RCU, and lock ordering + * chan->lock > devices_lock prevents taking write side lock, so copy + * then close. */ rcu_read_lock(); + list_for_each_entry_rcu(entry, &bt_6lowpan_devices, list) + list_for_each_entry_rcu(peer, &entry->peers, list) + clear_bit(LOWPAN_PEER_CLOSING, peer->flags); + rcu_read_unlock(); - list_for_each_entry_rcu(entry, &bt_6lowpan_devices, list) { - list_for_each_entry_rcu(peer, &entry->peers, list) { - new_peer = kmalloc(sizeof(*new_peer), GFP_ATOMIC); - if (!new_peer) - break; + do { + struct l2cap_chan *chans[32]; + int i; - new_peer->chan = peer->chan; - INIT_LIST_HEAD(&new_peer->list); + nchans = 0; - list_add(&new_peer->list, &peers); - } - } + spin_lock(&devices_lock); - rcu_read_unlock(); + list_for_each_entry_rcu(entry, &bt_6lowpan_devices, list) { + list_for_each_entry_rcu(peer, &entry->peers, list) { + if (test_and_set_bit(LOWPAN_PEER_CLOSING, + peer->flags)) + continue; - spin_lock(&devices_lock); - list_for_each_entry_safe(peer, tmp_peer, &peers, list) { - l2cap_chan_close(peer->chan, ENOENT); + l2cap_chan_hold(peer->chan); + chans[nchans++] = peer->chan; - list_del_rcu(&peer->list); - kfree_rcu(peer, rcu); - } - spin_unlock(&devices_lock); + if (nchans >= ARRAY_SIZE(chans)) + goto done; + } + } + +done: + spin_unlock(&devices_lock); + + for (i = 0; i < nchans; ++i) { + l2cap_chan_lock(chans[i]); + l2cap_chan_close(chans[i], ENOENT); + l2cap_chan_unlock(chans[i]); + l2cap_chan_put(chans[i]); + } + } while (nchans); } struct set_enable { @@ -1102,7 +1141,7 @@ static ssize_t lowpan_control_write(struct file *fp, buf[buf_size] = '\0'; if (memcmp(buf, "connect ", 8) == 0) { - ret = get_l2cap_conn(&buf[8], &addr, &addr_type, &conn); + ret = get_l2cap_conn(&buf[8], &addr, &addr_type, &conn, false); if (ret == -EINVAL) return ret; @@ -1139,7 +1178,7 @@ static ssize_t lowpan_control_write(struct file *fp, } if (memcmp(buf, "disconnect ", 11) == 0) { - ret = get_l2cap_conn(&buf[11], &addr, &addr_type, &conn); + ret = get_l2cap_conn(&buf[11], &addr, &addr_type, &conn, true); if (ret < 0) return ret; diff --git a/net/bluetooth/eir.c b/net/bluetooth/eir.c index 1bc51e2b05a34..3e1713673ecc9 100644 --- a/net/bluetooth/eir.c +++ b/net/bluetooth/eir.c @@ -366,17 +366,19 @@ u8 eir_create_scan_rsp(struct hci_dev *hdev, u8 instance, u8 *ptr) void *eir_get_service_data(u8 *eir, size_t eir_len, u16 uuid, size_t *len) { - while ((eir = eir_get_data(eir, eir_len, EIR_SERVICE_DATA, len))) { + size_t dlen; + + while ((eir = eir_get_data(eir, eir_len, EIR_SERVICE_DATA, &dlen))) { u16 value = get_unaligned_le16(eir); if (uuid == value) { if (len) - *len -= 2; + *len = dlen - 2; return &eir[2]; } - eir += *len; - eir_len -= *len; + eir += dlen; + eir_len -= dlen; } return NULL; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 549ee9e87d636..ff9d2520ba749 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -339,7 +339,8 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) case BT_CODEC_TRANSPARENT: if (!find_next_esco_param(conn, esco_param_msbc, ARRAY_SIZE(esco_param_msbc))) - return false; + return -EINVAL; + param = &esco_param_msbc[conn->attempt - 1]; cp.tx_coding_format.id = 0x03; cp.rx_coding_format.id = 0x03; diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 30519d47e8a69..023ad47a385bf 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -1862,10 +1862,8 @@ void hci_free_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor) if (monitor->handle) idr_remove(&hdev->adv_monitors_idr, monitor->handle); - if (monitor->state != ADV_MONITOR_STATE_NOT_REGISTERED) { + if (monitor->state != ADV_MONITOR_STATE_NOT_REGISTERED) hdev->adv_monitors_cnt--; - mgmt_adv_monitor_removed(hdev, monitor->handle); - } kfree(monitor); } @@ -3380,23 +3378,18 @@ static void hci_link_tx_to(struct hci_dev *hdev, __u8 type) bt_dev_err(hdev, "link tx timeout"); - rcu_read_lock(); + hci_dev_lock(hdev); /* Kill stalled connections */ - list_for_each_entry_rcu(c, &h->list, list) { + list_for_each_entry(c, &h->list, list) { if (c->type == type && c->sent) { bt_dev_err(hdev, "killing stalled connection %pMR", &c->dst); - /* hci_disconnect might sleep, so, we have to release - * the RCU read lock before calling it. - */ - rcu_read_unlock(); hci_disconnect(c, HCI_ERROR_REMOTE_USER_TERM); - rcu_read_lock(); } } - rcu_read_unlock(); + hci_dev_unlock(hdev); } static struct hci_chan *hci_chan_sent(struct hci_dev *hdev, __u8 type, diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 4029330e29a99..9356410eb4b80 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -1596,7 +1596,7 @@ static u8 hci_cc_le_set_ext_adv_enable(struct hci_dev *hdev, void *data, hci_dev_set_flag(hdev, HCI_LE_ADV); - if (adv && !adv->periodic) + if (adv) adv->enabled = true; conn = hci_lookup_le_connect(hdev); @@ -2139,40 +2139,6 @@ static u8 hci_cc_set_adv_param(struct hci_dev *hdev, void *data, return rp->status; } -static u8 hci_cc_set_ext_adv_param(struct hci_dev *hdev, void *data, - struct sk_buff *skb) -{ - struct hci_rp_le_set_ext_adv_params *rp = data; - struct hci_cp_le_set_ext_adv_params *cp; - struct adv_info *adv_instance; - - bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); - - if (rp->status) - return rp->status; - - cp = hci_sent_cmd_data(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS); - if (!cp) - return rp->status; - - hci_dev_lock(hdev); - hdev->adv_addr_type = cp->own_addr_type; - if (!cp->handle) { - /* Store in hdev for instance 0 */ - hdev->adv_tx_power = rp->tx_power; - } else { - adv_instance = hci_find_adv_instance(hdev, cp->handle); - if (adv_instance) - adv_instance->tx_power = rp->tx_power; - } - /* Update adv data as tx power is known now */ - hci_update_adv_data(hdev, cp->handle); - - hci_dev_unlock(hdev); - - return rp->status; -} - static u8 hci_cc_read_rssi(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -2726,7 +2692,7 @@ static void hci_cs_disconnect(struct hci_dev *hdev, u8 status) if (!conn) goto unlock; - if (status) { + if (status && status != HCI_ERROR_UNKNOWN_CONN_ID) { mgmt_disconnect_failed(hdev, &conn->dst, conn->type, conn->dst_type, status); @@ -2741,6 +2707,12 @@ static void hci_cs_disconnect(struct hci_dev *hdev, u8 status) goto done; } + /* During suspend, mark connection as closed immediately + * since we might not receive HCI_EV_DISCONN_COMPLETE + */ + if (hdev->suspended) + conn->state = BT_CLOSED; + mgmt_conn = test_and_clear_bit(HCI_CONN_MGMT_CONNECTED, &conn->flags); if (conn->type == ACL_LINK) { @@ -3076,8 +3048,18 @@ static void hci_conn_complete_evt(struct hci_dev *hdev, void *data, hci_dev_lock(hdev); + /* Check for existing connection: + * + * 1. If it doesn't exist then it must be receiver/slave role. + * 2. If it does exist confirm that it is connecting/BT_CONNECT in case + * of initiator/master role since there could be a collision where + * either side is attempting to connect or something like a fuzzing + * testing is trying to play tricks to destroy the hcon object before + * it even attempts to connect (e.g. hcon->state == BT_OPEN). + */ conn = hci_conn_hash_lookup_ba(hdev, ev->link_type, &ev->bdaddr); - if (!conn) { + if (!conn || + (conn->role == HCI_ROLE_MASTER && conn->state != BT_CONNECT)) { /* In case of error status and there is no connection pending * just unlock as there is nothing to cleanup. */ @@ -3967,8 +3949,11 @@ static u8 hci_cc_le_set_per_adv_enable(struct hci_dev *hdev, void *data, hci_dev_set_flag(hdev, HCI_LE_PER_ADV); if (adv) - adv->enabled = true; + adv->periodic_enabled = true; } else { + if (adv) + adv->periodic_enabled = false; + /* If just one instance was disabled check if there are * any other instance enabled before clearing HCI_LE_PER_ADV. * The current periodic adv instance will be marked as @@ -4153,8 +4138,6 @@ static const struct hci_cc { HCI_CC(HCI_OP_LE_READ_NUM_SUPPORTED_ADV_SETS, hci_cc_le_read_num_adv_sets, sizeof(struct hci_rp_le_read_num_supported_adv_sets)), - HCI_CC(HCI_OP_LE_SET_EXT_ADV_PARAMS, hci_cc_set_ext_adv_param, - sizeof(struct hci_rp_le_set_ext_adv_params)), HCI_CC_STATUS(HCI_OP_LE_SET_EXT_ADV_ENABLE, hci_cc_le_set_ext_adv_enable), HCI_CC_STATUS(HCI_OP_LE_SET_ADV_SET_RAND_ADDR, @@ -4221,6 +4204,13 @@ static void hci_cmd_complete_evt(struct hci_dev *hdev, void *data, } if (i == ARRAY_SIZE(hci_cc_table)) { + if (!skb->len) { + bt_dev_err(hdev, "Unexpected cc 0x%4.4x with no status", + *opcode); + *status = HCI_ERROR_UNSPECIFIED; + return; + } + /* Unknown opcode, assume byte 0 contains the status, so * that e.g. __hci_cmd_sync() properly returns errors * for vendor specific commands send by HCI drivers. @@ -4422,7 +4412,17 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data, if (!conn) continue; - conn->sent -= count; + /* Check if there is really enough packets outstanding before + * attempting to decrease the sent counter otherwise it could + * underflow.. + */ + if (conn->sent >= count) { + conn->sent -= count; + } else { + bt_dev_warn(hdev, "hcon %p sent %u < count %u", + conn, conn->sent, count); + conn->sent = 0; + } switch (conn->type) { case ACL_LINK: @@ -5635,8 +5635,18 @@ static void le_conn_complete_evt(struct hci_dev *hdev, u8 status, */ hci_dev_clear_flag(hdev, HCI_LE_ADV); - conn = hci_conn_hash_lookup_ba(hdev, LE_LINK, bdaddr); - if (!conn) { + /* Check for existing connection: + * + * 1. If it doesn't exist then use the role to create a new object. + * 2. If it does exist confirm that it is connecting/BT_CONNECT in case + * of initiator/master role since there could be a collision where + * either side is attempting to connect or something like a fuzzing + * testing is trying to play tricks to destroy the hcon object before + * it even attempts to connect (e.g. hcon->state == BT_OPEN). + */ + conn = hci_conn_hash_lookup_role(hdev, LE_LINK, role, bdaddr); + if (!conn || + (conn->role == HCI_ROLE_MASTER && conn->state != BT_CONNECT)) { /* In case of error status and there is no connection pending * just unlock as there is nothing to cleanup. */ @@ -6252,6 +6262,11 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, void *data, static u8 ext_evt_type_to_legacy(struct hci_dev *hdev, u16 evt_type) { + u16 pdu_type = evt_type & ~LE_EXT_ADV_DATA_STATUS_MASK; + + if (!pdu_type) + return LE_ADV_NONCONN_IND; + if (evt_type & LE_EXT_ADV_LEGACY_PDU) { switch (evt_type) { case LE_LEGACY_ADV_IND: @@ -6283,8 +6298,7 @@ static u8 ext_evt_type_to_legacy(struct hci_dev *hdev, u16 evt_type) if (evt_type & LE_EXT_ADV_SCAN_IND) return LE_ADV_SCAN_IND; - if (evt_type == LE_EXT_ADV_NON_CONN_IND || - evt_type & LE_EXT_ADV_DIRECT_IND) + if (evt_type & LE_EXT_ADV_DIRECT_IND) return LE_ADV_NONCONN_IND; invalid: @@ -6696,8 +6710,8 @@ static void hci_le_cis_estabilished_evt(struct hci_dev *hdev, void *data, qos->ucast.out.latency = DIV_ROUND_CLOSEST(get_unaligned_le24(ev->p_latency), 1000); - qos->ucast.in.sdu = le16_to_cpu(ev->c_mtu); - qos->ucast.out.sdu = le16_to_cpu(ev->p_mtu); + qos->ucast.in.sdu = ev->c_bn ? le16_to_cpu(ev->c_mtu) : 0; + qos->ucast.out.sdu = ev->p_bn ? le16_to_cpu(ev->p_mtu) : 0; qos->ucast.in.phy = ev->c_phy; qos->ucast.out.phy = ev->p_phy; break; @@ -6711,8 +6725,8 @@ static void hci_le_cis_estabilished_evt(struct hci_dev *hdev, void *data, qos->ucast.in.latency = DIV_ROUND_CLOSEST(get_unaligned_le24(ev->p_latency), 1000); - qos->ucast.out.sdu = le16_to_cpu(ev->c_mtu); - qos->ucast.in.sdu = le16_to_cpu(ev->p_mtu); + qos->ucast.out.sdu = ev->c_bn ? le16_to_cpu(ev->c_mtu) : 0; + qos->ucast.in.sdu = ev->p_bn ? le16_to_cpu(ev->p_mtu) : 0; qos->ucast.out.phy = ev->c_phy; qos->ucast.in.phy = ev->p_phy; break; @@ -6916,7 +6930,10 @@ static void hci_le_big_sync_established_evt(struct hci_dev *hdev, void *data, bis->iso_qos.bcast.in.sdu = le16_to_cpu(ev->max_pdu); if (!ev->status) { + bis->state = BT_CONNECTED; set_bit(HCI_CONN_BIG_SYNC, &bis->flags); + hci_debugfs_create_conn(bis); + hci_conn_add_sysfs(bis); hci_iso_setup_path(bis); } } diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 69c2ba1e843eb..d2613bd3e6db0 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -118,7 +118,7 @@ static void hci_sock_free_cookie(struct sock *sk) int id = hci_pi(sk)->cookie; if (id) { - hci_pi(sk)->cookie = 0xffffffff; + hci_pi(sk)->cookie = 0; ida_free(&sock_cookie_ida, id); } } diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index d6f40806ee512..985ca780a6694 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -881,11 +881,17 @@ bool hci_cmd_sync_dequeue_once(struct hci_dev *hdev, { struct hci_cmd_sync_work_entry *entry; - entry = hci_cmd_sync_lookup_entry(hdev, func, data, destroy); - if (!entry) + mutex_lock(&hdev->cmd_sync_work_lock); + + entry = _hci_cmd_sync_lookup_entry(hdev, func, data, destroy); + if (!entry) { + mutex_unlock(&hdev->cmd_sync_work_lock); return false; + } - hci_cmd_sync_cancel_entry(hdev, entry); + _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); + + mutex_unlock(&hdev->cmd_sync_work_lock); return true; } @@ -1224,10 +1230,130 @@ static int hci_set_adv_set_random_addr_sync(struct hci_dev *hdev, u8 instance, sizeof(cp), &cp, HCI_CMD_TIMEOUT); } +static int +hci_set_ext_adv_params_sync(struct hci_dev *hdev, struct adv_info *adv, + const struct hci_cp_le_set_ext_adv_params *cp, + struct hci_rp_le_set_ext_adv_params *rp) +{ + struct sk_buff *skb; + + skb = __hci_cmd_sync(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS, sizeof(*cp), + cp, HCI_CMD_TIMEOUT); + + /* If command return a status event, skb will be set to -ENODATA */ + if (skb == ERR_PTR(-ENODATA)) + return 0; + + if (IS_ERR(skb)) { + bt_dev_err(hdev, "Opcode 0x%4.4x failed: %ld", + HCI_OP_LE_SET_EXT_ADV_PARAMS, PTR_ERR(skb)); + return PTR_ERR(skb); + } + + if (skb->len != sizeof(*rp)) { + bt_dev_err(hdev, "Invalid response length for 0x%4.4x: %u", + HCI_OP_LE_SET_EXT_ADV_PARAMS, skb->len); + kfree_skb(skb); + return -EIO; + } + + memcpy(rp, skb->data, sizeof(*rp)); + kfree_skb(skb); + + if (!rp->status) { + hdev->adv_addr_type = cp->own_addr_type; + if (!cp->handle) { + /* Store in hdev for instance 0 */ + hdev->adv_tx_power = rp->tx_power; + } else if (adv) { + adv->tx_power = rp->tx_power; + } + } + + return rp->status; +} + +static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) +{ + struct { + struct hci_cp_le_set_ext_adv_data cp; + u8 data[HCI_MAX_EXT_AD_LENGTH]; + } pdu; + u8 len; + struct adv_info *adv = NULL; + int err; + + memset(&pdu, 0, sizeof(pdu)); + + if (instance) { + adv = hci_find_adv_instance(hdev, instance); + if (!adv || !adv->adv_data_changed) + return 0; + } + + len = eir_create_adv_data(hdev, instance, pdu.data); + + pdu.cp.length = len; + pdu.cp.handle = instance; + pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; + pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; + + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA, + sizeof(pdu.cp) + len, &pdu.cp, + HCI_CMD_TIMEOUT); + if (err) + return err; + + /* Update data if the command succeed */ + if (adv) { + adv->adv_data_changed = false; + } else { + memcpy(hdev->adv_data, pdu.data, len); + hdev->adv_data_len = len; + } + + return 0; +} + +static int hci_set_adv_data_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_adv_data cp; + u8 len; + + memset(&cp, 0, sizeof(cp)); + + len = eir_create_adv_data(hdev, instance, cp.data); + + /* There's nothing to do if the data hasn't changed */ + if (hdev->adv_data_len == len && + memcmp(cp.data, hdev->adv_data, len) == 0) + return 0; + + memcpy(hdev->adv_data, cp.data, sizeof(cp.data)); + hdev->adv_data_len = len; + + cp.length = len; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_DATA, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance) +{ + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + return 0; + + if (ext_adv_capable(hdev)) + return hci_set_ext_adv_data_sync(hdev, instance); + + return hci_set_adv_data_sync(hdev, instance); +} + int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) { struct hci_cp_le_set_ext_adv_params cp; - bool connectable; + struct hci_rp_le_set_ext_adv_params rp; + bool connectable, require_privacy; u32 flags; bdaddr_t random_addr; u8 own_addr_type; @@ -1247,7 +1373,7 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) * Command Disallowed error, so we must first disable the * instance if it is active. */ - if (adv && !adv->pending) { + if (adv) { err = hci_disable_ext_adv_instance_sync(hdev, instance); if (err) return err; @@ -1265,10 +1391,12 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) return -EPERM; /* Set require_privacy to true only when non-connectable - * advertising is used. In that case it is fine to use a - * non-resolvable private address. + * advertising is used and it is not periodic. + * In that case it is fine to use a non-resolvable private address. */ - err = hci_get_random_address(hdev, !connectable, + require_privacy = !connectable && !(adv && adv->periodic); + + err = hci_get_random_address(hdev, require_privacy, adv_use_rpa(hdev, flags), adv, &own_addr_type, &random_addr); if (err < 0) @@ -1333,8 +1461,12 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) cp.secondary_phy = HCI_ADV_PHY_1M; } - err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS, - sizeof(cp), &cp, HCI_CMD_TIMEOUT); + err = hci_set_ext_adv_params_sync(hdev, adv, &cp, &rp); + if (err) + return err; + + /* Update adv data as tx power is known now */ + err = hci_set_ext_adv_data_sync(hdev, cp.handle); if (err) return err; @@ -1499,7 +1631,7 @@ int hci_disable_per_advertising_sync(struct hci_dev *hdev, u8 instance) /* If periodic advertising already disabled there is nothing to do. */ adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->periodic || !adv->enabled) + if (!adv || !adv->periodic_enabled) return 0; memset(&cp, 0, sizeof(cp)); @@ -1568,7 +1700,7 @@ static int hci_enable_per_advertising_sync(struct hci_dev *hdev, u8 instance) /* If periodic advertising already enabled there is nothing to do. */ adv = hci_find_adv_instance(hdev, instance); - if (adv && adv->periodic && adv->enabled) + if (adv && adv->periodic_enabled) return 0; memset(&cp, 0, sizeof(cp)); @@ -1586,7 +1718,8 @@ static int hci_enable_per_advertising_sync(struct hci_dev *hdev, u8 instance) static int hci_adv_bcast_annoucement(struct hci_dev *hdev, struct adv_info *adv) { u8 bid[3]; - u8 ad[4 + 3]; + u8 ad[HCI_MAX_EXT_AD_LENGTH]; + u8 len; /* Skip if NULL adv as instance 0x00 is used for general purpose * advertising so it cannot used for the likes of Broadcast Announcement @@ -1612,8 +1745,10 @@ static int hci_adv_bcast_annoucement(struct hci_dev *hdev, struct adv_info *adv) /* Generate Broadcast ID */ get_random_bytes(bid, sizeof(bid)); - eir_append_service_data(ad, 0, 0x1852, bid, sizeof(bid)); - hci_set_adv_instance_data(hdev, adv->instance, sizeof(ad), ad, 0, NULL); + len = eir_append_service_data(ad, 0, 0x1852, bid, sizeof(bid)); + memcpy(ad + len, adv->adv_data, adv->adv_data_len); + hci_set_adv_instance_data(hdev, adv->instance, len + adv->adv_data_len, + ad, 0, NULL); return hci_update_adv_data_sync(hdev, adv->instance); } @@ -1630,8 +1765,15 @@ int hci_start_per_adv_sync(struct hci_dev *hdev, u8 instance, u8 data_len, if (instance) { adv = hci_find_adv_instance(hdev, instance); - /* Create an instance if that could not be found */ - if (!adv) { + if (adv) { + /* Turn it into periodic advertising */ + adv->periodic = true; + adv->per_adv_data_len = data_len; + if (data) + memcpy(adv->per_adv_data, data, data_len); + adv->flags = flags; + } else if (!adv) { + /* Create an instance if that could not be found */ adv = hci_add_per_instance(hdev, instance, flags, data_len, data, sync_interval, @@ -1849,82 +1991,6 @@ int hci_le_terminate_big_sync(struct hci_dev *hdev, u8 handle, u8 reason) sizeof(cp), &cp, HCI_CMD_TIMEOUT); } -static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) -{ - struct { - struct hci_cp_le_set_ext_adv_data cp; - u8 data[HCI_MAX_EXT_AD_LENGTH]; - } pdu; - u8 len; - struct adv_info *adv = NULL; - int err; - - memset(&pdu, 0, sizeof(pdu)); - - if (instance) { - adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->adv_data_changed) - return 0; - } - - len = eir_create_adv_data(hdev, instance, pdu.data); - - pdu.cp.length = len; - pdu.cp.handle = instance; - pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; - pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; - - err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA, - sizeof(pdu.cp) + len, &pdu.cp, - HCI_CMD_TIMEOUT); - if (err) - return err; - - /* Update data if the command succeed */ - if (adv) { - adv->adv_data_changed = false; - } else { - memcpy(hdev->adv_data, pdu.data, len); - hdev->adv_data_len = len; - } - - return 0; -} - -static int hci_set_adv_data_sync(struct hci_dev *hdev, u8 instance) -{ - struct hci_cp_le_set_adv_data cp; - u8 len; - - memset(&cp, 0, sizeof(cp)); - - len = eir_create_adv_data(hdev, instance, cp.data); - - /* There's nothing to do if the data hasn't changed */ - if (hdev->adv_data_len == len && - memcmp(cp.data, hdev->adv_data, len) == 0) - return 0; - - memcpy(hdev->adv_data, cp.data, sizeof(cp.data)); - hdev->adv_data_len = len; - - cp.length = len; - - return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_DATA, - sizeof(cp), &cp, HCI_CMD_TIMEOUT); -} - -int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance) -{ - if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) - return 0; - - if (ext_adv_capable(hdev)) - return hci_set_ext_adv_data_sync(hdev, instance); - - return hci_set_adv_data_sync(hdev, instance); -} - int hci_schedule_adv_instance_sync(struct hci_dev *hdev, u8 instance, bool force) { @@ -2000,13 +2066,10 @@ static int hci_clear_adv_sets_sync(struct hci_dev *hdev, struct sock *sk) static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force) { struct adv_info *adv, *n; - int err = 0; if (ext_adv_capable(hdev)) /* Remove all existing sets */ - err = hci_clear_adv_sets_sync(hdev, sk); - if (ext_adv_capable(hdev)) - return err; + return hci_clear_adv_sets_sync(hdev, sk); /* This is safe as long as there is no command send while the lock is * held. @@ -2034,13 +2097,11 @@ static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force) static int hci_remove_adv_sync(struct hci_dev *hdev, u8 instance, struct sock *sk) { - int err = 0; + int err; /* If we use extended advertising, instance has to be removed first. */ if (ext_adv_capable(hdev)) - err = hci_remove_ext_adv_instance_sync(hdev, instance, sk); - if (ext_adv_capable(hdev)) - return err; + return hci_remove_ext_adv_instance_sync(hdev, instance, sk); /* This is safe as long as there is no command send while the lock is * held. @@ -2139,16 +2200,13 @@ int hci_read_tx_power_sync(struct hci_dev *hdev, __le16 handle, u8 type) int hci_disable_advertising_sync(struct hci_dev *hdev) { u8 enable = 0x00; - int err = 0; /* If controller is not advertising we are done. */ if (!hci_dev_test_flag(hdev, HCI_LE_ADV)) return 0; if (ext_adv_capable(hdev)) - err = hci_disable_ext_adv_instance_sync(hdev, 0x00); - if (ext_adv_capable(hdev)) - return err; + return hci_disable_ext_adv_instance_sync(hdev, 0x00); return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE, sizeof(enable), &enable, HCI_CMD_TIMEOUT); @@ -2516,6 +2574,10 @@ static int hci_pause_advertising_sync(struct hci_dev *hdev) int err; int old_state; + /* If controller is not advertising we are done. */ + if (!hci_dev_test_flag(hdev, HCI_LE_ADV)) + return 0; + /* If already been paused there is nothing to do. */ if (hdev->advertising_paused) return 0; @@ -2585,6 +2647,13 @@ static int hci_resume_advertising_sync(struct hci_dev *hdev) hci_remove_ext_adv_instance_sync(hdev, adv->instance, NULL); } + + /* If current advertising instance is set to instance 0x00 + * then we need to re-enable it. + */ + if (!hdev->cur_adv_instance) + err = hci_enable_ext_advertising_sync(hdev, + hdev->cur_adv_instance); } else { /* Schedule for most recent instance to be restarted and begin * the software rotation loop @@ -3300,7 +3369,7 @@ static int hci_powered_update_adv_sync(struct hci_dev *hdev) * advertising data. This also applies to the case * where BR/EDR was toggled during the AUTO_OFF phase. */ - if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || + if (hci_dev_test_flag(hdev, HCI_ADVERTISING) && list_empty(&hdev->adv_instances)) { if (ext_adv_capable(hdev)) { err = hci_setup_ext_adv_instance_sync(hdev, 0x00); @@ -3437,13 +3506,13 @@ int hci_update_scan_sync(struct hci_dev *hdev) return hci_write_scan_enable_sync(hdev, scan); } -int hci_update_name_sync(struct hci_dev *hdev) +int hci_update_name_sync(struct hci_dev *hdev, const u8 *name) { struct hci_cp_write_local_name cp; memset(&cp, 0, sizeof(cp)); - memcpy(cp.name, hdev->dev_name, sizeof(cp.name)); + memcpy(cp.name, name, sizeof(cp.name)); return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_LOCAL_NAME, sizeof(cp), &cp, @@ -3496,7 +3565,7 @@ int hci_powered_update_sync(struct hci_dev *hdev) hci_write_fast_connectable_sync(hdev, false); hci_update_scan_sync(hdev); hci_update_class_sync(hdev); - hci_update_name_sync(hdev); + hci_update_name_sync(hdev, hdev->dev_name); hci_update_eir_sync(hdev); } @@ -6247,6 +6316,7 @@ static int hci_le_ext_directed_advertising_sync(struct hci_dev *hdev, struct hci_conn *conn) { struct hci_cp_le_set_ext_adv_params cp; + struct hci_rp_le_set_ext_adv_params rp; int err; bdaddr_t random_addr; u8 own_addr_type; @@ -6288,8 +6358,12 @@ static int hci_le_ext_directed_advertising_sync(struct hci_dev *hdev, if (err) return err; - err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS, - sizeof(cp), &cp, HCI_CMD_TIMEOUT); + err = hci_set_ext_adv_params_sync(hdev, NULL, &cp, &rp); + if (err) + return err; + + /* Update adv data as tx power is known now */ + err = hci_set_ext_adv_data_sync(hdev, cp.handle); if (err) return err; @@ -6737,8 +6811,8 @@ int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, return 0; } - /* No privacy so use a public address. */ - *own_addr_type = ADDR_LE_DEV_PUBLIC; + /* No privacy, use the current address */ + hci_copy_identity_address(hdev, rand_addr, own_addr_type); return 0; } diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index b94d202bf3745..1469e9b69e631 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -718,6 +718,13 @@ static void iso_sock_kill(struct sock *sk) BT_DBG("sk %p state %d", sk, sk->sk_state); + /* Sock is dead, so set conn->sk to NULL to avoid possible UAF */ + if (iso_pi(sk)->conn) { + iso_conn_lock(iso_pi(sk)->conn); + iso_pi(sk)->conn->sk = NULL; + iso_conn_unlock(iso_pi(sk)->conn); + } + /* Kill poor orphan */ bt_sock_unlink(&iso_sk_list, sk); sock_set_flag(sk, SOCK_DEAD); @@ -1775,7 +1782,13 @@ static void iso_conn_ready(struct iso_conn *conn) } bacpy(&iso_pi(sk)->dst, &hcon->dst); - iso_pi(sk)->dst_type = hcon->dst_type; + + /* Convert from HCI to three-value type */ + if (hcon->dst_type == ADDR_LE_DEV_PUBLIC) + iso_pi(sk)->dst_type = BDADDR_LE_PUBLIC; + else + iso_pi(sk)->dst_type = BDADDR_LE_RANDOM; + iso_pi(sk)->sync_handle = iso_pi(parent)->sync_handle; memcpy(iso_pi(sk)->base, iso_pi(parent)->base, iso_pi(parent)->base_len); iso_pi(sk)->base_len = iso_pi(parent)->base_len; @@ -2084,7 +2097,7 @@ void iso_recv(struct hci_conn *hcon, struct sk_buff *skb, u16 flags) skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), skb->len); conn->rx_len -= skb->len; - return; + break; case ISO_END: skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index d4dcdb2370cc9..9bbcae3edca29 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -497,6 +497,7 @@ void l2cap_chan_hold(struct l2cap_chan *c) kref_get(&c->kref); } +EXPORT_SYMBOL_GPL(l2cap_chan_hold); struct l2cap_chan *l2cap_chan_hold_unless_zero(struct l2cap_chan *c) { @@ -1411,7 +1412,8 @@ static void l2cap_request_info(struct l2cap_conn *conn) sizeof(req), &req); } -static bool l2cap_check_enc_key_size(struct hci_conn *hcon) +static bool l2cap_check_enc_key_size(struct hci_conn *hcon, + struct l2cap_chan *chan) { /* The minimum encryption key size needs to be enforced by the * host stack before establishing any L2CAP connections. The @@ -1425,7 +1427,7 @@ static bool l2cap_check_enc_key_size(struct hci_conn *hcon) int min_key_size = hcon->hdev->min_enc_key_size; /* On FIPS security level, key size must be 16 bytes */ - if (hcon->sec_level == BT_SECURITY_FIPS) + if (chan->sec_level == BT_SECURITY_FIPS) min_key_size = 16; return (!test_bit(HCI_CONN_ENCRYPT, &hcon->flags) || @@ -1453,7 +1455,7 @@ static void l2cap_do_start(struct l2cap_chan *chan) !__l2cap_no_conn_pending(chan)) return; - if (l2cap_check_enc_key_size(conn->hcon)) + if (l2cap_check_enc_key_size(conn->hcon, chan)) l2cap_start_connection(chan); else __set_chan_timer(chan, L2CAP_DISC_TIMEOUT); @@ -1528,7 +1530,7 @@ static void l2cap_conn_start(struct l2cap_conn *conn) continue; } - if (l2cap_check_enc_key_size(conn->hcon)) + if (l2cap_check_enc_key_size(conn->hcon, chan)) l2cap_start_connection(chan); else l2cap_chan_close(chan, ECONNREFUSED); @@ -3379,7 +3381,7 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data struct l2cap_conf_rfc rfc = { .mode = L2CAP_MODE_BASIC }; struct l2cap_conf_efs efs; u8 remote_efs = 0; - u16 mtu = L2CAP_DEFAULT_MTU; + u16 mtu = 0; u16 result = L2CAP_CONF_SUCCESS; u16 size; @@ -3484,6 +3486,29 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data /* Configure output options and let the other side know * which ones we don't like. */ + /* If MTU is not provided in configure request, try adjusting it + * to the current output MTU if it has been set + * + * Bluetooth Core 6.1, Vol 3, Part A, Section 4.5 + * + * Each configuration parameter value (if any is present) in an + * L2CAP_CONFIGURATION_RSP packet reflects an ‘adjustment’ to a + * configuration parameter value that has been sent (or, in case + * of default values, implied) in the corresponding + * L2CAP_CONFIGURATION_REQ packet. + */ + if (!mtu) { + /* Only adjust for ERTM channels as for older modes the + * remote stack may not be able to detect that the + * adjustment causing it to silently drop packets. + */ + if (chan->mode == L2CAP_MODE_ERTM && + chan->omtu && chan->omtu != L2CAP_DEFAULT_MTU) + mtu = chan->omtu; + else + mtu = L2CAP_DEFAULT_MTU; + } + if (mtu < L2CAP_DEFAULT_MIN_MTU) result = L2CAP_CONF_UNACCEPT; else { @@ -3955,7 +3980,7 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, /* Check if the ACL is secure enough (if not SDP) */ if (psm != cpu_to_le16(L2CAP_PSM_SDP) && (!hci_conn_check_link_mode(conn->hcon) || - !l2cap_check_enc_key_size(conn->hcon))) { + !l2cap_check_enc_key_size(conn->hcon, pchan))) { conn->disc_reason = HCI_ERROR_AUTH_FAILURE; result = L2CAP_CR_SEC_BLOCK; goto response; @@ -4832,7 +4857,8 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, if (!smp_sufficient_security(conn->hcon, pchan->sec_level, SMP_ALLOW_STK)) { - result = L2CAP_CR_LE_AUTHENTICATION; + result = pchan->sec_level == BT_SECURITY_MEDIUM ? + L2CAP_CR_LE_ENCRYPTION : L2CAP_CR_LE_AUTHENTICATION; chan = NULL; goto response_unlock; } @@ -7323,7 +7349,7 @@ static void l2cap_security_cfm(struct hci_conn *hcon, u8 status, u8 encrypt) } if (chan->state == BT_CONNECT) { - if (!status && l2cap_check_enc_key_size(hcon)) + if (!status && l2cap_check_enc_key_size(hcon, chan)) l2cap_start_connection(chan); else __set_chan_timer(chan, L2CAP_DISC_TIMEOUT); @@ -7333,7 +7359,7 @@ static void l2cap_security_cfm(struct hci_conn *hcon, u8 status, u8 encrypt) struct l2cap_conn_rsp rsp; __u16 res, stat; - if (!status && l2cap_check_enc_key_size(hcon)) { + if (!status && l2cap_check_enc_key_size(hcon, chan)) { if (test_bit(FLAG_DEFER_SETUP, &chan->flags)) { res = L2CAP_CR_PEND; stat = L2CAP_CS_AUTHOR_PEND; diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index aaaaf9733b589..59630dbeda20d 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -1406,7 +1406,10 @@ static int l2cap_sock_release(struct socket *sock) if (!sk) return 0; + lock_sock_nested(sk, L2CAP_NESTING_PARENT); l2cap_sock_cleanup_listen(sk); + release_sock(sk); + bt_sock_unlink(&l2cap_sk_list, sk); err = l2cap_sock_shutdown(sock, SHUT_RDWR); @@ -1687,6 +1690,9 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) { struct sock *sk = chan->data; + if (!sk) + return; + if (test_and_clear_bit(FLAG_PENDING_SECURITY, &chan->flags)) { sk->sk_state = BT_CONNECTED; chan->state = BT_CONNECTED; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 29e420e9754bb..de9b7517125ee 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -1074,7 +1074,8 @@ static int mesh_send_done_sync(struct hci_dev *hdev, void *data) struct mgmt_mesh_tx *mesh_tx; hci_dev_clear_flag(hdev, HCI_MESH_SENDING); - hci_disable_advertising_sync(hdev); + if (list_empty(&hdev->adv_instances)) + hci_disable_advertising_sync(hdev); mesh_tx = mgmt_mesh_next(hdev, NULL); if (mesh_tx) @@ -2147,6 +2148,9 @@ static int set_mesh_sync(struct hci_dev *hdev, void *data) else hci_dev_clear_flag(hdev, HCI_MESH); + hdev->le_scan_interval = __le16_to_cpu(cp->period); + hdev->le_scan_window = __le16_to_cpu(cp->window); + len -= sizeof(*cp); /* If filters don't fit, forward all adv pkts */ @@ -2161,6 +2165,7 @@ static int set_mesh(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_cp_set_mesh *cp = data; struct mgmt_pending_cmd *cmd; + __u16 period, window; int err = 0; bt_dev_dbg(hdev, "sock %p", sk); @@ -2174,6 +2179,23 @@ static int set_mesh(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, MGMT_STATUS_INVALID_PARAMS); + /* Keep allowed ranges in sync with set_scan_params() */ + period = __le16_to_cpu(cp->period); + + if (period < 0x0004 || period > 0x4000) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, + MGMT_STATUS_INVALID_PARAMS); + + window = __le16_to_cpu(cp->window); + + if (window < 0x0004 || window > 0x4000) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, + MGMT_STATUS_INVALID_PARAMS); + + if (window > period) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, + MGMT_STATUS_INVALID_PARAMS); + hci_dev_lock(hdev); cmd = mgmt_pending_add(sk, MGMT_OP_SET_MESH_RECEIVER, hdev, data, len); @@ -3804,8 +3826,11 @@ static void set_name_complete(struct hci_dev *hdev, void *data, int err) static int set_name_sync(struct hci_dev *hdev, void *data) { + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_set_local_name *cp = cmd->param; + if (lmp_bredr_capable(hdev)) { - hci_update_name_sync(hdev); + hci_update_name_sync(hdev, cp->name); hci_update_eir_sync(hdev); } @@ -4387,13 +4412,11 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, return -ENOMEM; #ifdef CONFIG_BT_FEATURE_DEBUG - if (!hdev) { - flags = bt_dbg_get() ? BIT(0) : 0; + flags = bt_dbg_get() ? BIT(0) : 0; - memcpy(rp->features[idx].uuid, debug_uuid, 16); - rp->features[idx].flags = cpu_to_le32(flags); - idx++; - } + memcpy(rp->features[idx].uuid, debug_uuid, 16); + rp->features[idx].flags = cpu_to_le32(flags); + idx++; #endif if (hdev && hci_dev_le_state_simultaneous(hdev)) { @@ -5165,24 +5188,14 @@ static void mgmt_adv_monitor_added(struct sock *sk, struct hci_dev *hdev, mgmt_event(MGMT_EV_ADV_MONITOR_ADDED, hdev, &ev, sizeof(ev), sk); } -void mgmt_adv_monitor_removed(struct hci_dev *hdev, u16 handle) +static void mgmt_adv_monitor_removed(struct sock *sk, struct hci_dev *hdev, + __le16 handle) { struct mgmt_ev_adv_monitor_removed ev; - struct mgmt_pending_cmd *cmd; - struct sock *sk_skip = NULL; - struct mgmt_cp_remove_adv_monitor *cp; - - cmd = pending_find(MGMT_OP_REMOVE_ADV_MONITOR, hdev); - if (cmd) { - cp = cmd->param; - if (cp->monitor_handle) - sk_skip = cmd->sk; - } - - ev.monitor_handle = cpu_to_le16(handle); + ev.monitor_handle = handle; - mgmt_event(MGMT_EV_ADV_MONITOR_REMOVED, hdev, &ev, sizeof(ev), sk_skip); + mgmt_event(MGMT_EV_ADV_MONITOR_REMOVED, hdev, &ev, sizeof(ev), sk); } static int read_adv_mon_features(struct sock *sk, struct hci_dev *hdev, @@ -5284,8 +5297,7 @@ static int __add_adv_patterns_monitor(struct sock *sk, struct hci_dev *hdev, if (pending_find(MGMT_OP_SET_LE, hdev) || pending_find(MGMT_OP_ADD_ADV_PATTERNS_MONITOR, hdev) || - pending_find(MGMT_OP_ADD_ADV_PATTERNS_MONITOR_RSSI, hdev) || - pending_find(MGMT_OP_REMOVE_ADV_MONITOR, hdev)) { + pending_find(MGMT_OP_ADD_ADV_PATTERNS_MONITOR_RSSI, hdev)) { status = MGMT_STATUS_BUSY; goto unlock; } @@ -5353,9 +5365,9 @@ static u8 parse_adv_monitor_pattern(struct adv_monitor *m, u8 pattern_count, for (i = 0; i < pattern_count; i++) { offset = patterns[i].offset; length = patterns[i].length; - if (offset >= HCI_MAX_EXT_AD_LENGTH || - length > HCI_MAX_EXT_AD_LENGTH || - (offset + length) > HCI_MAX_EXT_AD_LENGTH) + if (offset >= HCI_MAX_AD_LENGTH || + length > HCI_MAX_AD_LENGTH || + (offset + length) > HCI_MAX_AD_LENGTH) return MGMT_STATUS_INVALID_PARAMS; p = kmalloc(sizeof(*p), GFP_KERNEL); @@ -5455,8 +5467,7 @@ static void mgmt_remove_adv_monitor_complete(struct hci_dev *hdev, struct mgmt_pending_cmd *cmd = data; struct mgmt_cp_remove_adv_monitor *cp; - if (status == -ECANCELED || - cmd != pending_find(MGMT_OP_REMOVE_ADV_MONITOR, hdev)) + if (status == -ECANCELED) return; hci_dev_lock(hdev); @@ -5465,12 +5476,14 @@ static void mgmt_remove_adv_monitor_complete(struct hci_dev *hdev, rp.monitor_handle = cp->monitor_handle; - if (!status) + if (!status) { + mgmt_adv_monitor_removed(cmd->sk, hdev, cp->monitor_handle); hci_update_passive_scan(hdev); + } mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, mgmt_status(status), &rp, sizeof(rp)); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); hci_dev_unlock(hdev); bt_dev_dbg(hdev, "remove monitor %d complete, status %d", @@ -5480,10 +5493,6 @@ static void mgmt_remove_adv_monitor_complete(struct hci_dev *hdev, static int mgmt_remove_adv_monitor_sync(struct hci_dev *hdev, void *data) { struct mgmt_pending_cmd *cmd = data; - - if (cmd != pending_find(MGMT_OP_REMOVE_ADV_MONITOR, hdev)) - return -ECANCELED; - struct mgmt_cp_remove_adv_monitor *cp = cmd->param; u16 handle = __le16_to_cpu(cp->monitor_handle); @@ -5502,14 +5511,13 @@ static int remove_adv_monitor(struct sock *sk, struct hci_dev *hdev, hci_dev_lock(hdev); if (pending_find(MGMT_OP_SET_LE, hdev) || - pending_find(MGMT_OP_REMOVE_ADV_MONITOR, hdev) || pending_find(MGMT_OP_ADD_ADV_PATTERNS_MONITOR, hdev) || pending_find(MGMT_OP_ADD_ADV_PATTERNS_MONITOR_RSSI, hdev)) { status = MGMT_STATUS_BUSY; goto unlock; } - cmd = mgmt_pending_add(sk, MGMT_OP_REMOVE_ADV_MONITOR, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_REMOVE_ADV_MONITOR, hdev, data, len); if (!cmd) { status = MGMT_STATUS_NO_RESOURCES; goto unlock; @@ -5519,7 +5527,7 @@ static int remove_adv_monitor(struct sock *sk, struct hci_dev *hdev, mgmt_remove_adv_monitor_complete); if (err) { - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); if (err == -ENOMEM) status = MGMT_STATUS_NO_RESOURCES; @@ -6551,6 +6559,7 @@ static int set_scan_params(struct sock *sk, struct hci_dev *hdev, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_SCAN_PARAMS, MGMT_STATUS_NOT_SUPPORTED); + /* Keep allowed ranges in sync with set_mesh() */ interval = __le16_to_cpu(cp->interval); if (interval < 0x0004 || interval > 0x4000) @@ -7605,11 +7614,16 @@ static void add_device_complete(struct hci_dev *hdev, void *data, int err) struct mgmt_cp_add_device *cp = cmd->param; if (!err) { + struct hci_conn_params *params; + + params = hci_conn_params_lookup(hdev, &cp->addr.bdaddr, + le_addr_type(cp->addr.type)); + device_added(cmd->sk, hdev, &cp->addr.bdaddr, cp->addr.type, cp->action); device_flags_changed(NULL, hdev, &cp->addr.bdaddr, cp->addr.type, hdev->conn_flags, - PTR_UINT(cmd->user_data)); + params ? params->flags : 0); } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_ADD_DEVICE, @@ -7712,8 +7726,6 @@ static int add_device(struct sock *sk, struct hci_dev *hdev, goto unlock; } - cmd->user_data = UINT_PTR(current_flags); - err = hci_cmd_sync_queue(hdev, add_device_sync, cmd, add_device_complete); if (err < 0) { @@ -9435,6 +9447,7 @@ void mgmt_index_removed(struct hci_dev *hdev) cancel_delayed_work_sync(&hdev->discov_off); cancel_delayed_work_sync(&hdev->service_cache); cancel_delayed_work_sync(&hdev->rpa_expired); + cancel_delayed_work_sync(&hdev->mesh_send_done); } void mgmt_power_on(struct hci_dev *hdev, int err) diff --git a/net/bluetooth/mgmt_util.c b/net/bluetooth/mgmt_util.c index 0115f783bde80..dba6a0d66500f 100644 --- a/net/bluetooth/mgmt_util.c +++ b/net/bluetooth/mgmt_util.c @@ -229,23 +229,6 @@ struct mgmt_pending_cmd *mgmt_pending_find(unsigned short channel, u16 opcode, return NULL; } -struct mgmt_pending_cmd *mgmt_pending_find_data(unsigned short channel, - u16 opcode, - struct hci_dev *hdev, - const void *data) -{ - struct mgmt_pending_cmd *cmd; - - list_for_each_entry(cmd, &hdev->mgmt_pending, list) { - if (cmd->user_data != data) - continue; - if (cmd->opcode == opcode) - return cmd; - } - - return NULL; -} - void mgmt_pending_foreach(u16 opcode, struct hci_dev *hdev, void (*cb)(struct mgmt_pending_cmd *cmd, void *data), void *data) @@ -321,7 +304,7 @@ void mgmt_mesh_foreach(struct hci_dev *hdev, { struct mgmt_mesh_tx *mesh_tx, *tmp; - list_for_each_entry_safe(mesh_tx, tmp, &hdev->mgmt_pending, list) { + list_for_each_entry_safe(mesh_tx, tmp, &hdev->mesh_pending, list) { if (!sk || mesh_tx->sk == sk) cb(mesh_tx, data); } diff --git a/net/bluetooth/mgmt_util.h b/net/bluetooth/mgmt_util.h index bdf978605d5a8..f2ba994ab1d84 100644 --- a/net/bluetooth/mgmt_util.h +++ b/net/bluetooth/mgmt_util.h @@ -54,10 +54,6 @@ int mgmt_cmd_complete(struct sock *sk, u16 index, u16 cmd, u8 status, struct mgmt_pending_cmd *mgmt_pending_find(unsigned short channel, u16 opcode, struct hci_dev *hdev); -struct mgmt_pending_cmd *mgmt_pending_find_data(unsigned short channel, - u16 opcode, - struct hci_dev *hdev, - const void *data); void mgmt_pending_foreach(u16 opcode, struct hci_dev *hdev, void (*cb)(struct mgmt_pending_cmd *cmd, void *data), void *data); diff --git a/net/bluetooth/rfcomm/tty.c b/net/bluetooth/rfcomm/tty.c index 94ec913dfb76e..e258ca1914705 100644 --- a/net/bluetooth/rfcomm/tty.c +++ b/net/bluetooth/rfcomm/tty.c @@ -651,8 +651,8 @@ static void rfcomm_dev_modem_status(struct rfcomm_dlc *dlc, u8 v24_sig) tty_port_tty_hangup(&dev->port, true); dev->modem_status = - ((v24_sig & RFCOMM_V24_RTC) ? (TIOCM_DSR | TIOCM_DTR) : 0) | - ((v24_sig & RFCOMM_V24_RTR) ? (TIOCM_RTS | TIOCM_CTS) : 0) | + ((v24_sig & RFCOMM_V24_RTC) ? TIOCM_DSR : 0) | + ((v24_sig & RFCOMM_V24_RTR) ? TIOCM_CTS : 0) | ((v24_sig & RFCOMM_V24_IC) ? TIOCM_RI : 0) | ((v24_sig & RFCOMM_V24_DV) ? TIOCM_CD : 0); } @@ -1063,10 +1063,14 @@ static void rfcomm_tty_hangup(struct tty_struct *tty) static int rfcomm_tty_tiocmget(struct tty_struct *tty) { struct rfcomm_dev *dev = tty->driver_data; + struct rfcomm_dlc *dlc = dev->dlc; + u8 v24_sig; BT_DBG("tty %p dev %p", tty, dev); - return dev->modem_status; + rfcomm_dlc_get_modem_status(dlc, &v24_sig); + + return (v24_sig & (TIOCM_DTR | TIOCM_RTS)) | dev->modem_status; } static int rfcomm_tty_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear) @@ -1079,23 +1083,15 @@ static int rfcomm_tty_tiocmset(struct tty_struct *tty, unsigned int set, unsigne rfcomm_dlc_get_modem_status(dlc, &v24_sig); - if (set & TIOCM_DSR || set & TIOCM_DTR) + if (set & TIOCM_DTR) v24_sig |= RFCOMM_V24_RTC; - if (set & TIOCM_RTS || set & TIOCM_CTS) + if (set & TIOCM_RTS) v24_sig |= RFCOMM_V24_RTR; - if (set & TIOCM_RI) - v24_sig |= RFCOMM_V24_IC; - if (set & TIOCM_CD) - v24_sig |= RFCOMM_V24_DV; - if (clear & TIOCM_DSR || clear & TIOCM_DTR) + if (clear & TIOCM_DTR) v24_sig &= ~RFCOMM_V24_RTC; - if (clear & TIOCM_RTS || clear & TIOCM_CTS) + if (clear & TIOCM_RTS) v24_sig &= ~RFCOMM_V24_RTR; - if (clear & TIOCM_RI) - v24_sig &= ~RFCOMM_V24_IC; - if (clear & TIOCM_CD) - v24_sig &= ~RFCOMM_V24_DV; rfcomm_dlc_set_modem_status(dlc, v24_sig); diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 64d4d57c7033a..6635d155e360b 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -433,6 +433,13 @@ static void sco_sock_kill(struct sock *sk) BT_DBG("sk %p state %d", sk, sk->sk_state); + /* Sock is dead, so set conn->sk to NULL to avoid possible UAF */ + if (sco_pi(sk)->conn) { + sco_conn_lock(sco_pi(sk)->conn); + sco_pi(sk)->conn->sk = NULL; + sco_conn_unlock(sco_pi(sk)->conn); + } + /* Kill poor orphan */ bt_sock_unlink(&sco_sk_list, sk); sock_set_flag(sk, SOCK_DEAD); diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 56f7f041c9a60..4c00bc50de811 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -1380,7 +1380,7 @@ static void smp_timeout(struct work_struct *work) bt_dev_dbg(conn->hcon->hdev, "conn %p", conn); - hci_disconnect(conn->hcon, HCI_ERROR_REMOTE_USER_TERM); + hci_disconnect(conn->hcon, HCI_ERROR_AUTH_FAILURE); } static struct smp_chan *smp_chan_create(struct l2cap_conn *conn) @@ -2978,8 +2978,25 @@ static int smp_sig_channel(struct l2cap_chan *chan, struct sk_buff *skb) if (code > SMP_CMD_MAX) goto drop; - if (smp && !test_and_clear_bit(code, &smp->allow_cmd)) + if (smp && !test_and_clear_bit(code, &smp->allow_cmd)) { + /* If there is a context and the command is not allowed consider + * it a failure so the session is cleanup properly. + */ + switch (code) { + case SMP_CMD_IDENT_INFO: + case SMP_CMD_IDENT_ADDR_INFO: + case SMP_CMD_SIGN_INFO: + /* 3.6.1. Key distribution and generation + * + * A device may reject a distributed key by sending the + * Pairing Failed command with the reason set to + * "Key Rejected". + */ + smp_failure(conn, SMP_KEY_REJECTED); + break; + } goto drop; + } /* If we don't have a context the only allowed commands are * pairing request and security request. diff --git a/net/bluetooth/smp.h b/net/bluetooth/smp.h index 87a59ec2c9f02..c5da53dfab04f 100644 --- a/net/bluetooth/smp.h +++ b/net/bluetooth/smp.h @@ -138,6 +138,7 @@ struct smp_cmd_keypress_notify { #define SMP_NUMERIC_COMP_FAILED 0x0c #define SMP_BREDR_PAIRING_IN_PROGRESS 0x0d #define SMP_CROSS_TRANSP_NOT_ALLOWED 0x0e +#define SMP_KEY_REJECTED 0x0f #define SMP_MIN_ENC_KEY_SIZE 7 #define SMP_MAX_ENC_KEY_SIZE 16 diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 5918d1b32e196..c639bdafe6b01 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -12,6 +12,11 @@ extern struct bpf_struct_ops bpf_bpf_dummy_ops; /* A common type for test_N with return value in bpf_dummy_ops */ typedef int (*dummy_ops_test_ret_fn)(struct bpf_dummy_ops_state *state, ...); +static int dummy_ops_test_ret_function(struct bpf_dummy_ops_state *state, ...) +{ + return 0; +} + struct bpf_dummy_ops_test_args { u64 args[MAX_BPF_FUNC_ARGS]; struct bpf_dummy_ops_state state; @@ -62,7 +67,7 @@ static int dummy_ops_copy_args(struct bpf_dummy_ops_test_args *args) static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) { - dummy_ops_test_ret_fn test = (void *)image; + dummy_ops_test_ret_fn test = (void *)image + cfi_get_offset(); struct bpf_dummy_ops_state *state = NULL; /* state needs to be NULL if args[0] is 0 */ @@ -120,6 +125,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, op_idx = prog->expected_attach_type; err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[op_idx], + &dummy_ops_test_ret_function, image, image + PAGE_SIZE); if (err < 0) goto out; @@ -220,6 +226,28 @@ static void bpf_dummy_unreg(void *kdata) { } +static int bpf_dummy_test_1(struct bpf_dummy_ops_state *cb) +{ + return 0; +} + +static int bpf_dummy_test_2(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2, + char a3, unsigned long a4) +{ + return 0; +} + +static int bpf_dummy_test_sleepable(struct bpf_dummy_ops_state *cb) +{ + return 0; +} + +static struct bpf_dummy_ops __bpf_bpf_dummy_ops = { + .test_1 = bpf_dummy_test_1, + .test_2 = bpf_dummy_test_2, + .test_sleepable = bpf_dummy_test_sleepable, +}; + struct bpf_struct_ops bpf_bpf_dummy_ops = { .verifier_ops = &bpf_dummy_verifier_ops, .init = bpf_dummy_init, @@ -228,4 +256,5 @@ struct bpf_struct_ops bpf_bpf_dummy_ops = { .reg = bpf_dummy_reg, .unreg = bpf_dummy_unreg, .name = "bpf_dummy_ops", + .cfi_stubs = &__bpf_bpf_dummy_ops, }; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 73fb9db55798c..5ce32408bcffc 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -602,10 +602,21 @@ __bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) refcount_dec(&p->cnt); } +__bpf_kfunc void bpf_kfunc_call_test_release_dtor(void *p) +{ + bpf_kfunc_call_test_release(p); +} +CFI_NOSEAL(bpf_kfunc_call_test_release_dtor); + __bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p) { } +__bpf_kfunc void bpf_kfunc_call_memb_release_dtor(void *p) +{ +} +CFI_NOSEAL(bpf_kfunc_call_memb_release_dtor); + __diag_pop(); BTF_SET8_START(bpf_test_modify_return_ids) @@ -1676,9 +1687,9 @@ static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids) BTF_ID(struct, prog_test_ref_kfunc) -BTF_ID(func, bpf_kfunc_call_test_release) +BTF_ID(func, bpf_kfunc_call_test_release_dtor) BTF_ID(struct, prog_test_member) -BTF_ID(func, bpf_kfunc_call_memb_release) +BTF_ID(func, bpf_kfunc_call_memb_release_dtor) static int __init bpf_prog_test_run_init(void) { diff --git a/net/bridge/br.c b/net/bridge/br.c index a6e94ceb7c9a0..d466febcf9abe 100644 --- a/net/bridge/br.c +++ b/net/bridge/br.c @@ -37,6 +37,11 @@ static int br_device_event(struct notifier_block *unused, unsigned long event, v int err; if (netif_is_bridge_master(dev)) { + struct net_bridge *br = netdev_priv(dev); + + if (event == NETDEV_REGISTER) + br_fdb_change_mac_address(br, dev->dev_addr); + err = br_vlan_bridge_event(dev, event, ptr); if (err) return notifier_from_errno(err); @@ -312,6 +317,13 @@ int br_boolopt_multi_toggle(struct net_bridge *br, int err = 0; int opt_id; + opt_id = find_next_bit(&bitmap, BITS_PER_LONG, BR_BOOLOPT_MAX); + if (opt_id != BITS_PER_LONG) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unknown boolean option %d", + opt_id); + return -EINVAL; + } + for_each_set_bit(opt_id, &bitmap, BR_BOOLOPT_MAX) { bool on = !!(bm->optval & BIT(opt_id)); diff --git a/net/bridge/br_forward.c b/net/bridge/br_forward.c index e19b583ff2c6d..e9f09cdb9848e 100644 --- a/net/bridge/br_forward.c +++ b/net/bridge/br_forward.c @@ -25,7 +25,7 @@ static inline int should_deliver(const struct net_bridge_port *p, vg = nbp_vlan_group_rcu(p); return ((p->flags & BR_HAIRPIN_MODE) || skb->dev != p->dev) && - (br_mst_is_enabled(p->br) || p->state == BR_STATE_FORWARDING) && + (br_mst_is_enabled(p) || p->state == BR_STATE_FORWARDING) && br_allowed_egress(vg, skb) && nbp_switchdev_allowed_egress(p, skb) && !br_skb_isolated(p, skb); } @@ -148,7 +148,8 @@ void br_forward(const struct net_bridge_port *to, goto out; /* redirect to backup link if the destination port is down */ - if (rcu_access_pointer(to->backup_port) && !netif_carrier_ok(to->dev)) { + if (rcu_access_pointer(to->backup_port) && + (!netif_carrier_ok(to->dev) || !netif_running(to->dev))) { struct net_bridge_port *backup_port; backup_port = rcu_dereference(to->backup_port); diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index 2450690f98cfa..6ffc81eedf074 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -386,6 +386,7 @@ void br_dev_delete(struct net_device *dev, struct list_head *head) del_nbp(p); } + br_mst_uninit(br); br_recalculate_neigh_suppress_enabled(br); br_fdb_delete_by_port(br, NULL, 0, 1); diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index e09000e38d071..951330c1a813b 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -93,7 +93,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb br = p->br; - if (br_mst_is_enabled(br)) { + if (br_mst_is_enabled(p)) { state = BR_STATE_FORWARDING; } else { if (p->state == BR_STATE_DISABLED) @@ -411,7 +411,7 @@ static rx_handler_result_t br_handle_frame(struct sk_buff **pskb) return RX_HANDLER_PASS; forward: - if (br_mst_is_enabled(p->br)) + if (br_mst_is_enabled(p)) goto defer_stp_filtering; switch (p->state) { diff --git a/net/bridge/br_mdb.c b/net/bridge/br_mdb.c index 7305f5f8215ca..96bea0c8408fe 100644 --- a/net/bridge/br_mdb.c +++ b/net/bridge/br_mdb.c @@ -1030,7 +1030,7 @@ static int br_mdb_add_group(const struct br_mdb_config *cfg, /* host join */ if (!port) { - if (mp->host_joined) { + if (mp->host_joined && !(cfg->nlflags & NLM_F_REPLACE)) { NL_SET_ERR_MSG_MOD(extack, "Group is already joined by host"); return -EEXIST; } diff --git a/net/bridge/br_mst.c b/net/bridge/br_mst.c index 1820f09ff59ce..43a300ae6bfaf 100644 --- a/net/bridge/br_mst.c +++ b/net/bridge/br_mst.c @@ -22,6 +22,12 @@ bool br_mst_enabled(const struct net_device *dev) } EXPORT_SYMBOL_GPL(br_mst_enabled); +void br_mst_uninit(struct net_bridge *br) +{ + if (br_opt_get(br, BROPT_MST_ENABLED)) + static_branch_dec(&br_mst_used); +} + int br_mst_get_info(const struct net_device *dev, u16 msti, unsigned long *vids) { const struct net_bridge_vlan_group *vg; @@ -80,10 +86,10 @@ static void br_mst_vlan_set_state(struct net_bridge_vlan_group *vg, if (br_vlan_get_state(v) == state) return; - br_vlan_set_state(v, state); - if (v->vid == vg->pvid) br_vlan_set_pvid_state(vg, state); + + br_vlan_set_state(v, state); } int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state, @@ -225,9 +231,9 @@ int br_mst_set_enabled(struct net_bridge *br, bool on, return err; if (on) - static_branch_enable(&br_mst_used); + static_branch_inc(&br_mst_used); else - static_branch_disable(&br_mst_used); + static_branch_dec(&br_mst_used); br_opt_toggle(br, BROPT_MST_ENABLED, on); return 0; diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index c38244d60ff86..f42805d9b38fa 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -2104,12 +2104,17 @@ static void __br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) } } -void br_multicast_enable_port(struct net_bridge_port *port) +static void br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) { - struct net_bridge *br = port->br; + struct net_bridge *br = pmctx->port->br; spin_lock_bh(&br->multicast_lock); - __br_multicast_enable_port_ctx(&port->multicast_ctx); + if (br_multicast_port_ctx_is_vlan(pmctx) && + !(pmctx->vlan->priv_flags & BR_VLFLAG_MCAST_ENABLED)) { + spin_unlock_bh(&br->multicast_lock); + return; + } + __br_multicast_enable_port_ctx(pmctx); spin_unlock_bh(&br->multicast_lock); } @@ -2136,11 +2141,67 @@ static void __br_multicast_disable_port_ctx(struct net_bridge_mcast_port *pmctx) br_multicast_rport_del_notify(pmctx, del); } +static void br_multicast_disable_port_ctx(struct net_bridge_mcast_port *pmctx) +{ + struct net_bridge *br = pmctx->port->br; + + spin_lock_bh(&br->multicast_lock); + if (br_multicast_port_ctx_is_vlan(pmctx) && + !(pmctx->vlan->priv_flags & BR_VLFLAG_MCAST_ENABLED)) { + spin_unlock_bh(&br->multicast_lock); + return; + } + + __br_multicast_disable_port_ctx(pmctx); + spin_unlock_bh(&br->multicast_lock); +} + +static void br_multicast_toggle_port(struct net_bridge_port *port, bool on) +{ +#if IS_ENABLED(CONFIG_BRIDGE_VLAN_FILTERING) + if (br_opt_get(port->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) { + struct net_bridge_vlan_group *vg; + struct net_bridge_vlan *vlan; + + rcu_read_lock(); + vg = nbp_vlan_group_rcu(port); + if (!vg) { + rcu_read_unlock(); + return; + } + + /* iterate each vlan, toggle vlan multicast context */ + list_for_each_entry_rcu(vlan, &vg->vlan_list, vlist) { + struct net_bridge_mcast_port *pmctx = + &vlan->port_mcast_ctx; + u8 state = br_vlan_get_state(vlan); + /* enable vlan multicast context when state is + * LEARNING or FORWARDING + */ + if (on && br_vlan_state_allowed(state, true)) + br_multicast_enable_port_ctx(pmctx); + else + br_multicast_disable_port_ctx(pmctx); + } + rcu_read_unlock(); + return; + } +#endif + /* toggle port multicast context when vlan snooping is disabled */ + if (on) + br_multicast_enable_port_ctx(&port->multicast_ctx); + else + br_multicast_disable_port_ctx(&port->multicast_ctx); +} + +void br_multicast_enable_port(struct net_bridge_port *port) +{ + br_multicast_toggle_port(port, true); +} + void br_multicast_disable_port(struct net_bridge_port *port) { - spin_lock_bh(&port->br->multicast_lock); - __br_multicast_disable_port_ctx(&port->multicast_ctx); - spin_unlock_bh(&port->br->multicast_lock); + br_multicast_toggle_port(port, false); } static int __grp_src_delete_marked(struct net_bridge_port_group *pg) @@ -4210,6 +4271,32 @@ static void __br_multicast_stop(struct net_bridge_mcast *brmctx) #endif } +void br_multicast_update_vlan_mcast_ctx(struct net_bridge_vlan *v, u8 state) +{ +#if IS_ENABLED(CONFIG_BRIDGE_VLAN_FILTERING) + struct net_bridge *br; + + if (!br_vlan_should_use(v)) + return; + + if (br_vlan_is_master(v)) + return; + + br = v->port->br; + + if (!br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) + return; + + if (br_vlan_state_allowed(state, true)) + br_multicast_enable_port_ctx(&v->port_mcast_ctx); + + /* Multicast is not disabled for the vlan when it goes in + * blocking state because the timers will expire and stop by + * themselves without sending more queries. + */ +#endif +} + void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on) { struct net_bridge *br; @@ -4303,9 +4390,9 @@ int br_multicast_toggle_vlan_snooping(struct net_bridge *br, bool on, __br_multicast_open(&br->multicast_ctx); list_for_each_entry(p, &br->port_list, list) { if (on) - br_multicast_disable_port(p); + br_multicast_disable_port_ctx(&p->multicast_ctx); else - br_multicast_enable_port(p); + br_multicast_enable_port_ctx(&p->multicast_ctx); } list_for_each_entry(vlan, &vg->vlan_list, vlist) @@ -4720,6 +4807,14 @@ void br_multicast_set_query_intvl(struct net_bridge_mcast *brmctx, intvl_jiffies = BR_MULTICAST_QUERY_INTVL_MIN; } + if (intvl_jiffies > BR_MULTICAST_QUERY_INTVL_MAX) { + br_info(brmctx->br, + "trying to set multicast query interval above maximum, setting to %lu (%ums)\n", + jiffies_to_clock_t(BR_MULTICAST_QUERY_INTVL_MAX), + jiffies_to_msecs(BR_MULTICAST_QUERY_INTVL_MAX)); + intvl_jiffies = BR_MULTICAST_QUERY_INTVL_MAX; + } + brmctx->multicast_query_interval = intvl_jiffies; } @@ -4736,6 +4831,14 @@ void br_multicast_set_startup_query_intvl(struct net_bridge_mcast *brmctx, intvl_jiffies = BR_MULTICAST_STARTUP_QUERY_INTVL_MIN; } + if (intvl_jiffies > BR_MULTICAST_STARTUP_QUERY_INTVL_MAX) { + br_info(brmctx->br, + "trying to set multicast startup query interval above maximum, setting to %lu (%ums)\n", + jiffies_to_clock_t(BR_MULTICAST_STARTUP_QUERY_INTVL_MAX), + jiffies_to_msecs(BR_MULTICAST_STARTUP_QUERY_INTVL_MAX)); + intvl_jiffies = BR_MULTICAST_STARTUP_QUERY_INTVL_MAX; + } + brmctx->multicast_startup_query_interval = intvl_jiffies; } diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c index 2a4958e995f2d..e6962d693359b 100644 --- a/net/bridge/br_netfilter_hooks.c +++ b/net/bridge/br_netfilter_hooks.c @@ -648,9 +648,6 @@ static unsigned int br_nf_local_in(void *priv, break; } - ct = container_of(nfct, struct nf_conn, ct_general); - WARN_ON_ONCE(!nf_ct_is_confirmed(ct)); - return ret; } #endif diff --git a/net/bridge/br_nf_core.c b/net/bridge/br_nf_core.c index 98aea5485aaef..a8c67035e23c0 100644 --- a/net/bridge/br_nf_core.c +++ b/net/bridge/br_nf_core.c @@ -65,17 +65,14 @@ static struct dst_ops fake_dst_ops = { * ipt_REJECT needs it. Future netfilter modules might * require us to fill additional fields. */ -static const u32 br_dst_default_metrics[RTAX_MAX] = { - [RTAX_MTU - 1] = 1500, -}; - void br_netfilter_rtable_init(struct net_bridge *br) { struct rtable *rt = &br->fake_rtable; rcuref_init(&rt->dst.__rcuref, 1); rt->dst.dev = br->dev; - dst_init_metrics(&rt->dst, br_dst_default_metrics, true); + dst_init_metrics(&rt->dst, br->metrics, false); + dst_metric_set(&rt->dst, RTAX_MTU, br->dev->mtu); rt->dst.flags = DST_NOXFRM | DST_FAKE_RTABLE; rt->dst.ops = &fake_dst_ops; } diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index 72d80fd943a8a..c8a4e3b39b0e2 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -31,6 +31,8 @@ #define BR_MULTICAST_DEFAULT_HASH_MAX 4096 #define BR_MULTICAST_QUERY_INTVL_MIN msecs_to_jiffies(1000) #define BR_MULTICAST_STARTUP_QUERY_INTVL_MIN BR_MULTICAST_QUERY_INTVL_MIN +#define BR_MULTICAST_QUERY_INTVL_MAX msecs_to_jiffies(86400000) /* 24 hours */ +#define BR_MULTICAST_STARTUP_QUERY_INTVL_MAX BR_MULTICAST_QUERY_INTVL_MAX #define BR_HWDOM_MAX BITS_PER_LONG @@ -502,6 +504,7 @@ struct net_bridge { struct rtable fake_rtable; struct rt6_info fake_rt6_info; }; + u32 metrics[RTAX_MAX]; #endif u16 group_fwd_mask; u16 group_fwd_mask_required; @@ -1042,6 +1045,7 @@ void br_multicast_port_ctx_init(struct net_bridge_port *port, struct net_bridge_vlan *vlan, struct net_bridge_mcast_port *pmctx); void br_multicast_port_ctx_deinit(struct net_bridge_mcast_port *pmctx); +void br_multicast_update_vlan_mcast_ctx(struct net_bridge_vlan *v, u8 state); void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on); int br_multicast_toggle_vlan_snooping(struct net_bridge *br, bool on, struct netlink_ext_ack *extack); @@ -1478,6 +1482,11 @@ static inline void br_multicast_port_ctx_deinit(struct net_bridge_mcast_port *pm { } +static inline void br_multicast_update_vlan_mcast_ctx(struct net_bridge_vlan *v, + u8 state) +{ +} + static inline void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on) { @@ -1829,7 +1838,9 @@ bool br_vlan_global_opts_can_enter_range(const struct net_bridge_vlan *v_curr, bool br_vlan_global_opts_fill(struct sk_buff *skb, u16 vid, u16 vid_range, const struct net_bridge_vlan *v_opts); -/* vlan state manipulation helpers using *_ONCE to annotate lock-free access */ +/* vlan state manipulation helpers using *_ONCE to annotate lock-free access, + * while br_vlan_set_state() may access data protected by multicast_lock. + */ static inline u8 br_vlan_get_state(const struct net_bridge_vlan *v) { return READ_ONCE(v->state); @@ -1838,6 +1849,7 @@ static inline u8 br_vlan_get_state(const struct net_bridge_vlan *v) static inline void br_vlan_set_state(struct net_bridge_vlan *v, u8 state) { WRITE_ONCE(v->state, state); + br_multicast_update_vlan_mcast_ctx(v, state); } static inline u8 br_vlan_get_pvid_state(const struct net_bridge_vlan_group *vg) @@ -1868,10 +1880,12 @@ static inline bool br_vlan_state_allowed(u8 state, bool learn_allow) /* br_mst.c */ #ifdef CONFIG_BRIDGE_VLAN_FILTERING DECLARE_STATIC_KEY_FALSE(br_mst_used); -static inline bool br_mst_is_enabled(struct net_bridge *br) +static inline bool br_mst_is_enabled(const struct net_bridge_port *p) { + /* check the port's vlan group to avoid racing with port deletion */ return static_branch_unlikely(&br_mst_used) && - br_opt_get(br, BROPT_MST_ENABLED); + br_opt_get(p->br, BROPT_MST_ENABLED) && + rcu_access_pointer(p->vlgrp); } int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state, @@ -1885,8 +1899,9 @@ int br_mst_fill_info(struct sk_buff *skb, const struct net_bridge_vlan_group *vg); int br_mst_process(struct net_bridge_port *p, const struct nlattr *mst_attr, struct netlink_ext_ack *extack); +void br_mst_uninit(struct net_bridge *br); #else -static inline bool br_mst_is_enabled(struct net_bridge *br) +static inline bool br_mst_is_enabled(const struct net_bridge_port *p) { return false; } @@ -1920,6 +1935,10 @@ static inline int br_mst_process(struct net_bridge_port *p, { return -EOPNOTSUPP; } + +static inline void br_mst_uninit(struct net_bridge *br) +{ +} #endif struct nf_br_ops { diff --git a/net/bridge/br_switchdev.c b/net/bridge/br_switchdev.c index 7b41ee8740cbb..f10bd6a233dcf 100644 --- a/net/bridge/br_switchdev.c +++ b/net/bridge/br_switchdev.c @@ -17,6 +17,9 @@ static bool nbp_switchdev_can_offload_tx_fwd(const struct net_bridge_port *p, if (!static_branch_unlikely(&br_switchdev_tx_fwd_offload)) return false; + if (br_multicast_igmp_type(skb)) + return false; + return (p->flags & BR_TX_FWD_OFFLOAD) && (p->hwdom != BR_INPUT_SKB_CB(skb)->src_hwdom); } diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index a1c22eab71ffe..cc54b8267bcc7 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -1455,7 +1455,7 @@ void br_vlan_fill_forward_path_pvid(struct net_bridge *br, if (!br_opt_get(br, BROPT_VLAN_ENABLED)) return; - vg = br_vlan_group(br); + vg = br_vlan_group_rcu(br); if (idx >= 0 && ctx->vlan[idx].proto == br->vlan_proto) { diff --git a/net/bridge/netfilter/ebt_limit.c b/net/bridge/netfilter/ebt_limit.c index e16183bd1bb88..aa4899411449c 100644 --- a/net/bridge/netfilter/ebt_limit.c +++ b/net/bridge/netfilter/ebt_limit.c @@ -86,6 +86,17 @@ static int ebt_limit_mt_check(const struct xt_mtchk_param *par) return 0; } +#define DEFINE_EBT_LIMIT_MT_REG(compat) \ + { \ + .name = "limit", \ + .revision = 0, \ + .family = NFPROTO_BRIDGE, \ + .match = ebt_limit_mt, \ + .checkentry = ebt_limit_mt_check, \ + .matchsize = sizeof(struct ebt_limit_info), \ + .usersize = offsetof(struct ebt_limit_info, prev), \ + .has_compat_metadata = compat, \ + } #ifdef CONFIG_NETFILTER_XTABLES_COMPAT /* @@ -97,22 +108,17 @@ struct ebt_compat_limit_info { compat_ulong_t prev; compat_uint_t credit, credit_cap, cost; }; -#endif -static struct xt_match ebt_limit_mt_reg __read_mostly = { - .name = "limit", - .revision = 0, - .family = NFPROTO_BRIDGE, - .match = ebt_limit_mt, - .checkentry = ebt_limit_mt_check, - .matchsize = sizeof(struct ebt_limit_info), - .usersize = offsetof(struct ebt_limit_info, prev), -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT +static struct compat_xt_match_ext ebt_limit_mt_reg_ext __read_mostly = { .compatsize = sizeof(struct ebt_compat_limit_info), -#endif - .me = THIS_MODULE, + .match = DEFINE_EBT_LIMIT_MT_REG(true), }; +#define ebt_limit_mt_reg (ebt_limit_mt_reg_ext.match) +#else +static struct xt_match ebt_limit_mt_reg __read_mostly = DEFINE_EBT_LIMIT_MT_REG(false); +#endif + static int __init ebt_limit_init(void) { return xt_register_match(&ebt_limit_mt_reg); diff --git a/net/bridge/netfilter/ebt_mark.c b/net/bridge/netfilter/ebt_mark.c index 8cf653c72fd82..8fca05f1456a4 100644 --- a/net/bridge/netfilter/ebt_mark.c +++ b/net/bridge/netfilter/ebt_mark.c @@ -53,6 +53,19 @@ static int ebt_mark_tg_check(const struct xt_tgchk_param *par) return -EINVAL; return 0; } + +#define DEFINE_EBT_MARK_TG_REG(compat) \ + { \ + .name = "mark", \ + .revision = 0, \ + .family = NFPROTO_BRIDGE, \ + .target = ebt_mark_tg, \ + .checkentry = ebt_mark_tg_check, \ + .targetsize = sizeof(struct ebt_mark_t_info), \ + .me = THIS_MODULE, \ + .has_compat_metadata = compat \ + } + #ifdef CONFIG_NETFILTER_XTABLES_COMPAT struct compat_ebt_mark_t_info { compat_ulong_t mark; @@ -78,23 +91,19 @@ static int mark_tg_compat_to_user(void __user *dst, const void *src) return -EFAULT; return 0; } -#endif -static struct xt_target ebt_mark_tg_reg __read_mostly = { - .name = "mark", - .revision = 0, - .family = NFPROTO_BRIDGE, - .target = ebt_mark_tg, - .checkentry = ebt_mark_tg_check, - .targetsize = sizeof(struct ebt_mark_t_info), -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT +static struct compat_xt_target_ext ebt_mark_tg_reg_ext __read_mostly = { .compatsize = sizeof(struct compat_ebt_mark_t_info), .compat_from_user = mark_tg_compat_from_user, .compat_to_user = mark_tg_compat_to_user, -#endif - .me = THIS_MODULE, + .target = DEFINE_EBT_MARK_TG_REG(true), }; +#define ebt_mark_tg_reg (ebt_mark_tg_reg_ext.target) +#else +static struct xt_target ebt_mark_tg_reg __read_mostly = DEFINE_EBT_MARK_TG_REG(false); +#endif + static int __init ebt_mark_init(void) { return xt_register_target(&ebt_mark_tg_reg); diff --git a/net/bridge/netfilter/ebt_mark_m.c b/net/bridge/netfilter/ebt_mark_m.c index 5872e73c741e6..3a91306480cbc 100644 --- a/net/bridge/netfilter/ebt_mark_m.c +++ b/net/bridge/netfilter/ebt_mark_m.c @@ -36,6 +36,17 @@ static int ebt_mark_mt_check(const struct xt_mtchk_param *par) return 0; } +#define DEFINE_EBT_MARK_MT_REG(compat) \ + { \ + .name = "mark_m", \ + .revision = 0, \ + .family = NFPROTO_BRIDGE, \ + .match = ebt_mark_mt, \ + .checkentry = ebt_mark_mt_check, \ + .matchsize = sizeof(struct ebt_mark_m_info), \ + .me = THIS_MODULE, \ + .has_compat_metadata = compat, \ + } #ifdef CONFIG_NETFILTER_XTABLES_COMPAT struct compat_ebt_mark_m_info { @@ -66,23 +77,19 @@ static int mark_mt_compat_to_user(void __user *dst, const void *src) return -EFAULT; return 0; } -#endif -static struct xt_match ebt_mark_mt_reg __read_mostly = { - .name = "mark_m", - .revision = 0, - .family = NFPROTO_BRIDGE, - .match = ebt_mark_mt, - .checkentry = ebt_mark_mt_check, - .matchsize = sizeof(struct ebt_mark_m_info), -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT +static struct compat_xt_match_ext ebt_mark_mt_reg_ext __read_mostly = { .compatsize = sizeof(struct compat_ebt_mark_m_info), .compat_from_user = mark_mt_compat_from_user, .compat_to_user = mark_mt_compat_to_user, -#endif - .me = THIS_MODULE, + .match = DEFINE_EBT_MARK_MT_REG(true), }; +#define ebt_mark_mt_reg (ebt_mark_mt_reg_ext.match) +#else +static struct xt_match ebt_mark_mt_reg __read_mostly = DEFINE_EBT_MARK_MT_REG(false); +#endif + static int __init ebt_mark_m_init(void) { return xt_register_match(&ebt_mark_mt_reg); diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index ed62c1026fe93..6433d2e737aa9 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -56,6 +56,15 @@ static unsigned int ebt_pernet_id __read_mostly; static LIST_HEAD(template_tables); static DEFINE_MUTEX(ebt_mutex); +#define DEFINE_EBT_STANDARD_TARGET(compat) \ + { \ + .name = "standard", \ + .revision = 0, \ + .family = NFPROTO_BRIDGE, \ + .targetsize = sizeof(int), \ + .has_compat_metadata = compat, \ + } + #ifdef CONFIG_NETFILTER_XTABLES_COMPAT static void ebt_standard_compat_from_user(void *dst, const void *src) { @@ -74,21 +83,20 @@ static int ebt_standard_compat_to_user(void __user *dst, const void *src) cv -= xt_compat_calc_jump(NFPROTO_BRIDGE, cv); return copy_to_user(dst, &cv, sizeof(cv)) ? -EFAULT : 0; } -#endif - -static struct xt_target ebt_standard_target = { - .name = "standard", - .revision = 0, - .family = NFPROTO_BRIDGE, - .targetsize = sizeof(int), -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT +static struct compat_xt_target_ext ebt_standard_target_ext = { .compatsize = sizeof(compat_int_t), .compat_from_user = ebt_standard_compat_from_user, .compat_to_user = ebt_standard_compat_to_user, -#endif + .target = DEFINE_EBT_STANDARD_TARGET(true), }; +static struct xt_target *ebt_standard_target_ptr = &ebt_standard_target_ext.target; +#else +static struct xt_target ebt_standard_target = DEFINE_EBT_STANDARD_TARGET(false); +static struct xt_target *ebt_standard_target_ptr = &ebt_standard_target; +#endif + static inline int ebt_do_watcher(const struct ebt_entry_watcher *w, struct sk_buff *skb, struct xt_action_param *par) @@ -760,7 +768,7 @@ ebt_check_entry(struct ebt_entry *e, struct net *net, } t->u.target = target; - if (t->u.target == &ebt_standard_target) { + if (t->u.target == ebt_standard_target_ptr) { if (gap < sizeof(struct ebt_standard_target)) { ret = -EFAULT; goto cleanup_watchers; @@ -1641,6 +1649,7 @@ static int compat_match_to_user(struct ebt_entry_match *m, void __user **dstptr, unsigned int *size) { const struct xt_match *match = m->u.match; + compat_to_user_fn_t compat_to_user = get_xt_match_compat_to_user(match); struct compat_ebt_entry_mwt __user *cm = *dstptr; int off = ebt_compat_match_offset(match, m->match_size); compat_uint_t msize = m->match_size - off; @@ -1653,8 +1662,8 @@ static int compat_match_to_user(struct ebt_entry_match *m, void __user **dstptr, put_user(msize, &cm->match_size)) return -EFAULT; - if (match->compat_to_user) { - if (match->compat_to_user(cm->data, m->data)) + if (compat_to_user) { + if (compat_to_user(cm->data, m->data)) return -EFAULT; } else { if (xt_data_to_user(cm->data, m->data, match->usersize, msize, @@ -1673,6 +1682,7 @@ static int compat_target_to_user(struct ebt_entry_target *t, unsigned int *size) { const struct xt_target *target = t->u.target; + compat_to_user_fn_t compat_to_user = get_xt_target_compat_to_user(target); struct compat_ebt_entry_mwt __user *cm = *dstptr; int off = xt_compat_target_offset(target); compat_uint_t tsize = t->target_size - off; @@ -1685,8 +1695,8 @@ static int compat_target_to_user(struct ebt_entry_target *t, put_user(tsize, &cm->match_size)) return -EFAULT; - if (target->compat_to_user) { - if (target->compat_to_user(cm->data, t->data)) + if (compat_to_user) { + if (compat_to_user(cm->data, t->data)) return -EFAULT; } else { if (xt_data_to_user(cm->data, t->data, target->usersize, tsize, @@ -1960,6 +1970,8 @@ static int compat_mtw_from_user(const struct compat_ebt_entry_mwt *mwt, char name[EBT_EXTENSION_MAXNAMELEN]; struct xt_match *match; struct xt_target *wt; + compat_from_user_fn_t match_compat_from_user; + compat_from_user_fn_t wt_compat_from_user; void *dst = NULL; int off, pad = 0; unsigned int size_kern, match_size = mwt->match_size; @@ -1979,8 +1991,10 @@ static int compat_mtw_from_user(const struct compat_ebt_entry_mwt *mwt, off = ebt_compat_match_offset(match, match_size); if (dst) { - if (match->compat_from_user) - match->compat_from_user(dst, mwt->data); + match_compat_from_user = get_xt_match_compat_from_user(match); + + if (match_compat_from_user) + match_compat_from_user(dst, mwt->data); else memcpy(dst, mwt->data, match_size); } @@ -1999,8 +2013,10 @@ static int compat_mtw_from_user(const struct compat_ebt_entry_mwt *mwt, off = xt_compat_target_offset(wt); if (dst) { - if (wt->compat_from_user) - wt->compat_from_user(dst, mwt->data); + wt_compat_from_user = get_xt_target_compat_from_user(wt); + + if (wt_compat_from_user) + wt_compat_from_user(dst, mwt->data); else memcpy(dst, mwt->data, match_size); } @@ -2569,19 +2585,19 @@ static int __init ebtables_init(void) { int ret; - ret = xt_register_target(&ebt_standard_target); + ret = xt_register_target(ebt_standard_target_ptr); if (ret < 0) return ret; ret = nf_register_sockopt(&ebt_sockopts); if (ret < 0) { - xt_unregister_target(&ebt_standard_target); + xt_unregister_target(ebt_standard_target_ptr); return ret; } ret = register_pernet_subsys(&ebt_net_ops); if (ret < 0) { nf_unregister_sockopt(&ebt_sockopts); - xt_unregister_target(&ebt_standard_target); + xt_unregister_target(ebt_standard_target_ptr); return ret; } @@ -2591,7 +2607,7 @@ static int __init ebtables_init(void) static void ebtables_fini(void) { nf_unregister_sockopt(&ebt_sockopts); - xt_unregister_target(&ebt_standard_target); + xt_unregister_target(ebt_standard_target_ptr); unregister_pernet_subsys(&ebt_net_ops); } diff --git a/net/bridge/netfilter/nf_conntrack_bridge.c b/net/bridge/netfilter/nf_conntrack_bridge.c index 6ef04f9fe481b..8c7b697e8478f 100644 --- a/net/bridge/netfilter/nf_conntrack_bridge.c +++ b/net/bridge/netfilter/nf_conntrack_bridge.c @@ -60,19 +60,19 @@ static int nf_br_ip_fragment(struct net *net, struct sock *sk, struct ip_fraglist_iter iter; struct sk_buff *frag; - if (first_len - hlen > mtu || - skb_headroom(skb) < ll_rs) + if (first_len - hlen > mtu) goto blackhole; - if (skb_cloned(skb)) + if (skb_cloned(skb) || + skb_headroom(skb) < ll_rs) goto slow_path; skb_walk_frags(skb, frag) { - if (frag->len > mtu || - skb_headroom(frag) < hlen + ll_rs) + if (frag->len > mtu) goto blackhole; - if (skb_shared(frag)) + if (skb_shared(frag) || + skb_headroom(frag) < hlen + ll_rs) goto slow_path; } diff --git a/net/bridge/netfilter/nft_meta_bridge.c b/net/bridge/netfilter/nft_meta_bridge.c index bd4d1b4d745f6..affb740c8685e 100644 --- a/net/bridge/netfilter/nft_meta_bridge.c +++ b/net/bridge/netfilter/nft_meta_bridge.c @@ -168,8 +168,7 @@ static bool nft_meta_bridge_set_reduce(struct nft_regs_track *track, } static int nft_meta_bridge_set_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { struct nft_meta *priv = nft_expr_priv(expr); unsigned int hooks; @@ -179,7 +178,7 @@ static int nft_meta_bridge_set_validate(const struct nft_ctx *ctx, hooks = 1 << NF_BR_PRE_ROUTING; break; default: - return nft_meta_set_validate(ctx, expr, data); + return nft_meta_set_validate(ctx, expr); } return nft_chain_validate_hooks(ctx->chain, hooks); diff --git a/net/bridge/netfilter/nft_reject_bridge.c b/net/bridge/netfilter/nft_reject_bridge.c index 71b54fed7263d..1cb5c16e97b7f 100644 --- a/net/bridge/netfilter/nft_reject_bridge.c +++ b/net/bridge/netfilter/nft_reject_bridge.c @@ -170,8 +170,7 @@ static void nft_reject_bridge_eval(const struct nft_expr *expr, } static int nft_reject_bridge_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { return nft_chain_validate_hooks(ctx->chain, (1 << NF_BR_PRE_ROUTING) | (1 << NF_BR_LOCAL_IN)); diff --git a/net/caif/cfctrl.c b/net/caif/cfctrl.c index 8480684f27625..10eeace2278b7 100644 --- a/net/caif/cfctrl.c +++ b/net/caif/cfctrl.c @@ -351,17 +351,154 @@ int cfctrl_cancel_req(struct cflayer *layr, struct cflayer *adap_layer) return found; } +static int cfctrl_link_setup(struct cfctrl *cfctrl, struct cfpkt *pkt, u8 cmdrsp) +{ + u8 len; + u8 linkid = 0; + enum cfctrl_srv serv; + enum cfctrl_srv servtype; + u8 endpoint; + u8 physlinkid; + u8 prio; + u8 tmp; + u8 *cp; + int i; + struct cfctrl_link_param linkparam; + struct cfctrl_request_info rsp, *req; + + memset(&linkparam, 0, sizeof(linkparam)); + + tmp = cfpkt_extr_head_u8(pkt); + + serv = tmp & CFCTRL_SRV_MASK; + linkparam.linktype = serv; + + servtype = tmp >> 4; + linkparam.chtype = servtype; + + tmp = cfpkt_extr_head_u8(pkt); + physlinkid = tmp & 0x07; + prio = tmp >> 3; + + linkparam.priority = prio; + linkparam.phyid = physlinkid; + endpoint = cfpkt_extr_head_u8(pkt); + linkparam.endpoint = endpoint & 0x03; + + switch (serv) { + case CFCTRL_SRV_VEI: + case CFCTRL_SRV_DBG: + if (CFCTRL_ERR_BIT & cmdrsp) + break; + /* Link ID */ + linkid = cfpkt_extr_head_u8(pkt); + break; + case CFCTRL_SRV_VIDEO: + tmp = cfpkt_extr_head_u8(pkt); + linkparam.u.video.connid = tmp; + if (CFCTRL_ERR_BIT & cmdrsp) + break; + /* Link ID */ + linkid = cfpkt_extr_head_u8(pkt); + break; + + case CFCTRL_SRV_DATAGRAM: + linkparam.u.datagram.connid = cfpkt_extr_head_u32(pkt); + if (CFCTRL_ERR_BIT & cmdrsp) + break; + /* Link ID */ + linkid = cfpkt_extr_head_u8(pkt); + break; + case CFCTRL_SRV_RFM: + /* Construct a frame, convert + * DatagramConnectionID + * to network format long and copy it out... + */ + linkparam.u.rfm.connid = cfpkt_extr_head_u32(pkt); + cp = (u8 *) linkparam.u.rfm.volume; + for (tmp = cfpkt_extr_head_u8(pkt); + cfpkt_more(pkt) && tmp != '\0'; + tmp = cfpkt_extr_head_u8(pkt)) + *cp++ = tmp; + *cp = '\0'; + + if (CFCTRL_ERR_BIT & cmdrsp) + break; + /* Link ID */ + linkid = cfpkt_extr_head_u8(pkt); + + break; + case CFCTRL_SRV_UTIL: + /* Construct a frame, convert + * DatagramConnectionID + * to network format long and copy it out... + */ + /* Fifosize KB */ + linkparam.u.utility.fifosize_kb = cfpkt_extr_head_u16(pkt); + /* Fifosize bufs */ + linkparam.u.utility.fifosize_bufs = cfpkt_extr_head_u16(pkt); + /* name */ + cp = (u8 *) linkparam.u.utility.name; + caif_assert(sizeof(linkparam.u.utility.name) + >= UTILITY_NAME_LENGTH); + for (i = 0; i < UTILITY_NAME_LENGTH && cfpkt_more(pkt); i++) { + tmp = cfpkt_extr_head_u8(pkt); + *cp++ = tmp; + } + /* Length */ + len = cfpkt_extr_head_u8(pkt); + linkparam.u.utility.paramlen = len; + /* Param Data */ + cp = linkparam.u.utility.params; + while (cfpkt_more(pkt) && len--) { + tmp = cfpkt_extr_head_u8(pkt); + *cp++ = tmp; + } + if (CFCTRL_ERR_BIT & cmdrsp) + break; + /* Link ID */ + linkid = cfpkt_extr_head_u8(pkt); + /* Length */ + len = cfpkt_extr_head_u8(pkt); + /* Param Data */ + cfpkt_extr_head(pkt, NULL, len); + break; + default: + pr_warn("Request setup, invalid type (%d)\n", serv); + return -1; + } + + rsp.cmd = CFCTRL_CMD_LINK_SETUP; + rsp.param = linkparam; + spin_lock_bh(&cfctrl->info_list_lock); + req = cfctrl_remove_req(cfctrl, &rsp); + + if (CFCTRL_ERR_BIT == (CFCTRL_ERR_BIT & cmdrsp) || + cfpkt_erroneous(pkt)) { + pr_err("Invalid O/E bit or parse error " + "on CAIF control channel\n"); + cfctrl->res.reject_rsp(cfctrl->serv.layer.up, 0, + req ? req->client_layer : NULL); + } else { + cfctrl->res.linksetup_rsp(cfctrl->serv.layer.up, linkid, + serv, physlinkid, + req ? req->client_layer : NULL); + } + + kfree(req); + + spin_unlock_bh(&cfctrl->info_list_lock); + + return 0; +} + static int cfctrl_recv(struct cflayer *layer, struct cfpkt *pkt) { u8 cmdrsp; u8 cmd; - int ret = -1; - u8 len; - u8 param[255]; + int ret = 0; u8 linkid = 0; struct cfctrl *cfctrl = container_obj(layer); - struct cfctrl_request_info rsp, *req; - cmdrsp = cfpkt_extr_head_u8(pkt); cmd = cmdrsp & CFCTRL_CMD_MASK; @@ -374,150 +511,7 @@ static int cfctrl_recv(struct cflayer *layer, struct cfpkt *pkt) switch (cmd) { case CFCTRL_CMD_LINK_SETUP: - { - enum cfctrl_srv serv; - enum cfctrl_srv servtype; - u8 endpoint; - u8 physlinkid; - u8 prio; - u8 tmp; - u8 *cp; - int i; - struct cfctrl_link_param linkparam; - memset(&linkparam, 0, sizeof(linkparam)); - - tmp = cfpkt_extr_head_u8(pkt); - - serv = tmp & CFCTRL_SRV_MASK; - linkparam.linktype = serv; - - servtype = tmp >> 4; - linkparam.chtype = servtype; - - tmp = cfpkt_extr_head_u8(pkt); - physlinkid = tmp & 0x07; - prio = tmp >> 3; - - linkparam.priority = prio; - linkparam.phyid = physlinkid; - endpoint = cfpkt_extr_head_u8(pkt); - linkparam.endpoint = endpoint & 0x03; - - switch (serv) { - case CFCTRL_SRV_VEI: - case CFCTRL_SRV_DBG: - if (CFCTRL_ERR_BIT & cmdrsp) - break; - /* Link ID */ - linkid = cfpkt_extr_head_u8(pkt); - break; - case CFCTRL_SRV_VIDEO: - tmp = cfpkt_extr_head_u8(pkt); - linkparam.u.video.connid = tmp; - if (CFCTRL_ERR_BIT & cmdrsp) - break; - /* Link ID */ - linkid = cfpkt_extr_head_u8(pkt); - break; - - case CFCTRL_SRV_DATAGRAM: - linkparam.u.datagram.connid = - cfpkt_extr_head_u32(pkt); - if (CFCTRL_ERR_BIT & cmdrsp) - break; - /* Link ID */ - linkid = cfpkt_extr_head_u8(pkt); - break; - case CFCTRL_SRV_RFM: - /* Construct a frame, convert - * DatagramConnectionID - * to network format long and copy it out... - */ - linkparam.u.rfm.connid = - cfpkt_extr_head_u32(pkt); - cp = (u8 *) linkparam.u.rfm.volume; - for (tmp = cfpkt_extr_head_u8(pkt); - cfpkt_more(pkt) && tmp != '\0'; - tmp = cfpkt_extr_head_u8(pkt)) - *cp++ = tmp; - *cp = '\0'; - - if (CFCTRL_ERR_BIT & cmdrsp) - break; - /* Link ID */ - linkid = cfpkt_extr_head_u8(pkt); - - break; - case CFCTRL_SRV_UTIL: - /* Construct a frame, convert - * DatagramConnectionID - * to network format long and copy it out... - */ - /* Fifosize KB */ - linkparam.u.utility.fifosize_kb = - cfpkt_extr_head_u16(pkt); - /* Fifosize bufs */ - linkparam.u.utility.fifosize_bufs = - cfpkt_extr_head_u16(pkt); - /* name */ - cp = (u8 *) linkparam.u.utility.name; - caif_assert(sizeof(linkparam.u.utility.name) - >= UTILITY_NAME_LENGTH); - for (i = 0; - i < UTILITY_NAME_LENGTH - && cfpkt_more(pkt); i++) { - tmp = cfpkt_extr_head_u8(pkt); - *cp++ = tmp; - } - /* Length */ - len = cfpkt_extr_head_u8(pkt); - linkparam.u.utility.paramlen = len; - /* Param Data */ - cp = linkparam.u.utility.params; - while (cfpkt_more(pkt) && len--) { - tmp = cfpkt_extr_head_u8(pkt); - *cp++ = tmp; - } - if (CFCTRL_ERR_BIT & cmdrsp) - break; - /* Link ID */ - linkid = cfpkt_extr_head_u8(pkt); - /* Length */ - len = cfpkt_extr_head_u8(pkt); - /* Param Data */ - cfpkt_extr_head(pkt, ¶m, len); - break; - default: - pr_warn("Request setup, invalid type (%d)\n", - serv); - goto error; - } - - rsp.cmd = cmd; - rsp.param = linkparam; - spin_lock_bh(&cfctrl->info_list_lock); - req = cfctrl_remove_req(cfctrl, &rsp); - - if (CFCTRL_ERR_BIT == (CFCTRL_ERR_BIT & cmdrsp) || - cfpkt_erroneous(pkt)) { - pr_err("Invalid O/E bit or parse error " - "on CAIF control channel\n"); - cfctrl->res.reject_rsp(cfctrl->serv.layer.up, - 0, - req ? req->client_layer - : NULL); - } else { - cfctrl->res.linksetup_rsp(cfctrl->serv. - layer.up, linkid, - serv, physlinkid, - req ? req-> - client_layer : NULL); - } - - kfree(req); - - spin_unlock_bh(&cfctrl->info_list_lock); - } + ret = cfctrl_link_setup(cfctrl, pkt, cmdrsp); break; case CFCTRL_CMD_LINK_DESTROY: linkid = cfpkt_extr_head_u8(pkt); @@ -544,9 +538,9 @@ static int cfctrl_recv(struct cflayer *layer, struct cfpkt *pkt) break; default: pr_err("Unrecognized Control Frame\n"); + ret = -1; goto error; } - ret = 0; error: cfpkt_destroy(pkt); return ret; diff --git a/net/can/bcm.c b/net/can/bcm.c index a1f5db0fd5d4f..75653584f31b9 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #include @@ -120,6 +121,7 @@ struct bcm_op { struct canfd_frame last_sframe; struct sock *sk; struct net_device *rx_reg_dev; + spinlock_t bcm_tx_lock; /* protect currframe/count in runtime updates */ }; struct bcm_sock { @@ -205,7 +207,9 @@ static int bcm_proc_show(struct seq_file *m, void *v) seq_printf(m, " / bound %s", bcm_proc_getifname(net, ifname, bo->ifindex)); seq_printf(m, " <<<\n"); - list_for_each_entry(op, &bo->rx_ops, list) { + rcu_read_lock(); + + list_for_each_entry_rcu(op, &bo->rx_ops, list) { unsigned long reduction; @@ -261,6 +265,9 @@ static int bcm_proc_show(struct seq_file *m, void *v) seq_printf(m, "# sent %ld\n", op->frames_abs); } seq_putc(m, '\n'); + + rcu_read_unlock(); + return 0; } #endif /* CONFIG_PROC_FS */ @@ -273,13 +280,18 @@ static void bcm_can_tx(struct bcm_op *op) { struct sk_buff *skb; struct net_device *dev; - struct canfd_frame *cf = op->frames + op->cfsiz * op->currframe; + struct canfd_frame *cf; int err; /* no target device? => exit */ if (!op->ifindex) return; + /* read currframe under lock protection */ + spin_lock_bh(&op->bcm_tx_lock); + cf = op->frames + op->cfsiz * op->currframe; + spin_unlock_bh(&op->bcm_tx_lock); + dev = dev_get_by_index(sock_net(op->sk), op->ifindex); if (!dev) { /* RFC: should this bcm_op remove itself here? */ @@ -300,6 +312,10 @@ static void bcm_can_tx(struct bcm_op *op) skb->dev = dev; can_skb_set_owner(skb, op->sk); err = can_send(skb, 1); + + /* update currframe and count under lock protection */ + spin_lock_bh(&op->bcm_tx_lock); + if (!err) op->frames_abs++; @@ -308,6 +324,11 @@ static void bcm_can_tx(struct bcm_op *op) /* reached last frame? */ if (op->currframe >= op->nframes) op->currframe = 0; + + if (op->count > 0) + op->count--; + + spin_unlock_bh(&op->bcm_tx_lock); out: dev_put(dev); } @@ -404,7 +425,7 @@ static enum hrtimer_restart bcm_tx_timeout_handler(struct hrtimer *hrtimer) struct bcm_msg_head msg_head; if (op->kt_ival1 && (op->count > 0)) { - op->count--; + bcm_can_tx(op); if (!op->count && (op->flags & TX_COUNTEVT)) { /* create notification to user */ @@ -419,7 +440,6 @@ static enum hrtimer_restart bcm_tx_timeout_handler(struct hrtimer *hrtimer) bcm_send_to_user(op, &msg_head, NULL, 0); } - bcm_can_tx(op); } else if (op->kt_ival2) { bcm_can_tx(op); @@ -801,7 +821,7 @@ static int bcm_delete_rx_op(struct list_head *ops, struct bcm_msg_head *mh, REGMASK(op->can_id), bcm_rx_handler, op); - list_del(&op->list); + list_del_rcu(&op->list); bcm_remove_op(op); return 1; /* done */ } @@ -821,7 +841,7 @@ static int bcm_delete_tx_op(struct list_head *ops, struct bcm_msg_head *mh, list_for_each_entry_safe(op, n, ops, list) { if ((op->can_id == mh->can_id) && (op->ifindex == ifindex) && (op->flags & CAN_FD_FRAME) == (mh->flags & CAN_FD_FRAME)) { - list_del(&op->list); + list_del_rcu(&op->list); bcm_remove_op(op); return 1; /* done */ } @@ -914,6 +934,27 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, } op->flags = msg_head->flags; + /* only lock for unlikely count/nframes/currframe changes */ + if (op->nframes != msg_head->nframes || + op->flags & TX_RESET_MULTI_IDX || + op->flags & SETTIMER) { + + spin_lock_bh(&op->bcm_tx_lock); + + if (op->nframes != msg_head->nframes || + op->flags & TX_RESET_MULTI_IDX) { + /* potentially update changed nframes */ + op->nframes = msg_head->nframes; + /* restart multiple frame transmission */ + op->currframe = 0; + } + + if (op->flags & SETTIMER) + op->count = msg_head->count; + + spin_unlock_bh(&op->bcm_tx_lock); + } + } else { /* insert new BCM operation for the given can_id */ @@ -921,9 +962,14 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (!op) return -ENOMEM; + spin_lock_init(&op->bcm_tx_lock); op->can_id = msg_head->can_id; op->cfsiz = CFSIZ(msg_head->flags); op->flags = msg_head->flags; + op->nframes = msg_head->nframes; + + if (op->flags & SETTIMER) + op->count = msg_head->count; /* create array for CAN frames and copy the data */ if (msg_head->nframes > 1) { @@ -982,22 +1028,8 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, } /* if ((op = bcm_find_op(&bo->tx_ops, msg_head->can_id, ifindex))) */ - if (op->nframes != msg_head->nframes) { - op->nframes = msg_head->nframes; - /* start multiple frame transmission with index 0 */ - op->currframe = 0; - } - - /* check flags */ - - if (op->flags & TX_RESET_MULTI_IDX) { - /* start multiple frame transmission with index 0 */ - op->currframe = 0; - } - if (op->flags & SETTIMER) { /* set timer values */ - op->count = msg_head->count; op->ival1 = msg_head->ival1; op->ival2 = msg_head->ival2; op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); @@ -1014,11 +1046,8 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, op->flags |= TX_ANNOUNCE; } - if (op->flags & TX_ANNOUNCE) { + if (op->flags & TX_ANNOUNCE) bcm_can_tx(op); - if (op->count) - op->count--; - } if (op->flags & STARTTIMER) bcm_tx_start_timer(op); @@ -1234,7 +1263,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, bcm_rx_handler, op, "bcm", sk); if (err) { /* this bcm rx op is broken -> remove it */ - list_del(&op->list); + list_del_rcu(&op->list); bcm_remove_op(op); return err; } diff --git a/net/can/gw.c b/net/can/gw.c index 37528826935e7..e65500c52bf5c 100644 --- a/net/can/gw.c +++ b/net/can/gw.c @@ -130,7 +130,7 @@ struct cgw_job { u32 handled_frames; u32 dropped_frames; u32 deleted_frames; - struct cf_mod mod; + struct cf_mod __rcu *cf_mod; union { /* CAN frame data source */ struct net_device *dev; @@ -459,6 +459,7 @@ static void can_can_gw_rcv(struct sk_buff *skb, void *data) struct cgw_job *gwj = (struct cgw_job *)data; struct canfd_frame *cf; struct sk_buff *nskb; + struct cf_mod *mod; int modidx = 0; /* process strictly Classic CAN or CAN FD frames */ @@ -506,7 +507,8 @@ static void can_can_gw_rcv(struct sk_buff *skb, void *data) * When there is at least one modification function activated, * we need to copy the skb as we want to modify skb->data. */ - if (gwj->mod.modfunc[0]) + mod = rcu_dereference(gwj->cf_mod); + if (mod->modfunc[0]) nskb = skb_copy(skb, GFP_ATOMIC); else nskb = skb_clone(skb, GFP_ATOMIC); @@ -529,8 +531,8 @@ static void can_can_gw_rcv(struct sk_buff *skb, void *data) cf = (struct canfd_frame *)nskb->data; /* perform preprocessed modification functions if there are any */ - while (modidx < MAX_MODFUNCTIONS && gwj->mod.modfunc[modidx]) - (*gwj->mod.modfunc[modidx++])(cf, &gwj->mod); + while (modidx < MAX_MODFUNCTIONS && mod->modfunc[modidx]) + (*mod->modfunc[modidx++])(cf, mod); /* Has the CAN frame been modified? */ if (modidx) { @@ -546,11 +548,11 @@ static void can_can_gw_rcv(struct sk_buff *skb, void *data) } /* check for checksum updates */ - if (gwj->mod.csumfunc.crc8) - (*gwj->mod.csumfunc.crc8)(cf, &gwj->mod.csum.crc8); + if (mod->csumfunc.crc8) + (*mod->csumfunc.crc8)(cf, &mod->csum.crc8); - if (gwj->mod.csumfunc.xor) - (*gwj->mod.csumfunc.xor)(cf, &gwj->mod.csum.xor); + if (mod->csumfunc.xor) + (*mod->csumfunc.xor)(cf, &mod->csum.xor); } /* clear the skb timestamp if not configured the other way */ @@ -581,9 +583,20 @@ static void cgw_job_free_rcu(struct rcu_head *rcu_head) { struct cgw_job *gwj = container_of(rcu_head, struct cgw_job, rcu); + /* cgw_job::cf_mod is always accessed from the same cgw_job object within + * the same RCU read section. Once cgw_job is scheduled for removal, + * cf_mod can also be removed without mandating an additional grace period. + */ + kfree(rcu_access_pointer(gwj->cf_mod)); kmem_cache_free(cgw_cache, gwj); } +/* Return cgw_job::cf_mod with RTNL protected section */ +static struct cf_mod *cgw_job_cf_mod(struct cgw_job *gwj) +{ + return rcu_dereference_protected(gwj->cf_mod, rtnl_is_locked()); +} + static int cgw_notifier(struct notifier_block *nb, unsigned long msg, void *ptr) { @@ -616,6 +629,7 @@ static int cgw_put_job(struct sk_buff *skb, struct cgw_job *gwj, int type, { struct rtcanmsg *rtcan; struct nlmsghdr *nlh; + struct cf_mod *mod; nlh = nlmsg_put(skb, pid, seq, type, sizeof(*rtcan), flags); if (!nlh) @@ -650,82 +664,83 @@ static int cgw_put_job(struct sk_buff *skb, struct cgw_job *gwj, int type, goto cancel; } + mod = cgw_job_cf_mod(gwj); if (gwj->flags & CGW_FLAGS_CAN_FD) { struct cgw_fdframe_mod mb; - if (gwj->mod.modtype.and) { - memcpy(&mb.cf, &gwj->mod.modframe.and, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.and; + if (mod->modtype.and) { + memcpy(&mb.cf, &mod->modframe.and, sizeof(mb.cf)); + mb.modtype = mod->modtype.and; if (nla_put(skb, CGW_FDMOD_AND, sizeof(mb), &mb) < 0) goto cancel; } - if (gwj->mod.modtype.or) { - memcpy(&mb.cf, &gwj->mod.modframe.or, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.or; + if (mod->modtype.or) { + memcpy(&mb.cf, &mod->modframe.or, sizeof(mb.cf)); + mb.modtype = mod->modtype.or; if (nla_put(skb, CGW_FDMOD_OR, sizeof(mb), &mb) < 0) goto cancel; } - if (gwj->mod.modtype.xor) { - memcpy(&mb.cf, &gwj->mod.modframe.xor, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.xor; + if (mod->modtype.xor) { + memcpy(&mb.cf, &mod->modframe.xor, sizeof(mb.cf)); + mb.modtype = mod->modtype.xor; if (nla_put(skb, CGW_FDMOD_XOR, sizeof(mb), &mb) < 0) goto cancel; } - if (gwj->mod.modtype.set) { - memcpy(&mb.cf, &gwj->mod.modframe.set, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.set; + if (mod->modtype.set) { + memcpy(&mb.cf, &mod->modframe.set, sizeof(mb.cf)); + mb.modtype = mod->modtype.set; if (nla_put(skb, CGW_FDMOD_SET, sizeof(mb), &mb) < 0) goto cancel; } } else { struct cgw_frame_mod mb; - if (gwj->mod.modtype.and) { - memcpy(&mb.cf, &gwj->mod.modframe.and, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.and; + if (mod->modtype.and) { + memcpy(&mb.cf, &mod->modframe.and, sizeof(mb.cf)); + mb.modtype = mod->modtype.and; if (nla_put(skb, CGW_MOD_AND, sizeof(mb), &mb) < 0) goto cancel; } - if (gwj->mod.modtype.or) { - memcpy(&mb.cf, &gwj->mod.modframe.or, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.or; + if (mod->modtype.or) { + memcpy(&mb.cf, &mod->modframe.or, sizeof(mb.cf)); + mb.modtype = mod->modtype.or; if (nla_put(skb, CGW_MOD_OR, sizeof(mb), &mb) < 0) goto cancel; } - if (gwj->mod.modtype.xor) { - memcpy(&mb.cf, &gwj->mod.modframe.xor, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.xor; + if (mod->modtype.xor) { + memcpy(&mb.cf, &mod->modframe.xor, sizeof(mb.cf)); + mb.modtype = mod->modtype.xor; if (nla_put(skb, CGW_MOD_XOR, sizeof(mb), &mb) < 0) goto cancel; } - if (gwj->mod.modtype.set) { - memcpy(&mb.cf, &gwj->mod.modframe.set, sizeof(mb.cf)); - mb.modtype = gwj->mod.modtype.set; + if (mod->modtype.set) { + memcpy(&mb.cf, &mod->modframe.set, sizeof(mb.cf)); + mb.modtype = mod->modtype.set; if (nla_put(skb, CGW_MOD_SET, sizeof(mb), &mb) < 0) goto cancel; } } - if (gwj->mod.uid) { - if (nla_put_u32(skb, CGW_MOD_UID, gwj->mod.uid) < 0) + if (mod->uid) { + if (nla_put_u32(skb, CGW_MOD_UID, mod->uid) < 0) goto cancel; } - if (gwj->mod.csumfunc.crc8) { + if (mod->csumfunc.crc8) { if (nla_put(skb, CGW_CS_CRC8, CGW_CS_CRC8_LEN, - &gwj->mod.csum.crc8) < 0) + &mod->csum.crc8) < 0) goto cancel; } - if (gwj->mod.csumfunc.xor) { + if (mod->csumfunc.xor) { if (nla_put(skb, CGW_CS_XOR, CGW_CS_XOR_LEN, - &gwj->mod.csum.xor) < 0) + &mod->csum.xor) < 0) goto cancel; } @@ -1059,7 +1074,7 @@ static int cgw_create_job(struct sk_buff *skb, struct nlmsghdr *nlh, struct net *net = sock_net(skb->sk); struct rtcanmsg *r; struct cgw_job *gwj; - struct cf_mod mod; + struct cf_mod *mod; struct can_can_gw ccgw; u8 limhops = 0; int err = 0; @@ -1078,37 +1093,48 @@ static int cgw_create_job(struct sk_buff *skb, struct nlmsghdr *nlh, if (r->gwtype != CGW_TYPE_CAN_CAN) return -EINVAL; - err = cgw_parse_attr(nlh, &mod, CGW_TYPE_CAN_CAN, &ccgw, &limhops); + mod = kmalloc(sizeof(*mod), GFP_KERNEL); + if (!mod) + return -ENOMEM; + + err = cgw_parse_attr(nlh, mod, CGW_TYPE_CAN_CAN, &ccgw, &limhops); if (err < 0) - return err; + goto out_free_cf; - if (mod.uid) { + if (mod->uid) { ASSERT_RTNL(); /* check for updating an existing job with identical uid */ hlist_for_each_entry(gwj, &net->can.cgw_list, list) { - if (gwj->mod.uid != mod.uid) + struct cf_mod *old_cf; + + old_cf = cgw_job_cf_mod(gwj); + if (old_cf->uid != mod->uid) continue; /* interfaces & filters must be identical */ - if (memcmp(&gwj->ccgw, &ccgw, sizeof(ccgw))) - return -EINVAL; + if (memcmp(&gwj->ccgw, &ccgw, sizeof(ccgw))) { + err = -EINVAL; + goto out_free_cf; + } - /* update modifications with disabled softirq & quit */ - local_bh_disable(); - memcpy(&gwj->mod, &mod, sizeof(mod)); - local_bh_enable(); + rcu_assign_pointer(gwj->cf_mod, mod); + kfree_rcu_mightsleep(old_cf); return 0; } } /* ifindex == 0 is not allowed for job creation */ - if (!ccgw.src_idx || !ccgw.dst_idx) - return -ENODEV; + if (!ccgw.src_idx || !ccgw.dst_idx) { + err = -ENODEV; + goto out_free_cf; + } gwj = kmem_cache_alloc(cgw_cache, GFP_KERNEL); - if (!gwj) - return -ENOMEM; + if (!gwj) { + err = -ENOMEM; + goto out_free_cf; + } gwj->handled_frames = 0; gwj->dropped_frames = 0; @@ -1118,7 +1144,7 @@ static int cgw_create_job(struct sk_buff *skb, struct nlmsghdr *nlh, gwj->limit_hops = limhops; /* insert already parsed information */ - memcpy(&gwj->mod, &mod, sizeof(mod)); + RCU_INIT_POINTER(gwj->cf_mod, mod); memcpy(&gwj->ccgw, &ccgw, sizeof(ccgw)); err = -ENODEV; @@ -1152,9 +1178,11 @@ static int cgw_create_job(struct sk_buff *skb, struct nlmsghdr *nlh, if (!err) hlist_add_head_rcu(&gwj->list, &net->can.cgw_list); out: - if (err) + if (err) { kmem_cache_free(cgw_cache, gwj); - +out_free_cf: + kfree(mod); + } return err; } @@ -1214,19 +1242,22 @@ static int cgw_remove_job(struct sk_buff *skb, struct nlmsghdr *nlh, /* remove only the first matching entry */ hlist_for_each_entry_safe(gwj, nx, &net->can.cgw_list, list) { + struct cf_mod *cf_mod; + if (gwj->flags != r->flags) continue; if (gwj->limit_hops != limhops) continue; + cf_mod = cgw_job_cf_mod(gwj); /* we have a match when uid is enabled and identical */ - if (gwj->mod.uid || mod.uid) { - if (gwj->mod.uid != mod.uid) + if (cf_mod->uid || mod.uid) { + if (cf_mod->uid != mod.uid) continue; } else { /* no uid => check for identical modifications */ - if (memcmp(&gwj->mod, &mod, sizeof(mod))) + if (memcmp(cf_mod, &mod, sizeof(mod))) continue; } diff --git a/net/can/j1939/bus.c b/net/can/j1939/bus.c index 4866879016021..e0b966c2517cf 100644 --- a/net/can/j1939/bus.c +++ b/net/can/j1939/bus.c @@ -290,8 +290,11 @@ int j1939_local_ecu_get(struct j1939_priv *priv, name_t name, u8 sa) if (!ecu) ecu = j1939_ecu_create_locked(priv, name); err = PTR_ERR_OR_ZERO(ecu); - if (err) + if (err) { + if (j1939_address_is_unicast(sa)) + priv->ents[sa].nusers--; goto done; + } ecu->nusers++; /* TODO: do we care if ecu->addr != sa? */ diff --git a/net/can/j1939/socket.c b/net/can/j1939/socket.c index cf9a12d8da6f9..7bf4d4fb96735 100644 --- a/net/can/j1939/socket.c +++ b/net/can/j1939/socket.c @@ -520,6 +520,9 @@ static int j1939_sk_bind(struct socket *sock, struct sockaddr *uaddr, int len) ret = j1939_local_ecu_get(priv, jsk->addr.src_name, jsk->addr.sa); if (ret) { j1939_netdev_stop(priv); + jsk->priv = NULL; + synchronize_rcu(); + j1939_priv_put(priv); goto out_release_sock; } diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 3c8b78d9c4d1c..8add91385375e 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -1524,7 +1524,7 @@ static void con_fault_finish(struct ceph_connection *con) * in case we faulted due to authentication, invalidate our * current tickets so that we can get new ones. */ - if (con->v1.auth_retry) { + if (!ceph_msgr2(from_msgr(con->msgr)) && con->v1.auth_retry) { dout("auth_retry %d, invalidating\n", con->v1.auth_retry); if (con->ops->invalidate_authorizer) con->ops->invalidate_authorizer(con); @@ -1714,9 +1714,10 @@ static void clear_standby(struct ceph_connection *con) { /* come back from STANDBY? */ if (con->state == CEPH_CON_S_STANDBY) { - dout("clear_standby %p and ++connect_seq\n", con); + dout("clear_standby %p\n", con); con->state = CEPH_CON_S_PREOPEN; - con->v1.connect_seq++; + if (!ceph_msgr2(from_msgr(con->msgr))) + con->v1.connect_seq++; WARN_ON(ceph_con_flag_test(con, CEPH_CON_F_WRITE_PENDING)); WARN_ON(ceph_con_flag_test(con, CEPH_CON_F_KEEPALIVE_PENDING)); } diff --git a/net/core/datagram.c b/net/core/datagram.c index ef4e9e423d393..9a478b9dd6d8a 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -618,6 +618,20 @@ int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset, } EXPORT_SYMBOL(skb_copy_datagram_from_iter); +int skb_copy_datagram_from_iter_full(struct sk_buff *skb, int offset, + struct iov_iter *from, int len) +{ + struct iov_iter_state state; + int ret; + + iov_iter_save_state(from, &state); + ret = skb_copy_datagram_from_iter(skb, offset, from, len); + if (ret) + iov_iter_restore(from, &state); + return ret; +} +EXPORT_SYMBOL(skb_copy_datagram_from_iter_full); + int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, struct sk_buff *skb, struct iov_iter *from, size_t length) diff --git a/net/core/dev.c b/net/core/dev.c index f58d314e7bd3f..975513fb39a6d 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3567,6 +3567,18 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, features &= ~NETIF_F_TSO_MANGLEID; } + /* NETIF_F_IPV6_CSUM does not support IPv6 extension headers, + * so neither does TSO that depends on it. + */ + if (features & NETIF_F_IPV6_CSUM && + (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && + vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && + skb_transport_header_was_set(skb) && + skb_network_header_len(skb) != sizeof(struct ipv6hdr) && + !ipv6_has_hopopt_jumbo(skb)) + features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4); + return features; } diff --git a/net/core/dst.c b/net/core/dst.c index aad197e761cb4..193ed3fa3c902 100644 --- a/net/core/dst.c +++ b/net/core/dst.c @@ -150,9 +150,10 @@ void dst_dev_put(struct dst_entry *dst) dst->obsolete = DST_OBSOLETE_DEAD; if (dst->ops->ifdown) dst->ops->ifdown(dst, dev); - dst->input = dst_discard; - dst->output = dst_discard_out; - dst->dev = blackhole_netdev; + + WRITE_ONCE(dst->input, dst_discard); + WRITE_ONCE(dst->output, dst_discard_out); + WRITE_ONCE(dst->dev, blackhole_netdev); netdev_ref_replace(dev, blackhole_netdev, &dst->dev_tracker, GFP_ATOMIC); } @@ -265,7 +266,7 @@ unsigned int dst_blackhole_mtu(const struct dst_entry *dst) { unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); - return mtu ? : dst->dev->mtu; + return mtu ? : dst_dev(dst)->mtu; } EXPORT_SYMBOL_GPL(dst_blackhole_mtu); diff --git a/net/core/filter.c b/net/core/filter.c index d3ac5973fc81a..c4c13db233bfc 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1976,10 +1976,11 @@ BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset, bool is_pseudo = flags & BPF_F_PSEUDO_HDR; bool is_mmzero = flags & BPF_F_MARK_MANGLED_0; bool do_mforce = flags & BPF_F_MARK_ENFORCE; + bool is_ipv6 = flags & BPF_F_IPV6; __sum16 *ptr; if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE | - BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK))) + BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK | BPF_F_IPV6))) return -EINVAL; if (unlikely(offset > 0xffff || offset & 1)) return -EFAULT; @@ -1995,7 +1996,7 @@ BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset, if (unlikely(from != 0)) return -EINVAL; - inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo); + inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo, is_ipv6); break; case 2: inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo); @@ -2280,6 +2281,7 @@ static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev, if (IS_ERR(dst)) goto out_drop; + skb_dst_drop(skb); skb_dst_set(skb, dst); } else if (nh->nh_family != AF_INET6) { goto out_drop; @@ -2389,6 +2391,7 @@ static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev, goto out_drop; } + skb_dst_drop(skb); skb_dst_set(skb, &rt->dst); } @@ -2515,6 +2518,7 @@ int skb_do_redirect(struct sk_buff *skb) goto out_drop; skb->dev = dev; dev_sw_netstats_rx_add(dev, skb->len); + skb_scrub_packet(skb, false); return -EAGAIN; } return flags & BPF_F_NEIGH ? @@ -3236,6 +3240,13 @@ static const struct bpf_func_proto bpf_skb_vlan_pop_proto = { .arg1_type = ARG_PTR_TO_CTX, }; +static void bpf_skb_change_protocol(struct sk_buff *skb, u16 proto) +{ + skb->protocol = htons(proto); + if (skb_valid_dst(skb)) + skb_dst_drop(skb); +} + static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len) { /* Caller already did skb_cow() with len as headroom, @@ -3332,7 +3343,7 @@ static int bpf_skb_proto_4_to_6(struct sk_buff *skb) } } - skb->protocol = htons(ETH_P_IPV6); + bpf_skb_change_protocol(skb, ETH_P_IPV6); skb_clear_hash(skb); return 0; @@ -3362,7 +3373,7 @@ static int bpf_skb_proto_6_to_4(struct sk_buff *skb) } } - skb->protocol = htons(ETH_P_IP); + bpf_skb_change_protocol(skb, ETH_P_IP); skb_clear_hash(skb); return 0; @@ -3553,10 +3564,10 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff, /* Match skb->protocol to new outer l3 protocol */ if (skb->protocol == htons(ETH_P_IP) && flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6) - skb->protocol = htons(ETH_P_IPV6); + bpf_skb_change_protocol(skb, ETH_P_IPV6); else if (skb->protocol == htons(ETH_P_IPV6) && flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4) - skb->protocol = htons(ETH_P_IP); + bpf_skb_change_protocol(skb, ETH_P_IP); } if (skb_is_gso(skb)) { @@ -3609,10 +3620,10 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, /* Match skb->protocol to new outer l3 protocol */ if (skb->protocol == htons(ETH_P_IP) && flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV6) - skb->protocol = htons(ETH_P_IPV6); + bpf_skb_change_protocol(skb, ETH_P_IPV6); else if (skb->protocol == htons(ETH_P_IPV6) && flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV4) - skb->protocol = htons(ETH_P_IP); + bpf_skb_change_protocol(skb, ETH_P_IP); if (skb_is_gso(skb)) { struct skb_shared_info *shinfo = skb_shinfo(skb); @@ -4200,6 +4211,7 @@ static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset) if (unlikely(!sinfo->nr_frags)) { xdp_buff_clear_frags_flag(xdp); + xdp_buff_clear_frag_pfmemalloc(xdp); xdp->data_end -= offset; } @@ -6349,7 +6361,8 @@ static const struct bpf_func_proto bpf_skb_check_mtu_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_INT, + .arg3_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_ALIGNED, + .arg3_size = sizeof(u32), .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -6360,7 +6373,8 @@ static const struct bpf_func_proto bpf_xdp_check_mtu_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_INT, + .arg3_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_ALIGNED, + .arg3_size = sizeof(u32), .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -9170,13 +9184,17 @@ static bool sock_addr_is_valid_access(int off, int size, return false; info->reg_type = PTR_TO_SOCKET; break; - default: - if (type == BPF_READ) { - if (size != size_default) - return false; - } else { + case bpf_ctx_range(struct bpf_sock_addr, user_family): + case bpf_ctx_range(struct bpf_sock_addr, family): + case bpf_ctx_range(struct bpf_sock_addr, type): + case bpf_ctx_range(struct bpf_sock_addr, protocol): + if (type != BPF_READ) return false; - } + if (size != size_default) + return false; + break; + default: + return false; } return true; @@ -9344,6 +9362,9 @@ static bool flow_dissector_is_valid_access(int off, int size, if (off < 0 || off >= sizeof(struct __sk_buff)) return false; + if (off % size != 0) + return false; + if (type == BPF_WRITE) return false; diff --git a/net/core/gro.c b/net/core/gro.c index 397cf59842503..b8cc44406e69b 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -6,9 +6,6 @@ #define MAX_GRO_SKBS 8 -/* This should be increased if a protocol with a bigger head is added. */ -#define GRO_MAX_HEAD (MAX_HEADER + 128) - static DEFINE_SPINLOCK(offload_lock); struct list_head offload_base __read_mostly = LIST_HEAD_INIT(offload_base); /* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 018e213185a17..372db9e68fbde 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -442,7 +442,7 @@ static struct net *net_alloc(void) goto out; } -static void net_free(struct net *net) +void net_passive_dec(struct net *net) { if (refcount_dec_and_test(&net->passive)) { kfree(rcu_access_pointer(net->gen)); @@ -459,7 +459,7 @@ void net_drop_ns(void *p) struct net *net = (struct net *)p; if (net) - net_free(net); + net_passive_dec(net); } struct net *copy_net_ns(unsigned long flags, @@ -501,7 +501,7 @@ struct net *copy_net_ns(unsigned long flags, key_remove_domain(net->key_domain); #endif put_user_ns(user_ns); - net_free(net); + net_passive_dec(net); dec_ucounts: dec_net_namespaces(ucounts); return ERR_PTR(rv); @@ -635,7 +635,7 @@ static void cleanup_net(struct work_struct *work) key_remove_domain(net->key_domain); #endif put_user_ns(net->user_ns); - net_free(net); + net_passive_dec(net); } } diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 1a4d2a61b060b..a78340bb25ba6 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -45,11 +45,6 @@ #define MAX_UDP_CHUNK 1460 #define MAX_SKBS 32 - -static struct sk_buff_head skb_pool; - -DEFINE_STATIC_SRCU(netpoll_srcu); - #define USEC_PER_POLL 50 #define MAX_SKB_SIZE \ @@ -220,41 +215,41 @@ EXPORT_SYMBOL(netpoll_poll_dev); void netpoll_poll_disable(struct net_device *dev) { struct netpoll_info *ni; - int idx; + might_sleep(); - idx = srcu_read_lock(&netpoll_srcu); - ni = srcu_dereference(dev->npinfo, &netpoll_srcu); + ni = rtnl_dereference(dev->npinfo); if (ni) down(&ni->dev_lock); - srcu_read_unlock(&netpoll_srcu, idx); } EXPORT_SYMBOL(netpoll_poll_disable); void netpoll_poll_enable(struct net_device *dev) { struct netpoll_info *ni; - rcu_read_lock(); - ni = rcu_dereference(dev->npinfo); + + ni = rtnl_dereference(dev->npinfo); if (ni) up(&ni->dev_lock); - rcu_read_unlock(); } EXPORT_SYMBOL(netpoll_poll_enable); -static void refill_skbs(void) +static void refill_skbs(struct netpoll *np) { + struct sk_buff_head *skb_pool; struct sk_buff *skb; unsigned long flags; - spin_lock_irqsave(&skb_pool.lock, flags); - while (skb_pool.qlen < MAX_SKBS) { + skb_pool = &np->skb_pool; + + spin_lock_irqsave(&skb_pool->lock, flags); + while (skb_pool->qlen < MAX_SKBS) { skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC); if (!skb) break; - __skb_queue_tail(&skb_pool, skb); + __skb_queue_tail(skb_pool, skb); } - spin_unlock_irqrestore(&skb_pool.lock, flags); + spin_unlock_irqrestore(&skb_pool->lock, flags); } static void zap_completion_queue(void) @@ -291,12 +286,12 @@ static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve) struct sk_buff *skb; zap_completion_queue(); - refill_skbs(); + refill_skbs(np); repeat: skb = alloc_skb(len, GFP_ATOMIC); if (!skb) - skb = skb_dequeue(&skb_pool); + skb = skb_dequeue(&np->skb_pool); if (!skb) { if (++count < 10) { @@ -543,6 +538,14 @@ static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr) return -1; } +static void skb_pool_flush(struct netpoll *np) +{ + struct sk_buff_head *skb_pool; + + skb_pool = &np->skb_pool; + skb_queue_purge_reason(skb_pool, SKB_CONSUMED); +} + int netpoll_parse_options(struct netpoll *np, char *opt) { char *cur=opt, *delim; @@ -631,6 +634,8 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev) const struct net_device_ops *ops; int err; + skb_queue_head_init(&np->skb_pool); + if (ndev->priv_flags & IFF_DISABLE_NETPOLL) { np_err(np, "%s doesn't support polling, aborting\n", ndev->name); @@ -666,6 +671,9 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev) strscpy(np->dev_name, ndev->name, IFNAMSIZ); npinfo->netpoll = np; + /* fill up the skb queue */ + refill_skbs(np); + /* last thing to do is link it to the net device structure */ rcu_assign_pointer(ndev->npinfo, npinfo); @@ -784,15 +792,21 @@ int netpoll_setup(struct netpoll *np) } } - /* fill up the skb queue */ - refill_skbs(); - err = __netpoll_setup(np, ndev); if (err) - goto put; + goto flush; rtnl_unlock(); + + /* Make sure all NAPI polls which started before dev->npinfo + * was visible have exited before we start calling NAPI poll. + * NAPI skips locking if dev->npinfo is NULL. + */ + synchronize_rcu(); + return 0; +flush: + skb_pool_flush(np); put: DEBUG_NET_WARN_ON_ONCE(np->dev); if (ip_overwritten) @@ -804,13 +818,6 @@ int netpoll_setup(struct netpoll *np) } EXPORT_SYMBOL(netpoll_setup); -static int __init netpoll_init(void) -{ - skb_queue_head_init(&skb_pool); - return 0; -} -core_initcall(netpoll_init); - static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head) { struct netpoll_info *npinfo = @@ -836,8 +843,10 @@ void __netpoll_cleanup(struct netpoll *np) if (!npinfo) return; - synchronize_srcu(&netpoll_srcu); - + /* At this point, there is a single npinfo instance per netdevice, and + * its refcnt tracks how many netpoll structures are linked to it. We + * only perform npinfo cleanup when the refcnt decrements to zero. + */ if (refcount_dec_and_test(&npinfo->refcnt)) { const struct net_device_ops *ops; @@ -847,8 +856,9 @@ void __netpoll_cleanup(struct netpoll *np) RCU_INIT_POINTER(np->dev->npinfo, NULL); call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info); - } else - RCU_INIT_POINTER(np->dev->npinfo, NULL); + } + + skb_pool_flush(np); } EXPORT_SYMBOL_GPL(__netpoll_cleanup); diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 2f2f63c8cf4b0..0188d7f007857 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -180,11 +180,7 @@ static int page_pool_init(struct page_pool *pool, return -EINVAL; if (pool->p.pool_size) - ring_qsize = pool->p.pool_size; - - /* Sanity limit mem that can be pinned down */ - if (ring_qsize > 32768) - return -E2BIG; + ring_qsize = min(pool->p.pool_size, 16384); /* DMA direction is either DMA_FROM_DEVICE or DMA_BIDIRECTIONAL. * DMA_BIDIRECTIONAL is for allowing page used for DMA sending, @@ -422,6 +418,12 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, struct page *page; int i, nr_pages; + /* Unconditionally set NOWARN if allocating from NAPI. + * Drivers forget to set it, and OOM reports on packet Rx are useless. + */ + if ((gfp & GFP_ATOMIC) == GFP_ATOMIC) + gfp |= __GFP_NOWARN; + /* Don't support bulk alloc for high-order pages */ if (unlikely(pp_order)) return __page_pool_alloc_page_order(pool, gfp); diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 359e24c3f22ca..6afea369ca213 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -897,6 +897,10 @@ static ssize_t get_labels(const char __user *buffer, struct pktgen_dev *pkt_dev) pkt_dev->nr_labels = 0; do { __u32 tmp; + + if (n >= MAX_MPLS_LABELS) + return -E2BIG; + len = hex32_arg(&buffer[i], 8, &tmp); if (len <= 0) return len; @@ -908,8 +912,6 @@ static ssize_t get_labels(const char __user *buffer, struct pktgen_dev *pkt_dev) return -EFAULT; i++; n++; - if (n >= MAX_MPLS_LABELS) - return -E2BIG; } while (c == ','); pkt_dev->nr_labels = n; @@ -1875,8 +1877,8 @@ static ssize_t pktgen_thread_write(struct file *file, i = len; /* Read variable name */ - - len = strn_len(&user_buffer[i], sizeof(name) - 1); + max = min(sizeof(name) - 1, count - i); + len = strn_len(&user_buffer[i], max); if (len < 0) return len; @@ -1906,7 +1908,8 @@ static ssize_t pktgen_thread_write(struct file *file, if (!strcmp(name, "add_device")) { char f[32]; memset(f, 0, 32); - len = strn_len(&user_buffer[i], sizeof(f) - 1); + max = min(sizeof(f) - 1, count - i); + len = strn_len(&user_buffer[i], max); if (len < 0) { ret = len; goto out; diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 105db7712378f..0da3050ad60c0 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -4383,9 +4383,6 @@ static int rtnl_fdb_del(struct sk_buff *skb, struct nlmsghdr *nlh, int err; u16 vid; - if (!netlink_capable(skb, CAP_NET_ADMIN)) - return -EPERM; - if (!del_bulk) { err = nlmsg_parse_deprecated(nlh, sizeof(*ndm), tb, NDA_MAX, NULL, extack); diff --git a/net/core/selftests.c b/net/core/selftests.c index 7af99d07762ea..946e92cca2111 100644 --- a/net/core/selftests.c +++ b/net/core/selftests.c @@ -160,8 +160,9 @@ static struct sk_buff *net_test_get_skb(struct net_device *ndev, skb->csum = 0; skb->ip_summed = CHECKSUM_PARTIAL; if (attr->tcp) { - thdr->check = ~tcp_v4_check(skb->len, ihdr->saddr, - ihdr->daddr, 0); + int l4len = skb->len - skb_transport_offset(skb); + + thdr->check = ~tcp_v4_check(l4len, ihdr->saddr, ihdr->daddr, 0); skb->csum_start = skb_transport_header(skb) - skb->head; skb->csum_offset = offsetof(struct tcphdr, check); } else { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 3e3a3ecc48584..c146a59a67609 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -67,6 +67,9 @@ #include #include #include +#ifndef __GENKSYMS__ +#include +#endif #include #include #include @@ -98,7 +101,9 @@ static struct kmem_cache *skbuff_ext_cache __ro_after_init; static struct kmem_cache *skb_small_head_cache __ro_after_init; -#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER) +#define GRO_MAX_HEAD_PAD (GRO_MAX_HEAD + NET_SKB_PAD + NET_IP_ALIGN) +#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(max(MAX_TCP_HEADER, \ + GRO_MAX_HEAD_PAD)) /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two. * This should ensure that SKB_SMALL_HEAD_HEADROOM is a unique @@ -714,7 +719,7 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, /* If requested length is either too small or too big, * we use kmalloc() for skb->head allocation. */ - if (len <= SKB_WITH_OVERHEAD(1024) || + if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) || len > SKB_WITH_OVERHEAD(PAGE_SIZE) || (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); @@ -791,7 +796,8 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, * When the small frag allocator is available, prefer it over kmalloc * for small fragments */ - if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) || + if ((!NAPI_HAS_SMALL_PAGE_FRAG && + len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)) || len > SKB_WITH_OVERHEAD(PAGE_SIZE) || (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, @@ -6336,7 +6342,7 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, return NULL; while (data_len) { - if (nr_frags == MAX_SKB_FRAGS - 1) + if (nr_frags == MAX_SKB_FRAGS) goto failure; while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order)) order--; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index b9b941c487c8a..6225547808a6b 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -529,16 +529,22 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb, u32 off, u32 len, struct sk_psock *psock, struct sock *sk, - struct sk_msg *msg) + struct sk_msg *msg, + bool take_ref) { int num_sge, copied; + /* skb_to_sgvec will fail when the total number of fragments in + * frag_list and frags exceeds MAX_MSG_FRAGS. For example, the + * caller may aggregate multiple skbs. + */ num_sge = skb_to_sgvec(skb, msg->sg.data, off, len); if (num_sge < 0) { /* skb linearize may fail with ENOMEM, but lets simply try again * later if this happens. Under memory pressure we don't want to * drop the skb. We need to linearize the skb so that the mapping * in skb_to_sgvec can not error. + * Note that skb_linearize requires the skb not to be shared. */ if (skb_linearize(skb)) return -EAGAIN; @@ -555,7 +561,7 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb, msg->sg.start = 0; msg->sg.size = copied; msg->sg.end = num_sge; - msg->skb = skb; + msg->skb = take_ref ? skb_get(skb) : skb; sk_psock_queue_msg(psock, msg); sk_psock_data_ready(sk, psock); @@ -563,7 +569,7 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb, } static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb, - u32 off, u32 len); + u32 off, u32 len, bool take_ref); static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, u32 off, u32 len) @@ -577,7 +583,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, * correctly. */ if (unlikely(skb->sk == sk)) - return sk_psock_skb_ingress_self(psock, skb, off, len); + return sk_psock_skb_ingress_self(psock, skb, off, len, true); msg = sk_psock_create_ingress_msg(sk, skb); if (!msg) return -EAGAIN; @@ -589,7 +595,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, * into user buffers. */ skb_set_owner_r(skb, sk); - err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg); + err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, true); if (err < 0) kfree(msg); return err; @@ -600,7 +606,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, * because the skb is already accounted for here. */ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb, - u32 off, u32 len) + u32 off, u32 len, bool take_ref) { struct sk_msg *msg = alloc_sk_msg(GFP_ATOMIC); struct sock *sk = psock->sk; @@ -609,7 +615,7 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb if (unlikely(!msg)) return -EAGAIN; skb_set_owner_r(skb, sk); - err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg); + err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref); if (err < 0) kfree(msg); return err; @@ -618,18 +624,13 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, u32 off, u32 len, bool ingress) { - int err = 0; - if (!ingress) { if (!sock_writeable(psock->sk)) return -EAGAIN; return skb_send_sock(psock->sk, skb, off, len); } - skb_get(skb); - err = sk_psock_skb_ingress(psock, skb, off, len); - if (err < 0) - kfree_skb(skb); - return err; + + return sk_psock_skb_ingress(psock, skb, off, len); } static void sk_psock_skb_state(struct sk_psock *psock, @@ -654,12 +655,21 @@ static void sk_psock_backlog(struct work_struct *work) bool ingress; int ret; - mutex_lock(&psock->work_mutex); - if (unlikely(state->len)) { - len = state->len; - off = state->off; - } + /* If sk is quickly removed from the map and then added back, the old + * psock should not be scheduled, because there are now two psocks + * pointing to the same sk. + */ + if (!sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) + return; + /* Increment the psock refcnt to synchronize with close(fd) path in + * sock_map_close(), ensuring we wait for backlog thread completion + * before sk_socket freed. If refcnt increment fails, it indicates + * sock_map_close() completed with sk_socket potentially already freed. + */ + if (!sk_psock_get(psock->sk)) + return; + mutex_lock(&psock->work_mutex); while ((skb = skb_peek(&psock->ingress_skb))) { len = skb->len; off = 0; @@ -669,6 +679,13 @@ static void sk_psock_backlog(struct work_struct *work) off = stm->offset; len = stm->full_len; } + + /* Resume processing from previous partial state */ + if (unlikely(state->len)) { + len = state->len; + off = state->off; + } + ingress = skb_bpf_ingress(skb); skb_bpf_redirect_clear(skb); do { @@ -679,7 +696,8 @@ static void sk_psock_backlog(struct work_struct *work) if (ret <= 0) { if (ret == -EAGAIN) { sk_psock_skb_state(psock, state, len, off); - + /* Restore redir info we cleared before */ + skb_bpf_set_redir(skb, psock->sk, ingress); /* Delay slightly to prioritize any * other work that might be here. */ @@ -696,11 +714,14 @@ static void sk_psock_backlog(struct work_struct *work) len -= ret; } while (len); + /* The entire skb sent, clear state */ + sk_psock_skb_state(psock, state, 0, 0); skb = skb_dequeue(&psock->ingress_skb); kfree_skb(skb); } end: mutex_unlock(&psock->work_mutex); + sk_psock_put(psock->sk, psock); } struct sk_psock *sk_psock_init(struct sock *sk, int node) @@ -1013,7 +1034,7 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, off = stm->offset; len = stm->full_len; } - err = sk_psock_skb_ingress_self(psock, skb, off, len); + err = sk_psock_skb_ingress_self(psock, skb, off, len, false); } if (err < 0) { spin_lock_bh(&psock->ingress_lock); diff --git a/net/core/sock.c b/net/core/sock.c index f530e2efa0fb2..2508349c46bf9 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -2031,6 +2031,8 @@ int sk_getsockopt(struct sock *sk, int level, int optname, */ static inline void sock_lock_init(struct sock *sk) { + sk_owner_clear(sk); + if (sk->sk_kern_sock) sock_lock_init_class_and_name( sk, @@ -2127,6 +2129,9 @@ static void sk_prot_free(struct proto *prot, struct sock *sk) mem_cgroup_sk_free(sk); trace_android_vh_sk_free(sk); security_sk_free(sk); + + sk_owner_put(sk); + if (slab != NULL) kmem_cache_free(slab, sk); else @@ -2162,6 +2167,7 @@ struct sock *sk_alloc(struct net *net, int family, gfp_t priority, get_net_track(net, &sk->ns_tracker, priority); sock_inuse_add(net, 1); } else { + net_passive_inc(net); __netns_tracker_alloc(net, &sk->ns_tracker, false, priority); } @@ -2187,6 +2193,7 @@ EXPORT_SYMBOL(sk_alloc); static void __sk_destruct(struct rcu_head *head) { struct sock *sk = container_of(head, struct sock, sk_rcu); + struct net *net = sock_net(sk); struct sk_filter *filter; if (sk->sk_destruct) @@ -2218,14 +2225,28 @@ static void __sk_destruct(struct rcu_head *head) put_cred(sk->sk_peer_cred); put_pid(sk->sk_peer_pid); - if (likely(sk->sk_net_refcnt)) - put_net_track(sock_net(sk), &sk->ns_tracker); - else - __netns_tracker_free(sock_net(sk), &sk->ns_tracker, false); - + if (likely(sk->sk_net_refcnt)) { + put_net_track(net, &sk->ns_tracker); + } else { + __netns_tracker_free(net, &sk->ns_tracker, false); + net_passive_dec(net); + } sk_prot_free(sk->sk_prot_creator, sk); } +void sk_net_refcnt_upgrade(struct sock *sk) +{ + struct net *net = sock_net(sk); + + WARN_ON_ONCE(sk->sk_net_refcnt); + __netns_tracker_free(net, &sk->ns_tracker, false); + net_passive_dec(net); + sk->sk_net_refcnt = 1; + get_net_track(net, &sk->ns_tracker, GFP_KERNEL); + sock_inuse_add(net, 1); +} +EXPORT_SYMBOL_GPL(sk_net_refcnt_upgrade); + void sk_destruct(struct sock *sk) { bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); @@ -2318,6 +2339,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) * is not properly dismantling its kernel sockets at netns * destroy time. */ + net_passive_inc(sock_net(newsk)); __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, false, priority); } @@ -2433,8 +2455,8 @@ static u32 sk_dst_gso_max_size(struct sock *sk, struct dst_entry *dst) !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); #endif /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ - max_size = is_ipv6 ? READ_ONCE(dst->dev->gso_max_size) : - READ_ONCE(dst->dev->gso_ipv4_max_size); + max_size = is_ipv6 ? READ_ONCE(dst_dev(dst)->gso_max_size) : + READ_ONCE(dst_dev(dst)->gso_ipv4_max_size); if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) max_size = GSO_LEGACY_MAX_SIZE; @@ -2445,7 +2467,7 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) { u32 max_segs = 1; - sk->sk_route_caps = dst->dev->features; + sk->sk_route_caps = dst_dev(dst)->features; if (sk_is_tcp(sk)) sk->sk_route_caps |= NETIF_F_GSO; if (sk->sk_route_caps & NETIF_F_GSO) @@ -2459,7 +2481,7 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dst); /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ - max_segs = max_t(u32, READ_ONCE(dst->dev->gso_max_segs), 1); + max_segs = max_t(u32, READ_ONCE(dst_dev(dst)->gso_max_segs), 1); } } sk->sk_gso_max_segs = max_segs; @@ -2979,23 +3001,27 @@ void __release_sock(struct sock *sk) __acquires(&sk->sk_lock.slock) { struct sk_buff *skb, *next; + int nb = 0; while ((skb = sk->sk_backlog.head) != NULL) { sk->sk_backlog.head = sk->sk_backlog.tail = NULL; spin_unlock_bh(&sk->sk_lock.slock); - do { + while (1) { next = skb->next; prefetch(next); DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); skb_mark_not_on_list(skb); sk_backlog_rcv(sk, skb); - cond_resched(); - skb = next; - } while (skb != NULL); + if (!skb) + break; + + if (!(++nb & 15)) + cond_resched(); + } spin_lock_bh(&sk->sk_lock.slock); } @@ -3124,8 +3150,7 @@ int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) } } - if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) - trace_sock_exceed_buf_limit(sk, prot, allocated, kind); + trace_sock_exceed_buf_limit(sk, prot, allocated, kind); sk_memory_allocated_sub(sk, amt); @@ -3832,7 +3857,7 @@ static int assign_proto_idx(struct proto *prot) { prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); - if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { + if (unlikely(prot->inuse_idx == PROTO_INUSE_NR)) { pr_err("PROTO_INUSE_NR exhausted\n"); return -ENOSPC; } @@ -3843,7 +3868,7 @@ static int assign_proto_idx(struct proto *prot) static void release_proto_idx(struct proto *prot) { - if (prot->inuse_idx != PROTO_INUSE_NR - 1) + if (prot->inuse_idx != PROTO_INUSE_NR) clear_bit(prot->inuse_idx, proto_inuse_idx); } #else diff --git a/net/core/utils.c b/net/core/utils.c index c994e95172acf..5895d034bf279 100644 --- a/net/core/utils.c +++ b/net/core/utils.c @@ -473,11 +473,11 @@ void inet_proto_csum_replace16(__sum16 *sum, struct sk_buff *skb, EXPORT_SYMBOL(inet_proto_csum_replace16); void inet_proto_csum_replace_by_diff(__sum16 *sum, struct sk_buff *skb, - __wsum diff, bool pseudohdr) + __wsum diff, bool pseudohdr, bool ipv6) { if (skb->ip_summed != CHECKSUM_PARTIAL) { csum_replace_by_diff(sum, diff); - if (skb->ip_summed == CHECKSUM_COMPLETE && pseudohdr) + if (skb->ip_summed == CHECKSUM_COMPLETE && pseudohdr && !ipv6) skb->csum = ~csum_sub(diff, skb->csum); } else if (pseudohdr) { *sum = ~csum_fold(csum_add(diff, csum_unfold(*sum))); diff --git a/net/devlink/rate.c b/net/devlink/rate.c index dff1593b8406a..e7e80af4abaf6 100644 --- a/net/devlink/rate.c +++ b/net/devlink/rate.c @@ -702,13 +702,15 @@ void devl_rate_nodes_destroy(struct devlink *devlink) if (!devlink_rate->parent) continue; - refcount_dec(&devlink_rate->parent->refcnt); if (devlink_rate_is_leaf(devlink_rate)) ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv, NULL, NULL); else if (devlink_rate_is_node(devlink_rate)) ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv, NULL, NULL); + + refcount_dec(&devlink_rate->parent->refcnt); + devlink_rate->parent = NULL; } list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) { if (devlink_rate_is_node(devlink_rate)) { diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c index 07736edc8b6a5..c9bf1a9a6c99b 100644 --- a/net/dsa/dsa.c +++ b/net/dsa/dsa.c @@ -1613,6 +1613,7 @@ EXPORT_SYMBOL_GPL(dsa_unregister_switch); void dsa_switch_shutdown(struct dsa_switch *ds) { struct net_device *master, *slave_dev; + LIST_HEAD(close_list); struct dsa_port *dp; mutex_lock(&dsa2_mutex); @@ -1622,10 +1623,16 @@ void dsa_switch_shutdown(struct dsa_switch *ds) rtnl_lock(); + dsa_switch_for_each_cpu_port(dp, ds) + list_add(&dp->master->close_list, &close_list); + + dev_close_many(&close_list, true); + dsa_switch_for_each_user_port(dp, ds) { master = dsa_port_to_master(dp); slave_dev = dp->slave; + netif_device_detach(slave_dev); netdev_upper_dev_unlink(master, slave_dev); } diff --git a/net/dsa/tag_brcm.c b/net/dsa/tag_brcm.c index cacdafb41200e..385581cf3b7ba 100644 --- a/net/dsa/tag_brcm.c +++ b/net/dsa/tag_brcm.c @@ -255,12 +255,14 @@ static struct sk_buff *brcm_leg_tag_rcv(struct sk_buff *skb, { int len = BRCM_LEG_TAG_LEN; int source_port; + __be16 *proto; u8 *brcm_tag; - if (unlikely(!pskb_may_pull(skb, BRCM_LEG_PORT_ID))) + if (unlikely(!pskb_may_pull(skb, BRCM_LEG_TAG_LEN + VLAN_HLEN))) return NULL; brcm_tag = dsa_etype_header_pos_rx(skb); + proto = (__be16 *)(brcm_tag + BRCM_LEG_TAG_LEN); source_port = brcm_tag[5] & BRCM_LEG_PORT_ID; @@ -268,8 +270,12 @@ static struct sk_buff *brcm_leg_tag_rcv(struct sk_buff *skb, if (!skb->dev) return NULL; - /* VLAN tag is added by BCM63xx internal switch */ - if (netdev_uses_dsa(skb->dev)) + /* The internal switch in BCM63XX SoCs always tags on egress on the CPU + * port. We use VID 0 internally for untagged traffic, so strip the tag + * if the TCI field is all 0, and keep it otherwise to also retain + * e.g. 802.1p tagged packets. + */ + if (proto[0] == htons(ETH_P_8021Q) && proto[1] == 0) len += VLAN_HLEN; /* Remove Broadcom tag and update checksum */ diff --git a/net/dsa/tag_ksz.c b/net/dsa/tag_ksz.c index ea100bd25939b..0a16c04c4bfc4 100644 --- a/net/dsa/tag_ksz.c +++ b/net/dsa/tag_ksz.c @@ -139,7 +139,12 @@ static struct sk_buff *ksz8795_xmit(struct sk_buff *skb, struct net_device *dev) static struct sk_buff *ksz8795_rcv(struct sk_buff *skb, struct net_device *dev) { - u8 *tag = skb_tail_pointer(skb) - KSZ_EGRESS_TAG_LEN; + u8 *tag; + + if (skb_linearize(skb)) + return NULL; + + tag = skb_tail_pointer(skb) - KSZ_EGRESS_TAG_LEN; return ksz_common_rcv(skb, dev, tag[0] & 7, KSZ_EGRESS_TAG_LEN); } @@ -176,8 +181,9 @@ MODULE_ALIAS_DSA_TAG_DRIVER(DSA_TAG_PROTO_KSZ8795, KSZ8795_NAME); #define KSZ9477_INGRESS_TAG_LEN 2 #define KSZ9477_PTP_TAG_LEN 4 -#define KSZ9477_PTP_TAG_INDICATION 0x80 +#define KSZ9477_PTP_TAG_INDICATION BIT(7) +#define KSZ9477_TAIL_TAG_EG_PORT_M GENMASK(2, 0) #define KSZ9477_TAIL_TAG_PRIO GENMASK(8, 7) #define KSZ9477_TAIL_TAG_OVERRIDE BIT(9) #define KSZ9477_TAIL_TAG_LOOKUP BIT(10) @@ -300,10 +306,16 @@ static struct sk_buff *ksz9477_xmit(struct sk_buff *skb, static struct sk_buff *ksz9477_rcv(struct sk_buff *skb, struct net_device *dev) { - /* Tag decoding */ - u8 *tag = skb_tail_pointer(skb) - KSZ_EGRESS_TAG_LEN; - unsigned int port = tag[0] & 7; unsigned int len = KSZ_EGRESS_TAG_LEN; + unsigned int port; + u8 *tag; + + if (skb_linearize(skb)) + return NULL; + + /* Tag decoding */ + tag = skb_tail_pointer(skb) - KSZ_EGRESS_TAG_LEN; + port = tag[0] & KSZ9477_TAIL_TAG_EG_PORT_M; /* Extra 4-bytes PTP timestamp */ if (tag[0] & KSZ9477_PTP_TAG_INDICATION) { diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index 049c3adeb8504..b4a6e26ec2871 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -615,7 +615,10 @@ EXPORT_SYMBOL(fwnode_get_mac_address); */ int device_get_mac_address(struct device *dev, char *addr) { - return fwnode_get_mac_address(dev_fwnode(dev), addr); + if (!fwnode_get_mac_address(dev_fwnode(dev), addr)) + return 0; + + return nvmem_get_mac_address(dev, addr); } EXPORT_SYMBOL(device_get_mac_address); diff --git a/net/handshake/tlshd.c b/net/handshake/tlshd.c index bbfb4095ddd6b..06916a80cc130 100644 --- a/net/handshake/tlshd.c +++ b/net/handshake/tlshd.c @@ -254,6 +254,7 @@ static int tls_handshake_accept(struct handshake_req *req, out_cancel: genlmsg_cancel(msg, hdr); + nlmsg_free(msg); out: return ret; } diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index 906c38b9d66ff..70e958caa956d 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -59,7 +59,7 @@ static bool hsr_check_carrier(struct hsr_port *master) ASSERT_RTNL(); - hsr_for_each_port(master->hsr, port) { + hsr_for_each_port_rtnl(master->hsr, port) { if (port->type != HSR_PT_MASTER && is_slave_up(port->dev)) { netif_carrier_on(master->dev); return true; @@ -109,7 +109,7 @@ int hsr_get_max_mtu(struct hsr_priv *hsr) struct hsr_port *port; mtu_max = ETH_DATA_LEN; - hsr_for_each_port(hsr, port) + hsr_for_each_port_rtnl(hsr, port) if (port->type != HSR_PT_MASTER) mtu_max = min(port->dev->mtu, mtu_max); @@ -144,7 +144,7 @@ static int hsr_dev_open(struct net_device *dev) hsr = netdev_priv(dev); designation = '\0'; - hsr_for_each_port(hsr, port) { + hsr_for_each_port_rtnl(hsr, port) { if (port->type == HSR_PT_MASTER) continue; switch (port->type) { @@ -170,7 +170,24 @@ static int hsr_dev_open(struct net_device *dev) static int hsr_dev_close(struct net_device *dev) { - /* Nothing to do here. */ + struct hsr_port *port; + struct hsr_priv *hsr; + + hsr = netdev_priv(dev); + hsr_for_each_port_rtnl(hsr, port) { + if (port->type == HSR_PT_MASTER) + continue; + switch (port->type) { + case HSR_PT_SLAVE_A: + case HSR_PT_SLAVE_B: + dev_uc_unsync(port->dev, dev); + dev_mc_unsync(port->dev, dev); + break; + default: + break; + } + } + return 0; } @@ -190,7 +207,7 @@ static netdev_features_t hsr_features_recompute(struct hsr_priv *hsr, * may become enabled. */ features &= ~NETIF_F_ONE_FOR_ALL; - hsr_for_each_port(hsr, port) + hsr_for_each_port_rtnl(hsr, port) features = netdev_increment_features(features, port->dev->features, mask); @@ -211,6 +228,7 @@ static netdev_tx_t hsr_dev_xmit(struct sk_buff *skb, struct net_device *dev) struct hsr_priv *hsr = netdev_priv(dev); struct hsr_port *master; + rcu_read_lock(); master = hsr_port_get_hsr(hsr, HSR_PT_MASTER); if (master) { skb->dev = master->dev; @@ -223,6 +241,8 @@ static netdev_tx_t hsr_dev_xmit(struct sk_buff *skb, struct net_device *dev) dev_core_stats_tx_dropped_inc(dev); dev_kfree_skb_any(skb); } + rcu_read_unlock(); + return NETDEV_TX_OK; } @@ -293,6 +313,9 @@ static void send_hsr_supervision_frame(struct hsr_port *master, } hsr_stag = skb_put(skb, sizeof(struct hsr_sup_tag)); + skb_set_network_header(skb, ETH_HLEN + HSR_HLEN); + skb_reset_mac_len(skb); + set_hsr_stag_path(hsr_stag, (hsr->prot_version ? 0x0 : 0xf)); set_hsr_stag_HSR_ver(hsr_stag, hsr->prot_version); @@ -401,12 +424,133 @@ void hsr_del_ports(struct hsr_priv *hsr) hsr_del_port(port); } +static void hsr_set_rx_mode(struct net_device *dev) +{ + struct hsr_port *port; + struct hsr_priv *hsr; + + hsr = netdev_priv(dev); + + hsr_for_each_port_rtnl(hsr, port) { + if (port->type == HSR_PT_MASTER) + continue; + switch (port->type) { + case HSR_PT_SLAVE_A: + case HSR_PT_SLAVE_B: + dev_mc_sync_multiple(port->dev, dev); + dev_uc_sync_multiple(port->dev, dev); + break; + default: + break; + } + } +} + +static void hsr_change_rx_flags(struct net_device *dev, int change) +{ + struct hsr_port *port; + struct hsr_priv *hsr; + + hsr = netdev_priv(dev); + + hsr_for_each_port_rtnl(hsr, port) { + if (port->type == HSR_PT_MASTER) + continue; + switch (port->type) { + case HSR_PT_SLAVE_A: + case HSR_PT_SLAVE_B: + if (change & IFF_ALLMULTI) + dev_set_allmulti(port->dev, + dev->flags & + IFF_ALLMULTI ? 1 : -1); + break; + default: + break; + } + } +} + +static int hsr_ndo_vlan_rx_add_vid(struct net_device *dev, + __be16 proto, u16 vid) +{ + bool is_slave_a_added = false; + bool is_slave_b_added = false; + struct hsr_port *port; + struct hsr_priv *hsr; + int ret = 0; + + hsr = netdev_priv(dev); + + hsr_for_each_port_rtnl(hsr, port) { + if (port->type == HSR_PT_MASTER || + port->type == HSR_PT_INTERLINK) + continue; + + ret = vlan_vid_add(port->dev, proto, vid); + switch (port->type) { + case HSR_PT_SLAVE_A: + if (ret) { + /* clean up Slave-B */ + netdev_err(dev, "add vid failed for Slave-A\n"); + if (is_slave_b_added) + vlan_vid_del(port->dev, proto, vid); + return ret; + } + + is_slave_a_added = true; + break; + + case HSR_PT_SLAVE_B: + if (ret) { + /* clean up Slave-A */ + netdev_err(dev, "add vid failed for Slave-B\n"); + if (is_slave_a_added) + vlan_vid_del(port->dev, proto, vid); + return ret; + } + + is_slave_b_added = true; + break; + default: + break; + } + } + + return 0; +} + +static int hsr_ndo_vlan_rx_kill_vid(struct net_device *dev, + __be16 proto, u16 vid) +{ + struct hsr_port *port; + struct hsr_priv *hsr; + + hsr = netdev_priv(dev); + + hsr_for_each_port_rtnl(hsr, port) { + switch (port->type) { + case HSR_PT_SLAVE_A: + case HSR_PT_SLAVE_B: + vlan_vid_del(port->dev, proto, vid); + break; + default: + break; + } + } + + return 0; +} + static const struct net_device_ops hsr_device_ops = { .ndo_change_mtu = hsr_dev_change_mtu, .ndo_open = hsr_dev_open, .ndo_stop = hsr_dev_close, .ndo_start_xmit = hsr_dev_xmit, + .ndo_change_rx_flags = hsr_change_rx_flags, .ndo_fix_features = hsr_fix_features, + .ndo_set_rx_mode = hsr_set_rx_mode, + .ndo_vlan_rx_add_vid = hsr_ndo_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = hsr_ndo_vlan_rx_kill_vid, }; static struct device_type hsr_type = { @@ -447,7 +591,8 @@ void hsr_dev_setup(struct net_device *dev) dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA | NETIF_F_GSO_MASK | NETIF_F_HW_CSUM | - NETIF_F_HW_VLAN_CTAG_TX; + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_CTAG_FILTER; dev->features = dev->hw_features; @@ -533,6 +678,10 @@ int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2], (slave[1]->features & NETIF_F_HW_HSR_FWD)) hsr->fwd_offloaded = true; + if ((slave[0]->features & NETIF_F_HW_VLAN_CTAG_FILTER) && + (slave[1]->features & NETIF_F_HW_VLAN_CTAG_FILTER)) + hsr_dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; + res = register_netdevice(hsr_dev); if (res) goto err_unregister; diff --git a/net/hsr/hsr_main.c b/net/hsr/hsr_main.c index 257b50124cee5..76a1958609e29 100644 --- a/net/hsr/hsr_main.c +++ b/net/hsr/hsr_main.c @@ -22,7 +22,7 @@ static bool hsr_slave_empty(struct hsr_priv *hsr) { struct hsr_port *port; - hsr_for_each_port(hsr, port) + hsr_for_each_port_rtnl(hsr, port) if (port->type != HSR_PT_MASTER) return false; return true; @@ -125,7 +125,7 @@ struct hsr_port *hsr_port_get_hsr(struct hsr_priv *hsr, enum hsr_port_type pt) { struct hsr_port *port; - hsr_for_each_port(hsr, port) + hsr_for_each_port_rtnl(hsr, port) if (port->type == pt) return port; return NULL; diff --git a/net/hsr/hsr_main.h b/net/hsr/hsr_main.h index 18e01791ad799..2fcabe39e61f4 100644 --- a/net/hsr/hsr_main.h +++ b/net/hsr/hsr_main.h @@ -221,6 +221,9 @@ struct hsr_priv { #define hsr_for_each_port(hsr, port) \ list_for_each_entry_rcu((port), &(hsr)->ports, port_list) +#define hsr_for_each_port_rtnl(hsr, port) \ + list_for_each_entry_rcu((port), &(hsr)->ports, port_list, lockdep_rtnl_is_held()) + struct hsr_port *hsr_port_get_hsr(struct hsr_priv *hsr, enum hsr_port_type pt); /* Caller must ensure skb is a valid HSR frame */ diff --git a/net/hsr/hsr_slave.c b/net/hsr/hsr_slave.c index 1b6457f357bdb..b8230faa567f7 100644 --- a/net/hsr/hsr_slave.c +++ b/net/hsr/hsr_slave.c @@ -62,8 +62,14 @@ static rx_handler_result_t hsr_handle_frame(struct sk_buff **pskb) skb_push(skb, ETH_HLEN); skb_reset_mac_header(skb); if ((!hsr->prot_version && protocol == htons(ETH_P_PRP)) || - protocol == htons(ETH_P_HSR)) + protocol == htons(ETH_P_HSR)) { + if (!pskb_may_pull(skb, ETH_HLEN + HSR_HLEN)) { + kfree_skb(skb); + goto finish_consume; + } + skb_set_network_header(skb, ETH_HLEN + HSR_HLEN); + } skb_reset_mac_len(skb); hsr_forward_skb(skb, port); diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 39dcccf0f174b..ae8b15e6896fd 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -271,6 +271,74 @@ static int bpf_tcp_ca_validate(void *kdata) return tcp_validate_congestion_control(kdata); } +static u32 bpf_tcp_ca_ssthresh(struct sock *sk) +{ + return 0; +} + +static void bpf_tcp_ca_cong_avoid(struct sock *sk, u32 ack, u32 acked) +{ +} + +static void bpf_tcp_ca_set_state(struct sock *sk, u8 new_state) +{ +} + +static void bpf_tcp_ca_cwnd_event(struct sock *sk, enum tcp_ca_event ev) +{ +} + +static void bpf_tcp_ca_in_ack_event(struct sock *sk, u32 flags) +{ +} + +static void bpf_tcp_ca_pkts_acked(struct sock *sk, const struct ack_sample *sample) +{ +} + +static u32 bpf_tcp_ca_min_tso_segs(struct sock *sk) +{ + return 0; +} + +static void bpf_tcp_ca_cong_control(struct sock *sk, const struct rate_sample *rs) +{ +} + +static u32 bpf_tcp_ca_undo_cwnd(struct sock *sk) +{ + return 0; +} + +static u32 bpf_tcp_ca_sndbuf_expand(struct sock *sk) +{ + return 0; +} + +static void __bpf_tcp_ca_init(struct sock *sk) +{ +} + +static void __bpf_tcp_ca_release(struct sock *sk) +{ +} + +static struct tcp_congestion_ops __bpf_ops_tcp_congestion_ops = { + .ssthresh = bpf_tcp_ca_ssthresh, + .cong_avoid = bpf_tcp_ca_cong_avoid, + .set_state = bpf_tcp_ca_set_state, + .cwnd_event = bpf_tcp_ca_cwnd_event, + .in_ack_event = bpf_tcp_ca_in_ack_event, + .pkts_acked = bpf_tcp_ca_pkts_acked, + .min_tso_segs = bpf_tcp_ca_min_tso_segs, + .cong_control = bpf_tcp_ca_cong_control, + .undo_cwnd = bpf_tcp_ca_undo_cwnd, + .sndbuf_expand = bpf_tcp_ca_sndbuf_expand, + + .init = __bpf_tcp_ca_init, + .release = __bpf_tcp_ca_release, +}; + struct bpf_struct_ops bpf_tcp_congestion_ops = { .verifier_ops = &bpf_tcp_ca_verifier_ops, .reg = bpf_tcp_ca_reg, @@ -281,6 +349,7 @@ struct bpf_struct_ops bpf_tcp_congestion_ops = { .init = bpf_tcp_ca_init, .validate = bpf_tcp_ca_validate, .name = "tcp_congestion_ops", + .cfi_stubs = &__bpf_ops_tcp_congestion_ops, }; static int __init bpf_tcp_ca_kfunc_init(void) diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index c33b1ecc591e4..798497c8b1923 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -336,14 +336,13 @@ static void inetdev_destroy(struct in_device *in_dev) static int __init inet_blackhole_dev_init(void) { - int err = 0; + struct in_device *in_dev; rtnl_lock(); - if (!inetdev_init(blackhole_netdev)) - err = -ENOMEM; + in_dev = inetdev_init(blackhole_netdev); rtnl_unlock(); - return err; + return PTR_ERR_OR_ZERO(in_dev); } late_initcall(inet_blackhole_dev_init); diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index eeace9b509cec..ed5f37c6d9ba2 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -197,8 +197,10 @@ static int esp_output_tcp_finish(struct xfrm_state *x, struct sk_buff *skb) sk = esp_find_tcp_sk(x); err = PTR_ERR_OR_ZERO(sk); - if (err) + if (err) { + kfree_skb(skb); goto out; + } bh_lock_sock(sk); if (sock_owned_by_user(sk)) diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c index 10e96ed6c9e39..11e33a4312674 100644 --- a/net/ipv4/esp4_offload.c +++ b/net/ipv4/esp4_offload.c @@ -111,8 +111,10 @@ static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - __be16 type = x->inner_mode.family == AF_INET6 ? htons(ETH_P_IPV6) - : htons(ETH_P_IP); + const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, + XFRM_MODE_SKB_CB(skb)->protocol); + __be16 type = inner_mode->family == AF_INET6 ? htons(ETH_P_IPV6) + : htons(ETH_P_IP); return skb_eth_gso_segment(skb, features, type); } diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 90ce87ffed461..7993ff46de23c 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -829,19 +829,33 @@ static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, } } + if (cfg->fc_dst_len > 32) { + NL_SET_ERR_MSG(extack, "Invalid prefix length"); + err = -EINVAL; + goto errout; + } + + if (cfg->fc_dst_len < 32 && (ntohl(cfg->fc_dst) << cfg->fc_dst_len)) { + NL_SET_ERR_MSG(extack, "Invalid prefix for given prefix length"); + err = -EINVAL; + goto errout; + } + if (cfg->fc_nh_id) { if (cfg->fc_oif || cfg->fc_gw_family || cfg->fc_encap || cfg->fc_mp) { NL_SET_ERR_MSG(extack, "Nexthop specification and nexthop id are mutually exclusive"); - return -EINVAL; + err = -EINVAL; + goto errout; } } if (has_gw && has_via) { NL_SET_ERR_MSG(extack, "Nexthop configuration can not contain both GATEWAY and VIA"); - return -EINVAL; + err = -EINVAL; + goto errout; } if (!cfg->fc_table) diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c index 513f475c6a534..298a9944a3d1e 100644 --- a/net/ipv4/fib_rules.c +++ b/net/ipv4/fib_rules.c @@ -222,9 +222,9 @@ static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb, struct nlattr **tb, struct netlink_ext_ack *extack) { - struct net *net = sock_net(skb->sk); + struct fib4_rule *rule4 = (struct fib4_rule *)rule; + struct net *net = rule->fr_net; int err = -EINVAL; - struct fib4_rule *rule4 = (struct fib4_rule *) rule; if (!inet_validate_dscp(frh->tos)) { NL_SET_ERR_MSG(extack, diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 77b97c48da5ea..fa54b36b241ac 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -1192,22 +1192,6 @@ static int fib_insert_alias(struct trie *t, struct key_vector *tp, return 0; } -static bool fib_valid_key_len(u32 key, u8 plen, struct netlink_ext_ack *extack) -{ - if (plen > KEYLENGTH) { - NL_SET_ERR_MSG(extack, "Invalid prefix length"); - return false; - } - - if ((plen < KEYLENGTH) && (key << plen)) { - NL_SET_ERR_MSG(extack, - "Invalid prefix for given prefix length"); - return false; - } - - return true; -} - static void fib_remove_alias(struct trie *t, struct key_vector *tp, struct key_vector *l, struct fib_alias *old); @@ -1228,9 +1212,6 @@ int fib_table_insert(struct net *net, struct fib_table *tb, key = ntohl(cfg->fc_dst); - if (!fib_valid_key_len(key, plen, extack)) - return -EINVAL; - pr_debug("Insert table=%u %08x/%d\n", tb->tb_id, key, plen); fi = fib_create_info(cfg, extack); @@ -1723,9 +1704,6 @@ int fib_table_delete(struct net *net, struct fib_table *tb, key = ntohl(cfg->fc_dst); - if (!fib_valid_key_len(key, plen, extack)) - return -EINVAL; - l = fib_find_node(t, &tp, key); if (!l) return -ESRCH; diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index e710a05f5e734..156b9c18d3666 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -803,11 +803,12 @@ void icmp_ndo_send(struct sk_buff *skb_in, int type, int code, __be32 info) struct sk_buff *cloned_skb = NULL; struct ip_options opts = { 0 }; enum ip_conntrack_info ctinfo; + enum ip_conntrack_dir dir; struct nf_conn *ct; __be32 orig_ip; ct = nf_ct_get(skb_in, &ctinfo); - if (!ct || !(ct->status & IPS_SRC_NAT)) { + if (!ct || !(READ_ONCE(ct->status) & IPS_NAT_MASK)) { __icmp_send(skb_in, type, code, info, &opts); return; } @@ -822,7 +823,8 @@ void icmp_ndo_send(struct sk_buff *skb_in, int type, int code, __be32 info) goto out; orig_ip = ip_hdr(skb_in)->saddr; - ip_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.ip; + dir = CTINFO2DIR(ctinfo); + ip_hdr(skb_in)->saddr = ct->tuplehash[dir].tuple.src.u3.ip; __icmp_send(skb_in, type, code, info, &opts); ip_hdr(skb_in)->saddr = orig_ip; out: diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index 7967ff7e02f79..60e81f6b1c6d4 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -1231,22 +1231,37 @@ int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo) { unsigned int locksz = sizeof(spinlock_t); unsigned int i, nblocks = 1; + spinlock_t *ptr = NULL; - if (locksz != 0) { - /* allocate 2 cache lines or at least one spinlock per cpu */ - nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U); - nblocks = roundup_pow_of_two(nblocks * num_possible_cpus()); + if (locksz == 0) + goto set_mask; - /* no more locks than number of hash buckets */ - nblocks = min(nblocks, hashinfo->ehash_mask + 1); + /* Allocate 2 cache lines or at least one spinlock per cpu. */ + nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U) * num_possible_cpus(); - hashinfo->ehash_locks = kvmalloc_array(nblocks, locksz, GFP_KERNEL); - if (!hashinfo->ehash_locks) - return -ENOMEM; + /* At least one page per NUMA node. */ + nblocks = max(nblocks, num_online_nodes() * PAGE_SIZE / locksz); + + nblocks = roundup_pow_of_two(nblocks); + + /* No more locks than number of hash buckets. */ + nblocks = min(nblocks, hashinfo->ehash_mask + 1); - for (i = 0; i < nblocks; i++) - spin_lock_init(&hashinfo->ehash_locks[i]); + if (num_online_nodes() > 1) { + /* Use vmalloc() to allow NUMA policy to spread pages + * on all available nodes if desired. + */ + ptr = vmalloc_array(nblocks, locksz); + } + if (!ptr) { + ptr = kvmalloc_array(nblocks, locksz, GFP_KERNEL); + if (!ptr) + return -ENOMEM; } + for (i = 0; i < nblocks; i++) + spin_lock_init(&ptr[i]); + hashinfo->ehash_locks = ptr; +set_mask: hashinfo->ehash_locks_mask = nblocks - 1; return 0; } diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 890c15510b421..f261e29adc7c2 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -140,7 +140,6 @@ static int ipgre_err(struct sk_buff *skb, u32 info, const struct iphdr *iph; const int type = icmp_hdr(skb)->type; const int code = icmp_hdr(skb)->code; - unsigned int data_len = 0; struct ip_tunnel *t; if (tpi->proto == htons(ETH_P_TEB)) @@ -181,7 +180,6 @@ static int ipgre_err(struct sk_buff *skb, u32 info, case ICMP_TIME_EXCEEDED: if (code != ICMP_EXC_TTL) return 0; - data_len = icmp_hdr(skb)->un.reserved[1] * 4; /* RFC 4884 4.1 */ break; case ICMP_REDIRECT: @@ -189,10 +187,16 @@ static int ipgre_err(struct sk_buff *skb, u32 info, } #if IS_ENABLED(CONFIG_IPV6) - if (tpi->proto == htons(ETH_P_IPV6) && - !ip6_err_gen_icmpv6_unreach(skb, iph->ihl * 4 + tpi->hdr_len, - type, data_len)) - return 0; + if (tpi->proto == htons(ETH_P_IPV6)) { + unsigned int data_len = 0; + + if (type == ICMP_TIME_EXCEEDED) + data_len = icmp_hdr(skb)->un.reserved[1] * 4; /* RFC 4884 4.1 */ + + if (!ip6_err_gen_icmpv6_unreach(skb, iph->ihl * 4 + tpi->hdr_len, + type, data_len)) + return 0; + } #endif if (t->parms.iph.daddr == 0 || diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 765bd3f2a8408..cd0c8f6a79233 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -425,15 +425,20 @@ int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb) int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb) { - struct net_device *dev = skb_dst(skb)->dev, *indev = skb->dev; + struct net_device *dev, *indev = skb->dev; + int ret_val; + rcu_read_lock(); + dev = skb_dst_dev_rcu(skb); skb->dev = dev; skb->protocol = htons(ETH_P_IP); - return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, - net, sk, skb, indev, dev, - ip_finish_output, - !(IPCB(skb)->flags & IPSKB_REROUTED)); + ret_val = NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, + net, sk, skb, indev, dev, + ip_finish_output, + !(IPCB(skb)->flags & IPSKB_REROUTED)); + rcu_read_unlock(); + return ret_val; } EXPORT_SYMBOL(ip_output); diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index b5d64cd3ab0a2..090403c8cc6c3 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -567,20 +567,6 @@ static int tnl_update_pmtu(struct net_device *dev, struct sk_buff *skb, return 0; } -static void ip_tunnel_adj_headroom(struct net_device *dev, unsigned int headroom) -{ - /* we must cap headroom to some upperlimit, else pskb_expand_head - * will overflow header offsets in skb_headers_offset_update(). - */ - static const unsigned int max_allowed = 512; - - if (headroom > max_allowed) - headroom = max_allowed; - - if (headroom > READ_ONCE(dev->needed_headroom)) - WRITE_ONCE(dev->needed_headroom, headroom); -} - void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, u8 proto, int tunnel_hlen) { diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index deb08cab44640..75e3d7501752d 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -203,6 +203,9 @@ static int iptunnel_pmtud_build_icmp(struct sk_buff *skb, int mtu) if (!pskb_may_pull(skb, ETH_HLEN + sizeof(struct iphdr))) return -EINVAL; + if (skb_is_gso(skb)) + skb_gso_reset(skb); + skb_copy_bits(skb, skb_mac_offset(skb), &eh, ETH_HLEN); pskb_pull(skb, ETH_HLEN); skb_reset_network_header(skb); @@ -297,6 +300,9 @@ static int iptunnel_pmtud_build_icmpv6(struct sk_buff *skb, int mtu) if (!pskb_may_pull(skb, ETH_HLEN + sizeof(struct ipv6hdr))) return -EINVAL; + if (skb_is_gso(skb)) + skb_gso_reset(skb); + skb_copy_bits(skb, skb_mac_offset(skb), &eh, ETH_HLEN); pskb_pull(skb, ETH_HLEN); skb_reset_network_header(skb); diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c index 14365b20f1c5c..48459b89460d7 100644 --- a/net/ipv4/netfilter/arp_tables.c +++ b/net/ipv4/netfilter/arp_tables.c @@ -1586,18 +1586,29 @@ void arpt_unregister_table(struct net *net, const char *name) __arpt_unregister_table(net, table); } +#define DEFINE_ARPT_STANDARD_TARGET(compat) \ + { \ + .name = XT_STANDARD_TARGET, \ + .targetsize = sizeof(int), \ + .family = NFPROTO_ARP, \ + .has_compat_metadata = compat, \ + } + /* The built-in targets: standard (NULL) and error. */ -static struct xt_target arpt_builtin_tg[] __read_mostly = { - { - .name = XT_STANDARD_TARGET, - .targetsize = sizeof(int), - .family = NFPROTO_ARP, #ifdef CONFIG_NETFILTER_XTABLES_COMPAT - .compatsize = sizeof(compat_int_t), - .compat_from_user = compat_standard_from_user, - .compat_to_user = compat_standard_to_user, +static struct compat_xt_target_ext arpt_std_tg_ext __read_mostly = { + .compatsize = sizeof(compat_int_t), + .compat_from_user = compat_standard_from_user, + .compat_to_user = compat_standard_to_user, + .target = DEFINE_ARPT_STANDARD_TARGET(true), +}; + +#define arpt_std_tg (arpt_std_tg_ext.target) +#else +static struct xt_target arpt_std_tg __read_mostly = DEFINE_ARPT_STANDARD_TARGET(false); #endif - }, + +static struct xt_target arpt_builtin_tg[] __read_mostly = { { .name = XT_ERROR_TARGET, .target = arpt_error, @@ -1641,19 +1652,25 @@ static int __init arp_tables_init(void) goto err1; /* No one else will be downing sem now, so we won't sleep */ - ret = xt_register_targets(arpt_builtin_tg, ARRAY_SIZE(arpt_builtin_tg)); + ret = xt_register_target(&arpt_std_tg); if (ret < 0) goto err2; + ret = xt_register_targets(arpt_builtin_tg, ARRAY_SIZE(arpt_builtin_tg)); + if (ret < 0) + goto err4; + /* Register setsockopt */ ret = nf_register_sockopt(&arpt_sockopts); if (ret < 0) - goto err4; + goto err8; return 0; -err4: +err8: xt_unregister_targets(arpt_builtin_tg, ARRAY_SIZE(arpt_builtin_tg)); +err4: + xt_unregister_target(&arpt_std_tg); err2: unregister_pernet_subsys(&arp_tables_net_ops); err1: @@ -1664,6 +1681,7 @@ static void __exit arp_tables_fini(void) { nf_unregister_sockopt(&arpt_sockopts); xt_unregister_targets(arpt_builtin_tg, ARRAY_SIZE(arpt_builtin_tg)); + xt_unregister_target(&arpt_std_tg); unregister_pernet_subsys(&arp_tables_net_ops); } diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c index fe89a056eb06c..f48d63eb62e6a 100644 --- a/net/ipv4/netfilter/ip_tables.c +++ b/net/ipv4/netfilter/ip_tables.c @@ -1805,17 +1805,28 @@ void ipt_unregister_table_exit(struct net *net, const char *name) __ipt_unregister_table(net, table); } -static struct xt_target ipt_builtin_tg[] __read_mostly = { - { - .name = XT_STANDARD_TARGET, - .targetsize = sizeof(int), - .family = NFPROTO_IPV4, +#define DEFINE_IPT_STANDARD_TARGET(compat) \ + { \ + .name = XT_STANDARD_TARGET, \ + .targetsize = sizeof(int), \ + .family = NFPROTO_IPV4, \ + .has_compat_metadata = compat \ + } + #ifdef CONFIG_NETFILTER_XTABLES_COMPAT - .compatsize = sizeof(compat_int_t), - .compat_from_user = compat_standard_from_user, - .compat_to_user = compat_standard_to_user, +static struct compat_xt_target_ext ipt_std_tg_ext __read_mostly = { + .compatsize = sizeof(compat_int_t), + .compat_from_user = compat_standard_from_user, + .compat_to_user = compat_standard_to_user, + .target = DEFINE_IPT_STANDARD_TARGET(true), +}; + +#define ipt_std_tg (ipt_std_tg_ext.target) +#else +static struct xt_target ipt_std_tg __read_mostly = DEFINE_IPT_STANDARD_TARGET(false); #endif - }, + +static struct xt_target ipt_builtin_tg[] __read_mostly = { { .name = XT_ERROR_TARGET, .target = ipt_error, @@ -1859,19 +1870,25 @@ static int __init ip_tables_init(void) goto err1; /* No one else will be downing sem now, so we won't sleep */ - ret = xt_register_targets(ipt_builtin_tg, ARRAY_SIZE(ipt_builtin_tg)); + ret = xt_register_target(&ipt_std_tg); if (ret < 0) goto err2; + ret = xt_register_targets(ipt_builtin_tg, ARRAY_SIZE(ipt_builtin_tg)); + if (ret < 0) + goto err4; + /* Register setsockopt */ ret = nf_register_sockopt(&ipt_sockopts); if (ret < 0) - goto err4; + goto err8; return 0; -err4: +err8: xt_unregister_targets(ipt_builtin_tg, ARRAY_SIZE(ipt_builtin_tg)); +err4: + xt_unregister_target(&ipt_std_tg); err2: unregister_pernet_subsys(&ip_tables_net_ops); err1: @@ -1883,6 +1900,7 @@ static void __exit ip_tables_fini(void) nf_unregister_sockopt(&ipt_sockopts); xt_unregister_targets(ipt_builtin_tg, ARRAY_SIZE(ipt_builtin_tg)); + xt_unregister_target(&ipt_std_tg); unregister_pernet_subsys(&ip_tables_net_ops); } diff --git a/net/ipv4/netfilter/nf_reject_ipv4.c b/net/ipv4/netfilter/nf_reject_ipv4.c index 675b5bbed638e..2064b40130412 100644 --- a/net/ipv4/netfilter/nf_reject_ipv4.c +++ b/net/ipv4/netfilter/nf_reject_ipv4.c @@ -71,6 +71,27 @@ struct sk_buff *nf_reject_skb_v4_tcp_reset(struct net *net, } EXPORT_SYMBOL_GPL(nf_reject_skb_v4_tcp_reset); +static bool nf_skb_is_icmp_unreach(const struct sk_buff *skb) +{ + const struct iphdr *iph = ip_hdr(skb); + u8 *tp, _type; + int thoff; + + if (iph->protocol != IPPROTO_ICMP) + return false; + + thoff = skb_network_offset(skb) + sizeof(*iph); + + tp = skb_header_pointer(skb, + thoff + offsetof(struct icmphdr, type), + sizeof(_type), &_type); + + if (!tp) + return false; + + return *tp == ICMP_DEST_UNREACH; +} + struct sk_buff *nf_reject_skb_v4_unreach(struct net *net, struct sk_buff *oldskb, const struct net_device *dev, @@ -91,6 +112,10 @@ struct sk_buff *nf_reject_skb_v4_unreach(struct net *net, if (ip_hdr(oldskb)->frag_off & htons(IP_OFFSET)) return NULL; + /* don't reply to ICMP_DEST_UNREACH with ICMP_DEST_UNREACH. */ + if (nf_skb_is_icmp_unreach(oldskb)) + return NULL; + /* RFC says return as much as we can without exceeding 576 bytes. */ len = min_t(unsigned int, 536, oldskb->len); @@ -247,8 +272,7 @@ void nf_send_reset(struct net *net, struct sock *sk, struct sk_buff *oldskb, if (!oth) return; - if ((hook == NF_INET_PRE_ROUTING || hook == NF_INET_INGRESS) && - nf_reject_fill_skb_dst(oldskb) < 0) + if (!skb_dst(oldskb) && nf_reject_fill_skb_dst(oldskb) < 0) return; if (skb_rtable(oldskb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) @@ -321,8 +345,7 @@ void nf_send_unreach(struct sk_buff *skb_in, int code, int hook) if (iph->frag_off & htons(IP_OFFSET)) return; - if ((hook == NF_INET_PRE_ROUTING || hook == NF_INET_INGRESS) && - nf_reject_fill_skb_dst(skb_in) < 0) + if (!skb_dst(skb_in) && nf_reject_fill_skb_dst(skb_in) < 0) return; if (skb_csum_unnecessary(skb_in) || diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 8d41b03942197..99385fe34a1e5 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -1835,6 +1835,12 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, { struct nh_grp_entry *nhge, *tmp; + /* If there is nothing to do, let's avoid the costly call to + * synchronize_net() + */ + if (list_empty(&nh->grp_list)) + return; + list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list) remove_nh_grp_entry(net, nhge, nlinfo); @@ -2136,6 +2142,13 @@ static int replace_nexthop_single(struct net *net, struct nexthop *old, return -EINVAL; } + if (!list_empty(&old->grp_list) && + rtnl_dereference(new->nh_info)->fdb_nh != + rtnl_dereference(old->nh_info)->fdb_nh) { + NL_SET_ERR_MSG(extack, "Cannot change nexthop FDB status while in a group"); + return -EINVAL; + } + err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack); if (err) return err; diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 4cb0c896caf97..5c848136bc266 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -77,6 +77,7 @@ static inline struct hlist_head *ping_hashslot(struct ping_table *table, int ping_get_port(struct sock *sk, unsigned short ident) { + struct net *net = sock_net(sk); struct inet_sock *isk, *isk2; struct hlist_head *hlist; struct sock *sk2 = NULL; @@ -90,9 +91,10 @@ int ping_get_port(struct sock *sk, unsigned short ident) for (i = 0; i < (1L << 16); i++, result++) { if (!result) result++; /* avoid zero */ - hlist = ping_hashslot(&ping_table, sock_net(sk), - result); + hlist = ping_hashslot(&ping_table, net, result); sk_for_each(sk2, hlist) { + if (!net_eq(sock_net(sk2), net)) + continue; isk2 = inet_sk(sk2); if (isk2->inet_num == result) @@ -108,8 +110,10 @@ int ping_get_port(struct sock *sk, unsigned short ident) if (i >= (1L << 16)) goto fail; } else { - hlist = ping_hashslot(&ping_table, sock_net(sk), ident); + hlist = ping_hashslot(&ping_table, net, ident); sk_for_each(sk2, hlist) { + if (!net_eq(sock_net(sk2), net)) + continue; isk2 = inet_sk(sk2); /* BUG? Why is this reuse and not reuseaddr? ping.c @@ -129,7 +133,7 @@ int ping_get_port(struct sock *sk, unsigned short ident) pr_debug("was not hashed\n"); sk_add_node_rcu(sk, hlist); sock_set_flag(sk, SOCK_RCU_FREE); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); + sock_prot_inuse_add(net, sk->sk_prot, 1); } spin_unlock(&ping_table.lock); return 0; @@ -188,6 +192,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) } sk_for_each_rcu(sk, hslot) { + if (!net_eq(sock_net(sk), net)) + continue; isk = inet_sk(sk); pr_debug("iterate\n"); diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c index a85b0aba36462..b444767b15a5d 100644 --- a/net/ipv4/proc.c +++ b/net/ipv4/proc.c @@ -43,7 +43,7 @@ #include #include -#define TCPUDP_MIB_MAX max_t(u32, UDP_MIB_MAX, TCP_MIB_MAX) +#define TCPUDP_MIB_MAX MAX_T(u32, UDP_MIB_MAX, TCP_MIB_MAX) /* * Report socket allocation statistics [mea@utu.fi] diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 2e2d45e29530b..da837699b37bf 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -192,7 +192,11 @@ const __u8 ip_tos2prio[16] = { EXPORT_SYMBOL(ip_tos2prio); static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); +#ifndef CONFIG_PREEMPT_RT #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field) +#else +#define RT_CACHE_STAT_INC(field) this_cpu_inc(rt_cache_stat.field) +#endif #ifdef CONFIG_PROC_FS static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos) @@ -613,6 +617,11 @@ static void fnhe_remove_oldest(struct fnhe_hash_bucket *hash) oldest_p = fnhe_p; } } + + /* Clear oldest->fnhe_daddr to prevent this fnhe from being + * rebound with new dsts in rt_bind_exception(). + */ + oldest->fnhe_daddr = 0; fnhe_flush_routes(oldest); *oldest_p = oldest->fnhe_next; kfree_rcu(oldest, rcu); @@ -1695,8 +1704,8 @@ struct rtable *rt_dst_clone(struct net_device *dev, struct rtable *rt) else if (rt->rt_gw_family == AF_INET6) new_rt->rt_gw6 = rt->rt_gw6; - new_rt->dst.input = rt->dst.input; - new_rt->dst.output = rt->dst.output; + new_rt->dst.input = READ_ONCE(rt->dst.input); + new_rt->dst.output = READ_ONCE(rt->dst.output); new_rt->dst.error = rt->dst.error; new_rt->dst.lastuse = jiffies; new_rt->dst.lwtstate = lwtstate_get(rt->dst.lwtstate); @@ -2543,12 +2552,16 @@ static struct rtable *__mkroute_output(const struct fib_result *res, !netif_is_l3_master(dev_out)) return ERR_PTR(-EINVAL); - if (ipv4_is_lbcast(fl4->daddr)) + if (ipv4_is_lbcast(fl4->daddr)) { type = RTN_BROADCAST; - else if (ipv4_is_multicast(fl4->daddr)) + + /* reset fi to prevent gateway resolution */ + fi = NULL; + } else if (ipv4_is_multicast(fl4->daddr)) { type = RTN_MULTICAST; - else if (ipv4_is_zeronet(fl4->daddr)) + } else if (ipv4_is_zeronet(fl4->daddr)) { return ERR_PTR(-EINVAL); + } if (dev_out->flags & IFF_LOOPBACK) flags |= RTCF_LOCAL; @@ -2556,7 +2569,6 @@ static struct rtable *__mkroute_output(const struct fib_result *res, do_cache = true; if (type == RTN_BROADCAST) { flags |= RTCF_BROADCAST | RTCF_LOCAL; - fi = NULL; } else if (type == RTN_MULTICAST) { flags |= RTCF_MULTICAST | RTCF_LOCAL; if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr, diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 4a5d514fe2913..fb6eae6f34cd4 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1135,7 +1135,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) goto do_error; while (msg_data_left(msg)) { - ssize_t copy = 0; + int copy = 0; skb = tcp_write_queue_tail(sk); if (skb) @@ -1728,6 +1728,7 @@ EXPORT_SYMBOL(tcp_peek_len); /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ int tcp_set_rcvlowat(struct sock *sk, int val) { + struct tcp_sock *tp = tcp_sk(sk); int space, cap; if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) @@ -1746,7 +1747,9 @@ int tcp_set_rcvlowat(struct sock *sk, int val) space = tcp_space_from_win(sk, val); if (space > sk->sk_rcvbuf) { WRITE_ONCE(sk->sk_rcvbuf, space); - WRITE_ONCE(tcp_sk(sk)->window_clamp, val); + + if (tp->window_clamp && tp->window_clamp < val) + WRITE_ONCE(tp->window_clamp, val); } return 0; } @@ -2795,8 +2798,8 @@ bool tcp_check_oom(struct sock *sk, int shift) void __tcp_close(struct sock *sk, long timeout) { + bool data_was_unread = false; struct sk_buff *skb; - int data_was_unread = 0; int state; WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); @@ -2815,11 +2818,12 @@ void __tcp_close(struct sock *sk, long timeout) * reader process may not have drained the data yet! */ while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { - u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq; + u32 end_seq = TCP_SKB_CB(skb)->end_seq; if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) - len--; - data_was_unread += len; + end_seq--; + if (after(end_seq, tcp_sk(sk)->copied_seq)) + data_was_unread = true; __kfree_skb(skb); } @@ -3020,6 +3024,7 @@ int tcp_disconnect(struct sock *sk, int flags) struct inet_connection_sock *icsk = inet_csk(sk); struct tcp_sock *tp = tcp_sk(sk); int old_state = sk->sk_state; + struct request_sock *req; u32 seq; if (old_state != TCP_CLOSE) @@ -3130,6 +3135,10 @@ int tcp_disconnect(struct sock *sk, int flags) /* Clean up fastopen related fields */ + req = rcu_dereference_protected(tp->fastopen_rsk, + lockdep_sock_is_held(sk)); + if (req) + reqsk_fastopen_remove(sk, req, false); tcp_free_fastopen_req(tp); inet_clear_bit(DEFER_CONNECT, sk); tp->fastopen_client_fail = 0; diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 5312237e80409..7518d2af63088 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -408,8 +408,11 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, if (!psock->cork) { psock->cork = kzalloc(sizeof(*psock->cork), GFP_ATOMIC | __GFP_NOWARN); - if (!psock->cork) + if (!psock->cork) { + sk_msg_free(sk, msg); + *copied = 0; return -ENOMEM; + } } memcpy(psock->cork, msg, sizeof(*msg)); return 0; diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 5bf9acbe9c2fd..0cb0945eba192 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -3,6 +3,7 @@ #include #include #include +#include void tcp_fastopen_init_key_once(struct net *net) { @@ -278,6 +279,8 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, refcount_set(&req->rsk_refcnt, 2); + sk_mark_napi_id_set(child, skb); + /* Now finish processing the fastopen child socket. */ tcp_init_transfer(child, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB, skb); diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index d24d7ba958d1a..3839011c93f90 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -426,6 +426,20 @@ static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr return false; } +static void tcp_count_delivered_ce(struct tcp_sock *tp, u32 ecn_count) +{ + tp->delivered_ce += ecn_count; +} + +/* Updates the delivered and delivered_ce counts */ +static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered, + bool ece_ack) +{ + tp->delivered += delivered; + if (ece_ack) + tcp_count_delivered_ce(tp, delivered); +} + /* Buffer size and advertised window tuning. * * 1. Tuning sk->sk_sndbuf, when connection enters established state. @@ -658,10 +672,12 @@ EXPORT_SYMBOL(tcp_initialize_rcv_mss); */ static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep) { - u32 new_sample = tp->rcv_rtt_est.rtt_us; - long m = sample; + u32 new_sample, old_sample = tp->rcv_rtt_est.rtt_us; + long m = sample << 3; - if (new_sample != 0) { + if (old_sample == 0 || m < old_sample) { + new_sample = m; + } else { /* If we sample in larger samples in the non-timestamp * case, we could grossly overestimate the RTT especially * with chatty applications or bulk transfer apps which @@ -672,17 +688,9 @@ static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep) * else with timestamps disabled convergence takes too * long. */ - if (!win_dep) { - m -= (new_sample >> 3); - new_sample += m; - } else { - m <<= 3; - if (m < new_sample) - new_sample = m; - } - } else { - /* No previous measure. */ - new_sample = m << 3; + if (win_dep) + return; + new_sample = old_sample - (old_sample >> 3) + sample; } tp->rcv_rtt_est.rtt_us = new_sample; @@ -1140,15 +1148,6 @@ void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) } } -/* Updates the delivered and delivered_ce counts */ -static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered, - bool ece_ack) -{ - tp->delivered += delivered; - if (ece_ack) - tp->delivered_ce += delivered; -} - /* This procedure tags the retransmission queue when SACKs arrive. * * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L). @@ -2463,20 +2462,33 @@ static inline bool tcp_packet_delayed(const struct tcp_sock *tp) { const struct sock *sk = (const struct sock *)tp; - if (tp->retrans_stamp && - tcp_tsopt_ecr_before(tp, tp->retrans_stamp)) - return true; /* got echoed TS before first retransmission */ + /* Received an echoed timestamp before the first retransmission? */ + if (tp->retrans_stamp) + return tcp_tsopt_ecr_before(tp, tp->retrans_stamp); + + /* We set tp->retrans_stamp upon the first retransmission of a loss + * recovery episode, so normally if tp->retrans_stamp is 0 then no + * retransmission has happened yet (likely due to TSQ, which can cause + * fast retransmits to be delayed). So if snd_una advanced while + * (tp->retrans_stamp is 0 then apparently a packet was merely delayed, + * not lost. But there are exceptions where we retransmit but then + * clear tp->retrans_stamp, so we check for those exceptions. + */ + + /* (1) For non-SACK connections, tcp_is_non_sack_preventing_reopen() + * clears tp->retrans_stamp when snd_una == high_seq. + */ + if (!tcp_is_sack(tp) && !before(tp->snd_una, tp->high_seq)) + return false; - /* Check if nothing was retransmitted (retrans_stamp==0), which may - * happen in fast recovery due to TSQ. But we ignore zero retrans_stamp - * in TCP_SYN_SENT, since when we set FLAG_SYN_ACKED we also clear - * retrans_stamp even if we had retransmitted the SYN. + /* (2) In TCP_SYN_SENT tcp_clean_rtx_queue() clears tp->retrans_stamp + * when setting FLAG_SYN_ACKED is set, even if the SYN was + * retransmitted. */ - if (!tp->retrans_stamp && /* no record of a retransmit/SYN? */ - sk->sk_state != TCP_SYN_SENT) /* not the FLAG_SYN_ACKED case? */ - return true; /* nothing was retransmitted */ + if (sk->sk_state == TCP_SYN_SENT) + return false; - return false; + return true; /* tp->retrans_stamp is zero; no retransmit yet */ } /* Undo procedures. */ @@ -3808,12 +3820,23 @@ static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) } } -static inline void tcp_in_ack_event(struct sock *sk, u32 flags) +static void tcp_in_ack_event(struct sock *sk, int flag) { const struct inet_connection_sock *icsk = inet_csk(sk); - if (icsk->icsk_ca_ops->in_ack_event) - icsk->icsk_ca_ops->in_ack_event(sk, flags); + if (icsk->icsk_ca_ops->in_ack_event) { + u32 ack_ev_flags = 0; + + if (flag & FLAG_WIN_UPDATE) + ack_ev_flags |= CA_ACK_WIN_UPDATE; + if (flag & FLAG_SLOWPATH) { + ack_ev_flags |= CA_ACK_SLOWPATH; + if (flag & FLAG_ECE) + ack_ev_flags |= CA_ACK_ECE; + } + + icsk->icsk_ca_ops->in_ack_event(sk, ack_ev_flags); + } } /* Congestion control has updated the cwnd already. So if we're in @@ -3930,12 +3953,8 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) tcp_snd_una_update(tp, ack); flag |= FLAG_WIN_UPDATE; - tcp_in_ack_event(sk, CA_ACK_WIN_UPDATE); - NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPACKS); } else { - u32 ack_ev_flags = CA_ACK_SLOWPATH; - if (ack_seq != TCP_SKB_CB(skb)->end_seq) flag |= FLAG_DATA; else @@ -3947,19 +3966,12 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una, &sack_state); - if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb))) { + if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb))) flag |= FLAG_ECE; - ack_ev_flags |= CA_ACK_ECE; - } if (sack_state.sack_delivered) tcp_count_delivered(tp, sack_state.sack_delivered, flag & FLAG_ECE); - - if (flag & FLAG_WIN_UPDATE) - ack_ev_flags |= CA_ACK_WIN_UPDATE; - - tcp_in_ack_event(sk, ack_ev_flags); } /* This is a deviation from RFC3168 since it states that: @@ -3986,6 +3998,8 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) tcp_rack_update_reo_wnd(sk, &rs); + tcp_in_ack_event(sk, flag); + if (tp->tlp_high_seq) tcp_process_tlp_ack(sk, ack, flag); @@ -4017,6 +4031,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) return 1; no_queue: + tcp_in_ack_event(sk, flag); /* If data was DSACKed, see if we can undo a cwnd reduction. */ if (flag & FLAG_DSACKING_ACK) { tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag, @@ -4853,8 +4868,9 @@ static void tcp_ofo_queue(struct sock *sk) if (before(TCP_SKB_CB(skb)->seq, dsack_high)) { __u32 dsack = dsack_high; + if (before(TCP_SKB_CB(skb)->end_seq, dsack_high)) - dsack_high = TCP_SKB_CB(skb)->end_seq; + dsack = TCP_SKB_CB(skb)->end_seq; tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack); } p = rb_next(p); @@ -4921,6 +4937,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) return; } + tcp_measure_rcv_mss(sk, skb); /* Disable header prediction. */ tp->pred_flags = 0; inet_csk_schedule_ack(sk); @@ -6691,6 +6708,9 @@ int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) if (!tp->srtt_us) tcp_synack_rtt_meas(sk, req); + if (tp->rx_opt.tstamp_ok) + tp->advmss -= TCPOLEN_TSTAMP_ALIGNED; + if (req) { tcp_rcv_synrecv_state_fastopen(sk); } else { @@ -6715,9 +6735,6 @@ int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale; tcp_init_wl(tp, TCP_SKB_CB(skb)->seq); - if (tp->rx_opt.tstamp_ok) - tp->advmss -= TCPOLEN_TSTAMP_ALIGNED; - if (!inet_csk(sk)->icsk_ca_ops->cong_control) tcp_update_pacing_rate(sk); @@ -7177,7 +7194,6 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, &foc, TCP_SYNACK_FASTOPEN, skb); /* Add the child socket directly into the accept queue */ if (!inet_csk_reqsk_queue_add(sk, req, fastopen_sk)) { - reqsk_fastopen_remove(fastopen_sk, req, false); bh_unlock_sock(fastopen_sk); sock_put(fastopen_sk); goto drop_and_free; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 0105c7d3fe969..ff2630bf62ae5 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -2195,7 +2195,8 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb, u32 max_segs) { const struct inet_connection_sock *icsk = inet_csk(sk); - u32 send_win, cong_win, limit, in_flight; + u32 send_win, cong_win, limit, in_flight, threshold; + u64 srtt_in_ns, expected_ack, how_far_is_the_ack; struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *head; int win_divisor; @@ -2257,9 +2258,19 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb, head = tcp_rtx_queue_head(sk); if (!head) goto send_now; - delta = tp->tcp_clock_cache - head->tstamp; - /* If next ACK is likely to come too late (half srtt), do not defer */ - if ((s64)(delta - (u64)NSEC_PER_USEC * (tp->srtt_us >> 4)) < 0) + + srtt_in_ns = (u64)(NSEC_PER_USEC >> 3) * tp->srtt_us; + /* When is the ACK expected ? */ + expected_ack = head->tstamp + srtt_in_ns; + /* How far from now is the ACK expected ? */ + how_far_is_the_ack = expected_ack - tp->tcp_clock_cache; + + /* If next ACK is likely to come too late, + * ie in more than min(1ms, half srtt), do not defer. + */ + threshold = min(srtt_in_ns >> 1, NSEC_PER_MSEC); + + if ((s64)(how_far_is_the_ack - threshold) > 0) goto send_now; /* Ok, it looks like it is advisable to defer. diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 2ab16139c197b..9be9df2caf654 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -61,7 +61,7 @@ static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM); skb->remcsum_offload = remcsum; - need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb)); + need_ipsec = (skb_dst(skb) && dst_xfrm(skb_dst(skb))) || skb_sec_path(skb); /* Try to offload checksum if possible */ offload_csum = !!(need_csum && !need_ipsec && @@ -247,6 +247,62 @@ static struct sk_buff *__udpv4_gso_segment_list_csum(struct sk_buff *segs) return segs; } +static void __udpv6_gso_segment_csum(struct sk_buff *seg, + struct in6_addr *oldip, + const struct in6_addr *newip, + __be16 *oldport, __be16 newport) +{ + struct udphdr *uh = udp_hdr(seg); + + if (ipv6_addr_equal(oldip, newip) && *oldport == newport) + return; + + if (uh->check) { + inet_proto_csum_replace16(&uh->check, seg, oldip->s6_addr32, + newip->s6_addr32, true); + + inet_proto_csum_replace2(&uh->check, seg, *oldport, newport, + false); + if (!uh->check) + uh->check = CSUM_MANGLED_0; + } + + *oldip = *newip; + *oldport = newport; +} + +static struct sk_buff *__udpv6_gso_segment_list_csum(struct sk_buff *segs) +{ + const struct ipv6hdr *iph; + const struct udphdr *uh; + struct ipv6hdr *iph2; + struct sk_buff *seg; + struct udphdr *uh2; + + seg = segs; + uh = udp_hdr(seg); + iph = ipv6_hdr(seg); + uh2 = udp_hdr(seg->next); + iph2 = ipv6_hdr(seg->next); + + if (!(*(const u32 *)&uh->source ^ *(const u32 *)&uh2->source) && + ipv6_addr_equal(&iph->saddr, &iph2->saddr) && + ipv6_addr_equal(&iph->daddr, &iph2->daddr)) + return segs; + + while ((seg = seg->next)) { + uh2 = udp_hdr(seg); + iph2 = ipv6_hdr(seg); + + __udpv6_gso_segment_csum(seg, &iph2->saddr, &iph->saddr, + &uh2->source, uh->source); + __udpv6_gso_segment_csum(seg, &iph2->daddr, &iph->daddr, + &uh2->dest, uh->dest); + } + + return segs; +} + static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb, netdev_features_t features, bool is_ipv6) @@ -259,7 +315,10 @@ static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb, udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss); - return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb); + if (is_ipv6) + return __udpv6_gso_segment_list_csum(skb); + else + return __udpv4_gso_segment_list_csum(skb); } struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, @@ -273,6 +332,7 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, bool copy_dtor; __sum16 check; __be16 newlen; + int ret = 0; mss = skb_shinfo(gso_skb)->gso_size; if (gso_skb->len <= sizeof(*uh) + mss) @@ -295,6 +355,10 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, if (skb_pagelen(gso_skb) - sizeof(*uh) == skb_shinfo(gso_skb)->gso_size) return __udp_gso_segment_list(gso_skb, features, is_ipv6); + ret = __skb_linearize(gso_skb); + if (ret) + return ERR_PTR(ret); + /* Setup csum, as fraglist skips this in udp4_gro_receive. */ gso_skb->csum_start = skb_transport_header(gso_skb) - gso_skb->head; gso_skb->csum_offset = offsetof(struct udphdr, check); diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c index 0292197497850..a08b0b6e0727c 100644 --- a/net/ipv4/udp_tunnel_nic.c +++ b/net/ipv4/udp_tunnel_nic.c @@ -899,7 +899,7 @@ udp_tunnel_nic_netdevice_event(struct notifier_block *unused, err = udp_tunnel_nic_register(dev); if (err) - netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err); + netdev_warn(dev, "failed to register for UDP tunnel offloads: %d", err); return notifier_from_errno(err); } /* All other events will need the udp_tunnel_nic state */ diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index c6d3e20bff1e4..b60325a35b3bc 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -237,6 +237,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = { .ioam6_id = IOAM6_DEFAULT_IF_ID, .ioam6_id_wide = IOAM6_DEFAULT_IF_ID_WIDE, .ndisc_evict_nocarrier = 1, + .ra_honor_pio_pflag = 0, }; static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { @@ -299,6 +300,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { .ioam6_id = IOAM6_DEFAULT_IF_ID, .ioam6_id_wide = IOAM6_DEFAULT_IF_ID_WIDE, .ndisc_evict_nocarrier = 1, + .ra_honor_pio_pflag = 0, }; /* Check if link is ready: is it up and is a valid qdisc available */ @@ -2195,13 +2197,12 @@ void addrconf_dad_failure(struct sk_buff *skb, struct inet6_ifaddr *ifp) in6_ifa_put(ifp); } -/* Join to solicited addr multicast group. - * caller must hold RTNL */ +/* Join to solicited addr multicast group. */ void addrconf_join_solict(struct net_device *dev, const struct in6_addr *addr) { struct in6_addr maddr; - if (dev->flags&(IFF_LOOPBACK|IFF_NOARP)) + if (READ_ONCE(dev->flags) & (IFF_LOOPBACK | IFF_NOARP)) return; addrconf_addr_solict_mult(addr, &maddr); @@ -2765,6 +2766,7 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao) u32 addr_flags = 0; struct inet6_dev *in6_dev; struct net *net = dev_net(dev); + bool ignore_autoconf = false; pinfo = (struct prefix_info *) opt; @@ -2858,7 +2860,8 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao) /* Try to figure out our local address for this prefix */ - if (pinfo->autoconf && in6_dev->cnf.autoconf) { + ignore_autoconf = READ_ONCE(in6_dev->cnf.ra_honor_pio_pflag) && pinfo->preferpd; + if (pinfo->autoconf && in6_dev->cnf.autoconf && !ignore_autoconf) { struct in6_addr addr; bool tokenized = false, dev_addr_generated = false; @@ -3211,16 +3214,13 @@ static void add_v4_addrs(struct inet6_dev *idev) struct in6_addr addr; struct net_device *dev; struct net *net = dev_net(idev->dev); - int scope, plen, offset = 0; + int scope, plen; u32 pflags = 0; ASSERT_RTNL(); memset(&addr, 0, sizeof(struct in6_addr)); - /* in case of IP6GRE the dev_addr is an IPv6 and therefore we use only the last 4 bytes */ - if (idev->dev->addr_len == sizeof(struct in6_addr)) - offset = sizeof(struct in6_addr) - 4; - memcpy(&addr.s6_addr32[3], idev->dev->dev_addr + offset, 4); + memcpy(&addr.s6_addr32[3], idev->dev->dev_addr, 4); if (!(idev->dev->flags & IFF_POINTOPOINT) && idev->dev->type == ARPHRD_SIT) { scope = IPV6_ADDR_COMPATv4; @@ -3524,21 +3524,22 @@ static void addrconf_gre_config(struct net_device *dev) ASSERT_RTNL(); - idev = ipv6_find_idev(dev); - if (IS_ERR(idev)) { - pr_debug("%s: add_dev failed\n", __func__); + idev = addrconf_add_dev(dev); + if (IS_ERR(idev)) return; - } - if (dev->type == ARPHRD_ETHER) { + /* Generate the IPv6 link-local address using addrconf_addr_gen(), + * unless we have an IPv4 GRE device not bound to an IP address and + * which is in EUI64 mode (as __ipv6_isatap_ifid() would fail in this + * case). Such devices fall back to add_v4_addrs() instead. + */ + if (!(dev->type == ARPHRD_IPGRE && *(__be32 *)dev->dev_addr == 0 && + idev->cnf.addr_gen_mode == IN6_ADDR_GEN_MODE_EUI64)) { addrconf_addr_gen(idev, true); return; } add_v4_addrs(idev); - - if (dev->flags & IFF_POINTOPOINT) - addrconf_add_mroute(dev); } #endif @@ -3858,7 +3859,7 @@ static int addrconf_ifdown(struct net_device *dev, bool unregister) * Do not dev_put! */ if (unregister) { - idev->dead = 1; + WRITE_ONCE(idev->dead, 1); /* protected by rtnl_lock */ RCU_INIT_POINTER(dev->ip6_ptr, NULL); @@ -6913,6 +6914,15 @@ static const struct ctl_table addrconf_sysctl[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "ra_honor_pio_pflag", + .data = &ipv6_devconf.ra_honor_pio_pflag, + .maxlen = sizeof(u8), + .mode = 0644, + .proc_handler = proc_dou8vec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, #ifdef CONFIG_IPV6_ROUTER_PREF { .procname = "accept_ra_rtr_pref", @@ -7134,7 +7144,9 @@ static const struct ctl_table addrconf_sysctl[] = { .data = &ipv6_devconf.rpl_seg_enabled, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, }, { .procname = "ioam6_enabled", @@ -7414,7 +7426,8 @@ int __init addrconf_init(void) if (err < 0) goto out_addrlabel; - addrconf_wq = create_workqueue("ipv6_addrconf"); + /* All works using addrconf_wq need to lock rtnl. */ + addrconf_wq = create_singlethread_workqueue("ipv6_addrconf"); if (!addrconf_wq) { err = -ENOMEM; goto out_nowq; diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index 01005035ad101..5361e2107458f 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -46,6 +46,34 @@ struct ah_skb_cb { #define AH_SKB_CB(__skb) ((struct ah_skb_cb *)&((__skb)->cb[0])) +/* Helper to save IPv6 addresses and extension headers to temporary storage */ +static inline void ah6_save_hdrs(struct tmp_ext *iph_ext, + struct ipv6hdr *top_iph, int extlen) +{ + if (!extlen) + return; + +#if IS_ENABLED(CONFIG_IPV6_MIP6) + iph_ext->saddr = top_iph->saddr; +#endif + iph_ext->daddr = top_iph->daddr; + memcpy(&iph_ext->hdrs, top_iph + 1, extlen - sizeof(*iph_ext)); +} + +/* Helper to restore IPv6 addresses and extension headers from temporary storage */ +static inline void ah6_restore_hdrs(struct ipv6hdr *top_iph, + struct tmp_ext *iph_ext, int extlen) +{ + if (!extlen) + return; + +#if IS_ENABLED(CONFIG_IPV6_MIP6) + top_iph->saddr = iph_ext->saddr; +#endif + top_iph->daddr = iph_ext->daddr; + memcpy(top_iph + 1, &iph_ext->hdrs, extlen - sizeof(*iph_ext)); +} + static void *ah_alloc_tmp(struct crypto_ahash *ahash, int nfrags, unsigned int size) { @@ -304,13 +332,7 @@ static void ah6_output_done(void *data, int err) memcpy(ah->auth_data, icv, ahp->icv_trunc_len); memcpy(top_iph, iph_base, IPV6HDR_BASELEN); - if (extlen) { -#if IS_ENABLED(CONFIG_IPV6_MIP6) - memcpy(&top_iph->saddr, iph_ext, extlen); -#else - memcpy(&top_iph->daddr, iph_ext, extlen); -#endif - } + ah6_restore_hdrs(top_iph, iph_ext, extlen); kfree(AH_SKB_CB(skb)->tmp); xfrm_output_resume(skb->sk, skb, err); @@ -381,12 +403,8 @@ static int ah6_output(struct xfrm_state *x, struct sk_buff *skb) */ memcpy(iph_base, top_iph, IPV6HDR_BASELEN); + ah6_save_hdrs(iph_ext, top_iph, extlen); if (extlen) { -#if IS_ENABLED(CONFIG_IPV6_MIP6) - memcpy(iph_ext, &top_iph->saddr, extlen); -#else - memcpy(iph_ext, &top_iph->daddr, extlen); -#endif err = ipv6_clear_mutable_options(top_iph, extlen - sizeof(*iph_ext) + sizeof(*top_iph), @@ -437,13 +455,7 @@ static int ah6_output(struct xfrm_state *x, struct sk_buff *skb) memcpy(ah->auth_data, icv, ahp->icv_trunc_len); memcpy(top_iph, iph_base, IPV6HDR_BASELEN); - if (extlen) { -#if IS_ENABLED(CONFIG_IPV6_MIP6) - memcpy(&top_iph->saddr, iph_ext, extlen); -#else - memcpy(&top_iph->daddr, iph_ext, extlen); -#endif - } + ah6_restore_hdrs(top_iph, iph_ext, extlen); out_free: kfree(iph_base); diff --git a/net/ipv6/calipso.c b/net/ipv6/calipso.c index c07e3da08d2a8..24666291c54a8 100644 --- a/net/ipv6/calipso.c +++ b/net/ipv6/calipso.c @@ -1210,6 +1210,10 @@ static int calipso_req_setattr(struct request_sock *req, struct ipv6_opt_hdr *old, *new; struct sock *sk = sk_to_full_sk(req_to_sk(req)); + /* sk is NULL for SYN+ACK w/ SYN Cookie */ + if (!sk) + return -ENOMEM; + if (req_inet->ipv6_opt && req_inet->ipv6_opt->hopopt) old = req_inet->ipv6_opt->hopopt; else @@ -1250,6 +1254,10 @@ static void calipso_req_delattr(struct request_sock *req) struct ipv6_txoptions *txopts; struct sock *sk = sk_to_full_sk(req_to_sk(req)); + /* sk is NULL for SYN+ACK w/ SYN Cookie */ + if (!sk) + return; + if (!req_inet->ipv6_opt || !req_inet->ipv6_opt->hopopt) return; diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index 62bb9651133c4..43a9966e2064d 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -214,8 +214,10 @@ static int esp_output_tcp_finish(struct xfrm_state *x, struct sk_buff *skb) sk = esp6_find_tcp_sk(x); err = PTR_ERR_OR_ZERO(sk); - if (err) + if (err) { + kfree_skb(skb); goto out; + } bh_lock_sock(sk); if (sock_owned_by_user(sk)) diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c index a189e08370a5e..438f9cbdca299 100644 --- a/net/ipv6/esp6_offload.c +++ b/net/ipv6/esp6_offload.c @@ -145,8 +145,10 @@ static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - __be16 type = x->inner_mode.family == AF_INET ? htons(ETH_P_IP) - : htons(ETH_P_IPV6); + const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, + XFRM_MODE_SKB_CB(skb)->protocol); + __be16 type = inner_mode->family == AF_INET ? htons(ETH_P_IP) + : htons(ETH_P_IPV6); return skb_eth_gso_segment(skb, features, type); } diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c index 6eeab21512ba9..e0f0c5f8cccda 100644 --- a/net/ipv6/fib6_rules.c +++ b/net/ipv6/fib6_rules.c @@ -350,9 +350,9 @@ static int fib6_rule_configure(struct fib_rule *rule, struct sk_buff *skb, struct nlattr **tb, struct netlink_ext_ack *extack) { + struct fib6_rule *rule6 = (struct fib6_rule *)rule; + struct net *net = rule->fr_net; int err = -EINVAL; - struct net *net = sock_net(skb->sk); - struct fib6_rule *rule6 = (struct fib6_rule *) rule; if (!inet_validate_dscp(frh->tos)) { NL_SET_ERR_MSG(extack, diff --git a/net/ipv6/ila/ila_common.c b/net/ipv6/ila/ila_common.c index 95e9146918cc6..b8d43ed4689db 100644 --- a/net/ipv6/ila/ila_common.c +++ b/net/ipv6/ila/ila_common.c @@ -86,7 +86,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&th->check, skb, - diff, true); + diff, true, true); } break; case NEXTHDR_UDP: @@ -97,7 +97,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) { diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&uh->check, skb, - diff, true); + diff, true, true); if (!uh->check) uh->check = CSUM_MANGLED_0; } @@ -111,7 +111,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&ih->icmp6_cksum, skb, - diff, true); + diff, true, true); } break; } diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index 984276484ca1f..81de4c0ec957f 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -437,15 +437,17 @@ struct fib6_dump_arg { static int fib6_rt_dump(struct fib6_info *rt, struct fib6_dump_arg *arg) { enum fib_event_type fib_event = FIB_EVENT_ENTRY_REPLACE; + unsigned int nsiblings; int err; if (!rt || rt == arg->net->ipv6.fib6_null_entry) return 0; - if (rt->fib6_nsiblings) + nsiblings = READ_ONCE(rt->fib6_nsiblings); + if (nsiblings) err = call_fib6_multipath_entry_notifier(arg->nb, fib_event, rt, - rt->fib6_nsiblings, + nsiblings, arg->extack); else err = call_fib6_entry_notifier(arg->nb, fib_event, rt, @@ -1118,7 +1120,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, if (rt6_duplicate_nexthop(iter, rt)) { if (rt->fib6_nsiblings) - rt->fib6_nsiblings = 0; + WRITE_ONCE(rt->fib6_nsiblings, 0); if (!(iter->fib6_flags & RTF_EXPIRES)) return -EEXIST; if (!(rt->fib6_flags & RTF_EXPIRES)) @@ -1144,7 +1146,8 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, */ if (rt_can_ecmp && rt6_qualify_for_ecmp(iter)) - rt->fib6_nsiblings++; + WRITE_ONCE(rt->fib6_nsiblings, + rt->fib6_nsiblings + 1); } if (iter->fib6_metric > rt->fib6_metric) @@ -1194,7 +1197,8 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, fib6_nsiblings = 0; list_for_each_entry_safe(sibling, temp_sibling, &rt->fib6_siblings, fib6_siblings) { - sibling->fib6_nsiblings++; + WRITE_ONCE(sibling->fib6_nsiblings, + sibling->fib6_nsiblings + 1); BUG_ON(sibling->fib6_nsiblings != rt->fib6_nsiblings); fib6_nsiblings++; } @@ -1239,8 +1243,9 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, list_for_each_entry_safe(sibling, next_sibling, &rt->fib6_siblings, fib6_siblings) - sibling->fib6_nsiblings--; - rt->fib6_nsiblings = 0; + WRITE_ONCE(sibling->fib6_nsiblings, + sibling->fib6_nsiblings - 1); + WRITE_ONCE(rt->fib6_nsiblings, 0); list_del_rcu(&rt->fib6_siblings); rt6_multipath_rebalance(next_sibling); return err; @@ -1952,8 +1957,9 @@ static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn, notify_del = true; list_for_each_entry_safe(sibling, next_sibling, &rt->fib6_siblings, fib6_siblings) - sibling->fib6_nsiblings--; - rt->fib6_nsiblings = 0; + WRITE_ONCE(sibling->fib6_nsiblings, + sibling->fib6_nsiblings - 1); + WRITE_ONCE(rt->fib6_nsiblings, 0); list_del_rcu(&rt->fib6_siblings); rt6_multipath_rebalance(next_sibling); } diff --git a/net/ipv6/ip6_icmp.c b/net/ipv6/ip6_icmp.c index 9e3574880cb03..233914b63bdb8 100644 --- a/net/ipv6/ip6_icmp.c +++ b/net/ipv6/ip6_icmp.c @@ -54,11 +54,12 @@ void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info) struct inet6_skb_parm parm = { 0 }; struct sk_buff *cloned_skb = NULL; enum ip_conntrack_info ctinfo; + enum ip_conntrack_dir dir; struct in6_addr orig_ip; struct nf_conn *ct; ct = nf_ct_get(skb_in, &ctinfo); - if (!ct || !(ct->status & IPS_SRC_NAT)) { + if (!ct || !(READ_ONCE(ct->status) & IPS_NAT_MASK)) { __icmpv6_send(skb_in, type, code, info, &parm); return; } @@ -73,7 +74,8 @@ void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info) goto out; orig_ip = ipv6_hdr(skb_in)->saddr; - ipv6_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.in6; + dir = CTINFO2DIR(ctinfo); + ipv6_hdr(skb_in)->saddr = ct->tuplehash[dir].tuple.src.u3.in6; __icmpv6_send(skb_in, type, code, info, &parm); ipv6_hdr(skb_in)->saddr = orig_ip; out: diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index 7f014a8969fb2..84b17eeaa57cc 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -150,7 +150,9 @@ static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb, ops = rcu_dereference(inet6_offloads[proto]); if (likely(ops && ops->callbacks.gso_segment)) { - skb_reset_transport_header(skb); + if (!skb_reset_transport_header_careful(skb)) + goto out; + segs = ops->callbacks.gso_segment(skb, features); if (!segs) skb->network_header = skb_mac_header(skb) + nhoff - skb->head; diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index d645d022ce774..e635ddd41aba6 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1255,8 +1255,7 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, */ max_headroom = LL_RESERVED_SPACE(dst->dev) + sizeof(struct ipv6hdr) + dst->header_len + t->hlen; - if (max_headroom > READ_ONCE(dev->needed_headroom)) - WRITE_ONCE(dev->needed_headroom, max_headroom); + ip_tunnel_adj_headroom(dev, max_headroom); err = ip6_tnl_encap(skb, t, &proto, fl6); if (err) diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 7f19868d7d6c6..18451dfeae682 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -2035,6 +2035,7 @@ static int ip6mr_forward2(struct net *net, struct mr_table *mrt, struct sk_buff *skb, int vifi) { struct vif_device *vif = &mrt->vif_table[vifi]; + struct net_device *indev = skb->dev; struct net_device *vif_dev; struct ipv6hdr *ipv6h; struct dst_entry *dst; @@ -2097,7 +2098,7 @@ static int ip6mr_forward2(struct net *net, struct mr_table *mrt, IP6CB(skb)->flags |= IP6SKB_FORWARDED; return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, - net, NULL, skb, skb->dev, vif_dev, + net, NULL, skb, indev, skb->dev, ip6mr_forward2_finish); out_free: diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 9bb246c09fcee..160b452f75e7d 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -803,8 +803,8 @@ static void mld_del_delrec(struct inet6_dev *idev, struct ifmcaddr6 *im) } else { im->mca_crcount = idev->mc_qrv; } - in6_dev_put(pmc->idev); ip6_mc_clear_src(pmc); + in6_dev_put(pmc->idev); kfree_rcu(pmc, rcu); } } @@ -906,23 +906,22 @@ static struct ifmcaddr6 *mca_alloc(struct inet6_dev *idev, static int __ipv6_dev_mc_inc(struct net_device *dev, const struct in6_addr *addr, unsigned int mode) { - struct ifmcaddr6 *mc; struct inet6_dev *idev; - - ASSERT_RTNL(); + struct ifmcaddr6 *mc; /* we need to take a reference on idev */ idev = in6_dev_get(dev); - if (!idev) return -EINVAL; - if (idev->dead) { + mutex_lock(&idev->mc_lock); + + if (READ_ONCE(idev->dead)) { + mutex_unlock(&idev->mc_lock); in6_dev_put(idev); return -ENODEV; } - mutex_lock(&idev->mc_lock); for_each_mc_mclock(idev, mc) { if (ipv6_addr_equal(&mc->mca_addr, addr)) { mc->mca_users++; diff --git a/net/ipv6/netfilter.c b/net/ipv6/netfilter.c index 857713d7a38a5..d873658fc821f 100644 --- a/net/ipv6/netfilter.c +++ b/net/ipv6/netfilter.c @@ -163,20 +163,20 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, struct ip6_fraglist_iter iter; struct sk_buff *frag2; - if (first_len - hlen > mtu || - skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) + if (first_len - hlen > mtu) goto blackhole; - if (skb_cloned(skb)) + if (skb_cloned(skb) || + skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) goto slow_path; skb_walk_frags(skb, frag2) { - if (frag2->len > mtu || - skb_headroom(frag2) < (hlen + hroom + sizeof(struct frag_hdr))) + if (frag2->len > mtu) goto blackhole; /* Partially cloned skb? */ - if (skb_shared(frag2)) + if (skb_shared(frag2) || + skb_headroom(frag2) < (hlen + hroom + sizeof(struct frag_hdr))) goto slow_path; } diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c index 131f7bb2110d3..ce51c9f64515e 100644 --- a/net/ipv6/netfilter/ip6_tables.c +++ b/net/ipv6/netfilter/ip6_tables.c @@ -1811,18 +1811,29 @@ void ip6t_unregister_table_exit(struct net *net, const char *name) __ip6t_unregister_table(net, table); } +#define DEFINE_IP6T_STANDARD_TARGET(compat) \ + { \ + .name = XT_STANDARD_TARGET, \ + .targetsize = sizeof(int), \ + .family = NFPROTO_IPV6, \ + .has_compat_metadata = compat \ + } + /* The built-in targets: standard (NULL) and error. */ -static struct xt_target ip6t_builtin_tg[] __read_mostly = { - { - .name = XT_STANDARD_TARGET, - .targetsize = sizeof(int), - .family = NFPROTO_IPV6, #ifdef CONFIG_NETFILTER_XTABLES_COMPAT - .compatsize = sizeof(compat_int_t), - .compat_from_user = compat_standard_from_user, - .compat_to_user = compat_standard_to_user, +static struct compat_xt_target_ext ip6t_std_tg_ext = { + .compatsize = sizeof(compat_int_t), + .compat_from_user = compat_standard_from_user, + .compat_to_user = compat_standard_to_user, + .target = DEFINE_IP6T_STANDARD_TARGET(true), +}; + +#define ip6t_std_tg (ip6t_std_tg_ext.target) +#else +static struct xt_target ip6t_std_tg __read_mostly = DEFINE_IP6T_STANDARD_TARGET(false); #endif - }, + +static struct xt_target ip6t_builtin_tg[] __read_mostly = { { .name = XT_ERROR_TARGET, .target = ip6t_error, @@ -1866,19 +1877,25 @@ static int __init ip6_tables_init(void) goto err1; /* No one else will be downing sem now, so we won't sleep */ + ret = xt_register_target(&ip6t_std_tg); + if (ret) + goto err2; + ret = xt_register_targets(ip6t_builtin_tg, ARRAY_SIZE(ip6t_builtin_tg)); if (ret < 0) - goto err2; + goto err4; /* Register setsockopt */ ret = nf_register_sockopt(&ip6t_sockopts); if (ret < 0) - goto err4; + goto err8; return 0; -err4: +err8: xt_unregister_targets(ip6t_builtin_tg, ARRAY_SIZE(ip6t_builtin_tg)); +err4: + xt_unregister_target(&ip6t_std_tg); err2: unregister_pernet_subsys(&ip6_tables_net_ops); err1: @@ -1890,6 +1907,7 @@ static void __exit ip6_tables_fini(void) nf_unregister_sockopt(&ip6t_sockopts); xt_unregister_targets(ip6t_builtin_tg, ARRAY_SIZE(ip6t_builtin_tg)); + xt_unregister_target(&ip6t_std_tg); unregister_pernet_subsys(&ip6_tables_net_ops); } diff --git a/net/ipv6/netfilter/nf_reject_ipv6.c b/net/ipv6/netfilter/nf_reject_ipv6.c index e4776bd2ed89b..a19ca1907de36 100644 --- a/net/ipv6/netfilter/nf_reject_ipv6.c +++ b/net/ipv6/netfilter/nf_reject_ipv6.c @@ -91,6 +91,32 @@ struct sk_buff *nf_reject_skb_v6_tcp_reset(struct net *net, } EXPORT_SYMBOL_GPL(nf_reject_skb_v6_tcp_reset); +static bool nf_skb_is_icmp6_unreach(const struct sk_buff *skb) +{ + const struct ipv6hdr *ip6h = ipv6_hdr(skb); + u8 proto = ip6h->nexthdr; + u8 _type, *tp; + int thoff; + __be16 fo; + + thoff = ipv6_skip_exthdr(skb, ((u8 *)(ip6h + 1) - skb->data), &proto, &fo); + + if (thoff < 0 || thoff >= skb->len || fo != 0) + return false; + + if (proto != IPPROTO_ICMPV6) + return false; + + tp = skb_header_pointer(skb, + thoff + offsetof(struct icmp6hdr, icmp6_type), + sizeof(_type), &_type); + + if (!tp) + return false; + + return *tp == ICMPV6_DEST_UNREACH; +} + struct sk_buff *nf_reject_skb_v6_unreach(struct net *net, struct sk_buff *oldskb, const struct net_device *dev, @@ -104,6 +130,10 @@ struct sk_buff *nf_reject_skb_v6_unreach(struct net *net, if (!nf_reject_ip6hdr_validate(oldskb)) return NULL; + /* Don't reply to ICMPV6_DEST_UNREACH with ICMPV6_DEST_UNREACH */ + if (nf_skb_is_icmp6_unreach(oldskb)) + return NULL; + /* Include "As much of invoking packet as possible without the ICMPv6 * packet exceeding the minimum IPv6 MTU" in the ICMP payload. */ @@ -293,7 +323,7 @@ void nf_send_reset6(struct net *net, struct sock *sk, struct sk_buff *oldskb, fl6.fl6_sport = otcph->dest; fl6.fl6_dport = otcph->source; - if (hook == NF_INET_PRE_ROUTING || hook == NF_INET_INGRESS) { + if (!skb_dst(oldskb)) { nf_ip6_route(net, &dst, flowi6_to_flowi(&fl6), false); if (!dst) return; @@ -397,8 +427,7 @@ void nf_send_unreach6(struct net *net, struct sk_buff *skb_in, if (hooknum == NF_INET_LOCAL_OUT && skb_in->dev == NULL) skb_in->dev = net->loopback_dev; - if ((hooknum == NF_INET_PRE_ROUTING || hooknum == NF_INET_INGRESS) && - nf_reject6_fill_skb_dst(skb_in) < 0) + if (!skb_dst(skb_in) && nf_reject6_fill_skb_dst(skb_in) < 0) return; icmpv6_send(skb_in, ICMPV6_DEST_UNREACH, code, 0); diff --git a/net/ipv6/netfilter/nft_fib_ipv6.c b/net/ipv6/netfilter/nft_fib_ipv6.c index c9f1634b3838a..a89ce0fbfe4b1 100644 --- a/net/ipv6/netfilter/nft_fib_ipv6.c +++ b/net/ipv6/netfilter/nft_fib_ipv6.c @@ -158,6 +158,7 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs, { const struct nft_fib *priv = nft_expr_priv(expr); int noff = skb_network_offset(pkt->skb); + const struct net_device *found = NULL; const struct net_device *oif = NULL; u32 *dest = ®s->data[priv->dreg]; struct ipv6hdr *iph, _iph; @@ -202,11 +203,15 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs, if (rt->rt6i_flags & (RTF_REJECT | RTF_ANYCAST | RTF_LOCAL)) goto put_rt_err; - if (oif && oif != rt->rt6i_idev->dev && - l3mdev_master_ifindex_rcu(rt->rt6i_idev->dev) != oif->ifindex) - goto put_rt_err; + if (!oif) { + found = rt->rt6i_idev->dev; + } else { + if (oif == rt->rt6i_idev->dev || + l3mdev_master_ifindex_rcu(rt->rt6i_idev->dev) == oif->ifindex) + found = oif; + } - nft_fib_store_result(dest, priv, rt->rt6i_idev->dev); + nft_fib_store_result(dest, priv, found); put_rt_err: ip6_rt_put(rt); } diff --git a/net/ipv6/proc.c b/net/ipv6/proc.c index 6d1d9221649d5..752327b10dde7 100644 --- a/net/ipv6/proc.c +++ b/net/ipv6/proc.c @@ -27,7 +27,7 @@ #include #define MAX4(a, b, c, d) \ - max_t(u32, max_t(u32, a, b), max_t(u32, c, d)) + MAX_T(u32, MAX_T(u32, a, b), MAX_T(u32, c, d)) #define SNMP_MIB_MAX MAX4(UDP_MIB_MAX, TCP_MIB_MAX, \ IPSTATS_MIB_MAX, ICMP_MIB_MAX) diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 4f526606bc894..7d72633ea0198 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -438,7 +438,7 @@ static int rawv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, if (flags & MSG_ERRQUEUE) return ipv6_recv_error(sk, msg, len, addr_len); - if (np->rxpmtu && np->rxopt.bits.rxpmtu) + if (np->rxopt.bits.rxpmtu && READ_ONCE(np->rxpmtu)) return ipv6_recv_rxpmtu(sk, msg, len, addr_len); skb = skb_recv_datagram(sk, flags, &err); diff --git a/net/ipv6/route.c b/net/ipv6/route.c index f6a6a426fd6f8..1109eb5900ac8 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -5207,7 +5207,8 @@ static void ip6_route_mpath_notify(struct fib6_info *rt, */ rcu_read_lock(); - if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) { + if ((nlflags & NLM_F_APPEND) && rt_last && + READ_ONCE(rt_last->fib6_nsiblings)) { rt = list_first_or_null_rcu(&rt_last->fib6_siblings, struct fib6_info, fib6_siblings); @@ -5554,32 +5555,34 @@ static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg) static size_t rt6_nlmsg_size(struct fib6_info *f6i) { + struct fib6_info *sibling; + struct fib6_nh *nh; int nexthop_len; if (f6i->nh) { nexthop_len = nla_total_size(4); /* RTA_NH_ID */ nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size, &nexthop_len); - } else { - struct fib6_nh *nh = f6i->fib6_nh; - struct fib6_info *sibling; - - nexthop_len = 0; - if (f6i->fib6_nsiblings) { - rt6_nh_nlmsg_size(nh, &nexthop_len); - - rcu_read_lock(); + goto common; + } - list_for_each_entry_rcu(sibling, &f6i->fib6_siblings, - fib6_siblings) { - rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len); - } + rcu_read_lock(); +retry: + nh = f6i->fib6_nh; + nexthop_len = 0; + if (READ_ONCE(f6i->fib6_nsiblings)) { + rt6_nh_nlmsg_size(nh, &nexthop_len); - rcu_read_unlock(); + list_for_each_entry_rcu(sibling, &f6i->fib6_siblings, + fib6_siblings) { + rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len); + if (!READ_ONCE(f6i->fib6_nsiblings)) + goto retry; } - nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); } - + rcu_read_unlock(); + nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); +common: return NLMSG_ALIGN(sizeof(struct rtmsg)) + nla_total_size(16) /* RTA_SRC */ + nla_total_size(16) /* RTA_DST */ @@ -5738,7 +5741,7 @@ static int rt6_fill_node(struct net *net, struct sk_buff *skb, if (dst->lwtstate && lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) goto nla_put_failure; - } else if (rt->fib6_nsiblings) { + } else if (READ_ONCE(rt->fib6_nsiblings)) { struct fib6_info *sibling; struct nlattr *mp; @@ -5840,16 +5843,21 @@ static bool fib6_info_uses_dev(const struct fib6_info *f6i, if (f6i->fib6_nh->fib_nh_dev == dev) return true; - if (f6i->fib6_nsiblings) { - struct fib6_info *sibling, *next_sibling; + if (READ_ONCE(f6i->fib6_nsiblings)) { + const struct fib6_info *sibling; - list_for_each_entry_safe(sibling, next_sibling, - &f6i->fib6_siblings, fib6_siblings) { - if (sibling->fib6_nh->fib_nh_dev == dev) + rcu_read_lock(); + list_for_each_entry_rcu(sibling, &f6i->fib6_siblings, + fib6_siblings) { + if (sibling->fib6_nh->fib_nh_dev == dev) { + rcu_read_unlock(); return true; + } + if (!READ_ONCE(f6i->fib6_nsiblings)) + break; } + rcu_read_unlock(); } - return false; } diff --git a/net/ipv6/rpl_iptunnel.c b/net/ipv6/rpl_iptunnel.c index 28fc7fae57972..523aa8c9b382f 100644 --- a/net/ipv6/rpl_iptunnel.c +++ b/net/ipv6/rpl_iptunnel.c @@ -129,13 +129,13 @@ static int rpl_do_srh_inline(struct sk_buff *skb, const struct rpl_lwt *rlwt, struct dst_entry *cache_dst) { struct ipv6_rpl_sr_hdr *isrh, *csrh; - const struct ipv6hdr *oldhdr; + struct ipv6hdr oldhdr; struct ipv6hdr *hdr; unsigned char *buf; size_t hdrlen; int err; - oldhdr = ipv6_hdr(skb); + memcpy(&oldhdr, ipv6_hdr(skb), sizeof(oldhdr)); buf = kcalloc(struct_size(srh, segments.addr, srh->segments_left), 2, GFP_ATOMIC); if (!buf) @@ -147,7 +147,7 @@ static int rpl_do_srh_inline(struct sk_buff *skb, const struct rpl_lwt *rlwt, memcpy(isrh, srh, sizeof(*isrh)); memcpy(isrh->rpl_segaddr, &srh->rpl_segaddr[1], (srh->segments_left - 1) * 16); - isrh->rpl_segaddr[srh->segments_left - 1] = oldhdr->daddr; + isrh->rpl_segaddr[srh->segments_left - 1] = oldhdr.daddr; ipv6_rpl_srh_compress(csrh, isrh, &srh->rpl_segaddr[0], isrh->segments_left - 1); @@ -169,7 +169,7 @@ static int rpl_do_srh_inline(struct sk_buff *skb, const struct rpl_lwt *rlwt, skb_mac_header_rebuild(skb); hdr = ipv6_hdr(skb); - memmove(hdr, oldhdr, sizeof(*hdr)); + memmove(hdr, &oldhdr, sizeof(*hdr)); isrh = (void *)hdr + sizeof(*hdr); memcpy(isrh, csrh, hdrlen); diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index 3c3800223e0e0..6e15a65faeccd 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -269,7 +270,7 @@ bool seg6_hmac_validate_skb(struct sk_buff *skb) if (seg6_hmac_compute(hinfo, srh, &ipv6_hdr(skb)->saddr, hmac_output)) return false; - if (memcmp(hmac_output, tlv->hmac, SEG6_HMAC_FIELD_LEN) != 0) + if (crypto_memneq(hmac_output, tlv->hmac, SEG6_HMAC_FIELD_LEN)) return false; return true; @@ -293,6 +294,9 @@ int seg6_hmac_info_add(struct net *net, u32 key, struct seg6_hmac_info *hinfo) struct seg6_pernet_data *sdata = seg6_pernet(net); int err; + if (!__hmac_get_algo(hinfo->alg_id)) + return -EINVAL; + err = rhashtable_lookup_insert_fast(&sdata->hmac_infos, &hinfo->node, rht_params); diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index c434940131b1d..7f295b9c13744 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -1638,10 +1638,8 @@ static const struct nla_policy seg6_local_policy[SEG6_LOCAL_MAX + 1] = { [SEG6_LOCAL_SRH] = { .type = NLA_BINARY }, [SEG6_LOCAL_TABLE] = { .type = NLA_U32 }, [SEG6_LOCAL_VRFTABLE] = { .type = NLA_U32 }, - [SEG6_LOCAL_NH4] = { .type = NLA_BINARY, - .len = sizeof(struct in_addr) }, - [SEG6_LOCAL_NH6] = { .type = NLA_BINARY, - .len = sizeof(struct in6_addr) }, + [SEG6_LOCAL_NH4] = NLA_POLICY_EXACT_LEN(sizeof(struct in_addr)), + [SEG6_LOCAL_NH6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), [SEG6_LOCAL_IIF] = { .type = NLA_U32 }, [SEG6_LOCAL_OIF] = { .type = NLA_U32 }, [SEG6_LOCAL_BPF] = { .type = NLA_NESTED }, diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 9d39e25545bfa..f6d6497e49316 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -348,7 +348,7 @@ int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, if (flags & MSG_ERRQUEUE) return ipv6_recv_error(sk, msg, len, addr_len); - if (np->rxpmtu && np->rxopt.bits.rxpmtu) + if (np->rxopt.bits.rxpmtu && READ_ONCE(np->rxpmtu)) return ipv6_recv_rxpmtu(sk, msg, len, addr_len); try_again: diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index cc25fec44f850..19f3de3c24ef1 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -888,15 +888,15 @@ static int llc_ui_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (sk->sk_type != SOCK_STREAM) goto copy_uaddr; + /* Partial read */ + if (used + offset < skb_len) + continue; + if (!(flags & MSG_PEEK)) { skb_unlink(skb, &sk->sk_receive_queue); kfree_skb(skb); *seq = 0; } - - /* Partial read */ - if (used + offset < skb_len) - continue; } while (len > 0); out: diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index a3c5d4d995db0..1addfba4b2856 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1099,13 +1099,13 @@ ieee80211_copy_rnr_beacon(u8 *pos, struct cfg80211_rnr_elems *dst, { int i, offset = 0; + dst->cnt = src->cnt; for (i = 0; i < src->cnt; i++) { memcpy(pos + offset, src->elem[i].data, src->elem[i].len); dst->elem[i].len = src->elem[i].len; dst->elem[i].data = pos + offset; offset += dst->elem[i].len; } - dst->cnt = src->cnt; return offset; } @@ -1847,12 +1847,12 @@ static int sta_link_apply_parameters(struct ieee80211_local *local, } if (params->supported_rates && - params->supported_rates_len) { - ieee80211_parse_bitrates(link->conf->chandef.width, - sband, params->supported_rates, - params->supported_rates_len, - &link_sta->pub->supp_rates[sband->band]); - } + params->supported_rates_len && + !ieee80211_parse_bitrates(link->conf->chandef.width, + sband, params->supported_rates, + params->supported_rates_len, + &link_sta->pub->supp_rates[sband->band])) + return -EINVAL; if (params->ht_capa) ieee80211_ht_cap_ie_to_sta_ht_cap(sdata, sband, diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index 68952752b5990..4a21e53afa72e 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -89,11 +89,11 @@ ieee80211_chanctx_reserved_chandef(struct ieee80211_local *local, lockdep_assert_held(&local->chanctx_mtx); + if (WARN_ON(!compat)) + return NULL; + list_for_each_entry(link, &ctx->reserved_links, reserved_chanctx_list) { - if (!compat) - compat = &link->reserved_chandef; - compat = cfg80211_chandef_compatible(&link->reserved_chandef, compat); if (!compat) @@ -1313,6 +1313,7 @@ ieee80211_link_use_reserved_reassign(struct ieee80211_link_data *link) goto out; } + link->radar_required = link->reserved_radar_required; list_move(&link->assigned_chanctx_list, &new_ctx->assigned_links); rcu_assign_pointer(link_conf->chanctx_conf, &new_ctx->conf); diff --git a/net/mac80211/driver-ops.h b/net/mac80211/driver-ops.h index 78aa3bc51586e..5f8f72ca2769c 100644 --- a/net/mac80211/driver-ops.h +++ b/net/mac80211/driver-ops.h @@ -1273,7 +1273,7 @@ drv_get_ftm_responder_stats(struct ieee80211_local *local, struct ieee80211_sub_if_data *sdata, struct cfg80211_ftm_responder_stats *ftm_stats) { - u32 ret = -EOPNOTSUPP; + int ret = -EOPNOTSUPP; if (local->ops->get_ftm_responder_stats) ret = local->ops->get_ftm_responder_stats(&local->hw, diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index b8a580eb1705d..e08fac48b973d 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -1186,6 +1186,15 @@ ieee80211_vif_get_shift(struct ieee80211_vif *vif) return shift; } +#define for_each_link_data(sdata, __link) \ + struct ieee80211_sub_if_data *__sdata = sdata; \ + for (int __link_id = 0; \ + __link_id < ARRAY_SIZE((__sdata)->link); __link_id++) \ + if ((!(__sdata)->vif.valid_links || \ + (__sdata)->vif.valid_links & BIT(__link_id)) && \ + ((__link) = sdata_dereference((__sdata)->link[__link_id], \ + (__sdata)))) + static inline int ieee80211_get_mbssid_beacon_len(struct cfg80211_mbssid_elems *elems, struct cfg80211_rnr_elems *rnr_elems, diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index eaa4e5c6a5c3a..a531fb2b14dee 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -216,6 +216,10 @@ static int ieee80211_can_powered_addr_change(struct ieee80211_sub_if_data *sdata mutex_lock(&local->mtx); + /* if any stations are set known (so they know this vif too), reject */ + if (sta_info_get_by_idx(sdata, 0)) + return -EBUSY; + /* First check no ROC work is happening on this iface */ list_for_each_entry(roc, &local->roc_list, list) { if (roc->sdata != sdata) @@ -235,12 +239,16 @@ static int ieee80211_can_powered_addr_change(struct ieee80211_sub_if_data *sdata ret = -EBUSY; } + /* + * More interface types could be added here but changing the + * address while powered makes the most sense in client modes. + */ switch (sdata->vif.type) { case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: - /* More interface types could be added here but changing the - * address while powered makes the most sense in client modes. - */ + /* refuse while connecting */ + if (sdata->u.mgd.auth_data || sdata->u.mgd.assoc_data) + return -EBUSY; break; default: ret = -EOPNOTSUPP; diff --git a/net/mac80211/link.c b/net/mac80211/link.c index 16cbaea93fc32..af4d2b2e9a26f 100644 --- a/net/mac80211/link.c +++ b/net/mac80211/link.c @@ -28,8 +28,16 @@ void ieee80211_link_init(struct ieee80211_sub_if_data *sdata, if (link_id < 0) link_id = 0; - rcu_assign_pointer(sdata->vif.link_conf[link_id], link_conf); - rcu_assign_pointer(sdata->link[link_id], link); + if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) { + struct ieee80211_sub_if_data *ap_bss; + struct ieee80211_bss_conf *ap_bss_conf; + + ap_bss = container_of(sdata->bss, + struct ieee80211_sub_if_data, u.ap); + ap_bss_conf = sdata_dereference(ap_bss->vif.link_conf[link_id], + ap_bss); + memcpy(link_conf, ap_bss_conf, sizeof(*link_conf)); + } link->sdata = sdata; link->link_id = link_id; @@ -65,6 +73,9 @@ void ieee80211_link_init(struct ieee80211_sub_if_data *sdata, ieee80211_link_debugfs_add(link); } + + rcu_assign_pointer(sdata->vif.link_conf[link_id], link_conf); + rcu_assign_pointer(sdata->link[link_id], link); } void ieee80211_link_stop(struct ieee80211_link_data *link) diff --git a/net/mac80211/main.c b/net/mac80211/main.c index d1046f495e63f..80b143bde93d6 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -965,7 +965,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) int result, i; enum nl80211_band band; int channels, max_bitrates; - bool supp_ht, supp_vht, supp_he, supp_eht; + bool supp_ht, supp_vht, supp_he, supp_eht, supp_s1g; struct cfg80211_chan_def dflt_chandef = {}; if (ieee80211_hw_check(hw, QUEUE_CONTROL) && @@ -1081,6 +1081,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) supp_vht = false; supp_he = false; supp_eht = false; + supp_s1g = false; for (band = 0; band < NUM_NL80211_BANDS; band++) { struct ieee80211_supported_band *sband; @@ -1127,6 +1128,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) max_bitrates = sband->n_bitrates; supp_ht = supp_ht || sband->ht_cap.ht_supported; supp_vht = supp_vht || sband->vht_cap.vht_supported; + supp_s1g = supp_s1g || sband->s1g_cap.s1g; for (i = 0; i < sband->n_iftype_data; i++) { const struct ieee80211_sband_iftype_data *iftd; @@ -1186,10 +1188,12 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) return -EINVAL; } - local->int_scan_req = kzalloc(sizeof(*local->int_scan_req) + - sizeof(void *) * channels, GFP_KERNEL); + local->int_scan_req = kzalloc(struct_size(local->int_scan_req, + channels, channels), + GFP_KERNEL); if (!local->int_scan_req) return -ENOMEM; + local->int_scan_req->n_channels = channels; eth_broadcast_addr(local->int_scan_req->bssid); @@ -1251,6 +1255,9 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) local->scan_ies_len += 2 + sizeof(struct ieee80211_vht_cap); + if (supp_s1g) + local->scan_ies_len += 2 + sizeof(struct ieee80211_s1g_cap); + /* * HE cap element is variable in size - set len to allow max size */ if (supp_he) { diff --git a/net/mac80211/mesh_hwmp.c b/net/mac80211/mesh_hwmp.c index c6395551f5df0..54930b06c3a4a 100644 --- a/net/mac80211/mesh_hwmp.c +++ b/net/mac80211/mesh_hwmp.c @@ -634,7 +634,7 @@ static void hwmp_preq_frame_process(struct ieee80211_sub_if_data *sdata, mesh_path_add_gate(mpath); } rcu_read_unlock(); - } else { + } else if (ifmsh->mshcfg.dot11MeshForwarding) { rcu_read_lock(); mpath = mesh_path_lookup(sdata, target_addr); if (mpath) { @@ -652,6 +652,8 @@ static void hwmp_preq_frame_process(struct ieee80211_sub_if_data *sdata, } } rcu_read_unlock(); + } else { + forward = false; } if (reply) { @@ -669,7 +671,7 @@ static void hwmp_preq_frame_process(struct ieee80211_sub_if_data *sdata, } } - if (forward && ifmsh->mshcfg.dot11MeshForwarding) { + if (forward) { u32 preq_id; u8 hopcount; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index d5c8f8324a377..4e489d5c37853 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -2959,7 +2959,8 @@ static void ieee80211_set_disassoc(struct ieee80211_sub_if_data *sdata, if (tx) ieee80211_flush_queues(local, sdata, false); - drv_mgd_complete_tx(sdata->local, sdata, &info); + if (tx || frame_buf) + drv_mgd_complete_tx(sdata->local, sdata, &info); /* clear AP addr only after building the needed mgmt frames */ eth_zero_addr(sdata->deflink.u.mgd.bssid); @@ -3661,6 +3662,7 @@ static void ieee80211_rx_mgmt_auth(struct ieee80211_sub_if_data *sdata, struct ieee80211_prep_tx_info info = { .subtype = IEEE80211_STYPE_AUTH, }; + bool sae_need_confirm = false; sdata_assert_lock(sdata); @@ -3704,6 +3706,8 @@ static void ieee80211_rx_mgmt_auth(struct ieee80211_sub_if_data *sdata, jiffies + IEEE80211_AUTH_WAIT_SAE_RETRY; ifmgd->auth_data->timeout_started = true; run_again(sdata, ifmgd->auth_data->timeout); + if (auth_transaction == 1) + sae_need_confirm = true; goto notify_driver; } @@ -3746,6 +3750,9 @@ static void ieee80211_rx_mgmt_auth(struct ieee80211_sub_if_data *sdata, ifmgd->auth_data->expected_transaction == 2)) { if (!ieee80211_mark_sta_auth(sdata)) return; /* ignore frame -- wait for timeout */ + } else if (ifmgd->auth_data->algorithm == WLAN_AUTH_SAE && + auth_transaction == 1) { + sae_need_confirm = true; } else if (ifmgd->auth_data->algorithm == WLAN_AUTH_SAE && auth_transaction == 2) { sdata_info(sdata, "SAE peer confirmed\n"); @@ -3754,7 +3761,8 @@ static void ieee80211_rx_mgmt_auth(struct ieee80211_sub_if_data *sdata, cfg80211_rx_mlme_mgmt(sdata->dev, (u8 *)mgmt, len); notify_driver: - drv_mgd_complete_tx(sdata->local, sdata, &info); + if (!sae_need_confirm) + drv_mgd_complete_tx(sdata->local, sdata, &info); } #define case_WLAN(type) \ @@ -4500,7 +4508,7 @@ static u8 ieee80211_max_rx_chains(struct ieee80211_link_data *link, he_cap_elem = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_CAPABILITY, ies->data, ies->len); - if (!he_cap_elem || he_cap_elem->datalen < sizeof(*he_cap)) + if (!he_cap_elem || he_cap_elem->datalen < sizeof(*he_cap) + 1) return chains; /* skip one byte ext_tag_id */ @@ -7822,7 +7830,6 @@ int ieee80211_mgd_deauth(struct ieee80211_sub_if_data *sdata, ieee80211_report_disconnect(sdata, frame_buf, sizeof(frame_buf), true, req->reason_code, false); - drv_mgd_complete_tx(sdata->local, sdata, &info); return 0; } diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 5eb233f619817..e6a0a65d4d43a 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -4221,10 +4221,16 @@ static bool ieee80211_rx_data_set_sta(struct ieee80211_rx_data *rx, rx->link_sta = NULL; } - if (link_id < 0) - rx->link = &rx->sdata->deflink; - else if (!ieee80211_rx_data_set_link(rx, link_id)) + if (link_id < 0) { + if (ieee80211_vif_is_mld(&rx->sdata->vif) && + sta && !sta->sta.valid_links) + rx->link = + rcu_dereference(rx->sdata->link[sta->deflink.link_id]); + else + rx->link = &rx->sdata->deflink; + } else if (!ieee80211_rx_data_set_link(rx, link_id)) { return false; + } return true; } @@ -4419,6 +4425,10 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx) if (!multicast && !ether_addr_equal(sdata->dev->dev_addr, hdr->addr1)) return false; + /* reject invalid/our STA address */ + if (!is_valid_ether_addr(hdr->addr2) || + ether_addr_equal(sdata->dev->dev_addr, hdr->addr2)) + return false; if (!rx->sta) { int rate_idx; if (status->encoding != RX_ENC_LEGACY) @@ -5209,12 +5219,20 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, } rx.sdata = prev_sta->sdata; + if (!status->link_valid && prev_sta->sta.mlo) { + struct link_sta_info *link_sta; + + link_sta = link_sta_info_get_bss(rx.sdata, + hdr->addr2); + if (!link_sta) + continue; + + link_id = link_sta->link_id; + } + if (!ieee80211_rx_data_set_sta(&rx, prev_sta, link_id)) goto out; - if (!status->link_valid && prev_sta->sta.mlo) - continue; - ieee80211_prepare_and_rx_handle(&rx, skb, false); prev_sta = sta; @@ -5222,10 +5240,18 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, if (prev_sta) { rx.sdata = prev_sta->sdata; - if (!ieee80211_rx_data_set_sta(&rx, prev_sta, link_id)) - goto out; + if (!status->link_valid && prev_sta->sta.mlo) { + struct link_sta_info *link_sta; + + link_sta = link_sta_info_get_bss(rx.sdata, + hdr->addr2); + if (!link_sta) + goto out; - if (!status->link_valid && prev_sta->sta.mlo) + link_id = link_sta->link_id; + } + + if (!ieee80211_rx_data_set_sta(&rx, prev_sta, link_id)) goto out; if (ieee80211_prepare_and_rx_handle(&rx, skb, true)) @@ -5315,10 +5341,14 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta, if (WARN_ON(!local->started)) goto drop; - if (likely(!(status->flag & RX_FLAG_FAILED_PLCP_CRC))) { + if (likely(!(status->flag & RX_FLAG_FAILED_PLCP_CRC) && + !(status->flag & RX_FLAG_NO_PSDU && + status->zero_length_psdu_type == + IEEE80211_RADIOTAP_ZERO_LEN_PSDU_NOT_CAPTURED))) { /* - * Validate the rate, unless a PLCP error means that - * we probably can't have a valid rate here anyway. + * Validate the rate, unless there was a PLCP error which may + * have an invalid rate or the PSDU was not capture and may be + * missing rate information. */ switch (status->encoding) { diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c index a4af3b7675efa..f3cdbd2133f67 100644 --- a/net/mac80211/tdls.c +++ b/net/mac80211/tdls.c @@ -1450,7 +1450,7 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, if (!(wiphy->flags & WIPHY_FLAG_SUPPORTS_TDLS)) return -ENOTSUPP; - if (sdata->vif.type != NL80211_IFTYPE_STATION) + if (sdata->vif.type != NL80211_IFTYPE_STATION || !sdata->vif.cfg.assoc) return -EINVAL; switch (oper) { diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 45a093d3f1fa7..7eddcb6f96455 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -629,6 +629,12 @@ ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx) else tx->key = NULL; + if (info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) { + if (tx->key && tx->key->flags & KEY_FLAG_UPLOADED_TO_HARDWARE) + info->control.hw_key = &tx->key->conf; + return TX_CONTINUE; + } + if (tx->key) { bool skip_hw = false; @@ -1451,7 +1457,7 @@ static void ieee80211_txq_enqueue(struct ieee80211_local *local, { struct fq *fq = &local->fq; struct fq_tin *tin = &txqi->tin; - u32 flow_idx = fq_flow_idx(fq, skb); + u32 flow_idx; ieee80211_set_skb_enqueue_time(skb); @@ -1467,6 +1473,7 @@ static void ieee80211_txq_enqueue(struct ieee80211_local *local, IEEE80211_TX_INTCFL_NEED_TXPROCESSING; __skb_queue_tail(&txqi->frags, skb); } else { + flow_idx = fq_flow_idx(fq, skb); fq_tin_enqueue(fq, tin, flow_idx, skb, fq_skb_free_func); } @@ -3877,6 +3884,7 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw, * The key can be removed while the packet was queued, so need to call * this here to get the current key. */ + info->control.hw_key = NULL; r = ieee80211_tx_h_select_key(&tx); if (r != TX_CONTINUE) { ieee80211_free_txskb(&local->hw, skb); @@ -4099,7 +4107,9 @@ void __ieee80211_schedule_txq(struct ieee80211_hw *hw, spin_lock_bh(&local->active_txq_lock[txq->ac]); - has_queue = force || txq_has_queue(txq); + has_queue = force || + (!test_bit(IEEE80211_TXQ_STOP, &txqi->flags) && + txq_has_queue(txq)); if (list_empty(&txqi->schedule_order) && (has_queue || ieee80211_txq_keep_active(txqi))) { /* If airtime accounting is active, always enqueue STAs at the @@ -4507,8 +4517,10 @@ netdev_tx_t ieee80211_subif_start_xmit(struct sk_buff *skb, IEEE80211_TX_CTRL_MLO_LINK_UNSPEC, NULL); } else if (ieee80211_vif_is_mld(&sdata->vif) && - sdata->vif.type == NL80211_IFTYPE_AP && - !ieee80211_hw_check(&sdata->local->hw, MLO_MCAST_MULTI_LINK_TX)) { + ((sdata->vif.type == NL80211_IFTYPE_AP && + !ieee80211_hw_check(&sdata->local->hw, MLO_MCAST_MULTI_LINK_TX)) || + (sdata->vif.type == NL80211_IFTYPE_AP_VLAN && + !sdata->wdev.use_4addr))) { ieee80211_mlo_multicast_tx(dev, skb); } else { normal: diff --git a/net/mac80211/util.c b/net/mac80211/util.c index 154b41af4157d..3a3cd09bdab65 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -4753,7 +4753,7 @@ void ieee80211_recalc_dtim(struct ieee80211_local *local, { u64 tsf = drv_get_tsf(local, sdata); u64 dtim_count = 0; - u16 beacon_int = sdata->vif.bss_conf.beacon_int * 1024; + u32 beacon_int = sdata->vif.bss_conf.beacon_int * 1024; u8 dtim_period = sdata->vif.bss_conf.dtim_period; struct ps_data *ps; u8 bcns_from_dtim; diff --git a/net/mctp/af_mctp.c b/net/mctp/af_mctp.c index 8032cfba22d1c..805f7376cebe3 100644 --- a/net/mctp/af_mctp.c +++ b/net/mctp/af_mctp.c @@ -73,7 +73,6 @@ static int mctp_bind(struct socket *sock, struct sockaddr *addr, int addrlen) lock_sock(sk); - /* TODO: allow rebind */ if (sk_hashed(sk)) { rc = -EADDRINUSE; goto out_release; @@ -347,7 +346,7 @@ static int mctp_getsockopt(struct socket *sock, int level, int optname, return 0; } - return -EINVAL; + return -ENOPROTOOPT; } static int mctp_ioctl_alloctag(struct mctp_sock *msk, unsigned long arg) @@ -549,15 +548,36 @@ static void mctp_sk_close(struct sock *sk, long timeout) static int mctp_sk_hash(struct sock *sk) { struct net *net = sock_net(sk); + struct sock *existing; + struct mctp_sock *msk; + int rc; + + msk = container_of(sk, struct mctp_sock, sk); /* Bind lookup runs under RCU, remain live during that. */ sock_set_flag(sk, SOCK_RCU_FREE); mutex_lock(&net->mctp.bind_lock); + + /* Prevent duplicate binds. */ + sk_for_each(existing, &net->mctp.binds) { + struct mctp_sock *mex = + container_of(existing, struct mctp_sock, sk); + + if (mex->bind_type == msk->bind_type && + mex->bind_addr == msk->bind_addr && + mex->bind_net == msk->bind_net) { + rc = -EADDRINUSE; + goto out; + } + } + sk_add_node_rcu(sk, &net->mctp.binds); - mutex_unlock(&net->mctp.bind_lock); + rc = 0; - return 0; +out: + mutex_unlock(&net->mctp.bind_lock); + return rc; } static void mctp_sk_unhash(struct sock *sk) diff --git a/net/mctp/device.c b/net/mctp/device.c index 85cc5f31f1e7c..8d1386601bbe0 100644 --- a/net/mctp/device.c +++ b/net/mctp/device.c @@ -20,8 +20,7 @@ #include struct mctp_dump_cb { - int h; - int idx; + unsigned long ifindex; size_t a_idx; }; @@ -115,43 +114,36 @@ static int mctp_dump_addrinfo(struct sk_buff *skb, struct netlink_callback *cb) { struct mctp_dump_cb *mcb = (void *)cb->ctx; struct net *net = sock_net(skb->sk); - struct hlist_head *head; struct net_device *dev; struct ifaddrmsg *hdr; struct mctp_dev *mdev; - int ifindex; - int idx = 0, rc; - - hdr = nlmsg_data(cb->nlh); - // filter by ifindex if requested - ifindex = hdr->ifa_index; + int ifindex = 0, rc; + + /* Filter by ifindex if a header is provided */ + if (cb->nlh->nlmsg_len >= nlmsg_msg_size(sizeof(*hdr))) { + hdr = nlmsg_data(cb->nlh); + ifindex = hdr->ifa_index; + } else { + if (cb->strict_check) { + NL_SET_ERR_MSG(cb->extack, "mctp: Invalid header for addr dump request"); + return -EINVAL; + } + } rcu_read_lock(); - for (; mcb->h < NETDEV_HASHENTRIES; mcb->h++, mcb->idx = 0) { - idx = 0; - head = &net->dev_index_head[mcb->h]; - hlist_for_each_entry_rcu(dev, head, index_hlist) { - if (idx >= mcb->idx && - (ifindex == 0 || ifindex == dev->ifindex)) { - mdev = __mctp_dev_get(dev); - if (mdev) { - rc = mctp_dump_dev_addrinfo(mdev, - skb, cb); - mctp_dev_put(mdev); - // Error indicates full buffer, this - // callback will get retried. - if (rc < 0) - goto out; - } - } - idx++; - // reset for next iteration - mcb->a_idx = 0; - } + for_each_netdev_dump(net, dev, mcb->ifindex) { + if (ifindex && ifindex != dev->ifindex) + continue; + mdev = __mctp_dev_get(dev); + if (!mdev) + continue; + rc = mctp_dump_dev_addrinfo(mdev, skb, cb); + mctp_dev_put(mdev); + if (rc < 0) + break; + mcb->a_idx = 0; } -out: rcu_read_unlock(); - mcb->idx = idx; return skb->len; } @@ -525,9 +517,12 @@ static struct notifier_block mctp_dev_nb = { }; static const struct rtnl_msg_handler mctp_device_rtnl_msg_handlers[] = { - {THIS_MODULE, PF_MCTP, RTM_NEWADDR, mctp_rtm_newaddr, NULL, 0}, - {THIS_MODULE, PF_MCTP, RTM_DELADDR, mctp_rtm_deladdr, NULL, 0}, - {THIS_MODULE, PF_MCTP, RTM_GETADDR, NULL, mctp_dump_addrinfo, 0}, + {.owner = THIS_MODULE, .protocol = PF_MCTP, .msgtype = RTM_NEWADDR, + .doit = mctp_rtm_newaddr}, + {.owner = THIS_MODULE, .protocol = PF_MCTP, .msgtype = RTM_DELADDR, + .doit = mctp_rtm_deladdr}, + {.owner = THIS_MODULE, .protocol = PF_MCTP, .msgtype = RTM_GETADDR, + .dumpit = mctp_dump_addrinfo}, }; int __init mctp_device_init(void) diff --git a/net/mctp/route.c b/net/mctp/route.c index d3c1f54386efc..009ba5edbd525 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -274,8 +274,10 @@ static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev) key = flow->key; - if (WARN_ON(key->dev && key->dev != dev)) + if (key->dev) { + WARN_ON(key->dev != dev); return; + } mctp_dev_set_key(dev, key); } diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 1af29af653885..5cf549ede91e1 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -81,8 +81,8 @@ static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned index) if (index < net->mpls.platform_labels) { struct mpls_route __rcu **platform_label = - rcu_dereference(net->mpls.platform_label); - rt = rcu_dereference(platform_label[index]); + rcu_dereference_rtnl(net->mpls.platform_label); + rt = rcu_dereference_rtnl(platform_label[index]); } return rt; } diff --git a/net/mptcp/options.c b/net/mptcp/options.c index 6ef98608ac64c..f1edae5f90790 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -839,8 +839,11 @@ bool mptcp_established_options(struct sock *sk, struct sk_buff *skb, opts->suboptions = 0; + /* Force later mptcp_write_options(), but do not use any actual + * option space. + */ if (unlikely(__mptcp_check_fallback(msk) && !mptcp_check_infinite_map(skb))) - return false; + return true; if (unlikely(skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST)) { if (mptcp_established_options_fastclose(sk, &opt_size, remaining, opts) || @@ -979,18 +982,19 @@ static bool check_fully_established(struct mptcp_sock *msk, struct sock *ssk, if (subflow->mp_join) goto reset; subflow->mp_capable = 0; + if (!mptcp_try_fallback(ssk)) + goto reset; pr_fallback(msk); - mptcp_do_fallback(ssk); return false; } - if (mp_opt->deny_join_id0) - WRITE_ONCE(msk->pm.remote_deny_join_id0, true); - if (unlikely(!READ_ONCE(msk->pm.server_side))) pr_warn_once("bogus mpc option on established client sk"); set_fully_established: + if (mp_opt->deny_join_id0) + WRITE_ONCE(msk->pm.remote_deny_join_id0, true); + mptcp_data_lock((struct sock *)msk); __mptcp_subflow_fully_established(msk, subflow, mp_opt); mptcp_data_unlock((struct sock *)msk); @@ -1040,6 +1044,31 @@ static void __mptcp_snd_una_update(struct mptcp_sock *msk, u64 new_snd_una) msk->snd_una = new_snd_una; } +static void rwin_update(struct mptcp_sock *msk, struct sock *ssk, + struct sk_buff *skb) +{ + struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); + struct tcp_sock *tp = tcp_sk(ssk); + u64 mptcp_rcv_wnd; + + /* Avoid touching extra cachelines if TCP is going to accept this + * skb without filling the TCP-level window even with a possibly + * outdated mptcp-level rwin. + */ + if (!skb->len || skb->len < tcp_receive_window(tp)) + return; + + mptcp_rcv_wnd = atomic64_read(&msk->rcv_wnd_sent); + if (!after64(mptcp_rcv_wnd, subflow->rcv_wnd_sent)) + return; + + /* Some other subflow grew the mptcp-level rwin since rcv_wup, + * resync. + */ + tp->rcv_wnd += mptcp_rcv_wnd - subflow->rcv_wnd_sent; + subflow->rcv_wnd_sent = mptcp_rcv_wnd; +} + static void ack_update_msk(struct mptcp_sock *msk, struct sock *ssk, struct mptcp_options_received *mp_opt) @@ -1116,7 +1145,9 @@ static bool add_addr_hmac_valid(struct mptcp_sock *msk, return hmac == mp_opt->ahmac; } -/* Return false if a subflow has been reset, else return true */ +/* Return false in case of error (or subflow has been reset), + * else return true. + */ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); @@ -1204,6 +1235,7 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb) */ if (mp_opt.use_ack) ack_update_msk(msk, sk, &mp_opt); + rwin_update(msk, sk, skb); /* Zero-data-length packets are dropped by the caller and not * propagated to the MPTCP layer, so the skb extension does not @@ -1220,7 +1252,7 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb) mpext = skb_ext_add(skb, SKB_EXT_MPTCP); if (!mpext) - return true; + return false; memset(mpext, 0, sizeof(*mpext)); @@ -1290,6 +1322,10 @@ static void mptcp_set_rwin(struct tcp_sock *tp, struct tcphdr *th) if (rcv_wnd_new != rcv_wnd_old) { raise_win: + /* The msk-level rcv wnd is after the tcp level one, + * sync the latter. + */ + rcv_wnd_new = rcv_wnd_old; win = rcv_wnd_old - ack_seq; tp->rcv_wnd = min_t(u64, win, U32_MAX); new_win = tp->rcv_wnd; @@ -1313,6 +1349,21 @@ static void mptcp_set_rwin(struct tcp_sock *tp, struct tcphdr *th) update_wspace: WRITE_ONCE(msk->old_wspace, tp->rcv_wnd); + subflow->rcv_wnd_sent = rcv_wnd_new; +} + +static void mptcp_track_rwin(struct tcp_sock *tp) +{ + const struct sock *ssk = (const struct sock *)tp; + struct mptcp_subflow_context *subflow; + struct mptcp_sock *msk; + + if (!ssk) + return; + + subflow = mptcp_subflow_ctx(ssk); + msk = mptcp_sk(subflow->conn); + WRITE_ONCE(msk->old_wspace, tp->rcv_wnd); } __sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum) @@ -1607,6 +1658,10 @@ void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp, opts->reset_transient, opts->reset_reason); return; + } else if (unlikely(!opts->suboptions)) { + /* Fallback to TCP */ + mptcp_track_rwin(tp); + return; } if (OPTION_MPTCP_PRIO & opts->suboptions) { diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 157a574fab0cc..ab7bdb6531816 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -227,9 +227,12 @@ void mptcp_pm_add_addr_received(const struct sock *ssk, } else { __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP); } - /* id0 should not have a different address */ + /* - id0 should not have a different address + * - special case for C-flag: linked to fill_local_addresses_vec() + */ } else if ((addr->id == 0 && !mptcp_pm_nl_is_init_remote_addr(msk, addr)) || - (addr->id > 0 && !READ_ONCE(pm->accept_addr))) { + (addr->id > 0 && !READ_ONCE(pm->accept_addr) && + !mptcp_pm_add_addr_c_flag_case(msk))) { mptcp_pm_announce_addr(msk, addr, true); mptcp_pm_add_addr_send_ack(msk); } else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) { @@ -304,8 +307,14 @@ void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq) pr_debug("fail_seq=%llu\n", fail_seq); - if (!READ_ONCE(msk->allow_infinite_fallback)) + /* After accepting the fail, we can't create any other subflows */ + spin_lock_bh(&msk->fallback_lock); + if (!msk->allow_infinite_fallback) { + spin_unlock_bh(&msk->fallback_lock); return; + } + msk->allow_subflows = false; + spin_unlock_bh(&msk->fallback_lock); if (!subflow->fail_tout) { pr_debug("send MP_FAIL response and infinite map\n"); diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c index f7257de37bd09..8dbe826555d29 100644 --- a/net/mptcp/pm_netlink.c +++ b/net/mptcp/pm_netlink.c @@ -29,6 +29,7 @@ struct mptcp_pm_add_entry { u8 retrans_times; struct timer_list add_timer; struct mptcp_sock *sock; + struct rcu_head rcu; }; struct pm_nl_pernet { @@ -294,6 +295,7 @@ static void mptcp_pm_add_timer(struct timer_list *timer) struct mptcp_pm_add_entry *entry = from_timer(entry, timer, add_timer); struct mptcp_sock *msk = entry->sock; struct sock *sk = (struct sock *)msk; + unsigned int timeout; pr_debug("msk=%p\n", msk); @@ -311,6 +313,10 @@ static void mptcp_pm_add_timer(struct timer_list *timer) goto out; } + timeout = mptcp_get_add_addr_timeout(sock_net(sk)); + if (!timeout) + goto out; + spin_lock_bh(&msk->pm.lock); if (!mptcp_pm_should_add_signal_addr(msk)) { @@ -322,7 +328,7 @@ static void mptcp_pm_add_timer(struct timer_list *timer) if (entry->retrans_times < ADD_ADDR_RETRANS_MAX) sk_reset_timer(sk, timer, - jiffies + mptcp_get_add_addr_timeout(sock_net(sk))); + jiffies + timeout); spin_unlock_bh(&msk->pm.lock); @@ -339,22 +345,27 @@ mptcp_pm_del_add_timer(struct mptcp_sock *msk, { struct mptcp_pm_add_entry *entry; struct sock *sk = (struct sock *)msk; - struct timer_list *add_timer = NULL; + bool stop_timer = false; + + rcu_read_lock(); spin_lock_bh(&msk->pm.lock); entry = mptcp_lookup_anno_list_by_saddr(msk, addr); if (entry && (!check_id || entry->addr.id == addr->id)) { entry->retrans_times = ADD_ADDR_RETRANS_MAX; - add_timer = &entry->add_timer; + stop_timer = true; } if (!check_id && entry) list_del(&entry->list); spin_unlock_bh(&msk->pm.lock); - /* no lock, because sk_stop_timer_sync() is calling del_timer_sync() */ - if (add_timer) - sk_stop_timer_sync(sk, add_timer); + /* Note: entry might have been removed by another thread. + * We hold rcu_read_lock() to ensure it is not freed under us. + */ + if (stop_timer) + sk_stop_timer_sync(sk, &entry->add_timer); + rcu_read_unlock(); return entry; } @@ -364,6 +375,7 @@ bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, struct mptcp_pm_add_entry *add_entry = NULL; struct sock *sk = (struct sock *)msk; struct net *net = sock_net(sk); + unsigned int timeout; lockdep_assert_held(&msk->pm.lock); @@ -373,9 +385,7 @@ bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, if (WARN_ON_ONCE(mptcp_pm_is_kernel(msk))) return false; - sk_reset_timer(sk, &add_entry->add_timer, - jiffies + mptcp_get_add_addr_timeout(net)); - return true; + goto reset_timer; } add_entry = kmalloc(sizeof(*add_entry), GFP_ATOMIC); @@ -389,8 +399,10 @@ bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, add_entry->retrans_times = 0; timer_setup(&add_entry->add_timer, mptcp_pm_add_timer, 0); - sk_reset_timer(sk, &add_entry->add_timer, - jiffies + mptcp_get_add_addr_timeout(net)); +reset_timer: + timeout = mptcp_get_add_addr_timeout(net); + if (timeout) + sk_reset_timer(sk, &add_entry->add_timer, jiffies + timeout); return true; } @@ -409,7 +421,7 @@ void mptcp_pm_free_anno_list(struct mptcp_sock *msk) list_for_each_entry_safe(entry, tmp, &free_list, list) { sk_stop_timer_sync(sk, &entry->add_timer); - kfree(entry); + kfree_rcu(entry, rcu); } } @@ -613,6 +625,10 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) } subflow: + /* No need to try establishing subflows to remote id0 if not allowed */ + if (mptcp_pm_add_addr_c_flag_case(msk)) + goto exit; + /* check if should create a new subflow */ while (msk->pm.local_addr_used < local_addr_max && msk->pm.subflows < subflows_max) { @@ -644,6 +660,8 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) __mptcp_subflow_connect(sk, &local.addr, &addrs[i]); spin_lock_bh(&msk->pm.lock); } + +exit: mptcp_pm_nl_check_work_pending(msk); } @@ -669,10 +687,12 @@ static unsigned int fill_local_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info mpc_addr; struct pm_nl_pernet *pernet; unsigned int subflows_max; + bool c_flag_case; int i = 0; pernet = pm_nl_get_pernet_from_msk(msk); subflows_max = mptcp_pm_get_subflows_max(msk); + c_flag_case = remote->id && mptcp_pm_add_addr_c_flag_case(msk); mptcp_local_address((struct sock_common *)msk, &mpc_addr); @@ -685,11 +705,26 @@ static unsigned int fill_local_addresses_vec(struct mptcp_sock *msk, continue; if (msk->pm.subflows < subflows_max) { + bool is_id0; + msk->pm.subflows++; addrs[i] = entry->addr; + is_id0 = mptcp_addresses_equal(&entry->addr, + &mpc_addr, + entry->addr.port); + + if (c_flag_case && + (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)) { + __clear_bit(addrs[i].id, + msk->pm.id_avail_bitmap); + + if (!is_id0) + msk->pm.local_addr_used++; + } + /* Special case for ID0: set the correct ID */ - if (mptcp_addresses_equal(&entry->addr, &mpc_addr, entry->addr.port)) + if (is_id0) addrs[i].id = 0; i++; @@ -697,6 +732,39 @@ static unsigned int fill_local_addresses_vec(struct mptcp_sock *msk, } rcu_read_unlock(); + /* Special case: peer sets the C flag, accept one ADD_ADDR if default + * limits are used -- accepting no ADD_ADDR -- and use subflow endpoints + */ + if (!i && c_flag_case) { + unsigned int local_addr_max = mptcp_pm_get_local_addr_max(msk); + + while (msk->pm.local_addr_used < local_addr_max && + msk->pm.subflows < subflows_max) { + struct mptcp_pm_addr_entry local; + + if (!select_local_address(pernet, msk, &local)) + break; + + __clear_bit(local.addr.id, msk->pm.id_avail_bitmap); + + if (!mptcp_pm_addr_families_match(sk, &local.addr, + remote)) + continue; + + if (mptcp_addresses_equal(&local.addr, &mpc_addr, + local.addr.port)) + continue; + + addrs[i] = local.addr; + + msk->pm.local_addr_used++; + msk->pm.subflows++; + i++; + } + + return i; + } + /* If the array is empty, fill in the single * 'IPADDRANY' local address */ @@ -1511,7 +1579,7 @@ static bool remove_anno_list_by_saddr(struct mptcp_sock *msk, entry = mptcp_pm_del_add_timer(msk, addr, false); if (entry) { - kfree(entry); + kfree_rcu(entry, rcu); return true; } @@ -1783,7 +1851,6 @@ static void __flush_addrs(struct list_head *list) static void __reset_counters(struct pm_nl_pernet *pernet) { WRITE_ONCE(pernet->add_addr_signal_max, 0); - WRITE_ONCE(pernet->add_addr_accept_max, 0); WRITE_ONCE(pernet->local_addr_max, 0); pernet->addrs = 0; } @@ -2247,6 +2314,7 @@ static int mptcp_event_created(struct sk_buff *skb, const struct sock *ssk) { int err = nla_put_u32(skb, MPTCP_ATTR_TOKEN, msk->token); + u16 flags = 0; if (err) return err; @@ -2254,6 +2322,12 @@ static int mptcp_event_created(struct sk_buff *skb, if (nla_put_u8(skb, MPTCP_ATTR_SERVER_SIDE, READ_ONCE(msk->pm.server_side))) return -EMSGSIZE; + if (READ_ONCE(msk->pm.remote_deny_join_id0)) + flags |= MPTCP_PM_EV_FLAG_DENY_JOIN_ID0; + + if (flags && nla_put_u16(skb, MPTCP_ATTR_FLAGS, flags)) + return -EMSGSIZE; + return mptcp_event_add_subflow(skb, ssk); } diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 140c3ffcb86ba..73b3a44b183fc 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -57,11 +57,13 @@ static u64 mptcp_wnd_end(const struct mptcp_sock *msk) static const struct proto_ops *mptcp_fallback_tcp_ops(const struct sock *sk) { + unsigned short family = READ_ONCE(sk->sk_family); + #if IS_ENABLED(CONFIG_MPTCP_IPV6) - if (sk->sk_prot == &tcpv6_prot) + if (family == AF_INET6) return &inet6_stream_ops; #endif - WARN_ON_ONCE(sk->sk_prot != &tcp_prot); + WARN_ON_ONCE(family != AF_INET); return &inet_stream_ops; } @@ -415,6 +417,20 @@ static void mptcp_close_wake_up(struct sock *sk) sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); } +static void mptcp_shutdown_subflows(struct mptcp_sock *msk) +{ + struct mptcp_subflow_context *subflow; + + mptcp_for_each_subflow(msk, subflow) { + struct sock *ssk = mptcp_subflow_tcp_sock(subflow); + bool slow; + + slow = lock_sock_fast(ssk); + tcp_shutdown(ssk, SEND_SHUTDOWN); + unlock_sock_fast(ssk, slow); + } +} + static bool mptcp_pending_data_fin_ack(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); @@ -438,6 +454,7 @@ static void mptcp_check_data_fin_ack(struct sock *sk) break; case TCP_CLOSING: case TCP_LAST_ACK: + mptcp_shutdown_subflows(msk); mptcp_set_state(sk, TCP_CLOSE); break; } @@ -605,6 +622,7 @@ static bool mptcp_check_data_fin(struct sock *sk) mptcp_set_state(sk, TCP_CLOSING); break; case TCP_FIN_WAIT2: + mptcp_shutdown_subflows(msk); mptcp_set_state(sk, TCP_CLOSE); break; default: @@ -623,10 +641,9 @@ static bool mptcp_check_data_fin(struct sock *sk) static void mptcp_dss_corruption(struct mptcp_sock *msk, struct sock *ssk) { - if (READ_ONCE(msk->allow_infinite_fallback)) { + if (mptcp_try_fallback(ssk)) { MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSCORRUPTIONFALLBACK); - mptcp_do_fallback(ssk); } else { MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSCORRUPTIONRESET); mptcp_subflow_reset(ssk); @@ -876,7 +893,7 @@ void mptcp_data_ready(struct sock *sk, struct sock *ssk) static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk) { mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq); - WRITE_ONCE(msk->allow_infinite_fallback, false); + msk->allow_infinite_fallback = false; mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); } @@ -887,6 +904,21 @@ static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) if (sk->sk_state != TCP_ESTABLISHED) return false; + /* The caller possibly is not holding the msk socket lock, but + * in the fallback case only the current subflow is touching + * the OoO queue. + */ + if (!RB_EMPTY_ROOT(&msk->out_of_order_queue)) + return false; + + spin_lock_bh(&msk->fallback_lock); + if (!msk->allow_subflows) { + spin_unlock_bh(&msk->fallback_lock); + return false; + } + mptcp_subflow_joined(msk, ssk); + spin_unlock_bh(&msk->fallback_lock); + /* attach to msk socket only after we are sure we will deal with it * at close time */ @@ -895,7 +927,6 @@ static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++; mptcp_sockopt_sync_locked(msk, ssk); - mptcp_subflow_joined(msk, ssk); mptcp_stop_tout_timer(sk); __mptcp_propagate_sndbuf(sk, ssk); return true; @@ -937,14 +968,19 @@ static void mptcp_reset_rtx_timer(struct sock *sk) bool mptcp_schedule_work(struct sock *sk) { - if (inet_sk_state_load(sk) != TCP_CLOSE && - schedule_work(&mptcp_sk(sk)->work)) { - /* each subflow already holds a reference to the sk, and the - * workqueue is invoked by a subflow, so sk can't go away here. - */ - sock_hold(sk); + if (inet_sk_state_load(sk) == TCP_CLOSE) + return false; + + /* Get a reference on this socket, mptcp_worker() will release it. + * As mptcp_worker() might complete before us, we can not avoid + * a sock_hold()/sock_put() if schedule_work() returns false. + */ + sock_hold(sk); + + if (schedule_work(&mptcp_sk(sk)->work)) return true; - } + + sock_put(sk); return false; } @@ -1022,7 +1058,7 @@ static void __mptcp_clean_una(struct sock *sk) if (WARN_ON_ONCE(!msk->recovery)) break; - WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); + msk->first_pending = mptcp_send_next(sk); } dfrag_clear(sk, dfrag); @@ -1231,10 +1267,14 @@ static void mptcp_update_infinite_map(struct mptcp_sock *msk, mpext->infinite_map = 1; mpext->data_len = 0; + if (!mptcp_try_fallback(ssk)) { + mptcp_subflow_reset(ssk); + return; + } + MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPTX); mptcp_subflow_ctx(ssk)->send_infinite_map = 0; pr_fallback(msk); - mptcp_do_fallback(ssk); } #define MPTCP_MAX_GSO_SIZE (GSO_LEGACY_MAX_SIZE - (MAX_TCP_HEADER + 1)) @@ -1309,7 +1349,12 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, if (copy == 0) { u64 snd_una = READ_ONCE(msk->snd_una); - if (snd_una != msk->snd_nxt || tcp_write_queue_tail(ssk)) { + /* No need for zero probe if there are any data pending + * either at the msk or ssk level; skb is the current write + * queue tail and can be empty at this point. + */ + if (snd_una != msk->snd_nxt || skb->len || + skb != tcp_send_head(ssk)) { tcp_remove_empty_skb(ssk); return 0; } @@ -1562,7 +1607,7 @@ static int __subflow_push_pending(struct sock *sk, struct sock *ssk, mptcp_update_post_push(msk, dfrag, ret); } - WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); + msk->first_pending = mptcp_send_next(sk); if (msk->snd_burst <= 0 || !sk_stream_memory_free(ssk) || @@ -1874,7 +1919,7 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) get_page(dfrag->page); list_add_tail(&dfrag->list, &msk->rtx_queue); if (!msk->first_pending) - WRITE_ONCE(msk->first_pending, dfrag); + msk->first_pending = dfrag; } pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d\n", msk, dfrag->data_seq, dfrag->data_len, dfrag->already_sent, @@ -1909,19 +1954,35 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied); static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, struct msghdr *msg, - size_t len, int flags, + size_t len, int flags, int copied_total, struct scm_timestamping_internal *tss, int *cmsg_flags) { struct sk_buff *skb, *tmp; + int total_data_len = 0; int copied = 0; skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { - u32 offset = MPTCP_SKB_CB(skb)->offset; + u32 delta, offset = MPTCP_SKB_CB(skb)->offset; u32 data_len = skb->len - offset; - u32 count = min_t(size_t, len - copied, data_len); + u32 count; int err; + if (flags & MSG_PEEK) { + /* skip already peeked skbs */ + if (total_data_len + data_len <= copied_total) { + total_data_len += data_len; + continue; + } + + /* skip the already peeked data in the current skb */ + delta = copied_total - total_data_len; + offset += delta; + data_len -= delta; + } + + count = min_t(size_t, len - copied, data_len); + if (!(flags & MSG_TRUNC)) { err = skb_copy_datagram_msg(skb, offset, msg, count); if (unlikely(err < 0)) { @@ -1938,22 +1999,19 @@ static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, copied += count; - if (count < data_len) { - if (!(flags & MSG_PEEK)) { + if (!(flags & MSG_PEEK)) { + msk->bytes_consumed += count; + if (count < data_len) { MPTCP_SKB_CB(skb)->offset += count; MPTCP_SKB_CB(skb)->map_seq += count; - msk->bytes_consumed += count; + break; } - break; - } - if (!(flags & MSG_PEEK)) { /* we will bulk release the skb memory later */ skb->destructor = NULL; WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); __skb_unlink(skb, &msk->receive_queue); __kfree_skb(skb); - msk->bytes_consumed += count; } if (copied >= len) @@ -2177,7 +2235,8 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, while (copied < len) { int err, bytes_read; - bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); + bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, + copied, &tss, &cmsg_flags); if (unlikely(bytes_read < 0)) { if (!copied) copied = bytes_read; @@ -2533,7 +2592,8 @@ static void __mptcp_close_subflow(struct sock *sk) if (ssk_state != TCP_CLOSE && (ssk_state != TCP_CLOSE_WAIT || - inet_sk_state_load(sk) != TCP_ESTABLISHED)) + inet_sk_state_load(sk) != TCP_ESTABLISHED || + __mptcp_check_fallback(msk))) continue; /* 'subflow_data_ready' will re-sched once rx queue is empty */ @@ -2606,9 +2666,9 @@ static void mptcp_check_fastclose(struct mptcp_sock *msk) static void __mptcp_retrans(struct sock *sk) { + struct mptcp_sendmsg_info info = { .data_lock_held = true, }; struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_subflow_context *subflow; - struct mptcp_sendmsg_info info = {}; struct mptcp_data_frag *dfrag; struct sock *ssk; int ret, err; @@ -2653,6 +2713,18 @@ static void __mptcp_retrans(struct sock *sk) info.sent = 0; info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent; + + /* + * make the whole retrans decision, xmit, disallow + * fallback atomic + */ + spin_lock_bh(&msk->fallback_lock); + if (__mptcp_check_fallback(msk)) { + spin_unlock_bh(&msk->fallback_lock); + release_sock(ssk); + return; + } + while (info.sent < info.limit) { ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); if (ret <= 0) @@ -2666,8 +2738,9 @@ static void __mptcp_retrans(struct sock *sk) len = max(copied, len); tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, info.size_goal); - WRITE_ONCE(msk->allow_infinite_fallback, false); + msk->allow_infinite_fallback = false; } + spin_unlock_bh(&msk->fallback_lock); release_sock(ssk); } @@ -2756,7 +2829,11 @@ static void mptcp_worker(struct work_struct *work) __mptcp_close_subflow(sk); if (mptcp_close_tout_expired(sk)) { + struct mptcp_subflow_context *subflow, *tmp; + mptcp_do_fastclose(sk); + mptcp_for_each_subflow_safe(msk, subflow, tmp) + __mptcp_close_ssk(sk, subflow->tcp_sock, subflow, 0); mptcp_close_wake_up(sk); } @@ -2796,11 +2873,13 @@ static void __mptcp_init_sock(struct sock *sk) WRITE_ONCE(msk->first, NULL); inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); - WRITE_ONCE(msk->allow_infinite_fallback, true); + msk->allow_infinite_fallback = true; + msk->allow_subflows = true; msk->recovery = false; msk->subflow_id = 1; mptcp_pm_data_init(msk); + spin_lock_init(&msk->fallback_lock); /* re-use the csk retrans timer for MPTCP-level retrans */ timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); @@ -2858,7 +2937,7 @@ static void __mptcp_clear_xmit(struct sock *sk) struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_data_frag *dtmp, *dfrag; - WRITE_ONCE(msk->first_pending, NULL); + msk->first_pending = NULL; list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) dfrag_clear(sk, dfrag); } @@ -3182,8 +3261,18 @@ static int mptcp_disconnect(struct sock *sk, int flags) /* msk->subflow is still intact, the following will not free the first * subflow */ - mptcp_destroy_common(msk, MPTCP_CF_FASTCLOSE); + mptcp_do_fastclose(sk); + mptcp_destroy_common(msk); + + /* The first subflow is already in TCP_CLOSE status, the following + * can't overlap with a fallback anymore + */ + spin_lock_bh(&msk->fallback_lock); + msk->allow_subflows = true; + msk->allow_infinite_fallback = true; WRITE_ONCE(msk->flags, 0); + spin_unlock_bh(&msk->fallback_lock); + msk->cb_flags = 0; msk->recovery = false; msk->can_ack = false; @@ -3355,7 +3444,7 @@ void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; } -void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) +void mptcp_destroy_common(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow, *tmp; struct sock *sk = (struct sock *)msk; @@ -3364,7 +3453,7 @@ void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) /* join list will be eventually flushed (with rst) at sock lock release time */ mptcp_for_each_subflow_safe(msk, subflow, tmp) - __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags); + __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, 0); /* move to sk_receive_queue, sk_stream_kill_queues will purge it */ mptcp_data_lock(sk); @@ -3389,7 +3478,7 @@ static void mptcp_destroy(struct sock *sk) /* allow the following to close even the initial subflow */ msk->free_first = 1; - mptcp_destroy_common(msk, 0); + mptcp_destroy_common(msk); sk_sockets_allocated_dec(sk); } @@ -3406,9 +3495,6 @@ void __mptcp_data_acked(struct sock *sk) void __mptcp_check_push(struct sock *sk, struct sock *ssk) { - if (!mptcp_send_head(sk)) - return; - if (!sock_owned_by_user(sk)) __mptcp_subflow_push_pending(sk, ssk, false); else @@ -3599,7 +3685,13 @@ bool mptcp_finish_join(struct sock *ssk) /* active subflow, already present inside the conn_list */ if (!list_empty(&subflow->node)) { + spin_lock_bh(&msk->fallback_lock); + if (!msk->allow_subflows) { + spin_unlock_bh(&msk->fallback_lock); + return false; + } mptcp_subflow_joined(msk, ssk); + spin_unlock_bh(&msk->fallback_lock); mptcp_propagate_sndbuf(parent, ssk); return true; } @@ -3712,7 +3804,7 @@ static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, struct mptcp_subflow_context *subflow) { subflow->request_mptcp = 0; - __mptcp_do_fallback(msk); + WARN_ON_ONCE(!__mptcp_try_fallback(msk)); } static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index d67add91c9b90..5d218c27c0776 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -330,10 +330,16 @@ struct mptcp_sock { u64 rtt_us; /* last maximum rtt of subflows */ } rcvq_space; u8 scaling_ratio; + bool allow_subflows; u32 subflow_id; u32 setsockopt_seq; char ca_name[TCP_CA_NAME_MAX]; + + spinlock_t fallback_lock; /* protects fallback, + * allow_infinite_fallback and + * allow_join + */ }; #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock) @@ -373,7 +379,7 @@ static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk) { const struct mptcp_sock *msk = mptcp_sk(sk); - return READ_ONCE(msk->first_pending); + return msk->first_pending; } static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk) @@ -472,6 +478,7 @@ struct mptcp_subflow_context { u64 remote_key; u64 idsn; u64 map_seq; + u64 rcv_wnd_sent; u32 snd_isn; u32 token; u32 rel_write_seq; @@ -864,7 +871,7 @@ static inline void mptcp_propagate_sndbuf(struct sock *sk, struct sock *ssk) local_bh_enable(); } -void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags); +void mptcp_destroy_common(struct mptcp_sock *msk); #define MPTCP_TOKEN_MAX_RETRIES 4 @@ -1074,6 +1081,14 @@ static inline void mptcp_pm_close_subflow(struct mptcp_sock *msk) spin_unlock_bh(&msk->pm.lock); } +static inline bool mptcp_pm_add_addr_c_flag_case(struct mptcp_sock *msk) +{ + return READ_ONCE(msk->pm.remote_deny_join_id0) && + msk->pm.local_addr_used == 0 && + mptcp_pm_get_add_addr_accept_max(msk) == 0 && + msk->pm.subflows < mptcp_pm_get_subflows_max(msk); +} + void mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk); void mptcp_sockopt_sync_locked(struct mptcp_sock *msk, struct sock *ssk); @@ -1097,25 +1112,33 @@ static inline bool mptcp_check_fallback(const struct sock *sk) return __mptcp_check_fallback(msk); } -static inline void __mptcp_do_fallback(struct mptcp_sock *msk) +static inline bool __mptcp_try_fallback(struct mptcp_sock *msk) { if (test_bit(MPTCP_FALLBACK_DONE, &msk->flags)) { pr_debug("TCP fallback already done (msk=%p)\n", msk); - return; + return true; } - if (WARN_ON_ONCE(!READ_ONCE(msk->allow_infinite_fallback))) - return; + spin_lock_bh(&msk->fallback_lock); + if (!msk->allow_infinite_fallback) { + spin_unlock_bh(&msk->fallback_lock); + return false; + } + + msk->allow_subflows = false; set_bit(MPTCP_FALLBACK_DONE, &msk->flags); + spin_unlock_bh(&msk->fallback_lock); + return true; } -static inline void mptcp_do_fallback(struct sock *ssk) +static inline bool mptcp_try_fallback(struct sock *ssk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); struct sock *sk = subflow->conn; struct mptcp_sock *msk; msk = mptcp_sk(sk); - __mptcp_do_fallback(msk); + if (!__mptcp_try_fallback(msk)) + return false; if (READ_ONCE(msk->snd_data_fin_enable) && !(ssk->sk_shutdown & SEND_SHUTDOWN)) { gfp_t saved_allocation = ssk->sk_allocation; @@ -1127,6 +1150,7 @@ static inline void mptcp_do_fallback(struct sock *ssk) tcp_shutdown(ssk, SEND_SHUTDOWN); ssk->sk_allocation = saved_allocation; } + return true; } #define pr_fallback(a) pr_debug("%s:fallback to TCP (msk=%p)\n", __func__, a) diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index 31f6899ef71aa..b31ba9f905e30 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -1471,13 +1471,12 @@ static void sync_socket_options(struct mptcp_sock *msk, struct sock *ssk) { static const unsigned int tx_rx_locks = SOCK_RCVBUF_LOCK | SOCK_SNDBUF_LOCK; struct sock *sk = (struct sock *)msk; + bool keep_open; - if (ssk->sk_prot->keepalive) { - if (sock_flag(sk, SOCK_KEEPOPEN)) - ssk->sk_prot->keepalive(ssk, 1); - else - ssk->sk_prot->keepalive(ssk, 0); - } + keep_open = sock_flag(sk, SOCK_KEEPOPEN); + if (ssk->sk_prot->keepalive) + ssk->sk_prot->keepalive(ssk, keep_open); + sock_valbool_flag(ssk, SOCK_KEEPOPEN, keep_open); ssk->sk_priority = sk->sk_priority; ssk->sk_bound_dev_if = sk->sk_bound_dev_if; diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index f4b8ca8be81e8..b71675f46d107 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -524,9 +524,11 @@ static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) mptcp_get_options(skb, &mp_opt); if (subflow->request_mptcp) { if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) { + if (!mptcp_try_fallback(sk)) + goto do_reset; + MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEFALLBACK); - mptcp_do_fallback(sk); pr_fallback(msk); goto fallback; } @@ -861,6 +863,10 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, ctx->subflow_id = 1; owner = mptcp_sk(ctx->conn); + + if (mp_opt.deny_join_id0) + WRITE_ONCE(owner->pm.remote_deny_join_id0, true); + mptcp_pm_new_connection(owner, child, 1); /* with OoO packets we can reach here without ingress @@ -1255,20 +1261,29 @@ static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ss mptcp_schedule_work(sk); } -static void mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk) +static bool mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); unsigned long fail_tout; + /* we are really failing, prevent any later subflow join */ + spin_lock_bh(&msk->fallback_lock); + if (!msk->allow_infinite_fallback) { + spin_unlock_bh(&msk->fallback_lock); + return false; + } + msk->allow_subflows = false; + spin_unlock_bh(&msk->fallback_lock); + /* greceful failure can happen only on the MPC subflow */ if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first))) - return; + return false; /* since the close timeout take precedence on the fail one, * no need to start the latter when the first is already set */ if (sock_flag((struct sock *)msk, SOCK_DEAD)) - return; + return true; /* we don't need extreme accuracy here, use a zero fail_tout as special * value meaning no fail timeout at all; @@ -1280,6 +1295,7 @@ static void mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk) tcp_send_ack(ssk); mptcp_reset_tout_timer(msk, subflow->fail_tout); + return true; } static bool subflow_check_data_avail(struct sock *ssk) @@ -1340,17 +1356,16 @@ static bool subflow_check_data_avail(struct sock *ssk) (subflow->mp_join || subflow->valid_csum_seen)) { subflow->send_mp_fail = 1; - if (!READ_ONCE(msk->allow_infinite_fallback)) { + if (!mptcp_subflow_fail(msk, ssk)) { subflow->reset_transient = 0; subflow->reset_reason = MPTCP_RST_EMIDDLEBOX; goto reset; } - mptcp_subflow_fail(msk, ssk); WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL); return true; } - if (!READ_ONCE(msk->allow_infinite_fallback)) { + if (!mptcp_try_fallback(ssk)) { /* fatal protocol error, close the socket. * subflow_error_report() will introduce the appropriate barriers */ @@ -1366,8 +1381,6 @@ static bool subflow_check_data_avail(struct sock *ssk) WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA); return false; } - - mptcp_do_fallback(ssk); } skb = skb_peek(&ssk->sk_receive_queue); @@ -1612,7 +1625,6 @@ int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, /* discard the subflow socket */ mptcp_sock_graft(ssk, sk->sk_socket); iput(SOCK_INODE(sf)); - WRITE_ONCE(msk->allow_infinite_fallback, false); mptcp_stop_tout_timer(sk); return 0; @@ -1707,10 +1719,7 @@ int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, * needs it. * Update ns_tracker to current stack trace and refcounted tracker. */ - __netns_tracker_free(net, &sf->sk->ns_tracker, false); - sf->sk->sk_net_refcnt = 1; - get_net_track(net, &sf->sk->ns_tracker, GFP_KERNEL); - sock_inuse_add(net, 1); + sk_net_refcnt_upgrade(sf->sk); err = tcp_set_ulp(sf->sk, "mptcp"); release_ssk: @@ -1790,7 +1799,7 @@ static void subflow_state_change(struct sock *sk) msk = mptcp_sk(parent); if (subflow_simultaneous_connect(sk)) { - mptcp_do_fallback(sk); + WARN_ON_ONCE(!mptcp_try_fallback(sk)); pr_fallback(msk); subflow->conn_finished = 1; mptcp_propagate_state(parent, sk, subflow, NULL); @@ -2086,6 +2095,10 @@ void __init mptcp_subflow_init(void) tcp_prot_override = tcp_prot; tcp_prot_override.release_cb = tcp_release_cb_override; tcp_prot_override.diag_destroy = tcp_abort_override; +#ifdef CONFIG_BPF_SYSCALL + /* Disable sockmap processing for subflows */ + tcp_prot_override.psock_update_sk_prot = NULL; +#endif #if IS_ENABLED(CONFIG_MPTCP_IPV6) /* In struct mptcp_subflow_request_sock, we assume the TCP request sock @@ -2123,6 +2136,10 @@ void __init mptcp_subflow_init(void) tcpv6_prot_override = tcpv6_prot; tcpv6_prot_override.release_cb = tcp_release_cb_override; tcpv6_prot_override.diag_destroy = tcp_abort_override; +#ifdef CONFIG_BPF_SYSCALL + /* Disable sockmap processing for subflows */ + tcpv6_prot_override.psock_update_sk_prot = NULL; +#endif #endif mptcp_diag_subflow_init(&subflow_ulp_ops); diff --git a/net/ncsi/internal.h b/net/ncsi/internal.h index 4e0842df5234e..ad1f671ffc37f 100644 --- a/net/ncsi/internal.h +++ b/net/ncsi/internal.h @@ -110,7 +110,7 @@ struct ncsi_channel_version { u8 update; /* NCSI version update */ char alpha1; /* NCSI version alpha1 */ char alpha2; /* NCSI version alpha2 */ - u8 fw_name[12]; /* Firmware name string */ + u8 fw_name[12 + 1]; /* Firmware name string */ u32 fw_version; /* Firmware version */ u16 pci_ids[4]; /* PCI identification */ u32 mf_id; /* Manufacture ID */ @@ -143,16 +143,15 @@ struct ncsi_channel_vlan_filter { }; struct ncsi_channel_stats { - u32 hnc_cnt_hi; /* Counter cleared */ - u32 hnc_cnt_lo; /* Counter cleared */ - u32 hnc_rx_bytes; /* Rx bytes */ - u32 hnc_tx_bytes; /* Tx bytes */ - u32 hnc_rx_uc_pkts; /* Rx UC packets */ - u32 hnc_rx_mc_pkts; /* Rx MC packets */ - u32 hnc_rx_bc_pkts; /* Rx BC packets */ - u32 hnc_tx_uc_pkts; /* Tx UC packets */ - u32 hnc_tx_mc_pkts; /* Tx MC packets */ - u32 hnc_tx_bc_pkts; /* Tx BC packets */ + u64 hnc_cnt; /* Counter cleared */ + u64 hnc_rx_bytes; /* Rx bytes */ + u64 hnc_tx_bytes; /* Tx bytes */ + u64 hnc_rx_uc_pkts; /* Rx UC packets */ + u64 hnc_rx_mc_pkts; /* Rx MC packets */ + u64 hnc_rx_bc_pkts; /* Rx BC packets */ + u64 hnc_tx_uc_pkts; /* Tx UC packets */ + u64 hnc_tx_mc_pkts; /* Tx MC packets */ + u64 hnc_tx_bc_pkts; /* Tx BC packets */ u32 hnc_fcs_err; /* FCS errors */ u32 hnc_align_err; /* Alignment errors */ u32 hnc_false_carrier; /* False carrier detection */ @@ -181,7 +180,7 @@ struct ncsi_channel_stats { u32 hnc_tx_1023_frames; /* Tx 512-1023 bytes frames */ u32 hnc_tx_1522_frames; /* Tx 1024-1522 bytes frames */ u32 hnc_tx_9022_frames; /* Tx 1523-9022 bytes frames */ - u32 hnc_rx_valid_bytes; /* Rx valid bytes */ + u64 hnc_rx_valid_bytes; /* Rx valid bytes */ u32 hnc_rx_runt_pkts; /* Rx error runt packets */ u32 hnc_rx_jabber_pkts; /* Rx error jabber packets */ u32 ncsi_rx_cmds; /* Rx NCSI commands */ diff --git a/net/ncsi/ncsi-pkt.h b/net/ncsi/ncsi-pkt.h index f2f3b5c1b9412..24edb27379724 100644 --- a/net/ncsi/ncsi-pkt.h +++ b/net/ncsi/ncsi-pkt.h @@ -252,16 +252,15 @@ struct ncsi_rsp_gp_pkt { /* Get Controller Packet Statistics */ struct ncsi_rsp_gcps_pkt { struct ncsi_rsp_pkt_hdr rsp; /* Response header */ - __be32 cnt_hi; /* Counter cleared */ - __be32 cnt_lo; /* Counter cleared */ - __be32 rx_bytes; /* Rx bytes */ - __be32 tx_bytes; /* Tx bytes */ - __be32 rx_uc_pkts; /* Rx UC packets */ - __be32 rx_mc_pkts; /* Rx MC packets */ - __be32 rx_bc_pkts; /* Rx BC packets */ - __be32 tx_uc_pkts; /* Tx UC packets */ - __be32 tx_mc_pkts; /* Tx MC packets */ - __be32 tx_bc_pkts; /* Tx BC packets */ + __be64 cnt; /* Counter cleared */ + __be64 rx_bytes; /* Rx bytes */ + __be64 tx_bytes; /* Tx bytes */ + __be64 rx_uc_pkts; /* Rx UC packets */ + __be64 rx_mc_pkts; /* Rx MC packets */ + __be64 rx_bc_pkts; /* Rx BC packets */ + __be64 tx_uc_pkts; /* Tx UC packets */ + __be64 tx_mc_pkts; /* Tx MC packets */ + __be64 tx_bc_pkts; /* Tx BC packets */ __be32 fcs_err; /* FCS errors */ __be32 align_err; /* Alignment errors */ __be32 false_carrier; /* False carrier detection */ @@ -290,11 +289,11 @@ struct ncsi_rsp_gcps_pkt { __be32 tx_1023_frames; /* Tx 512-1023 bytes frames */ __be32 tx_1522_frames; /* Tx 1024-1522 bytes frames */ __be32 tx_9022_frames; /* Tx 1523-9022 bytes frames */ - __be32 rx_valid_bytes; /* Rx valid bytes */ + __be64 rx_valid_bytes; /* Rx valid bytes */ __be32 rx_runt_pkts; /* Rx error runt packets */ __be32 rx_jabber_pkts; /* Rx error jabber packets */ __be32 checksum; /* Checksum */ -}; +} __packed __aligned(4); /* Get NCSI Statistics */ struct ncsi_rsp_gns_pkt { diff --git a/net/ncsi/ncsi-rsp.c b/net/ncsi/ncsi-rsp.c index 4a8ce2949faea..d5ed80731e892 100644 --- a/net/ncsi/ncsi-rsp.c +++ b/net/ncsi/ncsi-rsp.c @@ -775,6 +775,7 @@ static int ncsi_rsp_handler_gvi(struct ncsi_request *nr) ncv->alpha1 = rsp->alpha1; ncv->alpha2 = rsp->alpha2; memcpy(ncv->fw_name, rsp->fw_name, 12); + ncv->fw_name[12] = '\0'; ncv->fw_version = ntohl(rsp->fw_version); for (i = 0; i < ARRAY_SIZE(ncv->pci_ids); i++) ncv->pci_ids[i] = ntohs(rsp->pci_ids[i]); @@ -926,16 +927,15 @@ static int ncsi_rsp_handler_gcps(struct ncsi_request *nr) /* Update HNC's statistics */ ncs = &nc->stats; - ncs->hnc_cnt_hi = ntohl(rsp->cnt_hi); - ncs->hnc_cnt_lo = ntohl(rsp->cnt_lo); - ncs->hnc_rx_bytes = ntohl(rsp->rx_bytes); - ncs->hnc_tx_bytes = ntohl(rsp->tx_bytes); - ncs->hnc_rx_uc_pkts = ntohl(rsp->rx_uc_pkts); - ncs->hnc_rx_mc_pkts = ntohl(rsp->rx_mc_pkts); - ncs->hnc_rx_bc_pkts = ntohl(rsp->rx_bc_pkts); - ncs->hnc_tx_uc_pkts = ntohl(rsp->tx_uc_pkts); - ncs->hnc_tx_mc_pkts = ntohl(rsp->tx_mc_pkts); - ncs->hnc_tx_bc_pkts = ntohl(rsp->tx_bc_pkts); + ncs->hnc_cnt = be64_to_cpu(rsp->cnt); + ncs->hnc_rx_bytes = be64_to_cpu(rsp->rx_bytes); + ncs->hnc_tx_bytes = be64_to_cpu(rsp->tx_bytes); + ncs->hnc_rx_uc_pkts = be64_to_cpu(rsp->rx_uc_pkts); + ncs->hnc_rx_mc_pkts = be64_to_cpu(rsp->rx_mc_pkts); + ncs->hnc_rx_bc_pkts = be64_to_cpu(rsp->rx_bc_pkts); + ncs->hnc_tx_uc_pkts = be64_to_cpu(rsp->tx_uc_pkts); + ncs->hnc_tx_mc_pkts = be64_to_cpu(rsp->tx_mc_pkts); + ncs->hnc_tx_bc_pkts = be64_to_cpu(rsp->tx_bc_pkts); ncs->hnc_fcs_err = ntohl(rsp->fcs_err); ncs->hnc_align_err = ntohl(rsp->align_err); ncs->hnc_false_carrier = ntohl(rsp->false_carrier); @@ -964,7 +964,7 @@ static int ncsi_rsp_handler_gcps(struct ncsi_request *nr) ncs->hnc_tx_1023_frames = ntohl(rsp->tx_1023_frames); ncs->hnc_tx_1522_frames = ntohl(rsp->tx_1522_frames); ncs->hnc_tx_9022_frames = ntohl(rsp->tx_9022_frames); - ncs->hnc_rx_valid_bytes = ntohl(rsp->rx_valid_bytes); + ncs->hnc_rx_valid_bytes = be64_to_cpu(rsp->rx_valid_bytes); ncs->hnc_rx_runt_pkts = ntohl(rsp->rx_runt_pkts); ncs->hnc_rx_jabber_pkts = ntohl(rsp->rx_jabber_pkts); diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index 20aad81fcad7e..20b223e6e93b9 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -62,8 +62,8 @@ struct hbucket { : jhash_size((htable_bits) - HTABLE_REGION_BITS)) #define ahash_sizeof_regions(htable_bits) \ (ahash_numof_locks(htable_bits) * sizeof(struct ip_set_region)) -#define ahash_region(n, htable_bits) \ - ((n) % ahash_numof_locks(htable_bits)) +#define ahash_region(n) \ + ((n) / jhash_size(HTABLE_REGION_BITS)) #define ahash_bucket_start(h, htable_bits) \ ((htable_bits) < HTABLE_REGION_BITS ? 0 \ : (h) * jhash_size(HTABLE_REGION_BITS)) @@ -701,7 +701,7 @@ mtype_resize(struct ip_set *set, bool retried) #endif key = HKEY(data, h->initval, htable_bits); m = __ipset_dereference(hbucket(t, key)); - nr = ahash_region(key, htable_bits); + nr = ahash_region(key); if (!m) { m = kzalloc(sizeof(*m) + AHASH_INIT_SIZE * dsize, @@ -851,7 +851,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, rcu_read_lock_bh(); t = rcu_dereference_bh(h->table); key = HKEY(value, h->initval, t->htable_bits); - r = ahash_region(key, t->htable_bits); + r = ahash_region(key); atomic_inc(&t->uref); elements = t->hregion[r].elements; maxelem = t->maxelem; @@ -1049,7 +1049,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext, rcu_read_lock_bh(); t = rcu_dereference_bh(h->table); key = HKEY(value, h->initval, t->htable_bits); - r = ahash_region(key, t->htable_bits); + r = ahash_region(key); atomic_inc(&t->uref); rcu_read_unlock_bh(); diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 8182833a35828..d9c3eefe8cbb0 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -884,7 +884,7 @@ static void ip_vs_conn_expire(struct timer_list *t) * conntrack cleanup for the net. */ smp_rmb(); - if (ipvs->enable) + if (READ_ONCE(ipvs->enable)) ip_vs_conn_drop_conntrack(cp); } @@ -1432,7 +1432,7 @@ void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs) cond_resched_rcu(); /* netns clean up started, abort delayed work */ - if (!ipvs->enable) + if (!READ_ONCE(ipvs->enable)) break; } rcu_read_unlock(); diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index 3230506ae3ffd..34c4648ba36ec 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -1353,9 +1353,6 @@ ip_vs_out_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *stat if (unlikely(!skb_dst(skb))) return NF_ACCEPT; - if (!ipvs->enable) - return NF_ACCEPT; - ip_vs_fill_iph_skb(af, skb, false, &iph); #ifdef CONFIG_IP_VS_IPV6 if (af == AF_INET6) { @@ -1936,7 +1933,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state return NF_ACCEPT; } /* ipvs enabled in this netns ? */ - if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable)) + if (unlikely(sysctl_backup_only(ipvs))) return NF_ACCEPT; ip_vs_fill_iph_skb(af, skb, false, &iph); @@ -2104,7 +2101,7 @@ ip_vs_forward_icmp(void *priv, struct sk_buff *skb, int r; /* ipvs enabled in this netns ? */ - if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable)) + if (unlikely(sysctl_backup_only(ipvs))) return NF_ACCEPT; if (state->pf == NFPROTO_IPV4) { @@ -2291,7 +2288,7 @@ static int __net_init __ip_vs_init(struct net *net) return -ENOMEM; /* Hold the beast until a service is registered */ - ipvs->enable = 0; + WRITE_ONCE(ipvs->enable, 0); ipvs->net = net; /* Counters used for creating unique names */ ipvs->gen = atomic_read(&ipvs_netns_cnt); @@ -2363,7 +2360,7 @@ static void __net_exit __ip_vs_dev_cleanup_batch(struct list_head *net_list) ipvs = net_ipvs(net); ip_vs_unregister_hooks(ipvs, AF_INET); ip_vs_unregister_hooks(ipvs, AF_INET6); - ipvs->enable = 0; /* Disable packet reception */ + WRITE_ONCE(ipvs->enable, 0); /* Disable packet reception */ smp_wmb(); ip_vs_sync_net_cleanup(ipvs); } diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index ae76542de3e98..c82dcbb4dabce 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -256,7 +256,7 @@ static void est_reload_work_handler(struct work_struct *work) struct ip_vs_est_kt_data *kd = ipvs->est_kt_arr[id]; /* netns clean up started, abort delayed work */ - if (!ipvs->enable) + if (!READ_ONCE(ipvs->enable)) goto unlock; if (!kd) continue; @@ -1482,9 +1482,9 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u, *svc_p = svc; - if (!ipvs->enable) { + if (!READ_ONCE(ipvs->enable)) { /* Now there is a service - full throttle */ - ipvs->enable = 1; + WRITE_ONCE(ipvs->enable, 1); /* Start estimation for first time */ ip_vs_est_reload_start(ipvs); diff --git a/net/netfilter/ipvs/ip_vs_est.c b/net/netfilter/ipvs/ip_vs_est.c index c5970ba416aea..b48d3a09e2174 100644 --- a/net/netfilter/ipvs/ip_vs_est.c +++ b/net/netfilter/ipvs/ip_vs_est.c @@ -230,7 +230,7 @@ static int ip_vs_estimation_kthread(void *data) void ip_vs_est_reload_start(struct netns_ipvs *ipvs) { /* Ignore reloads before first service is added */ - if (!ipvs->enable) + if (!READ_ONCE(ipvs->enable)) return; ip_vs_est_stopped_recalc(ipvs); /* Bump the kthread configuration genid */ @@ -304,7 +304,7 @@ static int ip_vs_est_add_kthread(struct netns_ipvs *ipvs) int i; if ((unsigned long)ipvs->est_kt_count >= ipvs->est_max_threads && - ipvs->enable && ipvs->est_max_threads) + READ_ONCE(ipvs->enable) && ipvs->est_max_threads) return -EINVAL; mutex_lock(&ipvs->est_mutex); @@ -341,7 +341,7 @@ static int ip_vs_est_add_kthread(struct netns_ipvs *ipvs) } /* Start kthread tasks only when services are present */ - if (ipvs->enable && !ip_vs_est_stopped(ipvs)) { + if (READ_ONCE(ipvs->enable) && !ip_vs_est_stopped(ipvs)) { ret = ip_vs_est_kthread_start(ipvs, kd); if (ret < 0) goto out; @@ -484,7 +484,7 @@ int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats) struct ip_vs_estimator *est = &stats->est; int ret; - if (!ipvs->est_max_threads && ipvs->enable) + if (!ipvs->est_max_threads && READ_ONCE(ipvs->enable)) ipvs->est_max_threads = ip_vs_est_max_threads(ipvs); est->ktid = -1; @@ -661,7 +661,7 @@ static int ip_vs_est_calc_limits(struct netns_ipvs *ipvs, int *chain_max) /* Wait for cpufreq frequency transition */ wait_event_idle_timeout(wq, kthread_should_stop(), HZ / 50); - if (!ipvs->enable || kthread_should_stop()) + if (!READ_ONCE(ipvs->enable) || kthread_should_stop()) goto stop; } @@ -679,7 +679,7 @@ static int ip_vs_est_calc_limits(struct netns_ipvs *ipvs, int *chain_max) rcu_read_unlock(); local_bh_enable(); - if (!ipvs->enable || kthread_should_stop()) + if (!READ_ONCE(ipvs->enable) || kthread_should_stop()) goto stop; cond_resched(); @@ -755,7 +755,7 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs) mutex_lock(&ipvs->est_mutex); for (id = 1; id < ipvs->est_kt_count; id++) { /* netns clean up started, abort */ - if (!ipvs->enable) + if (!READ_ONCE(ipvs->enable)) goto unlock2; kd = ipvs->est_kt_arr[id]; if (!kd) @@ -785,7 +785,7 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs) id = ipvs->est_kt_count; next_kt: - if (!ipvs->enable || kthread_should_stop()) + if (!READ_ONCE(ipvs->enable) || kthread_should_stop()) goto unlock; id--; if (id < 0) diff --git a/net/netfilter/ipvs/ip_vs_ftp.c b/net/netfilter/ipvs/ip_vs_ftp.c index ef1f45e43b630..61d3797fb7995 100644 --- a/net/netfilter/ipvs/ip_vs_ftp.c +++ b/net/netfilter/ipvs/ip_vs_ftp.c @@ -53,6 +53,7 @@ enum { IP_VS_FTP_EPSV, }; +static bool exiting_module; /* * List of ports (up to IP_VS_APP_MAX_PORTS) to be handled by helper * First port is set to the default port. @@ -605,7 +606,7 @@ static void __ip_vs_ftp_exit(struct net *net) { struct netns_ipvs *ipvs = net_ipvs(net); - if (!ipvs) + if (!ipvs || !exiting_module) return; unregister_ip_vs_app(ipvs, &ip_vs_ftp); @@ -627,6 +628,7 @@ static int __init ip_vs_ftp_init(void) */ static void __exit ip_vs_ftp_exit(void) { + exiting_module = true; unregister_pernet_subsys(&ip_vs_ftp_ops); /* rcu_barrier() is called by netns */ } diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index 5cd511162bc03..0103c4a4d10a5 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -119,13 +119,12 @@ __mtu_check_toobig_v6(const struct sk_buff *skb, u32 mtu) return false; } -/* Get route to daddr, update *saddr, optionally bind route to saddr */ +/* Get route to daddr, optionally bind route to saddr */ static struct rtable *do_output_route4(struct net *net, __be32 daddr, - int rt_mode, __be32 *saddr) + int rt_mode, __be32 *ret_saddr) { struct flowi4 fl4; struct rtable *rt; - bool loop = false; memset(&fl4, 0, sizeof(fl4)); fl4.daddr = daddr; @@ -135,23 +134,17 @@ static struct rtable *do_output_route4(struct net *net, __be32 daddr, retry: rt = ip_route_output_key(net, &fl4); if (IS_ERR(rt)) { - /* Invalid saddr ? */ - if (PTR_ERR(rt) == -EINVAL && *saddr && - rt_mode & IP_VS_RT_MODE_CONNECT && !loop) { - *saddr = 0; - flowi4_update_output(&fl4, 0, daddr, 0); - goto retry; - } IP_VS_DBG_RL("ip_route_output error, dest: %pI4\n", &daddr); return NULL; - } else if (!*saddr && rt_mode & IP_VS_RT_MODE_CONNECT && fl4.saddr) { + } + if (rt_mode & IP_VS_RT_MODE_CONNECT && fl4.saddr) { ip_rt_put(rt); - *saddr = fl4.saddr; flowi4_update_output(&fl4, 0, daddr, fl4.saddr); - loop = true; + rt_mode = 0; goto retry; } - *saddr = fl4.saddr; + if (ret_saddr) + *ret_saddr = fl4.saddr; return rt; } @@ -344,19 +337,15 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (ret_saddr) *ret_saddr = dest_dst->dst_saddr.ip; } else { - __be32 saddr = htonl(INADDR_ANY); - noref = 0; /* For such unconfigured boxes avoid many route lookups * for performance reasons because we do not remember saddr */ rt_mode &= ~IP_VS_RT_MODE_CONNECT; - rt = do_output_route4(net, daddr, rt_mode, &saddr); + rt = do_output_route4(net, daddr, rt_mode, ret_saddr); if (!rt) goto err_unreach; - if (ret_saddr) - *ret_saddr = saddr; } local = (rt->rt_flags & RTCF_LOCAL) ? 1 : 0; diff --git a/net/netfilter/nf_bpf_link.c b/net/netfilter/nf_bpf_link.c index 2aad0562a4135..658e401b7937e 100644 --- a/net/netfilter/nf_bpf_link.c +++ b/net/netfilter/nf_bpf_link.c @@ -17,7 +17,7 @@ static unsigned int nf_hook_run_bpf(void *bpf_prog, struct sk_buff *skb, .skb = skb, }; - return bpf_prog_run(prog, &ctx); + return bpf_prog_run_pin_on_cpu(prog, &ctx); } struct bpf_nf_link { @@ -295,6 +295,9 @@ static bool nf_is_valid_access(int off, int size, enum bpf_access_type type, if (off < 0 || off >= sizeof(struct bpf_nf_ctx)) return false; + if (off % size != 0) + return false; + if (type == BPF_WRITE) return false; diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 34ad5975fbf3b..0081c1a0d5e56 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -1075,6 +1075,12 @@ static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx) hlist_nulls_add_head_rcu(&loser_ct->tuplehash[IP_CT_DIR_REPLY].hnnode, &nf_conntrack_hash[repl_idx]); + /* confirmed bit must be set after hlist add, not before: + * loser_ct can still be visible to other cpu due to + * SLAB_TYPESAFE_BY_RCU. + */ + smp_mb__before_atomic(); + set_bit(IPS_CONFIRMED_BIT, &loser_ct->status); NF_CT_STAT_INC(net, clash_resolve); return NF_ACCEPT; @@ -1211,8 +1217,6 @@ __nf_conntrack_confirm(struct sk_buff *skb) * user context, else we insert an already 'dead' hash, blocking * further use of that particular connection -JM. */ - ct->status |= IPS_CONFIRMED; - if (unlikely(nf_ct_is_dying(ct))) { NF_CT_STAT_INC(net, insert_failed); goto dying; @@ -1244,7 +1248,7 @@ __nf_conntrack_confirm(struct sk_buff *skb) } } - /* Timer relative to confirmation time, not original + /* Timeout is relative to confirmation time, not original setting time, otherwise we'd get timer wrap in weird delay cases. */ ct->timeout += nfct_time_stamp; @@ -1252,11 +1256,21 @@ __nf_conntrack_confirm(struct sk_buff *skb) __nf_conntrack_insert_prepare(ct); /* Since the lookup is lockless, hash insertion must be done after - * starting the timer and setting the CONFIRMED bit. The RCU barriers - * guarantee that no other CPU can find the conntrack before the above - * stores are visible. + * setting ct->timeout. The RCU barriers guarantee that no other CPU + * can find the conntrack before the above stores are visible. */ __nf_conntrack_hash_insert(ct, hash, reply_hash); + + /* IPS_CONFIRMED unset means 'ct not (yet) in hash', conntrack lookups + * skip entries that lack this bit. This happens when a CPU is looking + * at a stale entry that is being recycled due to SLAB_TYPESAFE_BY_RCU + * or when another CPU encounters this entry right after the insertion + * but before the set-confirm-bit below. This bit must not be set until + * after __nf_conntrack_hash_insert(). + */ + smp_mb__before_atomic(); + set_bit(IPS_CONFIRMED_BIT, &ct->status); + nf_conntrack_double_unlock(hash, reply_hash); local_bh_enable(); diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index f22691f838536..10f72b5b4e1ad 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -373,7 +373,7 @@ int nf_conntrack_helper_register(struct nf_conntrack_helper *me) (cur->tuple.src.l3num == NFPROTO_UNSPEC || cur->tuple.src.l3num == me->tuple.src.l3num) && cur->tuple.dst.protonum == me->tuple.dst.protonum) { - ret = -EEXIST; + ret = -EBUSY; goto out; } } @@ -384,7 +384,7 @@ int nf_conntrack_helper_register(struct nf_conntrack_helper *me) hlist_for_each_entry(cur, &nf_ct_helper_hash[h], hnode) { if (nf_ct_tuple_src_mask_cmp(&cur->tuple, &me->tuple, &mask)) { - ret = -EEXIST; + ret = -EBUSY; goto out; } } diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 282e9644f6fdd..928bd2013289a 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -859,8 +859,6 @@ ctnetlink_conntrack_event(unsigned int events, const struct nf_ct_event *item) static int ctnetlink_done(struct netlink_callback *cb) { - if (cb->args[1]) - nf_ct_put((struct nf_conn *)cb->args[1]); kfree(cb->data); return 0; } @@ -1175,19 +1173,26 @@ static int ctnetlink_filter_match(struct nf_conn *ct, void *data) return 0; } +static unsigned long ctnetlink_get_id(const struct nf_conn *ct) +{ + unsigned long id = nf_ct_get_id(ct); + + return id ? id : 1; +} + static int ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb) { unsigned int flags = cb->data ? NLM_F_DUMP_FILTERED : 0; struct net *net = sock_net(skb->sk); - struct nf_conn *ct, *last; + unsigned long last_id = cb->args[1]; struct nf_conntrack_tuple_hash *h; struct hlist_nulls_node *n; struct nf_conn *nf_ct_evict[8]; + struct nf_conn *ct; int res, i; spinlock_t *lockp; - last = (struct nf_conn *)cb->args[1]; i = 0; local_bh_disable(); @@ -1224,7 +1229,7 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb) continue; if (cb->args[1]) { - if (ct != last) + if (ctnetlink_get_id(ct) != last_id) continue; cb->args[1] = 0; } @@ -1237,8 +1242,7 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb) NFNL_MSG_TYPE(cb->nlh->nlmsg_type), ct, true, flags); if (res < 0) { - nf_conntrack_get(&ct->ct_general); - cb->args[1] = (unsigned long)ct; + cb->args[1] = ctnetlink_get_id(ct); spin_unlock(lockp); goto out; } @@ -1251,12 +1255,10 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb) } out: local_bh_enable(); - if (last) { + if (last_id) { /* nf ct hash resize happened, now clear the leftover. */ - if ((struct nf_conn *)cb->args[1] == last) + if (cb->args[1] == last_id) cb->args[1] = 0; - - nf_ct_put(last); } while (i) { diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c index 559665467b04d..1d5de1d9f008d 100644 --- a/net/netfilter/nf_conntrack_standalone.c +++ b/net/netfilter/nf_conntrack_standalone.c @@ -621,7 +621,9 @@ static struct ctl_table nf_ct_sysctl_table[] = { .data = &nf_conntrack_max, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, }, [NF_SYSCTL_CT_COUNT] = { .procname = "nf_conntrack_count", @@ -657,7 +659,9 @@ static struct ctl_table nf_ct_sysctl_table[] = { .data = &nf_ct_expect_max, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_INT_MAX, }, [NF_SYSCTL_CT_ACCT] = { .procname = "nf_conntrack_acct", @@ -951,7 +955,9 @@ static struct ctl_table nf_ct_netfilter_table[] = { .data = &nf_conntrack_max, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, }, { } }; diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index ccca6e3848bcc..9df883d79acc9 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -248,7 +248,7 @@ static noinline bool nf_nat_used_tuple_new(const struct nf_conntrack_tuple *tuple, const struct nf_conn *ignored_ct) { - static const unsigned long uses_nat = IPS_NAT_MASK | IPS_SEQ_ADJUST_BIT; + static const unsigned long uses_nat = IPS_NAT_MASK | IPS_SEQ_ADJUST; const struct nf_conntrack_tuple_hash *thash; const struct nf_conntrack_zone *zone; struct nf_conn *ct; @@ -287,8 +287,14 @@ nf_nat_used_tuple_new(const struct nf_conntrack_tuple *tuple, zone = nf_ct_zone(ignored_ct); thash = nf_conntrack_find_get(net, zone, tuple); - if (unlikely(!thash)) /* clashing entry went away */ - return false; + if (unlikely(!thash)) { + struct nf_conntrack_tuple reply; + + nf_ct_invert_tuple(&reply, tuple); + thash = nf_conntrack_find_get(net, zone, &reply); + if (!thash) /* clashing entry went away */ + return false; + } ct = nf_ct_tuplehash_to_ctrack(thash); diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 18ae39cf41887..80443b4eaeff0 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -979,11 +979,6 @@ static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net, NFTA_TABLE_PAD)) goto nla_put_failure; - if (event == NFT_MSG_DELTABLE) { - nlmsg_end(skb, nlh); - return 0; - } - if (nla_put_be32(skb, NFTA_TABLE_FLAGS, htonl(table->flags & NFT_TABLE_F_MASK))) goto nla_put_failure; @@ -1827,11 +1822,6 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, NFTA_CHAIN_PAD)) goto nla_put_failure; - if (event == NFT_MSG_DELCHAIN && !hook_list) { - nlmsg_end(skb, nlh); - return 0; - } - if (nft_is_base_chain(chain)) { const struct nft_base_chain *basechain = nft_base_chain(chain); struct nft_stats __percpu *stats; @@ -2586,6 +2576,7 @@ static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy, struct nft_chain *chain = ctx->chain; struct nft_chain_hook hook = {}; struct nft_stats *stats = NULL; + struct nftables_pernet *nft_net; struct nft_hook *h, *next; struct nf_hook_ops *ops; struct nft_trans *trans; @@ -2626,6 +2617,20 @@ static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy, if (nft_hook_list_find(&basechain->hook_list, h)) { list_del(&h->list); kfree(h); + continue; + } + + nft_net = nft_pernet(ctx->net); + list_for_each_entry(trans, &nft_net->commit_list, list) { + if (trans->msg_type != NFT_MSG_NEWCHAIN || + trans->ctx.table != ctx->table || + !nft_trans_chain_update(trans)) + continue; + + if (nft_hook_list_find(&nft_trans_chain_hooks(trans), h)) { + nft_chain_release_hook(&hook); + return -EEXIST; + } } } } else { @@ -3785,7 +3790,7 @@ void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule) /* can only be used if rule is no longer visible to dumps */ static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule) { - lockdep_commit_lock_is_held(ctx->net); + WARN_ON_ONCE(!lockdep_commit_lock_is_held(ctx->net)); nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE); nf_tables_rule_destroy(ctx, rule); @@ -3803,7 +3808,6 @@ static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *r int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain) { struct nft_expr *expr, *last; - const struct nft_data *data; struct nft_rule *rule; int err; @@ -3824,7 +3828,7 @@ int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain) /* This may call nft_chain_validate() recursively, * callers that do so must increment ctx->level. */ - err = expr->ops->validate(ctx, expr, &data); + err = expr->ops->validate(ctx, expr); if (err < 0) return err; } @@ -4584,11 +4588,6 @@ static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx, NFTA_SET_PAD)) goto nla_put_failure; - if (event == NFT_MSG_DELSET) { - nlmsg_end(skb, nlh); - return 0; - } - if (set->flags != 0) if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags))) goto nla_put_failure; @@ -5586,7 +5585,7 @@ void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *binding, enum nft_trans_phase phase) { - lockdep_commit_lock_is_held(ctx->net); + WARN_ON_ONCE(!lockdep_commit_lock_is_held(ctx->net)); switch (phase) { case NFT_TRANS_PREPARE_ERROR: @@ -7828,11 +7827,6 @@ static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net, NFTA_OBJ_PAD)) goto nla_put_failure; - if (event == NFT_MSG_DELOBJ) { - nlmsg_end(skb, nlh); - return 0; - } - if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) || nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) || nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset)) @@ -8514,6 +8508,7 @@ static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh, { const struct nlattr * const *nla = ctx->nla; struct nft_flowtable_hook flowtable_hook; + struct nftables_pernet *nft_net; struct nft_hook *hook, *next; struct nft_trans *trans; bool unregister = false; @@ -8529,6 +8524,20 @@ static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh, if (nft_hook_list_find(&flowtable->hook_list, hook)) { list_del(&hook->list); kfree(hook); + continue; + } + + nft_net = nft_pernet(ctx->net); + list_for_each_entry(trans, &nft_net->commit_list, list) { + if (trans->msg_type != NFT_MSG_NEWFLOWTABLE || + trans->ctx.table != ctx->table || + !nft_trans_flowtable_update(trans)) + continue; + + if (nft_hook_list_find(&nft_trans_flowtable_hooks(trans), hook)) { + err = -EEXIST; + goto err_flowtable_update_hook; + } } } @@ -8851,11 +8860,6 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net, NFTA_FLOWTABLE_PAD)) goto nla_put_failure; - if (event == NFT_MSG_DELFLOWTABLE && !hook_list) { - nlmsg_end(skb, nlh); - return 0; - } - if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) || nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags))) goto nla_put_failure; diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index d3d11dede5450..52cdfee17f73f 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -350,8 +350,7 @@ static int nft_target_dump(struct sk_buff *skb, } static int nft_target_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { struct xt_target *target = expr->ops->data; unsigned int hook_mask = 0; @@ -611,8 +610,7 @@ static int nft_match_large_dump(struct sk_buff *skb, } static int nft_match_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { struct xt_match *match = expr->ops->data; unsigned int hook_mask = 0; diff --git a/net/netfilter/nft_fib.c b/net/netfilter/nft_fib.c index bf825f6cb974e..507b7eba9bcdc 100644 --- a/net/netfilter/nft_fib.c +++ b/net/netfilter/nft_fib.c @@ -26,8 +26,7 @@ const struct nla_policy nft_fib_policy[NFTA_FIB_MAX + 1] = { }; EXPORT_SYMBOL(nft_fib_policy); -int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, - const struct nft_data **data) +int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { const struct nft_fib *priv = nft_expr_priv(expr); unsigned int hooks; diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c index 5a3d685420420..1894032a7971c 100644 --- a/net/netfilter/nft_flow_offload.c +++ b/net/netfilter/nft_flow_offload.c @@ -385,8 +385,7 @@ static void nft_flow_offload_eval(const struct nft_expr *expr, } static int nft_flow_offload_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { unsigned int hook_mask = (1 << NF_INET_FORWARD); diff --git a/net/netfilter/nft_fwd_netdev.c b/net/netfilter/nft_fwd_netdev.c index a5268e6dd32f1..fa9e4ae00b16a 100644 --- a/net/netfilter/nft_fwd_netdev.c +++ b/net/netfilter/nft_fwd_netdev.c @@ -204,8 +204,7 @@ static int nft_fwd_neigh_dump(struct sk_buff *skb, } static int nft_fwd_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { return nft_chain_validate_hooks(ctx->chain, (1 << NF_NETDEV_INGRESS) | (1 << NF_NETDEV_EGRESS)); diff --git a/net/netfilter/nft_immediate.c b/net/netfilter/nft_immediate.c index ac2422c215e54..02ee5fb69871f 100644 --- a/net/netfilter/nft_immediate.c +++ b/net/netfilter/nft_immediate.c @@ -244,8 +244,7 @@ static int nft_immediate_dump(struct sk_buff *skb, } static int nft_immediate_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **d) + const struct nft_expr *expr) { const struct nft_immediate_expr *priv = nft_expr_priv(expr); struct nft_ctx *pctx = (struct nft_ctx *)ctx; diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index 1b9edf2b33937..dd5441f92fdb0 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -206,8 +206,7 @@ static int nft_lookup_dump(struct sk_buff *skb, } static int nft_lookup_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **d) + const struct nft_expr *expr) { const struct nft_lookup *priv = nft_expr_priv(expr); struct nft_set_iter iter; diff --git a/net/netfilter/nft_masq.c b/net/netfilter/nft_masq.c index 8a14aaca93bbd..eee05394c5339 100644 --- a/net/netfilter/nft_masq.c +++ b/net/netfilter/nft_masq.c @@ -27,8 +27,7 @@ static const struct nla_policy nft_masq_policy[NFTA_MASQ_MAX + 1] = { }; static int nft_masq_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { int err; diff --git a/net/netfilter/nft_meta.c b/net/netfilter/nft_meta.c index 9139ce38ea7b9..dec76d28a0ac6 100644 --- a/net/netfilter/nft_meta.c +++ b/net/netfilter/nft_meta.c @@ -581,8 +581,7 @@ static int nft_meta_get_validate_xfrm(const struct nft_ctx *ctx) } static int nft_meta_get_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { const struct nft_meta *priv = nft_expr_priv(expr); @@ -600,8 +599,7 @@ static int nft_meta_get_validate(const struct nft_ctx *ctx, } int nft_meta_set_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { struct nft_meta *priv = nft_expr_priv(expr); unsigned int hooks; diff --git a/net/netfilter/nft_nat.c b/net/netfilter/nft_nat.c index 808f5802c2704..3d3e639a7a837 100644 --- a/net/netfilter/nft_nat.c +++ b/net/netfilter/nft_nat.c @@ -137,8 +137,7 @@ static const struct nla_policy nft_nat_policy[NFTA_NAT_MAX + 1] = { }; static int nft_nat_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { struct nft_nat *priv = nft_expr_priv(expr); int err; diff --git a/net/netfilter/nft_objref.c b/net/netfilter/nft_objref.c index 509011b1ef597..08a27433e2f5f 100644 --- a/net/netfilter/nft_objref.c +++ b/net/netfilter/nft_objref.c @@ -22,6 +22,35 @@ void nft_objref_eval(const struct nft_expr *expr, obj->ops->eval(obj, regs, pkt); } +static int nft_objref_validate_obj_type(const struct nft_ctx *ctx, u32 type) +{ + unsigned int hooks; + + switch (type) { + case NFT_OBJECT_SYNPROXY: + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + + hooks = (1 << NF_INET_LOCAL_IN) | (1 << NF_INET_FORWARD); + + return nft_chain_validate_hooks(ctx->chain, hooks); + default: + break; + } + + return 0; +} + +static int nft_objref_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct nft_object *obj = nft_objref_priv(expr); + + return nft_objref_validate_obj_type(ctx, obj->ops->type->type); +} + static int nft_objref_init(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nlattr * const tb[]) @@ -93,6 +122,7 @@ static const struct nft_expr_ops nft_objref_ops = { .activate = nft_objref_activate, .deactivate = nft_objref_deactivate, .dump = nft_objref_dump, + .validate = nft_objref_validate, .reduce = NFT_REDUCE_READONLY, }; @@ -198,6 +228,14 @@ static void nft_objref_map_destroy(const struct nft_ctx *ctx, nf_tables_destroy_set(ctx, priv->set); } +static int nft_objref_map_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + const struct nft_objref_map *priv = nft_expr_priv(expr); + + return nft_objref_validate_obj_type(ctx, priv->set->objtype); +} + static const struct nft_expr_ops nft_objref_map_ops = { .type = &nft_objref_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_objref_map)), @@ -207,6 +245,7 @@ static const struct nft_expr_ops nft_objref_map_ops = { .deactivate = nft_objref_map_deactivate, .destroy = nft_objref_map_destroy, .dump = nft_objref_map_dump, + .validate = nft_objref_map_validate, .reduce = NFT_REDUCE_READONLY, }; diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c index 7f61506e5b44b..123b4f47ccef4 100644 --- a/net/netfilter/nft_osf.c +++ b/net/netfilter/nft_osf.c @@ -113,8 +113,7 @@ static int nft_osf_dump(struct sk_buff *skb, } static int nft_osf_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { unsigned int hooks; diff --git a/net/netfilter/nft_queue.c b/net/netfilter/nft_queue.c index b2b8127c8d438..b8ebb187814f2 100644 --- a/net/netfilter/nft_queue.c +++ b/net/netfilter/nft_queue.c @@ -69,8 +69,7 @@ static void nft_queue_sreg_eval(const struct nft_expr *expr, } static int nft_queue_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { static const unsigned int supported_hooks = ((1 << NF_INET_PRE_ROUTING) | (1 << NF_INET_LOCAL_IN) | diff --git a/net/netfilter/nft_quota.c b/net/netfilter/nft_quota.c index 9b2d7463d3d32..df0798da2329b 100644 --- a/net/netfilter/nft_quota.c +++ b/net/netfilter/nft_quota.c @@ -19,10 +19,16 @@ struct nft_quota { }; static inline bool nft_overquota(struct nft_quota *priv, - const struct sk_buff *skb) + const struct sk_buff *skb, + bool *report) { - return atomic64_add_return(skb->len, priv->consumed) >= - atomic64_read(&priv->quota); + u64 consumed = atomic64_add_return(skb->len, priv->consumed); + u64 quota = atomic64_read(&priv->quota); + + if (report) + *report = consumed >= quota; + + return consumed > quota; } static inline bool nft_quota_invert(struct nft_quota *priv) @@ -34,7 +40,7 @@ static inline void nft_quota_do_eval(struct nft_quota *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt) { - if (nft_overquota(priv, pkt->skb) ^ nft_quota_invert(priv)) + if (nft_overquota(priv, pkt->skb, NULL) ^ nft_quota_invert(priv)) regs->verdict.code = NFT_BREAK; } @@ -51,13 +57,13 @@ static void nft_quota_obj_eval(struct nft_object *obj, const struct nft_pktinfo *pkt) { struct nft_quota *priv = nft_obj_data(obj); - bool overquota; + bool overquota, report; - overquota = nft_overquota(priv, pkt->skb); + overquota = nft_overquota(priv, pkt->skb, &report); if (overquota ^ nft_quota_invert(priv)) regs->verdict.code = NFT_BREAK; - if (overquota && + if (report && !test_and_set_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags)) nft_obj_notify(nft_net(pkt), obj->key.table, obj, 0, 0, NFT_MSG_NEWOBJ, 0, nft_pf(pkt), 0, GFP_ATOMIC); diff --git a/net/netfilter/nft_redir.c b/net/netfilter/nft_redir.c index a58bd8d291ff2..9051863509f31 100644 --- a/net/netfilter/nft_redir.c +++ b/net/netfilter/nft_redir.c @@ -27,8 +27,7 @@ static const struct nla_policy nft_redir_policy[NFTA_REDIR_MAX + 1] = { }; static int nft_redir_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { int err; diff --git a/net/netfilter/nft_reject.c b/net/netfilter/nft_reject.c index ed2e668474d67..196a92c7ea09b 100644 --- a/net/netfilter/nft_reject.c +++ b/net/netfilter/nft_reject.c @@ -24,8 +24,7 @@ const struct nla_policy nft_reject_policy[NFTA_REJECT_MAX + 1] = { EXPORT_SYMBOL_GPL(nft_reject_policy); int nft_reject_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { return nft_chain_validate_hooks(ctx->chain, (1 << NF_INET_LOCAL_IN) | diff --git a/net/netfilter/nft_reject_inet.c b/net/netfilter/nft_reject_inet.c index 973fa31a9dd6c..49020e67304ad 100644 --- a/net/netfilter/nft_reject_inet.c +++ b/net/netfilter/nft_reject_inet.c @@ -61,8 +61,7 @@ static void nft_reject_inet_eval(const struct nft_expr *expr, } static int nft_reject_inet_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { return nft_chain_validate_hooks(ctx->chain, (1 << NF_INET_LOCAL_IN) | diff --git a/net/netfilter/nft_reject_netdev.c b/net/netfilter/nft_reject_netdev.c index 7865cd8b11bb6..2558ce1505d98 100644 --- a/net/netfilter/nft_reject_netdev.c +++ b/net/netfilter/nft_reject_netdev.c @@ -145,8 +145,7 @@ static void nft_reject_netdev_eval(const struct nft_expr *expr, } static int nft_reject_netdev_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { return nft_chain_validate_hooks(ctx->chain, (1 << NF_NETDEV_INGRESS)); } diff --git a/net/netfilter/nft_rt.c b/net/netfilter/nft_rt.c index 2434c624aafde..9525f1e02a7d8 100644 --- a/net/netfilter/nft_rt.c +++ b/net/netfilter/nft_rt.c @@ -160,8 +160,7 @@ static int nft_rt_get_dump(struct sk_buff *skb, return -1; } -static int nft_rt_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, - const struct nft_data **data) +static int nft_rt_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { const struct nft_rt *priv = nft_expr_priv(expr); unsigned int hooks; diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c index c15db28c5ebc4..be7c16c79f711 100644 --- a/net/netfilter/nft_set_pipapo_avx2.c +++ b/net/netfilter/nft_set_pipapo_avx2.c @@ -1113,6 +1113,25 @@ bool nft_pipapo_avx2_estimate(const struct nft_set_desc *desc, u32 features, return true; } +/** + * pipapo_resmap_init_avx2() - Initialise result map before first use + * @m: Matching data, including mapping table + * @res_map: Result map + * + * Like pipapo_resmap_init() but do not set start map bits covered by the first field. + */ +static inline void pipapo_resmap_init_avx2(const struct nft_pipapo_match *m, unsigned long *res_map) +{ + const struct nft_pipapo_field *f = m->f; + int i; + + /* Starting map doesn't need to be set to all-ones for this implementation, + * but we do need to zero the remaining bits, if any. + */ + for (i = f->bsize; i < m->bsize_max; i++) + res_map[i] = 0ul; +} + /** * nft_pipapo_avx2_lookup() - Lookup function for AVX2 implementation * @net: Network namespace @@ -1171,7 +1190,7 @@ bool nft_pipapo_avx2_lookup(const struct net *net, const struct nft_set *set, res = scratch->map + (map_index ? m->bsize_max : 0); fill = scratch->map + (map_index ? 0 : m->bsize_max); - /* Starting map doesn't need to be set for this implementation */ + pipapo_resmap_init_avx2(m, res); nft_pipapo_avx2_prepare(); diff --git a/net/netfilter/nft_socket.c b/net/netfilter/nft_socket.c index 187b667bad6c3..35d0409b00950 100644 --- a/net/netfilter/nft_socket.c +++ b/net/netfilter/nft_socket.c @@ -275,8 +275,7 @@ static bool nft_socket_reduce(struct nft_regs_track *track, } static int nft_socket_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { if (ctx->family != NFPROTO_IPV4 && ctx->family != NFPROTO_IPV6 && diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index 1d737f89dfc18..5d3e518259859 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -248,8 +248,7 @@ static void nft_synproxy_eval(const struct nft_expr *expr, } static int nft_synproxy_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { if (ctx->family != NFPROTO_IPV4 && ctx->family != NFPROTO_IPV6 && diff --git a/net/netfilter/nft_tproxy.c b/net/netfilter/nft_tproxy.c index 71412adb73d41..ed344af2a439b 100644 --- a/net/netfilter/nft_tproxy.c +++ b/net/netfilter/nft_tproxy.c @@ -313,8 +313,7 @@ static int nft_tproxy_dump(struct sk_buff *skb, } static int nft_tproxy_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr, - const struct nft_data **data) + const struct nft_expr *expr) { if (ctx->family != NFPROTO_IPV4 && ctx->family != NFPROTO_IPV6 && diff --git a/net/netfilter/nft_tunnel.c b/net/netfilter/nft_tunnel.c index d499eb3f4f297..3e3ae29dde335 100644 --- a/net/netfilter/nft_tunnel.c +++ b/net/netfilter/nft_tunnel.c @@ -617,10 +617,10 @@ static int nft_tunnel_opts_dump(struct sk_buff *skb, struct geneve_opt *opt; int offset = 0; - inner = nla_nest_start_noflag(skb, NFTA_TUNNEL_KEY_OPTS_GENEVE); - if (!inner) - goto failure; while (opts->len > offset) { + inner = nla_nest_start_noflag(skb, NFTA_TUNNEL_KEY_OPTS_GENEVE); + if (!inner) + goto failure; opt = (struct geneve_opt *)(opts->u.data + offset); if (nla_put_be16(skb, NFTA_TUNNEL_KEY_GENEVE_CLASS, opt->opt_class) || @@ -630,8 +630,8 @@ static int nft_tunnel_opts_dump(struct sk_buff *skb, opt->length * 4, opt->opt_data)) goto inner_failure; offset += sizeof(*opt) + opt->length * 4; + nla_nest_end(skb, inner); } - nla_nest_end(skb, inner); } nla_nest_end(skb, nest); return 0; diff --git a/net/netfilter/nft_xfrm.c b/net/netfilter/nft_xfrm.c index 1c866757db552..8a07b46cc8fb7 100644 --- a/net/netfilter/nft_xfrm.c +++ b/net/netfilter/nft_xfrm.c @@ -229,8 +229,7 @@ static int nft_xfrm_get_dump(struct sk_buff *skb, return 0; } -static int nft_xfrm_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, - const struct nft_data **data) +static int nft_xfrm_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { const struct nft_xfrm *priv = nft_expr_priv(expr); unsigned int hooks; diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index e50c23b9c9c41..667329ac7e63d 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -743,7 +743,7 @@ EXPORT_SYMBOL(xt_compat_init_offsets); int xt_compat_match_offset(const struct xt_match *match) { - u_int16_t csize = match->compatsize ? : match->matchsize; + u_int16_t csize = get_xt_match_compatsize(match) ? : match->matchsize; return XT_ALIGN(match->matchsize) - COMPAT_XT_ALIGN(csize); } EXPORT_SYMBOL_GPL(xt_compat_match_offset); @@ -752,6 +752,7 @@ void xt_compat_match_from_user(struct xt_entry_match *m, void **dstptr, unsigned int *size) { const struct xt_match *match = m->u.kernel.match; + compat_from_user_fn_t compat_from_user = get_xt_match_compat_from_user(match); struct compat_xt_entry_match *cm = (struct compat_xt_entry_match *)m; int off = xt_compat_match_offset(match); u_int16_t msize = cm->u.user.match_size; @@ -759,8 +760,8 @@ void xt_compat_match_from_user(struct xt_entry_match *m, void **dstptr, m = *dstptr; memcpy(m, cm, sizeof(*cm)); - if (match->compat_from_user) - match->compat_from_user(m->data, cm->data); + if (compat_from_user) + compat_from_user(m->data, cm->data); else memcpy(m->data, cm->data, msize - sizeof(*cm)); @@ -785,6 +786,7 @@ int xt_compat_match_to_user(const struct xt_entry_match *m, void __user **dstptr, unsigned int *size) { const struct xt_match *match = m->u.kernel.match; + compat_to_user_fn_t compat_to_user = get_xt_match_compat_to_user(match); struct compat_xt_entry_match __user *cm = *dstptr; int off = xt_compat_match_offset(match); u_int16_t msize = m->u.user.match_size - off; @@ -792,8 +794,8 @@ int xt_compat_match_to_user(const struct xt_entry_match *m, if (XT_OBJ_TO_USER(cm, m, match, msize)) return -EFAULT; - if (match->compat_to_user) { - if (match->compat_to_user((void __user *)cm->data, m->data)) + if (compat_to_user) { + if (compat_to_user((void __user *)cm->data, m->data)) return -EFAULT; } else { if (COMPAT_XT_DATA_TO_USER(cm, m, match, msize - sizeof(*cm))) @@ -1123,7 +1125,7 @@ EXPORT_SYMBOL_GPL(xt_copy_counters); #ifdef CONFIG_NETFILTER_XTABLES_COMPAT int xt_compat_target_offset(const struct xt_target *target) { - u_int16_t csize = target->compatsize ? : target->targetsize; + u_int16_t csize = get_xt_target_compatsize(target) ? : target->targetsize; return XT_ALIGN(target->targetsize) - COMPAT_XT_ALIGN(csize); } EXPORT_SYMBOL_GPL(xt_compat_target_offset); @@ -1132,6 +1134,7 @@ void xt_compat_target_from_user(struct xt_entry_target *t, void **dstptr, unsigned int *size) { const struct xt_target *target = t->u.kernel.target; + compat_from_user_fn_t compat_from_user = get_xt_target_compat_from_user(target); struct compat_xt_entry_target *ct = (struct compat_xt_entry_target *)t; int off = xt_compat_target_offset(target); u_int16_t tsize = ct->u.user.target_size; @@ -1139,8 +1142,8 @@ void xt_compat_target_from_user(struct xt_entry_target *t, void **dstptr, t = *dstptr; memcpy(t, ct, sizeof(*ct)); - if (target->compat_from_user) - target->compat_from_user(t->data, ct->data); + if (compat_from_user) + compat_from_user(t->data, ct->data); else memcpy(t->data, ct->data, tsize - sizeof(*ct)); @@ -1159,6 +1162,7 @@ int xt_compat_target_to_user(const struct xt_entry_target *t, void __user **dstptr, unsigned int *size) { const struct xt_target *target = t->u.kernel.target; + compat_to_user_fn_t compat_to_user = get_xt_target_compat_to_user(target); struct compat_xt_entry_target __user *ct = *dstptr; int off = xt_compat_target_offset(target); u_int16_t tsize = t->u.user.target_size - off; @@ -1166,8 +1170,8 @@ int xt_compat_target_to_user(const struct xt_entry_target *t, if (XT_OBJ_TO_USER(ct, t, target, tsize)) return -EFAULT; - if (target->compat_to_user) { - if (target->compat_to_user((void __user *)ct->data, t->data)) + if (compat_to_user) { + if (compat_to_user((void __user *)ct->data, t->data)) return -EFAULT; } else { if (COMPAT_XT_DATA_TO_USER(ct, t, target, tsize - sizeof(*ct))) diff --git a/net/netfilter/xt_limit.c b/net/netfilter/xt_limit.c index 8b4fd27857f2f..6220e10153d3c 100644 --- a/net/netfilter/xt_limit.c +++ b/net/netfilter/xt_limit.c @@ -140,6 +140,20 @@ static void limit_mt_destroy(const struct xt_mtdtor_param *par) kfree(info->master); } +#define DEFINE_LIMIT_MT_REG(compat) \ + { \ + .name = "limit", \ + .revision = 0, \ + .family = NFPROTO_UNSPEC, \ + .match = limit_mt, \ + .checkentry = limit_mt_check, \ + .destroy = limit_mt_destroy, \ + .matchsize = sizeof(struct xt_rateinfo), \ + .usersize = offsetof(struct xt_rateinfo, prev), \ + .me = THIS_MODULE, \ + .has_compat_metadata = compat, \ + } + #ifdef CONFIG_NETFILTER_XTABLES_COMPAT struct compat_xt_rateinfo { u_int32_t avg; @@ -182,25 +196,19 @@ static int limit_mt_compat_to_user(void __user *dst, const void *src) }; return copy_to_user(dst, &cm, sizeof(cm)) ? -EFAULT : 0; } -#endif /* CONFIG_NETFILTER_XTABLES_COMPAT */ -static struct xt_match limit_mt_reg __read_mostly = { - .name = "limit", - .revision = 0, - .family = NFPROTO_UNSPEC, - .match = limit_mt, - .checkentry = limit_mt_check, - .destroy = limit_mt_destroy, - .matchsize = sizeof(struct xt_rateinfo), -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT +static struct compat_xt_match_ext limit_mt_reg_ext __read_mostly = { .compatsize = sizeof(struct compat_xt_rateinfo), .compat_from_user = limit_mt_compat_from_user, .compat_to_user = limit_mt_compat_to_user, -#endif - .usersize = offsetof(struct xt_rateinfo, prev), - .me = THIS_MODULE, + .match = DEFINE_LIMIT_MT_REG(true), }; +#define limit_mt_reg (limit_mt_reg_ext.match) +#else +static struct xt_match limit_mt_reg __read_mostly = DEFINE_LIMIT_MT_REG(false); +#endif /* CONFIG_NETFILTER_XTABLES_COMPAT */ + static int __init limit_mt_init(void) { return xt_register_match(&limit_mt_reg); diff --git a/net/netfilter/xt_nfacct.c b/net/netfilter/xt_nfacct.c index 7c6bf1c168131..0ca1cdfc4095b 100644 --- a/net/netfilter/xt_nfacct.c +++ b/net/netfilter/xt_nfacct.c @@ -38,8 +38,8 @@ nfacct_mt_checkentry(const struct xt_mtchk_param *par) nfacct = nfnl_acct_find_get(par->net, info->name); if (nfacct == NULL) { - pr_info_ratelimited("accounting object `%s' does not exists\n", - info->name); + pr_info_ratelimited("accounting object `%.*s' does not exist\n", + NFACCT_NAME_MAX, info->name); return -ENOENT; } info->nfacct = nfacct; diff --git a/net/netlabel/netlabel_kapi.c b/net/netlabel/netlabel_kapi.c index 27511c90a26f4..daef0eeaea2c7 100644 --- a/net/netlabel/netlabel_kapi.c +++ b/net/netlabel/netlabel_kapi.c @@ -1140,6 +1140,11 @@ int netlbl_conn_setattr(struct sock *sk, break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: + if (sk->sk_family != AF_INET6) { + ret_val = -EAFNOSUPPORT; + goto conn_setattr_return; + } + addr6 = (struct sockaddr_in6 *)addr; entry = netlbl_domhsh_getentry_af6(secattr->domain, &addr6->sin6_addr); diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 7b609284bf78d..dcc895da1a62d 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -387,7 +387,6 @@ static void netlink_skb_set_owner_r(struct sk_buff *skb, struct sock *sk) WARN_ON(skb->sk != NULL); skb->sk = sk; skb->destructor = netlink_skb_destructor; - atomic_add(skb->truesize, &sk->sk_rmem_alloc); sk_mem_charge(sk, skb->truesize); } @@ -803,16 +802,6 @@ static int netlink_release(struct socket *sock) sock_prot_inuse_add(sock_net(sk), &netlink_proto, -1); - /* Because struct net might disappear soon, do not keep a pointer. */ - if (!sk->sk_net_refcnt && sock_net(sk) != &init_net) { - __netns_tracker_free(sock_net(sk), &sk->ns_tracker, false); - /* Because of deferred_put_nlk_sk and use of work queue, - * it is possible netns will be freed before this socket. - */ - sock_net_set(sk, &init_net); - __netns_tracker_alloc(&init_net, &sk->ns_tracker, - false, GFP_KERNEL); - } call_rcu(&nlk->rcu, deferred_put_nlk_sk); return 0; } @@ -1227,41 +1216,48 @@ static struct sk_buff *netlink_alloc_large_skb(unsigned int size, int netlink_attachskb(struct sock *sk, struct sk_buff *skb, long *timeo, struct sock *ssk) { + DECLARE_WAITQUEUE(wait, current); struct netlink_sock *nlk; + unsigned int rmem; nlk = nlk_sk(sk); + rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc); - if ((atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || - test_bit(NETLINK_S_CONGESTED, &nlk->state))) { - DECLARE_WAITQUEUE(wait, current); - if (!*timeo) { - if (!ssk || netlink_is_kernel(ssk)) - netlink_overrun(sk); - sock_put(sk); - kfree_skb(skb); - return -EAGAIN; - } - - __set_current_state(TASK_INTERRUPTIBLE); - add_wait_queue(&nlk->wait, &wait); + if ((rmem == skb->truesize || rmem <= READ_ONCE(sk->sk_rcvbuf)) && + !test_bit(NETLINK_S_CONGESTED, &nlk->state)) { + netlink_skb_set_owner_r(skb, sk); + return 0; + } - if ((atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || - test_bit(NETLINK_S_CONGESTED, &nlk->state)) && - !sock_flag(sk, SOCK_DEAD)) - *timeo = schedule_timeout(*timeo); + atomic_sub(skb->truesize, &sk->sk_rmem_alloc); - __set_current_state(TASK_RUNNING); - remove_wait_queue(&nlk->wait, &wait); + if (!*timeo) { + if (!ssk || netlink_is_kernel(ssk)) + netlink_overrun(sk); sock_put(sk); + kfree_skb(skb); + return -EAGAIN; + } - if (signal_pending(current)) { - kfree_skb(skb); - return sock_intr_errno(*timeo); - } - return 1; + __set_current_state(TASK_INTERRUPTIBLE); + add_wait_queue(&nlk->wait, &wait); + rmem = atomic_read(&sk->sk_rmem_alloc); + + if (((rmem && rmem + skb->truesize > READ_ONCE(sk->sk_rcvbuf)) || + test_bit(NETLINK_S_CONGESTED, &nlk->state)) && + !sock_flag(sk, SOCK_DEAD)) + *timeo = schedule_timeout(*timeo); + + __set_current_state(TASK_RUNNING); + remove_wait_queue(&nlk->wait, &wait); + sock_put(sk); + + if (signal_pending(current)) { + kfree_skb(skb); + return sock_intr_errno(*timeo); } - netlink_skb_set_owner_r(skb, sk); - return 0; + + return 1; } static int __netlink_sendskb(struct sock *sk, struct sk_buff *skb) @@ -1321,6 +1317,7 @@ static int netlink_unicast_kernel(struct sock *sk, struct sk_buff *skb, ret = -ECONNREFUSED; if (nlk->netlink_rcv != NULL) { ret = skb->len; + atomic_add(skb->truesize, &sk->sk_rmem_alloc); netlink_skb_set_owner_r(skb, sk); NETLINK_CB(skb).sk = ssk; netlink_deliver_tap_kernel(sk, ssk, skb); @@ -1397,13 +1394,19 @@ EXPORT_SYMBOL_GPL(netlink_strict_get_check); static int netlink_broadcast_deliver(struct sock *sk, struct sk_buff *skb) { struct netlink_sock *nlk = nlk_sk(sk); + unsigned int rmem, rcvbuf; - if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf && + rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc); + rcvbuf = READ_ONCE(sk->sk_rcvbuf); + + if ((rmem == skb->truesize || rmem <= rcvbuf) && !test_bit(NETLINK_S_CONGESTED, &nlk->state)) { netlink_skb_set_owner_r(skb, sk); __netlink_sendskb(sk, skb); - return atomic_read(&sk->sk_rmem_alloc) > (sk->sk_rcvbuf >> 1); + return rmem > (rcvbuf >> 1); } + + atomic_sub(skb->truesize, &sk->sk_rmem_alloc); return -1; } @@ -2190,6 +2193,7 @@ static int netlink_dump(struct sock *sk, bool lock_taken) struct netlink_ext_ack extack = {}; struct netlink_callback *cb; struct sk_buff *skb = NULL; + unsigned int rmem, rcvbuf; size_t max_recvmsg_len; struct module *module; int err = -ENOBUFS; @@ -2203,9 +2207,6 @@ static int netlink_dump(struct sock *sk, bool lock_taken) goto errout_skb; } - if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) - goto errout_skb; - /* NLMSG_GOODSIZE is small to avoid high order allocations being * required, but it makes sense to _attempt_ a 16K bytes allocation * to reduce number of system calls on dump operations, if user @@ -2228,6 +2229,13 @@ static int netlink_dump(struct sock *sk, bool lock_taken) if (!skb) goto errout_skb; + rcvbuf = READ_ONCE(sk->sk_rcvbuf); + rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc); + if (rmem != skb->truesize && rmem >= rcvbuf) { + atomic_sub(skb->truesize, &sk->sk_rmem_alloc); + goto errout_skb; + } + /* Trim skb to allocated size. User is expected to provide buffer as * large as max(min_dump_alloc, 16KiB (mac_recvmsg_len capped at * netlink_recvmsg())). dump will pack as many smaller messages as diff --git a/net/netlink/policy.c b/net/netlink/policy.c index 87e3de0fde896..ef542a142b980 100644 --- a/net/netlink/policy.c +++ b/net/netlink/policy.c @@ -229,6 +229,8 @@ int netlink_policy_dump_attr_size_estimate(const struct nla_policy *pt) case NLA_S16: case NLA_S32: case NLA_S64: + case NLA_SINT: + case NLA_UINT: /* maximum is common, u64 min/max with padding */ return common + 2 * (nla_attr_size(0) + nla_attr_size(sizeof(u64))); @@ -287,6 +289,7 @@ __netlink_policy_dump_write_attr(struct netlink_policy_dump_state *state, case NLA_U16: case NLA_U32: case NLA_U64: + case NLA_UINT: case NLA_MSECS: { struct netlink_range_validation range; @@ -296,8 +299,10 @@ __netlink_policy_dump_write_attr(struct netlink_policy_dump_state *state, type = NL_ATTR_TYPE_U16; else if (pt->type == NLA_U32) type = NL_ATTR_TYPE_U32; - else + else if (pt->type == NLA_U64) type = NL_ATTR_TYPE_U64; + else + type = NL_ATTR_TYPE_UINT; if (pt->validation_type == NLA_VALIDATE_MASK) { if (nla_put_u64_64bit(skb, NL_POLICY_TYPE_ATTR_MASK, @@ -319,7 +324,8 @@ __netlink_policy_dump_write_attr(struct netlink_policy_dump_state *state, case NLA_S8: case NLA_S16: case NLA_S32: - case NLA_S64: { + case NLA_S64: + case NLA_SINT: { struct netlink_range_validation_signed range; if (pt->type == NLA_S8) @@ -328,8 +334,10 @@ __netlink_policy_dump_write_attr(struct netlink_policy_dump_state *state, type = NL_ATTR_TYPE_S16; else if (pt->type == NLA_S32) type = NL_ATTR_TYPE_S32; - else + else if (pt->type == NLA_S64) type = NL_ATTR_TYPE_S64; + else + type = NL_ATTR_TYPE_SINT; nla_get_range_signed(pt, &range); diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index 994a0a1efb589..cb2a672105dc1 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -27,11 +27,16 @@ /* Handle NCI Notification packets */ -static void nci_core_reset_ntf_packet(struct nci_dev *ndev, - const struct sk_buff *skb) +static int nci_core_reset_ntf_packet(struct nci_dev *ndev, + const struct sk_buff *skb) { /* Handle NCI 2.x core reset notification */ - const struct nci_core_reset_ntf *ntf = (void *)skb->data; + const struct nci_core_reset_ntf *ntf; + + if (skb->len < sizeof(struct nci_core_reset_ntf)) + return -EINVAL; + + ntf = (struct nci_core_reset_ntf *)skb->data; ndev->nci_ver = ntf->nci_ver; pr_debug("nci_ver 0x%x, config_status 0x%x\n", @@ -42,15 +47,22 @@ static void nci_core_reset_ntf_packet(struct nci_dev *ndev, __le32_to_cpu(ntf->manufact_specific_info); nci_req_complete(ndev, NCI_STATUS_OK); + + return 0; } -static void nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, - struct sk_buff *skb) +static int nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, + struct sk_buff *skb) { - struct nci_core_conn_credit_ntf *ntf = (void *) skb->data; + struct nci_core_conn_credit_ntf *ntf; struct nci_conn_info *conn_info; int i; + if (skb->len < sizeof(struct nci_core_conn_credit_ntf)) + return -EINVAL; + + ntf = (struct nci_core_conn_credit_ntf *)skb->data; + pr_debug("num_entries %d\n", ntf->num_entries); if (ntf->num_entries > NCI_MAX_NUM_CONN) @@ -68,7 +80,7 @@ static void nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, conn_info = nci_get_conn_info_by_conn_id(ndev, ntf->conn_entries[i].conn_id); if (!conn_info) - return; + return 0; atomic_add(ntf->conn_entries[i].credits, &conn_info->credits_cnt); @@ -77,12 +89,19 @@ static void nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, /* trigger the next tx */ if (!skb_queue_empty(&ndev->tx_q)) queue_work(ndev->tx_wq, &ndev->tx_work); + + return 0; } -static void nci_core_generic_error_ntf_packet(struct nci_dev *ndev, - const struct sk_buff *skb) +static int nci_core_generic_error_ntf_packet(struct nci_dev *ndev, + const struct sk_buff *skb) { - __u8 status = skb->data[0]; + __u8 status; + + if (skb->len < 1) + return -EINVAL; + + status = skb->data[0]; pr_debug("status 0x%x\n", status); @@ -91,12 +110,19 @@ static void nci_core_generic_error_ntf_packet(struct nci_dev *ndev, (the state remains the same) */ nci_req_complete(ndev, status); } + + return 0; } -static void nci_core_conn_intf_error_ntf_packet(struct nci_dev *ndev, - struct sk_buff *skb) +static int nci_core_conn_intf_error_ntf_packet(struct nci_dev *ndev, + struct sk_buff *skb) { - struct nci_core_intf_error_ntf *ntf = (void *) skb->data; + struct nci_core_intf_error_ntf *ntf; + + if (skb->len < sizeof(struct nci_core_intf_error_ntf)) + return -EINVAL; + + ntf = (struct nci_core_intf_error_ntf *)skb->data; ntf->conn_id = nci_conn_id(&ntf->conn_id); @@ -105,6 +131,8 @@ static void nci_core_conn_intf_error_ntf_packet(struct nci_dev *ndev, /* complete the data exchange transaction, if exists */ if (test_bit(NCI_DATA_EXCHANGE, &ndev->flags)) nci_data_exchange_complete(ndev, NULL, ntf->conn_id, -EIO); + + return 0; } static const __u8 * @@ -329,13 +357,18 @@ void nci_clear_target_list(struct nci_dev *ndev) ndev->n_targets = 0; } -static void nci_rf_discover_ntf_packet(struct nci_dev *ndev, - const struct sk_buff *skb) +static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, + const struct sk_buff *skb) { struct nci_rf_discover_ntf ntf; - const __u8 *data = skb->data; + const __u8 *data; bool add_target = true; + if (skb->len < sizeof(struct nci_rf_discover_ntf)) + return -EINVAL; + + data = skb->data; + ntf.rf_discovery_id = *data++; ntf.rf_protocol = *data++; ntf.rf_tech_and_mode = *data++; @@ -390,6 +423,8 @@ static void nci_rf_discover_ntf_packet(struct nci_dev *ndev, nfc_targets_found(ndev->nfc_dev, ndev->targets, ndev->n_targets); } + + return 0; } static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, @@ -531,14 +566,19 @@ static int nci_store_general_bytes_nfc_dep(struct nci_dev *ndev, return NCI_STATUS_OK; } -static void nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, - const struct sk_buff *skb) +static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, + const struct sk_buff *skb) { struct nci_conn_info *conn_info; struct nci_rf_intf_activated_ntf ntf; - const __u8 *data = skb->data; + const __u8 *data; int err = NCI_STATUS_OK; + if (skb->len < sizeof(struct nci_rf_intf_activated_ntf)) + return -EINVAL; + + data = skb->data; + ntf.rf_discovery_id = *data++; ntf.rf_interface = *data++; ntf.rf_protocol = *data++; @@ -645,7 +685,7 @@ static void nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, if (err == NCI_STATUS_OK) { conn_info = ndev->rf_conn_info; if (!conn_info) - return; + return 0; conn_info->max_pkt_payload_len = ntf.max_data_pkt_payload_size; conn_info->initial_num_credits = ntf.initial_num_credits; @@ -691,19 +731,26 @@ static void nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, pr_err("error when signaling tm activation\n"); } } + + return 0; } -static void nci_rf_deactivate_ntf_packet(struct nci_dev *ndev, - const struct sk_buff *skb) +static int nci_rf_deactivate_ntf_packet(struct nci_dev *ndev, + const struct sk_buff *skb) { const struct nci_conn_info *conn_info; - const struct nci_rf_deactivate_ntf *ntf = (void *)skb->data; + const struct nci_rf_deactivate_ntf *ntf; + + if (skb->len < sizeof(struct nci_rf_deactivate_ntf)) + return -EINVAL; + + ntf = (struct nci_rf_deactivate_ntf *)skb->data; pr_debug("entry, type 0x%x, reason 0x%x\n", ntf->type, ntf->reason); conn_info = ndev->rf_conn_info; if (!conn_info) - return; + return 0; /* drop tx data queue */ skb_queue_purge(&ndev->tx_q); @@ -735,14 +782,20 @@ static void nci_rf_deactivate_ntf_packet(struct nci_dev *ndev, } nci_req_complete(ndev, NCI_STATUS_OK); + + return 0; } -static void nci_nfcee_discover_ntf_packet(struct nci_dev *ndev, - const struct sk_buff *skb) +static int nci_nfcee_discover_ntf_packet(struct nci_dev *ndev, + const struct sk_buff *skb) { u8 status = NCI_STATUS_OK; - const struct nci_nfcee_discover_ntf *nfcee_ntf = - (struct nci_nfcee_discover_ntf *)skb->data; + const struct nci_nfcee_discover_ntf *nfcee_ntf; + + if (skb->len < sizeof(struct nci_nfcee_discover_ntf)) + return -EINVAL; + + nfcee_ntf = (struct nci_nfcee_discover_ntf *)skb->data; /* NFCForum NCI 9.2.1 HCI Network Specific Handling * If the NFCC supports the HCI Network, it SHALL return one, @@ -753,6 +806,8 @@ static void nci_nfcee_discover_ntf_packet(struct nci_dev *ndev, ndev->cur_params.id = nfcee_ntf->nfcee_id; nci_req_complete(ndev, status); + + return 0; } void nci_ntf_packet(struct nci_dev *ndev, struct sk_buff *skb) @@ -779,35 +834,43 @@ void nci_ntf_packet(struct nci_dev *ndev, struct sk_buff *skb) switch (ntf_opcode) { case NCI_OP_CORE_RESET_NTF: - nci_core_reset_ntf_packet(ndev, skb); + if (nci_core_reset_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_CORE_CONN_CREDITS_NTF: - nci_core_conn_credits_ntf_packet(ndev, skb); + if (nci_core_conn_credits_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_CORE_GENERIC_ERROR_NTF: - nci_core_generic_error_ntf_packet(ndev, skb); + if (nci_core_generic_error_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_CORE_INTF_ERROR_NTF: - nci_core_conn_intf_error_ntf_packet(ndev, skb); + if (nci_core_conn_intf_error_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_RF_DISCOVER_NTF: - nci_rf_discover_ntf_packet(ndev, skb); + if (nci_rf_discover_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_RF_INTF_ACTIVATED_NTF: - nci_rf_intf_activated_ntf_packet(ndev, skb); + if (nci_rf_intf_activated_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_RF_DEACTIVATE_NTF: - nci_rf_deactivate_ntf_packet(ndev, skb); + if (nci_rf_deactivate_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_NFCEE_DISCOVER_NTF: - nci_nfcee_discover_ntf_packet(ndev, skb); + if (nci_nfcee_discover_ntf_packet(ndev, skb)) + goto end; break; case NCI_OP_RF_NFCEE_ACTION_NTF: diff --git a/net/nfc/nci/uart.c b/net/nfc/nci/uart.c index ed1508a9e093e..aab107727f186 100644 --- a/net/nfc/nci/uart.c +++ b/net/nfc/nci/uart.c @@ -119,22 +119,22 @@ static int nci_uart_set_driver(struct tty_struct *tty, unsigned int driver) memcpy(nu, nci_uart_drivers[driver], sizeof(struct nci_uart)); nu->tty = tty; - tty->disc_data = nu; skb_queue_head_init(&nu->tx_q); INIT_WORK(&nu->write_work, nci_uart_write_work); spin_lock_init(&nu->rx_lock); ret = nu->ops.open(nu); if (ret) { - tty->disc_data = NULL; kfree(nu); + return ret; } else if (!try_module_get(nu->owner)) { nu->ops.close(nu); - tty->disc_data = NULL; kfree(nu); return -ENOENT; } - return ret; + tty->disc_data = nu; + + return 0; } /* ------ LDISC part ------ */ diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c index 6c5afb4ad67bb..0ea4fc2a755bf 100644 --- a/net/openvswitch/actions.c +++ b/net/openvswitch/actions.c @@ -597,69 +597,6 @@ static int set_ipv6(struct sk_buff *skb, struct sw_flow_key *flow_key, return 0; } -static int set_nsh(struct sk_buff *skb, struct sw_flow_key *flow_key, - const struct nlattr *a) -{ - struct nshhdr *nh; - size_t length; - int err; - u8 flags; - u8 ttl; - int i; - - struct ovs_key_nsh key; - struct ovs_key_nsh mask; - - err = nsh_key_from_nlattr(a, &key, &mask); - if (err) - return err; - - /* Make sure the NSH base header is there */ - if (!pskb_may_pull(skb, skb_network_offset(skb) + NSH_BASE_HDR_LEN)) - return -ENOMEM; - - nh = nsh_hdr(skb); - length = nsh_hdr_len(nh); - - /* Make sure the whole NSH header is there */ - err = skb_ensure_writable(skb, skb_network_offset(skb) + - length); - if (unlikely(err)) - return err; - - nh = nsh_hdr(skb); - skb_postpull_rcsum(skb, nh, length); - flags = nsh_get_flags(nh); - flags = OVS_MASKED(flags, key.base.flags, mask.base.flags); - flow_key->nsh.base.flags = flags; - ttl = nsh_get_ttl(nh); - ttl = OVS_MASKED(ttl, key.base.ttl, mask.base.ttl); - flow_key->nsh.base.ttl = ttl; - nsh_set_flags_and_ttl(nh, flags, ttl); - nh->path_hdr = OVS_MASKED(nh->path_hdr, key.base.path_hdr, - mask.base.path_hdr); - flow_key->nsh.base.path_hdr = nh->path_hdr; - switch (nh->mdtype) { - case NSH_M_TYPE1: - for (i = 0; i < NSH_MD1_CONTEXT_SIZE; i++) { - nh->md1.context[i] = - OVS_MASKED(nh->md1.context[i], key.context[i], - mask.context[i]); - } - memcpy(flow_key->nsh.context, nh->md1.context, - sizeof(nh->md1.context)); - break; - case NSH_M_TYPE2: - memset(flow_key->nsh.context, 0, - sizeof(flow_key->nsh.context)); - break; - default: - return -EINVAL; - } - skb_postpush_rcsum(skb, nh, length); - return 0; -} - /* Must follow skb_ensure_writable() since that can move the skb data. */ static void set_tp_port(struct sk_buff *skb, __be16 *port, __be16 new_port, __sum16 *check) @@ -959,8 +896,7 @@ static int output_userspace(struct datapath *dp, struct sk_buff *skb, upcall.cmd = OVS_PACKET_CMD_ACTION; upcall.mru = OVS_CB(skb)->mru; - for (a = nla_data(attr), rem = nla_len(attr); rem > 0; - a = nla_next(a, &rem)) { + nla_for_each_nested(a, attr, rem) { switch (nla_type(a)) { case OVS_USERSPACE_ATTR_USERDATA: upcall.userdata = a; @@ -1144,10 +1080,6 @@ static int execute_masked_set_action(struct sk_buff *skb, get_mask(a, struct ovs_key_ethernet *)); break; - case OVS_KEY_ATTR_NSH: - err = set_nsh(skb, flow_key, a); - break; - case OVS_KEY_ATTR_IPV4: err = set_ipv4(skb, flow_key, nla_data(a), get_mask(a, struct ovs_key_ipv4 *)); @@ -1184,6 +1116,7 @@ static int execute_masked_set_action(struct sk_buff *skb, case OVS_KEY_ATTR_CT_LABELS: case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4: case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6: + case OVS_KEY_ATTR_NSH: err = -EINVAL; break; } diff --git a/net/openvswitch/flow.c b/net/openvswitch/flow.c index 8a848ce72e291..b80bd3a907739 100644 --- a/net/openvswitch/flow.c +++ b/net/openvswitch/flow.c @@ -788,7 +788,7 @@ static int key_extract_l3l4(struct sk_buff *skb, struct sw_flow_key *key) memset(&key->ipv4, 0, sizeof(key->ipv4)); } } else if (eth_p_mpls(key->eth.type)) { - u8 label_count = 1; + size_t label_count = 1; memset(&key->mpls, 0, sizeof(key->mpls)); skb_set_inner_network_header(skb, skb->mac_len); diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c index 089ab1826e1d5..836e8e705d40e 100644 --- a/net/openvswitch/flow_netlink.c +++ b/net/openvswitch/flow_netlink.c @@ -1292,6 +1292,11 @@ static int metadata_from_nlattrs(struct net *net, struct sw_flow_match *match, return 0; } +/* + * Constructs NSH header 'nh' from attributes of OVS_ACTION_ATTR_PUSH_NSH, + * where 'nh' points to a memory block of 'size' bytes. It's assumed that + * attributes were previously validated with validate_push_nsh(). + */ int nsh_hdr_from_nlattr(const struct nlattr *attr, struct nshhdr *nh, size_t size) { @@ -1301,8 +1306,6 @@ int nsh_hdr_from_nlattr(const struct nlattr *attr, u8 ttl = 0; int mdlen = 0; - /* validate_nsh has check this, so we needn't do duplicate check here - */ if (size < NSH_BASE_HDR_LEN) return -ENOBUFS; @@ -1346,46 +1349,6 @@ int nsh_hdr_from_nlattr(const struct nlattr *attr, return 0; } -int nsh_key_from_nlattr(const struct nlattr *attr, - struct ovs_key_nsh *nsh, struct ovs_key_nsh *nsh_mask) -{ - struct nlattr *a; - int rem; - - /* validate_nsh has check this, so we needn't do duplicate check here - */ - nla_for_each_nested(a, attr, rem) { - int type = nla_type(a); - - switch (type) { - case OVS_NSH_KEY_ATTR_BASE: { - const struct ovs_nsh_key_base *base = nla_data(a); - const struct ovs_nsh_key_base *base_mask = base + 1; - - nsh->base = *base; - nsh_mask->base = *base_mask; - break; - } - case OVS_NSH_KEY_ATTR_MD1: { - const struct ovs_nsh_key_md1 *md1 = nla_data(a); - const struct ovs_nsh_key_md1 *md1_mask = md1 + 1; - - memcpy(nsh->context, md1->context, sizeof(*md1)); - memcpy(nsh_mask->context, md1_mask->context, - sizeof(*md1_mask)); - break; - } - case OVS_NSH_KEY_ATTR_MD2: - /* Not supported yet */ - return -ENOTSUPP; - default: - return -EINVAL; - } - } - - return 0; -} - static int nsh_key_put_from_nlattr(const struct nlattr *attr, struct sw_flow_match *match, bool is_mask, bool is_push_nsh, bool log) @@ -2825,17 +2788,13 @@ static int validate_and_copy_set_tun(const struct nlattr *attr, return err; } -static bool validate_nsh(const struct nlattr *attr, bool is_mask, - bool is_push_nsh, bool log) +static bool validate_push_nsh(const struct nlattr *attr, bool log) { struct sw_flow_match match; struct sw_flow_key key; - int ret = 0; ovs_match_init(&match, &key, true, NULL); - ret = nsh_key_put_from_nlattr(attr, &match, is_mask, - is_push_nsh, log); - return !ret; + return !nsh_key_put_from_nlattr(attr, &match, false, true, log); } /* Return false if there are any non-masked bits set. @@ -2983,13 +2942,6 @@ static int validate_set(const struct nlattr *a, break; - case OVS_KEY_ATTR_NSH: - if (eth_type != htons(ETH_P_NSH)) - return -EINVAL; - if (!validate_nsh(nla_data(a), masked, false, log)) - return -EINVAL; - break; - default: return -EINVAL; } @@ -3399,7 +3351,7 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr, return -EINVAL; } mac_proto = MAC_PROTO_NONE; - if (!validate_nsh(nla_data(a), false, true, true)) + if (!validate_push_nsh(nla_data(a), log)) return -EINVAL; break; diff --git a/net/openvswitch/flow_netlink.h b/net/openvswitch/flow_netlink.h index fe7f77fc5f189..ff8cdecbe3465 100644 --- a/net/openvswitch/flow_netlink.h +++ b/net/openvswitch/flow_netlink.h @@ -65,8 +65,6 @@ int ovs_nla_put_actions(const struct nlattr *attr, void ovs_nla_free_flow_actions(struct sw_flow_actions *); void ovs_nla_free_flow_actions_rcu(struct sw_flow_actions *); -int nsh_key_from_nlattr(const struct nlattr *attr, struct ovs_key_nsh *nsh, - struct ovs_key_nsh *nsh_mask); int nsh_hdr_from_nlattr(const struct nlattr *attr, struct nshhdr *nh, size_t size); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 4abf7e9ac4f2f..9620f160be70b 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2791,7 +2791,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) int len_sum = 0; int status = TP_STATUS_AVAILABLE; int hlen, tlen, copylen = 0; - long timeo = 0; + long timeo; mutex_lock(&po->pg_vec_lock); @@ -2845,22 +2845,28 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !vnet_hdr_sz) size_max = dev->mtu + reserve + VLAN_HLEN; + timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT); reinit_completion(&po->skb_completion); do { ph = packet_current_frame(po, &po->tx_ring, TP_STATUS_SEND_REQUEST); if (unlikely(ph == NULL)) { - if (need_wait && skb) { - timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT); + /* Note: packet_read_pending() might be slow if we + * have to call it as it's per_cpu variable, but in + * fast-path we don't have to call it, only when ph + * is NULL, we need to check the pending_refcnt. + */ + if (need_wait && packet_read_pending(&po->tx_ring)) { timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo); if (timeo <= 0) { err = !timeo ? -ETIMEDOUT : -ERESTARTSYS; goto out_put; } - } - /* check for additional frames */ - continue; + /* check for additional frames */ + continue; + } else + break; } skb = NULL; @@ -2949,14 +2955,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) } packet_increment_head(&po->tx_ring); len_sum += tp_len; - } while (likely((ph != NULL) || - /* Note: packet_read_pending() might be slow if we have - * to call it as it's per_cpu variable, but in fast-path - * we already short-circuit the loop with the first - * condition, and luckily don't have to go that path - * anyway. - */ - (need_wait && packet_read_pending(&po->tx_ring)))); + } while (1); err = len_sum; goto out_put; @@ -4567,10 +4566,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, spin_lock(&po->bind_lock); was_running = packet_sock_flag(po, PACKET_SOCK_RUNNING); num = po->num; - if (was_running) { - WRITE_ONCE(po->num, 0); + WRITE_ONCE(po->num, 0); + if (was_running) __unregister_prot_hook(sk, false); - } + spin_unlock(&po->bind_lock); synchronize_net(); @@ -4602,10 +4601,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, mutex_unlock(&po->pg_vec_lock); spin_lock(&po->bind_lock); - if (was_running) { - WRITE_ONCE(po->num, num); + WRITE_ONCE(po->num, num); + if (was_running) register_prot_hook(sk); - } + spin_unlock(&po->bind_lock); if (pg_vec && (po->tp_version > TPACKET_V2)) { /* Because we don't support block-based V3 on tx-ring */ diff --git a/net/phonet/pep.c b/net/phonet/pep.c index 3dd5f52bc1b58..e14b2b2d3b5cd 100644 --- a/net/phonet/pep.c +++ b/net/phonet/pep.c @@ -826,6 +826,7 @@ static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp, } /* Check for duplicate pipe handle */ + pn_skb_get_dst_sockaddr(skb, &dst); newsk = pep_find_pipe(&pn->hlist, &dst, pipe_handle); if (unlikely(newsk)) { __sock_put(newsk); @@ -850,7 +851,6 @@ static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp, newsk->sk_destruct = pipe_destruct; newpn = pep_sk(newsk); - pn_skb_get_dst_sockaddr(skb, &dst); pn_skb_get_src_sockaddr(skb, &src); newpn->pn_sk.sobject = pn_sockaddr_get_object(&dst); newpn->pn_sk.dobject = pn_sockaddr_get_object(&src); diff --git a/net/rds/ib_frmr.c b/net/rds/ib_frmr.c index 28c1b00221780..bd861191157b5 100644 --- a/net/rds/ib_frmr.c +++ b/net/rds/ib_frmr.c @@ -133,12 +133,15 @@ static int rds_ib_post_reg_frmr(struct rds_ib_mr *ibmr) ret = ib_map_mr_sg_zbva(frmr->mr, ibmr->sg, ibmr->sg_dma_len, &off, PAGE_SIZE); - if (unlikely(ret != ibmr->sg_dma_len)) - return ret < 0 ? ret : -EINVAL; + if (unlikely(ret != ibmr->sg_dma_len)) { + ret = ret < 0 ? ret : -EINVAL; + goto out_inc; + } - if (cmpxchg(&frmr->fr_state, - FRMR_IS_FREE, FRMR_IS_INUSE) != FRMR_IS_FREE) - return -EBUSY; + if (cmpxchg(&frmr->fr_state, FRMR_IS_FREE, FRMR_IS_INUSE) != FRMR_IS_FREE) { + ret = -EBUSY; + goto out_inc; + } atomic_inc(&ibmr->ic->i_fastreg_inuse_count); @@ -166,11 +169,10 @@ static int rds_ib_post_reg_frmr(struct rds_ib_mr *ibmr) /* Failure here can be because of -ENOMEM as well */ rds_transition_frwr_state(ibmr, FRMR_IS_INUSE, FRMR_IS_STALE); - atomic_inc(&ibmr->ic->i_fastreg_wrs); if (printk_ratelimit()) pr_warn("RDS/IB: %s returned error(%d)\n", __func__, ret); - goto out; + goto out_inc; } /* Wait for the registration to complete in order to prevent an invalid @@ -179,8 +181,10 @@ static int rds_ib_post_reg_frmr(struct rds_ib_mr *ibmr) */ wait_event(frmr->fr_reg_done, !frmr->fr_reg); -out: + return ret; +out_inc: + atomic_inc(&ibmr->ic->i_fastreg_wrs); return ret; } diff --git a/net/rds/rds.h b/net/rds/rds.h index dc360252c5157..5b1c072e2e7ff 100644 --- a/net/rds/rds.h +++ b/net/rds/rds.h @@ -93,7 +93,7 @@ enum { /* Max number of multipaths per RDS connection. Must be a power of 2 */ #define RDS_MPATH_WORKERS 8 -#define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \ +#define RDS_MPATH_HASH(rs, n) (jhash_1word(ntohs((rs)->rs_bound_port), \ (rs)->rs_hash_initval) & ((n) - 1)) #define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr)) diff --git a/net/rds/tcp.c b/net/rds/tcp.c index 2dba7505b4148..985b05f38b674 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -503,12 +503,8 @@ bool rds_tcp_tune(struct socket *sock) release_sock(sk); return false; } - /* Update ns_tracker to current stack trace and refcounted tracker */ - __netns_tracker_free(net, &sk->ns_tracker, false); - - sk->sk_net_refcnt = 1; - netns_tracker_alloc(net, &sk->ns_tracker, GFP_KERNEL); - sock_inuse_add(net, 1); + sk_net_refcnt_upgrade(sk); + put_net(net); } rtn = net_generic(net, rds_tcp_netid); if (rtn->sndbuf_size > 0) { diff --git a/net/rfkill/rfkill-gpio.c b/net/rfkill/rfkill-gpio.c index b12edbe0ef45c..badc71acbbdd8 100644 --- a/net/rfkill/rfkill-gpio.c +++ b/net/rfkill/rfkill-gpio.c @@ -79,10 +79,10 @@ static int rfkill_gpio_acpi_probe(struct device *dev, static int rfkill_gpio_probe(struct platform_device *pdev) { struct rfkill_gpio_data *rfkill; - struct gpio_desc *gpio; + const char *type_name = NULL; const char *name_property; const char *type_property; - const char *type_name; + struct gpio_desc *gpio; int ret; rfkill = devm_kzalloc(&pdev->dev, sizeof(*rfkill), GFP_KERNEL); diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index 66e9ceaaa43a1..614695444b6ac 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -170,7 +170,7 @@ void rose_kill_by_neigh(struct rose_neigh *neigh) if (rose->neighbour == neigh) { rose_disconnect(s, ENETUNREACH, ROSE_OUT_OF_ORDER, 0); - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); rose->neighbour = NULL; } } @@ -212,7 +212,7 @@ static void rose_kill_by_device(struct net_device *dev) if (rose->device == dev) { rose_disconnect(sk, ENETUNREACH, ROSE_OUT_OF_ORDER, 0); if (rose->neighbour) - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); netdev_put(rose->device, &rose->dev_tracker); rose->device = NULL; } @@ -655,7 +655,7 @@ static int rose_release(struct socket *sock) break; case ROSE_STATE_2: - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); release_sock(sk); rose_disconnect(sk, 0, -1, -1); lock_sock(sk); @@ -823,6 +823,7 @@ static int rose_connect(struct socket *sock, struct sockaddr *uaddr, int addr_le rose->lci = rose_new_lci(rose->neighbour); if (!rose->lci) { err = -ENETUNREACH; + rose_neigh_put(rose->neighbour); goto out_release; } @@ -834,12 +835,14 @@ static int rose_connect(struct socket *sock, struct sockaddr *uaddr, int addr_le dev = rose_dev_first(); if (!dev) { err = -ENETUNREACH; + rose_neigh_put(rose->neighbour); goto out_release; } user = ax25_findbyuid(current_euid()); if (!user) { err = -EINVAL; + rose_neigh_put(rose->neighbour); dev_put(dev); goto out_release; } @@ -874,8 +877,6 @@ static int rose_connect(struct socket *sock, struct sockaddr *uaddr, int addr_le rose->state = ROSE_STATE_1; - rose->neighbour->use++; - rose_write_internal(sk, ROSE_CALL_REQUEST); rose_start_heartbeat(sk); rose_start_t1timer(sk); @@ -1077,7 +1078,7 @@ int rose_rx_call_request(struct sk_buff *skb, struct net_device *dev, struct ros GFP_ATOMIC); make_rose->facilities = facilities; - make_rose->neighbour->use++; + rose_neigh_hold(make_rose->neighbour); if (rose_sk(sk)->defer) { make_rose->state = ROSE_STATE_5; diff --git a/net/rose/rose_in.c b/net/rose/rose_in.c index 4d67f36dce1b4..7caae93937ee9 100644 --- a/net/rose/rose_in.c +++ b/net/rose/rose_in.c @@ -56,7 +56,7 @@ static int rose_state1_machine(struct sock *sk, struct sk_buff *skb, int framety case ROSE_CLEAR_REQUEST: rose_write_internal(sk, ROSE_CLEAR_CONFIRMATION); rose_disconnect(sk, ECONNREFUSED, skb->data[3], skb->data[4]); - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); break; default: @@ -79,12 +79,12 @@ static int rose_state2_machine(struct sock *sk, struct sk_buff *skb, int framety case ROSE_CLEAR_REQUEST: rose_write_internal(sk, ROSE_CLEAR_CONFIRMATION); rose_disconnect(sk, 0, skb->data[3], skb->data[4]); - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); break; case ROSE_CLEAR_CONFIRMATION: rose_disconnect(sk, 0, -1, -1); - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); break; default: @@ -120,7 +120,7 @@ static int rose_state3_machine(struct sock *sk, struct sk_buff *skb, int framety case ROSE_CLEAR_REQUEST: rose_write_internal(sk, ROSE_CLEAR_CONFIRMATION); rose_disconnect(sk, 0, skb->data[3], skb->data[4]); - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); break; case ROSE_RR: @@ -233,7 +233,7 @@ static int rose_state4_machine(struct sock *sk, struct sk_buff *skb, int framety case ROSE_CLEAR_REQUEST: rose_write_internal(sk, ROSE_CLEAR_CONFIRMATION); rose_disconnect(sk, 0, skb->data[3], skb->data[4]); - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); break; default: @@ -253,7 +253,7 @@ static int rose_state5_machine(struct sock *sk, struct sk_buff *skb, int framety if (frametype == ROSE_CLEAR_REQUEST) { rose_write_internal(sk, ROSE_CLEAR_CONFIRMATION); rose_disconnect(sk, 0, skb->data[3], skb->data[4]); - rose_sk(sk)->neighbour->use--; + rose_neigh_put(rose_sk(sk)->neighbour); } return 0; diff --git a/net/rose/rose_route.c b/net/rose/rose_route.c index fee772b4637c8..28746ae5a2582 100644 --- a/net/rose/rose_route.c +++ b/net/rose/rose_route.c @@ -93,11 +93,11 @@ static int __must_check rose_add_node(struct rose_route_struct *rose_route, rose_neigh->ax25 = NULL; rose_neigh->dev = dev; rose_neigh->count = 0; - rose_neigh->use = 0; rose_neigh->dce_mode = 0; rose_neigh->loopback = 0; rose_neigh->number = rose_neigh_no++; rose_neigh->restarted = 0; + refcount_set(&rose_neigh->use, 1); skb_queue_head_init(&rose_neigh->queue); @@ -178,6 +178,7 @@ static int __must_check rose_add_node(struct rose_route_struct *rose_route, } } rose_neigh->count++; + rose_neigh_hold(rose_neigh); goto out; } @@ -187,6 +188,7 @@ static int __must_check rose_add_node(struct rose_route_struct *rose_route, rose_node->neighbour[rose_node->count] = rose_neigh; rose_node->count++; rose_neigh->count++; + rose_neigh_hold(rose_neigh); } out: @@ -234,20 +236,12 @@ static void rose_remove_neigh(struct rose_neigh *rose_neigh) if ((s = rose_neigh_list) == rose_neigh) { rose_neigh_list = rose_neigh->next; - if (rose_neigh->ax25) - ax25_cb_put(rose_neigh->ax25); - kfree(rose_neigh->digipeat); - kfree(rose_neigh); return; } while (s != NULL && s->next != NULL) { if (s->next == rose_neigh) { s->next = rose_neigh->next; - if (rose_neigh->ax25) - ax25_cb_put(rose_neigh->ax25); - kfree(rose_neigh->digipeat); - kfree(rose_neigh); return; } @@ -263,10 +257,10 @@ static void rose_remove_route(struct rose_route *rose_route) struct rose_route *s; if (rose_route->neigh1 != NULL) - rose_route->neigh1->use--; + rose_neigh_put(rose_route->neigh1); if (rose_route->neigh2 != NULL) - rose_route->neigh2->use--; + rose_neigh_put(rose_route->neigh2); if ((s = rose_route_list) == rose_route) { rose_route_list = rose_route->next; @@ -330,9 +324,12 @@ static int rose_del_node(struct rose_route_struct *rose_route, for (i = 0; i < rose_node->count; i++) { if (rose_node->neighbour[i] == rose_neigh) { rose_neigh->count--; + rose_neigh_put(rose_neigh); - if (rose_neigh->count == 0 && rose_neigh->use == 0) + if (rose_neigh->count == 0) { rose_remove_neigh(rose_neigh); + rose_neigh_put(rose_neigh); + } rose_node->count--; @@ -381,11 +378,11 @@ void rose_add_loopback_neigh(void) sn->ax25 = NULL; sn->dev = NULL; sn->count = 0; - sn->use = 0; sn->dce_mode = 1; sn->loopback = 1; sn->number = rose_neigh_no++; sn->restarted = 1; + refcount_set(&sn->use, 1); skb_queue_head_init(&sn->queue); @@ -436,6 +433,7 @@ int rose_add_loopback_node(const rose_address *address) rose_node_list = rose_node; rose_loopback_neigh->count++; + rose_neigh_hold(rose_loopback_neigh); out: spin_unlock_bh(&rose_node_list_lock); @@ -467,6 +465,7 @@ void rose_del_loopback_node(const rose_address *address) rose_remove_node(rose_node); rose_loopback_neigh->count--; + rose_neigh_put(rose_loopback_neigh); out: spin_unlock_bh(&rose_node_list_lock); @@ -497,22 +496,16 @@ void rose_rt_device_down(struct net_device *dev) t = rose_node; rose_node = rose_node->next; - for (i = 0; i < t->count; i++) { + for (i = t->count - 1; i >= 0; i--) { if (t->neighbour[i] != s) continue; t->count--; - switch (i) { - case 0: - t->neighbour[0] = t->neighbour[1]; - fallthrough; - case 1: - t->neighbour[1] = t->neighbour[2]; - break; - case 2: - break; - } + memmove(&t->neighbour[i], &t->neighbour[i + 1], + sizeof(t->neighbour[0]) * + (t->count - i)); + rose_neigh_put(s); } if (t->count <= 0) @@ -520,6 +513,7 @@ void rose_rt_device_down(struct net_device *dev) } rose_remove_neigh(s); + rose_neigh_put(s); } spin_unlock_bh(&rose_neigh_list_lock); spin_unlock_bh(&rose_node_list_lock); @@ -555,6 +549,7 @@ static int rose_clear_routes(void) { struct rose_neigh *s, *rose_neigh; struct rose_node *t, *rose_node; + int i; spin_lock_bh(&rose_node_list_lock); spin_lock_bh(&rose_neigh_list_lock); @@ -565,17 +560,21 @@ static int rose_clear_routes(void) while (rose_node != NULL) { t = rose_node; rose_node = rose_node->next; - if (!t->loopback) + + if (!t->loopback) { + for (i = 0; i < t->count; i++) + rose_neigh_put(t->neighbour[i]); rose_remove_node(t); + } } while (rose_neigh != NULL) { s = rose_neigh; rose_neigh = rose_neigh->next; - if (s->use == 0 && !s->loopback) { - s->count = 0; + if (!s->loopback) { rose_remove_neigh(s); + rose_neigh_put(s); } } @@ -691,6 +690,7 @@ struct rose_neigh *rose_get_neigh(rose_address *addr, unsigned char *cause, for (i = 0; i < node->count; i++) { if (node->neighbour[i]->restarted) { res = node->neighbour[i]; + rose_neigh_hold(node->neighbour[i]); goto out; } } @@ -702,6 +702,7 @@ struct rose_neigh *rose_get_neigh(rose_address *addr, unsigned char *cause, for (i = 0; i < node->count; i++) { if (!rose_ftimer_running(node->neighbour[i])) { res = node->neighbour[i]; + rose_neigh_hold(node->neighbour[i]); goto out; } failed = 1; @@ -791,13 +792,13 @@ static void rose_del_route_by_neigh(struct rose_neigh *rose_neigh) } if (rose_route->neigh1 == rose_neigh) { - rose_route->neigh1->use--; + rose_neigh_put(rose_route->neigh1); rose_route->neigh1 = NULL; rose_transmit_clear_request(rose_route->neigh2, rose_route->lci2, ROSE_OUT_OF_ORDER, 0); } if (rose_route->neigh2 == rose_neigh) { - rose_route->neigh2->use--; + rose_neigh_put(rose_route->neigh2); rose_route->neigh2 = NULL; rose_transmit_clear_request(rose_route->neigh1, rose_route->lci1, ROSE_OUT_OF_ORDER, 0); } @@ -926,7 +927,7 @@ int rose_route_frame(struct sk_buff *skb, ax25_cb *ax25) rose_clear_queues(sk); rose->cause = ROSE_NETWORK_CONGESTION; rose->diagnostic = 0; - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); rose->neighbour = NULL; rose->lci = 0; rose->state = ROSE_STATE_0; @@ -1051,12 +1052,12 @@ int rose_route_frame(struct sk_buff *skb, ax25_cb *ax25) if ((new_lci = rose_new_lci(new_neigh)) == 0) { rose_transmit_clear_request(rose_neigh, lci, ROSE_NETWORK_CONGESTION, 71); - goto out; + goto put_neigh; } if ((rose_route = kmalloc(sizeof(*rose_route), GFP_ATOMIC)) == NULL) { rose_transmit_clear_request(rose_neigh, lci, ROSE_NETWORK_CONGESTION, 120); - goto out; + goto put_neigh; } rose_route->lci1 = lci; @@ -1069,8 +1070,8 @@ int rose_route_frame(struct sk_buff *skb, ax25_cb *ax25) rose_route->lci2 = new_lci; rose_route->neigh2 = new_neigh; - rose_route->neigh1->use++; - rose_route->neigh2->use++; + rose_neigh_hold(rose_route->neigh1); + rose_neigh_hold(rose_route->neigh2); rose_route->next = rose_route_list; rose_route_list = rose_route; @@ -1082,6 +1083,8 @@ int rose_route_frame(struct sk_buff *skb, ax25_cb *ax25) rose_transmit_link(skb, rose_route->neigh2); res = 1; +put_neigh: + rose_neigh_put(new_neigh); out: spin_unlock_bh(&rose_route_list_lock); spin_unlock_bh(&rose_neigh_list_lock); @@ -1197,7 +1200,7 @@ static int rose_neigh_show(struct seq_file *seq, void *v) (rose_neigh->loopback) ? "RSLOOP-0" : ax2asc(buf, &rose_neigh->callsign), rose_neigh->dev ? rose_neigh->dev->name : "???", rose_neigh->count, - rose_neigh->use, + refcount_read(&rose_neigh->use) - rose_neigh->count - 1, (rose_neigh->dce_mode) ? "DCE" : "DTE", (rose_neigh->restarted) ? "yes" : "no", ax25_display_timer(&rose_neigh->t0timer) / HZ, @@ -1302,18 +1305,22 @@ void __exit rose_rt_free(void) struct rose_neigh *s, *rose_neigh = rose_neigh_list; struct rose_node *t, *rose_node = rose_node_list; struct rose_route *u, *rose_route = rose_route_list; + int i; while (rose_neigh != NULL) { s = rose_neigh; rose_neigh = rose_neigh->next; rose_remove_neigh(s); + rose_neigh_put(s); } while (rose_node != NULL) { t = rose_node; rose_node = rose_node->next; + for (i = 0; i < t->count; i++) + rose_neigh_put(t->neighbour[i]); rose_remove_node(t); } diff --git a/net/rose/rose_timer.c b/net/rose/rose_timer.c index 1525773e94aa1..c52d7d20c5199 100644 --- a/net/rose/rose_timer.c +++ b/net/rose/rose_timer.c @@ -180,7 +180,7 @@ static void rose_timer_expiry(struct timer_list *t) break; case ROSE_STATE_2: /* T3 */ - rose->neighbour->use--; + rose_neigh_put(rose->neighbour); rose_disconnect(sk, ETIMEDOUT, -1, -1); break; diff --git a/net/rxrpc/call_accept.c b/net/rxrpc/call_accept.c index 0f5a1d77b890f..37ac8a6656786 100644 --- a/net/rxrpc/call_accept.c +++ b/net/rxrpc/call_accept.c @@ -149,6 +149,7 @@ static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx, id_in_use: write_unlock(&rx->call_lock); + rxrpc_prefail_call(call, RXRPC_CALL_LOCAL_ERROR, -EBADSLT); rxrpc_cleanup_call(call); _leave(" = -EBADSLT"); return -EBADSLT; @@ -218,6 +219,7 @@ void rxrpc_discard_prealloc(struct rxrpc_sock *rx) tail = b->call_backlog_tail; while (CIRC_CNT(head, tail, size) > 0) { struct rxrpc_call *call = b->call_backlog[tail]; + rxrpc_see_call(call, rxrpc_call_see_discard); rcu_assign_pointer(call->socket, rx); if (rx->discard_new_call) { _debug("discard %lx", call->user_call_ID); @@ -253,6 +255,9 @@ static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx, unsigned short call_tail, conn_tail, peer_tail; unsigned short call_count, conn_count; + if (!b) + return NULL; + /* #calls >= #conns >= #peers must hold true. */ call_head = smp_load_acquire(&b->call_backlog_head); call_tail = b->call_backlog_tail; diff --git a/net/rxrpc/output.c b/net/rxrpc/output.c index cad6a7d18e040..4bbb27a48bd8a 100644 --- a/net/rxrpc/output.c +++ b/net/rxrpc/output.c @@ -589,6 +589,9 @@ void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb) __be32 code; int ret, ioc; + if (sp->hdr.type == RXRPC_PACKET_TYPE_ABORT) + return; /* Never abort an abort. */ + rxrpc_see_skb(skb, rxrpc_skb_see_reject); iov[0].iov_base = &whdr; diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c index a482f88c5fc5b..e24a44bae9a32 100644 --- a/net/rxrpc/recvmsg.c +++ b/net/rxrpc/recvmsg.c @@ -351,6 +351,16 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, goto try_again; } + rxrpc_see_call(call, rxrpc_call_see_recvmsg); + if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) { + rxrpc_see_call(call, rxrpc_call_see_already_released); + list_del_init(&call->recvmsg_link); + spin_unlock_irq(&rx->recvmsg_lock); + release_sock(&rx->sk); + trace_rxrpc_recvmsg(call->debug_id, rxrpc_recvmsg_unqueue, 0); + rxrpc_put_call(call, rxrpc_call_put_recvmsg); + goto try_again; + } if (!(flags & MSG_PEEK)) list_del_init(&call->recvmsg_link); else @@ -374,8 +384,13 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, release_sock(&rx->sk); - if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) - BUG(); + if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) { + rxrpc_see_call(call, rxrpc_call_see_already_released); + mutex_unlock(&call->user_mutex); + if (!(flags & MSG_PEEK)) + rxrpc_put_call(call, rxrpc_call_put_recvmsg); + goto try_again; + } if (test_bit(RXRPC_CALL_HAS_USERID, &call->flags)) { if (flags & MSG_CMSG_COMPAT) { diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index b0455fda7d0ba..223cc157312ab 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -47,12 +47,10 @@ TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb, filter = rcu_dereference(prog->filter); if (at_ingress) { __skb_push(skb, skb->mac_len); - bpf_compute_data_pointers(skb); - filter_res = bpf_prog_run(filter, skb); + filter_res = bpf_prog_run_data_pointers(filter, skb); __skb_pull(skb, skb->mac_len); } else { - bpf_compute_data_pointers(skb); - filter_res = bpf_prog_run(filter, skb); + filter_res = bpf_prog_run_data_pointers(filter, skb); } if (unlikely(!skb->tstamp && skb->mono_delivery_time)) skb->mono_delivery_time = 0; diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c index 0d7aee8933c5f..d90a2fa5966b7 100644 --- a/net/sched/act_connmark.c +++ b/net/sched/act_connmark.c @@ -88,7 +88,7 @@ TC_INDIRECT_SCOPE int tcf_connmark_act(struct sk_buff *skb, /* using overlimits stats to count how many packets marked */ tcf_action_inc_overlimit_qstats(&ca->common); out: - return READ_ONCE(ca->tcf_action); + return parms->action; } static const struct nla_policy connmark_policy[TCA_CONNMARK_MAX + 1] = { @@ -167,6 +167,8 @@ static int tcf_connmark_init(struct net *net, struct nlattr *nla, if (err < 0) goto release_idr; + nparms->action = parm->action; + spin_lock_bh(&ci->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); oparms = rcu_replace_pointer(ci->parms, nparms, lockdep_is_held(&ci->tcf_lock)); @@ -190,20 +192,22 @@ static int tcf_connmark_init(struct net *net, struct nlattr *nla, static inline int tcf_connmark_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { + const struct tcf_connmark_info *ci = to_connmark(a); unsigned char *b = skb_tail_pointer(skb); - struct tcf_connmark_info *ci = to_connmark(a); - struct tc_connmark opt = { - .index = ci->tcf_index, - .refcnt = refcount_read(&ci->tcf_refcnt) - ref, - .bindcnt = atomic_read(&ci->tcf_bindcnt) - bind, - }; - struct tcf_connmark_parms *parms; + const struct tcf_connmark_parms *parms; + struct tc_connmark opt; struct tcf_t t; - spin_lock_bh(&ci->tcf_lock); - parms = rcu_dereference_protected(ci->parms, lockdep_is_held(&ci->tcf_lock)); + memset(&opt, 0, sizeof(opt)); - opt.action = ci->tcf_action; + opt.index = ci->tcf_index; + opt.refcnt = refcount_read(&ci->tcf_refcnt) - ref; + opt.bindcnt = atomic_read(&ci->tcf_bindcnt) - bind; + + rcu_read_lock(); + parms = rcu_dereference(ci->parms); + + opt.action = parms->action; opt.zone = parms->zone; if (nla_put(skb, TCA_CONNMARK_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -212,12 +216,12 @@ static inline int tcf_connmark_dump(struct sk_buff *skb, struct tc_action *a, if (nla_put_64bit(skb, TCA_CONNMARK_TM, sizeof(t), &t, TCA_CONNMARK_PAD)) goto nla_put_failure; - spin_unlock_bh(&ci->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&ci->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 4d15b6a6169c4..8792244620b9d 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -44,9 +44,9 @@ static void tcf_ctinfo_dscp_set(struct nf_conn *ct, struct tcf_ctinfo *ca, ipv4_change_dsfield(ip_hdr(skb), INET_ECN_MASK, newdscp); - ca->stats_dscp_set++; + atomic64_inc(&ca->stats_dscp_set); } else { - ca->stats_dscp_error++; + atomic64_inc(&ca->stats_dscp_error); } } break; @@ -57,9 +57,9 @@ static void tcf_ctinfo_dscp_set(struct nf_conn *ct, struct tcf_ctinfo *ca, ipv6_change_dsfield(ipv6_hdr(skb), INET_ECN_MASK, newdscp); - ca->stats_dscp_set++; + atomic64_inc(&ca->stats_dscp_set); } else { - ca->stats_dscp_error++; + atomic64_inc(&ca->stats_dscp_error); } } break; @@ -72,7 +72,7 @@ static void tcf_ctinfo_cpmark_set(struct nf_conn *ct, struct tcf_ctinfo *ca, struct tcf_ctinfo_params *cp, struct sk_buff *skb) { - ca->stats_cpmark_set++; + atomic64_inc(&ca->stats_cpmark_set); skb->mark = READ_ONCE(ct->mark) & cp->cpmarkmask; } @@ -323,15 +323,18 @@ static int tcf_ctinfo_dump(struct sk_buff *skb, struct tc_action *a, } if (nla_put_u64_64bit(skb, TCA_CTINFO_STATS_DSCP_SET, - ci->stats_dscp_set, TCA_CTINFO_PAD)) + atomic64_read(&ci->stats_dscp_set), + TCA_CTINFO_PAD)) goto nla_put_failure; if (nla_put_u64_64bit(skb, TCA_CTINFO_STATS_DSCP_ERROR, - ci->stats_dscp_error, TCA_CTINFO_PAD)) + atomic64_read(&ci->stats_dscp_error), + TCA_CTINFO_PAD)) goto nla_put_failure; if (nla_put_u64_64bit(skb, TCA_CTINFO_STATS_CPMARK_SET, - ci->stats_cpmark_set, TCA_CTINFO_PAD)) + atomic64_read(&ci->stats_cpmark_set), + TCA_CTINFO_PAD)) goto nla_put_failure; spin_unlock_bh(&ci->tcf_lock); diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index bc7611b0744c4..431921204f660 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -644,13 +644,15 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, unsigned char *b = skb_tail_pointer(skb); struct tcf_ife_info *ife = to_ife(a); struct tcf_ife_params *p; - struct tc_ife opt = { - .index = ife->tcf_index, - .refcnt = refcount_read(&ife->tcf_refcnt) - ref, - .bindcnt = atomic_read(&ife->tcf_bindcnt) - bind, - }; + struct tc_ife opt; struct tcf_t t; + memset(&opt, 0, sizeof(opt)); + + opt.index = ife->tcf_index, + opt.refcnt = refcount_read(&ife->tcf_refcnt) - ref, + opt.bindcnt = atomic_read(&ife->tcf_bindcnt) - bind, + spin_lock_bh(&ife->tcf_lock); opt.action = ife->tcf_action; p = rcu_dereference_protected(ife->params, diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 382c7a71f81f2..05f718dd09a63 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -97,12 +97,10 @@ TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb, } else if (at_ingress) { /* It is safe to push/pull even if skb_shared() */ __skb_push(skb, skb->mac_len); - bpf_compute_data_pointers(skb); - filter_res = bpf_prog_run(prog->filter, skb); + filter_res = bpf_prog_run_data_pointers(prog->filter, skb); __skb_pull(skb, skb->mac_len); } else { - bpf_compute_data_pointers(skb); - filter_res = bpf_prog_run(prog->filter, skb); + filter_res = bpf_prog_run_data_pointers(prog->filter, skb); } if (unlikely(!skb->tstamp && skb->mono_delivery_time)) skb->mono_delivery_time = 0; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 698ecc727d0c9..9dccad1fd9542 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -334,17 +334,22 @@ struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle) return q; } -static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid) +static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid, + struct netlink_ext_ack *extack) { unsigned long cl; const struct Qdisc_class_ops *cops = p->ops->cl_ops; - if (cops == NULL) - return NULL; + if (cops == NULL) { + NL_SET_ERR_MSG(extack, "Parent qdisc is not classful"); + return ERR_PTR(-EOPNOTSUPP); + } cl = cops->find(p, classid); - if (cl == 0) - return NULL; + if (cl == 0) { + NL_SET_ERR_MSG(extack, "Specified class not found"); + return ERR_PTR(-ENOENT); + } return cops->leaf(p, cl); } @@ -595,16 +600,6 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, } EXPORT_SYMBOL(__qdisc_calculate_pkt_len); -void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc) -{ - if (!(qdisc->flags & TCQ_F_WARN_NONWC)) { - pr_warn("%s: %s qdisc %X: is non-work-conserving?\n", - txt, qdisc->ops->id, qdisc->handle >> 16); - qdisc->flags |= TCQ_F_WARN_NONWC; - } -} -EXPORT_SYMBOL(qdisc_warn_nonwc); - static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) { struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog, @@ -780,15 +775,12 @@ static u32 qdisc_alloc_handle(struct net_device *dev) void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) { - bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED; const struct Qdisc_class_ops *cops; unsigned long cl; u32 parentid; bool notify; int drops; - if (n == 0 && len == 0) - return; drops = max_t(int, n, 0); rcu_read_lock(); while ((parentid = sch->parent)) { @@ -797,17 +789,8 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) if (sch->flags & TCQ_F_NOPARENT) break; - /* Notify parent qdisc only if child qdisc becomes empty. - * - * If child was empty even before update then backlog - * counter is screwed and we skip notification because - * parent class is already passive. - * - * If the original child was offloaded then it is allowed - * to be seem as empty, so the parent is notified anyway. - */ - notify = !sch->q.qlen && !WARN_ON_ONCE(!n && - !qdisc_is_offloaded); + /* Notify parent qdisc only if child qdisc becomes empty. */ + notify = !sch->q.qlen; /* TODO: perform the search on a per txq basis */ sch = qdisc_lookup_rcu(qdisc_dev(sch), TC_H_MAJ(parentid)); if (sch == NULL) { @@ -816,6 +799,9 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) } cops = sch->ops->cl_ops; if (notify && cops->qlen_notify) { + /* Note that qlen_notify must be idempotent as it may get called + * multiple times. + */ cl = cops->find(sch, parentid); cops->qlen_notify(sch, cl); } @@ -1506,7 +1492,7 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid"); return -ENOENT; } - q = qdisc_leaf(p, clid); + q = qdisc_leaf(p, clid, extack); } else if (dev_ingress_queue(dev)) { q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping); } @@ -1517,6 +1503,8 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device"); return -ENOENT; } + if (IS_ERR(q)) + return PTR_ERR(q); if (tcm->tcm_handle && q->handle != tcm->tcm_handle) { NL_SET_ERR_MSG(extack, "Invalid handle"); @@ -1610,7 +1598,9 @@ static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, NL_SET_ERR_MSG(extack, "Failed to find specified qdisc"); return -ENOENT; } - q = qdisc_leaf(p, clid); + q = qdisc_leaf(p, clid, extack); + if (IS_ERR(q)) + return PTR_ERR(q); } else if (dev_ingress_queue_create(dev)) { q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping); } diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index 09242578dac5b..85984c91cf51f 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1762,7 +1762,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, ktime_t now = ktime_get(); struct cake_tin_data *b; struct cake_flow *flow; - u32 idx; + u32 idx, tin; /* choose flow to insert into */ idx = cake_classify(sch, &b, skb, q->flow_mode, &ret); @@ -1772,6 +1772,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, __qdisc_drop(skb, to_free); return ret; } + tin = (u32)(b - q->tins); idx--; flow = &b->flows[idx]; @@ -1939,13 +1940,22 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, q->buffer_max_used = q->buffer_used; if (q->buffer_used > q->buffer_limit) { + bool same_flow = false; u32 dropped = 0; + u32 drop_id; while (q->buffer_used > q->buffer_limit) { dropped++; - cake_drop(sch, to_free); + drop_id = cake_drop(sch, to_free); + + if ((drop_id >> 16) == tin && + (drop_id & 0xFFFF) == idx) + same_flow = true; } b->drop_overlimit += dropped; + + if (same_flow) + return NET_XMIT_CN; } return NET_XMIT_SUCCESS; } diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index 5f2e068157456..63c02040b426a 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -168,7 +168,7 @@ static int codel_change(struct Qdisc *sch, struct nlattr *opt, qlen = sch->q.qlen; while (sch->q.qlen > sch->limit) { - struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); + struct sk_buff *skb = qdisc_dequeue_internal(sch, true); dropped += qdisc_pkt_len(skb); qdisc_qstats_backlog_dec(sch, skb); diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 19901e77cd3b7..9b36955e32b14 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -35,6 +35,11 @@ struct drr_sched { struct Qdisc_class_hash clhash; }; +static bool cl_is_active(struct drr_class *cl) +{ + return !list_empty(&cl->alist); +} + static struct drr_class *drr_find_class(struct Qdisc *sch, u32 classid) { struct drr_sched *q = qdisc_priv(sch); @@ -105,6 +110,7 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid, return -ENOBUFS; gnet_stats_basic_sync_init(&cl->bstats); + INIT_LIST_HEAD(&cl->alist); cl->common.classid = classid; cl->quantum = quantum; cl->qdisc = qdisc_create_dflt(sch->dev_queue, @@ -229,7 +235,7 @@ static void drr_qlen_notify(struct Qdisc *csh, unsigned long arg) { struct drr_class *cl = (struct drr_class *)arg; - list_del(&cl->alist); + list_del_init(&cl->alist); } static int drr_dump_class(struct Qdisc *sch, unsigned long arg, @@ -336,7 +342,6 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct drr_sched *q = qdisc_priv(sch); struct drr_class *cl; int err = 0; - bool first; cl = drr_classify(skb, sch, &err); if (cl == NULL) { @@ -346,7 +351,6 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, return err; } - first = !cl->qdisc->q.qlen; err = qdisc_enqueue(skb, cl->qdisc, to_free); if (unlikely(err != NET_XMIT_SUCCESS)) { if (net_xmit_drop_count(err)) { @@ -356,7 +360,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, return err; } - if (first) { + if (!cl_is_active(cl)) { list_add_tail(&cl->alist, &q->active); cl->deficit = cl->quantum; } @@ -390,7 +394,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch) if (unlikely(skb == NULL)) goto out; if (cl->qdisc->q.qlen == 0) - list_del(&cl->alist); + list_del_init(&cl->alist); bstats_update(&cl->bstats, skb); qdisc_bstats_update(sch, skb); @@ -431,7 +435,7 @@ static void drr_reset_qdisc(struct Qdisc *sch) for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { if (cl->qdisc->q.qlen) - list_del(&cl->alist); + list_del_init(&cl->alist); qdisc_reset(cl->qdisc); } } diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index 9fd70462b41d5..9873f4ae90c3a 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -74,6 +74,11 @@ static const struct nla_policy ets_class_policy[TCA_ETS_MAX + 1] = { [TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 }, }; +static bool cl_is_active(struct ets_class *cl) +{ + return !list_empty(&cl->alist); +} + static int ets_quantum_parse(struct Qdisc *sch, const struct nlattr *attr, unsigned int *quantum, struct netlink_ext_ack *extack) @@ -293,7 +298,7 @@ static void ets_class_qlen_notify(struct Qdisc *sch, unsigned long arg) * to remove them. */ if (!ets_class_is_strict(q, cl) && sch->q.qlen) - list_del(&cl->alist); + list_del_init(&cl->alist); } static int ets_class_dump(struct Qdisc *sch, unsigned long arg, @@ -416,7 +421,6 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct ets_sched *q = qdisc_priv(sch); struct ets_class *cl; int err = 0; - bool first; cl = ets_classify(skb, sch, &err); if (!cl) { @@ -426,7 +430,6 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, return err; } - first = !cl->qdisc->q.qlen; err = qdisc_enqueue(skb, cl->qdisc, to_free); if (unlikely(err != NET_XMIT_SUCCESS)) { if (net_xmit_drop_count(err)) { @@ -436,7 +439,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, return err; } - if (first && !ets_class_is_strict(q, cl)) { + if (!cl_is_active(cl) && !ets_class_is_strict(q, cl)) { list_add_tail(&cl->alist, &q->active); cl->deficit = cl->quantum; } @@ -488,7 +491,7 @@ static struct sk_buff *ets_qdisc_dequeue(struct Qdisc *sch) if (unlikely(!skb)) goto out; if (cl->qdisc->q.qlen == 0) - list_del(&cl->alist); + list_del_init(&cl->alist); return ets_qdisc_dequeue_skb(sch, skb); } @@ -648,23 +651,24 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, sch_tree_lock(sch); - q->nbands = nbands; + for (i = nbands; i < oldbands; i++) { + if (i >= q->nstrict && q->classes[i].qdisc->q.qlen) + list_del_init(&q->classes[i].alist); + qdisc_purge_queue(q->classes[i].qdisc); + } + + WRITE_ONCE(q->nbands, nbands); for (i = nstrict; i < q->nstrict; i++) { if (q->classes[i].qdisc->q.qlen) { list_add_tail(&q->classes[i].alist, &q->active); q->classes[i].deficit = quanta[i]; } } - for (i = q->nbands; i < oldbands; i++) { - if (i >= q->nstrict && q->classes[i].qdisc->q.qlen) - list_del(&q->classes[i].alist); - qdisc_tree_flush_backlog(q->classes[i].qdisc); - } - q->nstrict = nstrict; + WRITE_ONCE(q->nstrict, nstrict); memcpy(q->prio2band, priomap, sizeof(priomap)); for (i = 0; i < q->nbands; i++) - q->classes[i].quantum = quanta[i]; + WRITE_ONCE(q->classes[i].quantum, quanta[i]); for (i = oldbands; i < q->nbands; i++) { q->classes[i].qdisc = queues[i]; @@ -678,7 +682,7 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, for (i = q->nbands; i < oldbands; i++) { qdisc_put(q->classes[i].qdisc); q->classes[i].qdisc = NULL; - q->classes[i].quantum = 0; + WRITE_ONCE(q->classes[i].quantum, 0); q->classes[i].deficit = 0; gnet_stats_basic_sync_init(&q->classes[i].bstats); memset(&q->classes[i].qstats, 0, sizeof(q->classes[i].qstats)); @@ -713,7 +717,7 @@ static void ets_qdisc_reset(struct Qdisc *sch) for (band = q->nstrict; band < q->nbands; band++) { if (q->classes[band].qdisc->q.qlen) - list_del(&q->classes[band].alist); + list_del_init(&q->classes[band].alist); } for (band = 0; band < q->nbands; band++) qdisc_reset(q->classes[band].qdisc); @@ -735,6 +739,7 @@ static int ets_qdisc_dump(struct Qdisc *sch, struct sk_buff *skb) struct ets_sched *q = qdisc_priv(sch); struct nlattr *opts; struct nlattr *nest; + u8 nbands, nstrict; int band; int prio; int err; @@ -747,21 +752,22 @@ static int ets_qdisc_dump(struct Qdisc *sch, struct sk_buff *skb) if (!opts) goto nla_err; - if (nla_put_u8(skb, TCA_ETS_NBANDS, q->nbands)) + nbands = READ_ONCE(q->nbands); + if (nla_put_u8(skb, TCA_ETS_NBANDS, nbands)) goto nla_err; - if (q->nstrict && - nla_put_u8(skb, TCA_ETS_NSTRICT, q->nstrict)) + nstrict = READ_ONCE(q->nstrict); + if (nstrict && nla_put_u8(skb, TCA_ETS_NSTRICT, nstrict)) goto nla_err; - if (q->nbands > q->nstrict) { + if (nbands > nstrict) { nest = nla_nest_start(skb, TCA_ETS_QUANTA); if (!nest) goto nla_err; - for (band = q->nstrict; band < q->nbands; band++) { + for (band = nstrict; band < nbands; band++) { if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND, - q->classes[band].quantum)) + READ_ONCE(q->classes[band].quantum))) goto nla_err; } @@ -773,7 +779,8 @@ static int ets_qdisc_dump(struct Qdisc *sch, struct sk_buff *skb) goto nla_err; for (prio = 0; prio <= TC_PRIO_MAX; prio++) { - if (nla_put_u8(skb, TCA_ETS_PRIOMAP_BAND, q->prio2band[prio])) + if (nla_put_u8(skb, TCA_ETS_PRIOMAP_BAND, + READ_ONCE(q->prio2band[prio]))) goto nla_err; } diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index f59a2cb2c803d..91f5ef6be0f23 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -901,7 +901,7 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt, sch_tree_lock(sch); } while (sch->q.qlen > sch->limit) { - struct sk_buff *skb = fq_dequeue(sch); + struct sk_buff *skb = qdisc_dequeue_internal(sch, false); if (!skb) break; diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 9330923a624c0..47b5a056165cb 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -431,7 +431,7 @@ static int fq_codel_change(struct Qdisc *sch, struct nlattr *opt, while (sch->q.qlen > sch->limit || q->memory_usage > q->memory_limit) { - struct sk_buff *skb = fq_codel_dequeue(sch); + struct sk_buff *skb = qdisc_dequeue_internal(sch, false); q->cstats.drop_len += qdisc_pkt_len(skb); rtnl_kfree_skbs(skb, skb); diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 68e6acd0f130d..607c580d75e4b 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -357,7 +357,7 @@ static int fq_pie_change(struct Qdisc *sch, struct nlattr *opt, /* Drop excess packets if new limit is lower */ while (sch->q.qlen > sch->limit) { - struct sk_buff *skb = fq_pie_qdisc_dequeue(sch); + struct sk_buff *skb = qdisc_dequeue_internal(sch, false); len_dropped += qdisc_pkt_len(skb); num_dropped += 1; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 57c05ef8f1fbc..fab7b3557c7ee 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -178,9 +178,10 @@ static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) static void try_bulk_dequeue_skb(struct Qdisc *q, struct sk_buff *skb, const struct netdev_queue *txq, - int *packets) + int *packets, int budget) { int bytelimit = qdisc_avail_bulklimit(txq) - skb->len; + int cnt = 0; while (bytelimit > 0) { struct sk_buff *nskb = q->dequeue(q); @@ -191,8 +192,10 @@ static void try_bulk_dequeue_skb(struct Qdisc *q, bytelimit -= nskb->len; /* covers GSO len */ skb->next = nskb; skb = nskb; - (*packets)++; /* GSO counts as one pkt */ + if (++cnt >= budget) + break; } + (*packets) += cnt; skb_mark_not_on_list(skb); } @@ -226,7 +229,7 @@ static void try_bulk_dequeue_skb_slow(struct Qdisc *q, * A requeued skb (via q->gso_skb) can also be a SKB list. */ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, - int *packets) + int *packets, int budget) { const struct netdev_queue *txq = q->dev_queue; struct sk_buff *skb = NULL; @@ -293,7 +296,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, if (skb) { bulk: if (qdisc_may_bulk(q)) - try_bulk_dequeue_skb(q, skb, txq, packets); + try_bulk_dequeue_skb(q, skb, txq, packets, budget); else try_bulk_dequeue_skb_slow(q, skb, packets); } @@ -385,7 +388,7 @@ bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q, * >0 - queue is not empty. * */ -static inline bool qdisc_restart(struct Qdisc *q, int *packets) +static inline bool qdisc_restart(struct Qdisc *q, int *packets, int budget) { spinlock_t *root_lock = NULL; struct netdev_queue *txq; @@ -394,7 +397,7 @@ static inline bool qdisc_restart(struct Qdisc *q, int *packets) bool validate; /* Dequeue packet */ - skb = dequeue_skb(q, &validate, packets); + skb = dequeue_skb(q, &validate, packets, budget); if (unlikely(!skb)) return false; @@ -412,7 +415,7 @@ void __qdisc_run(struct Qdisc *q) int quota = READ_ONCE(dev_tx_weight); int packets; - while (qdisc_restart(q, &packets)) { + while (qdisc_restart(q, &packets, quota)) { quota -= packets; if (quota <= 0) { if (q->flags & TCQ_F_NOLOCK) diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 371255e624332..751b1e2c35b3f 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -175,6 +175,11 @@ struct hfsc_sched { #define HT_INFINITY 0xffffffffffffffffULL /* infinite time value */ +static bool cl_in_el_or_vttree(struct hfsc_class *cl) +{ + return ((cl->cl_flags & HFSC_FSC) && cl->cl_nactive) || + ((cl->cl_flags & HFSC_RSC) && !RB_EMPTY_NODE(&cl->el_node)); +} /* * eligible tree holds backlogged classes being sorted by their eligible times. @@ -203,7 +208,10 @@ eltree_insert(struct hfsc_class *cl) static inline void eltree_remove(struct hfsc_class *cl) { - rb_erase(&cl->el_node, &cl->sched->eligible); + if (!RB_EMPTY_NODE(&cl->el_node)) { + rb_erase(&cl->el_node, &cl->sched->eligible); + RB_CLEAR_NODE(&cl->el_node); + } } static inline void @@ -827,22 +835,6 @@ update_vf(struct hfsc_class *cl, unsigned int len, u64 cur_time) } } -static unsigned int -qdisc_peek_len(struct Qdisc *sch) -{ - struct sk_buff *skb; - unsigned int len; - - skb = sch->ops->peek(sch); - if (unlikely(skb == NULL)) { - qdisc_warn_nonwc("qdisc_peek_len", sch); - return 0; - } - len = qdisc_pkt_len(skb); - - return len; -} - static void hfsc_adjust_levels(struct hfsc_class *cl) { @@ -1037,6 +1029,8 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid, if (cl == NULL) return -ENOBUFS; + RB_CLEAR_NODE(&cl->el_node); + err = tcf_block_get(&cl->block, &cl->filter_list, sch, extack); if (err) { kfree(cl); @@ -1224,7 +1218,8 @@ hfsc_qlen_notify(struct Qdisc *sch, unsigned long arg) /* vttree is now handled in update_vf() so that update_vf(cl, 0, 0) * needs to be called explicitly to remove a class from vttree. */ - update_vf(cl, 0, 0); + if (cl->cl_nactive) + update_vf(cl, 0, 0); if (cl->cl_flags & HFSC_RSC) eltree_remove(cl); } @@ -1566,7 +1561,10 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) return err; } - if (first) { + sch->qstats.backlog += len; + sch->q.qlen++; + + if (first && !cl_in_el_or_vttree(cl)) { if (cl->cl_flags & HFSC_RSC) init_ed(cl, len); if (cl->cl_flags & HFSC_FSC) @@ -1581,9 +1579,6 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) } - sch->qstats.backlog += len; - sch->q.qlen++; - return NET_XMIT_SUCCESS; } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index d26cd436cbe31..83fc44f20e31c 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -560,7 +560,7 @@ static int hhf_change(struct Qdisc *sch, struct nlattr *opt, qlen = sch->q.qlen; prev_backlog = sch->qstats.backlog; while (sch->q.qlen > sch->limit) { - struct sk_buff *skb = hhf_dequeue(sch); + struct sk_buff *skb = qdisc_dequeue_internal(sch, false); rtnl_kfree_skbs(skb, skb); } diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index 0d947414e6161..b5dc7f57aacc8 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -348,7 +348,8 @@ static void htb_add_to_wait_tree(struct htb_sched *q, */ static inline void htb_next_rb_node(struct rb_node **n) { - *n = rb_next(*n); + if (*n) + *n = rb_next(*n); } /** @@ -591,7 +592,7 @@ htb_change_class_mode(struct htb_sched *q, struct htb_class *cl, s64 *diff) */ static inline void htb_activate(struct htb_sched *q, struct htb_class *cl) { - WARN_ON(cl->level || !cl->leaf.q || !cl->leaf.q->q.qlen); + WARN_ON(cl->level || !cl->leaf.q); if (!cl->prio_activity) { cl->prio_activity = 1 << cl->prio; @@ -609,8 +610,8 @@ static inline void htb_activate(struct htb_sched *q, struct htb_class *cl) */ static inline void htb_deactivate(struct htb_sched *q, struct htb_class *cl) { - WARN_ON(!cl->prio_activity); - + if (!cl->prio_activity) + return; htb_deactivate_prios(q, cl); cl->prio_activity = 0; } @@ -820,7 +821,9 @@ static struct htb_class *htb_lookup_leaf(struct htb_prio *hprio, const int prio) u32 *pid; } stk[TC_HTB_MAXDEPTH], *sp = stk; - BUG_ON(!hprio->row.rb_node); + if (unlikely(!hprio->row.rb_node)) + return NULL; + sp->root = hprio->row.rb_node; sp->pptr = &hprio->ptr; sp->pid = &hprio->last_ptr_id; @@ -1750,8 +1753,7 @@ static int htb_delete(struct Qdisc *sch, unsigned long arg, if (cl->parent) cl->parent->children--; - if (cl->prio_activity) - htb_deactivate(q, cl); + htb_deactivate(q, cl); if (cl->cmode != HTB_CAN_SEND) htb_safe_rb_erase(&cl->pq_node, @@ -1963,8 +1965,7 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, /* turn parent into inner node */ qdisc_purge_queue(parent->leaf.q); parent_qdisc = parent->leaf.q; - if (parent->prio_activity) - htb_deactivate(q, parent); + htb_deactivate(q, parent); /* remove from evt list because of level change */ if (parent->cmode != HTB_CAN_SEND) { diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c index 793009f445c03..a0e3f3bae5361 100644 --- a/net/sched/sch_mqprio.c +++ b/net/sched/sch_mqprio.c @@ -152,7 +152,7 @@ static int mqprio_parse_opt(struct net_device *dev, struct tc_mqprio_qopt *qopt, static const struct nla_policy mqprio_tc_entry_policy[TCA_MQPRIO_TC_ENTRY_MAX + 1] = { [TCA_MQPRIO_TC_ENTRY_INDEX] = NLA_POLICY_MAX(NLA_U32, - TC_QOPT_MAX_QUEUE), + TC_QOPT_MAX_QUEUE - 1), [TCA_MQPRIO_TC_ENTRY_FP] = NLA_POLICY_RANGE(NLA_U32, TC_FP_EXPRESS, TC_FP_PREEMPTIBLE), diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 447d3e836a24f..0ad231e94e14d 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -972,6 +972,41 @@ static int parse_attr(struct nlattr *tb[], int maxtype, struct nlattr *nla, return 0; } +static const struct Qdisc_class_ops netem_class_ops; + +static int check_netem_in_tree(struct Qdisc *sch, bool duplicates, + struct netlink_ext_ack *extack) +{ + struct Qdisc *root, *q; + unsigned int i; + + root = qdisc_root_sleeping(sch); + + if (sch != root && root->ops->cl_ops == &netem_class_ops) { + if (duplicates || + ((struct netem_sched_data *)qdisc_priv(root))->duplicate) + goto err; + } + + if (!qdisc_dev(root)) + return 0; + + hash_for_each(qdisc_dev(root)->qdisc_hash, i, q, hash) { + if (sch != q && q->ops->cl_ops == &netem_class_ops) { + if (duplicates || + ((struct netem_sched_data *)qdisc_priv(q))->duplicate) + goto err; + } + } + + return 0; + +err: + NL_SET_ERR_MSG(extack, + "netem: cannot mix duplicating netems with other netems in tree"); + return -EINVAL; +} + /* Parse netlink message to set options */ static int netem_change(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) @@ -1030,6 +1065,11 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt, q->gap = qopt->gap; q->counter = 0; q->loss = qopt->loss; + + ret = check_netem_in_tree(sch, qopt->duplicate, extack); + if (ret) + goto unlock; + q->duplicate = qopt->duplicate; /* for compatibility with earlier versions. diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c index 2da6250ec3463..48c5ab8ec143c 100644 --- a/net/sched/sch_pie.c +++ b/net/sched/sch_pie.c @@ -190,7 +190,7 @@ static int pie_change(struct Qdisc *sch, struct nlattr *opt, /* Drop excess packets if new limit is lower */ qlen = sch->q.qlen; while (sch->q.qlen > sch->limit) { - struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); + struct sk_buff *skb = qdisc_dequeue_internal(sch, true); dropped += qdisc_pkt_len(skb); qdisc_qstats_backlog_dec(sch, skb); diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index fdc5ef52c3ee9..fdd9caa41e80f 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -211,7 +211,7 @@ static int prio_tune(struct Qdisc *sch, struct nlattr *opt, memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1); for (i = q->bands; i < oldbands; i++) - qdisc_tree_flush_backlog(q->queues[i]); + qdisc_purge_queue(q->queues[i]); for (i = oldbands; i < q->bands; i++) { q->queues[i] = queues[i]; diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index 546c10adcacde..69fdbbbb3b634 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -202,6 +202,11 @@ struct qfq_sched { */ enum update_reason {enqueue, requeue}; +static bool cl_is_active(struct qfq_class *cl) +{ + return !list_empty(&cl->alist); +} + static struct qfq_class *qfq_find_class(struct Qdisc *sch, u32 classid) { struct qfq_sched *q = qdisc_priv(sch); @@ -347,7 +352,7 @@ static void qfq_deactivate_class(struct qfq_sched *q, struct qfq_class *cl) struct qfq_aggregate *agg = cl->agg; - list_del(&cl->alist); /* remove from RR queue of the aggregate */ + list_del_init(&cl->alist); /* remove from RR queue of the aggregate */ if (list_empty(&agg->active)) /* agg is now inactive */ qfq_deactivate_agg(q, agg); } @@ -407,7 +412,7 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, bool existing = false; struct nlattr *tb[TCA_QFQ_MAX + 1]; struct qfq_aggregate *new_agg = NULL; - u32 weight, lmax, inv_w; + u32 weight, lmax, inv_w, old_weight, old_lmax; int err; int delta_w; @@ -441,12 +446,16 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, inv_w = ONE_FP / weight; weight = ONE_FP / inv_w; - if (cl != NULL && - lmax == cl->agg->lmax && - weight == cl->agg->class_weight) - return 0; /* nothing to change */ + if (cl != NULL) { + sch_tree_lock(sch); + old_weight = cl->agg->class_weight; + old_lmax = cl->agg->lmax; + sch_tree_unlock(sch); + if (lmax == old_lmax && weight == old_weight) + return 0; /* nothing to change */ + } - delta_w = weight - (cl ? cl->agg->class_weight : 0); + delta_w = weight - (cl ? old_weight : 0); if (q->wsum + delta_w > QFQ_MAX_WSUM) { NL_SET_ERR_MSG_FMT_MOD(extack, @@ -477,6 +486,7 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, gnet_stats_basic_sync_init(&cl->bstats); cl->common.classid = classid; cl->deficit = lmax; + INIT_LIST_HEAD(&cl->alist); cl->qdisc = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, classid, NULL); @@ -529,9 +539,6 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, static void qfq_destroy_class(struct Qdisc *sch, struct qfq_class *cl) { - struct qfq_sched *q = qdisc_priv(sch); - - qfq_rm_from_agg(q, cl); gen_kill_estimator(&cl->rate_est); qdisc_put(cl->qdisc); kfree(cl); @@ -552,6 +559,7 @@ static int qfq_delete_class(struct Qdisc *sch, unsigned long arg, qdisc_purge_queue(cl->qdisc); qdisc_class_hash_remove(&q->clhash, &cl->common); + qfq_rm_from_agg(q, cl); sch_tree_unlock(sch); @@ -622,6 +630,7 @@ static int qfq_dump_class(struct Qdisc *sch, unsigned long arg, { struct qfq_class *cl = (struct qfq_class *)arg; struct nlattr *nest; + u32 class_weight, lmax; tcm->tcm_parent = TC_H_ROOT; tcm->tcm_handle = cl->common.classid; @@ -630,8 +639,13 @@ static int qfq_dump_class(struct Qdisc *sch, unsigned long arg, nest = nla_nest_start_noflag(skb, TCA_OPTIONS); if (nest == NULL) goto nla_put_failure; - if (nla_put_u32(skb, TCA_QFQ_WEIGHT, cl->agg->class_weight) || - nla_put_u32(skb, TCA_QFQ_LMAX, cl->agg->lmax)) + + sch_tree_lock(sch); + class_weight = cl->agg->class_weight; + lmax = cl->agg->lmax; + sch_tree_unlock(sch); + if (nla_put_u32(skb, TCA_QFQ_WEIGHT, class_weight) || + nla_put_u32(skb, TCA_QFQ_LMAX, lmax)) goto nla_put_failure; return nla_nest_end(skb, nest); @@ -648,8 +662,10 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg, memset(&xstats, 0, sizeof(xstats)); + sch_tree_lock(sch); xstats.weight = cl->agg->class_weight; xstats.lmax = cl->agg->lmax; + sch_tree_unlock(sch); if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 || gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 || @@ -985,8 +1001,8 @@ static struct sk_buff *agg_dequeue(struct qfq_aggregate *agg, cl->deficit -= (int) len; if (cl->qdisc->q.qlen == 0) /* no more packets, remove from list */ - list_del(&cl->alist); - else if (cl->deficit < qdisc_pkt_len(cl->qdisc->ops->peek(cl->qdisc))) { + list_del_init(&cl->alist); + else if (cl->deficit < qdisc_peek_len(cl->qdisc)) { cl->deficit += agg->lmax; list_move_tail(&cl->alist, &agg->active); } @@ -1217,7 +1233,6 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct qfq_class *cl; struct qfq_aggregate *agg; int err = 0; - bool first; cl = qfq_classify(skb, sch, &err); if (cl == NULL) { @@ -1239,7 +1254,6 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, } gso_segs = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; - first = !cl->qdisc->q.qlen; err = qdisc_enqueue(skb, cl->qdisc, to_free); if (unlikely(err != NET_XMIT_SUCCESS)) { pr_debug("qfq_enqueue: enqueue failed %d\n", err); @@ -1255,8 +1269,8 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, ++sch->q.qlen; agg = cl->agg; - /* if the queue was not empty, then done here */ - if (!first) { + /* if the class is active, then done here */ + if (cl_is_active(cl)) { if (unlikely(skb == cl->qdisc->ops->peek(cl->qdisc)) && list_first_entry(&agg->active, struct qfq_class, alist) == cl && cl->deficit < len) @@ -1418,6 +1432,8 @@ static void qfq_qlen_notify(struct Qdisc *sch, unsigned long arg) struct qfq_sched *q = qdisc_priv(sch); struct qfq_class *cl = (struct qfq_class *)arg; + if (list_empty(&cl->alist)) + return; qfq_deactivate_class(q, cl); } @@ -1488,6 +1504,7 @@ static void qfq_destroy_qdisc(struct Qdisc *sch) for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry_safe(cl, next, &q->clhash.hash[i], common.hnode) { + qfq_rm_from_agg(q, cl); qfq_destroy_class(sch, cl); } } diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 16277b6a0238d..3c6b4460cf2c0 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -283,7 +283,7 @@ static int __red_change(struct Qdisc *sch, struct nlattr **tb, q->userbits = userbits; q->limit = ctl->limit; if (child) { - qdisc_tree_flush_backlog(q->qdisc); + qdisc_purge_queue(q->qdisc); old_child = q->qdisc; q->qdisc = child; } diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index 002941d35b643..e871cb53946a3 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -310,7 +310,10 @@ static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free) /* It is difficult to believe, but ALL THE SLOTS HAVE LENGTH 1. */ x = q->tail->next; slot = &q->slots[x]; - q->tail->next = slot->next; + if (slot->next == x) + q->tail = NULL; /* no more active slots */ + else + q->tail->next = slot->next; q->ht[slot->hash] = SFQ_EMPTY_SLOT; goto drop; } @@ -653,6 +656,14 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt, NL_SET_ERR_MSG_MOD(extack, "invalid quantum"); return -EINVAL; } + + if (ctl->perturb_period < 0 || + ctl->perturb_period > INT_MAX / HZ) { + NL_SET_ERR_MSG_MOD(extack, "invalid perturb period"); + return -EINVAL; + } + perturb_period = ctl->perturb_period * HZ; + if (ctl_v1 && !red_check_params(ctl_v1->qth_min, ctl_v1->qth_max, ctl_v1->Wlog, ctl_v1->Scell_log, NULL)) return -EINVAL; @@ -669,14 +680,12 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt, headdrop = q->headdrop; maxdepth = q->maxdepth; maxflows = q->maxflows; - perturb_period = q->perturb_period; quantum = q->quantum; flags = q->flags; /* update and validate configuration */ if (ctl->quantum) quantum = ctl->quantum; - perturb_period = ctl->perturb_period * HZ; if (ctl->flows) maxflows = min_t(u32, ctl->flows, SFQ_MAX_FLOWS); if (ctl->divisor) { diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 951a87909c297..a01d17d03bf57 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -41,6 +41,11 @@ static struct static_key_false taprio_have_working_mqprio; #define TXTIME_ASSIST_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST) #define FULL_OFFLOAD_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD) #define TAPRIO_FLAGS_INVALID U32_MAX +/* Minimum value for picos_per_byte to ensure non-zero duration + * for minimum-sized Ethernet frames (ETH_ZLEN = 60). + * 60 * 17 > PSEC_PER_NSEC (1000) + */ +#define TAPRIO_PICOS_PER_BYTE_MIN 17 struct sched_entry { /* Durations between this GCL entry and the GCL entry where the @@ -1294,7 +1299,8 @@ static void taprio_start_sched(struct Qdisc *sch, } static void taprio_set_picos_per_byte(struct net_device *dev, - struct taprio_sched *q) + struct taprio_sched *q, + struct netlink_ext_ack *extack) { struct ethtool_link_ksettings ecmd; int speed = SPEED_10; @@ -1310,6 +1316,15 @@ static void taprio_set_picos_per_byte(struct net_device *dev, skip: picos_per_byte = (USEC_PER_SEC * 8) / speed; + if (picos_per_byte < TAPRIO_PICOS_PER_BYTE_MIN) { + if (!extack) + pr_warn("Link speed %d is too high. Schedule may be inaccurate.\n", + speed); + NL_SET_ERR_MSG_FMT_MOD(extack, + "Link speed %d is too high. Schedule may be inaccurate.", + speed); + picos_per_byte = TAPRIO_PICOS_PER_BYTE_MIN; + } atomic64_set(&q->picos_per_byte, picos_per_byte); netdev_dbg(dev, "taprio: set %s's picos_per_byte to: %lld, linkspeed: %d\n", @@ -1334,17 +1349,19 @@ static int taprio_dev_notifier(struct notifier_block *nb, unsigned long event, if (dev != qdisc_dev(q->root)) continue; - taprio_set_picos_per_byte(dev, q); + taprio_set_picos_per_byte(dev, q, NULL); stab = rtnl_dereference(q->root->stab); - oper = rtnl_dereference(q->oper_sched); + rcu_read_lock(); + oper = rcu_dereference(q->oper_sched); if (oper) taprio_update_queue_max_sdu(q, oper, stab); - admin = rtnl_dereference(q->admin_sched); + admin = rcu_dereference(q->admin_sched); if (admin) taprio_update_queue_max_sdu(q, admin, stab); + rcu_read_unlock(); break; } @@ -1869,7 +1886,7 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt, q->flags = err; /* Needed for length_to_duration() during netlink attribute parsing */ - taprio_set_picos_per_byte(dev, q); + taprio_set_picos_per_byte(dev, q, extack); err = taprio_parse_mqprio_opt(dev, mqprio, extack, q->flags); if (err < 0) diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c index f92174008499b..61dd5c8f23101 100644 --- a/net/sched/sch_tbf.c +++ b/net/sched/sch_tbf.c @@ -452,7 +452,7 @@ static int tbf_change(struct Qdisc *sch, struct nlattr *opt, sch_tree_lock(sch); if (child) { - qdisc_tree_flush_backlog(q->qdisc); + qdisc_purge_queue(q->qdisc); old = q->qdisc; q->qdisc = child; } diff --git a/net/sctp/diag.c b/net/sctp/diag.c index c3d6b92dd3862..ad3e1462a896e 100644 --- a/net/sctp/diag.c +++ b/net/sctp/diag.c @@ -73,19 +73,26 @@ static int inet_diag_msg_sctpladdrs_fill(struct sk_buff *skb, struct nlattr *attr; void *info = NULL; + rcu_read_lock(); list_for_each_entry_rcu(laddr, address_list, list) addrcnt++; + rcu_read_unlock(); attr = nla_reserve(skb, INET_DIAG_LOCALS, addrlen * addrcnt); if (!attr) return -EMSGSIZE; info = nla_data(attr); + rcu_read_lock(); list_for_each_entry_rcu(laddr, address_list, list) { memcpy(info, &laddr->a, sizeof(laddr->a)); memset(info + sizeof(laddr->a), 0, addrlen - sizeof(laddr->a)); info += addrlen; + + if (!--addrcnt) + break; } + rcu_read_unlock(); return 0; } @@ -223,14 +230,15 @@ struct sctp_comm_param { bool net_admin; }; -static size_t inet_assoc_attr_size(struct sctp_association *asoc) +static size_t inet_assoc_attr_size(struct sock *sk, + struct sctp_association *asoc) { int addrlen = sizeof(struct sockaddr_storage); int addrcnt = 0; struct sctp_sockaddr_entry *laddr; list_for_each_entry_rcu(laddr, &asoc->base.bind_addr.address_list, - list) + list, lockdep_sock_is_held(sk)) addrcnt++; return nla_total_size(sizeof(struct sctp_info)) @@ -256,11 +264,14 @@ static int sctp_sock_dump_one(struct sctp_endpoint *ep, struct sctp_transport *t if (err) return err; - rep = nlmsg_new(inet_assoc_attr_size(assoc), GFP_KERNEL); - if (!rep) + lock_sock(sk); + + rep = nlmsg_new(inet_assoc_attr_size(sk, assoc), GFP_KERNEL); + if (!rep) { + release_sock(sk); return -ENOMEM; + } - lock_sock(sk); if (ep != assoc->ep) { err = -EAGAIN; goto out; diff --git a/net/sctp/input.c b/net/sctp/input.c index a8a254a5008e5..032a10d82302c 100644 --- a/net/sctp/input.c +++ b/net/sctp/input.c @@ -117,7 +117,7 @@ int sctp_rcv(struct sk_buff *skb) * it's better to just linearize it otherwise crc computing * takes longer. */ - if ((!is_gso && skb_linearize(skb)) || + if (((!is_gso || skb_cloned(skb)) && skb_linearize(skb)) || !pskb_may_pull(skb, sizeof(struct sctphdr))) goto discard_it; diff --git a/net/sctp/inqueue.c b/net/sctp/inqueue.c index 5c16521818058..f5a7d5a387555 100644 --- a/net/sctp/inqueue.c +++ b/net/sctp/inqueue.c @@ -169,13 +169,14 @@ struct sctp_chunk *sctp_inq_pop(struct sctp_inq *queue) chunk->head_skb = chunk->skb; /* skbs with "cover letter" */ - if (chunk->head_skb && chunk->skb->data_len == chunk->skb->len) + if (chunk->head_skb && chunk->skb->data_len == chunk->skb->len) { + if (WARN_ON(!skb_shinfo(chunk->skb)->frag_list)) { + __SCTP_INC_STATS(dev_net(chunk->skb->dev), + SCTP_MIB_IN_PKT_DISCARDS); + sctp_chunk_free(chunk); + goto next_chunk; + } chunk->skb = skb_shinfo(chunk->skb)->frag_list; - - if (WARN_ON(!chunk->skb)) { - __SCTP_INC_STATS(dev_net(chunk->skb->dev), SCTP_MIB_IN_PKT_DISCARDS); - sctp_chunk_free(chunk); - goto next_chunk; } } diff --git a/net/sctp/ipv6.c b/net/sctp/ipv6.c index 717828e531621..0673857cb3d8b 100644 --- a/net/sctp/ipv6.c +++ b/net/sctp/ipv6.c @@ -547,7 +547,9 @@ static void sctp_v6_from_sk(union sctp_addr *addr, struct sock *sk) { addr->v6.sin6_family = AF_INET6; addr->v6.sin6_port = 0; + addr->v6.sin6_flowinfo = 0; addr->v6.sin6_addr = sk->sk_v6_rcv_saddr; + addr->v6.sin6_scope_id = 0; } /* Initialize sk->sk_rcv_saddr from sctp_addr. */ diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index 08527d882e56e..eb2ed7db3fe16 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -31,6 +31,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -1796,7 +1797,7 @@ struct sctp_association *sctp_unpack_cookie( } } - if (memcmp(digest, cookie->signature, SCTP_SIGNATURE_SIZE)) { + if (crypto_memneq(digest, cookie->signature, SCTP_SIGNATURE_SIZE)) { *error = -SCTP_IERROR_BAD_SIG; goto fail; } diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 808863e047e0c..cd18b22b2bbae 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -30,6 +30,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -884,7 +885,8 @@ enum sctp_disposition sctp_sf_do_5_1D_ce(struct net *net, return SCTP_DISPOSITION_CONSUME; nomem_authev: - sctp_ulpevent_free(ai_ev); + if (ai_ev) + sctp_ulpevent_free(ai_ev); nomem_aiev: sctp_ulpevent_free(ev); nomem_ev: @@ -4415,7 +4417,7 @@ static enum sctp_ierror sctp_sf_authenticate( sh_key, GFP_ATOMIC); /* Discard the packet if the digests do not match */ - if (memcmp(save_digest, digest, sig_len)) { + if (crypto_memneq(save_digest, digest, sig_len)) { kfree(save_digest); return SCTP_IERROR_BAD_SIG; } diff --git a/net/sctp/socket.c b/net/sctp/socket.c index b84c5e0a76f52..adc04e88f349f 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -9094,7 +9094,8 @@ static void __sctp_write_space(struct sctp_association *asoc) wq = rcu_dereference(sk->sk_wq); if (wq) { if (waitqueue_active(&wq->wait)) - wake_up_interruptible(&wq->wait); + wake_up_interruptible_poll(&wq->wait, EPOLLOUT | + EPOLLWRNORM | EPOLLWRBAND); /* Note that we try to include the Async I/O support * here by modeling from the current TCP/UDP code. diff --git a/net/sctp/sysctl.c b/net/sctp/sysctl.c index fd73be940f460..77a76634014a5 100644 --- a/net/sctp/sysctl.c +++ b/net/sctp/sysctl.c @@ -529,6 +529,8 @@ static int proc_sctp_do_auth(struct ctl_table *ctl, int write, return ret; } +static DEFINE_MUTEX(sctp_sysctl_mutex); + static int proc_sctp_do_udp_port(struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -553,6 +555,7 @@ static int proc_sctp_do_udp_port(struct ctl_table *ctl, int write, if (new_value > max || new_value < min) return -EINVAL; + mutex_lock(&sctp_sysctl_mutex); net->sctp.udp_port = new_value; sctp_udp_sock_stop(net); if (new_value) { @@ -565,6 +568,7 @@ static int proc_sctp_do_udp_port(struct ctl_table *ctl, int write, lock_sock(sk); sctp_sk(sk)->udp_port = htons(net->sctp.udp_port); release_sock(sk); + mutex_unlock(&sctp_sysctl_mutex); } return ret; diff --git a/net/sctp/transport.c b/net/sctp/transport.c index 31eca29b6cfbf..abb44c0ac1a0b 100644 --- a/net/sctp/transport.c +++ b/net/sctp/transport.c @@ -495,6 +495,7 @@ void sctp_transport_update_rto(struct sctp_transport *tp, __u32 rtt) if (tp->rttvar || tp->srtt) { struct net *net = tp->asoc->base.net; + unsigned int rto_beta, rto_alpha; /* 6.3.1 C3) When a new RTT measurement R' is made, set * RTTVAR <- (1 - RTO.Beta) * RTTVAR + RTO.Beta * |SRTT - R'| * SRTT <- (1 - RTO.Alpha) * SRTT + RTO.Alpha * R' @@ -506,10 +507,14 @@ void sctp_transport_update_rto(struct sctp_transport *tp, __u32 rtt) * For example, assuming the default value of RTO.Alpha of * 1/8, rto_alpha would be expressed as 3. */ - tp->rttvar = tp->rttvar - (tp->rttvar >> net->sctp.rto_beta) - + (((__u32)abs((__s64)tp->srtt - (__s64)rtt)) >> net->sctp.rto_beta); - tp->srtt = tp->srtt - (tp->srtt >> net->sctp.rto_alpha) - + (rtt >> net->sctp.rto_alpha); + rto_beta = READ_ONCE(net->sctp.rto_beta); + if (rto_beta < 32) + tp->rttvar = tp->rttvar - (tp->rttvar >> rto_beta) + + (((__u32)abs((__s64)tp->srtt - (__s64)rtt)) >> rto_beta); + rto_alpha = READ_ONCE(net->sctp.rto_alpha); + if (rto_alpha < 32) + tp->srtt = tp->srtt - (tp->srtt >> rto_alpha) + + (rtt >> rto_alpha); } else { /* 6.3.1 C2) When the first RTT measurement R is made, set * SRTT <- R, RTTVAR <- R/2. diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 45efbbfff94ae..b3bfd0f18d418 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -2553,8 +2553,9 @@ static void smc_listen_work(struct work_struct *work) goto out_decl; } - smc_listen_out_connected(new_smc); SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini); + /* smc_listen_out() will release smcsk */ + smc_listen_out_connected(new_smc); goto out_free; out_unlock: @@ -3343,10 +3344,7 @@ int smc_create_clcsk(struct net *net, struct sock *sk, int family) * which need net ref. */ sk = smc->clcsock->sk; - __netns_tracker_free(net, &sk->ns_tracker, false); - sk->sk_net_refcnt = 1; - get_net_track(net, &sk->ns_tracker, GFP_KERNEL); - sock_inuse_add(net, 1); + sk_net_refcnt_upgrade(sk); return 0; } diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c index dbce904c03cf7..2f748226f1433 100644 --- a/net/smc/smc_clc.c +++ b/net/smc/smc_clc.c @@ -426,8 +426,6 @@ smc_clc_msg_decl_valid(struct smc_clc_msg_decline *dclc) { struct smc_clc_msg_hdr *hdr = &dclc->hdr; - if (hdr->typev1 != SMC_TYPE_R && hdr->typev1 != SMC_TYPE_D) - return false; if (hdr->version == SMC_V1) { if (ntohs(hdr->length) != sizeof(struct smc_clc_msg_decline)) return false; @@ -885,6 +883,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini) return SMC_CLC_DECL_CNFERR; } pclc_base->hdr.typev1 = SMC_TYPE_N; + ini->smc_type_v1 = SMC_TYPE_N; } else { pclc_base->iparea_offset = htons(sizeof(*pclc_smcd)); plen += sizeof(*pclc_prfx) + diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index 598ac9ead64b7..6df543e083fb3 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -743,6 +743,9 @@ bool smc_ib_is_sg_need_sync(struct smc_link *lnk, unsigned int i; bool ret = false; + if (!lnk->smcibdev->ibdev->dma_device) + return ret; + /* for now there is just one DMA address */ for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, buf_slot->sgt[lnk->link_idx].nents, i) { diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c index dbcc72b43d0c0..d44d7f427fc94 100644 --- a/net/smc/smc_pnet.c +++ b/net/smc/smc_pnet.c @@ -1084,14 +1084,16 @@ static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev, struct smc_init_info *ini) { u8 ndev_pnetid[SMC_MAX_PNETID_LEN]; + struct net_device *base_ndev; struct net *net; - ndev = pnet_find_base_ndev(ndev); + base_ndev = pnet_find_base_ndev(ndev); net = dev_net(ndev); - if (smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port, + if (smc_pnetid_by_dev_port(base_ndev->dev.parent, base_ndev->dev_port, ndev_pnetid) && + smc_pnet_find_ndev_pnetid_by_table(base_ndev, ndev_pnetid) && smc_pnet_find_ndev_pnetid_by_table(ndev, ndev_pnetid)) { - smc_pnet_find_rdma_dev(ndev, ini); + smc_pnet_find_rdma_dev(base_ndev, ini); return; /* pnetid could not be determined */ } _smc_pnet_find_roce_by_pnetid(ndev_pnetid, ini, NULL, net); diff --git a/net/strparser/strparser.c b/net/strparser/strparser.c index 95696f42647ec..b61384b08e7c3 100644 --- a/net/strparser/strparser.c +++ b/net/strparser/strparser.c @@ -238,7 +238,7 @@ static int __strp_recv(read_descriptor_t *desc, struct sk_buff *orig_skb, strp_parser_err(strp, -EMSGSIZE, desc); break; } else if (len <= (ssize_t)head->len - - skb->len - stm->strp.offset) { + (ssize_t)skb->len - stm->strp.offset) { /* Length must be into new skb (and also * greater than zero) */ diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index cf30bd649e270..d063b63516f81 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -724,7 +724,7 @@ svcauth_gss_verify_header(struct svc_rqst *rqstp, struct rsc *rsci, rqstp->rq_auth_stat = rpc_autherr_badverf; return SVC_DENIED; } - if (flavor != RPC_AUTH_GSS) { + if (flavor != RPC_AUTH_GSS || checksum.len < XDR_UNIT) { rqstp->rq_auth_stat = rpc_autherr_badverf; return SVC_DENIED; } diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 142ee6554848a..63756607f6327 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -275,9 +275,6 @@ static struct rpc_xprt *rpc_clnt_set_transport(struct rpc_clnt *clnt, old = rcu_dereference_protected(clnt->cl_xprt, lockdep_is_held(&clnt->cl_lock)); - if (!xprt_bound(xprt)) - clnt->cl_autobind = 1; - clnt->cl_timeout = timeout; rcu_assign_pointer(clnt->cl_xprt, xprt); spin_unlock(&clnt->cl_lock); @@ -2736,8 +2733,13 @@ rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) case -EPROTONOSUPPORT: goto out_err; case -EACCES: - /* Re-encode with a fresh cred */ - fallthrough; + /* possible RPCSEC_GSS out-of-sequence event (RFC2203), + * reset recv state and keep waiting, don't retransmit + */ + task->tk_rqstp->rq_reply_bytes_recvd = 0; + task->tk_status = xprt_request_enqueue_receive(task); + task->tk_action = call_transmit_status; + return -EBADMSG; default: goto out_garbage; } diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c index 102c3818bc54d..53bcca365fb1c 100644 --- a/net/sunrpc/rpcb_clnt.c +++ b/net/sunrpc/rpcb_clnt.c @@ -820,9 +820,10 @@ static void rpcb_getport_done(struct rpc_task *child, void *data) } trace_rpcb_setport(child, map->r_status, map->r_port); - xprt->ops->set_port(xprt, map->r_port); - if (map->r_port) + if (map->r_port) { + xprt->ops->set_port(xprt, map->r_port); xprt_set_bound(xprt); + } } /* diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index b43dc8409b1fb..4d8d460747164 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -1344,7 +1344,8 @@ svc_process_common(struct svc_rqst *rqstp) case SVC_OK: break; case SVC_GARBAGE: - goto err_garbage_args; + rqstp->rq_auth_stat = rpc_autherr_badcred; + goto err_bad_auth; case SVC_SYSERR: goto err_system_err; case SVC_DENIED: @@ -1485,14 +1486,6 @@ svc_process_common(struct svc_rqst *rqstp) *rqstp->rq_accept_statp = rpc_proc_unavail; goto sendit; -err_garbage_args: - svc_printk(rqstp, "failed to decode RPC header\n"); - - if (serv->sv_stats) - serv->sv_stats->rpcbadfmt++; - *rqstp->rq_accept_statp = rpc_garbage_args; - goto sendit; - err_system_err: if (serv->sv_stats) serv->sv_stats->rpcbadfmt++; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 8d760f8fc4b5a..78b139d8c1f3b 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -257,20 +257,47 @@ svc_tcp_sock_process_cmsg(struct socket *sock, struct msghdr *msg, } static int -svc_tcp_sock_recv_cmsg(struct svc_sock *svsk, struct msghdr *msg) +svc_tcp_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags) { union { struct cmsghdr cmsg; u8 buf[CMSG_SPACE(sizeof(u8))]; } u; - struct socket *sock = svsk->sk_sock; + u8 alert[2]; + struct kvec alert_kvec = { + .iov_base = alert, + .iov_len = sizeof(alert), + }; + struct msghdr msg = { + .msg_flags = *msg_flags, + .msg_control = &u, + .msg_controllen = sizeof(u), + }; + int ret; + + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &alert_kvec, 1, + alert_kvec.iov_len); + ret = sock_recvmsg(sock, &msg, MSG_DONTWAIT); + if (ret > 0 && + tls_get_record_type(sock->sk, &u.cmsg) == TLS_RECORD_TYPE_ALERT) { + iov_iter_revert(&msg.msg_iter, ret); + ret = svc_tcp_sock_process_cmsg(sock, &msg, &u.cmsg, -EAGAIN); + } + return ret; +} + +static int +svc_tcp_sock_recvmsg(struct svc_sock *svsk, struct msghdr *msg) +{ int ret; + struct socket *sock = svsk->sk_sock; - msg->msg_control = &u; - msg->msg_controllen = sizeof(u); ret = sock_recvmsg(sock, msg, MSG_DONTWAIT); - if (unlikely(msg->msg_controllen != sizeof(u))) - ret = svc_tcp_sock_process_cmsg(sock, msg, &u.cmsg, ret); + if (msg->msg_flags & MSG_CTRUNC) { + msg->msg_flags &= ~(MSG_CTRUNC | MSG_EOR); + if (ret == 0 || ret == -EIO) + ret = svc_tcp_sock_recv_cmsg(sock, &msg->msg_flags); + } return ret; } @@ -321,7 +348,7 @@ static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen, iov_iter_advance(&msg.msg_iter, seek); buflen -= seek; } - len = svc_tcp_sock_recv_cmsg(svsk, &msg); + len = svc_tcp_sock_recvmsg(svsk, &msg); if (len > 0) svc_flush_bvec(bvec, len, seek); @@ -1019,7 +1046,7 @@ static ssize_t svc_tcp_read_marker(struct svc_sock *svsk, iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen; iov.iov_len = want; iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, want); - len = svc_tcp_sock_recv_cmsg(svsk, &msg); + len = svc_tcp_sock_recvmsg(svsk, &msg); if (len < 0) return len; svsk->sk_tcplen += len; @@ -1552,10 +1579,7 @@ static struct svc_xprt *svc_create_socket(struct svc_serv *serv, newlen = error; if (protocol == IPPROTO_TCP) { - __netns_tracker_free(net, &sock->sk->ns_tracker, false); - sock->sk->sk_net_refcnt = 1; - get_net_track(net, &sock->sk->ns_tracker, GFP_KERNEL); - sock_inuse_add(net, 1); + sk_net_refcnt_upgrade(sock->sk); if ((error = kernel_listen(sock, 64)) < 0) goto bummer; } diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 29df05879c8e9..3660ef2647112 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -358,7 +358,7 @@ xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp) static int xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, - struct cmsghdr *cmsg, int ret) + unsigned int *msg_flags, struct cmsghdr *cmsg, int ret) { u8 content_type = tls_get_record_type(sock->sk, cmsg); u8 level, description; @@ -371,7 +371,7 @@ xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, * record, even though there might be more frames * waiting to be decrypted. */ - msg->msg_flags &= ~MSG_EOR; + *msg_flags &= ~MSG_EOR; break; case TLS_RECORD_TYPE_ALERT: tls_alert_recv(sock->sk, msg, &level, &description); @@ -386,19 +386,33 @@ xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, } static int -xs_sock_recv_cmsg(struct socket *sock, struct msghdr *msg, int flags) +xs_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags, int flags) { union { struct cmsghdr cmsg; u8 buf[CMSG_SPACE(sizeof(u8))]; } u; + u8 alert[2]; + struct kvec alert_kvec = { + .iov_base = alert, + .iov_len = sizeof(alert), + }; + struct msghdr msg = { + .msg_flags = *msg_flags, + .msg_control = &u, + .msg_controllen = sizeof(u), + }; int ret; - msg->msg_control = &u; - msg->msg_controllen = sizeof(u); - ret = sock_recvmsg(sock, msg, flags); - if (msg->msg_controllen != sizeof(u)) - ret = xs_sock_process_cmsg(sock, msg, &u.cmsg, ret); + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &alert_kvec, 1, + alert_kvec.iov_len); + ret = sock_recvmsg(sock, &msg, flags); + if (ret > 0) { + if (tls_get_record_type(sock->sk, &u.cmsg) == TLS_RECORD_TYPE_ALERT) + iov_iter_revert(&msg.msg_iter, ret); + ret = xs_sock_process_cmsg(sock, &msg, msg_flags, &u.cmsg, + -EAGAIN); + } return ret; } @@ -408,7 +422,13 @@ xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek) ssize_t ret; if (seek != 0) iov_iter_advance(&msg->msg_iter, seek); - ret = xs_sock_recv_cmsg(sock, msg, flags); + ret = sock_recvmsg(sock, msg, flags); + /* Handle TLS inband control message lazily */ + if (msg->msg_flags & MSG_CTRUNC) { + msg->msg_flags &= ~(MSG_CTRUNC | MSG_EOR); + if (ret == 0 || ret == -EIO) + ret = xs_sock_recv_cmsg(sock, &msg->msg_flags, flags); + } return ret > 0 ? ret + seek : ret; } @@ -434,7 +454,7 @@ xs_read_discard(struct socket *sock, struct msghdr *msg, int flags, size_t count) { iov_iter_discard(&msg->msg_iter, ITER_DEST, count); - return xs_sock_recv_cmsg(sock, msg, flags); + return xs_sock_recvmsg(sock, msg, flags, 0); } #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE @@ -1921,12 +1941,8 @@ static struct socket *xs_create_sock(struct rpc_xprt *xprt, goto out; } - if (protocol == IPPROTO_TCP) { - __netns_tracker_free(xprt->xprt_net, &sock->sk->ns_tracker, false); - sock->sk->sk_net_refcnt = 1; - get_net_track(xprt->xprt_net, &sock->sk->ns_tracker, GFP_KERNEL); - sock_inuse_add(xprt->xprt_net, 1); - } + if (protocol == IPPROTO_TCP) + sk_net_refcnt_upgrade(sock->sk); filp = sock_alloc_file(sock, O_NONBLOCK, NULL); if (IS_ERR(filp)) @@ -2724,6 +2740,11 @@ static void xs_tcp_tls_setup_socket(struct work_struct *work) } rpc_shutdown_client(lower_clnt); + /* Check for ingress data that arrived before the socket's + * ->data_ready callback was set up. + */ + xs_poll_check_readable(upper_transport); + out_unlock: current_restore_flags(pflags, PF_MEMALLOC); upper_transport->clnt = NULL; diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index c524421ec6525..ea5bb131ebd06 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -425,7 +425,7 @@ static void tipc_aead_free(struct rcu_head *rp) } free_percpu(aead->tfm_entry); kfree_sensitive(aead->key); - kfree(aead); + kfree_sensitive(aead); } static int tipc_aead_users(struct tipc_aead __rcu *aead) @@ -817,12 +817,20 @@ static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb, goto exit; } + /* Get net to avoid freed tipc_crypto when delete namespace */ + if (!maybe_get_net(aead->crypto->net)) { + tipc_bearer_put(b); + rc = -ENODEV; + goto exit; + } + /* Now, do encrypt */ rc = crypto_aead_encrypt(req); if (rc == -EINPROGRESS || rc == -EBUSY) return rc; tipc_bearer_put(b); + put_net(aead->crypto->net); exit: kfree(ctx); @@ -860,6 +868,7 @@ static void tipc_aead_encrypt_done(void *data, int err) kfree(tx_ctx); tipc_bearer_put(b); tipc_aead_put(aead); + put_net(net); } /** diff --git a/net/tipc/net.c b/net/tipc/net.c index 0e95572e56b41..7e65d0b0c4a8d 100644 --- a/net/tipc/net.c +++ b/net/tipc/net.c @@ -145,7 +145,9 @@ void tipc_net_finalize_work(struct work_struct *work) { struct tipc_net *tn = container_of(work, struct tipc_net, work); + rtnl_lock(); tipc_net_finalize(tipc_link_net(tn->bcl), tn->trial_addr); + rtnl_unlock(); } void tipc_net_stop(struct net *net) diff --git a/net/tipc/topsrv.c b/net/tipc/topsrv.c index 8ee0c07d00e9b..ffe577bf6b515 100644 --- a/net/tipc/topsrv.c +++ b/net/tipc/topsrv.c @@ -704,8 +704,10 @@ static void tipc_topsrv_stop(struct net *net) for (id = 0; srv->idr_in_use; id++) { con = idr_find(&srv->conn_idr, id); if (con) { + conn_get(con); spin_unlock_bh(&srv->idr_lock); tipc_conn_close(con); + conn_put(con); spin_lock_bh(&srv->idr_lock); } } diff --git a/net/tipc/udp_media.c b/net/tipc/udp_media.c index b16ca400ff559..e993bd6ed7c26 100644 --- a/net/tipc/udp_media.c +++ b/net/tipc/udp_media.c @@ -489,7 +489,7 @@ int tipc_udp_nl_dump_remoteip(struct sk_buff *skb, struct netlink_callback *cb) rtnl_lock(); b = tipc_bearer_find(net, bname); - if (!b) { + if (!b || b->bcast_addr.media_id != TIPC_MEDIA_TYPE_UDP) { rtnl_unlock(); return -EINVAL; } @@ -500,7 +500,7 @@ int tipc_udp_nl_dump_remoteip(struct sk_buff *skb, struct netlink_callback *cb) rtnl_lock(); b = rtnl_dereference(tn->bearer_list[bid]); - if (!b) { + if (!b || b->bcast_addr.media_id != TIPC_MEDIA_TYPE_UDP) { rtnl_unlock(); return -EINVAL; } diff --git a/net/tls/tls.h b/net/tls/tls.h index c13f3dc3b0fc0..d2c982f690cbb 100644 --- a/net/tls/tls.h +++ b/net/tls/tls.h @@ -147,6 +147,7 @@ void update_sk_prot(struct sock *sk, struct tls_context *ctx); int wait_on_pending_writer(struct sock *sk, long *timeo); void tls_err_abort(struct sock *sk, int err); +void tls_strp_abort_strp(struct tls_strparser *strp, int err); int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx); void tls_update_rx_zc_capable(struct tls_context *tls_ctx); @@ -198,7 +199,7 @@ void tls_strp_msg_done(struct tls_strparser *strp); int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb); void tls_rx_msg_ready(struct tls_strparser *strp); -void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh); +bool tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh); int tls_strp_msg_cow(struct tls_sw_context_rx *ctx); struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx); int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst); diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 8c94c926606ad..4f72fd26ab405 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -727,8 +727,10 @@ tls_device_rx_resync_async(struct tls_offload_resync_async *resync_async, /* shouldn't get to wraparound: * too long in async stage, something bad happened */ - if (WARN_ON_ONCE(resync_async->rcd_delta == USHRT_MAX)) + if (WARN_ON_ONCE(resync_async->rcd_delta == USHRT_MAX)) { + tls_offload_rx_resync_async_request_cancel(resync_async); return false; + } /* asynchronous stage: log all headers seq such that * req_seq <= seq <= end_seq, and wait for real resync request diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c index e1fe3092505c5..9f060b427c216 100644 --- a/net/tls/tls_main.c +++ b/net/tls/tls_main.c @@ -254,12 +254,9 @@ int tls_process_cmsg(struct sock *sk, struct msghdr *msg, if (msg->msg_flags & MSG_MORE) return -EINVAL; - rc = tls_handle_open_record(sk, msg->msg_flags); - if (rc) - return rc; - *record_type = *(unsigned char *)CMSG_DATA(cmsg); - rc = 0; + + rc = tls_handle_open_record(sk, msg->msg_flags); break; default: return -EINVAL; diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index 1852fac3e72b7..ae723cd6af397 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -12,7 +12,7 @@ static struct workqueue_struct *tls_strp_wq; -static void tls_strp_abort_strp(struct tls_strparser *strp, int err) +void tls_strp_abort_strp(struct tls_strparser *strp, int err) { if (strp->stopped) return; @@ -210,11 +210,17 @@ static int tls_strp_copyin_frag(struct tls_strparser *strp, struct sk_buff *skb, struct sk_buff *in_skb, unsigned int offset, size_t in_len) { + unsigned int nfrag = skb->len / PAGE_SIZE; size_t len, chunk; skb_frag_t *frag; int sz; - frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE]; + if (unlikely(nfrag >= skb_shinfo(skb)->nr_frags)) { + DEBUG_NET_WARN_ON_ONCE(1); + return -EMSGSIZE; + } + + frag = &skb_shinfo(skb)->frags[nfrag]; len = in_len; /* First make sure we got the header */ @@ -474,7 +480,7 @@ static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len) strp->stm.offset = offset; } -void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) +bool tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) { struct strp_msg *rxm; struct tls_msg *tlm; @@ -483,8 +489,11 @@ void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len); if (!strp->copy_mode && force_refresh) { - if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len)) - return; + if (unlikely(tcp_inq(strp->sk) < strp->stm.full_len)) { + WRITE_ONCE(strp->msg_ready, 0); + memset(&strp->stm, 0, sizeof(strp->stm)); + return false; + } tls_strp_load_anchor_with_queue(strp, strp->stm.full_len); } @@ -494,6 +503,8 @@ void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) rxm->offset = strp->stm.offset; tlm = tls_msg(strp->anchor); tlm->control = strp->mark; + + return true; } /* Called with lock held on lower socket */ @@ -511,14 +522,11 @@ static int tls_strp_read_sock(struct tls_strparser *strp) if (inq < strp->stm.full_len) return tls_strp_read_copy(strp, true); + tls_strp_load_anchor_with_queue(strp, inq); if (!strp->stm.full_len) { - tls_strp_load_anchor_with_queue(strp, inq); - sz = tls_rx_msg_size(strp, strp->anchor); - if (sz < 0) { - tls_strp_abort_strp(strp, sz); + if (sz < 0) return sz; - } strp->stm.full_len = sz; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 6e30fe879d538..410e39e4b79fd 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -872,6 +872,19 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, delta = msg->sg.size; psock->eval = sk_psock_msg_verdict(sk, psock, msg); delta -= msg->sg.size; + + if ((s32)delta > 0) { + /* It indicates that we executed bpf_msg_pop_data(), + * causing the plaintext data size to decrease. + * Therefore the encrypted data size also needs to + * correspondingly decrease. We only need to subtract + * delta to calculate the new ciphertext length since + * ktls does not support block encryption. + */ + struct sk_msg *enc = &ctx->open_rec->msg_encrypted; + + sk_msg_trim(sk, enc, enc->sg.size - delta); + } } if (msg->cork_bytes && msg->cork_bytes > msg->sg.size && !enospc && !full_record) { @@ -908,6 +921,13 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, &msg_redir, send, flags); lock_sock(sk); if (err < 0) { + /* Regardless of whether the data represented by + * msg_redir is sent successfully, we have already + * uncharged it via sk_msg_return_zero(). The + * msg->sg.size represents the remaining unprocessed + * data, which needs to be uncharged here. + */ + sk_mem_uncharge(sk, msg->sg.size); *copied -= sk_msg_free_nocharge(sk, &msg_redir); msg->sg.size = 0; } @@ -1034,7 +1054,7 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, if (ret == -EINPROGRESS) num_async++; else if (ret != -EAGAIN) - goto send_end; + goto end; } } @@ -1092,8 +1112,11 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, goto send_end; tls_ctx->pending_open_record_frags = true; - if (sk_msg_full(msg_pl)) + if (sk_msg_full(msg_pl)) { full_record = true; + sk_msg_trim(sk, msg_en, + msg_pl->sg.size + prot->overhead_size); + } if (full_record || eor) goto copied; @@ -1120,11 +1143,22 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, num_async++; else if (ret == -ENOMEM) goto wait_for_memory; - else if (ctx->open_rec && ret == -ENOSPC) + else if (ctx->open_rec && ret == -ENOSPC) { + if (msg_pl->cork_bytes) { + ret = 0; + goto send_end; + } goto rollback_iter; - else if (ret != -EAGAIN) + } else if (ret != -EAGAIN) goto send_end; } + + /* Transmit if any encryptions have completed */ + if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { + cancel_delayed_work(&ctx->tx_work.work); + tls_tx_records(sk, msg->msg_flags); + } + continue; rollback_iter: copied -= try_to_copy; @@ -1180,6 +1214,12 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, goto send_end; } } + + /* Transmit if any encryptions have completed */ + if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { + cancel_delayed_work(&ctx->tx_work.work); + tls_tx_records(sk, msg->msg_flags); + } } continue; @@ -1199,9 +1239,10 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, goto alloc_encrypted; } +send_end: if (!num_async) { - goto send_end; - } else if (num_zc) { + goto end; + } else if (num_zc || eor) { int err; /* Wait for pending encryptions to get completed */ @@ -1218,7 +1259,7 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, tls_tx_records(sk, msg->msg_flags); } -send_end: +end: ret = sk_stream_error(sk, msg->msg_flags, ret); return copied > 0 ? copied : ret; } @@ -1356,7 +1397,8 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, return sock_intr_errno(timeo); } - tls_strp_msg_load(&ctx->strp, released); + if (unlikely(!tls_strp_msg_load(&ctx->strp, released))) + return tls_rx_rec_wait(sk, psock, nonblock, false); return 1; } @@ -1608,8 +1650,10 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, if (unlikely(darg->async)) { err = tls_strp_msg_hold(&ctx->strp, &ctx->async_hold); - if (err) - __skb_queue_tail(&ctx->async_hold, darg->skb); + if (err) { + err = tls_decrypt_async_wait(ctx); + darg->async = false; + } return err; } @@ -1749,6 +1793,9 @@ int decrypt_skb(struct sock *sk, struct scatterlist *sgout) return tls_decrypt_sg(sk, NULL, sgout, &darg); } +/* All records returned from a recvmsg() call must have the same type. + * 0 is not a valid content type. Use it as "no type reported, yet". + */ static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm, u8 *control) { @@ -1992,8 +2039,10 @@ int tls_sw_recvmsg(struct sock *sk, if (err < 0) goto end; + /* process_rx_list() will set @control if it processed any records */ copied = err; - if (len <= copied || (copied && control != TLS_RECORD_TYPE_DATA) || rx_more) + if (len <= copied || rx_more || + (control && control != TLS_RECORD_TYPE_DATA)) goto end; target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); @@ -2411,8 +2460,7 @@ int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb) return data_len + TLS_HEADER_SIZE; read_failure: - tls_err_abort(strp->sk, ret); - + tls_strp_abort_strp(strp, ret); return ret; } diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 542b4610a95e7..5733f659022c4 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -125,6 +125,46 @@ static spinlock_t bsd_socket_locks[UNIX_HASH_SIZE / 2]; * hash table is protected with spinlock. * each socket state is protected by separate spinlock. */ +#ifdef CONFIG_PROVE_LOCKING +#define cmp_ptr(l, r) (((l) > (r)) - ((l) < (r))) + +static int unix_table_lock_cmp_fn(const struct lockdep_map *a, + const struct lockdep_map *b) +{ + return cmp_ptr(a, b); +} + +static int unix_state_lock_cmp_fn(const struct lockdep_map *_a, + const struct lockdep_map *_b) +{ + const struct unix_sock *a, *b; + + a = container_of(_a, struct unix_sock, lock.dep_map); + b = container_of(_b, struct unix_sock, lock.dep_map); + + /* unix_state_double_lock(): ascending address order. */ + return cmp_ptr(a, b); +} + +static int unix_recvq_lock_cmp_fn(const struct lockdep_map *_a, + const struct lockdep_map *_b) +{ + const struct sock *a, *b; + + a = container_of(_a, struct sock, sk_receive_queue.lock.dep_map); + b = container_of(_b, struct sock, sk_receive_queue.lock.dep_map); + + /* unix_collect_skb(): listener -> embryo order. */ + if (a->sk_state == TCP_LISTEN && unix_sk(b)->listener == a) + return -1; + + /* Should never happen. Just to be symmetric. */ + if (b->sk_state == TCP_LISTEN && unix_sk(a)->listener == b) + return 1; + + return 0; +} +#endif static unsigned int unix_unbound_hash(struct sock *sk) { @@ -167,7 +207,7 @@ static void unix_table_double_lock(struct net *net, swap(hash1, hash2); spin_lock(&net->unx.table.locks[hash1]); - spin_lock_nested(&net->unx.table.locks[hash2], SINGLE_DEPTH_NESTING); + spin_lock(&net->unx.table.locks[hash2]); } static void unix_table_double_unlock(struct net *net, @@ -590,6 +630,11 @@ static void unix_sock_destructor(struct sock *sk) #endif } +static unsigned int unix_skb_len(const struct sk_buff *skb) +{ + return skb->len - UNIXCB(skb).consumed; +} + static void unix_release_sock(struct sock *sk, int embrion) { struct unix_sock *u = unix_sk(sk); @@ -617,20 +662,23 @@ static void unix_release_sock(struct sock *sk, int embrion) unix_state_unlock(sk); #if IS_ENABLED(CONFIG_AF_UNIX_OOB) - if (u->oob_skb) { - kfree_skb(u->oob_skb); - u->oob_skb = NULL; - } + u->oob_skb = NULL; #endif wake_up_interruptible_all(&u->peer_wait); if (skpair != NULL) { if (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) { + struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); + +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (skb && !unix_skb_len(skb)) + skb = skb_peek_next(skb, &sk->sk_receive_queue); +#endif unix_state_lock(skpair); /* No more writes */ WRITE_ONCE(skpair->sk_shutdown, SHUTDOWN_MASK); - if (!skb_queue_empty_lockless(&sk->sk_receive_queue) || embrion) + if (skb || embrion) WRITE_ONCE(skpair->sk_err, ECONNRESET); unix_state_unlock(skpair); skpair->sk_state_change(skpair); @@ -977,12 +1025,15 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, sk->sk_write_space = unix_write_space; sk->sk_max_ack_backlog = READ_ONCE(net->unx.sysctl_max_dgram_qlen); sk->sk_destruct = unix_sock_destructor; + lock_set_cmp_fn(&sk->sk_receive_queue.lock, unix_recvq_lock_cmp_fn, NULL); + u = unix_sk(sk); u->listener = NULL; u->vertex = NULL; u->path.dentry = NULL; u->path.mnt = NULL; spin_lock_init(&u->lock); + lock_set_cmp_fn(&u->lock, unix_state_lock_cmp_fn, NULL); mutex_init(&u->iolock); /* single task reading lock */ mutex_init(&u->bindlock); /* single task binding lock */ init_waitqueue_head(&u->peer_wait); @@ -1331,11 +1382,12 @@ static void unix_state_double_lock(struct sock *sk1, struct sock *sk2) unix_state_lock(sk1); return; } + if (sk1 > sk2) swap(sk1, sk2); unix_state_lock(sk1); - unix_state_lock_nested(sk2, U_LOCK_SECOND); + unix_state_lock(sk2); } static void unix_state_double_unlock(struct sock *sk1, struct sock *sk2) @@ -2145,13 +2197,9 @@ static int queue_oob(struct socket *sock, struct msghdr *msg, struct sock *other } maybe_add_creds(skb, sock, other); - skb_get(skb); - scm_stat_add(other, skb); spin_lock(&other->sk_receive_queue.lock); - if (ousk->oob_skb) - consume_skb(ousk->oob_skb); WRITE_ONCE(ousk->oob_skb, skb); __skb_queue_tail(&other->sk_receive_queue, skb); spin_unlock(&other->sk_receive_queue.lock); @@ -2515,11 +2563,6 @@ static long unix_stream_data_wait(struct sock *sk, long timeo, return timeo; } -static unsigned int unix_skb_len(const struct sk_buff *skb) -{ - return skb->len - UNIXCB(skb).consumed; -} - struct unix_stream_read_state { int (*recv_actor)(struct sk_buff *, int, int, struct unix_stream_read_state *); @@ -2534,11 +2577,11 @@ struct unix_stream_read_state { #if IS_ENABLED(CONFIG_AF_UNIX_OOB) static int unix_stream_recv_urg(struct unix_stream_read_state *state) { + struct sk_buff *oob_skb, *read_skb = NULL; struct socket *sock = state->socket; struct sock *sk = sock->sk; struct unix_sock *u = unix_sk(sk); int chunk = 1; - struct sk_buff *oob_skb; mutex_lock(&u->iolock); unix_state_lock(sk); @@ -2553,10 +2596,15 @@ static int unix_stream_recv_urg(struct unix_stream_read_state *state) oob_skb = u->oob_skb; - if (!(state->flags & MSG_PEEK)) + if (!(state->flags & MSG_PEEK)) { WRITE_ONCE(u->oob_skb, NULL); - else - skb_get(oob_skb); + + if (oob_skb->prev != (struct sk_buff *)&sk->sk_receive_queue && + !unix_skb_len(oob_skb->prev)) { + read_skb = oob_skb->prev; + __skb_unlink(read_skb, &sk->sk_receive_queue); + } + } spin_unlock(&sk->sk_receive_queue.lock); unix_state_unlock(sk); @@ -2566,10 +2614,10 @@ static int unix_stream_recv_urg(struct unix_stream_read_state *state) if (!(state->flags & MSG_PEEK)) UNIXCB(oob_skb).consumed += 1; - consume_skb(oob_skb); - mutex_unlock(&u->iolock); + consume_skb(read_skb); + if (chunk < 0) return -EFAULT; @@ -2595,12 +2643,10 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, if (copied) { skb = NULL; } else if (!(flags & MSG_PEEK)) { - if (sock_flag(sk, SOCK_URGINLINE)) { - WRITE_ONCE(u->oob_skb, NULL); - consume_skb(skb); - } else { + WRITE_ONCE(u->oob_skb, NULL); + + if (!sock_flag(sk, SOCK_URGINLINE)) { __skb_unlink(skb, &sk->sk_receive_queue); - WRITE_ONCE(u->oob_skb, NULL); unlinked_skb = skb; skb = skb_peek(&sk->sk_receive_queue); } @@ -2611,10 +2657,7 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, spin_unlock(&sk->sk_receive_queue.lock); - if (unlinked_skb) { - WARN_ON_ONCE(skb_unref(unlinked_skb)); - kfree_skb(unlinked_skb); - } + kfree_skb(unlinked_skb); } return skb; } @@ -2657,7 +2700,6 @@ static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor) unix_state_unlock(sk); if (drop) { - WARN_ON_ONCE(skb_unref(skb)); kfree_skb(skb); return -EAGAIN; } @@ -3598,6 +3640,7 @@ static int __net_init unix_net_init(struct net *net) for (i = 0; i < UNIX_HASH_SIZE; i++) { spin_lock_init(&net->unx.table.locks[i]); + lock_set_cmp_fn(&net->unx.table.locks[i], unix_table_lock_cmp_fn, NULL); INIT_HLIST_HEAD(&net->unx.table.buckets[i]); } diff --git a/net/unix/garbage.c b/net/unix/garbage.c index 23efb78fe9ef4..1f839af199d38 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -106,6 +106,7 @@ static struct unix_vertex *unix_edge_successor(struct unix_edge *edge) /* If an embryo socket has a fd, * the listener indirectly holds the fd's refcnt. */ + if (edge->successor->listener) return unix_sk(edge->successor->listener)->vertex; @@ -136,6 +137,7 @@ enum unix_vertex_index { }; static unsigned long unix_vertex_unvisited_index = UNIX_VERTEX_INDEX_MARK1; +static unsigned long unix_vertex_max_scc_index = UNIX_VERTEX_INDEX_START; static void unix_add_edge(struct scm_fp_list *fpl, struct unix_edge *edge) { @@ -144,6 +146,7 @@ static void unix_add_edge(struct scm_fp_list *fpl, struct unix_edge *edge) if (!vertex) { vertex = list_first_entry(&fpl->vertices, typeof(*vertex), entry); vertex->index = unix_vertex_unvisited_index; + vertex->scc_index = ++unix_vertex_max_scc_index; vertex->out_degree = 0; INIT_LIST_HEAD(&vertex->edges); INIT_LIST_HEAD(&vertex->scc_entry); @@ -337,23 +340,6 @@ static bool unix_vertex_dead(struct unix_vertex *vertex) return true; } -enum unix_recv_queue_lock_class { - U_RECVQ_LOCK_NORMAL, - U_RECVQ_LOCK_EMBRYO, -}; - -static void unix_collect_queue(struct unix_sock *u, struct sk_buff_head *hitlist) -{ - skb_queue_splice_init(&u->sk.sk_receive_queue, hitlist); - -#if IS_ENABLED(CONFIG_AF_UNIX_OOB) - if (u->oob_skb) { - WARN_ON_ONCE(skb_unref(u->oob_skb)); - u->oob_skb = NULL; - } -#endif -} - static void unix_collect_skb(struct list_head *scc, struct sk_buff_head *hitlist) { struct unix_vertex *vertex; @@ -375,13 +361,12 @@ static void unix_collect_skb(struct list_head *scc, struct sk_buff_head *hitlist skb_queue_walk(queue, skb) { struct sk_buff_head *embryo_queue = &skb->sk->sk_receive_queue; - /* listener -> embryo order, the inversion never happens. */ - spin_lock_nested(&embryo_queue->lock, U_RECVQ_LOCK_EMBRYO); - unix_collect_queue(unix_sk(skb->sk), hitlist); + spin_lock(&embryo_queue->lock); + skb_queue_splice_init(embryo_queue, hitlist); spin_unlock(&embryo_queue->lock); } } else { - unix_collect_queue(u, hitlist); + skb_queue_splice_init(queue, hitlist); } spin_unlock(&queue->lock); @@ -498,10 +483,15 @@ static void __unix_walk_scc(struct unix_vertex *vertex, unsigned long *last_inde scc_dead = unix_vertex_dead(v); } - if (scc_dead) + if (scc_dead) { unix_collect_skb(&scc, hitlist); - else if (!unix_graph_maybe_cyclic) - unix_graph_maybe_cyclic = unix_scc_cyclic(&scc); + } else { + if (unix_vertex_max_scc_index < vertex->scc_index) + unix_vertex_max_scc_index = vertex->scc_index; + + if (!unix_graph_maybe_cyclic) + unix_graph_maybe_cyclic = unix_scc_cyclic(&scc); + } list_del(&scc); } @@ -516,6 +506,7 @@ static void unix_walk_scc(struct sk_buff_head *hitlist) unsigned long last_index = UNIX_VERTEX_INDEX_START; unix_graph_maybe_cyclic = false; + unix_vertex_max_scc_index = UNIX_VERTEX_INDEX_START; /* Visit every vertex exactly once. * __unix_walk_scc() moves visited vertices to unix_visited_vertices. diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index f8f1a49689da6..ca1289e64bcc8 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -406,6 +406,8 @@ EXPORT_SYMBOL_GPL(vsock_enqueue_accept); static bool vsock_use_local_transport(unsigned int remote_cid) { + lockdep_assert_held(&vsock_register_mutex); + if (!transport_local) return false; @@ -463,6 +465,8 @@ int vsock_assign_transport(struct vsock_sock *vsk, struct vsock_sock *psk) remote_flags = vsk->remote_addr.svm_flags; + mutex_lock(&vsock_register_mutex); + switch (sk->sk_type) { case SOCK_DGRAM: new_transport = transport_dgram; @@ -478,13 +482,30 @@ int vsock_assign_transport(struct vsock_sock *vsk, struct vsock_sock *psk) new_transport = transport_h2g; break; default: - return -ESOCKTNOSUPPORT; + ret = -ESOCKTNOSUPPORT; + goto err; } - if (vsk->transport) { - if (vsk->transport == new_transport) - return 0; + if (vsk->transport && vsk->transport == new_transport) { + ret = 0; + goto err; + } + /* We increase the module refcnt to prevent the transport unloading + * while there are open sockets assigned to it. + */ + if (!new_transport || !try_module_get(new_transport->module)) { + ret = -ENODEV; + goto err; + } + + /* It's safe to release the mutex after a successful try_module_get(). + * Whichever transport `new_transport` points at, it won't go away until + * the last module_put() below or in vsock_deassign_transport(). + */ + mutex_unlock(&vsock_register_mutex); + + if (vsk->transport) { /* transport->release() must be called with sock lock acquired. * This path can only be taken during vsock_connect(), where we * have already held the sock lock. In the other cases, this @@ -504,12 +525,6 @@ int vsock_assign_transport(struct vsock_sock *vsk, struct vsock_sock *psk) vsk->peer_shutdown = 0; } - /* We increase the module refcnt to prevent the transport unloading - * while there are open sockets assigned to it. - */ - if (!new_transport || !try_module_get(new_transport->module)) - return -ENODEV; - if (sk->sk_type == SOCK_SEQPACKET) { if (!new_transport->seqpacket_allow || !new_transport->seqpacket_allow(remote_cid)) { @@ -527,12 +542,31 @@ int vsock_assign_transport(struct vsock_sock *vsk, struct vsock_sock *psk) vsk->transport = new_transport; return 0; +err: + mutex_unlock(&vsock_register_mutex); + return ret; } EXPORT_SYMBOL_GPL(vsock_assign_transport); +/* + * Provide safe access to static transport_{h2g,g2h,dgram,local} callbacks. + * Otherwise we may race with module removal. Do not use on `vsk->transport`. + */ +static u32 vsock_registered_transport_cid(const struct vsock_transport **transport) +{ + u32 cid = VMADDR_CID_ANY; + + mutex_lock(&vsock_register_mutex); + if (*transport) + cid = (*transport)->get_local_cid(); + mutex_unlock(&vsock_register_mutex); + + return cid; +} + bool vsock_find_cid(unsigned int cid) { - if (transport_g2h && cid == transport_g2h->get_local_cid()) + if (cid == vsock_registered_transport_cid(&transport_g2h)) return true; if (transport_h2g && cid == VMADDR_CID_HOST) @@ -654,7 +688,8 @@ static int __vsock_bind_connectible(struct vsock_sock *vsk, unsigned int i; for (i = 0; i < MAX_PORT_RETRIES; i++) { - if (port <= LAST_RESERVED_PORT) + if (port == VMADDR_PORT_ANY || + port <= LAST_RESERVED_PORT) port = LAST_RESERVED_PORT + 1; new_addr.svm_port = port++; @@ -1515,18 +1550,40 @@ static int vsock_connect(struct socket *sock, struct sockaddr *addr, timeout = schedule_timeout(timeout); lock_sock(sk); - if (signal_pending(current)) { - err = sock_intr_errno(timeout); - sk->sk_state = sk->sk_state == TCP_ESTABLISHED ? TCP_CLOSING : TCP_CLOSE; - sock->state = SS_UNCONNECTED; - vsock_transport_cancel_pkt(vsk); - vsock_remove_connected(vsk); - goto out_wait; - } else if ((sk->sk_state != TCP_ESTABLISHED) && (timeout == 0)) { - err = -ETIMEDOUT; + /* Connection established. Whatever happens to socket once we + * release it, that's not connect()'s concern. No need to go + * into signal and timeout handling. Call it a day. + * + * Note that allowing to "reset" an already established socket + * here is racy and insecure. + */ + if (sk->sk_state == TCP_ESTABLISHED) + break; + + /* If connection was _not_ established and a signal/timeout came + * to be, we want the socket's state reset. User space may want + * to retry. + * + * sk_state != TCP_ESTABLISHED implies that socket is not on + * vsock_connected_table. We keep the binding and the transport + * assigned. + */ + if (signal_pending(current) || timeout == 0) { + err = timeout == 0 ? -ETIMEDOUT : sock_intr_errno(timeout); + + /* Listener might have already responded with + * VIRTIO_VSOCK_OP_RESPONSE. Its handling expects our + * sk_state == TCP_SYN_SENT, which hereby we break. + * In such case VIRTIO_VSOCK_OP_RST will follow. + */ sk->sk_state = TCP_CLOSE; sock->state = SS_UNCONNECTED; + + /* Try to cancel VIRTIO_VSOCK_OP_REQUEST skb sent out by + * transport->connect(). + */ vsock_transport_cancel_pkt(vsk); + goto out_wait; } @@ -2391,18 +2448,19 @@ static long vsock_dev_do_ioctl(struct file *filp, unsigned int cmd, void __user *ptr) { u32 __user *p = ptr; - u32 cid = VMADDR_CID_ANY; int retval = 0; + u32 cid; switch (cmd) { case IOCTL_VM_SOCKETS_GET_LOCAL_CID: /* To be compatible with the VMCI behavior, we prioritize the * guest CID instead of well-know host CID (VMADDR_CID_HOST). */ - if (transport_g2h) - cid = transport_g2h->get_local_cid(); - else if (transport_h2g) - cid = transport_h2g->get_local_cid(); + cid = vsock_registered_transport_cid(&transport_g2h); + if (cid == VMADDR_CID_ANY) + cid = vsock_registered_transport_cid(&transport_h2g); + if (cid == VMADDR_CID_ANY) + cid = vsock_registered_transport_cid(&transport_local); if (put_user(cid, p) != 0) retval = -EFAULT; diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index 2925f5d27ad3f..5d7c68214d574 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -63,6 +63,17 @@ struct virtio_vsock { u32 guest_cid; bool seqpacket_allow; + + /* These fields are used only in tx path in function + * 'virtio_transport_send_pkt_work()', so to save + * stack space in it, place both of them here. Each + * pointer from 'out_sgs' points to the corresponding + * element in 'out_bufs' - this is initialized in + * 'virtio_vsock_probe()'. Both fields are protected + * by 'tx_lock'. +1 is needed for packet header. + */ + struct scatterlist *out_sgs[MAX_SKB_FRAGS + 1]; + struct scatterlist out_bufs[MAX_SKB_FRAGS + 1]; }; static u32 virtio_transport_get_local_cid(void) @@ -100,8 +111,8 @@ virtio_transport_send_pkt_work(struct work_struct *work) vq = vsock->vqs[VSOCK_VQ_TX]; for (;;) { - struct scatterlist hdr, buf, *sgs[2]; int ret, in_sg = 0, out_sg = 0; + struct scatterlist **sgs; struct sk_buff *skb; bool reply; @@ -110,12 +121,43 @@ virtio_transport_send_pkt_work(struct work_struct *work) break; reply = virtio_vsock_skb_reply(skb); - - sg_init_one(&hdr, virtio_vsock_hdr(skb), sizeof(*virtio_vsock_hdr(skb))); - sgs[out_sg++] = &hdr; - if (skb->len > 0) { - sg_init_one(&buf, skb->data, skb->len); - sgs[out_sg++] = &buf; + sgs = vsock->out_sgs; + sg_init_one(sgs[out_sg], virtio_vsock_hdr(skb), + sizeof(*virtio_vsock_hdr(skb))); + out_sg++; + + if (!skb_is_nonlinear(skb)) { + if (skb->len > 0) { + sg_init_one(sgs[out_sg], skb->data, skb->len); + out_sg++; + } + } else { + struct skb_shared_info *si; + int i; + + /* If skb is nonlinear, then its buffer must contain + * only header and nothing more. Data is stored in + * the fragged part. + */ + WARN_ON_ONCE(skb_headroom(skb) != sizeof(*virtio_vsock_hdr(skb))); + + si = skb_shinfo(skb); + + for (i = 0; i < si->nr_frags; i++) { + skb_frag_t *skb_frag = &si->frags[i]; + void *va; + + /* We will use 'page_to_virt()' for the userspace page + * here, because virtio or dma-mapping layers will call + * 'virt_to_phys()' later to fill the buffer descriptor. + * We don't touch memory at "virtual" address of this page. + */ + va = page_to_virt(skb_frag->bv_page); + sg_init_one(sgs[out_sg], + va + skb_frag->bv_offset, + skb_frag->bv_len); + out_sg++; + } } ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, skb, GFP_KERNEL); @@ -221,7 +263,7 @@ virtio_transport_cancel_pkt(struct vsock_sock *vsk) static void virtio_vsock_rx_fill(struct virtio_vsock *vsock) { - int total_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM; + int total_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE; struct scatterlist pkt, *p; struct virtqueue *vq; struct sk_buff *skb; @@ -230,7 +272,7 @@ static void virtio_vsock_rx_fill(struct virtio_vsock *vsock) vq = vsock->vqs[VSOCK_VQ_RX]; do { - skb = virtio_vsock_alloc_skb(total_len, GFP_KERNEL); + skb = virtio_vsock_alloc_linear_skb(total_len, GFP_KERNEL); if (!skb) break; @@ -497,8 +539,9 @@ static void virtio_transport_rx_work(struct work_struct *work) do { virtqueue_disable_cb(vq); for (;;) { + unsigned int len, payload_len; + struct virtio_vsock_hdr *hdr; struct sk_buff *skb; - unsigned int len; if (!virtio_transport_more_replies(vsock)) { /* Stop rx until the device processes already @@ -515,13 +558,22 @@ static void virtio_transport_rx_work(struct work_struct *work) vsock->rx_buf_nr--; /* Drop short/long packets */ - if (unlikely(len < sizeof(struct virtio_vsock_hdr) || + if (unlikely(len < sizeof(*hdr) || len > virtio_vsock_skb_len(skb))) { kfree_skb(skb); continue; } - virtio_vsock_skb_rx_put(skb); + hdr = virtio_vsock_hdr(skb); + payload_len = le32_to_cpu(hdr->len); + if (unlikely(payload_len > len - sizeof(*hdr))) { + kfree_skb(skb); + continue; + } + + if (payload_len) + virtio_vsock_skb_put(skb, payload_len); + virtio_transport_deliver_tap_pkt(skb); virtio_transport_recv_pkt(&virtio_transport, skb); } @@ -637,6 +689,7 @@ static int virtio_vsock_probe(struct virtio_device *vdev) { struct virtio_vsock *vsock = NULL; int ret; + int i; ret = mutex_lock_interruptible(&the_virtio_vsock_mutex); if (ret) @@ -679,6 +732,9 @@ static int virtio_vsock_probe(struct virtio_device *vdev) if (ret < 0) goto out; + for (i = 0; i < ARRAY_SIZE(vsock->out_sgs); i++) + vsock->out_sgs[i] = &vsock->out_bufs[i]; + rcu_assign_pointer(the_virtio_vsock, vsock); virtio_vsock_vqs_start(vsock); diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index e289b8f9d75c0..603fa52ac204e 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -29,10 +29,6 @@ static void virtio_transport_cancel_close_work(struct vsock_sock *vsk, bool cancel_timeout); -uint virtio_transport_max_vsock_pkt_buf_size = 64 * 1024; -module_param(virtio_transport_max_vsock_pkt_buf_size, uint, 0444); -EXPORT_SYMBOL_GPL(virtio_transport_max_vsock_pkt_buf_size); - static const struct virtio_transport * virtio_transport_get_ops(struct vsock_sock *vsk) { @@ -59,7 +55,6 @@ virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info, const size_t skb_len = VIRTIO_VSOCK_SKB_HEADROOM + len; struct virtio_vsock_hdr *hdr; struct sk_buff *skb; - void *payload; int err; skb = virtio_vsock_alloc_skb(skb_len, GFP_KERNEL); @@ -79,8 +74,8 @@ virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info, hdr->fwd_cnt = cpu_to_le32(0); if (info->msg && len > 0) { - payload = skb_put(skb, len); - err = memcpy_from_msg(payload, info->msg, len); + virtio_vsock_skb_put(skb, len); + err = skb_copy_datagram_from_iter_full(skb, 0, &info->msg->msg_iter, len); if (err) goto out; @@ -115,6 +110,27 @@ virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info, return NULL; } +static void virtio_transport_copy_nonlinear_skb(const struct sk_buff *skb, + void *dst, + size_t len) +{ + struct iov_iter iov_iter = { 0 }; + struct kvec kvec; + size_t to_copy; + + kvec.iov_base = dst; + kvec.iov_len = len; + + iov_iter.iter_type = ITER_KVEC; + iov_iter.kvec = &kvec; + iov_iter.nr_segs = 1; + + to_copy = min_t(size_t, len, skb->len); + + skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset, + &iov_iter, to_copy); +} + /* Packet capture */ static struct sk_buff *virtio_transport_build_skb(void *opaque) { @@ -123,7 +139,6 @@ static struct sk_buff *virtio_transport_build_skb(void *opaque) struct af_vsockmon_hdr *hdr; struct sk_buff *skb; size_t payload_len; - void *payload_buf; /* A packet could be split to fit the RX buffer, so we can retrieve * the payload length from the header and the buffer pointer taking @@ -131,7 +146,6 @@ static struct sk_buff *virtio_transport_build_skb(void *opaque) */ pkt_hdr = virtio_vsock_hdr(pkt); payload_len = pkt->len; - payload_buf = pkt->data; skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len, GFP_ATOMIC); @@ -174,7 +188,13 @@ static struct sk_buff *virtio_transport_build_skb(void *opaque) skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr)); if (payload_len) { - skb_put_data(skb, payload_buf, payload_len); + if (skb_is_nonlinear(pkt)) { + void *data = skb_put(skb, payload_len); + + virtio_transport_copy_nonlinear_skb(pkt, data, payload_len); + } else { + skb_put_data(skb, pkt->data, payload_len); + } } return skb; @@ -206,6 +226,24 @@ static u16 virtio_transport_get_type(struct sock *sk) static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, struct virtio_vsock_pkt_info *info) { + /* ANDROID: + * + * Older host kernels (including the 5.10-based images used by + * Cuttlefish) only support linear SKBs on the RX path. + * Consequently, if we transmit a VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + * packet, the host allocation can fail and the packet will be + * silently dropped. + * + * As a nasty workaround, limit the entire SKB to ~28KiB, which + * allows for 4KiB of SKB wiggle room whilst keeping the + * allocation below PAGE_ALLOC_COSTLY_ORDER. + * + * This can be removed when all supported host kernels have + * support for non-linear RX buffers introduced by Change-Id + * I4212a8daf9f19b5bbffc06ce93338c823de7bb19. + */ + u32 max_skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE, + SKB_WITH_OVERHEAD(SZ_32K - VIRTIO_VSOCK_SKB_HEADROOM) - SZ_4K); u32 src_cid, src_port, dst_cid, dst_port; const struct virtio_transport *t_ops; struct virtio_vsock_sock *vvs; @@ -244,7 +282,7 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, struct sk_buff *skb; size_t skb_len; - skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE, rest_len); + skb_len = min_t(u32, max_skb_len, rest_len); skb = virtio_transport_alloc_skb(info, skb_len, src_cid, src_port, @@ -373,9 +411,10 @@ virtio_transport_stream_do_peek(struct vsock_sock *vsk, spin_unlock_bh(&vvs->rx_lock); /* sk_lock is held by caller so no one else can dequeue. - * Unlock rx_lock since memcpy_to_msg() may sleep. + * Unlock rx_lock since skb_copy_datagram_iter() may sleep. */ - err = memcpy_to_msg(msg, skb->data, bytes); + err = skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset, + &msg->msg_iter, bytes); if (err) goto out; @@ -421,25 +460,27 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, while (total < len && !skb_queue_empty(&vvs->rx_queue)) { skb = skb_peek(&vvs->rx_queue); - bytes = len - total; - if (bytes > skb->len) - bytes = skb->len; + bytes = min_t(size_t, len - total, + skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset); /* sk_lock is held by caller so no one else can dequeue. - * Unlock rx_lock since memcpy_to_msg() may sleep. + * Unlock rx_lock since skb_copy_datagram_iter() may sleep. */ spin_unlock_bh(&vvs->rx_lock); - err = memcpy_to_msg(msg, skb->data, bytes); + err = skb_copy_datagram_iter(skb, + VIRTIO_VSOCK_SKB_CB(skb)->offset, + &msg->msg_iter, bytes); if (err) goto out; spin_lock_bh(&vvs->rx_lock); total += bytes; - skb_pull(skb, bytes); - if (skb->len == 0) { + VIRTIO_VSOCK_SKB_CB(skb)->offset += bytes; + + if (skb->len == VIRTIO_VSOCK_SKB_CB(skb)->offset) { u32 pkt_len = le32_to_cpu(virtio_vsock_hdr(skb)->len); virtio_transport_dec_rx_pkt(vvs, pkt_len); @@ -508,9 +549,10 @@ virtio_transport_seqpacket_do_peek(struct vsock_sock *vsk, spin_unlock_bh(&vvs->rx_lock); /* sk_lock is held by caller so no one else can dequeue. - * Unlock rx_lock since memcpy_to_msg() may sleep. + * Unlock rx_lock since skb_copy_datagram_iter() may sleep. */ - err = memcpy_to_msg(msg, skb->data, bytes); + err = skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset, + &msg->msg_iter, bytes); if (err) return err; @@ -569,11 +611,13 @@ static int virtio_transport_seqpacket_do_dequeue(struct vsock_sock *vsk, int err; /* sk_lock is held by caller so no one else can dequeue. - * Unlock rx_lock since memcpy_to_msg() may sleep. + * Unlock rx_lock since skb_copy_datagram_iter() may sleep. */ spin_unlock_bh(&vvs->rx_lock); - err = memcpy_to_msg(msg, skb->data, bytes_to_copy); + err = skb_copy_datagram_iter(skb, 0, + &msg->msg_iter, + bytes_to_copy); if (err) { /* Copy of message failed. Rest of * fragments will be freed without copy. diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index b370070194fa4..7eccd6708d664 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -119,6 +119,8 @@ vmci_transport_packet_init(struct vmci_transport_packet *pkt, u16 proto, struct vmci_handle handle) { + memset(pkt, 0, sizeof(*pkt)); + /* We register the stream control handler as an any cid handle so we * must always send from a source address of VMADDR_CID_ANY */ @@ -131,8 +133,6 @@ vmci_transport_packet_init(struct vmci_transport_packet *pkt, pkt->type = type; pkt->src_port = src->svm_port; pkt->dst_port = dst->svm_port; - memset(&pkt->proto, 0, sizeof(pkt->proto)); - memset(&pkt->_reserved2, 0, sizeof(pkt->_reserved2)); switch (pkt->type) { case VMCI_TRANSPORT_PACKET_TYPE_INVALID: diff --git a/net/wireless/core.c b/net/wireless/core.c index a2b15349324b6..f6693983b5e98 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -550,6 +550,9 @@ struct wiphy *wiphy_new_nm(const struct cfg80211_ops *ops, int sizeof_priv, INIT_WORK(&rdev->mgmt_registrations_update_wk, cfg80211_mgmt_registrations_update_wk); spin_lock_init(&rdev->mgmt_registrations_lock); + INIT_WORK(&rdev->wiphy_work, cfg80211_wiphy_work); + INIT_LIST_HEAD(&rdev->wiphy_work_list); + spin_lock_init(&rdev->wiphy_work_lock); #ifdef CONFIG_CFG80211_DEFAULT_PS rdev->wiphy.flags |= WIPHY_FLAG_PS_ON_BY_DEFAULT; @@ -567,9 +570,6 @@ struct wiphy *wiphy_new_nm(const struct cfg80211_ops *ops, int sizeof_priv, return NULL; } - INIT_WORK(&rdev->wiphy_work, cfg80211_wiphy_work); - INIT_LIST_HEAD(&rdev->wiphy_work_list); - spin_lock_init(&rdev->wiphy_work_lock); INIT_WORK(&rdev->rfkill_block, cfg80211_rfkill_block_work); INIT_WORK(&rdev->conn_work, cfg80211_conn_work); INIT_WORK(&rdev->event_work, cfg80211_event_work); diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index 55a1d3633853f..3d631f8073f06 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -739,7 +739,8 @@ int cfg80211_mlme_mgmt_tx(struct cfg80211_registered_device *rdev, mgmt = (const struct ieee80211_mgmt *)params->buf; - if (!ieee80211_is_mgmt(mgmt->frame_control)) + if (!ieee80211_is_mgmt(mgmt->frame_control) || + ieee80211_has_order(mgmt->frame_control)) return -EINVAL; stype = le16_to_cpu(mgmt->frame_control) & IEEE80211_FCTL_STYPE; diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index f2cc9fa33f3c3..6eb357ca75b2b 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -1208,12 +1208,6 @@ static int nl80211_msg_put_channel(struct sk_buff *msg, struct wiphy *wiphy, if ((chan->flags & IEEE80211_CHAN_DFS_CONCURRENT) && nla_put_flag(msg, NL80211_FREQUENCY_ATTR_DFS_CONCURRENT)) goto nla_put_failure; - if ((chan->flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT) && - nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_6GHZ_VLP_CLIENT)) - goto nla_put_failure; - if ((chan->flags & IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT) && - nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_6GHZ_AFC_CLIENT)) - goto nla_put_failure; } if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_MAX_TX_POWER, diff --git a/net/wireless/reg.c b/net/wireless/reg.c index 8ee613917a3fe..d88b4745faf43 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -5,7 +5,7 @@ * Copyright 2008-2011 Luis R. Rodriguez * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2017 Intel Deutschland GmbH - * Copyright (C) 2018 - 2024 Intel Corporation + * Copyright (C) 2018 - 2023 Intel Corporation * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -1592,10 +1592,6 @@ static u32 map_regdom_flags(u32 rd_flags) channel_flags |= IEEE80211_CHAN_NO_EHT; if (rd_flags & NL80211_RRF_DFS_CONCURRENT) channel_flags |= IEEE80211_CHAN_DFS_CONCURRENT; - if (rd_flags & NL80211_RRF_NO_6GHZ_VLP_CLIENT) - channel_flags |= IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT; - if (rd_flags & NL80211_RRF_NO_6GHZ_AFC_CLIENT) - channel_flags |= IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT; if (rd_flags & NL80211_RRF_PSD) channel_flags |= IEEE80211_CHAN_PSD; return channel_flags; @@ -4235,6 +4231,9 @@ EXPORT_SYMBOL(regulatory_pre_cac_allowed); static void cfg80211_check_and_end_cac(struct cfg80211_registered_device *rdev) { struct wireless_dev *wdev; + + wiphy_lock(&rdev->wiphy); + /* If we finished CAC or received radar, we should end any * CAC running on the same channels. * the check !cfg80211_chandef_dfs_usable contain 2 options: @@ -4258,6 +4257,8 @@ static void cfg80211_check_and_end_cac(struct cfg80211_registered_device *rdev) if (!cfg80211_chandef_dfs_usable(&rdev->wiphy, chandef)) rdev_end_cac(rdev, wdev->netdev); } + + wiphy_unlock(&rdev->wiphy); } void regulatory_propagate_dfs_state(struct wiphy *wiphy, diff --git a/net/wireless/scan.c b/net/wireless/scan.c index dc16dac10b3e6..d28a48cdf88d6 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -5,7 +5,7 @@ * Copyright 2008 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2016 Intel Deutschland GmbH - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2023 Intel Corporation */ #include #include @@ -1850,7 +1850,8 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, */ f = rcu_access_pointer(new->pub.beacon_ies); - kfree_rcu((struct cfg80211_bss_ies *)f, rcu_head); + if (!new->pub.hidden_beacon_bss) + kfree_rcu((struct cfg80211_bss_ies *)f, rcu_head); return false; } @@ -2139,36 +2140,6 @@ struct cfg80211_inform_single_bss_data { u64 cannot_use_reasons; }; -static bool cfg80211_6ghz_power_type_valid(const u8 *ie, size_t ielen, - const u32 flags) -{ - const struct element *tmp; - struct ieee80211_he_operation *he_oper; - - tmp = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ie, ielen); - if (tmp && tmp->datalen >= sizeof(*he_oper) + 1 && - tmp->datalen >= ieee80211_he_oper_size(tmp->data + 1)) { - const struct ieee80211_he_6ghz_oper *he_6ghz_oper; - - he_oper = (void *)&tmp->data[1]; - he_6ghz_oper = ieee80211_he_6ghz_oper(he_oper); - - if (!he_6ghz_oper) - return false; - - switch (u8_get_bits(he_6ghz_oper->control, - IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO)) { - case IEEE80211_6GHZ_CTRL_REG_LPI_AP: - return true; - case IEEE80211_6GHZ_CTRL_REG_SP_AP: - return !(flags & IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT); - case IEEE80211_6GHZ_CTRL_REG_VLP_AP: - return !(flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT); - } - } - return false; -} - /* Returned bss is reference counted and must be cleaned up appropriately. */ static struct cfg80211_bss * cfg80211_inform_single_bss_data(struct wiphy *wiphy, @@ -2201,14 +2172,6 @@ cfg80211_inform_single_bss_data(struct wiphy *wiphy, if (!channel) return NULL; - if (channel->band == NL80211_BAND_6GHZ && - !cfg80211_6ghz_power_type_valid(data->ie, data->ielen, - channel->flags)) { - data->use_for = 0; - data->cannot_use_reasons = - NL80211_BSS_CANNOT_USE_6GHZ_PWR_MISMATCH; - } - memcpy(tmp.pub.bssid, data->bssid, ETH_ALEN); tmp.pub.channel = channel; if (data->bss_source != BSS_SOURCE_STA_PROFILE) @@ -2607,7 +2570,7 @@ cfg80211_defrag_mle(const struct element *mle, const u8 *ie, size_t ielen, /* Required length for first defragmentation */ buf_len = mle->datalen - 1; for_each_element(elem, mle->data + mle->datalen, - ielen - sizeof(*mle) + mle->datalen) { + ie + ielen - mle->data - mle->datalen) { if (elem->id != WLAN_EID_FRAGMENT) break; diff --git a/net/wireless/sme.c b/net/wireless/sme.c index e4d0abf7884c2..4219f0cb9436d 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -915,13 +915,16 @@ void __cfg80211_connect_result(struct net_device *dev, if (!wdev->u.client.ssid_len) { rcu_read_lock(); for_each_valid_link(cr, link) { + u32 ssid_len; + ssid = ieee80211_bss_get_elem(cr->links[link].bss, WLAN_EID_SSID); if (!ssid || !ssid->datalen) continue; - memcpy(wdev->u.client.ssid, ssid->data, ssid->datalen); + ssid_len = min(ssid->datalen, IEEE80211_MAX_SSID_LEN); + memcpy(wdev->u.client.ssid, ssid->data, ssid_len); wdev->u.client.ssid_len = ssid->datalen; break; } diff --git a/net/wireless/util.c b/net/wireless/util.c index 10c461a7f5a60..2fd94d3b740d7 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -820,6 +820,52 @@ bool ieee80211_is_valid_amsdu(struct sk_buff *skb, u8 mesh_hdr) } EXPORT_SYMBOL(ieee80211_is_valid_amsdu); + +/* + * Detects if an MSDU frame was maliciously converted into an A-MSDU + * frame by an adversary. This is done by parsing the received frame + * as if it were a regular MSDU, even though the A-MSDU flag is set. + * + * For non-mesh interfaces, detection involves checking whether the + * payload, when interpreted as an MSDU, begins with a valid RFC1042 + * header. This is done by comparing the A-MSDU subheader's destination + * address to the start of the RFC1042 header. + * + * For mesh interfaces, the MSDU includes a 6-byte Mesh Control field + * and an optional variable-length Mesh Address Extension field before + * the RFC1042 header. The position of the RFC1042 header must therefore + * be calculated based on the mesh header length. + * + * Since this function intentionally parses an A-MSDU frame as an MSDU, + * it only assumes that the A-MSDU subframe header is present, and + * beyond this it performs its own bounds checks under the assumption + * that the frame is instead parsed as a non-aggregated MSDU. + */ +static bool +is_amsdu_aggregation_attack(struct ethhdr *eth, struct sk_buff *skb, + enum nl80211_iftype iftype) +{ + int offset; + + /* Non-mesh case can be directly compared */ + if (iftype != NL80211_IFTYPE_MESH_POINT) + return ether_addr_equal(eth->h_dest, rfc1042_header); + + offset = __ieee80211_get_mesh_hdrlen(eth->h_dest[0]); + if (offset == 6) { + /* Mesh case with empty address extension field */ + return ether_addr_equal(eth->h_source, rfc1042_header); + } else if (offset + ETH_ALEN <= skb->len) { + /* Mesh case with non-empty address extension field */ + u8 temp[ETH_ALEN]; + + skb_copy_bits(skb, offset, temp, ETH_ALEN); + return ether_addr_equal(temp, rfc1042_header); + } + + return false; +} + void ieee80211_amsdu_to_8023s(struct sk_buff *skb, struct sk_buff_head *list, const u8 *addr, enum nl80211_iftype iftype, const unsigned int extra_headroom, @@ -861,8 +907,10 @@ void ieee80211_amsdu_to_8023s(struct sk_buff *skb, struct sk_buff_head *list, /* the last MSDU has no padding */ if (subframe_len > remaining) goto purge; - /* mitigate A-MSDU aggregation injection attacks */ - if (ether_addr_equal(hdr.eth.h_dest, rfc1042_header)) + /* mitigate A-MSDU aggregation injection attacks, to be + * checked when processing first subframe (offset == 0). + */ + if (offset == 0 && is_amsdu_aggregation_attack(&hdr.eth, skb, iftype)) goto purge; offset += sizeof(struct ethhdr); diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index d3b3f9e720b3b..427072285b8c7 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -169,8 +169,10 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) { struct espintcp_ctx *ctx = espintcp_getctx(sk); - if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog)) + if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog)) { + kfree_skb(skb); return -ENOBUFS; + } __skb_queue_tail(&ctx->out_queue, skb); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 04dc0c8a83707..7188d3592dde4 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -371,7 +371,6 @@ int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, xdo->dev = dev; netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC); - xdo->real_dev = dev; xdo->type = XFRM_DEV_OFFLOAD_PACKET; switch (dir) { case XFRM_POLICY_IN: @@ -393,7 +392,6 @@ int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, err = dev->xfrmdev_ops->xdo_dev_policy_add(xp, extack); if (err) { xdo->dev = NULL; - xdo->real_dev = NULL; xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; xdo->dir = 0; netdev_put(dev, &xdo->dev_tracker); diff --git a/net/xfrm/xfrm_interface_core.c b/net/xfrm/xfrm_interface_core.c index e21cc71095bb2..ca6db1e960ce6 100644 --- a/net/xfrm/xfrm_interface_core.c +++ b/net/xfrm/xfrm_interface_core.c @@ -880,7 +880,7 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[], return -EINVAL; } - if (p.collect_md) { + if (p.collect_md || xi->p.collect_md) { NL_SET_ERR_MSG(extack, "collect_md can't be changed"); return -EINVAL; } @@ -891,11 +891,6 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[], } else { if (xi->dev != dev) return -EEXIST; - if (xi->p.collect_md) { - NL_SET_ERR_MSG(extack, - "device can't be changed to collect_md"); - return -EINVAL; - } } return xfrmi_update(xi, &p); diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index db3116646040f..85f8acd20b721 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -764,8 +764,12 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb) /* Exclusive direct xmit for tunnel mode, as * some filtering or matching rules may apply * in transport mode. + * Locally generated packets also require + * the normal XFRM path for L2 header setup, + * as the hardware needs the L2 header to match + * for encryption, so skip direct output as well. */ - if (x->props.mode == XFRM_MODE_TUNNEL) + if (x->props.mode == XFRM_MODE_TUNNEL && !skb->sk) return xfrm_dev_direct_output(sk, x, skb); return xfrm_output_resume(sk, skb, 0); diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index f973e140facf5..95c11e1b24c4c 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -1603,6 +1603,9 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) struct xfrm_policy *delpol; struct hlist_head *chain; + /* Sanitize mark before store */ + policy->mark.v &= policy->mark.m; + spin_lock_bh(&net->xfrm.xfrm_policy_lock); chain = policy_hash_bysel(net, &policy->selector, policy->family, dir); if (chain) diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 35a375605606b..39758d8e78ce8 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -1271,7 +1271,6 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, xso->type = XFRM_DEV_OFFLOAD_PACKET; xso->dir = xdo->dir; xso->dev = xdo->dev; - xso->real_dev = xdo->real_dev; xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ; netdev_hold(xso->dev, &xso->dev_tracker, GFP_ATOMIC); error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL); @@ -1279,7 +1278,6 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, xso->dir = 0; netdev_put(xso->dev, &xso->dev_tracker); xso->dev = NULL; - xso->real_dev = NULL; xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; x->km.state = XFRM_STATE_DEAD; to_put = x; @@ -1420,6 +1418,9 @@ static void __xfrm_state_insert(struct xfrm_state *x) list_add(&x->km.all, &net->xfrm.state_all); + /* Sanitize mark before store */ + x->mark.v &= x->mark.m; + h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, x->props.reqid, x->props.family); XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index c91a3c24f6070..5416f21918e04 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -12,8 +12,10 @@ #include #include #include +#include /* `bindgen` gets confused at certain things. */ const size_t BINDINGS_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO; +const vm_flags_t BINDINGS_VM_MERGEABLE = VM_MERGEABLE; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 9bcbea04dac30..7d9078b94a8f0 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -51,3 +51,4 @@ pub use bindings_raw::*; pub const GFP_KERNEL: gfp_t = BINDINGS_GFP_KERNEL; pub const __GFP_ZERO: gfp_t = BINDINGS___GFP_ZERO; +pub const VM_MERGEABLE: vm_flags_t = BINDINGS_VM_MERGEABLE; diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 7dee348ef0cc8..7614a7198ce20 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -249,6 +249,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(MODULE)] #[doc(hidden)] #[no_mangle] + #[link_section = \".exit.text\"] pub extern \"C\" fn cleanup_module() {{ // SAFETY: // - This function is inaccessible to the outside due to the double diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 3fa16412db15c..927d72659173e 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -392,7 +392,7 @@ $(obj)/%.o: $(src)/%.c @echo " CLANG-bpf " $@ $(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(BPF_EXTRA_CFLAGS) \ -I$(obj) -I$(srctree)/tools/testing/selftests/bpf/ \ - -I$(LIBBPF_INCLUDE) \ + -I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \ -D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \ -D__TARGET_ARCH_$(SRCARCH) -Wno-compare-distinct-pointer-types \ -Wno-gnu-variable-sized-type-not-at-end \ diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c index e2a6a69352dfb..b40f85e3806fc 100644 --- a/samples/ftrace/ftrace-direct-modify.c +++ b/samples/ftrace/ftrace-direct-modify.c @@ -40,8 +40,8 @@ asm ( CALL_DEPTH_ACCOUNT " call my_direct_func1\n" " leave\n" -" .size my_tramp1, .-my_tramp1\n" ASM_RET +" .size my_tramp1, .-my_tramp1\n" " .type my_tramp2, @function\n" " .globl my_tramp2\n" diff --git a/samples/ftrace/sample-trace-array.c b/samples/ftrace/sample-trace-array.c index d1b2f4f9c5137..2cb61492102da 100644 --- a/samples/ftrace/sample-trace-array.c +++ b/samples/ftrace/sample-trace-array.c @@ -113,7 +113,7 @@ static int __init sample_trace_array_init(void) /* * If context specific per-cpu buffers havent already been allocated. */ - trace_printk_init_buffers(); + trace_array_init_printk(tr); simple_tsk = kthread_run(simple_thread, NULL, "sample-instance"); if (IS_ERR(simple_tsk)) { diff --git a/samples/mei/mei-amt-version.c b/samples/mei/mei-amt-version.c index 867debd3b9124..1d7254bcb44cb 100644 --- a/samples/mei/mei-amt-version.c +++ b/samples/mei/mei-amt-version.c @@ -69,11 +69,11 @@ #include #include #include +#include #include #include #include #include -#include #include /***************************************************************************** diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index 0ea3281a92e1e..570f64e2ddf10 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -29,6 +29,18 @@ KBUILD_CFLAGS-$(CONFIG_CC_NO_ARRAY_BOUNDS) += -Wno-array-bounds ifdef CONFIG_CC_IS_CLANG # The kernel builds with '-std=gnu11' so use of GNU extensions is acceptable. KBUILD_CFLAGS += -Wno-gnu + +# Clang may emit a warning when a const variable, such as the dummy variables +# in typecheck(), or const member of an aggregate type are not initialized, +# which can result in unexpected behavior. However, in many audited cases of +# the "field" variant of the warning, this is intentional because the field is +# never used within a particular call path, the field is within a union with +# other non-const members, or the containing object is not const so the field +# can be modified via memcpy() / memset(). While the variable warning also gets +# disabled with this same switch, there should not be too much coverage lost +# because -Wuninitialized will still flag when an uninitialized const variable +# is used. +KBUILD_CFLAGS += $(call cc-disable-warning, default-const-init-unsafe) else # gcc inanely warns about local variables called 'main' diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index 390658a2d5b74..a57c24c129720 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -68,10 +68,14 @@ CFLAGS_KASAN := -fsanitize=kernel-hwaddress \ $(call cc-param,hwasan-inline-all-checks=0) \ $(instrumentation_flags) -# Instrument memcpy/memset/memmove calls by using instrumented __hwasan_mem*(). -ifeq ($(call clang-min-version, 150000)$(call gcc-min-version, 130000),y) -CFLAGS_KASAN += $(call cc-param,hwasan-kernel-mem-intrinsic-prefix=1) -endif +# Instrument memcpy/memset/memmove calls by using instrumented __(hw)asan_mem*(). +ifdef CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX + ifdef CONFIG_CC_IS_GCC + CFLAGS_KASAN += $(call cc-param,asan-kernel-mem-intrinsic-prefix=1) + else + CFLAGS_KASAN += $(call cc-param,hwasan-kernel-mem-intrinsic-prefix=1) + endif +endif # CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX endif # CONFIG_KASAN_SW_TAGS diff --git a/scripts/checkstack.pl b/scripts/checkstack.pl index f27d552aec43f..aad423c5181a9 100755 --- a/scripts/checkstack.pl +++ b/scripts/checkstack.pl @@ -68,9 +68,6 @@ # 2f60: 48 81 ec e8 05 00 00 sub $0x5e8,%rsp $re = qr/^.*[as][du][db] \$(0x$x{1,8}),\%(e|r)sp$/o; $dre = qr/^.*[as][du][db] (%.*),\%(e|r)sp$/o; - } elsif ($arch eq 'ia64') { - #e0000000044011fc: 01 0f fc 8c adds r12=-384,r12 - $re = qr/.*adds.*r12=-(([0-9]{2}|[3-9])[0-9]{2}),r12/o; } elsif ($arch eq 'm68k') { # 2b6c: 4e56 fb70 linkw %fp,#-1168 # 1df770: defc ffe4 addaw #-28,%sp diff --git a/scripts/config b/scripts/config index ff88e2faefd35..ea475c07de283 100755 --- a/scripts/config +++ b/scripts/config @@ -32,6 +32,7 @@ commands: Disable option directly after other option --module-after|-M beforeopt option Turn option into module directly after other option + --refresh Refresh the config using old settings commands can be repeated multiple times @@ -124,16 +125,22 @@ undef_var() { txt_delete "^# $name is not set" "$FN" } -if [ "$1" = "--file" ]; then - FN="$2" - if [ "$FN" = "" ] ; then - usage +FN=.config +CMDS=() +while [[ $# -gt 0 ]]; do + if [ "$1" = "--file" ]; then + if [ "$2" = "" ]; then + usage + fi + FN="$2" + shift 2 + else + CMDS+=("$1") + shift fi - shift 2 -else - FN=.config -fi +done +set -- "${CMDS[@]}" if [ "$1" = "" ] ; then usage fi @@ -217,9 +224,8 @@ while [ "$1" != "" ] ; do set_var "${CONFIG_}$B" "${CONFIG_}$B=m" "${CONFIG_}$A" ;; - # undocumented because it ignores --file (fixme) --refresh) - yes "" | make oldconfig + yes "" | make oldconfig KCONFIG_CONFIG=$FN ;; *) diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 1ae39b9f4a95e..de1e4c927cbc7 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -128,6 +128,38 @@ static inline tree build_const_char_string(int len, const char *str) return cstr; } +static inline void __add_type_attr(tree type, const char *attr, tree args) +{ + tree oldattr; + + if (type == NULL_TREE) + return; + oldattr = lookup_attribute(attr, TYPE_ATTRIBUTES(type)); + if (oldattr != NULL_TREE) { + gcc_assert(TREE_VALUE(oldattr) == args || TREE_VALUE(TREE_VALUE(oldattr)) == TREE_VALUE(args)); + return; + } + + TYPE_ATTRIBUTES(type) = copy_list(TYPE_ATTRIBUTES(type)); + TYPE_ATTRIBUTES(type) = tree_cons(get_identifier(attr), args, TYPE_ATTRIBUTES(type)); +} + +static inline void add_type_attr(tree type, const char *attr, tree args) +{ + tree main_variant = TYPE_MAIN_VARIANT(type); + + __add_type_attr(TYPE_CANONICAL(type), attr, args); + __add_type_attr(TYPE_CANONICAL(main_variant), attr, args); + __add_type_attr(main_variant, attr, args); + + for (type = TYPE_NEXT_VARIANT(main_variant); type; type = TYPE_NEXT_VARIANT(type)) { + if (!lookup_attribute(attr, TYPE_ATTRIBUTES(type))) + TYPE_ATTRIBUTES(type) = TYPE_ATTRIBUTES(main_variant); + + __add_type_attr(TYPE_CANONICAL(type), attr, args); + } +} + #define PASS_INFO(NAME, REF, ID, POS) \ struct register_pass_info NAME##_pass_info = { \ .pass = make_##NAME##_pass(), \ @@ -164,10 +196,17 @@ inline bool is_a_helper::test(const_gimple gs) } #endif +#if BUILDING_GCC_VERSION < 16000 #define TODO_verify_ssa TODO_verify_il #define TODO_verify_flow TODO_verify_il #define TODO_verify_stmts TODO_verify_il #define TODO_verify_rtl_sharing TODO_verify_il +#else +#define TODO_verify_ssa 0 +#define TODO_verify_flow 0 +#define TODO_verify_stmts 0 +#define TODO_verify_rtl_sharing 0 +#endif #define INSN_DELETED_P(insn) (insn)->deleted() diff --git a/scripts/gcc-plugins/randomize_layout_plugin.c b/scripts/gcc-plugins/randomize_layout_plugin.c index 746ff2d272f25..e70eef049ada6 100644 --- a/scripts/gcc-plugins/randomize_layout_plugin.c +++ b/scripts/gcc-plugins/randomize_layout_plugin.c @@ -77,6 +77,9 @@ static tree handle_randomize_layout_attr(tree *node, tree name, tree args, int f if (TYPE_P(*node)) { type = *node; + } else if (TREE_CODE(*node) == FIELD_DECL) { + *no_add_attrs = false; + return NULL_TREE; } else { gcc_assert(TREE_CODE(*node) == TYPE_DECL); type = TREE_TYPE(*node); @@ -348,35 +351,18 @@ static int relayout_struct(tree type) shuffle(type, (tree *)newtree, shuffle_length); - /* - * set up a bogus anonymous struct field designed to error out on unnamed struct initializers - * as gcc provides no other way to detect such code - */ - list = make_node(FIELD_DECL); - TREE_CHAIN(list) = newtree[0]; - TREE_TYPE(list) = void_type_node; - DECL_SIZE(list) = bitsize_zero_node; - DECL_NONADDRESSABLE_P(list) = 1; - DECL_FIELD_BIT_OFFSET(list) = bitsize_zero_node; - DECL_SIZE_UNIT(list) = size_zero_node; - DECL_FIELD_OFFSET(list) = size_zero_node; - DECL_CONTEXT(list) = type; - // to satisfy the constify plugin - TREE_READONLY(list) = 1; - for (i = 0; i < num_fields - 1; i++) TREE_CHAIN(newtree[i]) = newtree[i+1]; TREE_CHAIN(newtree[num_fields - 1]) = NULL_TREE; + add_type_attr(type, "randomize_performed", NULL_TREE); + add_type_attr(type, "designated_init", NULL_TREE); + if (has_flexarray) + add_type_attr(type, "has_flexarray", NULL_TREE); + main_variant = TYPE_MAIN_VARIANT(type); - for (variant = main_variant; variant; variant = TYPE_NEXT_VARIANT(variant)) { - TYPE_FIELDS(variant) = list; - TYPE_ATTRIBUTES(variant) = copy_list(TYPE_ATTRIBUTES(variant)); - TYPE_ATTRIBUTES(variant) = tree_cons(get_identifier("randomize_performed"), NULL_TREE, TYPE_ATTRIBUTES(variant)); - TYPE_ATTRIBUTES(variant) = tree_cons(get_identifier("designated_init"), NULL_TREE, TYPE_ATTRIBUTES(variant)); - if (has_flexarray) - TYPE_ATTRIBUTES(type) = tree_cons(get_identifier("has_flexarray"), NULL_TREE, TYPE_ATTRIBUTES(type)); - } + for (variant = main_variant; variant; variant = TYPE_NEXT_VARIANT(variant)) + TYPE_FIELDS(variant) = newtree[0]; /* * force a re-layout of the main variant @@ -444,10 +430,8 @@ static void randomize_type(tree type) if (lookup_attribute("randomize_layout", TYPE_ATTRIBUTES(TYPE_MAIN_VARIANT(type))) || is_pure_ops_struct(type)) relayout_struct(type); - for (variant = TYPE_MAIN_VARIANT(type); variant; variant = TYPE_NEXT_VARIANT(variant)) { - TYPE_ATTRIBUTES(type) = copy_list(TYPE_ATTRIBUTES(type)); - TYPE_ATTRIBUTES(type) = tree_cons(get_identifier("randomize_considered"), NULL_TREE, TYPE_ATTRIBUTES(type)); - } + add_type_attr(type, "randomize_considered", NULL_TREE); + #ifdef __DEBUG_PLUGIN fprintf(stderr, "Marking randomize_considered on struct %s\n", ORIG_TYPE_NAME(type)); #ifdef __DEBUG_VERBOSE diff --git a/scripts/gdb/linux/constants.py.in b/scripts/gdb/linux/constants.py.in index e810e0c27ff18..866c13468ebf8 100644 --- a/scripts/gdb/linux/constants.py.in +++ b/scripts/gdb/linux/constants.py.in @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -93,6 +94,12 @@ LX_GDBPARSED(RADIX_TREE_MAP_SIZE) LX_GDBPARSED(RADIX_TREE_MAP_SHIFT) LX_GDBPARSED(RADIX_TREE_MAP_MASK) +/* linux/maple_tree.h */ +LX_VALUE(MAPLE_NODE_SLOTS) +LX_VALUE(MAPLE_RANGE64_SLOTS) +LX_VALUE(MAPLE_ARANGE64_SLOTS) +LX_GDBPARSED(MAPLE_NODE_MASK) + /* linux/vmalloc.h */ LX_VALUE(VM_IOREMAP) LX_VALUE(VM_ALLOC) diff --git a/scripts/gdb/linux/interrupts.py b/scripts/gdb/linux/interrupts.py index ef478e273791f..b371a62213a00 100644 --- a/scripts/gdb/linux/interrupts.py +++ b/scripts/gdb/linux/interrupts.py @@ -7,7 +7,7 @@ from linux import constants from linux import cpus from linux import utils -from linux import radixtree +from linux import mapletree irq_desc_type = utils.CachedType("struct irq_desc") @@ -23,12 +23,12 @@ def irqd_is_level(desc): def show_irq_desc(prec, irq): text = "" - desc = radixtree.lookup(gdb.parse_and_eval("&irq_desc_tree"), irq) + desc = mapletree.mtree_load(gdb.parse_and_eval("&sparse_irqs"), irq) if desc is None: return text - desc = desc.cast(irq_desc_type.get_type()) - if desc is None: + desc = desc.cast(irq_desc_type.get_type().pointer()) + if desc == 0: return text if irq_settings_is_hidden(desc): @@ -110,7 +110,7 @@ def x86_show_mce(prec, var, pfx, desc): pvar = gdb.parse_and_eval(var) text = "%*s: " % (prec, pfx) for cpu in cpus.each_online_cpu(): - text += "%10u " % (cpus.per_cpu(pvar, cpu)) + text += "%10u " % (cpus.per_cpu(pvar, cpu).dereference()) text += " %s\n" % (desc) return text @@ -142,7 +142,7 @@ def x86_show_interupts(prec): if constants.LX_CONFIG_X86_MCE: text += x86_show_mce(prec, "&mce_exception_count", "MCE", "Machine check exceptions") - text == x86_show_mce(prec, "&mce_poll_count", "MCP", "Machine check polls") + text += x86_show_mce(prec, "&mce_poll_count", "MCP", "Machine check polls") text += show_irq_err_count(prec) @@ -221,8 +221,8 @@ def invoke(self, arg, from_tty): gdb.write("CPU%-8d" % cpu) gdb.write("\n") - if utils.gdb_eval_or_none("&irq_desc_tree") is None: - return + if utils.gdb_eval_or_none("&sparse_irqs") is None: + raise gdb.GdbError("Unable to find the sparse IRQ tree, is CONFIG_SPARSE_IRQ enabled?") for irq in range(nr_irqs): gdb.write(show_irq_desc(prec, irq)) diff --git a/scripts/gdb/linux/mapletree.py b/scripts/gdb/linux/mapletree.py new file mode 100644 index 0000000000000..d52d51c0a03fc --- /dev/null +++ b/scripts/gdb/linux/mapletree.py @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Maple tree helpers +# +# Copyright (c) 2025 Broadcom +# +# Authors: +# Florian Fainelli + +import gdb + +from linux import utils +from linux import constants +from linux import xarray + +maple_tree_root_type = utils.CachedType("struct maple_tree") +maple_node_type = utils.CachedType("struct maple_node") +maple_enode_type = utils.CachedType("void") + +maple_dense = 0 +maple_leaf_64 = 1 +maple_range_64 = 2 +maple_arange_64 = 3 + +class Mas(object): + ma_active = 0 + ma_start = 1 + ma_root = 2 + ma_none = 3 + ma_pause = 4 + ma_overflow = 5 + ma_underflow = 6 + ma_error = 7 + + def __init__(self, mt, first, end): + if mt.type == maple_tree_root_type.get_type().pointer(): + self.tree = mt.dereference() + elif mt.type != maple_tree_root_type.get_type(): + raise gdb.GdbError("must be {} not {}" + .format(maple_tree_root_type.get_type().pointer(), mt.type)) + self.tree = mt + self.index = first + self.last = end + self.node = None + self.status = self.ma_start + self.min = 0 + self.max = -1 + + def is_start(self): + # mas_is_start() + return self.status == self.ma_start + + def is_ptr(self): + # mas_is_ptr() + return self.status == self.ma_root + + def is_none(self): + # mas_is_none() + return self.status == self.ma_none + + def root(self): + # mas_root() + return self.tree['ma_root'].cast(maple_enode_type.get_type().pointer()) + + def start(self): + # mas_start() + if self.is_start() is False: + return None + + self.min = 0 + self.max = ~0 + + while True: + self.depth = 0 + root = self.root() + if xarray.xa_is_node(root): + self.depth = 0 + self.status = self.ma_active + self.node = mte_safe_root(root) + self.offset = 0 + if mte_dead_node(self.node) is True: + continue + + return None + + self.node = None + # Empty tree + if root is None: + self.status = self.ma_none + self.offset = constants.LX_MAPLE_NODE_SLOTS + return None + + # Single entry tree + self.status = self.ma_root + self.offset = constants.LX_MAPLE_NODE_SLOTS + + if self.index != 0: + return None + + return root + + return None + + def reset(self): + # mas_reset() + self.status = self.ma_start + self.node = None + +def mte_safe_root(node): + if node.type != maple_enode_type.get_type().pointer(): + raise gdb.GdbError("{} must be {} not {}" + .format(mte_safe_root.__name__, maple_enode_type.get_type().pointer(), node.type)) + ulong_type = utils.get_ulong_type() + indirect_ptr = node.cast(ulong_type) & ~0x2 + val = indirect_ptr.cast(maple_enode_type.get_type().pointer()) + return val + +def mte_node_type(entry): + ulong_type = utils.get_ulong_type() + val = None + if entry.type == maple_enode_type.get_type().pointer(): + val = entry.cast(ulong_type) + elif entry.type == ulong_type: + val = entry + else: + raise gdb.GdbError("{} must be {} not {}" + .format(mte_node_type.__name__, maple_enode_type.get_type().pointer(), entry.type)) + return (val >> 0x3) & 0xf + +def ma_dead_node(node): + if node.type != maple_node_type.get_type().pointer(): + raise gdb.GdbError("{} must be {} not {}" + .format(ma_dead_node.__name__, maple_node_type.get_type().pointer(), node.type)) + ulong_type = utils.get_ulong_type() + parent = node['parent'] + indirect_ptr = node['parent'].cast(ulong_type) & ~constants.LX_MAPLE_NODE_MASK + return indirect_ptr == node + +def mte_to_node(enode): + ulong_type = utils.get_ulong_type() + if enode.type == maple_enode_type.get_type().pointer(): + indirect_ptr = enode.cast(ulong_type) + elif enode.type == ulong_type: + indirect_ptr = enode + else: + raise gdb.GdbError("{} must be {} not {}" + .format(mte_to_node.__name__, maple_enode_type.get_type().pointer(), enode.type)) + indirect_ptr = indirect_ptr & ~constants.LX_MAPLE_NODE_MASK + return indirect_ptr.cast(maple_node_type.get_type().pointer()) + +def mte_dead_node(enode): + if enode.type != maple_enode_type.get_type().pointer(): + raise gdb.GdbError("{} must be {} not {}" + .format(mte_dead_node.__name__, maple_enode_type.get_type().pointer(), enode.type)) + node = mte_to_node(enode) + return ma_dead_node(node) + +def ma_is_leaf(tp): + result = tp < maple_range_64 + return tp < maple_range_64 + +def mt_pivots(t): + if t == maple_dense: + return 0 + elif t == maple_leaf_64 or t == maple_range_64: + return constants.LX_MAPLE_RANGE64_SLOTS - 1 + elif t == maple_arange_64: + return constants.LX_MAPLE_ARANGE64_SLOTS - 1 + +def ma_pivots(node, t): + if node.type != maple_node_type.get_type().pointer(): + raise gdb.GdbError("{}: must be {} not {}" + .format(ma_pivots.__name__, maple_node_type.get_type().pointer(), node.type)) + if t == maple_arange_64: + return node['ma64']['pivot'] + elif t == maple_leaf_64 or t == maple_range_64: + return node['mr64']['pivot'] + else: + return None + +def ma_slots(node, tp): + if node.type != maple_node_type.get_type().pointer(): + raise gdb.GdbError("{}: must be {} not {}" + .format(ma_slots.__name__, maple_node_type.get_type().pointer(), node.type)) + if tp == maple_arange_64: + return node['ma64']['slot'] + elif tp == maple_range_64 or tp == maple_leaf_64: + return node['mr64']['slot'] + elif tp == maple_dense: + return node['slot'] + else: + return None + +def mt_slot(mt, slots, offset): + ulong_type = utils.get_ulong_type() + return slots[offset].cast(ulong_type) + +def mtree_lookup_walk(mas): + ulong_type = utils.get_ulong_type() + n = mas.node + + while True: + node = mte_to_node(n) + tp = mte_node_type(n) + pivots = ma_pivots(node, tp) + end = mt_pivots(tp) + offset = 0 + while True: + if pivots[offset] >= mas.index: + break + if offset >= end: + break + offset += 1 + + slots = ma_slots(node, tp) + n = mt_slot(mas.tree, slots, offset) + if ma_dead_node(node) is True: + mas.reset() + return None + break + + if ma_is_leaf(tp) is True: + break + + return n + +def mtree_load(mt, index): + ulong_type = utils.get_ulong_type() + # MT_STATE(...) + mas = Mas(mt, index, index) + entry = None + + while True: + entry = mas.start() + if mas.is_none(): + return None + + if mas.is_ptr(): + if index != 0: + entry = None + return entry + + entry = mtree_lookup_walk(mas) + if entry is None and mas.is_start(): + continue + else: + break + + if xarray.xa_is_zero(entry): + return None + + return entry diff --git a/scripts/gdb/linux/tasks.py b/scripts/gdb/linux/tasks.py index 17ec19e9b5bf6..5be53b372a693 100644 --- a/scripts/gdb/linux/tasks.py +++ b/scripts/gdb/linux/tasks.py @@ -86,21 +86,12 @@ def invoke(self, arg, from_tty): thread_info_type = utils.CachedType("struct thread_info") -ia64_task_size = None - def get_thread_info(task): thread_info_ptr_type = thread_info_type.get_type().pointer() - if utils.is_target_arch("ia64"): - global ia64_task_size - if ia64_task_size is None: - ia64_task_size = gdb.parse_and_eval("sizeof(struct task_struct)") - thread_info_addr = task.address + ia64_task_size - thread_info = thread_info_addr.cast(thread_info_ptr_type) - else: - if task.type.fields()[0].type == thread_info_type.get_type(): - return task['thread_info'] - thread_info = task['stack'].cast(thread_info_ptr_type) + if task.type.fields()[0].type == thread_info_type.get_type(): + return task['thread_info'] + thread_info = task['stack'].cast(thread_info_ptr_type) return thread_info.dereference() diff --git a/scripts/gdb/linux/xarray.py b/scripts/gdb/linux/xarray.py new file mode 100644 index 0000000000000..f4477b5def75f --- /dev/null +++ b/scripts/gdb/linux/xarray.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Xarray helpers +# +# Copyright (c) 2025 Broadcom +# +# Authors: +# Florian Fainelli + +import gdb + +from linux import utils +from linux import constants + +def xa_is_internal(entry): + ulong_type = utils.get_ulong_type() + return ((entry.cast(ulong_type) & 3) == 2) + +def xa_mk_internal(v): + return ((v << 2) | 2) + +def xa_is_zero(entry): + ulong_type = utils.get_ulong_type() + return entry.cast(ulong_type) == xa_mk_internal(257) + +def xa_is_node(entry): + ulong_type = utils.get_ulong_type() + return xa_is_internal(entry) and (entry.cast(ulong_type) > 4096) diff --git a/scripts/head-object-list.txt b/scripts/head-object-list.txt index ffde8d41f2b36..f1167ce808ba3 100644 --- a/scripts/head-object-list.txt +++ b/scripts/head-object-list.txt @@ -17,7 +17,6 @@ arch/arm/kernel/head-nommu.o arch/arm/kernel/head.o arch/csky/kernel/head.o arch/hexagon/kernel/head.o -arch/ia64/kernel/head.o arch/loongarch/kernel/head.o arch/m68k/68000/head.o arch/m68k/coldfire/head.o diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 9e52c7360e55b..2b99d18e703d8 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -780,7 +780,7 @@ static void renderer_edited(GtkCellRendererText * cell, struct symbol *sym; if (!gtk_tree_model_get_iter(model2, &iter, path)) - return; + goto free; gtk_tree_model_get(model2, &iter, COL_MENU, &menu, -1); sym = menu->sym; @@ -792,6 +792,7 @@ static void renderer_edited(GtkCellRendererText * cell, update_tree(&rootmenu, NULL); +free: gtk_tree_path_free(path); } @@ -974,13 +975,14 @@ on_treeview2_key_press_event(GtkWidget * widget, void on_treeview2_cursor_changed(GtkTreeView * treeview, gpointer user_data) { + GtkTreeModel *model = gtk_tree_view_get_model(treeview); GtkTreeSelection *selection; GtkTreeIter iter; struct menu *menu; selection = gtk_tree_view_get_selection(treeview); - if (gtk_tree_selection_get_selected(selection, &model2, &iter)) { - gtk_tree_model_get(model2, &iter, COL_MENU, &menu, -1); + if (gtk_tree_selection_get_selected(selection, &model, &iter)) { + gtk_tree_model_get(model, &iter, COL_MENU, &menu, -1); text_insert_help(menu); } } diff --git a/scripts/kconfig/lxdialog/inputbox.c b/scripts/kconfig/lxdialog/inputbox.c index 1dcfb288ee636..327b60cdb8dac 100644 --- a/scripts/kconfig/lxdialog/inputbox.c +++ b/scripts/kconfig/lxdialog/inputbox.c @@ -39,8 +39,10 @@ int dialog_inputbox(const char *title, const char *prompt, int height, int width if (!init) instr[0] = '\0'; - else - strcpy(instr, init); + else { + strncpy(instr, init, sizeof(dialog_input_result) - 1); + instr[sizeof(dialog_input_result) - 1] = '\0'; + } do_resize: if (getmaxy(stdscr) <= (height - INPUTBOX_HEIGTH_MIN)) diff --git a/scripts/kconfig/lxdialog/menubox.c b/scripts/kconfig/lxdialog/menubox.c index 0e333284e947b..6bb8a320a4cb8 100644 --- a/scripts/kconfig/lxdialog/menubox.c +++ b/scripts/kconfig/lxdialog/menubox.c @@ -264,7 +264,7 @@ int dialog_menu(const char *title, const char *prompt, if (key < 256 && isalpha(key)) key = tolower(key); - if (strchr("ynmh", key)) + if (strchr("ynmh ", key)) i = max_choice; else { for (i = choice + 1; i < max_choice; i++) { diff --git a/scripts/kconfig/mconf.c b/scripts/kconfig/mconf.c index eccc87a441e71..2364140596c5d 100644 --- a/scripts/kconfig/mconf.c +++ b/scripts/kconfig/mconf.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -247,7 +248,7 @@ search_help[] = " -> PCI support (PCI [=y])\n" "(1) -> PCI access mode ( [=y])\n" " Defined at drivers/pci/Kconfig:47\n" - " Depends on: X86_LOCAL_APIC && X86_IO_APIC || IA64\n" + " Depends on: X86_LOCAL_APIC && X86_IO_APIC\n" " Selects: LIBCRC32\n" " Selected by: BAR [=n]\n" "-----------------------------------------------------------------\n" @@ -1015,6 +1016,8 @@ int main(int ac, char **av) signal(SIGINT, sig_handler); + setlocale(LC_ALL, ""); + if (ac > 1 && strcmp(av[1], "-s") == 0) { silent = 1; /* Silence conf_read() until the real callback is set up */ diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh index 0b7952471c18f..79c09b378be81 100755 --- a/scripts/kconfig/merge_config.sh +++ b/scripts/kconfig/merge_config.sh @@ -112,8 +112,8 @@ INITFILE=$1 shift; if [ ! -r "$INITFILE" ]; then - echo "The base file '$INITFILE' does not exist. Exit." >&2 - exit 1 + echo "The base file '$INITFILE' does not exist. Creating one..." >&2 + touch "$INITFILE" fi MERGE_LIST=$* diff --git a/scripts/kconfig/nconf.c b/scripts/kconfig/nconf.c index 143a2c351d576..a7cfa6813c63f 100644 --- a/scripts/kconfig/nconf.c +++ b/scripts/kconfig/nconf.c @@ -7,6 +7,7 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif +#include #include #include #include @@ -216,7 +217,7 @@ search_help[] = "Symbol: FOO [ = m]\n" "Prompt: Foo bus is used to drive the bar HW\n" "Defined at drivers/pci/Kconfig:47\n" -"Depends on: X86_LOCAL_APIC && X86_IO_APIC || IA64\n" +"Depends on: X86_LOCAL_APIC && X86_IO_APIC\n" "Location:\n" " -> Bus options (PCI, PCMCIA, EISA, ISA)\n" " -> PCI support (PCI [ = y])\n" @@ -591,6 +592,8 @@ static void item_add_str(const char *fmt, ...) tmp_str, sizeof(k_menu_items[index].str)); + k_menu_items[index].str[sizeof(k_menu_items[index].str) - 1] = '\0'; + free_item(curses_menu_items[index]); curses_menu_items[index] = new_item( k_menu_items[index].str, @@ -1567,6 +1570,8 @@ int main(int ac, char **av) int lines, columns; char *mode; + setlocale(LC_ALL, ""); + if (ac > 1 && strcmp(av[1], "-s") == 0) { /* Silence conf_read() until the real callback is set up */ conf_set_message_callback(NULL); diff --git a/scripts/kconfig/nconf.gui.c b/scripts/kconfig/nconf.gui.c index 25a7263ef3c8c..5f13a0a7fb0be 100644 --- a/scripts/kconfig/nconf.gui.c +++ b/scripts/kconfig/nconf.gui.c @@ -349,6 +349,7 @@ int dialog_inputbox(WINDOW *main_window, x = (columns-win_cols)/2; strncpy(result, init, *result_len); + result[*result_len - 1] = '\0'; /* create the windows */ win = newwin(win_lines, win_cols, y, x); diff --git a/scripts/kconfig/qconf.cc b/scripts/kconfig/qconf.cc index 4f3ba3debc08e..119997c8bf1f7 100644 --- a/scripts/kconfig/qconf.cc +++ b/scripts/kconfig/qconf.cc @@ -480,7 +480,7 @@ void ConfigList::updateListAllForAll() while (it.hasNext()) { ConfigList *list = it.next(); - list->updateList(); + list->updateListAll(); } } diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 3eee0143e0c5c..f58726671fb37 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -55,18 +55,12 @@ patch -p1 < %{SOURCE2} %{make} %{makeflags} KERNELRELEASE=%{KERNELRELEASE} KBUILD_BUILD_VERSION=%{release} %install -mkdir -p %{buildroot}/boot -%ifarch ia64 -mkdir -p %{buildroot}/boot/efi -cp $(%{make} %{makeflags} -s image_name) %{buildroot}/boot/efi/vmlinuz-%{KERNELRELEASE} -ln -s efi/vmlinuz-%{KERNELRELEASE} %{buildroot}/boot/ -%else -cp $(%{make} %{makeflags} -s image_name) %{buildroot}/boot/vmlinuz-%{KERNELRELEASE} -%endif +mkdir -p %{buildroot}/lib/modules/%{KERNELRELEASE} +cp $(%{make} %{makeflags} -s image_name) %{buildroot}/lib/modules/%{KERNELRELEASE}/vmlinuz %{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} modules_install %{make} %{makeflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install -cp System.map %{buildroot}/boot/System.map-%{KERNELRELEASE} -cp .config %{buildroot}/boot/config-%{KERNELRELEASE} +cp System.map %{buildroot}/lib/modules/%{KERNELRELEASE} +cp .config %{buildroot}/lib/modules/%{KERNELRELEASE}/config ln -fns /usr/src/kernels/%{KERNELRELEASE} %{buildroot}/lib/modules/%{KERNELRELEASE}/build %if %{with_devel} %{make} %{makeflags} run-command KBUILD_RUN_COMMAND='${srctree}/scripts/package/install-extmod-build %{buildroot}/usr/src/kernels/%{KERNELRELEASE}' @@ -76,13 +70,14 @@ ln -fns /usr/src/kernels/%{KERNELRELEASE} %{buildroot}/lib/modules/%{KERNELRELEA rm -rf %{buildroot} %post -if [ -x /sbin/installkernel -a -r /boot/vmlinuz-%{KERNELRELEASE} -a -r /boot/System.map-%{KERNELRELEASE} ]; then -cp /boot/vmlinuz-%{KERNELRELEASE} /boot/.vmlinuz-%{KERNELRELEASE}-rpm -cp /boot/System.map-%{KERNELRELEASE} /boot/.System.map-%{KERNELRELEASE}-rpm -rm -f /boot/vmlinuz-%{KERNELRELEASE} /boot/System.map-%{KERNELRELEASE} -/sbin/installkernel %{KERNELRELEASE} /boot/.vmlinuz-%{KERNELRELEASE}-rpm /boot/.System.map-%{KERNELRELEASE}-rpm -rm -f /boot/.vmlinuz-%{KERNELRELEASE}-rpm /boot/.System.map-%{KERNELRELEASE}-rpm +if [ -x /usr/bin/kernel-install ]; then + /usr/bin/kernel-install add %{KERNELRELEASE} /lib/modules/%{KERNELRELEASE}/vmlinuz fi +for file in vmlinuz System.map config; do + if ! cmp --silent "/lib/modules/%{KERNELRELEASE}/${file}" "/boot/${file}-%{KERNELRELEASE}"; then + cp "/lib/modules/%{KERNELRELEASE}/${file}" "/boot/${file}-%{KERNELRELEASE}" + fi +done %preun if [ -x /sbin/new-kernel-pkg ]; then @@ -100,7 +95,6 @@ fi %defattr (-, root, root) /lib/modules/%{KERNELRELEASE} %exclude /lib/modules/%{KERNELRELEASE}/build -/boot/* %files headers %defattr (-, root, root) diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index 5044224cf6714..c1a36da85e84f 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -26,7 +26,7 @@ set_debarch() { # Attempt to find the correct Debian architecture case "$UTS_MACHINE" in - i386|ia64|alpha|m68k|riscv*) + i386|alpha|m68k|riscv*) debarch="$UTS_MACHINE" ;; x86_64) debarch=amd64 ;; diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 40ae6b2c7a6da..3e4f54799cc0a 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -590,7 +590,6 @@ static int do_file(char const *const fname) ideal_nop = ideal_nop4_arm64; is_fake_mcount64 = arm64_is_fake_mcount; break; - case EM_IA_64: reltype = R_IA64_IMM64; break; case EM_MIPS: /* reltype: e_class */ break; case EM_LOONGARCH: /* reltype: e_class */ break; case EM_PPC: reltype = R_PPC_ADDR32; break; diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 6a4645a579760..f84df9e383fd0 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -275,13 +275,6 @@ $section_type = '%progbits'; $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_AARCH64_CALL26\\s+_mcount\$"; $type = ".quad"; -} elsif ($arch eq "ia64") { - $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\s_mcount\$"; - $type = "data8"; - - if ($is_module eq "0") { - $cc .= " -mconstant-gp"; - } } elsif ($arch eq "sparc64") { # In the objdump output there are giblets like: # 0000000000000000 : diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh index 76e9cbcfbeab4..d06baf626abe7 100755 --- a/scripts/xz_wrap.sh +++ b/scripts/xz_wrap.sh @@ -15,7 +15,6 @@ LZMA2OPTS= case $SRCARCH in x86) BCJ=--x86 ;; powerpc) BCJ=--powerpc ;; - ia64) BCJ=--ia64; LZMA2OPTS=pb=4 ;; arm) BCJ=--arm ;; sparc) BCJ=--sparc ;; esac diff --git a/security/apparmor/file.c b/security/apparmor/file.c index 6fd21324a097f..a51b83cf69689 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -436,9 +436,11 @@ int aa_path_link(const struct cred *subj_cred, { struct path link = { .mnt = new_dir->mnt, .dentry = new_dentry }; struct path target = { .mnt = new_dir->mnt, .dentry = old_dentry }; + struct inode *inode = d_backing_inode(old_dentry); + vfsuid_t vfsuid = i_uid_into_vfsuid(mnt_idmap(target.mnt), inode); struct path_cond cond = { - d_backing_inode(old_dentry)->i_uid, - d_backing_inode(old_dentry)->i_mode + .uid = vfsuid_into_kuid(vfsuid), + .mode = inode->i_mode, }; char *buffer = NULL, *buffer2 = NULL; struct aa_profile *profile; diff --git a/security/apparmor/include/lib.h b/security/apparmor/include/lib.h index 73c8a32c68613..6e88e99da80f6 100644 --- a/security/apparmor/include/lib.h +++ b/security/apparmor/include/lib.h @@ -46,7 +46,11 @@ #define AA_BUG_FMT(X, fmt, args...) \ WARN((X), "AppArmor WARN %s: (" #X "): " fmt, __func__, ##args) #else -#define AA_BUG_FMT(X, fmt, args...) no_printk(fmt, ##args) +#define AA_BUG_FMT(X, fmt, args...) \ + do { \ + BUILD_BUG_ON_INVALID(X); \ + no_printk(fmt, ##args); \ + } while (0) #endif #define AA_ERROR(fmt, args...) \ diff --git a/security/apparmor/include/match.h b/security/apparmor/include/match.h index 58fbf67139b9c..e59305abb85a3 100644 --- a/security/apparmor/include/match.h +++ b/security/apparmor/include/match.h @@ -141,17 +141,15 @@ aa_state_t aa_dfa_matchn_until(struct aa_dfa *dfa, aa_state_t start, void aa_dfa_free_kref(struct kref *kref); -#define WB_HISTORY_SIZE 24 +/* This needs to be a power of 2 */ +#define WB_HISTORY_SIZE 32 struct match_workbuf { - unsigned int count; unsigned int pos; unsigned int len; - unsigned int size; /* power of 2, same as history size */ - unsigned int history[WB_HISTORY_SIZE]; + aa_state_t history[WB_HISTORY_SIZE]; }; #define DEFINE_MATCH_WB(N) \ struct match_workbuf N = { \ - .count = 0, \ .pos = 0, \ .len = 0, \ } diff --git a/security/apparmor/match.c b/security/apparmor/match.c index b97ef5e1db732..3667b79e9366b 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -668,34 +668,35 @@ aa_state_t aa_dfa_matchn_until(struct aa_dfa *dfa, aa_state_t start, return state; } -#define inc_wb_pos(wb) \ -do { \ +#define inc_wb_pos(wb) \ +do { \ + BUILD_BUG_ON_NOT_POWER_OF_2(WB_HISTORY_SIZE); \ wb->pos = (wb->pos + 1) & (WB_HISTORY_SIZE - 1); \ - wb->len = (wb->len + 1) & (WB_HISTORY_SIZE - 1); \ + wb->len = (wb->len + 1) > WB_HISTORY_SIZE ? WB_HISTORY_SIZE : \ + wb->len + 1; \ } while (0) /* For DFAs that don't support extended tagging of states */ +/* adjust is only set if is_loop returns true */ static bool is_loop(struct match_workbuf *wb, aa_state_t state, unsigned int *adjust) { - aa_state_t pos = wb->pos; - aa_state_t i; + int pos = wb->pos; + int i; if (wb->history[pos] < state) return false; - for (i = 0; i <= wb->len; i++) { + for (i = 0; i < wb->len; i++) { if (wb->history[pos] == state) { *adjust = i; return true; } - if (pos == 0) - pos = WB_HISTORY_SIZE; - pos--; + /* -1 wraps to WB_HISTORY_SIZE - 1 */ + pos = (pos - 1) & (WB_HISTORY_SIZE - 1); } - *adjust = i; - return true; + return false; } static aa_state_t leftmatch_fb(struct aa_dfa *dfa, aa_state_t start, diff --git a/security/inode.c b/security/inode.c index 3aa75fffa8c92..a90b043695d92 100644 --- a/security/inode.c +++ b/security/inode.c @@ -159,7 +159,6 @@ static struct dentry *securityfs_create_dentry(const char *name, umode_t mode, inode->i_fop = fops; } d_instantiate(dentry, inode); - dget(dentry); inode_unlock(dir); return dentry; @@ -306,7 +305,6 @@ void securityfs_remove(struct dentry *dentry) simple_rmdir(dir, dentry); else simple_unlink(dir, dentry); - dput(dentry); } inode_unlock(dir); simple_release_fs(&mount, &mount_count); diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 870dde67707b1..0acf7d6baa37f 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -671,6 +671,15 @@ static int ima_protect_xattr(struct dentry *dentry, const char *xattr_name, return 0; } +/* + * ima_reset_appraise_flags - reset ima_iint_cache flags + * + * @digsig: whether to clear/set IMA_DIGSIG flag, tristate values + * 0: clear IMA_DIGSIG + * 1: set IMA_DIGSIG + * -1: don't change IMA_DIGSIG + * + */ static void ima_reset_appraise_flags(struct inode *inode, int digsig) { struct integrity_iint_cache *iint; @@ -683,9 +692,9 @@ static void ima_reset_appraise_flags(struct inode *inode, int digsig) return; iint->measured_pcrs = 0; set_bit(IMA_CHANGE_XATTR, &iint->atomic_flags); - if (digsig) + if (digsig == 1) set_bit(IMA_DIGSIG, &iint->atomic_flags); - else + else if (digsig == 0) clear_bit(IMA_DIGSIG, &iint->atomic_flags); } @@ -770,6 +779,8 @@ int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, digsig = (xvalue->type == EVM_IMA_XATTR_DIGSIG); } else if (!strcmp(xattr_name, XATTR_NAME_EVM) && xattr_value_len > 0) { digsig = (xvalue->type == EVM_XATTR_PORTABLE_DIGSIG); + } else { + digsig = -1; } if (result == 1 || evm_revalidate_status(xattr_name)) { ima_reset_appraise_flags(d_backing_inode(dentry), digsig); @@ -783,18 +794,20 @@ int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) { if (evm_revalidate_status(acl_name)) - ima_reset_appraise_flags(d_backing_inode(dentry), 0); + ima_reset_appraise_flags(d_backing_inode(dentry), -1); return 0; } int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name) { - int result; + int result, digsig = -1; result = ima_protect_xattr(dentry, xattr_name, NULL, 0); if (result == 1 || evm_revalidate_status(xattr_name)) { - ima_reset_appraise_flags(d_backing_inode(dentry), 0); + if (!strcmp(xattr_name, XATTR_NAME_IMA)) + digsig = 0; + ima_reset_appraise_flags(d_backing_inode(dentry), digsig); if (result == 1) result = 0; } diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 98308a2bdef6e..3b734a4dfcbe4 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -128,16 +128,22 @@ static void ima_rdwr_violation_check(struct file *file, if (atomic_read(&inode->i_readcount) && IS_IMA(inode)) { if (!iint) iint = integrity_iint_find(inode); + /* IMA_MEASURE is set from reader side */ - if (iint && test_bit(IMA_MUST_MEASURE, - &iint->atomic_flags)) + if (iint && test_and_clear_bit(IMA_MAY_EMIT_TOMTOU, + &iint->atomic_flags)) send_tomtou = true; } } else { if (must_measure) - set_bit(IMA_MUST_MEASURE, &iint->atomic_flags); - if (inode_is_open_for_write(inode) && must_measure) - send_writers = true; + set_bit(IMA_MAY_EMIT_TOMTOU, &iint->atomic_flags); + + /* Limit number of open_writers violations */ + if (inode_is_open_for_write(inode) && must_measure) { + if (!test_and_set_bit(IMA_EMITTED_OPENWRITERS, + &iint->atomic_flags)) + send_writers = true; + } } if (!send_tomtou && !send_writers) @@ -235,7 +241,9 @@ static int process_measurement(struct file *file, const struct cred *cred, &allowed_algos); violation_check = ((func == FILE_CHECK || func == MMAP_CHECK || func == MMAP_CHECK_REQPROT) && - (ima_policy_flag & IMA_MEASURE)); + (ima_policy_flag & IMA_MEASURE) && + ((action & IMA_MEASURE) || + (file->f_mode & FMODE_WRITE))); if (!action && !violation_check) return 0; diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index ad20ff7f5dfaa..a007edae938ae 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -74,7 +74,8 @@ #define IMA_UPDATE_XATTR 1 #define IMA_CHANGE_ATTR 2 #define IMA_DIGSIG 3 -#define IMA_MUST_MEASURE 4 +#define IMA_MAY_EMIT_TOMTOU 4 +#define IMA_EMITTED_OPENWRITERS 5 enum evm_ima_xattr_type { IMA_XATTR_DIGEST = 0x01, diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index aa108bea6739b..ddcbfaede6849 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -241,7 +242,7 @@ int TSS_checkhmac1(unsigned char *buffer, if (ret < 0) goto out; - if (memcmp(testhmac, authdata, SHA1_DIGEST_SIZE)) + if (crypto_memneq(testhmac, authdata, SHA1_DIGEST_SIZE)) ret = -EINVAL; out: kfree_sensitive(sdesc); @@ -334,7 +335,7 @@ static int TSS_checkhmac2(unsigned char *buffer, TPM_NONCE_SIZE, ononce, 1, continueflag1, 0, 0); if (ret < 0) goto out; - if (memcmp(testhmac1, authdata1, SHA1_DIGEST_SIZE)) { + if (crypto_memneq(testhmac1, authdata1, SHA1_DIGEST_SIZE)) { ret = -EINVAL; goto out; } @@ -343,7 +344,7 @@ static int TSS_checkhmac2(unsigned char *buffer, TPM_NONCE_SIZE, ononce, 1, continueflag2, 0, 0); if (ret < 0) goto out; - if (memcmp(testhmac2, authdata2, SHA1_DIGEST_SIZE)) + if (crypto_memneq(testhmac2, authdata2, SHA1_DIGEST_SIZE)) ret = -EINVAL; out: kfree_sensitive(sdesc); diff --git a/security/selinux/Kconfig b/security/selinux/Kconfig index d30348fbe0df3..c74d531f7b0d5 100644 --- a/security/selinux/Kconfig +++ b/security/selinux/Kconfig @@ -69,6 +69,17 @@ config SECURITY_SELINUX_SID2STR_CACHE_SIZE If unsure, keep the default value. +config SECURITY_SELINUX_AVC_HASH_BITS + int "SELinux avc hashtable size" + depends on SECURITY_SELINUX + range 9 14 + default 9 + help + This option sets the number of buckets used in the AVC hash table + to 2^SECURITY_SELINUX_AVC_HASH_BITS. A higher value helps maintain + shorter chain lengths especially when expanding AVC nodes via + /sys/fs/selinux/avc/cache_threshold. + config SECURITY_SELINUX_DEBUG bool "SELinux kernel debugging support" depends on SECURITY_SELINUX diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 9f44a0f446ec4..8e40b0941d1c8 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -30,13 +30,14 @@ #include "avc.h" #include "avc_ss.h" #include "classmap.h" +#include "hash.h" #define CREATE_TRACE_POINTS #include -#define AVC_CACHE_SLOTS 512 -#define AVC_DEF_CACHE_THRESHOLD 512 -#define AVC_CACHE_RECLAIM 16 +#define AVC_CACHE_SLOTS (1 << CONFIG_SECURITY_SELINUX_AVC_HASH_BITS) +#define AVC_DEF_CACHE_THRESHOLD AVC_CACHE_SLOTS +#define AVC_CACHE_RECLAIM 16 #ifdef CONFIG_SECURITY_SELINUX_AVC_STATS #define avc_cache_stats_incr(field) this_cpu_inc(avc_cache_stats.field) @@ -127,7 +128,7 @@ static struct kmem_cache *avc_xperms_cachep __ro_after_init; static inline u32 avc_hash(u32 ssid, u32 tsid, u16 tclass) { - return (ssid ^ (tsid<<2) ^ (tclass<<4)) & (AVC_CACHE_SLOTS - 1); + return av_hash(ssid, tsid, (u32)tclass, (u32)(AVC_CACHE_SLOTS - 1)); } /** diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index d4a99d98ec774..7851c9fd4cbd8 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -462,7 +462,10 @@ static int selinux_is_genfs_special_handling(struct super_block *sb) !strcmp(sb->s_type->name, "rootfs") || (selinux_policycap_cgroupseclabel() && (!strcmp(sb->s_type->name, "cgroup") || - !strcmp(sb->s_type->name, "cgroup2"))); + !strcmp(sb->s_type->name, "cgroup2"))) || + // Android: remove functionfs policycap check due to + // ABI breakage with policycap array. + !strcmp(sb->s_type->name, "functionfs"); } static int selinux_is_sblabel_mnt(struct super_block *sb) @@ -727,7 +730,10 @@ static int selinux_set_mnt_opts(struct super_block *sb, !strcmp(sb->s_type->name, "binder") || !strcmp(sb->s_type->name, "bpf") || !strcmp(sb->s_type->name, "pstore") || - !strcmp(sb->s_type->name, "securityfs")) + !strcmp(sb->s_type->name, "securityfs") || + // Android: remove functionfs policycap check due to + // ABI breakage with policycap array. + !strcmp(sb->s_type->name, "functionfs")) sbsec->flags |= SE_SBGENFS; if (!strcmp(sb->s_type->name, "sysfs") || diff --git a/security/selinux/include/hash.h b/security/selinux/include/hash.h new file mode 100644 index 0000000000000..c0d13f4ba24c4 --- /dev/null +++ b/security/selinux/include/hash.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _SELINUX_HASH_H_ +#define _SELINUX_HASH_H_ + +/* Based on MurmurHash3, written by Austin Appleby and placed in the + * public domain. + */ +static inline u32 av_hash(u32 key1, u32 key2, u32 key3, u32 mask) +{ + static const u32 c1 = 0xcc9e2d51; + static const u32 c2 = 0x1b873593; + static const u32 r1 = 15; + static const u32 r2 = 13; + static const u32 m = 5; + static const u32 n = 0xe6546b64; + + u32 hash = 0; + +#define mix(input) \ + do { \ + u32 v = input; \ + v *= c1; \ + v = (v << r1) | (v >> (32 - r1)); \ + v *= c2; \ + hash ^= v; \ + hash = (hash << r2) | (hash >> (32 - r2)); \ + hash = hash * m + n; \ + } while (0) + + mix(key1); + mix(key2); + mix(key3); + +#undef mix + + hash ^= hash >> 16; + hash *= 0x85ebca6b; + hash ^= hash >> 13; + hash *= 0xc2b2ae35; + hash ^= hash >> 16; + + return hash & mask; +} + +#endif /* _SELINUX_HASH_H_ */ + diff --git a/security/selinux/ss/avtab.c b/security/selinux/ss/avtab.c index 86d98a8e291b4..6e51ab9aecece 100644 --- a/security/selinux/ss/avtab.c +++ b/security/selinux/ss/avtab.c @@ -22,47 +22,15 @@ #include #include "avtab.h" #include "policydb.h" +#include "hash.h" static struct kmem_cache *avtab_node_cachep __ro_after_init; static struct kmem_cache *avtab_xperms_cachep __ro_after_init; -/* Based on MurmurHash3, written by Austin Appleby and placed in the - * public domain. - */ static inline u32 avtab_hash(const struct avtab_key *keyp, u32 mask) { - static const u32 c1 = 0xcc9e2d51; - static const u32 c2 = 0x1b873593; - static const u32 r1 = 15; - static const u32 r2 = 13; - static const u32 m = 5; - static const u32 n = 0xe6546b64; - - u32 hash = 0; - -#define mix(input) do { \ - u32 v = input; \ - v *= c1; \ - v = (v << r1) | (v >> (32 - r1)); \ - v *= c2; \ - hash ^= v; \ - hash = (hash << r2) | (hash >> (32 - r2)); \ - hash = hash * m + n; \ - } while (0) - - mix(keyp->target_class); - mix(keyp->target_type); - mix(keyp->source_type); - -#undef mix - - hash ^= hash >> 16; - hash *= 0x85ebca6b; - hash ^= hash >> 13; - hash *= 0xc2b2ae35; - hash ^= hash >> 16; - - return hash & mask; + return av_hash((u32)keyp->target_class, (u32)keyp->target_type, + (u32)keyp->source_type, mask); } static struct avtab_node* diff --git a/security/selinux/xfrm.c b/security/selinux/xfrm.c index 95fcd2d3433e4..0fbbc62ed3ede 100644 --- a/security/selinux/xfrm.c +++ b/security/selinux/xfrm.c @@ -95,7 +95,7 @@ static int selinux_xfrm_alloc_user(struct xfrm_sec_ctx **ctxp, ctx->ctx_doi = XFRM_SC_DOI_LSM; ctx->ctx_alg = XFRM_SC_ALG_SELINUX; - ctx->ctx_len = str_len; + ctx->ctx_len = str_len + 1; memcpy(ctx->ctx_str, &uctx[1], str_len); ctx->ctx_str[str_len] = '\0'; rc = security_context_to_sid(ctx->ctx_str, str_len, diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index 5dd1e164f9b13..1e35c9f807b2b 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -830,7 +830,7 @@ static int smk_open_cipso(struct inode *inode, struct file *file) static ssize_t smk_set_cipso(struct file *file, const char __user *buf, size_t count, loff_t *ppos, int format) { - struct netlbl_lsm_catmap *old_cat, *new_cat = NULL; + struct netlbl_lsm_catmap *old_cat; struct smack_known *skp; struct netlbl_lsm_secattr ncats; char mapcatset[SMK_CIPSOLEN]; @@ -917,22 +917,15 @@ static ssize_t smk_set_cipso(struct file *file, const char __user *buf, smack_catset_bit(cat, mapcatset); } - ncats.flags = 0; - if (catlen == 0) { - ncats.attr.mls.cat = NULL; - ncats.attr.mls.lvl = maplevel; - new_cat = netlbl_catmap_alloc(GFP_ATOMIC); - if (new_cat) - new_cat->next = ncats.attr.mls.cat; - ncats.attr.mls.cat = new_cat; - skp->smk_netlabel.flags &= ~(1U << 3); - rc = 0; - } else { - rc = smk_netlbl_mls(maplevel, mapcatset, &ncats, SMK_CIPSOLEN); - } + + rc = smk_netlbl_mls(maplevel, mapcatset, &ncats, SMK_CIPSOLEN); if (rc >= 0) { old_cat = skp->smk_netlabel.attr.mls.cat; rcu_assign_pointer(skp->smk_netlabel.attr.mls.cat, ncats.attr.mls.cat); + if (ncats.attr.mls.cat) + skp->smk_netlabel.flags |= NETLBL_SECATTR_MLS_CAT; + else + skp->smk_netlabel.flags &= ~(u32)NETLBL_SECATTR_MLS_CAT; skp->smk_netlabel.attr.mls.lvl = ncats.attr.mls.lvl; synchronize_rcu(); netlbl_catmap_free(old_cat); diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index 728c211142d14..471de2d1b37ad 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -1085,8 +1085,7 @@ static int snd_pcm_oss_change_params_locked(struct snd_pcm_substream *substream) runtime->oss.params = 0; runtime->oss.prepare = 1; runtime->oss.buffer_used = 0; - if (runtime->dma_area) - snd_pcm_format_set_silence(runtime->format, runtime->dma_area, bytes_to_samples(runtime, runtime->dma_bytes)); + snd_pcm_runtime_buffer_set_silence(runtime); runtime->oss.period_frames = snd_pcm_alsa_frames(substream, oss_period_size); diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index e40de64ec85cb..f37fd1e48740f 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "pcm_local.h" @@ -703,6 +704,17 @@ static void snd_pcm_buffer_access_unlock(struct snd_pcm_runtime *runtime) atomic_inc(&runtime->buffer_accessing); } +/* fill the PCM buffer with the current silence format; called from pcm_oss.c */ +void snd_pcm_runtime_buffer_set_silence(struct snd_pcm_runtime *runtime) +{ + snd_pcm_buffer_access_lock(runtime); + if (runtime->dma_area) + snd_pcm_format_set_silence(runtime->format, runtime->dma_area, + bytes_to_samples(runtime, runtime->dma_bytes)); + snd_pcm_buffer_access_unlock(runtime); +} +EXPORT_SYMBOL_GPL(snd_pcm_runtime_buffer_set_silence); + #if IS_ENABLED(CONFIG_SND_PCM_OSS) #define is_oss_stream(substream) ((substream)->oss.oss) #else @@ -3114,13 +3126,23 @@ struct snd_pcm_sync_ptr32 { static snd_pcm_uframes_t recalculate_boundary(struct snd_pcm_runtime *runtime) { snd_pcm_uframes_t boundary; + snd_pcm_uframes_t border; + int order; if (! runtime->buffer_size) return 0; - boundary = runtime->buffer_size; - while (boundary * 2 <= 0x7fffffffUL - runtime->buffer_size) - boundary *= 2; - return boundary; + + border = 0x7fffffffUL - runtime->buffer_size; + if (runtime->buffer_size > border) + return runtime->buffer_size; + + order = __fls(border) - __fls(runtime->buffer_size); + boundary = runtime->buffer_size << order; + + if (boundary <= border) + return boundary; + else + return boundary / 2; } static int snd_pcm_ioctl_sync_ptr_compat(struct snd_pcm_substream *substream, diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 6195fe9dda179..31428cdc0f63d 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -736,15 +736,21 @@ static int snd_seq_deliver_single_event(struct snd_seq_client *client, */ static int __deliver_to_subscribers(struct snd_seq_client *client, struct snd_seq_event *event, - struct snd_seq_client_port *src_port, - int atomic, int hop) + int port, int atomic, int hop) { + struct snd_seq_client_port *src_port; struct snd_seq_subscribers *subs; int err, result = 0, num_ev = 0; union __snd_seq_event event_saved; size_t saved_size; struct snd_seq_port_subs_info *grp; + if (port < 0) + return 0; + src_port = snd_seq_port_use_ptr(client, port); + if (!src_port) + return 0; + /* save original event record */ saved_size = snd_seq_event_packet_size(event); memcpy(&event_saved, event, saved_size); @@ -780,6 +786,7 @@ static int __deliver_to_subscribers(struct snd_seq_client *client, read_unlock(&grp->list_lock); else up_read(&grp->list_mutex); + snd_seq_port_unlock(src_port); memcpy(event, &event_saved, saved_size); return (result < 0) ? result : num_ev; } @@ -788,25 +795,32 @@ static int deliver_to_subscribers(struct snd_seq_client *client, struct snd_seq_event *event, int atomic, int hop) { - struct snd_seq_client_port *src_port; - int ret = 0, ret2; - - src_port = snd_seq_port_use_ptr(client, event->source.port); - if (src_port) { - ret = __deliver_to_subscribers(client, event, src_port, atomic, hop); - snd_seq_port_unlock(src_port); - } - - if (client->ump_endpoint_port < 0 || - event->source.port == client->ump_endpoint_port) - return ret; + int ret; +#if IS_ENABLED(CONFIG_SND_SEQ_UMP) + int ret2; +#endif - src_port = snd_seq_port_use_ptr(client, client->ump_endpoint_port); - if (!src_port) + ret = __deliver_to_subscribers(client, event, + event->source.port, atomic, hop); +#if IS_ENABLED(CONFIG_SND_SEQ_UMP) + if (!snd_seq_client_is_ump(client) || client->ump_endpoint_port < 0) return ret; - ret2 = __deliver_to_subscribers(client, event, src_port, atomic, hop); - snd_seq_port_unlock(src_port); - return ret2 < 0 ? ret2 : ret; + /* If it's an event from EP port (and with a UMP group), + * deliver to subscribers of the corresponding UMP group port, too. + * Or, if it's from non-EP port, deliver to subscribers of EP port, too. + */ + if (event->source.port == client->ump_endpoint_port) + ret2 = __deliver_to_subscribers(client, event, + snd_seq_ump_group_port(event), + atomic, hop); + else + ret2 = __deliver_to_subscribers(client, event, + client->ump_endpoint_port, + atomic, hop); + if (ret2 < 0) + return ret2; +#endif + return ret; } /* deliver an event to the destination port(s). @@ -1155,8 +1169,7 @@ static __poll_t snd_seq_poll(struct file *file, poll_table * wait) if (snd_seq_file_flags(file) & SNDRV_SEQ_LFLG_OUTPUT) { /* check if data is available in the pool */ - if (!snd_seq_write_pool_allocated(client) || - snd_seq_pool_poll_wait(client->pool, file, wait)) + if (snd_seq_pool_poll_wait(client->pool, file, wait)) mask |= EPOLLOUT | EPOLLWRNORM; } @@ -2570,8 +2583,6 @@ int snd_seq_kernel_client_write_poll(int clientid, struct file *file, poll_table if (client == NULL) return -ENXIO; - if (! snd_seq_write_pool_allocated(client)) - return 1; if (snd_seq_pool_poll_wait(client->pool, file, wait)) return 1; return 0; diff --git a/sound/core/seq/seq_memory.c b/sound/core/seq/seq_memory.c index b603bb93f8960..692860deec0c3 100644 --- a/sound/core/seq/seq_memory.c +++ b/sound/core/seq/seq_memory.c @@ -429,6 +429,7 @@ int snd_seq_pool_poll_wait(struct snd_seq_pool *pool, struct file *file, poll_table *wait) { poll_wait(file, &pool->output_sleep, wait); + guard(spinlock_irq)(&pool->lock); return snd_seq_output_ok(pool); } diff --git a/sound/core/seq/seq_ump_convert.c b/sound/core/seq/seq_ump_convert.c index 4dd540cbb1cbb..83a27362b7a06 100644 --- a/sound/core/seq/seq_ump_convert.c +++ b/sound/core/seq/seq_ump_convert.c @@ -1284,3 +1284,21 @@ int snd_seq_deliver_to_ump(struct snd_seq_client *source, else return cvt_to_ump_midi1(dest, dest_port, event, atomic, hop); } + +/* return the UMP group-port number of the event; + * return -1 if groupless or non-UMP event + */ +int snd_seq_ump_group_port(const struct snd_seq_event *event) +{ + const struct snd_seq_ump_event *ump_ev = + (const struct snd_seq_ump_event *)event; + unsigned char type; + + if (!snd_seq_ev_is_ump(event)) + return -1; + type = ump_message_type(ump_ev->ump[0]); + if (ump_is_groupless_msg(type)) + return -1; + /* group-port number starts from 1 */ + return ump_message_group(ump_ev->ump[0]) + 1; +} diff --git a/sound/core/seq/seq_ump_convert.h b/sound/core/seq/seq_ump_convert.h index 6c146d8032804..4abf0a7637d70 100644 --- a/sound/core/seq/seq_ump_convert.h +++ b/sound/core/seq/seq_ump_convert.h @@ -18,5 +18,6 @@ int snd_seq_deliver_to_ump(struct snd_seq_client *source, struct snd_seq_client_port *dest_port, struct snd_seq_event *event, int atomic, int hop); +int snd_seq_ump_group_port(const struct snd_seq_event *event); #endif /* __SEQ_UMP_CONVERT_H */ diff --git a/sound/drivers/serial-generic.c b/sound/drivers/serial-generic.c index c8db6c75d133d..8b8e9e871ed37 100644 --- a/sound/drivers/serial-generic.c +++ b/sound/drivers/serial-generic.c @@ -37,6 +37,8 @@ MODULE_LICENSE("GPL"); #define SERIAL_TX_STATE_ACTIVE 1 #define SERIAL_TX_STATE_WAKEUP 2 +#define INTERNAL_BUF_SIZE 256 + struct snd_serial_generic { struct serdev_device *serdev; @@ -51,6 +53,7 @@ struct snd_serial_generic { struct work_struct tx_work; unsigned long tx_state; + char tx_buf[INTERNAL_BUF_SIZE]; }; static void snd_serial_generic_tx_wakeup(struct snd_serial_generic *drvdata) @@ -61,11 +64,8 @@ static void snd_serial_generic_tx_wakeup(struct snd_serial_generic *drvdata) schedule_work(&drvdata->tx_work); } -#define INTERNAL_BUF_SIZE 256 - static void snd_serial_generic_tx_work(struct work_struct *work) { - static char buf[INTERNAL_BUF_SIZE]; int num_bytes; struct snd_serial_generic *drvdata = container_of(work, struct snd_serial_generic, tx_work); @@ -78,8 +78,10 @@ static void snd_serial_generic_tx_work(struct work_struct *work) if (!test_bit(SERIAL_MODE_OUTPUT_OPEN, &drvdata->filemode)) break; - num_bytes = snd_rawmidi_transmit_peek(substream, buf, INTERNAL_BUF_SIZE); - num_bytes = serdev_device_write_buf(drvdata->serdev, buf, num_bytes); + num_bytes = snd_rawmidi_transmit_peek(substream, drvdata->tx_buf, + INTERNAL_BUF_SIZE); + num_bytes = serdev_device_write_buf(drvdata->serdev, drvdata->tx_buf, + num_bytes); if (!num_bytes) break; diff --git a/sound/firewire/amdtp-stream.h b/sound/firewire/amdtp-stream.h index 775db3fc4959f..ec10270c2cce3 100644 --- a/sound/firewire/amdtp-stream.h +++ b/sound/firewire/amdtp-stream.h @@ -32,7 +32,7 @@ * allows 5 times as large as IEC 61883-6 defines. * @CIP_HEADER_WITHOUT_EOH: Only for in-stream. CIP Header doesn't include * valid EOH. - * @CIP_NO_HEADERS: a lack of headers in packets + * @CIP_NO_HEADER: a lack of headers in packets * @CIP_UNALIGHED_DBC: Only for in-stream. The value of dbc is not alighed to * the value of current SYT_INTERVAL; e.g. initial value is not zero. * @CIP_UNAWARE_SYT: For outgoing packet, the value in SYT field of CIP is 0xffff. diff --git a/sound/firewire/motu/motu-hwdep.c b/sound/firewire/motu/motu-hwdep.c index 88d1f4b56e4be..a220ac0c8eb83 100644 --- a/sound/firewire/motu/motu-hwdep.c +++ b/sound/firewire/motu/motu-hwdep.c @@ -111,7 +111,7 @@ static __poll_t hwdep_poll(struct snd_hwdep *hwdep, struct file *file, events = 0; spin_unlock_irq(&motu->lock); - return events | EPOLLOUT; + return events; } static int hwdep_get_info(struct snd_motu *motu, void __user *arg) diff --git a/sound/isa/sb/sb16_main.c b/sound/isa/sb/sb16_main.c index a9b87e159b2d1..1497a7822eee6 100644 --- a/sound/isa/sb/sb16_main.c +++ b/sound/isa/sb/sb16_main.c @@ -703,6 +703,9 @@ static int snd_sb16_dma_control_put(struct snd_kcontrol *kcontrol, struct snd_ct unsigned char nval, oval; int change; + if (chip->mode & (SB_MODE_PLAYBACK | SB_MODE_CAPTURE)) + return -EBUSY; + nval = ucontrol->value.enumerated.item[0]; if (nval > 2) return -EINVAL; @@ -711,6 +714,10 @@ static int snd_sb16_dma_control_put(struct snd_kcontrol *kcontrol, struct snd_ct change = nval != oval; snd_sb16_set_dma_mode(chip, nval); spin_unlock_irqrestore(&chip->reg_lock, flags); + if (change) { + snd_dma_disable(chip->dma8); + snd_dma_disable(chip->dma16); + } return change; } diff --git a/sound/pci/es1968.c b/sound/pci/es1968.c index 4bc0f53c223b7..bc9203da35fb6 100644 --- a/sound/pci/es1968.c +++ b/sound/pci/es1968.c @@ -1569,7 +1569,7 @@ static int snd_es1968_capture_open(struct snd_pcm_substream *substream) struct snd_pcm_runtime *runtime = substream->runtime; struct es1968 *chip = snd_pcm_substream_chip(substream); struct esschan *es; - int apu1, apu2; + int err, apu1, apu2; apu1 = snd_es1968_alloc_apu_pair(chip, ESM_APU_PCM_CAPTURE); if (apu1 < 0) @@ -1613,7 +1613,9 @@ static int snd_es1968_capture_open(struct snd_pcm_substream *substream) runtime->hw = snd_es1968_capture; runtime->hw.buffer_bytes_max = runtime->hw.period_bytes_max = calc_available_memory_size(chip) - 1024; /* keep MIXBUF size */ - snd_pcm_hw_constraint_pow2(runtime, 0, SNDRV_PCM_HW_PARAM_BUFFER_BYTES); + err = snd_pcm_hw_constraint_pow2(runtime, 0, SNDRV_PCM_HW_PARAM_BUFFER_BYTES); + if (err < 0) + return err; spin_lock_irq(&chip->substream_lock); list_add(&es->list, &chip->substream_list); diff --git a/sound/pci/hda/hda_beep.c b/sound/pci/hda/hda_beep.c index e63621bcb2142..1a684e47d4d18 100644 --- a/sound/pci/hda/hda_beep.c +++ b/sound/pci/hda/hda_beep.c @@ -31,8 +31,9 @@ static void generate_tone(struct hda_beep *beep, int tone) beep->power_hook(beep, true); beep->playing = 1; } - snd_hda_codec_write(codec, beep->nid, 0, - AC_VERB_SET_BEEP_CONTROL, tone); + if (!codec->beep_just_power_on) + snd_hda_codec_write(codec, beep->nid, 0, + AC_VERB_SET_BEEP_CONTROL, tone); if (!tone && beep->playing) { beep->playing = 0; if (beep->power_hook) @@ -212,10 +213,12 @@ int snd_hda_attach_beep_device(struct hda_codec *codec, int nid) struct hda_beep *beep; int err; - if (!snd_hda_get_bool_hint(codec, "beep")) - return 0; /* disabled explicitly by hints */ - if (codec->beep_mode == HDA_BEEP_MODE_OFF) - return 0; /* disabled by module option */ + if (!codec->beep_just_power_on) { + if (!snd_hda_get_bool_hint(codec, "beep")) + return 0; /* disabled explicitly by hints */ + if (codec->beep_mode == HDA_BEEP_MODE_OFF) + return 0; /* disabled by module option */ + } beep = kzalloc(sizeof(*beep), GFP_KERNEL); if (beep == NULL) diff --git a/sound/pci/hda/hda_bind.c b/sound/pci/hda/hda_bind.c index b7ca2a83fbb08..95786bdadfe6a 100644 --- a/sound/pci/hda/hda_bind.c +++ b/sound/pci/hda/hda_bind.c @@ -44,7 +44,7 @@ static void hda_codec_unsol_event(struct hdac_device *dev, unsigned int ev) struct hda_codec *codec = container_of(dev, struct hda_codec, core); /* ignore unsol events during shutdown */ - if (codec->bus->shutdown) + if (codec->card->shutdown || codec->bus->shutdown) return; /* ignore unsol events during system suspend/resume */ diff --git a/sound/pci/hda/hda_codec.c b/sound/pci/hda/hda_codec.c index aa6dc00985b51..80c3084189b03 100644 --- a/sound/pci/hda/hda_codec.c +++ b/sound/pci/hda/hda_codec.c @@ -641,24 +641,16 @@ static void hda_jackpoll_work(struct work_struct *work) struct hda_codec *codec = container_of(work, struct hda_codec, jackpoll_work.work); - /* for non-polling trigger: we need nothing if already powered on */ - if (!codec->jackpoll_interval && snd_hdac_is_power_on(&codec->core)) + if (!codec->jackpoll_interval) return; /* the power-up/down sequence triggers the runtime resume */ - snd_hda_power_up_pm(codec); + snd_hda_power_up(codec); /* update jacks manually if polling is required, too */ - if (codec->jackpoll_interval) { - snd_hda_jack_set_dirty_all(codec); - snd_hda_jack_poll_all(codec); - } - snd_hda_power_down_pm(codec); - - if (!codec->jackpoll_interval) - return; - - schedule_delayed_work(&codec->jackpoll_work, - codec->jackpoll_interval); + snd_hda_jack_set_dirty_all(codec); + snd_hda_jack_poll_all(codec); + schedule_delayed_work(&codec->jackpoll_work, codec->jackpoll_interval); + snd_hda_power_down(codec); } /* release all pincfg lists */ @@ -2920,12 +2912,12 @@ static void hda_call_codec_resume(struct hda_codec *codec) snd_hda_regmap_sync(codec); } - if (codec->jackpoll_interval) - hda_jackpoll_work(&codec->jackpoll_work.work); - else - snd_hda_jack_report_sync(codec); + snd_hda_jack_report_sync(codec); codec->core.dev.power.power_state = PMSG_ON; snd_hdac_leave_pm(&codec->core); + if (codec->jackpoll_interval) + schedule_delayed_work(&codec->jackpoll_work, + codec->jackpoll_interval); } static int hda_codec_runtime_suspend(struct device *dev) @@ -2937,8 +2929,6 @@ static int hda_codec_runtime_suspend(struct device *dev) if (!codec->card) return 0; - cancel_delayed_work_sync(&codec->jackpoll_work); - state = hda_call_codec_suspend(codec); if (codec->link_down_at_suspend || (codec_has_clkstop(codec) && codec_has_epss(codec) && @@ -2946,10 +2936,6 @@ static int hda_codec_runtime_suspend(struct device *dev) snd_hdac_codec_link_down(&codec->core); snd_hda_codec_display_power(codec, false); - if (codec->bus->jackpoll_in_suspend && - (dev->power.power_state.event != PM_EVENT_SUSPEND)) - schedule_delayed_work(&codec->jackpoll_work, - codec->jackpoll_interval); return 0; } @@ -3052,6 +3038,7 @@ void snd_hda_codec_shutdown(struct hda_codec *codec) if (!codec->core.registered) return; + codec->jackpoll_interval = 0; /* don't poll any longer */ cancel_delayed_work_sync(&codec->jackpoll_work); list_for_each_entry(cpcm, &codec->pcm_list_head, list) snd_pcm_suspend_all(cpcm->pcm); @@ -3118,10 +3105,11 @@ int snd_hda_codec_build_controls(struct hda_codec *codec) if (err < 0) return err; + snd_hda_jack_report_sync(codec); /* call at the last init point */ if (codec->jackpoll_interval) - hda_jackpoll_work(&codec->jackpoll_work.work); - else - snd_hda_jack_report_sync(codec); /* call at the last init point */ + schedule_delayed_work(&codec->jackpoll_work, + codec->jackpoll_interval); + sync_power_up_states(codec); return 0; } diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c index 3a0df631d25d9..059693e03fd96 100644 --- a/sound/pci/hda/hda_intel.c +++ b/sound/pci/hda/hda_intel.c @@ -2266,6 +2266,8 @@ static const struct snd_pci_quirk power_save_denylist[] = { SND_PCI_QUIRK(0x1734, 0x1232, "KONTRON SinglePC", 0), /* Dell ALC3271 */ SND_PCI_QUIRK(0x1028, 0x0962, "Dell ALC3271", 0), + /* https://bugzilla.kernel.org/show_bug.cgi?id=220210 */ + SND_PCI_QUIRK(0x17aa, 0x5079, "Lenovo Thinkpad E15", 0), {} }; #endif /* CONFIG_PM */ @@ -2725,6 +2727,9 @@ static const struct pci_device_id azx_ids[] = { { PCI_VDEVICE(ATI, 0xab38), .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS | AZX_DCAPS_PM_RUNTIME }, + { PCI_VDEVICE(ATI, 0xab40), + .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS | + AZX_DCAPS_PM_RUNTIME }, /* GLENFLY */ { PCI_DEVICE(PCI_VENDOR_ID_GLENFLY, PCI_ANY_ID), .class = PCI_CLASS_MULTIMEDIA_HD_AUDIO << 8, diff --git a/sound/pci/hda/hda_tegra.c b/sound/pci/hda/hda_tegra.c index d967e70a70585..12a144a269ee8 100644 --- a/sound/pci/hda/hda_tegra.c +++ b/sound/pci/hda/hda_tegra.c @@ -72,6 +72,10 @@ struct hda_tegra_soc { bool has_hda2codec_2x_reset; bool has_hda2hdmi; + bool has_hda2codec_2x; + bool input_stream; + bool always_on; + bool requires_init; }; struct hda_tegra { @@ -187,7 +191,9 @@ static int __maybe_unused hda_tegra_runtime_resume(struct device *dev) if (rc != 0) return rc; if (chip->running) { - hda_tegra_init(hda); + if (hda->soc->requires_init) + hda_tegra_init(hda); + azx_init_chip(chip, 1); /* disable controller wake up event*/ azx_writew(chip, WAKEEN, azx_readw(chip, WAKEEN) & @@ -252,7 +258,8 @@ static int hda_tegra_init_chip(struct azx *chip, struct platform_device *pdev) bus->remap_addr = hda->regs + HDA_BAR0; bus->addr = res->start + HDA_BAR0; - hda_tegra_init(hda); + if (hda->soc->requires_init) + hda_tegra_init(hda); return 0; } @@ -325,7 +332,7 @@ static int hda_tegra_first_init(struct azx *chip, struct platform_device *pdev) * starts with offset 0 which is wrong as HW register for output stream * offset starts with 4. */ - if (of_device_is_compatible(np, "nvidia,tegra234-hda")) + if (!hda->soc->input_stream) chip->capture_streams = 4; chip->playback_streams = (gcap >> 12) & 0x0f; @@ -421,7 +428,6 @@ static int hda_tegra_create(struct snd_card *card, chip->driver_caps = driver_caps; chip->driver_type = driver_caps & 0xff; chip->dev_index = 0; - chip->jackpoll_interval = msecs_to_jiffies(5000); INIT_LIST_HEAD(&chip->pcm_list); chip->codec_probe_mask = -1; @@ -438,7 +444,16 @@ static int hda_tegra_create(struct snd_card *card, chip->bus.core.sync_write = 0; chip->bus.core.needs_damn_long_delay = 1; chip->bus.core.aligned_mmio = 1; - chip->bus.jackpoll_in_suspend = 1; + + /* + * HDA power domain and clocks are always on for Tegra264 and + * the jack detection logic would work always, so no need of + * jack polling mechanism running. + */ + if (!hda->soc->always_on) { + chip->jackpoll_interval = msecs_to_jiffies(5000); + chip->bus.jackpoll_in_suspend = 1; + } err = snd_device_new(card, SNDRV_DEV_LOWLEVEL, chip, &ops); if (err < 0) { @@ -452,22 +467,44 @@ static int hda_tegra_create(struct snd_card *card, static const struct hda_tegra_soc tegra30_data = { .has_hda2codec_2x_reset = true, .has_hda2hdmi = true, + .has_hda2codec_2x = true, + .input_stream = true, + .always_on = false, + .requires_init = true, }; static const struct hda_tegra_soc tegra194_data = { .has_hda2codec_2x_reset = false, .has_hda2hdmi = true, + .has_hda2codec_2x = true, + .input_stream = true, + .always_on = false, + .requires_init = true, }; static const struct hda_tegra_soc tegra234_data = { .has_hda2codec_2x_reset = true, .has_hda2hdmi = false, + .has_hda2codec_2x = true, + .input_stream = false, + .always_on = false, + .requires_init = true, +}; + +static const struct hda_tegra_soc tegra264_data = { + .has_hda2codec_2x_reset = true, + .has_hda2hdmi = false, + .has_hda2codec_2x = false, + .input_stream = false, + .always_on = true, + .requires_init = false, }; static const struct of_device_id hda_tegra_match[] = { { .compatible = "nvidia,tegra30-hda", .data = &tegra30_data }, { .compatible = "nvidia,tegra194-hda", .data = &tegra194_data }, { .compatible = "nvidia,tegra234-hda", .data = &tegra234_data }, + { .compatible = "nvidia,tegra264-hda", .data = &tegra264_data }, {}, }; MODULE_DEVICE_TABLE(of, hda_tegra_match); @@ -522,7 +559,9 @@ static int hda_tegra_probe(struct platform_device *pdev) hda->clocks[hda->nclocks++].id = "hda"; if (hda->soc->has_hda2hdmi) hda->clocks[hda->nclocks++].id = "hda2hdmi"; - hda->clocks[hda->nclocks++].id = "hda2codec_2x"; + + if (hda->soc->has_hda2codec_2x) + hda->clocks[hda->nclocks++].id = "hda2codec_2x"; err = devm_clk_bulk_get(&pdev->dev, hda->nclocks, hda->clocks); if (err < 0) diff --git a/sound/pci/hda/patch_ca0132.c b/sound/pci/hda/patch_ca0132.c index 27e48fdbbf3aa..851e9231bbbd5 100644 --- a/sound/pci/hda/patch_ca0132.c +++ b/sound/pci/hda/patch_ca0132.c @@ -4411,7 +4411,7 @@ static int add_tuning_control(struct hda_codec *codec, } knew.private_value = HDA_COMPOSE_AMP_VAL(nid, 1, 0, type); - sprintf(namestr, "%s %s Volume", name, dirstr[dir]); + snprintf(namestr, sizeof(namestr), "%s %s Volume", name, dirstr[dir]); return snd_hda_ctl_add(codec, nid, snd_ctl_new1(&knew, codec)); } @@ -4803,7 +4803,8 @@ static int ca0132_alt_select_out(struct hda_codec *codec) if (err < 0) goto exit; - if (ca0132_alt_select_out_quirk_set(codec) < 0) + err = ca0132_alt_select_out_quirk_set(codec); + if (err < 0) goto exit; switch (spec->cur_out_type) { @@ -4893,6 +4894,8 @@ static int ca0132_alt_select_out(struct hda_codec *codec) spec->bass_redirection_val); else err = ca0132_alt_surround_set_bass_redirection(codec, 0); + if (err < 0) + goto exit; /* Unmute DSP now that we're done with output selection. */ err = dspio_set_uint_param(codec, 0x96, diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index f030700cd60d7..80c015af09efd 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -1991,6 +1991,7 @@ static int hdmi_add_cvt(struct hda_codec *codec, hda_nid_t cvt_nid) static const struct snd_pci_quirk force_connect_list[] = { SND_PCI_QUIRK(0x103c, 0x83e2, "HP EliteDesk 800 G4", 1), SND_PCI_QUIRK(0x103c, 0x83ef, "HP MP9 G4 Retail System AMS", 1), + SND_PCI_QUIRK(0x103c, 0x845a, "HP EliteDesk 800 G4 DM 65W", 1), SND_PCI_QUIRK(0x103c, 0x870f, "HP", 1), SND_PCI_QUIRK(0x103c, 0x871a, "HP", 1), SND_PCI_QUIRK(0x103c, 0x8711, "HP", 1), @@ -4559,6 +4560,9 @@ HDA_CODEC_ENTRY(0x10de002e, "Tegra186 HDMI/DP1", patch_tegra_hdmi), HDA_CODEC_ENTRY(0x10de002f, "Tegra194 HDMI/DP2", patch_tegra_hdmi), HDA_CODEC_ENTRY(0x10de0030, "Tegra194 HDMI/DP3", patch_tegra_hdmi), HDA_CODEC_ENTRY(0x10de0031, "Tegra234 HDMI/DP", patch_tegra234_hdmi), +HDA_CODEC_ENTRY(0x10de0033, "SoC 33 HDMI/DP", patch_tegra234_hdmi), +HDA_CODEC_ENTRY(0x10de0034, "Tegra264 HDMI/DP", patch_tegra234_hdmi), +HDA_CODEC_ENTRY(0x10de0035, "SoC 35 HDMI/DP", patch_tegra234_hdmi), HDA_CODEC_ENTRY(0x10de0040, "GPU 40 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0041, "GPU 41 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0042, "GPU 42 HDMI/DP", patch_nvhdmi), @@ -4597,15 +4601,32 @@ HDA_CODEC_ENTRY(0x10de0097, "GPU 97 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0098, "GPU 98 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0099, "GPU 99 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de009a, "GPU 9a HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de009b, "GPU 9b HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de009c, "GPU 9c HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de009d, "GPU 9d HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de009e, "GPU 9e HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de009f, "GPU 9f HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de00a0, "GPU a0 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00a1, "GPU a1 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de00a3, "GPU a3 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de00a4, "GPU a4 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de00a5, "GPU a5 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de00a6, "GPU a6 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de00a7, "GPU a7 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00a8, "GPU a8 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00a9, "GPU a9 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00aa, "GPU aa HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00ab, "GPU ab HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00ad, "GPU ad HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00ae, "GPU ae HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00af, "GPU af HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00b0, "GPU b0 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00b1, "GPU b1 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00c0, "GPU c0 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00c1, "GPU c1 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00c3, "GPU c3 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00c4, "GPU c4 HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de00c5, "GPU c5 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de8001, "MCP73 HDMI", patch_nvhdmi_2ch), HDA_CODEC_ENTRY(0x10de8067, "MCP67/68 HDMI", patch_nvhdmi_2ch), HDA_CODEC_ENTRY(0x67663d82, "Arise 82 HDMI/DP", patch_gf_hdmi), diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 2f3f295f2b0cb..65c9d47f03af5 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -24,6 +24,7 @@ #include #include "hda_local.h" #include "hda_auto_parser.h" +#include "hda_beep.h" #include "hda_jack.h" #include "hda_generic.h" #include "hda_component.h" @@ -3643,6 +3644,15 @@ static void alc256_shutup(struct hda_codec *codec) hp_pin = 0x21; alc_update_coefex_idx(codec, 0x57, 0x04, 0x0007, 0x1); /* Low power */ + + /* 3k pull low control for Headset jack. */ + /* NOTE: call this before clearing the pin, otherwise codec stalls */ + /* If disable 3k pulldown control for alc257, the Mic detection will not work correctly + * when booting with headset plugged. So skip setting it for the codec alc257 + */ + if (spec->en_3kpull_low) + alc_update_coef_idx(codec, 0x46, 0, 3 << 12); + hp_pin_sense = snd_hda_jack_detect(codec, hp_pin); if (hp_pin_sense) { @@ -3653,14 +3663,6 @@ static void alc256_shutup(struct hda_codec *codec) msleep(75); - /* 3k pull low control for Headset jack. */ - /* NOTE: call this before clearing the pin, otherwise codec stalls */ - /* If disable 3k pulldown control for alc257, the Mic detection will not work correctly - * when booting with headset plugged. So skip setting it for the codec alc257 - */ - if (spec->en_3kpull_low) - alc_update_coef_idx(codec, 0x46, 0, 3 << 12); - if (!spec->no_shutup_pins) snd_hda_codec_write(codec, hp_pin, 0, AC_VERB_SET_PIN_WIDGET_CONTROL, 0x0); @@ -6789,6 +6791,41 @@ static void alc285_fixup_hp_spectre_x360_eb1(struct hda_codec *codec, } } +/* GPIO1 = amplifier on/off */ +static void alc285_fixup_hp_spectre_x360_df1(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + struct alc_spec *spec = codec->spec; + static const hda_nid_t conn[] = { 0x02 }; + static const struct hda_pintbl pincfgs[] = { + { 0x14, 0x90170110 }, /* front/high speakers */ + { 0x17, 0x90170130 }, /* back/bass speakers */ + { } + }; + + // enable mute led + alc285_fixup_hp_mute_led_coefbit(codec, fix, action); + + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + /* needed for amp of back speakers */ + spec->gpio_mask |= 0x01; + spec->gpio_dir |= 0x01; + snd_hda_apply_pincfgs(codec, pincfgs); + /* share DAC to have unified volume control */ + snd_hda_override_conn_list(codec, 0x14, ARRAY_SIZE(conn), conn); + snd_hda_override_conn_list(codec, 0x17, ARRAY_SIZE(conn), conn); + break; + case HDA_FIXUP_ACT_INIT: + /* need to toggle GPIO to enable the amp of back speakers */ + alc_update_gpio_data(codec, 0x01, true); + msleep(100); + alc_update_gpio_data(codec, 0x01, false); + break; + } +} + static void alc285_fixup_hp_spectre_x360(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -6861,6 +6898,30 @@ static void alc285_fixup_hp_envy_x360(struct hda_codec *codec, } } +static void alc285_fixup_hp_beep(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + if (action == HDA_FIXUP_ACT_PRE_PROBE) { + codec->beep_just_power_on = true; + } else if (action == HDA_FIXUP_ACT_INIT) { +#ifdef CONFIG_SND_HDA_INPUT_BEEP + /* + * Just enable loopback to internal speaker and headphone jack. + * Disable amplification to get about the same beep volume as + * was on pure BIOS setup before loading the driver. + */ + alc_update_coef_idx(codec, 0x36, 0x7070, BIT(13)); + + snd_hda_enable_beep_device(codec, 1); + +#if !IS_ENABLED(CONFIG_INPUT_PCSPKR) + dev_warn_once(hda_codec_dev(codec), + "enable CONFIG_INPUT_PCSPKR to get PC beeps\n"); +#endif +#endif + } +} + /* for hda_fixup_thinkpad_acpi() */ #include "thinkpad_helper.c" @@ -7376,6 +7437,7 @@ enum { ALC280_FIXUP_HP_9480M, ALC245_FIXUP_HP_X360_AMP, ALC285_FIXUP_HP_SPECTRE_X360_EB1, + ALC285_FIXUP_HP_SPECTRE_X360_DF1, ALC285_FIXUP_HP_ENVY_X360, ALC288_FIXUP_DELL_HEADSET_MODE, ALC288_FIXUP_DELL1_MIC_NO_PRESENCE, @@ -7477,6 +7539,7 @@ enum { ALC285_FIXUP_HP_GPIO_LED, ALC285_FIXUP_HP_MUTE_LED, ALC285_FIXUP_HP_SPECTRE_X360_MUTE_LED, + ALC285_FIXUP_HP_BEEP_MICMUTE_LED, ALC236_FIXUP_HP_MUTE_LED_COEFBIT2, ALC236_FIXUP_HP_GPIO_LED, ALC236_FIXUP_HP_MUTE_LED, @@ -9064,6 +9127,12 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_spectre_x360_mute_led, }, + [ALC285_FIXUP_HP_BEEP_MICMUTE_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc285_fixup_hp_beep, + .chained = true, + .chain_id = ALC285_FIXUP_HP_MUTE_LED, + }, [ALC236_FIXUP_HP_MUTE_LED_COEFBIT2] = { .type = HDA_FIXUP_FUNC, .v.func = alc236_fixup_hp_mute_led_coefbit2, @@ -9407,6 +9476,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_spectre_x360_eb1 }, + [ALC285_FIXUP_HP_SPECTRE_X360_DF1] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc285_fixup_hp_spectre_x360_df1 + }, [ALC285_FIXUP_HP_ENVY_X360] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_envy_x360, @@ -9880,6 +9953,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x0871, "Dell Precision 3630", ALC255_FIXUP_DELL_HEADSET_MIC), SND_PCI_QUIRK(0x1028, 0x0872, "Dell Precision 3630", ALC255_FIXUP_DELL_HEADSET_MIC), SND_PCI_QUIRK(0x1028, 0x0873, "Dell Precision 3930", ALC255_FIXUP_DUMMY_LINEOUT_VERB), + SND_PCI_QUIRK(0x1028, 0x0879, "Dell Latitude 5420 Rugged", ALC269_FIXUP_DELL4_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x08ad, "Dell WYSE AIO", ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x08ae, "Dell WYSE NB", ALC225_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x0935, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB), @@ -9998,6 +10072,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x84e7, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x8519, "HP Spectre x360 15-df0xxx", ALC285_FIXUP_HP_SPECTRE_X360), SND_PCI_QUIRK(0x103c, 0x8537, "HP ProBook 440 G6", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), + SND_PCI_QUIRK(0x103c, 0x8548, "HP EliteBook x360 830 G6", ALC285_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x854a, "HP EliteBook 830 G6", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x85c6, "HP Pavilion x360 Convertible 14-dy1xxx", ALC295_FIXUP_HP_MUTE_LED_COEFBIT11), SND_PCI_QUIRK(0x103c, 0x85de, "HP Envy x360 13-ar0xxx", ALC285_FIXUP_HP_ENVY_X360), SND_PCI_QUIRK(0x103c, 0x860f, "HP ZBook 15 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT), @@ -10006,6 +10082,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x86c1, "HP Laptop 15-da3001TU", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x86c7, "HP Envy AiO 32", ALC274_FIXUP_HP_ENVY_GPIO), SND_PCI_QUIRK(0x103c, 0x86e7, "HP Spectre x360 15-eb0xxx", ALC285_FIXUP_HP_SPECTRE_X360_EB1), + SND_PCI_QUIRK(0x103c, 0x863e, "HP Spectre x360 15-df1xxx", ALC285_FIXUP_HP_SPECTRE_X360_DF1), SND_PCI_QUIRK(0x103c, 0x86e8, "HP Spectre x360 15-eb0xxx", ALC285_FIXUP_HP_SPECTRE_X360_EB1), SND_PCI_QUIRK(0x103c, 0x86f9, "HP Spectre x360 13-aw0xxx", ALC285_FIXUP_HP_SPECTRE_X360_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8716, "HP Elite Dragonfly G2 Notebook PC", ALC285_FIXUP_HP_GPIO_AMP_INIT), @@ -10016,7 +10093,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8730, "HP ProBook 445 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8735, "HP ProBook 435 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8736, "HP", ALC285_FIXUP_HP_GPIO_AMP_INIT), - SND_PCI_QUIRK(0x103c, 0x8760, "HP", ALC285_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x8760, "HP EliteBook 8{4,5}5 G7", ALC285_FIXUP_HP_BEEP_MICMUTE_LED), SND_PCI_QUIRK(0x103c, 0x876e, "HP ENVY x360 Convertible 13-ay0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x877a, "HP", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x877d, "HP", ALC236_FIXUP_HP_MUTE_LED), @@ -10029,6 +10106,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8788, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x87b7, "HP Laptop 14-fq0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x87c8, "HP", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x87cc, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87d3, "HP Laptop 15-gw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x87df, "HP ProBook 430 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87e5, "HP ProBook 440 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), @@ -10077,12 +10155,14 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8975, "HP EliteBook x360 840 Aero G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x897d, "HP mt440 Mobile Thin Client U74", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8981, "HP Elite Dragonfly G3", ALC245_FIXUP_CS35L41_SPI_4), + SND_PCI_QUIRK(0x103c, 0x898a, "HP Pavilion 15-eg100", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x898e, "HP EliteBook 835 G9", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x898f, "HP EliteBook 835 G9", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8991, "HP EliteBook 845 G9", ALC287_FIXUP_CS35L41_I2C_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8992, "HP EliteBook 845 G9", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8994, "HP EliteBook 855 G9", ALC287_FIXUP_CS35L41_I2C_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8995, "HP EliteBook 855 G9", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x103c, 0x89a0, "HP Laptop 15-dw4xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x89a4, "HP ProBook 440 G9", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89a6, "HP ProBook 450 G9", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89aa, "HP EliteBook 630 G9", ALC236_FIXUP_HP_GPIO_LED), @@ -10171,6 +10251,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8e18, "HP ZBook Firefly 14 G12A", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8e19, "HP ZBook Firefly 14 G12A", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8e1a, "HP ZBook Firefly 14 G12A", ALC285_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8e1d, "HP ZBook X Gli 16 G12", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8e3a, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x103c, 0x8e3b, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x103e, "ASUS X540SA", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x103f, "ASUS TX300", ALC282_FIXUP_ASUS_TX300), SND_PCI_QUIRK(0x1043, 0x1054, "ASUS G614FH/FM/FP", ALC287_FIXUP_CS35L41_I2C_2), @@ -10228,6 +10311,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1a13, "Asus G73Jw", ALC269_FIXUP_ASUS_G73JW), SND_PCI_QUIRK(0x1043, 0x1a63, "ASUS UX3405MA", ALC245_FIXUP_CS35L41_SPI_2), SND_PCI_QUIRK(0x1043, 0x1a83, "ASUS UM5302LA", ALC294_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x1043, 0x1a8e, "ASUS G712LWS", ALC294_FIXUP_LENOVO_MIC_LOCATION), SND_PCI_QUIRK(0x1043, 0x1a8f, "ASUS UX582ZS", ALC245_FIXUP_CS35L41_SPI_2), SND_PCI_QUIRK(0x1043, 0x1b11, "ASUS UX431DA", ALC294_FIXUP_ASUS_COEF_1B), SND_PCI_QUIRK(0x1043, 0x1b13, "Asus U41SV", ALC269_FIXUP_INV_DMIC), @@ -10251,6 +10335,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1d4e, "ASUS TM420", ALC256_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x1da2, "ASUS UP6502ZA/ZD", ALC245_FIXUP_CS35L41_SPI_2), SND_PCI_QUIRK(0x1043, 0x1e02, "ASUS UX3402ZA", ALC245_FIXUP_CS35L41_SPI_2), + SND_PCI_QUIRK(0x1043, 0x1e10, "ASUS VivoBook X507UAR", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1e11, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA502), SND_PCI_QUIRK(0x1043, 0x1e12, "ASUS UM3402", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1e51, "ASUS Zephyrus M15", ALC294_FIXUP_ASUS_GU502_PINS), @@ -10501,6 +10586,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38f9, "Thinkbook 16P Gen5", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x38fa, "Thinkbook 16P Gen5", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI), + SND_PCI_QUIRK(0x17aa, 0x390d, "Lenovo Yoga Pro 7 14ASP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3913, "Lenovo 145", ALC236_FIXUP_LENOVO_INV_DMIC), SND_PCI_QUIRK(0x17aa, 0x3977, "IdeaPad S210", ALC283_FIXUP_INT_MIC), SND_PCI_QUIRK(0x17aa, 0x3978, "Lenovo B50-70", ALC269_FIXUP_DMIC_THINKPAD_ACPI), @@ -10551,12 +10637,15 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1d05, 0x121b, "TongFang GMxAGxx", ALC269_FIXUP_NO_SHUTUP), SND_PCI_QUIRK(0x1d05, 0x1387, "TongFang GMxIXxx", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x1409, "TongFang GMxIXxx", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x300f, "TongFang X6AR5xxY", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x3019, "TongFang X6FR5xxY", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d17, 0x3288, "Haier Boyue G42", ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS), SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1945, "Redmi G", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC), + SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1f66, 0x0105, "Ayaneo Portable Game Player", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2782, 0x0214, "VAIO VJFE-CL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), @@ -10572,6 +10661,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0xf111, 0x0001, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x0006, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x0009, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0xf111, 0x000b, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x000c, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), #if 0 @@ -10754,6 +10844,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC295_FIXUP_HP_OMEN, .name = "alc295-hp-omen"}, {.id = ALC285_FIXUP_HP_SPECTRE_X360, .name = "alc285-hp-spectre-x360"}, {.id = ALC285_FIXUP_HP_SPECTRE_X360_EB1, .name = "alc285-hp-spectre-x360-eb1"}, + {.id = ALC285_FIXUP_HP_SPECTRE_X360_DF1, .name = "alc285-hp-spectre-x360-df1"}, {.id = ALC285_FIXUP_HP_ENVY_X360, .name = "alc285-hp-envy-x360"}, {.id = ALC287_FIXUP_IDEAPAD_BASS_SPK_AMP, .name = "alc287-ideapad-bass-spk-amp"}, {.id = ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN, .name = "alc287-yoga9-bass-spk-pin"}, diff --git a/sound/pci/intel8x0.c b/sound/pci/intel8x0.c index ae285c0a629c8..f3df6fe2b7f18 100644 --- a/sound/pci/intel8x0.c +++ b/sound/pci/intel8x0.c @@ -2252,7 +2252,7 @@ static int snd_intel8x0_mixer(struct intel8x0 *chip, int ac97_clock, tmp |= chip->ac97_sdin[0] << ICH_DI1L_SHIFT; for (i = 1; i < 4; i++) { if (pcm->r[0].codec[i]) { - tmp |= chip->ac97_sdin[pcm->r[0].codec[1]->num] << ICH_DI2L_SHIFT; + tmp |= chip->ac97_sdin[pcm->r[0].codec[i]->num] << ICH_DI2L_SHIFT; break; } } diff --git a/sound/pci/lx6464es/lx_core.c b/sound/pci/lx6464es/lx_core.c index b5b0d43bb8dcd..c3f2717aebf25 100644 --- a/sound/pci/lx6464es/lx_core.c +++ b/sound/pci/lx6464es/lx_core.c @@ -316,7 +316,7 @@ static int lx_message_send_atomic(struct lx6464es *chip, struct lx_rmh *rmh) /* low-level dsp access */ int lx_dsp_get_version(struct lx6464es *chip, u32 *rdsp_version) { - u16 ret; + int ret; mutex_lock(&chip->msg_lock); @@ -330,10 +330,10 @@ int lx_dsp_get_version(struct lx6464es *chip, u32 *rdsp_version) int lx_dsp_get_clock_frequency(struct lx6464es *chip, u32 *rfreq) { - u16 ret = 0; u32 freq_raw = 0; u32 freq = 0; u32 frequency = 0; + int ret; mutex_lock(&chip->msg_lock); diff --git a/sound/sh/Kconfig b/sound/sh/Kconfig index b75fbb3236a7b..f5fa09d740b4c 100644 --- a/sound/sh/Kconfig +++ b/sound/sh/Kconfig @@ -14,7 +14,7 @@ if SND_SUPERH config SND_AICA tristate "Dreamcast Yamaha AICA sound" - depends on SH_DREAMCAST + depends on SH_DREAMCAST && SH_DMA_API select SND_PCM select G2_DMA help diff --git a/sound/soc/amd/acp/amd.h b/sound/soc/amd/acp/amd.h index d6cfae6ec5f74..3f1761755b866 100644 --- a/sound/soc/amd/acp/amd.h +++ b/sound/soc/amd/acp/amd.h @@ -116,7 +116,7 @@ #define PDM_DMA_INTR_MASK 0x10000 #define PDM_DEC_64 0x2 #define PDM_CLK_FREQ_MASK 0x07 -#define PDM_MISC_CTRL_MASK 0x10 +#define PDM_MISC_CTRL_MASK 0x18 #define PDM_ENABLE 0x01 #define PDM_DISABLE 0x00 #define DMA_EN_MASK 0x02 diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index 622df58a96942..24919e68b3468 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -311,6 +311,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "83AS"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83HN"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -339,6 +346,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "83Q3"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "RB"), + DMI_MATCH(DMI_PRODUCT_NAME, "Nitro ANV15-41"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -346,6 +360,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "83J2"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83J3"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -360,7 +381,7 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "M5402RA"), } }, - { + { .driver_data = &acp6x_card, .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), @@ -388,6 +409,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "M6500RC"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "M6501RM"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -437,6 +465,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Bravo 17 D7VEK"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."), + DMI_MATCH(DMI_PRODUCT_NAME, "Bravo 17 D7VF"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -500,6 +535,20 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "OMEN by HP Gaming Laptop 16z-n000"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "Victus by HP Gaming Laptop 15-fb1xxx"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "Victus by HP Gaming Laptop 15-fb2xxx"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -542,6 +591,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "8A7F"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_BOARD_NAME, "8A81"), + } + }, { .driver_data = &acp6x_card, .matches = { diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index ce77934f3eef3..0e96caa607fb8 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -464,6 +464,28 @@ static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, return -EINVAL; } +static int mca_fe_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + unsigned int mask, nchannels; + + if (cl->tdm_slots) { + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + mask = cl->tdm_tx_mask; + else + mask = cl->tdm_rx_mask; + + nchannels = hweight32(mask); + } else { + nchannels = 2; + } + + return snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_CHANNELS, + 1, nchannels); +} + static int mca_fe_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, unsigned int rx_mask, int slots, int slot_width) { @@ -680,6 +702,7 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, } static const struct snd_soc_dai_ops mca_fe_ops = { + .startup = mca_fe_startup, .set_fmt = mca_fe_set_fmt, .set_bclk_ratio = mca_set_bclk_ratio, .set_tdm_slot = mca_fe_set_tdm_slot, diff --git a/sound/soc/codecs/ak4613.c b/sound/soc/codecs/ak4613.c index 619a817ee91cb..4c1318ce8ae58 100644 --- a/sound/soc/codecs/ak4613.c +++ b/sound/soc/codecs/ak4613.c @@ -840,14 +840,14 @@ static void ak4613_parse_of(struct ak4613_priv *priv, /* Input 1 - 2 */ for (i = 0; i < 2; i++) { snprintf(prop, sizeof(prop), "asahi-kasei,in%d-single-end", i + 1); - if (!of_get_property(np, prop, NULL)) + if (!of_property_read_bool(np, prop)) priv->ic |= 1 << i; } /* Output 1 - 6 */ for (i = 0; i < 6; i++) { snprintf(prop, sizeof(prop), "asahi-kasei,out%d-single-end", i + 1); - if (!of_get_property(np, prop, NULL)) + if (!of_property_read_bool(np, prop)) priv->oc |= 1 << i; } diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c index d3db89c93b331..25ee7477709e7 100644 --- a/sound/soc/codecs/cs35l56-shared.c +++ b/sound/soc/codecs/cs35l56-shared.c @@ -661,7 +661,7 @@ int cs35l56_hw_init(struct cs35l56_base *cs35l56_base) break; default: dev_err(cs35l56_base->dev, "Unknown device %x\n", devid); - return ret; + return -ENODEV; } ret = regmap_read(cs35l56_base->regmap, CS35L56_DSP_RESTRICT_STS1, &secured); diff --git a/sound/soc/codecs/cs4271.c b/sound/soc/codecs/cs4271.c index 188b8b43c524f..b7529e2900572 100644 --- a/sound/soc/codecs/cs4271.c +++ b/sound/soc/codecs/cs4271.c @@ -594,17 +594,17 @@ static int cs4271_component_probe(struct snd_soc_component *component) ret = regcache_sync(cs4271->regmap); if (ret < 0) - return ret; + goto err_disable_regulator; ret = regmap_update_bits(cs4271->regmap, CS4271_MODE2, CS4271_MODE2_PDN | CS4271_MODE2_CPEN, CS4271_MODE2_PDN | CS4271_MODE2_CPEN); if (ret < 0) - return ret; + goto err_disable_regulator; ret = regmap_update_bits(cs4271->regmap, CS4271_MODE2, CS4271_MODE2_PDN, 0); if (ret < 0) - return ret; + goto err_disable_regulator; /* Power-up sequence requires 85 uS */ udelay(85); @@ -614,6 +614,10 @@ static int cs4271_component_probe(struct snd_soc_component *component) CS4271_MODE2_MUTECAEQUB); return 0; + +err_disable_regulator: + regulator_bulk_disable(ARRAY_SIZE(cs4271->supplies), cs4271->supplies); + return ret; } static void cs4271_component_remove(struct snd_soc_component *component) diff --git a/sound/soc/codecs/cs42l43-jack.c b/sound/soc/codecs/cs42l43-jack.c index 0b8e88b19888e..6d8455c1bee6d 100644 --- a/sound/soc/codecs/cs42l43-jack.c +++ b/sound/soc/codecs/cs42l43-jack.c @@ -642,6 +642,10 @@ static int cs42l43_run_type_detect(struct cs42l43_codec *priv) reinit_completion(&priv->type_detect); + regmap_update_bits(cs42l43->regmap, CS42L43_STEREO_MIC_CLAMP_CTRL, + CS42L43_SMIC_HPAMP_CLAMP_DIS_FRC_VAL_MASK, + CS42L43_SMIC_HPAMP_CLAMP_DIS_FRC_VAL_MASK); + cs42l43_start_hs_bias(priv, true); regmap_update_bits(cs42l43->regmap, CS42L43_HS2, CS42L43_HSDET_MODE_MASK, 0x3 << CS42L43_HSDET_MODE_SHIFT); @@ -653,6 +657,9 @@ static int cs42l43_run_type_detect(struct cs42l43_codec *priv) CS42L43_HSDET_MODE_MASK, 0x2 << CS42L43_HSDET_MODE_SHIFT); cs42l43_stop_hs_bias(priv); + regmap_update_bits(cs42l43->regmap, CS42L43_STEREO_MIC_CLAMP_CTRL, + CS42L43_SMIC_HPAMP_CLAMP_DIS_FRC_VAL_MASK, 0); + if (!time_left) return -ETIMEDOUT; diff --git a/sound/soc/codecs/hda.c b/sound/soc/codecs/hda.c index d57b043d6bfef..42aca0a63c441 100644 --- a/sound/soc/codecs/hda.c +++ b/sound/soc/codecs/hda.c @@ -150,7 +150,7 @@ int hda_codec_probe_complete(struct hda_codec *codec) ret = snd_hda_codec_build_controls(codec); if (ret < 0) { dev_err(&hdev->dev, "unable to create controls %d\n", ret); - goto out; + return ret; } /* Bus suspended codecs as it does not manage their pm */ @@ -158,7 +158,7 @@ int hda_codec_probe_complete(struct hda_codec *codec) /* rpm was forbidden in snd_hda_codec_device_new() */ snd_hda_codec_set_power_save(codec, 2000); snd_hda_codec_register(codec); -out: + /* Complement pm_runtime_get_sync(bus) in probe */ pm_runtime_mark_last_busy(bus->dev); pm_runtime_put_autosuspend(bus->dev); diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c index 8b6b760296948..0ddfb0cb376fd 100644 --- a/sound/soc/codecs/hdac_hdmi.c +++ b/sound/soc/codecs/hdac_hdmi.c @@ -1230,7 +1230,8 @@ static int hdac_hdmi_parse_eld(struct hdac_device *hdev, >> DRM_ELD_VER_SHIFT; if (ver != ELD_VER_CEA_861D && ver != ELD_VER_PARTIAL) { - dev_err(&hdev->dev, "HDMI: Unknown ELD version %d\n", ver); + dev_err_ratelimited(&hdev->dev, + "HDMI: Unknown ELD version %d\n", ver); return -EINVAL; } @@ -1238,7 +1239,8 @@ static int hdac_hdmi_parse_eld(struct hdac_device *hdev, DRM_ELD_MNL_MASK) >> DRM_ELD_MNL_SHIFT; if (mnl > ELD_MAX_MNL) { - dev_err(&hdev->dev, "HDMI: MNL Invalid %d\n", mnl); + dev_err_ratelimited(&hdev->dev, + "HDMI: MNL Invalid %d\n", mnl); return -EINVAL; } @@ -1297,8 +1299,8 @@ static void hdac_hdmi_present_sense(struct hdac_hdmi_pin *pin, if (!port->eld.monitor_present || !port->eld.eld_valid) { - dev_err(&hdev->dev, "%s: disconnect for pin:port %d:%d\n", - __func__, pin->nid, port->id); + dev_dbg(&hdev->dev, "%s: disconnect for pin:port %d:%d\n", + __func__, pin->nid, port->id); /* * PCMs are not registered during device probe, so don't diff --git a/sound/soc/codecs/idt821034.c b/sound/soc/codecs/idt821034.c index 2cc7b9166e695..068a5448e273e 100644 --- a/sound/soc/codecs/idt821034.c +++ b/sound/soc/codecs/idt821034.c @@ -548,14 +548,14 @@ static int idt821034_kctrl_mute_put(struct snd_kcontrol *kcontrol, return ret; } -static const DECLARE_TLV_DB_LINEAR(idt821034_gain_in, -6520, 1306); -#define IDT821034_GAIN_IN_MIN_RAW 1 /* -65.20 dB -> 10^(-65.2/20.0) * 1820 = 1 */ -#define IDT821034_GAIN_IN_MAX_RAW 8191 /* 13.06 dB -> 10^(13.06/20.0) * 1820 = 8191 */ +static const DECLARE_TLV_DB_LINEAR(idt821034_gain_in, -300, 1300); +#define IDT821034_GAIN_IN_MIN_RAW 1288 /* -3.0 dB -> 10^(-3.0/20.0) * 1820 = 1288 */ +#define IDT821034_GAIN_IN_MAX_RAW 8130 /* 13.0 dB -> 10^(13.0/20.0) * 1820 = 8130 */ #define IDT821034_GAIN_IN_INIT_RAW 1820 /* 0dB -> 10^(0/20) * 1820 = 1820 */ -static const DECLARE_TLV_DB_LINEAR(idt821034_gain_out, -6798, 1029); -#define IDT821034_GAIN_OUT_MIN_RAW 1 /* -67.98 dB -> 10^(-67.98/20.0) * 2506 = 1*/ -#define IDT821034_GAIN_OUT_MAX_RAW 8191 /* 10.29 dB -> 10^(10.29/20.0) * 2506 = 8191 */ +static const DECLARE_TLV_DB_LINEAR(idt821034_gain_out, -1300, 300); +#define IDT821034_GAIN_OUT_MIN_RAW 561 /* -13.0 dB -> 10^(-13.0/20.0) * 2506 = 561 */ +#define IDT821034_GAIN_OUT_MAX_RAW 3540 /* 3.0 dB -> 10^(3.0/20.0) * 2506 = 3540 */ #define IDT821034_GAIN_OUT_INIT_RAW 2506 /* 0dB -> 10^(0/20) * 2506 = 2506 */ static const struct snd_kcontrol_new idt821034_controls[] = { diff --git a/sound/soc/codecs/lpass-tx-macro.c b/sound/soc/codecs/lpass-tx-macro.c index ebddfa74ce0a0..150ed10c8377a 100644 --- a/sound/soc/codecs/lpass-tx-macro.c +++ b/sound/soc/codecs/lpass-tx-macro.c @@ -1940,7 +1940,7 @@ static int tx_macro_register_mclk_output(struct tx_macro *tx) } static const struct snd_soc_component_driver tx_macro_component_drv = { - .name = "RX-MACRO", + .name = "TX-MACRO", .probe = tx_macro_component_probe, .controls = tx_macro_snd_controls, .num_controls = ARRAY_SIZE(tx_macro_snd_controls), diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c index b71ef03c4aef0..c7d6696b1bfdf 100644 --- a/sound/soc/codecs/lpass-va-macro.c +++ b/sound/soc/codecs/lpass-va-macro.c @@ -1555,7 +1555,7 @@ static int va_macro_probe(struct platform_device *pdev) if (ret) goto err_clkout; - va->fsgen = clk_hw_get_clk(&va->hw, "fsgen"); + va->fsgen = devm_clk_hw_get_clk(dev, &va->hw, "fsgen"); if (IS_ERR(va->fsgen)) { ret = PTR_ERR(va->fsgen); goto err_clkout; diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c index 2adf744c65263..4023b88e7bc13 100644 --- a/sound/soc/codecs/max98090.c +++ b/sound/soc/codecs/max98090.c @@ -1234,9 +1234,11 @@ static const struct snd_soc_dapm_widget max98091_dapm_widgets[] = { SND_SOC_DAPM_INPUT("DMIC4"), SND_SOC_DAPM_SUPPLY("DMIC3_ENA", M98090_REG_DIGITAL_MIC_ENABLE, - M98090_DIGMIC3_SHIFT, 0, NULL, 0), + M98090_DIGMIC3_SHIFT, 0, max98090_shdn_event, + SND_SOC_DAPM_POST_PMU), SND_SOC_DAPM_SUPPLY("DMIC4_ENA", M98090_REG_DIGITAL_MIC_ENABLE, - M98090_DIGMIC4_SHIFT, 0, NULL, 0), + M98090_DIGMIC4_SHIFT, 0, max98090_shdn_event, + SND_SOC_DAPM_POST_PMU), }; static const struct snd_soc_dapm_route max98090_dapm_routes[] = { diff --git a/sound/soc/codecs/mt6359-accdet.h b/sound/soc/codecs/mt6359-accdet.h index c234f2f4276a1..78ada3a5bfae5 100644 --- a/sound/soc/codecs/mt6359-accdet.h +++ b/sound/soc/codecs/mt6359-accdet.h @@ -123,6 +123,15 @@ struct mt6359_accdet { struct workqueue_struct *jd_workqueue; }; +#if IS_ENABLED(CONFIG_SND_SOC_MT6359_ACCDET) int mt6359_accdet_enable_jack_detect(struct snd_soc_component *component, struct snd_soc_jack *jack); +#else +static inline int +mt6359_accdet_enable_jack_detect(struct snd_soc_component *component, + struct snd_soc_jack *jack) +{ + return -EOPNOTSUPP; +} +#endif #endif diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c index f307374834b55..48ed75c3a7db2 100644 --- a/sound/soc/codecs/nau8821.c +++ b/sound/soc/codecs/nau8821.c @@ -26,7 +26,8 @@ #include #include "nau8821.h" -#define NAU8821_JD_ACTIVE_HIGH BIT(0) +#define NAU8821_QUIRK_JD_ACTIVE_HIGH BIT(0) +#define NAU8821_QUIRK_JD_DB_BYPASS BIT(1) static int nau8821_quirk; static int quirk_override = -1; @@ -1030,12 +1031,17 @@ static bool nau8821_is_jack_inserted(struct regmap *regmap) return active_high == is_high; } -static void nau8821_int_status_clear_all(struct regmap *regmap) +static void nau8821_irq_status_clear(struct regmap *regmap, int active_irq) { - int active_irq, clear_irq, i; + int clear_irq, i; - /* Reset the intrruption status from rightmost bit if the corres- - * ponding irq event occurs. + if (active_irq) { + regmap_write(regmap, NAU8821_R11_INT_CLR_KEY_STATUS, active_irq); + return; + } + + /* Reset the interruption status from rightmost bit if the + * corresponding irq event occurs. */ regmap_read(regmap, NAU8821_R10_IRQ_STATUS, &active_irq); for (i = 0; i < NAU8821_REG_DATA_LEN; i++) { @@ -1062,7 +1068,7 @@ static void nau8821_eject_jack(struct nau8821 *nau8821) snd_soc_dapm_sync(dapm); /* Clear all interruption status */ - nau8821_int_status_clear_all(regmap); + nau8821_irq_status_clear(regmap, 0); /* Enable the insertion interruption, disable the ejection inter- * ruption, and then bypass de-bounce circuit. @@ -1166,9 +1172,10 @@ static void nau8821_setup_inserted_irq(struct nau8821 *nau8821) regmap_update_bits(regmap, NAU8821_R1D_I2S_PCM_CTRL2, NAU8821_I2S_MS_MASK, NAU8821_I2S_MS_SLAVE); - /* Not bypass de-bounce circuit */ - regmap_update_bits(regmap, NAU8821_R0D_JACK_DET_CTRL, - NAU8821_JACK_DET_DB_BYPASS, 0); + /* Do not bypass de-bounce circuit */ + if (!(nau8821_quirk & NAU8821_QUIRK_JD_DB_BYPASS)) + regmap_update_bits(regmap, NAU8821_R0D_JACK_DET_CTRL, + NAU8821_JACK_DET_DB_BYPASS, 0); regmap_update_bits(regmap, NAU8821_R0F_INTERRUPT_MASK, NAU8821_IRQ_EJECT_EN, 0); @@ -1191,6 +1198,7 @@ static irqreturn_t nau8821_interrupt(int irq, void *data) if ((active_irq & NAU8821_JACK_EJECT_IRQ_MASK) == NAU8821_JACK_EJECT_DETECTED) { + cancel_work_sync(&nau8821->jdet_work); regmap_update_bits(regmap, NAU8821_R71_ANALOG_ADC_1, NAU8821_MICDET_MASK, NAU8821_MICDET_DIS); nau8821_eject_jack(nau8821); @@ -1205,11 +1213,11 @@ static irqreturn_t nau8821_interrupt(int irq, void *data) clear_irq = NAU8821_KEY_RELEASE_IRQ; } else if ((active_irq & NAU8821_JACK_INSERT_IRQ_MASK) == NAU8821_JACK_INSERT_DETECTED) { + cancel_work_sync(&nau8821->jdet_work); regmap_update_bits(regmap, NAU8821_R71_ANALOG_ADC_1, NAU8821_MICDET_MASK, NAU8821_MICDET_EN); if (nau8821_is_jack_inserted(regmap)) { /* detect microphone and jack type */ - cancel_work_sync(&nau8821->jdet_work); schedule_work(&nau8821->jdet_work); /* Turn off insertion interruption at manual mode */ regmap_update_bits(regmap, @@ -1527,7 +1535,7 @@ static int nau8821_resume_setup(struct nau8821 *nau8821) nau8821_configure_sysclk(nau8821, NAU8821_CLK_DIS, 0); if (nau8821->irq) { /* Clear all interruption status */ - nau8821_int_status_clear_all(regmap); + nau8821_irq_status_clear(regmap, 0); /* Enable both insertion and ejection interruptions, and then * bypass de-bounce circuit. @@ -1848,7 +1856,23 @@ static const struct dmi_system_id nau8821_quirk_table[] = { DMI_MATCH(DMI_SYS_VENDOR, "Positivo Tecnologia SA"), DMI_MATCH(DMI_BOARD_NAME, "CW14Q01P-V2"), }, - .driver_data = (void *)(NAU8821_JD_ACTIVE_HIGH), + .driver_data = (void *)(NAU8821_QUIRK_JD_ACTIVE_HIGH), + }, + { + /* Valve Steam Deck LCD */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Valve"), + DMI_MATCH(DMI_PRODUCT_NAME, "Jupiter"), + }, + .driver_data = (void *)(NAU8821_QUIRK_JD_DB_BYPASS), + }, + { + /* Valve Steam Deck OLED */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Valve"), + DMI_MATCH(DMI_PRODUCT_NAME, "Galileo"), + }, + .driver_data = (void *)(NAU8821_QUIRK_JD_DB_BYPASS), }, {} }; @@ -1890,9 +1914,12 @@ static int nau8821_i2c_probe(struct i2c_client *i2c) nau8821_check_quirks(); - if (nau8821_quirk & NAU8821_JD_ACTIVE_HIGH) + if (nau8821_quirk & NAU8821_QUIRK_JD_ACTIVE_HIGH) nau8821->jkdet_polarity = 0; + if (nau8821_quirk & NAU8821_QUIRK_JD_DB_BYPASS) + dev_dbg(dev, "Force bypassing jack detection debounce circuit\n"); + nau8821_print_device_properties(nau8821); nau8821_reset_chip(nau8821->regmap); diff --git a/sound/soc/codecs/pcm3168a.c b/sound/soc/codecs/pcm3168a.c index 9d6431338fb71..329549936bd5c 100644 --- a/sound/soc/codecs/pcm3168a.c +++ b/sound/soc/codecs/pcm3168a.c @@ -494,9 +494,9 @@ static int pcm3168a_hw_params(struct snd_pcm_substream *substream, } break; case 24: - if (provider_mode || (format == SND_SOC_DAIFMT_DSP_A) || - (format == SND_SOC_DAIFMT_DSP_B)) { - dev_err(component->dev, "24-bit slots not supported in provider mode, or consumer mode using DSP\n"); + if (!provider_mode && ((format == SND_SOC_DAIFMT_DSP_A) || + (format == SND_SOC_DAIFMT_DSP_B))) { + dev_err(component->dev, "24-bit slots not supported in consumer mode using DSP\n"); return -EINVAL; } break; diff --git a/sound/soc/codecs/rt5640.c b/sound/soc/codecs/rt5640.c index 1955d77cffd99..0f250e8e216a4 100644 --- a/sound/soc/codecs/rt5640.c +++ b/sound/soc/codecs/rt5640.c @@ -3016,6 +3016,11 @@ static int rt5640_i2c_probe(struct i2c_client *i2c) } regmap_read(rt5640->regmap, RT5640_VENDOR_ID2, &val); + if (val != RT5640_DEVICE_ID) { + usleep_range(60000, 100000); + regmap_read(rt5640->regmap, RT5640_VENDOR_ID2, &val); + } + if (val != RT5640_DEVICE_ID) { dev_err(&i2c->dev, "Device with ID register %#x is not rt5640/39\n", val); diff --git a/sound/soc/codecs/rt5682s.c b/sound/soc/codecs/rt5682s.c index 92c647d439ec7..89de427c19e79 100644 --- a/sound/soc/codecs/rt5682s.c +++ b/sound/soc/codecs/rt5682s.c @@ -653,14 +653,15 @@ static void rt5682s_sar_power_mode(struct snd_soc_component *component, int mode switch (mode) { case SAR_PWR_SAVING: snd_soc_component_update_bits(component, RT5682S_CBJ_CTRL_3, - RT5682S_CBJ_IN_BUF_MASK, RT5682S_CBJ_IN_BUF_DIS); + RT5682S_CBJ_IN_BUF_MASK, RT5682S_CBJ_IN_BUF_EN); snd_soc_component_update_bits(component, RT5682S_CBJ_CTRL_1, - RT5682S_MB1_PATH_MASK | RT5682S_MB2_PATH_MASK, - RT5682S_CTRL_MB1_REG | RT5682S_CTRL_MB2_REG); + RT5682S_MB1_PATH_MASK | RT5682S_MB2_PATH_MASK | + RT5682S_VREF_POW_MASK, RT5682S_CTRL_MB1_FSM | + RT5682S_CTRL_MB2_FSM | RT5682S_VREF_POW_FSM); snd_soc_component_update_bits(component, RT5682S_SAR_IL_CMD_1, RT5682S_SAR_BUTDET_MASK | RT5682S_SAR_BUTDET_POW_MASK | RT5682S_SAR_SEL_MB1_2_CTL_MASK, RT5682S_SAR_BUTDET_DIS | - RT5682S_SAR_BUTDET_POW_SAV | RT5682S_SAR_SEL_MB1_2_MANU); + RT5682S_SAR_BUTDET_POW_NORM | RT5682S_SAR_SEL_MB1_2_MANU); usleep_range(5000, 5500); snd_soc_component_update_bits(component, RT5682S_SAR_IL_CMD_1, RT5682S_SAR_BUTDET_MASK, RT5682S_SAR_BUTDET_EN); @@ -688,7 +689,7 @@ static void rt5682s_sar_power_mode(struct snd_soc_component *component, int mode snd_soc_component_update_bits(component, RT5682S_SAR_IL_CMD_1, RT5682S_SAR_BUTDET_MASK | RT5682S_SAR_BUTDET_POW_MASK | RT5682S_SAR_SEL_MB1_2_CTL_MASK, RT5682S_SAR_BUTDET_DIS | - RT5682S_SAR_BUTDET_POW_SAV | RT5682S_SAR_SEL_MB1_2_MANU); + RT5682S_SAR_BUTDET_POW_NORM | RT5682S_SAR_SEL_MB1_2_MANU); break; default: dev_err(component->dev, "Invalid SAR Power mode: %d\n", mode); @@ -725,7 +726,7 @@ static void rt5682s_disable_push_button_irq(struct snd_soc_component *component) snd_soc_component_update_bits(component, RT5682S_SAR_IL_CMD_1, RT5682S_SAR_BUTDET_MASK | RT5682S_SAR_BUTDET_POW_MASK | RT5682S_SAR_SEL_MB1_2_CTL_MASK, RT5682S_SAR_BUTDET_DIS | - RT5682S_SAR_BUTDET_POW_SAV | RT5682S_SAR_SEL_MB1_2_MANU); + RT5682S_SAR_BUTDET_POW_NORM | RT5682S_SAR_SEL_MB1_2_MANU); } /** @@ -786,7 +787,7 @@ static int rt5682s_headset_detect(struct snd_soc_component *component, int jack_ jack_type = SND_JACK_HEADSET; snd_soc_component_write(component, RT5682S_SAR_IL_CMD_3, 0x024c); snd_soc_component_update_bits(component, RT5682S_CBJ_CTRL_1, - RT5682S_FAST_OFF_MASK, RT5682S_FAST_OFF_EN); + RT5682S_FAST_OFF_MASK, RT5682S_FAST_OFF_DIS); snd_soc_component_update_bits(component, RT5682S_SAR_IL_CMD_1, RT5682S_SAR_SEL_MB1_2_MASK, val << RT5682S_SAR_SEL_MB1_2_SFT); rt5682s_enable_push_button_irq(component); @@ -966,7 +967,7 @@ static int rt5682s_set_jack_detect(struct snd_soc_component *component, RT5682S_EMB_JD_MASK | RT5682S_DET_TYPE | RT5682S_POL_FAST_OFF_MASK | RT5682S_MIC_CAP_MASK, RT5682S_EMB_JD_EN | RT5682S_DET_TYPE | - RT5682S_POL_FAST_OFF_HIGH | RT5682S_MIC_CAP_HS); + RT5682S_POL_FAST_OFF_LOW | RT5682S_MIC_CAP_HS); regmap_update_bits(rt5682s->regmap, RT5682S_SAR_IL_CMD_1, RT5682S_SAR_POW_MASK, RT5682S_SAR_POW_EN); regmap_update_bits(rt5682s->regmap, RT5682S_GPIO_CTRL_1, diff --git a/sound/soc/codecs/rt722-sdca-sdw.c b/sound/soc/codecs/rt722-sdca-sdw.c index c382cb6be6025..c0bb1b4b2dcb5 100644 --- a/sound/soc/codecs/rt722-sdca-sdw.c +++ b/sound/soc/codecs/rt722-sdca-sdw.c @@ -28,9 +28,50 @@ static bool rt722_sdca_readable_register(struct device *dev, unsigned int reg) 0): case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_GE49, RT722_SDCA_CTL_DETECTED_MODE, 0): - case SDW_SDCA_CTL(FUNC_NUM_HID, RT722_SDCA_ENT_HID01, RT722_SDCA_CTL_HIDTX_CURRENT_OWNER, - 0) ... SDW_SDCA_CTL(FUNC_NUM_HID, RT722_SDCA_ENT_HID01, - RT722_SDCA_CTL_HIDTX_MESSAGE_LENGTH, 0): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_XU03, RT722_SDCA_CTL_SELECTED_MODE, + 0): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_USER_FU05, + RT722_SDCA_CTL_FU_MUTE, CH_L) ... + SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_USER_FU05, + RT722_SDCA_CTL_FU_MUTE, CH_R): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_XU0D, + RT722_SDCA_CTL_SELECTED_MODE, 0): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_USER_FU0F, + RT722_SDCA_CTL_FU_MUTE, CH_L) ... + SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_USER_FU0F, + RT722_SDCA_CTL_FU_MUTE, CH_R): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_PDE40, + RT722_SDCA_CTL_REQ_POWER_STATE, 0): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_PDE12, + RT722_SDCA_CTL_REQ_POWER_STATE, 0): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_CS01, + RT722_SDCA_CTL_SAMPLE_FREQ_INDEX, 0): + case SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT722_SDCA_ENT_CS11, + RT722_SDCA_CTL_SAMPLE_FREQ_INDEX, 0): + case SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT722_SDCA_ENT_USER_FU1E, + RT722_SDCA_CTL_FU_MUTE, CH_01) ... + SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT722_SDCA_ENT_USER_FU1E, + RT722_SDCA_CTL_FU_MUTE, CH_04): + case SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT722_SDCA_ENT_IT26, + RT722_SDCA_CTL_VENDOR_DEF, 0): + case SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT722_SDCA_ENT_PDE2A, + RT722_SDCA_CTL_REQ_POWER_STATE, 0): + case SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT722_SDCA_ENT_CS1F, + RT722_SDCA_CTL_SAMPLE_FREQ_INDEX, 0): + case SDW_SDCA_CTL(FUNC_NUM_HID, RT722_SDCA_ENT_HID01, + RT722_SDCA_CTL_HIDTX_CURRENT_OWNER, 0) ... + SDW_SDCA_CTL(FUNC_NUM_HID, RT722_SDCA_ENT_HID01, + RT722_SDCA_CTL_HIDTX_MESSAGE_LENGTH, 0): + case SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06, + RT722_SDCA_CTL_FU_MUTE, CH_L) ... + SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06, + RT722_SDCA_CTL_FU_MUTE, CH_R): + case SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_OT23, + RT722_SDCA_CTL_VENDOR_DEF, CH_08): + case SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_PDE23, + RT722_SDCA_CTL_REQ_POWER_STATE, 0): + case SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_CS31, + RT722_SDCA_CTL_SAMPLE_FREQ_INDEX, 0): case RT722_BUF_ADDR_HID1 ... RT722_BUF_ADDR_HID2: return true; default: @@ -74,6 +115,7 @@ static bool rt722_sdca_mbq_readable_register(struct device *dev, unsigned int re case 0x5600000 ... 0x5600007: case 0x5700000 ... 0x5700004: case 0x5800000 ... 0x5800004: + case 0x5810000: case 0x5b00003: case 0x5c00011: case 0x5d00006: @@ -81,6 +123,7 @@ static bool rt722_sdca_mbq_readable_register(struct device *dev, unsigned int re case 0x5f00030: case 0x6100000 ... 0x6100051: case 0x6100055 ... 0x6100057: + case 0x6100060: case 0x6100062: case 0x6100064 ... 0x6100065: case 0x6100067: diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index e87a07eee9737..054c6f860675a 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -182,33 +182,6 @@ static SOC_ENUM_SINGLE_DECL( static const struct snd_kcontrol_new tas2764_asi1_mux = SOC_DAPM_ENUM("ASI1 Source", tas2764_ASI1_src_enum); -static int tas2764_dac_event(struct snd_soc_dapm_widget *w, - struct snd_kcontrol *kcontrol, int event) -{ - struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); - struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); - int ret; - - switch (event) { - case SND_SOC_DAPM_POST_PMU: - tas2764->dac_powered = true; - ret = tas2764_update_pwr_ctrl(tas2764); - break; - case SND_SOC_DAPM_PRE_PMD: - tas2764->dac_powered = false; - ret = tas2764_update_pwr_ctrl(tas2764); - break; - default: - dev_err(tas2764->dev, "Unsupported event\n"); - return -EINVAL; - } - - if (ret < 0) - return ret; - - return 0; -} - static const struct snd_kcontrol_new isense_switch = SOC_DAPM_SINGLE("Switch", TAS2764_PWR_CTRL, TAS2764_ISENSE_POWER_EN, 1, 1); static const struct snd_kcontrol_new vsense_switch = @@ -221,8 +194,7 @@ static const struct snd_soc_dapm_widget tas2764_dapm_widgets[] = { 1, &isense_switch), SND_SOC_DAPM_SWITCH("VSENSE", TAS2764_PWR_CTRL, TAS2764_VSENSE_POWER_EN, 1, &vsense_switch), - SND_SOC_DAPM_DAC_E("DAC", NULL, SND_SOC_NOPM, 0, 0, tas2764_dac_event, - SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD), + SND_SOC_DAPM_DAC("DAC", NULL, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_OUTPUT("OUT"), SND_SOC_DAPM_SIGGEN("VMON"), SND_SOC_DAPM_SIGGEN("IMON") @@ -243,9 +215,28 @@ static int tas2764_mute(struct snd_soc_dai *dai, int mute, int direction) { struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(dai->component); + int ret; + + if (!mute) { + tas2764->dac_powered = true; + ret = tas2764_update_pwr_ctrl(tas2764); + if (ret) + return ret; + } tas2764->unmuted = !mute; - return tas2764_update_pwr_ctrl(tas2764); + ret = tas2764_update_pwr_ctrl(tas2764); + if (ret) + return ret; + + if (mute) { + tas2764->dac_powered = false; + ret = tas2764_update_pwr_ctrl(tas2764); + if (ret) + return ret; + } + + return 0; } static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) @@ -551,7 +542,7 @@ static int tas2764_codec_probe(struct snd_soc_component *component) tas2764_reset(tas2764); if (tas2764->irq) { - ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK0, 0xff); + ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK0, 0x00); if (ret < 0) return ret; @@ -636,6 +627,7 @@ static const struct reg_default tas2764_reg_defaults[] = { { TAS2764_TDM_CFG2, 0x0a }, { TAS2764_TDM_CFG3, 0x10 }, { TAS2764_TDM_CFG5, 0x42 }, + { TAS2764_INT_CLK_CFG, 0x19 }, }; static const struct regmap_range_cfg tas2764_regmap_ranges[] = { @@ -653,6 +645,7 @@ static const struct regmap_range_cfg tas2764_regmap_ranges[] = { static bool tas2764_volatile_register(struct device *dev, unsigned int reg) { switch (reg) { + case TAS2764_SW_RST: case TAS2764_INT_LTCH0 ... TAS2764_INT_LTCH4: case TAS2764_INT_CLK_CFG: return true; diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c index 5c6b825c757b3..181b16530e5bc 100644 --- a/sound/soc/codecs/tas2770.c +++ b/sound/soc/codecs/tas2770.c @@ -158,11 +158,37 @@ static const struct snd_kcontrol_new isense_switch = static const struct snd_kcontrol_new vsense_switch = SOC_DAPM_SINGLE("Switch", TAS2770_PWR_CTRL, 2, 1, 1); +static int sense_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *kcontrol, int event) +{ + struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); + struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); + + /* + * Powering up ISENSE/VSENSE requires a trip through the shutdown state. + * Do that here to ensure that our changes are applied properly, otherwise + * we might end up with non-functional IVSENSE if playback started earlier, + * which would break software speaker protection. + */ + switch (event) { + case SND_SOC_DAPM_PRE_REG: + return snd_soc_component_update_bits(component, TAS2770_PWR_CTRL, + TAS2770_PWR_CTRL_MASK, + TAS2770_PWR_CTRL_SHUTDOWN); + case SND_SOC_DAPM_POST_REG: + return tas2770_update_pwr_ctrl(tas2770); + default: + return 0; + } +} + static const struct snd_soc_dapm_widget tas2770_dapm_widgets[] = { SND_SOC_DAPM_AIF_IN("ASI1", "ASI1 Playback", 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_MUX("ASI1 Sel", SND_SOC_NOPM, 0, 0, &tas2770_asi1_mux), - SND_SOC_DAPM_SWITCH("ISENSE", TAS2770_PWR_CTRL, 3, 1, &isense_switch), - SND_SOC_DAPM_SWITCH("VSENSE", TAS2770_PWR_CTRL, 2, 1, &vsense_switch), + SND_SOC_DAPM_SWITCH_E("ISENSE", TAS2770_PWR_CTRL, 3, 1, &isense_switch, + sense_event, SND_SOC_DAPM_PRE_REG | SND_SOC_DAPM_POST_REG), + SND_SOC_DAPM_SWITCH_E("VSENSE", TAS2770_PWR_CTRL, 2, 1, &vsense_switch, + sense_event, SND_SOC_DAPM_PRE_REG | SND_SOC_DAPM_POST_REG), SND_SOC_DAPM_DAC_E("DAC", NULL, SND_SOC_NOPM, 0, 0, tas2770_dac_event, SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD), SND_SOC_DAPM_OUTPUT("OUT"), diff --git a/sound/soc/codecs/tas2781-i2c.c b/sound/soc/codecs/tas2781-i2c.c index 43775c1944452..836cf06a45266 100644 --- a/sound/soc/codecs/tas2781-i2c.c +++ b/sound/soc/codecs/tas2781-i2c.c @@ -616,7 +616,8 @@ static void tasdevice_parse_dt(struct tasdevice_priv *tas_priv) { struct i2c_client *client = (struct i2c_client *)tas_priv->client; unsigned int dev_addrs[TASDEVICE_MAX_CHANNELS]; - int i, ndev = 0; + int ndev = 0; + int i, rc; if (tas_priv->isacpi) { ndev = device_property_read_u32_array(&client->dev, @@ -627,8 +628,12 @@ static void tasdevice_parse_dt(struct tasdevice_priv *tas_priv) } else { ndev = (ndev < ARRAY_SIZE(dev_addrs)) ? ndev : ARRAY_SIZE(dev_addrs); - ndev = device_property_read_u32_array(&client->dev, + rc = device_property_read_u32_array(&client->dev, "ti,audio-slots", dev_addrs, ndev); + if (rc != 0) { + ndev = 1; + dev_addrs[0] = client->addr; + } } tas_priv->irq = diff --git a/sound/soc/codecs/tlv320aic3x.c b/sound/soc/codecs/tlv320aic3x.c index 56e795a00e22f..591f6c9f9d3a6 100644 --- a/sound/soc/codecs/tlv320aic3x.c +++ b/sound/soc/codecs/tlv320aic3x.c @@ -121,6 +121,16 @@ static const struct reg_default aic3x_reg[] = { { 108, 0x00 }, { 109, 0x00 }, }; +static const struct reg_sequence aic3007_class_d[] = { + /* Class-D speaker driver init; datasheet p. 46 */ + { AIC3X_PAGE_SELECT, 0x0D }, + { 0xD, 0x0D }, + { 0x8, 0x5C }, + { 0x8, 0x5D }, + { 0x8, 0x5C }, + { AIC3X_PAGE_SELECT, 0x00 }, +}; + static bool aic3x_volatile_reg(struct device *dev, unsigned int reg) { switch (reg) { @@ -1393,6 +1403,10 @@ static int aic3x_set_power(struct snd_soc_component *component, int power) gpiod_set_value(aic3x->gpio_reset, 0); } + if (aic3x->model == AIC3X_MODEL_3007) + regmap_multi_reg_write_bypassed(aic3x->regmap, aic3007_class_d, + ARRAY_SIZE(aic3007_class_d)); + /* Sync reg_cache with the hardware */ regcache_cache_only(aic3x->regmap, false); regcache_sync(aic3x->regmap); @@ -1723,17 +1737,6 @@ static void aic3x_configure_ocmv(struct device *dev, struct aic3x_priv *aic3x) } } - -static const struct reg_sequence aic3007_class_d[] = { - /* Class-D speaker driver init; datasheet p. 46 */ - { AIC3X_PAGE_SELECT, 0x0D }, - { 0xD, 0x0D }, - { 0x8, 0x5C }, - { 0x8, 0x5D }, - { 0x8, 0x5C }, - { AIC3X_PAGE_SELECT, 0x00 }, -}; - int aic3x_probe(struct device *dev, struct regmap *regmap, kernel_ulong_t driver_data) { struct aic3x_priv *aic3x; @@ -1825,13 +1828,6 @@ int aic3x_probe(struct device *dev, struct regmap *regmap, kernel_ulong_t driver aic3x_configure_ocmv(dev, aic3x); - if (aic3x->model == AIC3X_MODEL_3007) { - ret = regmap_register_patch(aic3x->regmap, aic3007_class_d, - ARRAY_SIZE(aic3007_class_d)); - if (ret != 0) - dev_err(dev, "Failed to init class D: %d\n", ret); - } - ret = devm_snd_soc_register_component(dev, &soc_component_dev_aic3x, &aic3x_dai, 1); if (ret) return ret; diff --git a/sound/soc/codecs/wcd9335.c b/sound/soc/codecs/wcd9335.c index a05b553e6472f..8d5d186fd5802 100644 --- a/sound/soc/codecs/wcd9335.c +++ b/sound/soc/codecs/wcd9335.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -329,8 +329,7 @@ struct wcd9335_codec { int comp_enabled[COMPANDER_MAX]; int intr1; - int reset_gpio; - struct regulator_bulk_data supplies[WCD9335_MAX_SUPPLY]; + struct gpio_desc *reset_gpio; unsigned int rx_port_value[WCD9335_RX_MAX]; unsigned int tx_port_value[WCD9335_TX_MAX]; @@ -357,6 +356,10 @@ struct wcd9335_irq { char *name; }; +static const char * const wcd9335_supplies[] = { + "vdd-buck", "vdd-buck-sido", "vdd-tx", "vdd-rx", "vdd-io", +}; + static const struct wcd9335_slim_ch wcd9335_tx_chs[WCD9335_TX_MAX] = { WCD9335_SLIM_TX_CH(0), WCD9335_SLIM_TX_CH(1), @@ -5032,53 +5035,30 @@ static const struct regmap_irq_chip wcd9335_regmap_irq1_chip = { static int wcd9335_parse_dt(struct wcd9335_codec *wcd) { struct device *dev = wcd->dev; - struct device_node *np = dev->of_node; int ret; - wcd->reset_gpio = of_get_named_gpio(np, "reset-gpios", 0); - if (wcd->reset_gpio < 0) { - dev_err(dev, "Reset GPIO missing from DT\n"); - return wcd->reset_gpio; - } + wcd->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(wcd->reset_gpio)) + return dev_err_probe(dev, PTR_ERR(wcd->reset_gpio), "Reset GPIO missing from DT\n"); wcd->mclk = devm_clk_get(dev, "mclk"); - if (IS_ERR(wcd->mclk)) { - dev_err(dev, "mclk not found\n"); - return PTR_ERR(wcd->mclk); - } + if (IS_ERR(wcd->mclk)) + return dev_err_probe(dev, PTR_ERR(wcd->mclk), "mclk not found\n"); wcd->native_clk = devm_clk_get(dev, "slimbus"); - if (IS_ERR(wcd->native_clk)) { - dev_err(dev, "slimbus clock not found\n"); - return PTR_ERR(wcd->native_clk); - } + if (IS_ERR(wcd->native_clk)) + return dev_err_probe(dev, PTR_ERR(wcd->native_clk), "slimbus clock not found\n"); - wcd->supplies[0].supply = "vdd-buck"; - wcd->supplies[1].supply = "vdd-buck-sido"; - wcd->supplies[2].supply = "vdd-tx"; - wcd->supplies[3].supply = "vdd-rx"; - wcd->supplies[4].supply = "vdd-io"; - - ret = regulator_bulk_get(dev, WCD9335_MAX_SUPPLY, wcd->supplies); - if (ret) { - dev_err(dev, "Failed to get supplies: err = %d\n", ret); - return ret; - } + ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(wcd9335_supplies), + wcd9335_supplies); + if (ret) + return dev_err_probe(dev, ret, "Failed to get and enable supplies\n"); return 0; } static int wcd9335_power_on_reset(struct wcd9335_codec *wcd) { - struct device *dev = wcd->dev; - int ret; - - ret = regulator_bulk_enable(WCD9335_MAX_SUPPLY, wcd->supplies); - if (ret) { - dev_err(dev, "Failed to get supplies: err = %d\n", ret); - return ret; - } - /* * For WCD9335, it takes about 600us for the Vout_A and * Vout_D to be ready after BUCK_SIDO is powered up. @@ -5088,9 +5068,9 @@ static int wcd9335_power_on_reset(struct wcd9335_codec *wcd) */ usleep_range(600, 650); - gpio_direction_output(wcd->reset_gpio, 0); + gpiod_set_value(wcd->reset_gpio, 1); msleep(20); - gpio_set_value(wcd->reset_gpio, 1); + gpiod_set_value(wcd->reset_gpio, 0); msleep(20); return 0; @@ -5163,10 +5143,8 @@ static int wcd9335_slim_probe(struct slim_device *slim) wcd->dev = dev; ret = wcd9335_parse_dt(wcd); - if (ret) { - dev_err(dev, "Error parsing DT: %d\n", ret); + if (ret) return ret; - } ret = wcd9335_power_on_reset(wcd); if (ret) diff --git a/sound/soc/codecs/wcd934x.c b/sound/soc/codecs/wcd934x.c index fe222c4b74c00..c04be088f17a2 100644 --- a/sound/soc/codecs/wcd934x.c +++ b/sound/soc/codecs/wcd934x.c @@ -5858,6 +5858,13 @@ static const struct snd_soc_component_driver wcd934x_component_drv = { .endianness = 1, }; +static void wcd934x_put_device_action(void *data) +{ + struct device *dev = data; + + put_device(dev); +} + static int wcd934x_codec_parse_data(struct wcd934x_codec *wcd) { struct device *dev = &wcd->sdev->dev; @@ -5878,11 +5885,13 @@ static int wcd934x_codec_parse_data(struct wcd934x_codec *wcd) } slim_get_logical_addr(wcd->sidev); - wcd->if_regmap = regmap_init_slimbus(wcd->sidev, + wcd->if_regmap = devm_regmap_init_slimbus(wcd->sidev, &wcd934x_ifc_regmap_config); - if (IS_ERR(wcd->if_regmap)) + if (IS_ERR(wcd->if_regmap)) { + put_device(&wcd->sidev->dev); return dev_err_probe(dev, PTR_ERR(wcd->if_regmap), "Failed to allocate ifc register map\n"); + } of_property_read_u32(dev->parent->of_node, "qcom,dmic-sample-rate", &wcd->dmic_sample_rate); @@ -5926,6 +5935,10 @@ static int wcd934x_codec_probe(struct platform_device *pdev) return ret; } + ret = devm_add_action_or_reset(dev, wcd934x_put_device_action, &wcd->sidev->dev); + if (ret) + return ret; + /* set default rate 9P6MHz */ regmap_update_bits(wcd->regmap, WCD934X_CODEC_RPM_CLK_MCLK_CFG, WCD934X_CODEC_RPM_CLK_MCLK_CFG_MCLK_MASK, diff --git a/sound/soc/codecs/wm8940.c b/sound/soc/codecs/wm8940.c index b9432f8b64e5b..39d2c8e85d9db 100644 --- a/sound/soc/codecs/wm8940.c +++ b/sound/soc/codecs/wm8940.c @@ -220,7 +220,7 @@ static const struct snd_kcontrol_new wm8940_snd_controls[] = { SOC_SINGLE_TLV("Digital Capture Volume", WM8940_ADCVOL, 0, 255, 0, wm8940_adc_tlv), SOC_ENUM("Mic Bias Level", wm8940_mic_bias_level_enum), - SOC_SINGLE_TLV("Capture Boost Volue", WM8940_ADCBOOST, + SOC_SINGLE_TLV("Capture Boost Volume", WM8940_ADCBOOST, 8, 1, 0, wm8940_capture_boost_vol_tlv), SOC_SINGLE_TLV("Speaker Playback Volume", WM8940_SPKVOL, 0, 63, 0, wm8940_spk_vol_tlv), @@ -693,7 +693,12 @@ static int wm8940_update_clocks(struct snd_soc_dai *dai) f = wm8940_get_mclkdiv(priv->mclk, fs256, &mclkdiv); if (f != priv->mclk) { /* The PLL performs best around 90MHz */ - fpll = wm8940_get_mclkdiv(22500000, fs256, &mclkdiv); + if (fs256 % 8000) + f = 22579200; + else + f = 24576000; + + fpll = wm8940_get_mclkdiv(f, fs256, &mclkdiv); } wm8940_set_dai_pll(dai, 0, 0, priv->mclk, fpll); diff --git a/sound/soc/codecs/wm8974.c b/sound/soc/codecs/wm8974.c index 260bac695b20a..2aaa1cbe68b71 100644 --- a/sound/soc/codecs/wm8974.c +++ b/sound/soc/codecs/wm8974.c @@ -419,10 +419,14 @@ static int wm8974_update_clocks(struct snd_soc_dai *dai) fs256 = 256 * priv->fs; f = wm8974_get_mclkdiv(priv->mclk, fs256, &mclkdiv); - if (f != priv->mclk) { /* The PLL performs best around 90MHz */ - fpll = wm8974_get_mclkdiv(22500000, fs256, &mclkdiv); + if (fs256 % 8000) + f = 22579200; + else + f = 24576000; + + fpll = wm8974_get_mclkdiv(f, fs256, &mclkdiv); } wm8974_set_dai_pll(dai, 0, 0, priv->mclk, fpll); diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c index b793263291dc8..72dc23c994bcd 100644 --- a/sound/soc/fsl/fsl_asrc.c +++ b/sound/soc/fsl/fsl_asrc.c @@ -517,7 +517,8 @@ static int fsl_asrc_config_pair(struct fsl_asrc_pair *pair, bool use_ideal_rate) regmap_update_bits(asrc->regmap, REG_ASRCTR, ASRCTR_ATSi_MASK(index), ASRCTR_ATS(index)); regmap_update_bits(asrc->regmap, REG_ASRCTR, - ASRCTR_USRi_MASK(index), 0); + ASRCTR_IDRi_MASK(index) | ASRCTR_USRi_MASK(index), + ASRCTR_USR(index)); /* Set the input and output clock sources */ regmap_update_bits(asrc->regmap, REG_ASRCSR, diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c index aa15f56ca139d..0de878d64a3bd 100644 --- a/sound/soc/fsl/fsl_sai.c +++ b/sound/soc/fsl/fsl_sai.c @@ -322,7 +322,6 @@ static int fsl_sai_set_dai_fmt_tr(struct snd_soc_dai *cpu_dai, break; case SND_SOC_DAIFMT_PDM: val_cr2 |= FSL_SAI_CR2_BCP; - val_cr4 &= ~FSL_SAI_CR4_MF; sai->is_pdm_mode = true; break; case SND_SOC_DAIFMT_RIGHT_J: @@ -597,7 +596,7 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream, val_cr5 |= FSL_SAI_CR5_WNW(slot_width); val_cr5 |= FSL_SAI_CR5_W0W(slot_width); - if (sai->is_lsb_first || sai->is_pdm_mode) + if (sai->is_lsb_first) val_cr5 |= FSL_SAI_CR5_FBT(0); else val_cr5 |= FSL_SAI_CR5_FBT(word_width - 1); @@ -762,13 +761,15 @@ static void fsl_sai_config_disable(struct fsl_sai *sai, int dir) * anymore. Add software reset to fix this issue. * This is a hardware bug, and will be fix in the * next sai version. + * + * In consumer mode, this can happen even after a + * single open/close, especially if both tx and rx + * are running concurrently. */ - if (!sai->is_consumer_mode) { - /* Software Reset */ - regmap_write(sai->regmap, FSL_SAI_xCSR(tx, ofs), FSL_SAI_CSR_SR); - /* Clear SR bit to finish the reset */ - regmap_write(sai->regmap, FSL_SAI_xCSR(tx, ofs), 0); - } + /* Software Reset */ + regmap_update_bits(sai->regmap, FSL_SAI_xCSR(tx, ofs), FSL_SAI_CSR_SR, FSL_SAI_CSR_SR); + /* Clear SR bit to finish the reset */ + regmap_update_bits(sai->regmap, FSL_SAI_xCSR(tx, ofs), FSL_SAI_CSR_SR, 0); } static int fsl_sai_trigger(struct snd_pcm_substream *substream, int cmd, @@ -887,11 +888,11 @@ static int fsl_sai_dai_probe(struct snd_soc_dai *cpu_dai) unsigned int ofs = sai->soc_data->reg_offset; /* Software Reset for both Tx and Rx */ - regmap_write(sai->regmap, FSL_SAI_TCSR(ofs), FSL_SAI_CSR_SR); - regmap_write(sai->regmap, FSL_SAI_RCSR(ofs), FSL_SAI_CSR_SR); + regmap_update_bits(sai->regmap, FSL_SAI_TCSR(ofs), FSL_SAI_CSR_SR, FSL_SAI_CSR_SR); + regmap_update_bits(sai->regmap, FSL_SAI_RCSR(ofs), FSL_SAI_CSR_SR, FSL_SAI_CSR_SR); /* Clear SR bit to finish the reset */ - regmap_write(sai->regmap, FSL_SAI_TCSR(ofs), 0); - regmap_write(sai->regmap, FSL_SAI_RCSR(ofs), 0); + regmap_update_bits(sai->regmap, FSL_SAI_TCSR(ofs), FSL_SAI_CSR_SR, 0); + regmap_update_bits(sai->regmap, FSL_SAI_RCSR(ofs), FSL_SAI_CSR_SR, 0); regmap_update_bits(sai->regmap, FSL_SAI_TCR1(ofs), FSL_SAI_CR1_RFW_MASK(sai->soc_data->fifo_depth), @@ -1708,11 +1709,11 @@ static int fsl_sai_runtime_resume(struct device *dev) regcache_cache_only(sai->regmap, false); regcache_mark_dirty(sai->regmap); - regmap_write(sai->regmap, FSL_SAI_TCSR(ofs), FSL_SAI_CSR_SR); - regmap_write(sai->regmap, FSL_SAI_RCSR(ofs), FSL_SAI_CSR_SR); + regmap_update_bits(sai->regmap, FSL_SAI_TCSR(ofs), FSL_SAI_CSR_SR, FSL_SAI_CSR_SR); + regmap_update_bits(sai->regmap, FSL_SAI_RCSR(ofs), FSL_SAI_CSR_SR, FSL_SAI_CSR_SR); usleep_range(1000, 2000); - regmap_write(sai->regmap, FSL_SAI_TCSR(ofs), 0); - regmap_write(sai->regmap, FSL_SAI_RCSR(ofs), 0); + regmap_update_bits(sai->regmap, FSL_SAI_TCSR(ofs), FSL_SAI_CSR_SR, 0); + regmap_update_bits(sai->regmap, FSL_SAI_RCSR(ofs), FSL_SAI_CSR_SR, 0); ret = regcache_sync(sai->regmap); if (ret) diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c index c46f64557a7ff..1d7791c7fb4ec 100644 --- a/sound/soc/fsl/fsl_xcvr.c +++ b/sound/soc/fsl/fsl_xcvr.c @@ -1197,6 +1197,26 @@ static irqreturn_t irq0_isr(int irq, void *devid) /* clear CS control register */ memset_io(reg_ctrl, 0, sizeof(val)); } + } else { + regmap_read(xcvr->regmap, FSL_XCVR_RX_CS_DATA_0, + (u32 *)&xcvr->rx_iec958.status[0]); + regmap_read(xcvr->regmap, FSL_XCVR_RX_CS_DATA_1, + (u32 *)&xcvr->rx_iec958.status[4]); + regmap_read(xcvr->regmap, FSL_XCVR_RX_CS_DATA_2, + (u32 *)&xcvr->rx_iec958.status[8]); + regmap_read(xcvr->regmap, FSL_XCVR_RX_CS_DATA_3, + (u32 *)&xcvr->rx_iec958.status[12]); + regmap_read(xcvr->regmap, FSL_XCVR_RX_CS_DATA_4, + (u32 *)&xcvr->rx_iec958.status[16]); + regmap_read(xcvr->regmap, FSL_XCVR_RX_CS_DATA_5, + (u32 *)&xcvr->rx_iec958.status[20]); + for (i = 0; i < 6; i++) { + val = *(u32 *)(xcvr->rx_iec958.status + i * 4); + *(u32 *)(xcvr->rx_iec958.status + i * 4) = + bitrev32(val); + } + regmap_set_bits(xcvr->regmap, FSL_XCVR_RX_DPTH_CTRL, + FSL_XCVR_RX_DPTH_CTRL_CSA); } } if (isr & FSL_XCVR_IRQ_NEW_UD) { diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c index 7128bcf3a743e..bb304de5cc38a 100644 --- a/sound/soc/fsl/imx-card.c +++ b/sound/soc/fsl/imx-card.c @@ -517,7 +517,7 @@ static int imx_card_parse_of(struct imx_card_data *data) if (!card->dai_link) return -ENOMEM; - data->link_data = devm_kcalloc(dev, num_links, sizeof(*link), GFP_KERNEL); + data->link_data = devm_kcalloc(dev, num_links, sizeof(*link_data), GFP_KERNEL); if (!data->link_data) return -ENOMEM; diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c index 63e4356e8caf9..8f36cef88fe60 100644 --- a/sound/soc/intel/avs/core.c +++ b/sound/soc/intel/avs/core.c @@ -415,6 +415,8 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) adev = devm_kzalloc(dev, sizeof(*adev), GFP_KERNEL); if (!adev) return -ENOMEM; + bus = &adev->base.core; + ret = avs_bus_init(adev, pci, id); if (ret < 0) { dev_err(dev, "failed to init avs bus: %d\n", ret); @@ -425,7 +427,6 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) if (ret < 0) return ret; - bus = &adev->base.core; bus->addr = pci_resource_start(pci, 0); bus->remap_addr = pci_ioremap_bar(pci, 0); if (!bus->remap_addr) { diff --git a/sound/soc/intel/avs/debugfs.c b/sound/soc/intel/avs/debugfs.c index bdd388ec01eaf..26d0c3a5a9542 100644 --- a/sound/soc/intel/avs/debugfs.c +++ b/sound/soc/intel/avs/debugfs.c @@ -371,7 +371,10 @@ static ssize_t trace_control_write(struct file *file, const char __user *from, s return ret; num_elems = *array; - resource_mask = array[1]; + if (!num_elems) { + ret = -EINVAL; + goto free_array; + } /* * Disable if just resource mask is provided - no log priority flags. @@ -379,6 +382,7 @@ static ssize_t trace_control_write(struct file *file, const char __user *from, s * Enable input format: mask, prio1, .., prioN * Where 'N' equals number of bits set in the 'mask'. */ + resource_mask = array[1]; if (num_elems == 1) { ret = disable_logs(adev, resource_mask); } else { diff --git a/sound/soc/intel/avs/ipc.c b/sound/soc/intel/avs/ipc.c index 74f676fdfba29..afd472906ede4 100644 --- a/sound/soc/intel/avs/ipc.c +++ b/sound/soc/intel/avs/ipc.c @@ -169,7 +169,9 @@ static void avs_dsp_exception_caught(struct avs_dev *adev, union avs_notify_msg dev_crit(adev->dev, "communication severed, rebooting dsp..\n"); - cancel_delayed_work_sync(&ipc->d0ix_work); + /* Avoid deadlock as the exception may be the response to SET_D0IX. */ + if (current_work() != &ipc->d0ix_work.work) + cancel_delayed_work_sync(&ipc->d0ix_work); ipc->in_d0ix = false; /* Re-enabled on recovery completion. */ pm_runtime_disable(adev->dev); diff --git a/sound/soc/intel/avs/pcm.c b/sound/soc/intel/avs/pcm.c index 781019685b941..9251c38cf9d12 100644 --- a/sound/soc/intel/avs/pcm.c +++ b/sound/soc/intel/avs/pcm.c @@ -611,6 +611,8 @@ static int avs_dai_fe_prepare(struct snd_pcm_substream *substream, struct snd_so data = snd_soc_dai_get_dma_data(dai, substream); host_stream = data->host_stream; + if (runtime->state == SNDRV_PCM_STATE_XRUN) + hdac_stream(host_stream)->prepared = false; if (hdac_stream(host_stream)->prepared) return 0; diff --git a/sound/soc/intel/boards/Kconfig b/sound/soc/intel/boards/Kconfig index 0ae6eecc88519..033be4d3e02db 100644 --- a/sound/soc/intel/boards/Kconfig +++ b/sound/soc/intel/boards/Kconfig @@ -11,7 +11,7 @@ menuconfig SND_SOC_INTEL_MACH kernel: saying N will just cause the configurator to skip all the questions about Intel ASoC machine drivers. -if SND_SOC_INTEL_MACH +if SND_SOC_INTEL_MACH && (SND_SOC_SOF_INTEL_COMMON || !SND_SOC_SOF_INTEL_COMMON) config SND_SOC_INTEL_USER_FRIENDLY_LONG_NAMES bool "Use more user friendly long card names" diff --git a/sound/soc/intel/boards/bytcht_es8316.c b/sound/soc/intel/boards/bytcht_es8316.c index 7a30d2d36f199..fbd7e1f0783fd 100644 --- a/sound/soc/intel/boards/bytcht_es8316.c +++ b/sound/soc/intel/boards/bytcht_es8316.c @@ -46,7 +46,8 @@ enum { BYT_CHT_ES8316_INTMIC_IN2_MAP, }; -#define BYT_CHT_ES8316_MAP(quirk) ((quirk) & GENMASK(3, 0)) +#define BYT_CHT_ES8316_MAP_MASK GENMASK(3, 0) +#define BYT_CHT_ES8316_MAP(quirk) ((quirk) & BYT_CHT_ES8316_MAP_MASK) #define BYT_CHT_ES8316_SSP0 BIT(16) #define BYT_CHT_ES8316_MONO_SPEAKER BIT(17) #define BYT_CHT_ES8316_JD_INVERTED BIT(18) @@ -59,10 +60,23 @@ MODULE_PARM_DESC(quirk, "Board-specific quirk override"); static void log_quirks(struct device *dev) { - if (BYT_CHT_ES8316_MAP(quirk) == BYT_CHT_ES8316_INTMIC_IN1_MAP) + int map; + + map = BYT_CHT_ES8316_MAP(quirk); + switch (map) { + case BYT_CHT_ES8316_INTMIC_IN1_MAP: dev_info(dev, "quirk IN1_MAP enabled"); - if (BYT_CHT_ES8316_MAP(quirk) == BYT_CHT_ES8316_INTMIC_IN2_MAP) + break; + case BYT_CHT_ES8316_INTMIC_IN2_MAP: dev_info(dev, "quirk IN2_MAP enabled"); + break; + default: + dev_warn_once(dev, "quirk sets invalid input map: 0x%x, default to INTMIC_IN1_MAP\n", map); + quirk &= ~BYT_CHT_ES8316_MAP_MASK; + quirk |= BYT_CHT_ES8316_INTMIC_IN1_MAP; + break; + } + if (quirk & BYT_CHT_ES8316_SSP0) dev_info(dev, "quirk SSP0 enabled"); if (quirk & BYT_CHT_ES8316_MONO_SPEAKER) diff --git a/sound/soc/intel/boards/bytcr_rt5640.c b/sound/soc/intel/boards/bytcr_rt5640.c index ce80adc30fe94..beca466dd468e 100644 --- a/sound/soc/intel/boards/bytcr_rt5640.c +++ b/sound/soc/intel/boards/bytcr_rt5640.c @@ -68,7 +68,8 @@ enum { BYT_RT5640_OVCD_SF_1P5 = (RT5640_OVCD_SF_1P5 << 13), }; -#define BYT_RT5640_MAP(quirk) ((quirk) & GENMASK(3, 0)) +#define BYT_RT5640_MAP_MASK GENMASK(3, 0) +#define BYT_RT5640_MAP(quirk) ((quirk) & BYT_RT5640_MAP_MASK) #define BYT_RT5640_JDSRC(quirk) (((quirk) & GENMASK(7, 4)) >> 4) #define BYT_RT5640_OVCD_TH(quirk) (((quirk) & GENMASK(12, 8)) >> 8) #define BYT_RT5640_OVCD_SF(quirk) (((quirk) & GENMASK(14, 13)) >> 13) @@ -140,7 +141,9 @@ static void log_quirks(struct device *dev) dev_info(dev, "quirk NO_INTERNAL_MIC_MAP enabled\n"); break; default: - dev_err(dev, "quirk map 0x%x is not supported, microphone input will not work\n", map); + dev_warn_once(dev, "quirk sets invalid input map: 0x%x, default to DMIC1_MAP\n", map); + byt_rt5640_quirk &= ~BYT_RT5640_MAP_MASK; + byt_rt5640_quirk |= BYT_RT5640_DMIC1_MAP; break; } if (byt_rt5640_quirk & BYT_RT5640_HSMIC2_ON_IN1) @@ -576,6 +579,19 @@ static const struct dmi_system_id byt_rt5640_quirk_table[] = { BYT_RT5640_SSP0_AIF2 | BYT_RT5640_MCLK_EN), }, + { /* Acer Aspire SW3-013 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "Aspire SW3-013"), + }, + .driver_data = (void *)(BYT_RT5640_DMIC1_MAP | + BYT_RT5640_JD_SRC_JD2_IN4N | + BYT_RT5640_OVCD_TH_2000UA | + BYT_RT5640_OVCD_SF_0P75 | + BYT_RT5640_DIFF_MIC | + BYT_RT5640_SSP0_AIF1 | + BYT_RT5640_MCLK_EN), + }, { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Acer"), diff --git a/sound/soc/intel/boards/bytcr_rt5651.c b/sound/soc/intel/boards/bytcr_rt5651.c index 805afaf47b290..2abe47303ecdb 100644 --- a/sound/soc/intel/boards/bytcr_rt5651.c +++ b/sound/soc/intel/boards/bytcr_rt5651.c @@ -58,7 +58,8 @@ enum { BYT_RT5651_OVCD_SF_1P5 = (RT5651_OVCD_SF_1P5 << 13), }; -#define BYT_RT5651_MAP(quirk) ((quirk) & GENMASK(3, 0)) +#define BYT_RT5651_MAP_MASK GENMASK(3, 0) +#define BYT_RT5651_MAP(quirk) ((quirk) & BYT_RT5651_MAP_MASK) #define BYT_RT5651_JDSRC(quirk) (((quirk) & GENMASK(7, 4)) >> 4) #define BYT_RT5651_OVCD_TH(quirk) (((quirk) & GENMASK(12, 8)) >> 8) #define BYT_RT5651_OVCD_SF(quirk) (((quirk) & GENMASK(14, 13)) >> 13) @@ -100,14 +101,29 @@ MODULE_PARM_DESC(quirk, "Board-specific quirk override"); static void log_quirks(struct device *dev) { - if (BYT_RT5651_MAP(byt_rt5651_quirk) == BYT_RT5651_DMIC_MAP) + int map; + + map = BYT_RT5651_MAP(byt_rt5651_quirk); + switch (map) { + case BYT_RT5651_DMIC_MAP: dev_info(dev, "quirk DMIC_MAP enabled"); - if (BYT_RT5651_MAP(byt_rt5651_quirk) == BYT_RT5651_IN1_MAP) + break; + case BYT_RT5651_IN1_MAP: dev_info(dev, "quirk IN1_MAP enabled"); - if (BYT_RT5651_MAP(byt_rt5651_quirk) == BYT_RT5651_IN2_MAP) + break; + case BYT_RT5651_IN2_MAP: dev_info(dev, "quirk IN2_MAP enabled"); - if (BYT_RT5651_MAP(byt_rt5651_quirk) == BYT_RT5651_IN1_IN2_MAP) + break; + case BYT_RT5651_IN1_IN2_MAP: dev_info(dev, "quirk IN1_IN2_MAP enabled"); + break; + default: + dev_warn_once(dev, "quirk sets invalid input map: 0x%x, default to DMIC_MAP\n", map); + byt_rt5651_quirk &= ~BYT_RT5651_MAP_MASK; + byt_rt5651_quirk |= BYT_RT5651_DMIC_MAP; + break; + } + if (BYT_RT5651_JDSRC(byt_rt5651_quirk)) { dev_info(dev, "quirk realtek,jack-detect-source %ld\n", BYT_RT5651_JDSRC(byt_rt5651_quirk)); diff --git a/sound/soc/mediatek/common/mtk-afe-platform-driver.c b/sound/soc/mediatek/common/mtk-afe-platform-driver.c index 52495c930ca3b..56a704ec2ea94 100644 --- a/sound/soc/mediatek/common/mtk-afe-platform-driver.c +++ b/sound/soc/mediatek/common/mtk-afe-platform-driver.c @@ -120,7 +120,9 @@ int mtk_afe_pcm_new(struct snd_soc_component *component, struct mtk_base_afe *afe = snd_soc_component_get_drvdata(component); size = afe->mtk_afe_hardware->buffer_bytes_max; - snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV, afe->dev, 0, size); + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV, afe->dev, + afe->preallocate_buffers ? size : 0, + size); return 0; } diff --git a/sound/soc/mediatek/common/mtk-base-afe.h b/sound/soc/mediatek/common/mtk-base-afe.h index f51578b6c50a3..a406f2e3e7a87 100644 --- a/sound/soc/mediatek/common/mtk-base-afe.h +++ b/sound/soc/mediatek/common/mtk-base-afe.h @@ -117,6 +117,7 @@ struct mtk_base_afe { struct mtk_base_afe_irq *irqs; int irqs_size; int memif_32bit_supported; + bool preallocate_buffers; struct list_head sub_dais; struct snd_soc_dai_driver *dai_drivers; diff --git a/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c b/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c index 06269f7e37566..240b2f041a3f8 100644 --- a/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c +++ b/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1070,6 +1071,12 @@ static int mt8173_afe_pcm_dev_probe(struct platform_device *pdev) afe->dev = &pdev->dev; + ret = of_reserved_mem_device_init(&pdev->dev); + if (ret) { + dev_info(&pdev->dev, "no reserved memory found, pre-allocating buffers instead\n"); + afe->preallocate_buffers = true; + } + irq_id = platform_get_irq(pdev, 0); if (irq_id <= 0) return irq_id < 0 ? irq_id : -ENXIO; diff --git a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c index 90422ed2bbcc2..cbee5a8764c37 100644 --- a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c +++ b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -1106,6 +1107,12 @@ static int mt8183_afe_pcm_dev_probe(struct platform_device *pdev) afe->dev = &pdev->dev; dev = afe->dev; + ret = of_reserved_mem_device_init(dev); + if (ret) { + dev_info(dev, "no reserved memory found, pre-allocating buffers instead\n"); + afe->preallocate_buffers = true; + } + /* initial audio related clock */ ret = mt8183_init_clock(afe); if (ret) { diff --git a/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c b/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c index b86159f70a33a..9ee431304a431 100644 --- a/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c +++ b/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -2835,6 +2836,12 @@ static int mt8186_afe_pcm_dev_probe(struct platform_device *pdev) afe_priv = afe->platform_priv; afe->dev = &pdev->dev; + ret = of_reserved_mem_device_init(dev); + if (ret) { + dev_info(dev, "no reserved memory found, pre-allocating buffers instead\n"); + afe->preallocate_buffers = true; + } + afe->base_addr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(afe->base_addr)) return PTR_ERR(afe->base_addr); diff --git a/sound/soc/mediatek/mt8188/mt8188-afe-clk.c b/sound/soc/mediatek/mt8188/mt8188-afe-clk.c index e69c1bb2cb239..7f411b8577823 100644 --- a/sound/soc/mediatek/mt8188/mt8188-afe-clk.c +++ b/sound/soc/mediatek/mt8188/mt8188-afe-clk.c @@ -58,7 +58,15 @@ static const char *aud_clks[MT8188_CLK_NUM] = { [MT8188_CLK_AUD_ADC] = "aud_adc", [MT8188_CLK_AUD_DAC_HIRES] = "aud_dac_hires", [MT8188_CLK_AUD_A1SYS_HP] = "aud_a1sys_hp", + [MT8188_CLK_AUD_AFE_DMIC1] = "aud_afe_dmic1", + [MT8188_CLK_AUD_AFE_DMIC2] = "aud_afe_dmic2", + [MT8188_CLK_AUD_AFE_DMIC3] = "aud_afe_dmic3", + [MT8188_CLK_AUD_AFE_DMIC4] = "aud_afe_dmic4", [MT8188_CLK_AUD_ADC_HIRES] = "aud_adc_hires", + [MT8188_CLK_AUD_DMIC_HIRES1] = "aud_dmic_hires1", + [MT8188_CLK_AUD_DMIC_HIRES2] = "aud_dmic_hires2", + [MT8188_CLK_AUD_DMIC_HIRES3] = "aud_dmic_hires3", + [MT8188_CLK_AUD_DMIC_HIRES4] = "aud_dmic_hires4", [MT8188_CLK_AUD_I2SIN] = "aud_i2sin", [MT8188_CLK_AUD_TDM_IN] = "aud_tdm_in", [MT8188_CLK_AUD_I2S_OUT] = "aud_i2s_out", diff --git a/sound/soc/mediatek/mt8188/mt8188-afe-clk.h b/sound/soc/mediatek/mt8188/mt8188-afe-clk.h index ec53c171c170a..c6c78d684f3ee 100644 --- a/sound/soc/mediatek/mt8188/mt8188-afe-clk.h +++ b/sound/soc/mediatek/mt8188/mt8188-afe-clk.h @@ -54,7 +54,15 @@ enum { MT8188_CLK_AUD_ADC, MT8188_CLK_AUD_DAC_HIRES, MT8188_CLK_AUD_A1SYS_HP, + MT8188_CLK_AUD_AFE_DMIC1, + MT8188_CLK_AUD_AFE_DMIC2, + MT8188_CLK_AUD_AFE_DMIC3, + MT8188_CLK_AUD_AFE_DMIC4, MT8188_CLK_AUD_ADC_HIRES, + MT8188_CLK_AUD_DMIC_HIRES1, + MT8188_CLK_AUD_DMIC_HIRES2, + MT8188_CLK_AUD_DMIC_HIRES3, + MT8188_CLK_AUD_DMIC_HIRES4, MT8188_CLK_AUD_I2SIN, MT8188_CLK_AUD_TDM_IN, MT8188_CLK_AUD_I2S_OUT, diff --git a/sound/soc/mediatek/mt8188/mt8188-afe-pcm.c b/sound/soc/mediatek/mt8188/mt8188-afe-pcm.c index 11f30b183520f..4a304bffef8ba 100644 --- a/sound/soc/mediatek/mt8188/mt8188-afe-pcm.c +++ b/sound/soc/mediatek/mt8188/mt8188-afe-pcm.c @@ -2855,10 +2855,6 @@ static bool mt8188_is_volatile_reg(struct device *dev, unsigned int reg) case AFE_DMIC3_SRC_DEBUG_MON0: case AFE_DMIC3_UL_SRC_MON0: case AFE_DMIC3_UL_SRC_MON1: - case DMIC_GAIN1_CUR: - case DMIC_GAIN2_CUR: - case DMIC_GAIN3_CUR: - case DMIC_GAIN4_CUR: case ETDM_IN1_MONITOR: case ETDM_IN2_MONITOR: case ETDM_OUT1_MONITOR: diff --git a/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c b/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c index d0520e7e1d79a..364e43da0e241 100644 --- a/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c +++ b/sound/soc/mediatek/mt8192/mt8192-afe-pcm.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -2196,6 +2197,12 @@ static int mt8192_afe_pcm_dev_probe(struct platform_device *pdev) afe->dev = &pdev->dev; dev = afe->dev; + ret = of_reserved_mem_device_init(dev); + if (ret) { + dev_info(dev, "no reserved memory found, pre-allocating buffers instead\n"); + afe->preallocate_buffers = true; + } + /* init audio related clock */ ret = mt8192_init_clock(afe); if (ret) { diff --git a/sound/soc/meson/aiu-encoder-i2s.c b/sound/soc/meson/aiu-encoder-i2s.c index a0dd914c8ed13..3b4061508c180 100644 --- a/sound/soc/meson/aiu-encoder-i2s.c +++ b/sound/soc/meson/aiu-encoder-i2s.c @@ -236,8 +236,12 @@ static int aiu_encoder_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) inv == SND_SOC_DAIFMT_IB_IF) val |= AIU_CLK_CTRL_LRCLK_INVERT; - if (inv == SND_SOC_DAIFMT_IB_NF || - inv == SND_SOC_DAIFMT_IB_IF) + /* + * The SoC changes data on the rising edge of the bitclock + * so an inversion of the bitclock is required in normal mode + */ + if (inv == SND_SOC_DAIFMT_NB_NF || + inv == SND_SOC_DAIFMT_NB_IF) val |= AIU_CLK_CTRL_AOCLK_INVERT; /* Signal skew */ @@ -328,4 +332,3 @@ const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops = { .startup = aiu_encoder_i2s_startup, .shutdown = aiu_encoder_i2s_shutdown, }; - diff --git a/sound/soc/meson/meson-card-utils.c b/sound/soc/meson/meson-card-utils.c index f7fd9c013e19f..1adaf0041a480 100644 --- a/sound/soc/meson/meson-card-utils.c +++ b/sound/soc/meson/meson-card-utils.c @@ -231,7 +231,7 @@ static int meson_card_parse_of_optional(struct snd_soc_card *card, const char *p)) { /* If property is not provided, don't fail ... */ - if (!of_property_read_bool(card->dev->of_node, propname)) + if (!of_property_present(card->dev->of_node, propname)) return 0; /* ... but do fail if it is provided and the parsing fails */ diff --git a/sound/soc/qcom/lpass-platform.c b/sound/soc/qcom/lpass-platform.c index f918d9e16dc04..f342bc4b3a146 100644 --- a/sound/soc/qcom/lpass-platform.c +++ b/sound/soc/qcom/lpass-platform.c @@ -201,7 +201,6 @@ static int lpass_platform_pcmops_open(struct snd_soc_component *component, struct regmap *map; unsigned int dai_id = cpu_dai->driver->id; - component->id = dai_id; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -1189,13 +1188,14 @@ static int lpass_platform_pcmops_suspend(struct snd_soc_component *component) { struct lpass_data *drvdata = snd_soc_component_get_drvdata(component); struct regmap *map; - unsigned int dai_id = component->id; - if (dai_id == LPASS_DP_RX) + if (drvdata->hdmi_port_enable) { map = drvdata->hdmiif_map; - else - map = drvdata->lpaif_map; + regcache_cache_only(map, true); + regcache_mark_dirty(map); + } + map = drvdata->lpaif_map; regcache_cache_only(map, true); regcache_mark_dirty(map); @@ -1206,14 +1206,19 @@ static int lpass_platform_pcmops_resume(struct snd_soc_component *component) { struct lpass_data *drvdata = snd_soc_component_get_drvdata(component); struct regmap *map; - unsigned int dai_id = component->id; + int ret; - if (dai_id == LPASS_DP_RX) + if (drvdata->hdmi_port_enable) { map = drvdata->hdmiif_map; - else - map = drvdata->lpaif_map; + regcache_cache_only(map, false); + ret = regcache_sync(map); + if (ret) + return ret; + } + map = drvdata->lpaif_map; regcache_cache_only(map, false); + return regcache_sync(map); } @@ -1223,7 +1228,9 @@ static int lpass_platform_copy(struct snd_soc_component *component, unsigned long bytes) { struct snd_pcm_runtime *rt = substream->runtime; - unsigned int dai_id = component->id; + struct snd_soc_pcm_runtime *soc_runtime = snd_soc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(soc_runtime, 0); + unsigned int dai_id = cpu_dai->driver->id; int ret = 0; void __iomem *dma_buf = (void __iomem *) (rt->dma_area + pos + diff --git a/sound/soc/qcom/qdsp6/audioreach.c b/sound/soc/qcom/qdsp6/audioreach.c index 5974c7929dd37..3eab43e9efb32 100644 --- a/sound/soc/qcom/qdsp6/audioreach.c +++ b/sound/soc/qcom/qdsp6/audioreach.c @@ -967,6 +967,7 @@ static int audioreach_i2s_set_media_format(struct q6apm_graph *graph, param_data->param_id = PARAM_ID_I2S_INTF_CFG; param_data->param_size = ic_sz - APM_MODULE_PARAM_DATA_SIZE; + intf_cfg->cfg.lpaif_type = module->hw_interface_type; intf_cfg->cfg.intf_idx = module->hw_interface_idx; intf_cfg->cfg.sd_line_idx = module->sd_line_idx; diff --git a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c index 6511f0a08de16..3813fe5e01819 100644 --- a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c +++ b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c @@ -207,8 +207,10 @@ static int q6apm_lpass_dai_prepare(struct snd_pcm_substream *substream, struct s return 0; err: - q6apm_graph_close(dai_data->graph[dai->id]); - dai_data->graph[dai->id] = NULL; + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + q6apm_graph_close(dai_data->graph[dai->id]); + dai_data->graph[dai->id] = NULL; + } return rc; } @@ -254,6 +256,7 @@ static const struct snd_soc_dai_ops q6i2s_ops = { .shutdown = q6apm_lpass_dai_shutdown, .set_channel_map = q6dma_set_channel_map, .hw_params = q6dma_hw_params, + .set_fmt = q6i2s_set_fmt, }; static const struct snd_soc_dai_ops q6hdmi_ops = { diff --git a/sound/soc/qcom/qdsp6/q6asm.c b/sound/soc/qcom/qdsp6/q6asm.c index 195780f75d05d..9f53cffe5184e 100644 --- a/sound/soc/qcom/qdsp6/q6asm.c +++ b/sound/soc/qcom/qdsp6/q6asm.c @@ -376,9 +376,9 @@ static void q6asm_audio_client_free_buf(struct audio_client *ac, spin_lock_irqsave(&ac->lock, flags); port->num_periods = 0; + spin_unlock_irqrestore(&ac->lock, flags); kfree(port->buf); port->buf = NULL; - spin_unlock_irqrestore(&ac->lock, flags); } /** diff --git a/sound/soc/qcom/qdsp6/topology.c b/sound/soc/qcom/qdsp6/topology.c index 70572c83e1017..d4c3bc85fb356 100644 --- a/sound/soc/qcom/qdsp6/topology.c +++ b/sound/soc/qcom/qdsp6/topology.c @@ -586,8 +586,8 @@ static int audioreach_widget_load_module_common(struct snd_soc_component *compon return PTR_ERR(cont); mod = audioreach_parse_common_tokens(apm, cont, &tplg_w->priv, w); - if (IS_ERR(mod)) - return PTR_ERR(mod); + if (IS_ERR_OR_NULL(mod)) + return mod ? PTR_ERR(mod) : -ENODEV; dobj = &w->dobj; dobj->private = mod; diff --git a/sound/soc/qcom/sc8280xp.c b/sound/soc/qcom/sc8280xp.c index d5cc967992d16..ee195a54d0c3b 100644 --- a/sound/soc/qcom/sc8280xp.c +++ b/sound/soc/qcom/sc8280xp.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -58,8 +59,10 @@ static int sc8280xp_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd, SNDRV_PCM_HW_PARAM_RATE); struct snd_interval *channels = hw_param_interval(params, SNDRV_PCM_HW_PARAM_CHANNELS); + struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT); rate->min = rate->max = 48000; + snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S16_LE); channels->min = 2; channels->max = 2; switch (cpu_dai->id) { diff --git a/sound/soc/qcom/sdm845.c b/sound/soc/qcom/sdm845.c index 3eb29645a6377..39d22304ce43f 100644 --- a/sound/soc/qcom/sdm845.c +++ b/sound/soc/qcom/sdm845.c @@ -90,6 +90,10 @@ static int sdm845_slim_snd_hw_params(struct snd_pcm_substream *substream, else ret = snd_soc_dai_set_channel_map(cpu_dai, tx_ch_cnt, tx_ch, 0, NULL); + if (ret != 0 && ret != -ENOTSUPP) { + dev_err(rtd->dev, "failed to set cpu chan map, err:%d\n", ret); + return ret; + } } return 0; diff --git a/sound/soc/qcom/sm8250.c b/sound/soc/qcom/sm8250.c index 88a7169336d61..580eb20b0771a 100644 --- a/sound/soc/qcom/sm8250.c +++ b/sound/soc/qcom/sm8250.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -39,9 +40,11 @@ static int sm8250_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd, SNDRV_PCM_HW_PARAM_RATE); struct snd_interval *channels = hw_param_interval(params, SNDRV_PCM_HW_PARAM_CHANNELS); + struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT); rate->min = rate->max = 48000; channels->min = channels->max = 2; + snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S16_LE); return 0; } diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index e65fe3a7c3e42..dc95b6f415558 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -1025,6 +1025,9 @@ static int soc_dai_link_sanity_check(struct snd_soc_card *card, void snd_soc_remove_pcm_runtime(struct snd_soc_card *card, struct snd_soc_pcm_runtime *rtd) { + if (!rtd) + return; + lockdep_assert_held(&client_mutex); /* @@ -2935,7 +2938,7 @@ int snd_soc_of_parse_pin_switches(struct snd_soc_card *card, const char *prop) unsigned int i, nb_controls; int ret; - if (!of_property_read_bool(dev->of_node, prop)) + if (!of_property_present(dev->of_node, prop)) return 0; strings = devm_kcalloc(dev, nb_controls_max, @@ -3009,23 +3012,17 @@ int snd_soc_of_parse_tdm_slot(struct device_node *np, if (rx_mask) snd_soc_of_get_slot_mask(np, "dai-tdm-slot-rx-mask", rx_mask); - if (of_property_read_bool(np, "dai-tdm-slot-num")) { - ret = of_property_read_u32(np, "dai-tdm-slot-num", &val); - if (ret) - return ret; - - if (slots) - *slots = val; - } - - if (of_property_read_bool(np, "dai-tdm-slot-width")) { - ret = of_property_read_u32(np, "dai-tdm-slot-width", &val); - if (ret) - return ret; + ret = of_property_read_u32(np, "dai-tdm-slot-num", &val); + if (ret && ret != -EINVAL) + return ret; + if (!ret && slots) + *slots = val; - if (slot_width) - *slot_width = val; - } + ret = of_property_read_u32(np, "dai-tdm-slot-width", &val); + if (ret && ret != -EINVAL) + return ret; + if (!ret && slot_width) + *slot_width = val; return 0; } @@ -3249,10 +3246,10 @@ unsigned int snd_soc_daifmt_parse_format(struct device_node *np, * SND_SOC_DAIFMT_INV_MASK area */ snprintf(prop, sizeof(prop), "%sbitclock-inversion", prefix); - bit = !!of_get_property(np, prop, NULL); + bit = of_property_read_bool(np, prop); snprintf(prop, sizeof(prop), "%sframe-inversion", prefix); - frame = !!of_get_property(np, prop, NULL); + frame = of_property_read_bool(np, prop); switch ((bit << 4) + frame) { case 0x11: @@ -3289,12 +3286,12 @@ unsigned int snd_soc_daifmt_parse_clock_provider_raw(struct device_node *np, * check "[prefix]frame-master" */ snprintf(prop, sizeof(prop), "%sbitclock-master", prefix); - bit = !!of_get_property(np, prop, NULL); + bit = of_property_present(np, prop); if (bit && bitclkmaster) *bitclkmaster = of_parse_phandle(np, prop, 0); snprintf(prop, sizeof(prop), "%sframe-master", prefix); - frame = !!of_get_property(np, prop, NULL); + frame = of_property_present(np, prop); if (frame && framemaster) *framemaster = of_parse_phandle(np, prop, 0); diff --git a/sound/soc/soc-dai.c b/sound/soc/soc-dai.c index 9a828e55c4f9e..5a0fa8d1f38b5 100644 --- a/sound/soc/soc-dai.c +++ b/sound/soc/soc-dai.c @@ -273,12 +273,15 @@ int snd_soc_dai_set_tdm_slot(struct snd_soc_dai *dai, &rx_mask, }; - if (dai->driver->ops && - dai->driver->ops->xlate_tdm_slot_mask) - dai->driver->ops->xlate_tdm_slot_mask(slots, - &tx_mask, &rx_mask); - else - snd_soc_xlate_tdm_slot_mask(slots, &tx_mask, &rx_mask); + if (slots) { + if (dai->driver->ops && + dai->driver->ops->xlate_tdm_slot_mask) + ret = dai->driver->ops->xlate_tdm_slot_mask(slots, &tx_mask, &rx_mask); + else + ret = snd_soc_xlate_tdm_slot_mask(slots, &tx_mask, &rx_mask); + if (ret) + goto err; + } for_each_pcm_streams(stream) snd_soc_dai_tdm_mask_set(dai, stream, *tdm_mask[stream]); @@ -287,6 +290,7 @@ int snd_soc_dai_set_tdm_slot(struct snd_soc_dai *dai, dai->driver->ops->set_tdm_slot) ret = dai->driver->ops->set_tdm_slot(dai, tx_mask, rx_mask, slots, slot_width); +err: return soc_dai_ret(dai, ret); } EXPORT_SYMBOL_GPL(snd_soc_dai_set_tdm_slot); diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 7729f8f4d5e61..7facb7b2dba19 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -734,6 +734,10 @@ static int snd_soc_dapm_set_bias_level(struct snd_soc_dapm_context *dapm, out: trace_snd_soc_bias_level_done(card, level); + /* success */ + if (ret == 0) + snd_soc_dapm_init_bias_level(dapm, level); + return ret; } diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index b4cfc34d00ee6..5be32c37bb8a0 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -638,6 +638,37 @@ int snd_soc_get_volsw_range(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_get_volsw_range); +static int snd_soc_clip_to_platform_max(struct snd_kcontrol *kctl) +{ + struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; + struct snd_ctl_elem_value *uctl; + int ret; + + if (!mc->platform_max) + return 0; + + uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + if (!uctl) + return -ENOMEM; + + ret = kctl->get(kctl, uctl); + if (ret < 0) + goto out; + + if (uctl->value.integer.value[0] > mc->platform_max) + uctl->value.integer.value[0] = mc->platform_max; + + if (snd_soc_volsw_is_stereo(mc) && + uctl->value.integer.value[1] > mc->platform_max) + uctl->value.integer.value[1] = mc->platform_max; + + ret = kctl->put(kctl, uctl); + +out: + kfree(uctl); + return ret; +} + /** * snd_soc_limit_volume - Set new limit to an existing volume control. * @@ -662,7 +693,7 @@ int snd_soc_limit_volume(struct snd_soc_card *card, struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; if (max <= mc->max - mc->min) { mc->platform_max = max; - ret = 0; + ret = snd_soc_clip_to_platform_max(kctl); } } return ret; diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c index 25bbba49b522f..efe068daff59f 100644 --- a/sound/soc/soc-pcm.c +++ b/sound/soc/soc-pcm.c @@ -1542,10 +1542,13 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream, /* * Filter for systems with 'component_chaining' enabled. * This helps to avoid unnecessary re-configuration of an - * already active BE on such systems. + * already active BE on such systems and ensures the BE DAI + * widget is powered ON after hw_params() BE DAI callback. */ if (fe->card->component_chaining && (be->dpcm[stream].state != SND_SOC_DPCM_STATE_NEW) && + (be->dpcm[stream].state != SND_SOC_DPCM_STATE_OPEN) && + (be->dpcm[stream].state != SND_SOC_DPCM_STATE_HW_PARAMS) && (be->dpcm[stream].state != SND_SOC_DPCM_STATE_CLOSE)) continue; diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c index 0b0087abcc50e..3bb743cb167a5 100644 --- a/sound/soc/sof/intel/hda-stream.c +++ b/sound/soc/sof/intel/hda-stream.c @@ -842,7 +842,7 @@ int hda_dsp_stream_init(struct snd_sof_dev *sdev) if (num_capture >= SOF_HDA_CAPTURE_STREAMS) { dev_err(sdev->dev, "error: too many capture streams %d\n", - num_playback); + num_capture); return -EINVAL; } diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c index 0e5ae7fa0ef7a..0db8c4b2f0ff2 100644 --- a/sound/soc/sof/ipc3-topology.c +++ b/sound/soc/sof/ipc3-topology.c @@ -2394,11 +2394,6 @@ static int sof_ipc3_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verif if (ret < 0) return ret; - /* free all the scheduler widgets now */ - ret = sof_ipc3_free_widgets_in_list(sdev, true, &dyn_widgets, verify); - if (ret < 0) - return ret; - /* * Tear down all pipelines associated with PCMs that did not get suspended * and unset the prepare flag so that they can be set up again during resume. @@ -2414,6 +2409,11 @@ static int sof_ipc3_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verif } } + /* free all the scheduler widgets now. This will also power down the secondary cores */ + ret = sof_ipc3_free_widgets_in_list(sdev, true, &dyn_widgets, verify); + if (ret < 0) + return ret; + list_for_each_entry(sroute, &sdev->route_list, list) sroute->setup = false; diff --git a/sound/soc/sof/ipc4-control.c b/sound/soc/sof/ipc4-control.c index b4cdcec33e120..84145209dec49 100644 --- a/sound/soc/sof/ipc4-control.c +++ b/sound/soc/sof/ipc4-control.c @@ -483,6 +483,14 @@ static int sof_ipc4_bytes_ext_put(struct snd_sof_control *scontrol, return -EINVAL; } + /* Check header id */ + if (header.numid != SOF_CTRL_CMD_BINARY) { + dev_err_ratelimited(scomp->dev, + "Incorrect numid for bytes put %d\n", + header.numid); + return -EINVAL; + } + /* Verify the ABI header first */ if (copy_from_user(&abi_hdr, tlvd->tlv, sizeof(abi_hdr))) return -EFAULT; @@ -565,7 +573,8 @@ static int _sof_ipc4_bytes_ext_get(struct snd_sof_control *scontrol, if (data_size > size) return -ENOSPC; - header.numid = scontrol->comp_id; + /* Set header id and length */ + header.numid = SOF_CTRL_CMD_BINARY; header.length = data_size; if (copy_to_user(tlvd, &header, sizeof(struct snd_ctl_tlv))) diff --git a/sound/soc/sof/ipc4-pcm.c b/sound/soc/sof/ipc4-pcm.c index e8acf60c27a74..a29632423ccda 100644 --- a/sound/soc/sof/ipc4-pcm.c +++ b/sound/soc/sof/ipc4-pcm.c @@ -615,13 +615,15 @@ static int sof_ipc4_pcm_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm /* allocate memory for max number of pipeline IDs */ pipeline_list->pipelines = kcalloc(ipc4_data->max_num_pipelines, - sizeof(struct snd_sof_widget *), GFP_KERNEL); + sizeof(*pipeline_list->pipelines), + GFP_KERNEL); if (!pipeline_list->pipelines) { sof_ipc4_pcm_free(sdev, spcm); return -ENOMEM; } - if (!support_info) + /* Delay reporting is only supported on playback */ + if (!support_info || stream == SNDRV_PCM_STREAM_CAPTURE) continue; stream_info = kzalloc(sizeof(*stream_info), GFP_KERNEL); diff --git a/sound/soc/sof/ipc4-topology.h b/sound/soc/sof/ipc4-topology.h index 0fb759c6eeaf9..07bd84204baa7 100644 --- a/sound/soc/sof/ipc4-topology.h +++ b/sound/soc/sof/ipc4-topology.h @@ -59,8 +59,8 @@ #define SOF_IPC4_INVALID_NODE_ID 0xffffffff -/* FW requires minimum 2ms DMA buffer size */ -#define SOF_IPC4_MIN_DMA_BUFFER_SIZE 2 +/* FW requires minimum 4ms DMA buffer size */ +#define SOF_IPC4_MIN_DMA_BUFFER_SIZE 4 /* * The base of multi-gateways. Multi-gateways addressing starts from diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 7afded323150c..c18a1fdd40ee3 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -1057,7 +1057,7 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, struct snd_sof_dai *dai) { struct snd_soc_card *card = scomp->card; - struct snd_soc_pcm_runtime *rtd; + struct snd_soc_pcm_runtime *rtd, *full, *partial; struct snd_soc_dai *cpu_dai; int stream; int i; @@ -1074,12 +1074,22 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, else goto end; + full = NULL; + partial = NULL; list_for_each_entry(rtd, &card->rtd_list, list) { /* does stream match DAI link ? */ - if (!rtd->dai_link->stream_name || - !strstr(rtd->dai_link->stream_name, w->sname)) - continue; + if (rtd->dai_link->stream_name) { + if (!strcmp(rtd->dai_link->stream_name, w->sname)) { + full = rtd; + break; + } else if (strstr(rtd->dai_link->stream_name, w->sname)) { + partial = rtd; + } + } + } + rtd = full ? full : partial; + if (rtd) { for_each_rtd_cpu_dais(rtd, i, cpu_dai) { /* * Please create DAI widget in the right order diff --git a/sound/soc/stm/stm32_sai_sub.c b/sound/soc/stm/stm32_sai_sub.c index dcbcd1a59a3aa..351e96163406b 100644 --- a/sound/soc/stm/stm32_sai_sub.c +++ b/sound/soc/stm/stm32_sai_sub.c @@ -551,6 +551,14 @@ static int stm32_sai_set_sysclk(struct snd_soc_dai *cpu_dai, struct stm32_sai_sub_data *sai = snd_soc_dai_get_drvdata(cpu_dai); int ret; + /* + * The mclk rate is determined at runtime from the audio stream rate. + * Skip calls to the set_sysclk callback that are not relevant during the + * initialization phase. + */ + if (!snd_soc_card_is_instantiated(cpu_dai->component->card)) + return 0; + if (dir == SND_SOC_CLOCK_OUT && sai->sai_mclk) { ret = stm32_sai_sub_reg_up(sai, STM_SAI_CR1_REGX, SAI_XCR1_NODIV, diff --git a/sound/soc/sunxi/sun4i-codec.c b/sound/soc/sunxi/sun4i-codec.c index f0a5fd9011018..0d7758cc84c63 100644 --- a/sound/soc/sunxi/sun4i-codec.c +++ b/sound/soc/sunxi/sun4i-codec.c @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -239,6 +240,7 @@ struct sun4i_codec { struct clk *clk_module; struct reset_control *rst; struct gpio_desc *gpio_pa; + struct gpio_desc *gpio_hp; /* ADC_FIFOC register is at different offset on different SoCs */ struct regmap_field *reg_adc_fifoc; @@ -1277,6 +1279,49 @@ static struct snd_soc_dai_driver dummy_cpu_dai = { .ops = &dummy_dai_ops, }; +static struct snd_soc_jack sun4i_headphone_jack; + +static struct snd_soc_jack_pin sun4i_headphone_jack_pins[] = { + { .pin = "Headphone", .mask = SND_JACK_HEADPHONE }, +}; + +static struct snd_soc_jack_gpio sun4i_headphone_jack_gpio = { + .name = "hp-det", + .report = SND_JACK_HEADPHONE, + .debounce_time = 150, +}; + +static int sun4i_codec_machine_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct sun4i_codec *scodec = snd_soc_card_get_drvdata(card); + int ret; + + if (scodec->gpio_hp) { + ret = snd_soc_card_jack_new_pins(card, "Headphone Jack", + SND_JACK_HEADPHONE, + &sun4i_headphone_jack, + sun4i_headphone_jack_pins, + ARRAY_SIZE(sun4i_headphone_jack_pins)); + if (ret) { + dev_err(rtd->dev, + "Headphone jack creation failed: %d\n", ret); + return ret; + } + + sun4i_headphone_jack_gpio.desc = scodec->gpio_hp; + ret = snd_soc_jack_add_gpios(&sun4i_headphone_jack, 1, + &sun4i_headphone_jack_gpio); + + if (ret) { + dev_err(rtd->dev, "Headphone GPIO not added: %d\n", ret); + return ret; + } + } + + return 0; +} + static struct snd_soc_dai_link *sun4i_codec_create_link(struct device *dev, int *num_links) { @@ -1302,6 +1347,7 @@ static struct snd_soc_dai_link *sun4i_codec_create_link(struct device *dev, link->codecs->name = dev_name(dev); link->platforms->name = dev_name(dev); link->dai_fmt = SND_SOC_DAIFMT_I2S; + link->init = sun4i_codec_machine_init; *num_links = 1; @@ -1742,6 +1788,13 @@ static int sun4i_codec_probe(struct platform_device *pdev) return ret; } + scodec->gpio_hp = devm_gpiod_get_optional(&pdev->dev, "hp-det", GPIOD_IN); + if (IS_ERR(scodec->gpio_hp)) { + ret = PTR_ERR(scodec->gpio_hp); + dev_err_probe(&pdev->dev, ret, "Failed to get hp-det gpio\n"); + return ret; + } + /* reg_field setup */ scodec->reg_adc_fifoc = devm_regmap_field_alloc(&pdev->dev, scodec->regmap, diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c index ab3c6b2544d20..140cb27f73287 100644 --- a/sound/soc/tegra/tegra210_ahub.c +++ b/sound/soc/tegra/tegra210_ahub.c @@ -1359,6 +1359,8 @@ static int tegra_ahub_probe(struct platform_device *pdev) return -ENOMEM; ahub->soc_data = of_device_get_match_data(&pdev->dev); + if (!ahub->soc_data) + return -ENODEV; platform_set_drvdata(pdev, ahub); diff --git a/sound/soc/ti/omap-hdmi.c b/sound/soc/ti/omap-hdmi.c index 0a731b21e5a58..72fabf22a02ee 100644 --- a/sound/soc/ti/omap-hdmi.c +++ b/sound/soc/ti/omap-hdmi.c @@ -361,17 +361,20 @@ static int omap_hdmi_audio_probe(struct platform_device *pdev) if (!card->dai_link) return -ENOMEM; - compnent = devm_kzalloc(dev, sizeof(*compnent), GFP_KERNEL); + compnent = devm_kzalloc(dev, 2 * sizeof(*compnent), GFP_KERNEL); if (!compnent) return -ENOMEM; - card->dai_link->cpus = compnent; + card->dai_link->cpus = &compnent[0]; card->dai_link->num_cpus = 1; card->dai_link->codecs = &asoc_dummy_dlc; card->dai_link->num_codecs = 1; + card->dai_link->platforms = &compnent[1]; + card->dai_link->num_platforms = 1; card->dai_link->name = card->name; card->dai_link->stream_name = card->name; card->dai_link->cpus->dai_name = dev_name(ad->dssdev); + card->dai_link->platforms->name = dev_name(ad->dssdev); card->num_links = 1; card->dev = dev; diff --git a/sound/usb/card.c b/sound/usb/card.c index 66455a0fb1cb7..42bc603a2ed69 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -790,10 +790,16 @@ get_alias_quirk(struct usb_device *dev, unsigned int id) */ static int try_to_register_card(struct snd_usb_audio *chip, int ifnum) { + struct usb_interface *iface; + if (check_delayed_register_option(chip) == ifnum || - chip->last_iface == ifnum || - usb_interface_claimed(usb_ifnum_to_if(chip->dev, chip->last_iface))) + chip->last_iface == ifnum) + return snd_card_register(chip->card); + + iface = usb_ifnum_to_if(chip->dev, chip->last_iface); + if (iface && usb_interface_claimed(iface)) return snd_card_register(chip->card); + return 0; } diff --git a/sound/usb/clock.c b/sound/usb/clock.c old mode 100644 new mode 100755 diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index d09f33f83af24..b756332c659c0 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -926,14 +926,21 @@ static int endpoint_set_interface(struct snd_usb_audio *chip, { int altset = set ? ep->altsetting : 0; int err; + int retries = 0; + const int max_retries = 5; if (ep->iface_ref->altset == altset) return 0; usb_audio_dbg(chip, "Setting usb interface %d:%d for EP 0x%x\n", ep->iface, altset, ep->ep_num); +retry: err = usb_set_interface(chip->dev, ep->iface, altset); if (err < 0) { + if (err == -EPROTO && ++retries <= max_retries) { + msleep(5 * (1 << (retries - 1))); + goto retry; + } usb_audio_err_ratelimited( chip, "%d:%d: usb_set_interface failed (%d)\n", ep->iface, altset, err); @@ -1376,6 +1383,11 @@ int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, ep->sample_rem = ep->cur_rate % ep->pps; ep->packsize[0] = ep->cur_rate / ep->pps; ep->packsize[1] = (ep->cur_rate + (ep->pps - 1)) / ep->pps; + if (ep->packsize[1] > ep->maxpacksize) { + usb_audio_dbg(chip, "Too small maxpacksize %u for rate %u / pps %u\n", + ep->maxpacksize, ep->cur_rate, ep->pps); + return -EINVAL; + } /* calculate the frequency in 16.16 format */ ep->freqm = ep->freqn; diff --git a/sound/usb/format.c b/sound/usb/format.c index 3b3a5ea6fcbfc..f33d25a4e4cc7 100644 --- a/sound/usb/format.c +++ b/sound/usb/format.c @@ -263,7 +263,8 @@ static int parse_audio_format_rates_v1(struct snd_usb_audio *chip, struct audiof } /* Jabra Evolve 65 headset */ - if (chip->usb_id == USB_ID(0x0b0e, 0x030b)) { + if (chip->usb_id == USB_ID(0x0b0e, 0x030b) || + chip->usb_id == USB_ID(0x0b0e, 0x030c)) { /* only 48kHz for playback while keeping 16kHz for capture */ if (fp->nr_rates != 1) return set_fixed_rate(fp, 48000, SNDRV_PCM_RATE_48000); diff --git a/sound/usb/implicit.c b/sound/usb/implicit.c index 4727043fd7458..77f06da93151e 100644 --- a/sound/usb/implicit.c +++ b/sound/usb/implicit.c @@ -57,6 +57,7 @@ static const struct snd_usb_implicit_fb_match playback_implicit_fb_quirks[] = { IMPLICIT_FB_FIXED_DEV(0x31e9, 0x0002, 0x81, 2), /* Solid State Logic SSL2+ */ IMPLICIT_FB_FIXED_DEV(0x0499, 0x172f, 0x81, 2), /* Steinberg UR22C */ IMPLICIT_FB_FIXED_DEV(0x0d9a, 0x00df, 0x81, 2), /* RTX6001 */ + IMPLICIT_FB_FIXED_DEV(0x19f7, 0x000a, 0x84, 3), /* RODE AI-1 */ IMPLICIT_FB_FIXED_DEV(0x22f0, 0x0006, 0x81, 3), /* Allen&Heath Qu-16 */ IMPLICIT_FB_FIXED_DEV(0x1686, 0xf029, 0x82, 2), /* Zoom UAC-2 */ IMPLICIT_FB_FIXED_DEV(0x2466, 0x8003, 0x86, 2), /* Fractal Audio Axe-Fx II */ diff --git a/sound/usb/midi.c b/sound/usb/midi.c index 3059f814eb5c2..fdf69917d19cc 100644 --- a/sound/usb/midi.c +++ b/sound/usb/midi.c @@ -1522,12 +1522,12 @@ static void snd_usbmidi_free(struct snd_usb_midi *umidi) { int i; + if (!umidi->disconnected) + snd_usbmidi_disconnect(&umidi->list); + for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) { struct snd_usb_midi_endpoint *ep = &umidi->endpoints[i]; - if (ep->out) - snd_usbmidi_out_endpoint_delete(ep->out); - if (ep->in) - snd_usbmidi_in_endpoint_delete(ep->in); + kfree(ep->out); } mutex_destroy(&umidi->mutex); kfree(umidi); @@ -1553,7 +1553,7 @@ void snd_usbmidi_disconnect(struct list_head *p) spin_unlock_irq(&umidi->disc_lock); up_write(&umidi->disc_rwsem); - del_timer_sync(&umidi->error_timer); + timer_shutdown_sync(&umidi->error_timer); for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) { struct snd_usb_midi_endpoint *ep = &umidi->endpoints[i]; diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index b8fa0a866153b..e19d962fab870 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -930,7 +930,7 @@ static int parse_term_uac2_clock_source(struct mixer_build *state, { struct uac_clock_source_descriptor *d = p1; - term->type = UAC3_CLOCK_SOURCE << 16; /* virtual type */ + term->type = UAC2_CLOCK_SOURCE << 16; /* virtual type */ term->id = id; term->name = d->iClockSource; return 0; @@ -1191,6 +1191,13 @@ static void volume_control_quirks(struct usb_mixer_elem_info *cval, cval->res = 1; } break; + case USB_ID(0x3302, 0x12db): /* MOONDROP Quark2 */ + if (!strcmp(kctl->id.name, "PCM Playback Volume")) { + usb_audio_info(chip, + "set volume quirk for MOONDROP Quark2\n"); + cval->min = -14208; /* Mute under it */ + } + break; } } @@ -3072,6 +3079,8 @@ static int snd_usb_mixer_controls_badd(struct usb_mixer_interface *mixer, int i; assoc = usb_ifnum_to_if(dev, ctrlif)->intf_assoc; + if (!assoc) + return -EINVAL; /* Detect BADD capture/playback channels from AS EP descriptors */ for (i = 0; i < assoc->bInterfaceCount; i++) { diff --git a/sound/usb/mixer_maps.c b/sound/usb/mixer_maps.c index 0e9b5431a47f2..faac7df1fbcf0 100644 --- a/sound/usb/mixer_maps.c +++ b/sound/usb/mixer_maps.c @@ -383,6 +383,13 @@ static const struct usbmix_name_map ms_usb_link_map[] = { { 0 } /* terminator */ }; +/* KTMicro USB */ +static struct usbmix_name_map s31b2_0022_map[] = { + { 23, "Speaker Playback" }, + { 18, "Headphone Playback" }, + { 0 } +}; + /* ASUS ROG Zenith II with Realtek ALC1220-VB */ static const struct usbmix_name_map asus_zenith_ii_map[] = { { 19, NULL, 12 }, /* FU, Input Gain Pad - broken response, disabled */ @@ -692,6 +699,11 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = { .id = USB_ID(0x045e, 0x083c), .map = ms_usb_link_map, }, + { + /* KTMicro USB */ + .id = USB_ID(0X31b2, 0x0022), + .map = s31b2_0022_map, + }, { 0 } /* terminator */ }; diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index be0b3c8ac7055..270a0be672b7e 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -76,7 +77,8 @@ static int snd_create_std_mono_ctl_offset(struct usb_mixer_interface *mixer, cval->idx_off = idx_off; /* get_min_max() is called only for integer volumes later, - * so provide a short-cut for booleans */ + * so provide a short-cut for booleans + */ cval->min = 0; cval->max = 1; cval->res = 0; @@ -125,7 +127,7 @@ static int snd_create_std_mono_table(struct usb_mixer_interface *mixer, { int err; - while (t->name != NULL) { + while (t->name) { err = snd_create_std_mono_ctl(mixer, t->unitid, t->control, t->cmask, t->val_type, t->name, t->tlv_callback); if (err < 0) @@ -207,7 +209,6 @@ static void snd_usb_soundblaster_remote_complete(struct urb *urb) if (code == rc->mute_code) snd_usb_mixer_notify_id(mixer, rc->mute_mixer_id); mixer->rc_code = code; - wmb(); wake_up(&mixer->rc_waitq); } @@ -375,10 +376,10 @@ static int snd_audigy2nx_controls_create(struct usb_mixer_interface *mixer) struct snd_kcontrol_new knew; /* USB X-Fi S51 doesn't have a CMSS LED */ - if ((mixer->chip->usb_id == USB_ID(0x041e, 0x3042)) && i == 0) + if (mixer->chip->usb_id == USB_ID(0x041e, 0x3042) && i == 0) continue; /* USB X-Fi S51 Pro doesn't have one either */ - if ((mixer->chip->usb_id == USB_ID(0x041e, 0x30df)) && i == 0) + if (mixer->chip->usb_id == USB_ID(0x041e, 0x30df) && i == 0) continue; if (i > 1 && /* Live24ext has 2 LEDs only */ (mixer->chip->usb_id == USB_ID(0x041e, 0x3040) || @@ -527,6 +528,265 @@ static int snd_emu0204_controls_create(struct usb_mixer_interface *mixer) &snd_emu0204_control, NULL); } +#if IS_REACHABLE(CONFIG_INPUT) +/* + * Sony DualSense controller (PS5) jack detection + * + * Since this is an UAC 1 device, it doesn't support jack detection. + * However, the controller hid-playstation driver reports HP & MIC + * insert events through a dedicated input device. + */ + +#define SND_DUALSENSE_JACK_OUT_TERM_ID 3 +#define SND_DUALSENSE_JACK_IN_TERM_ID 4 + +struct dualsense_mixer_elem_info { + struct usb_mixer_elem_info info; + struct input_handler ih; + struct input_device_id id_table[2]; + bool connected; +}; + +static void snd_dualsense_ih_event(struct input_handle *handle, + unsigned int type, unsigned int code, + int value) +{ + struct dualsense_mixer_elem_info *mei; + struct usb_mixer_elem_list *me; + + if (type != EV_SW) + return; + + mei = container_of(handle->handler, struct dualsense_mixer_elem_info, ih); + me = &mei->info.head; + + if ((me->id == SND_DUALSENSE_JACK_OUT_TERM_ID && code == SW_HEADPHONE_INSERT) || + (me->id == SND_DUALSENSE_JACK_IN_TERM_ID && code == SW_MICROPHONE_INSERT)) { + mei->connected = !!value; + snd_ctl_notify(me->mixer->chip->card, SNDRV_CTL_EVENT_MASK_VALUE, + &me->kctl->id); + } +} + +static bool snd_dualsense_ih_match(struct input_handler *handler, + struct input_dev *dev) +{ + struct dualsense_mixer_elem_info *mei; + struct usb_device *snd_dev; + char *input_dev_path, *usb_dev_path; + size_t usb_dev_path_len; + bool match = false; + + mei = container_of(handler, struct dualsense_mixer_elem_info, ih); + snd_dev = mei->info.head.mixer->chip->dev; + + input_dev_path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL); + if (!input_dev_path) { + dev_warn(&snd_dev->dev, "Failed to get input dev path\n"); + return false; + } + + usb_dev_path = kobject_get_path(&snd_dev->dev.kobj, GFP_KERNEL); + if (!usb_dev_path) { + dev_warn(&snd_dev->dev, "Failed to get USB dev path\n"); + goto free_paths; + } + + /* + * Ensure the VID:PID matched input device supposedly owned by the + * hid-playstation driver belongs to the actual hardware handled by + * the current USB audio device, which implies input_dev_path being + * a subpath of usb_dev_path. + * + * This verification is necessary when there is more than one identical + * controller attached to the host system. + */ + usb_dev_path_len = strlen(usb_dev_path); + if (usb_dev_path_len >= strlen(input_dev_path)) + goto free_paths; + + usb_dev_path[usb_dev_path_len] = '/'; + match = !memcmp(input_dev_path, usb_dev_path, usb_dev_path_len + 1); + +free_paths: + kfree(input_dev_path); + kfree(usb_dev_path); + + return match; +} + +static int snd_dualsense_ih_connect(struct input_handler *handler, + struct input_dev *dev, + const struct input_device_id *id) +{ + struct input_handle *handle; + int err; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = handler->name; + + err = input_register_handle(handle); + if (err) + goto err_free; + + err = input_open_device(handle); + if (err) + goto err_unregister; + + return 0; + +err_unregister: + input_unregister_handle(handle); +err_free: + kfree(handle); + return err; +} + +static void snd_dualsense_ih_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} + +static void snd_dualsense_ih_start(struct input_handle *handle) +{ + struct dualsense_mixer_elem_info *mei; + struct usb_mixer_elem_list *me; + int status = -1; + + mei = container_of(handle->handler, struct dualsense_mixer_elem_info, ih); + me = &mei->info.head; + + if (me->id == SND_DUALSENSE_JACK_OUT_TERM_ID && + test_bit(SW_HEADPHONE_INSERT, handle->dev->swbit)) + status = test_bit(SW_HEADPHONE_INSERT, handle->dev->sw); + else if (me->id == SND_DUALSENSE_JACK_IN_TERM_ID && + test_bit(SW_MICROPHONE_INSERT, handle->dev->swbit)) + status = test_bit(SW_MICROPHONE_INSERT, handle->dev->sw); + + if (status >= 0) { + mei->connected = !!status; + snd_ctl_notify(me->mixer->chip->card, SNDRV_CTL_EVENT_MASK_VALUE, + &me->kctl->id); + } +} + +static int snd_dualsense_jack_get(struct snd_kcontrol *kctl, + struct snd_ctl_elem_value *ucontrol) +{ + struct dualsense_mixer_elem_info *mei = snd_kcontrol_chip(kctl); + + ucontrol->value.integer.value[0] = mei->connected; + + return 0; +} + +static const struct snd_kcontrol_new snd_dualsense_jack_control = { + .iface = SNDRV_CTL_ELEM_IFACE_CARD, + .access = SNDRV_CTL_ELEM_ACCESS_READ, + .info = snd_ctl_boolean_mono_info, + .get = snd_dualsense_jack_get, +}; + +static int snd_dualsense_resume_jack(struct usb_mixer_elem_list *list) +{ + snd_ctl_notify(list->mixer->chip->card, SNDRV_CTL_EVENT_MASK_VALUE, + &list->kctl->id); + return 0; +} + +static void snd_dualsense_mixer_elem_free(struct snd_kcontrol *kctl) +{ + struct dualsense_mixer_elem_info *mei = snd_kcontrol_chip(kctl); + + if (mei->ih.event) + input_unregister_handler(&mei->ih); + + snd_usb_mixer_elem_free(kctl); +} + +static int snd_dualsense_jack_create(struct usb_mixer_interface *mixer, + const char *name, bool is_output) +{ + struct dualsense_mixer_elem_info *mei; + struct input_device_id *idev_id; + struct snd_kcontrol *kctl; + int err; + + mei = kzalloc(sizeof(*mei), GFP_KERNEL); + if (!mei) + return -ENOMEM; + + snd_usb_mixer_elem_init_std(&mei->info.head, mixer, + is_output ? SND_DUALSENSE_JACK_OUT_TERM_ID : + SND_DUALSENSE_JACK_IN_TERM_ID); + + mei->info.head.resume = snd_dualsense_resume_jack; + mei->info.val_type = USB_MIXER_BOOLEAN; + mei->info.channels = 1; + mei->info.min = 0; + mei->info.max = 1; + + kctl = snd_ctl_new1(&snd_dualsense_jack_control, mei); + if (!kctl) { + kfree(mei); + return -ENOMEM; + } + + strscpy(kctl->id.name, name, sizeof(kctl->id.name)); + kctl->private_free = snd_dualsense_mixer_elem_free; + + err = snd_usb_mixer_add_control(&mei->info.head, kctl); + if (err) + return err; + + idev_id = &mei->id_table[0]; + idev_id->flags = INPUT_DEVICE_ID_MATCH_VENDOR | INPUT_DEVICE_ID_MATCH_PRODUCT | + INPUT_DEVICE_ID_MATCH_EVBIT | INPUT_DEVICE_ID_MATCH_SWBIT; + idev_id->vendor = USB_ID_VENDOR(mixer->chip->usb_id); + idev_id->product = USB_ID_PRODUCT(mixer->chip->usb_id); + idev_id->evbit[BIT_WORD(EV_SW)] = BIT_MASK(EV_SW); + if (is_output) + idev_id->swbit[BIT_WORD(SW_HEADPHONE_INSERT)] = BIT_MASK(SW_HEADPHONE_INSERT); + else + idev_id->swbit[BIT_WORD(SW_MICROPHONE_INSERT)] = BIT_MASK(SW_MICROPHONE_INSERT); + + mei->ih.event = snd_dualsense_ih_event; + mei->ih.match = snd_dualsense_ih_match; + mei->ih.connect = snd_dualsense_ih_connect; + mei->ih.disconnect = snd_dualsense_ih_disconnect; + mei->ih.start = snd_dualsense_ih_start; + mei->ih.name = name; + mei->ih.id_table = mei->id_table; + + err = input_register_handler(&mei->ih); + if (err) { + dev_warn(&mixer->chip->dev->dev, + "Could not register input handler: %d\n", err); + mei->ih.event = NULL; + } + + return 0; +} + +static int snd_dualsense_controls_create(struct usb_mixer_interface *mixer) +{ + int err; + + err = snd_dualsense_jack_create(mixer, "Headphone Jack", true); + if (err < 0) + return err; + + return snd_dualsense_jack_create(mixer, "Headset Mic Jack", false); +} +#endif /* IS_REACHABLE(CONFIG_INPUT) */ + /* ASUS Xonar U1 / U3 controls */ static int snd_xonar_u1_switch_get(struct snd_kcontrol *kcontrol, @@ -1733,7 +1993,8 @@ static int snd_microii_spdif_default_put(struct snd_kcontrol *kcontrol, unsigned int pval, pval_old; int err; - pval = pval_old = kcontrol->private_value; + pval = kcontrol->private_value; + pval_old = pval; pval &= 0xfffff0f0; pval |= (ucontrol->value.iec958.status[1] & 0x0f) << 8; pval |= (ucontrol->value.iec958.status[0] & 0x0f); @@ -2150,15 +2411,15 @@ static int dell_dock_mixer_init(struct usb_mixer_interface *mixer) #define SND_RME_CLK_FREQMUL_SHIFT 18 #define SND_RME_CLK_FREQMUL_MASK 0x7 #define SND_RME_CLK_SYSTEM(x) \ - ((x >> SND_RME_CLK_SYSTEM_SHIFT) & SND_RME_CLK_SYSTEM_MASK) + (((x) >> SND_RME_CLK_SYSTEM_SHIFT) & SND_RME_CLK_SYSTEM_MASK) #define SND_RME_CLK_AES(x) \ - ((x >> SND_RME_CLK_AES_SHIFT) & SND_RME_CLK_AES_SPDIF_MASK) + (((x) >> SND_RME_CLK_AES_SHIFT) & SND_RME_CLK_AES_SPDIF_MASK) #define SND_RME_CLK_SPDIF(x) \ - ((x >> SND_RME_CLK_SPDIF_SHIFT) & SND_RME_CLK_AES_SPDIF_MASK) + (((x) >> SND_RME_CLK_SPDIF_SHIFT) & SND_RME_CLK_AES_SPDIF_MASK) #define SND_RME_CLK_SYNC(x) \ - ((x >> SND_RME_CLK_SYNC_SHIFT) & SND_RME_CLK_SYNC_MASK) + (((x) >> SND_RME_CLK_SYNC_SHIFT) & SND_RME_CLK_SYNC_MASK) #define SND_RME_CLK_FREQMUL(x) \ - ((x >> SND_RME_CLK_FREQMUL_SHIFT) & SND_RME_CLK_FREQMUL_MASK) + (((x) >> SND_RME_CLK_FREQMUL_SHIFT) & SND_RME_CLK_FREQMUL_MASK) #define SND_RME_CLK_AES_LOCK 0x1 #define SND_RME_CLK_AES_SYNC 0x4 #define SND_RME_CLK_SPDIF_LOCK 0x2 @@ -2167,9 +2428,9 @@ static int dell_dock_mixer_init(struct usb_mixer_interface *mixer) #define SND_RME_SPDIF_FORMAT_SHIFT 5 #define SND_RME_BINARY_MASK 0x1 #define SND_RME_SPDIF_IF(x) \ - ((x >> SND_RME_SPDIF_IF_SHIFT) & SND_RME_BINARY_MASK) + (((x) >> SND_RME_SPDIF_IF_SHIFT) & SND_RME_BINARY_MASK) #define SND_RME_SPDIF_FORMAT(x) \ - ((x >> SND_RME_SPDIF_FORMAT_SHIFT) & SND_RME_BINARY_MASK) + (((x) >> SND_RME_SPDIF_FORMAT_SHIFT) & SND_RME_BINARY_MASK) static const u32 snd_rme_rate_table[] = { 32000, 44100, 48000, 50000, @@ -3271,7 +3532,7 @@ static int snd_djm_controls_update(struct usb_mixer_interface *mixer, int err; const struct snd_djm_device *device = &snd_djm_devices[device_idx]; - if ((group >= device->ncontrols) || value >= device->controls[group].noptions) + if (group >= device->ncontrols || value >= device->controls[group].noptions) return -EINVAL; err = snd_usb_lock_shutdown(mixer->chip); @@ -3389,6 +3650,13 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer) err = snd_emu0204_controls_create(mixer); break; +#if IS_REACHABLE(CONFIG_INPUT) + case USB_ID(0x054c, 0x0ce6): /* Sony DualSense controller (PS5) */ + case USB_ID(0x054c, 0x0df2): /* Sony DualSense Edge controller (PS5) */ + err = snd_dualsense_controls_create(mixer); + break; +#endif /* IS_REACHABLE(CONFIG_INPUT) */ + case USB_ID(0x0763, 0x2030): /* M-Audio Fast Track C400 */ case USB_ID(0x0763, 0x2031): /* M-Audio Fast Track C400 */ err = snd_c400_create_mixer(mixer); @@ -3546,7 +3814,8 @@ static void snd_dragonfly_quirk_db_scale(struct usb_mixer_interface *mixer, struct snd_kcontrol *kctl) { /* Approximation using 10 ranges based on output measurement on hw v1.2. - * This seems close to the cubic mapping e.g. alsamixer uses. */ + * This seems close to the cubic mapping e.g. alsamixer uses. + */ static const DECLARE_TLV_DB_RANGE(scale, 0, 1, TLV_DB_MINMAX_ITEM(-5300, -4970), 2, 5, TLV_DB_MINMAX_ITEM(-4710, -4160), @@ -3630,14 +3899,12 @@ void snd_usb_mixer_fu_apply_quirk(struct usb_mixer_interface *mixer, if (unitid == 7 && cval->control == UAC_FU_VOLUME) snd_dragonfly_quirk_db_scale(mixer, cval, kctl); break; + } + /* lowest playback value is muted on some devices */ - case USB_ID(0x0d8c, 0x000c): /* C-Media */ - case USB_ID(0x0d8c, 0x0014): /* C-Media */ - case USB_ID(0x19f7, 0x0003): /* RODE NT-USB */ + if (mixer->chip->quirk_flags & QUIRK_FLAG_MIXER_MIN_MUTE) if (strstr(kctl->id.name, "Playback")) cval->min_mute = 1; - break; - } /* ALSA-ify some Plantronics headset control names */ if (USB_ID_VENDOR(mixer->chip->usb_id) == 0x047f && diff --git a/sound/usb/mixer_s1810c.c b/sound/usb/mixer_s1810c.c index fac4bbc6b2757..2413a6d96971c 100644 --- a/sound/usb/mixer_s1810c.c +++ b/sound/usb/mixer_s1810c.c @@ -93,6 +93,7 @@ struct s1810c_ctl_packet { #define SC1810C_CTL_LINE_SW 0 #define SC1810C_CTL_MUTE_SW 1 +#define SC1824C_CTL_MONO_SW 2 #define SC1810C_CTL_AB_SW 3 #define SC1810C_CTL_48V_SW 4 @@ -123,6 +124,7 @@ struct s1810c_state_packet { #define SC1810C_STATE_48V_SW 58 #define SC1810C_STATE_LINE_SW 59 #define SC1810C_STATE_MUTE_SW 60 +#define SC1824C_STATE_MONO_SW 61 #define SC1810C_STATE_AB_SW 62 struct s1810_mixer_state { @@ -181,7 +183,7 @@ snd_sc1810c_get_status_field(struct usb_device *dev, pkt_out.fields[SC1810C_STATE_F1_IDX] = SC1810C_SET_STATE_F1; pkt_out.fields[SC1810C_STATE_F2_IDX] = SC1810C_SET_STATE_F2; - ret = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), + ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), SC1810C_SET_STATE_REQ, SC1810C_SET_STATE_REQTYPE, (*seqnum), 0, &pkt_out, sizeof(pkt_out)); @@ -502,6 +504,15 @@ static const struct snd_kcontrol_new snd_s1810c_mute_sw = { .private_value = (SC1810C_STATE_MUTE_SW | SC1810C_CTL_MUTE_SW << 8) }; +static const struct snd_kcontrol_new snd_s1824c_mono_sw = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "Mono Main Out Switch", + .info = snd_ctl_boolean_mono_info, + .get = snd_s1810c_switch_get, + .put = snd_s1810c_switch_set, + .private_value = (SC1824C_STATE_MONO_SW | SC1824C_CTL_MONO_SW << 8) +}; + static const struct snd_kcontrol_new snd_s1810c_48v_sw = { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = "48V Phantom Power On Mic Inputs Switch", @@ -588,8 +599,17 @@ int snd_sc1810_init_mixer(struct usb_mixer_interface *mixer) if (ret < 0) return ret; - ret = snd_s1810c_switch_init(mixer, &snd_s1810c_ab_sw); - if (ret < 0) - return ret; + // The 1824c has a Mono Main switch instead of a + // A/B select switch. + if (mixer->chip->usb_id == USB_ID(0x194f, 0x010d)) { + ret = snd_s1810c_switch_init(mixer, &snd_s1824c_mono_sw); + if (ret < 0) + return ret; + } else if (mixer->chip->usb_id == USB_ID(0x194f, 0x010c)) { + ret = snd_s1810c_switch_init(mixer, &snd_s1810c_ab_sw); + if (ret < 0) + return ret; + } + return ret; } diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index 90480b9b9b089..316f1abefd51d 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -1279,6 +1279,8 @@ static int scarlett2_usb( struct scarlett2_usb_packet *req, *resp = NULL; size_t req_buf_size = struct_size(req, data, req_size); size_t resp_buf_size = struct_size(resp, data, resp_size); + int retries = 0; + const int max_retries = 5; int err; req = kmalloc(req_buf_size, GFP_KERNEL); @@ -1302,10 +1304,15 @@ static int scarlett2_usb( if (req_size) memcpy(req->data, req_data, req_size); +retry: err = scarlett2_usb_tx(dev, private->bInterfaceNumber, req, req_buf_size); if (err != req_buf_size) { + if (err == -EPROTO && ++retries <= max_retries) { + msleep(5 * (1 << (retries - 1))); + goto retry; + } usb_audio_err( mixer->chip, "%s USB request result cmd %x was %d\n", diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c old mode 100644 new mode 100755 index d9d4c5922a50b..7d747677b58da --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -2096,6 +2096,10 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_SET_IFACE_FIRST), DEVICE_FLG(0x0556, 0x0014, /* Phoenix Audio TMX320VC */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x0572, 0x1b08, /* Conexant Systems (Rockwell), Inc. */ + QUIRK_FLAG_MIXER_MIN_MUTE), + DEVICE_FLG(0x0572, 0x1b09, /* Conexant Systems (Rockwell), Inc. */ + QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x05a3, 0x9420, /* ELP HD USB Camera */ QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x05a7, 0x1020, /* Bose Companion 5 */ @@ -2138,16 +2142,26 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY_1M), DEVICE_FLG(0x0b0e, 0x0349, /* Jabra 550a */ QUIRK_FLAG_CTL_MSG_DELAY_1M), + DEVICE_FLG(0x0bda, 0x498a, /* Realtek Semiconductor Corp. */ + QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x0c45, 0x6340, /* Sonix HD USB Camera */ QUIRK_FLAG_GET_SAMPLE_RATE), - DEVICE_FLG(0x0d8c, 0x0014, /* USB Audio Device */ - QUIRK_FLAG_CTL_MSG_DELAY_1M), + DEVICE_FLG(0x0c45, 0x636b, /* Microdia JP001 USB Camera */ + QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x0d8c, 0x000c, /* C-Media */ + QUIRK_FLAG_MIXER_MIN_MUTE), + DEVICE_FLG(0x0d8c, 0x0014, /* C-Media */ + QUIRK_FLAG_CTL_MSG_DELAY_1M | QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x0ecb, 0x205c, /* JBL Quantum610 Wireless */ QUIRK_FLAG_FIXED_RATE), DEVICE_FLG(0x0ecb, 0x2069, /* JBL Quantum810 Wireless */ QUIRK_FLAG_FIXED_RATE), DEVICE_FLG(0x0fd9, 0x0008, /* Hauppauge HVR-950Q */ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), + DEVICE_FLG(0x1101, 0x0003, /* Audioengine D1 */ + QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x12d1, 0x3a07, /* Huawei Technologies Co., Ltd. */ + QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x1224, 0x2a25, /* Jieli Technology USB PHY 2.0 */ QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), DEVICE_FLG(0x1395, 0x740a, /* Sennheiser DECT */ @@ -2178,12 +2192,16 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_DISABLE_AUTOSUSPEND), DEVICE_FLG(0x17aa, 0x104d, /* Lenovo ThinkStation P620 Internal Speaker + Front Headset */ QUIRK_FLAG_DISABLE_AUTOSUSPEND), + DEVICE_FLG(0x17ef, 0x3083, /* Lenovo TBT3 dock */ + QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x1852, 0x5062, /* Luxman D-08u */ QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY), DEVICE_FLG(0x1852, 0x5065, /* Luxman DA-06 */ QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY), DEVICE_FLG(0x1901, 0x0191, /* GE B850V3 CP2114 audio interface */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x19f7, 0x0003, /* RODE NT-USB */ + QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x19f7, 0x0035, /* RODE NT-USB+ */ QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x1bcf, 0x2281, /* HD Webcam */ @@ -2234,6 +2252,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x2912, 0x30c8, /* Audioengine D1 */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x2a70, 0x1881, /* OnePlus Technology (Shenzhen) Co., Ltd. BE02T */ + QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x2b53, 0x0023, /* Fiero SC-01 (firmware v1.0.0 @ 48 kHz) */ QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x2b53, 0x0024, /* Fiero SC-01 (firmware v1.0.0 @ 96 kHz) */ @@ -2244,10 +2264,14 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY_1M), DEVICE_FLG(0x2d95, 0x8021, /* VIVO USB-C-XE710 HEADSET */ QUIRK_FLAG_CTL_MSG_DELAY_1M), + DEVICE_FLG(0x2d99, 0x0026, /* HECATE G2 GAMING HEADSET */ + QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x2fc6, 0xf0b7, /* iBasso DC07 Pro */ QUIRK_FLAG_CTL_MSG_DELAY_1M), DEVICE_FLG(0x30be, 0x0101, /* Schiit Hel */ QUIRK_FLAG_IGNORE_CTL_ERROR), + DEVICE_FLG(0x339b, 0x3a07, /* Synaptics HONOR USB-C HEADSET */ + QUIRK_FLAG_MIXER_MIN_MUTE), DEVICE_FLG(0x413c, 0xa506, /* Dell AE515 sound bar */ QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x534d, 0x0021, /* MacroSilicon MS2100/MS2106 */ @@ -2299,6 +2323,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x2d87, /* Cayin device */ QUIRK_FLAG_DSD_RAW), + VENDOR_FLG(0x2fc6, /* Comture-inc devices */ + QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x3336, /* HEM devices */ QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x3353, /* Khadas devices */ diff --git a/sound/usb/stream.c b/sound/usb/stream.c index e14c725acebf2..12a5e053ec54f 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -341,20 +341,28 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor len = le16_to_cpu(cluster->wLength); c = 0; - p += sizeof(struct uac3_cluster_header_descriptor); + p += sizeof(*cluster); + len -= sizeof(*cluster); - while (((p - (void *)cluster) < len) && (c < channels)) { + while (len > 0 && (c < channels)) { struct uac3_cluster_segment_descriptor *cs_desc = p; u16 cs_len; u8 cs_type; + if (len < sizeof(*cs_desc)) + break; cs_len = le16_to_cpu(cs_desc->wLength); + if (len < cs_len) + break; cs_type = cs_desc->bSegmentType; if (cs_type == UAC3_CHANNEL_INFORMATION) { struct uac3_cluster_information_segment_descriptor *is = p; unsigned char map; + if (cs_len < sizeof(*is)) + break; + /* * TODO: this conversion is not complete, update it * after adding UAC3 values to asound.h @@ -456,6 +464,7 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor chmap->map[c++] = map; } p += cs_len; + len -= cs_len; } if (channels < c) @@ -876,7 +885,7 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip, u64 badd_formats = 0; unsigned int num_channels; struct audioformat *fp; - u16 cluster_id, wLength; + u16 cluster_id, wLength, cluster_wLength; int clock = 0; int err; @@ -982,6 +991,8 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip, * and request Cluster Descriptor */ wLength = le16_to_cpu(hc_header.wLength); + if (wLength < sizeof(cluster)) + return NULL; cluster = kzalloc(wLength, GFP_KERNEL); if (!cluster) return ERR_PTR(-ENOMEM); @@ -1003,6 +1014,16 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip, return ERR_PTR(-EIO); } + cluster_wLength = le16_to_cpu(cluster->wLength); + if (cluster_wLength < sizeof(*cluster) || + cluster_wLength > wLength) { + dev_err(&dev->dev, + "%u:%d : invalid Cluster Descriptor size\n", + iface_no, altno); + kfree(cluster); + return ERR_PTR(-EIO); + } + num_channels = cluster->bNrChannels; chmap = convert_chmap_v3(cluster); kfree(cluster); diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index e9f1d102e53dc..6a85d9d9168a9 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -190,6 +190,9 @@ extern bool snd_usb_skip_validation; * for the given endpoint. * QUIRK_FLAG_MIC_RES_16 and QUIRK_FLAG_MIC_RES_384 * Set the fixed resolution for Mic Capture Volume (mostly for webcams) + * QUIRK_FLAG_MIXER_MIN_MUTE + * Set minimum volume control value as mute for devices where the lowest + * playback value represents muted state instead of minimum audible volume */ #define QUIRK_FLAG_GET_SAMPLE_RATE (1U << 0) @@ -216,5 +219,6 @@ extern bool snd_usb_skip_validation; #define QUIRK_FLAG_FIXED_RATE (1U << 21) #define QUIRK_FLAG_MIC_RES_16 (1U << 22) #define QUIRK_FLAG_MIC_RES_384 (1U << 23) +#define QUIRK_FLAG_MIXER_MIN_MUTE (1U << 24) #endif /* __USBAUDIO_H */ diff --git a/sound/usb/validate.c b/sound/usb/validate.c index 6fe206f6e9110..4bb4893f6e74f 100644 --- a/sound/usb/validate.c +++ b/sound/usb/validate.c @@ -221,6 +221,17 @@ static bool validate_uac3_feature_unit(const void *p, return d->bLength >= sizeof(*d) + 4 + 2; } +static bool validate_uac3_power_domain_unit(const void *p, + const struct usb_desc_validator *v) +{ + const struct uac3_power_domain_descriptor *d = p; + + if (d->bLength < sizeof(*d)) + return false; + /* baEntities[] + wPDomainDescrStr */ + return d->bLength >= sizeof(*d) + d->bNrEntities + 2; +} + static bool validate_midi_out_jack(const void *p, const struct usb_desc_validator *v) { @@ -255,7 +266,11 @@ static const struct usb_desc_validator audio_validators[] = { FUNC(UAC_VERSION_2, UAC_MIXER_UNIT, validate_mixer_unit), FUNC(UAC_VERSION_2, UAC_SELECTOR_UNIT, validate_selector_unit), FUNC(UAC_VERSION_2, UAC_FEATURE_UNIT, validate_uac2_feature_unit), - /* UAC_VERSION_2, UAC2_EFFECT_UNIT: not implemented yet */ + /* just a stop-gap, it should be a proper function for the array + * once if the unit is really parsed/used + */ + FIXED(UAC_VERSION_2, UAC2_EFFECT_UNIT, + struct uac2_effect_unit_descriptor), FUNC(UAC_VERSION_2, UAC2_PROCESSING_UNIT_V2, validate_processing_unit), FUNC(UAC_VERSION_2, UAC2_EXTENSION_UNIT_V2, validate_processing_unit), FIXED(UAC_VERSION_2, UAC2_CLOCK_SOURCE, @@ -274,8 +289,9 @@ static const struct usb_desc_validator audio_validators[] = { /* UAC_VERSION_3, UAC3_EXTENDED_TERMINAL: not implemented yet */ FUNC(UAC_VERSION_3, UAC3_MIXER_UNIT, validate_mixer_unit), FUNC(UAC_VERSION_3, UAC3_SELECTOR_UNIT, validate_selector_unit), - FUNC(UAC_VERSION_3, UAC_FEATURE_UNIT, validate_uac3_feature_unit), - /* UAC_VERSION_3, UAC3_EFFECT_UNIT: not implemented yet */ + FUNC(UAC_VERSION_3, UAC3_FEATURE_UNIT, validate_uac3_feature_unit), + FIXED(UAC_VERSION_3, UAC3_EFFECT_UNIT, + struct uac2_effect_unit_descriptor), /* sharing the same struct */ FUNC(UAC_VERSION_3, UAC3_PROCESSING_UNIT, validate_processing_unit), FUNC(UAC_VERSION_3, UAC3_EXTENSION_UNIT, validate_processing_unit), FIXED(UAC_VERSION_3, UAC3_CLOCK_SOURCE, @@ -285,6 +301,7 @@ static const struct usb_desc_validator audio_validators[] = { struct uac3_clock_multiplier_descriptor), /* UAC_VERSION_3, UAC3_SAMPLE_RATE_CONVERTER: not implemented yet */ /* UAC_VERSION_3, UAC3_CONNECTORS: not implemented yet */ + FUNC(UAC_VERSION_3, UAC3_POWER_DOMAIN, validate_uac3_power_domain_unit), { } /* terminator */ }; diff --git a/sound/x86/intel_hdmi_audio.c b/sound/x86/intel_hdmi_audio.c index ab95fb34a6358..7b9292cf839f2 100644 --- a/sound/x86/intel_hdmi_audio.c +++ b/sound/x86/intel_hdmi_audio.c @@ -1766,7 +1766,7 @@ static int __hdmi_lpe_audio_probe(struct platform_device *pdev) /* setup private data which can be retrieved when required */ pcm->private_data = ctx; pcm->info_flags = 0; - strscpy(pcm->name, card->shortname, strlen(card->shortname)); + strscpy(pcm->name, card->shortname, sizeof(pcm->name)); /* setup the ops for playback */ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &had_pcm_ops); diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index 1d111350197f3..76f9cad9fb62b 100644 --- a/tools/arch/x86/include/asm/msr-index.h +++ b/tools/arch/x86/include/asm/msr-index.h @@ -550,6 +550,7 @@ #define MSR_AMD64_OSVW_STATUS 0xc0010141 #define MSR_AMD_PPIN_CTL 0xc00102f0 #define MSR_AMD_PPIN 0xc00102f1 +#define MSR_AMD64_CPUID_FN_7 0xc0011002 #define MSR_AMD64_CPUID_FN_1 0xc0011004 #define MSR_AMD64_LS_CFG 0xc0011020 #define MSR_AMD64_DC_CFG 0xc0011022 diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index b7965dfff33a9..8c2644f3497e6 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 #define _GNU_SOURCE -#include +#include +#include #include +#include #include #include -#include #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define min(a, b) (((a) < (b)) ? (a) : (b)) @@ -156,14 +157,14 @@ static bool cpuid_store(struct cpuid_range *range, u32 f, int subleaf, if (!func->leafs) { func->leafs = malloc(sizeof(struct subleaf)); if (!func->leafs) - perror("malloc func leaf"); + err(EXIT_FAILURE, NULL); func->nr = 1; } else { s = func->nr; func->leafs = realloc(func->leafs, (s + 1) * sizeof(*leaf)); if (!func->leafs) - perror("realloc f->leafs"); + err(EXIT_FAILURE, NULL); func->nr++; } @@ -222,7 +223,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) range = malloc(sizeof(struct cpuid_range)); if (!range) - perror("malloc range"); + err(EXIT_FAILURE, NULL); if (input_eax & 0x80000000) range->is_ext = true; @@ -231,7 +232,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) range->funcs = malloc(sizeof(struct cpuid_func) * idx_func); if (!range->funcs) - perror("malloc range->funcs"); + err(EXIT_FAILURE, NULL); range->nr = idx_func; memset(range->funcs, 0, sizeof(struct cpuid_func) * idx_func); @@ -387,8 +388,8 @@ static int parse_line(char *line) return 0; err_exit: - printf("Warning: wrong line format:\n"); - printf("\tline[%d]: %s\n", flines, line); + warnx("Wrong line format:\n" + "\tline[%d]: %s", flines, line); return -1; } @@ -410,10 +411,8 @@ static void parse_text(void) file = fopen("./cpuid.csv", "r"); } - if (!file) { - printf("Fail to open '%s'\n", filename); - return; - } + if (!file) + err(EXIT_FAILURE, "%s", filename); while (1) { ret = getline(&line, &len, file); @@ -521,7 +520,7 @@ static inline struct cpuid_func *index_to_func(u32 index) func_idx = index & 0xffff; if ((func_idx + 1) > (u32)range->nr) { - printf("ERR: invalid input index (0x%x)\n", index); + warnx("Invalid input index (0x%x)", index); return NULL; } return &range->funcs[func_idx]; @@ -553,7 +552,7 @@ static void show_info(void) return; } - printf("ERR: invalid input subleaf (0x%x)\n", user_sub); + warnx("Invalid input subleaf (0x%x)", user_sub); } show_func(func); @@ -584,15 +583,15 @@ static void setup_platform_cpuid(void) static void usage(void) { - printf("kcpuid [-abdfhr] [-l leaf] [-s subleaf]\n" - "\t-a|--all Show both bit flags and complex bit fields info\n" - "\t-b|--bitflags Show boolean flags only\n" - "\t-d|--detail Show details of the flag/fields (default)\n" - "\t-f|--flags Specify the cpuid csv file\n" - "\t-h|--help Show usage info\n" - "\t-l|--leaf=index Specify the leaf you want to check\n" - "\t-r|--raw Show raw cpuid data\n" - "\t-s|--subleaf=sub Specify the subleaf you want to check\n" + warnx("kcpuid [-abdfhr] [-l leaf] [-s subleaf]\n" + "\t-a|--all Show both bit flags and complex bit fields info\n" + "\t-b|--bitflags Show boolean flags only\n" + "\t-d|--detail Show details of the flag/fields (default)\n" + "\t-f|--flags Specify the CPUID CSV file\n" + "\t-h|--help Show usage info\n" + "\t-l|--leaf=index Specify the leaf you want to check\n" + "\t-r|--raw Show raw CPUID data\n" + "\t-s|--subleaf=sub Specify the subleaf you want to check" ); } @@ -643,7 +642,7 @@ static int parse_options(int argc, char *argv[]) user_sub = strtoul(optarg, NULL, 0); break; default: - printf("%s: Invalid option '%c'\n", argv[0], optopt); + warnx("Invalid option '%c'", optopt); return -1; } diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 1b7f697146041..42c7b70f6996c 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -38,7 +38,7 @@ static int dump_prog_id_as_func_ptr(const struct btf_dumper *d, __u32 info_len = sizeof(info); const char *prog_name = NULL; struct btf *prog_btf = NULL; - struct bpf_func_info finfo; + struct bpf_func_info finfo = {}; __u32 finfo_rec_size; char prog_str[1024]; int err; diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 9b75639434b81..0a764426d9358 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -461,10 +461,11 @@ int get_fd_type(int fd) p_err("can't read link type: %s", strerror(errno)); return -1; } - if (n == sizeof(path)) { + if (n == sizeof(buf)) { p_err("can't read link type: path too long!"); return -1; } + buf[n] = '\0'; if (strstr(buf, "bpf-map")) return BPF_OBJ_MAP; diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 08d0ac543c674..a0536528dfde2 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -534,9 +534,9 @@ int main(int argc, char **argv) usage(); if (version_requested) - return do_version(argc, argv); - - ret = cmd_select(commands, argc, argv, do_help); + ret = do_version(argc, argv); + else + ret = cmd_select(commands, argc, argv, do_help); if (json_output) jsonw_destroy(&json_wtr); diff --git a/tools/bpf/bpftool/net.c b/tools/bpf/bpftool/net.c index 28e9417a5c2e3..c2ca82fc21e21 100644 --- a/tools/bpf/bpftool/net.c +++ b/tools/bpf/bpftool/net.c @@ -360,17 +360,18 @@ static int dump_link_nlmsg(void *cookie, void *msg, struct nlattr **tb) { struct bpf_netdev_t *netinfo = cookie; struct ifinfomsg *ifinfo = msg; + struct ip_devname_ifindex *tmp; if (netinfo->filter_idx > 0 && netinfo->filter_idx != ifinfo->ifi_index) return 0; if (netinfo->used_len == netinfo->array_len) { - netinfo->devices = realloc(netinfo->devices, - (netinfo->array_len + 16) * - sizeof(struct ip_devname_ifindex)); - if (!netinfo->devices) + tmp = realloc(netinfo->devices, + (netinfo->array_len + 16) * sizeof(struct ip_devname_ifindex)); + if (!tmp) return -ENOMEM; + netinfo->devices = tmp; netinfo->array_len += 16; } netinfo->devices[netinfo->used_len].ifindex = ifinfo->ifi_index; @@ -389,6 +390,7 @@ static int dump_class_qdisc_nlmsg(void *cookie, void *msg, struct nlattr **tb) { struct bpf_tcinfo_t *tcinfo = cookie; struct tcmsg *info = msg; + struct tc_kind_handle *tmp; if (tcinfo->is_qdisc) { /* skip clsact qdisc */ @@ -400,11 +402,12 @@ static int dump_class_qdisc_nlmsg(void *cookie, void *msg, struct nlattr **tb) } if (tcinfo->used_len == tcinfo->array_len) { - tcinfo->handle_array = realloc(tcinfo->handle_array, + tmp = realloc(tcinfo->handle_array, (tcinfo->array_len + 16) * sizeof(struct tc_kind_handle)); - if (!tcinfo->handle_array) + if (!tmp) return -ENOMEM; + tcinfo->handle_array = tmp; tcinfo->array_len += 16; } tcinfo->handle_array[tcinfo->used_len].handle = info->tcm_handle; diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 174e076e56af2..90fa7a8d0f4b3 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -2203,7 +2203,7 @@ static void profile_print_readings(void) static char *profile_target_name(int tgt_fd) { - struct bpf_func_info func_info; + struct bpf_func_info func_info = {}; struct bpf_prog_info info = {}; __u32 info_len = sizeof(info); const struct btf_type *t; diff --git a/tools/bpf/resolve_btfids/Makefile b/tools/bpf/resolve_btfids/Makefile index 4b8079f294f65..b0072e64b0102 100644 --- a/tools/bpf/resolve_btfids/Makefile +++ b/tools/bpf/resolve_btfids/Makefile @@ -19,7 +19,7 @@ endif # Overrides for the prepare step libraries. HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \ - CROSS_COMPILE="" EXTRA_CFLAGS="$(HOSTCFLAGS)" + CROSS_COMPILE="" CLANG_CROSS_FLAGS="" EXTRA_CFLAGS="$(HOSTCFLAGS)" RM ?= rm HOSTCC ?= gcc diff --git a/tools/build/Makefile.build b/tools/build/Makefile.build index fac42486a8cf0..27f4ee9cb4db4 100644 --- a/tools/build/Makefile.build +++ b/tools/build/Makefile.build @@ -141,6 +141,10 @@ objprefix := $(subst ./,,$(OUTPUT)$(dir)/) obj-y := $(addprefix $(objprefix),$(obj-y)) subdir-obj-y := $(addprefix $(objprefix),$(subdir-obj-y)) +# Separate out test log files from real build objects. +test-y := $(filter %_log, $(obj-y)) +obj-y := $(filter-out %_log, $(obj-y)) + # Final '$(obj)-in.o' object in-target := $(objprefix)$(obj)-in.o @@ -151,7 +155,7 @@ $(subdir-y): $(sort $(subdir-obj-y)): $(subdir-y) ; -$(in-target): $(obj-y) FORCE +$(in-target): $(obj-y) $(test-y) FORCE $(call rule_mkdir) $(call if_changed,$(host)ld_multi) diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index dad79ede4e0ae..17e03236e3461 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -80,7 +80,30 @@ FILES= \ FILES := $(addprefix $(OUTPUT),$(FILES)) -PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config +# Some distros provide the command $(CROSS_COMPILE)pkg-config for +# searching packges installed with Multiarch. Use it for cross +# compilation if it is existed. +ifneq (, $(shell which $(CROSS_COMPILE)pkg-config)) + PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config +else + PKG_CONFIG ?= pkg-config + + # PKG_CONFIG_PATH or PKG_CONFIG_LIBDIR, alongside PKG_CONFIG_SYSROOT_DIR + # for modified system root, are required for the cross compilation. + # If these PKG_CONFIG environment variables are not set, Multiarch library + # paths are used instead. + ifdef CROSS_COMPILE + ifeq ($(PKG_CONFIG_LIBDIR)$(PKG_CONFIG_PATH)$(PKG_CONFIG_SYSROOT_DIR),) + CROSS_ARCH = $(shell $(CC) -dumpmachine) + PKG_CONFIG_LIBDIR := /usr/local/$(CROSS_ARCH)/lib/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/local/lib/$(CROSS_ARCH)/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/lib/$(CROSS_ARCH)/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/local/share/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/share/pkgconfig/ + export PKG_CONFIG_LIBDIR + endif + endif +endif all: $(FILES) @@ -283,10 +306,10 @@ $(OUTPUT)test-libbabeltrace.bin: $(BUILD) # -lbabeltrace provided by $(FEATURE_CHECK_LDFLAGS-libbabeltrace) $(OUTPUT)test-compile-32.bin: - $(CC) -m32 -o $@ test-compile.c + $(CC) -m32 -Wall -Werror -o $@ test-compile.c $(OUTPUT)test-compile-x32.bin: - $(CC) -mx32 -o $@ test-compile.c + $(CC) -mx32 -Wall -Werror -o $@ test-compile.c $(OUTPUT)test-zlib.bin: $(BUILD) -lz diff --git a/tools/gpio/Makefile b/tools/gpio/Makefile index d29c9c49e2512..342e056c8c665 100644 --- a/tools/gpio/Makefile +++ b/tools/gpio/Makefile @@ -77,8 +77,8 @@ $(OUTPUT)gpio-watch: $(GPIO_WATCH_IN) clean: rm -f $(ALL_PROGRAMS) - rm -f $(OUTPUT)include/linux/gpio.h - find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + rm -rf $(OUTPUT)include + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete -o -name '\.*.cmd' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/include/linux/bitmap.h b/tools/include/linux/bitmap.h index 210c13b1b8570..926f3117d8e4f 100644 --- a/tools/include/linux/bitmap.h +++ b/tools/include/linux/bitmap.h @@ -3,6 +3,7 @@ #define _TOOLS_LINUX_BITMAP_H #include +#include #include #include #include diff --git a/tools/include/linux/kallsyms.h b/tools/include/linux/kallsyms.h index 5a37ccbec54fb..f61a01dd7eb7c 100644 --- a/tools/include/linux/kallsyms.h +++ b/tools/include/linux/kallsyms.h @@ -18,6 +18,7 @@ static inline const char *kallsyms_lookup(unsigned long addr, return NULL; } +#ifdef HAVE_BACKTRACE_SUPPORT #include #include static inline void print_ip_sym(const char *loglvl, unsigned long ip) @@ -30,5 +31,8 @@ static inline void print_ip_sym(const char *loglvl, unsigned long ip) free(name); } +#else +static inline void print_ip_sym(const char *loglvl, unsigned long ip) {} +#endif #endif diff --git a/tools/include/nolibc/std.h b/tools/include/nolibc/std.h index 933bc0be7e1c6..f24953f8b949c 100644 --- a/tools/include/nolibc/std.h +++ b/tools/include/nolibc/std.h @@ -20,6 +20,8 @@ #include "stdint.h" +#include + /* those are commonly provided by sys/types.h */ typedef unsigned int dev_t; typedef unsigned long ino_t; @@ -31,6 +33,6 @@ typedef unsigned long nlink_t; typedef signed long off_t; typedef signed long blksize_t; typedef signed long blkcnt_t; -typedef signed long time_t; +typedef __kernel_time_t time_t; #endif /* _NOLIBC_STD_H */ diff --git a/tools/include/nolibc/types.h b/tools/include/nolibc/types.h index 8cfc4c860fa44..48dca7b188d06 100644 --- a/tools/include/nolibc/types.h +++ b/tools/include/nolibc/types.h @@ -128,7 +128,7 @@ typedef struct { int __fd = (fd); \ if (__fd >= 0) \ __set->fds[__fd / FD_SETIDXMASK] &= \ - ~(1U << (__fd & FX_SETBITMASK)); \ + ~(1U << (__fd & FD_SETBITMASK)); \ } while (0) #define FD_SET(fd, set) do { \ @@ -145,7 +145,7 @@ typedef struct { int __r = 0; \ if (__fd >= 0) \ __r = !!(__set->fds[__fd / FD_SETIDXMASK] & \ -1U << (__fd & FD_SET_BITMASK)); \ +1U << (__fd & FD_SETBITMASK)); \ __r; \ }) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index ba6e346c8d669..6bd8768b8fe6f 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1140,6 +1140,7 @@ enum bpf_perf_event_type { #define BPF_F_BEFORE (1U << 3) #define BPF_F_AFTER (1U << 4) #define BPF_F_ID (1U << 5) +#define BPF_F_PREORDER (1U << 6) #define BPF_F_LINK BPF_F_LINK /* 1 << 13 */ /* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the @@ -1912,6 +1913,7 @@ union bpf_attr { * for updates resulting in a null checksum the value is set to * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates * the checksum is to be computed against a pseudo-header. + * Flag **BPF_F_IPV6** should be set for IPv6 packets. * * This helper works in combination with **bpf_csum_diff**\ (), * which does not update the checksum in-place, but offers more @@ -5919,6 +5921,7 @@ enum { BPF_F_PSEUDO_HDR = (1ULL << 4), BPF_F_MARK_MANGLED_0 = (1ULL << 5), BPF_F_MARK_ENFORCE = (1ULL << 6), + BPF_F_IPV6 = (1ULL << 7), }; /* BPF_FUNC_clone_redirect and BPF_FUNC_redirect flags. */ diff --git a/tools/include/uapi/linux/if_link.h b/tools/include/uapi/linux/if_link.h index 39e659c83cfd2..cb8b0a3029d3d 100644 --- a/tools/include/uapi/linux/if_link.h +++ b/tools/include/uapi/linux/if_link.h @@ -865,6 +865,7 @@ enum { IFLA_BOND_AD_LACP_ACTIVE, IFLA_BOND_MISSED_MAX, IFLA_BOND_NS_IP6_TARGET, + IFLA_BOND_COUPLED_CONTROL, __IFLA_BOND_MAX, }; diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h index e2b9e8415c044..b3fc384595a9f 100644 --- a/tools/lib/bpf/bpf_core_read.h +++ b/tools/lib/bpf/bpf_core_read.h @@ -312,7 +312,13 @@ enum bpf_enum_value_kind { #define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j #define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__) +#if defined(__clang__) && (__clang_major__ >= 19) +#define ___type(...) __typeof_unqual__(___arrow(__VA_ARGS__)) +#elif defined(__GNUC__) && (__GNUC__ >= 14) +#define ___type(...) __typeof_unqual__(___arrow(__VA_ARGS__)) +#else #define ___type(...) typeof(___arrow(__VA_ARGS__)) +#endif #define ___read(read_fn, dst, src_type, src, accessor) \ read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor) diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h index 1c13f8e88833b..66b925bd954eb 100644 --- a/tools/lib/bpf/bpf_tracing.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -311,7 +311,7 @@ struct pt_regs___arm64 { #define __PT_RET_REG regs[31] #define __PT_FP_REG __unsupported__ #define __PT_RC_REG gpr[3] -#define __PT_SP_REG sp +#define __PT_SP_REG gpr[1] #define __PT_IP_REG nip #elif defined(bpf_target_sparc) diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 8484b563b53d0..2e9f28cece3ff 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -3922,6 +3922,19 @@ static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id return true; } +static bool btf_dedup_identical_ptrs(struct btf_dedup *d, __u32 id1, __u32 id2) +{ + struct btf_type *t1, *t2; + + t1 = btf_type_by_id(d->btf, id1); + t2 = btf_type_by_id(d->btf, id2); + + if (!btf_is_ptr(t1) || !btf_is_ptr(t2)) + return false; + + return t1->type == t2->type; +} + /* * Check equivalence of BTF type graph formed by candidate struct/union (we'll * call it "candidate graph" in this description for brevity) to a type graph @@ -4054,6 +4067,9 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, */ if (btf_dedup_identical_structs(d, hypot_type_id, cand_id)) return 1; + /* A similar case is again observed for PTRs. */ + if (btf_dedup_identical_ptrs(d, hypot_type_id, cand_id)) + return 1; return 0; } diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index ebf56d21d08ee..cf4db51b99eb5 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -225,6 +225,9 @@ static void btf_dump_free_names(struct hashmap *map) size_t bkt; struct hashmap_entry *cur; + if (!map) + return; + hashmap__for_each_entry(map, cur, bkt) free((void *)cur->pkey); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 2fad178949efe..95f2ffefff3ff 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -246,7 +246,7 @@ void libbpf_print(enum libbpf_print_level level, const char *format, ...) old_errno = errno; va_start(args, format); - __libbpf_pr(level, format, args); + print_fn(level, format, args); va_end(args); errno = old_errno; @@ -554,7 +554,7 @@ struct extern_desc { int sym_idx; int btf_id; int sec_btf_id; - const char *name; + char *name; char *essent_name; bool is_set; bool is_weak; @@ -838,7 +838,7 @@ bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, return -LIBBPF_ERRNO__FORMAT; } - if (sec_off + prog_sz > sec_sz) { + if (sec_off + prog_sz > sec_sz || sec_off + prog_sz < sec_off) { pr_warn("sec '%s': program at offset %zu crosses section boundary\n", sec_name, sec_off); return -LIBBPF_ERRNO__FORMAT; @@ -1802,7 +1802,7 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, } len = strlen(value); - if (value[len - 1] != '"') { + if (len < 2 || value[len - 1] != '"') { pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", ext->name, value); return -EINVAL; @@ -3822,7 +3822,9 @@ static int bpf_object__collect_externs(struct bpf_object *obj) return ext->btf_id; } t = btf__type_by_id(obj->btf, ext->btf_id); - ext->name = btf__name_by_offset(obj->btf, t->name_off); + ext->name = strdup(btf__name_by_offset(obj->btf, t->name_off)); + if (!ext->name) + return -ENOMEM; ext->sym_idx = i; ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; @@ -5011,6 +5013,16 @@ static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) return false; } + /* + * bpf_get_map_info_by_fd() for DEVMAP will always return flags with + * BPF_F_RDONLY_PROG set, but it generally is not set at map creation time. + * Thus, ignore the BPF_F_RDONLY_PROG flag in the flags returned from + * bpf_get_map_info_by_fd() when checking for compatibility with an + * existing DEVMAP. + */ + if (map->def.type == BPF_MAP_TYPE_DEVMAP || map->def.type == BPF_MAP_TYPE_DEVMAP_HASH) + map_info.map_flags &= ~BPF_F_RDONLY_PROG; + return (map_info.type == map->def.type && map_info.key_size == map->def.key_size && map_info.value_size == map->def.value_size && @@ -8457,8 +8469,10 @@ void bpf_object__close(struct bpf_object *obj) zfree(&obj->btf_custom_path); zfree(&obj->kconfig); - for (i = 0; i < obj->nr_extern; i++) + for (i = 0; i < obj->nr_extern; i++) { + zfree(&obj->externs[i].name); zfree(&obj->externs[i].essent_name); + } zfree(&obj->externs); obj->nr_extern = 0; @@ -12453,7 +12467,6 @@ struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, attr.config = PERF_COUNT_SW_BPF_OUTPUT; attr.type = PERF_TYPE_SOFTWARE; attr.sample_type = PERF_SAMPLE_RAW; - attr.sample_period = sample_period; attr.wakeup_events = sample_period; p.attr = &attr; diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index a3a190d13db8a..e1b3136643aa8 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -1187,7 +1187,7 @@ static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj } else { if (!secs_match(dst_sec, src_sec)) { pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); - return -1; + return -EINVAL; } /* "license" and "version" sections are deduped */ @@ -2034,7 +2034,7 @@ static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *ob } } else if (!secs_match(dst_sec, src_sec)) { pr_warn("sections %s are not compatible\n", src_sec->sec_name); - return -1; + return -EINVAL; } /* shdr->sh_link points to SYMTAB */ diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c index 975e265eab3bf..06663f9ea581f 100644 --- a/tools/lib/bpf/nlattr.c +++ b/tools/lib/bpf/nlattr.c @@ -63,16 +63,16 @@ static int validate_nla(struct nlattr *nla, int maxtype, minlen = nla_attr_minlen[pt->type]; if (libbpf_nla_len(nla) < minlen) - return -1; + return -EINVAL; if (pt->maxlen && libbpf_nla_len(nla) > pt->maxlen) - return -1; + return -EINVAL; if (pt->type == LIBBPF_NLA_STRING) { char *data = libbpf_nla_data(nla); if (data[libbpf_nla_len(nla) - 1] != '\0') - return -1; + return -EINVAL; } return 0; @@ -118,19 +118,18 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, if (policy) { err = validate_nla(nla, maxtype, policy); if (err < 0) - goto errout; + return err; } - if (tb[type]) + if (tb[type]) { pr_warn("Attribute of type %#x found multiple times in message, " "previous attribute is being ignored.\n", type); + } tb[type] = nla; } - err = 0; -errout: - return err; + return 0; } /** diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index ae64090184d32..b50960cce23f2 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -285,6 +285,7 @@ struct perf_record_header_event_type { struct perf_record_header_tracing_data { struct perf_event_header header; __u32 size; + __u32 pad; }; #define PERF_RECORD_MISC_BUILD_ID_SIZE (1 << 15) diff --git a/tools/lib/subcmd/help.c b/tools/lib/subcmd/help.c index 8561b0f01a247..ddaeb4eb3e249 100644 --- a/tools/lib/subcmd/help.c +++ b/tools/lib/subcmd/help.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "subcmd-util.h" #include "help.h" #include "exec-cmd.h" @@ -74,6 +75,9 @@ void exclude_cmds(struct cmdnames *cmds, struct cmdnames *excludes) size_t ci, cj, ei; int cmp; + if (!excludes->cnt) + return; + ci = cj = ei = 0; while (ci < cmds->cnt && ei < excludes->cnt) { cmp = strcmp(cmds->names[ci]->name, excludes->names[ei]->name); @@ -82,10 +86,11 @@ void exclude_cmds(struct cmdnames *cmds, struct cmdnames *excludes) ci++; cj++; } else { - zfree(&cmds->names[cj]); - cmds->names[cj++] = cmds->names[ci++]; + cmds->names[cj++] = cmds->names[ci]; + cmds->names[ci++] = NULL; } } else if (cmp == 0) { + zfree(&cmds->names[ci]); ci++; ei++; } else if (cmp > 0) { @@ -94,12 +99,12 @@ void exclude_cmds(struct cmdnames *cmds, struct cmdnames *excludes) } if (ci != cj) { while (ci < cmds->cnt) { - zfree(&cmds->names[cj]); - cmds->names[cj++] = cmds->names[ci++]; + cmds->names[cj++] = cmds->names[ci]; + cmds->names[ci++] = NULL; } } for (ci = cj; ci < cmds->cnt; ci++) - zfree(&cmds->names[ci]); + assert(cmds->names[ci] == NULL); cmds->cnt = cj; } diff --git a/tools/lib/thermal/Makefile b/tools/lib/thermal/Makefile index 8890fd57b110c..8d21ea1950a31 100644 --- a/tools/lib/thermal/Makefile +++ b/tools/lib/thermal/Makefile @@ -59,8 +59,12 @@ else CFLAGS := -g -Wall endif +NL3_CFLAGS = $(shell pkg-config --cflags libnl-3.0 2>/dev/null) +ifeq ($(NL3_CFLAGS),) +NL3_CFLAGS = -I/usr/include/libnl3 +endif + INCLUDES = \ --I/usr/include/libnl3 \ -I$(srctree)/tools/lib/thermal/include \ -I$(srctree)/tools/lib/ \ -I$(srctree)/tools/include \ @@ -72,6 +76,7 @@ INCLUDES = \ override CFLAGS += $(EXTRA_WARNINGS) override CFLAGS += -Werror -Wall override CFLAGS += -fPIC +override CFLAGS += $(NL3_CFLAGS) override CFLAGS += $(INCLUDES) override CFLAGS += -fvisibility=hidden override CFGLAS += -Wl,-L. @@ -147,7 +152,7 @@ endef install_lib: libs $(call QUIET_INSTALL, $(LIBTHERMAL_ALL)) \ $(call do_install_mkdir,$(libdir_SQ)); \ - cp -fpR $(LIBTHERMAL_ALL) $(DESTDIR)$(libdir_SQ) + cp -fR --preserve=mode,timestamp $(LIBTHERMAL_ALL) $(DESTDIR)$(libdir_SQ) install_headers: $(call QUIET_INSTALL, headers) \ diff --git a/tools/net/ynl/ethtool.py b/tools/net/ynl/ethtool.py index 6c9f7e31250cd..ffd8eb6d19048 100755 --- a/tools/net/ynl/ethtool.py +++ b/tools/net/ynl/ethtool.py @@ -320,20 +320,37 @@ def main(): return if args.show_time_stamping: - tsinfo = dumpit(ynl, args, 'tsinfo-get') + req = { + 'header': { + 'flags': 'stats', + }, + } + + tsinfo = dumpit(ynl, args, 'tsinfo-get', req) print(f'Time stamping parameters for {args.device}:') print('Capabilities:') [print(f'\t{v}') for v in bits_to_dict(tsinfo['timestamping'])] - print(f'PTP Hardware Clock: {tsinfo["phc-index"]}') + print(f'PTP Hardware Clock: {tsinfo.get("phc-index", "none")}') + + if 'tx-types' in tsinfo: + print('Hardware Transmit Timestamp Modes:') + [print(f'\t{v}') for v in bits_to_dict(tsinfo['tx-types'])] + else: + print('Hardware Transmit Timestamp Modes: none') + + if 'rx-filters' in tsinfo: + print('Hardware Receive Filter Modes:') + [print(f'\t{v}') for v in bits_to_dict(tsinfo['rx-filters'])] + else: + print('Hardware Receive Filter Modes: none') - print('Hardware Transmit Timestamp Modes:') - [print(f'\t{v}') for v in bits_to_dict(tsinfo['tx-types'])] + if 'stats' in tsinfo and tsinfo['stats']: + print('Statistics:') + [print(f'\t{k}: {v}') for k, v in tsinfo['stats'].items()] - print('Hardware Receive Filter Modes:') - [print(f'\t{v}') for v in bits_to_dict(tsinfo['rx-filters'])] return print(f'Settings for {args.device}:') diff --git a/tools/net/ynl/lib/ynl.c b/tools/net/ynl/lib/ynl.c index ae61ae5b02bf8..0871f86c6b666 100644 --- a/tools/net/ynl/lib/ynl.c +++ b/tools/net/ynl/lib/ynl.c @@ -368,7 +368,7 @@ int ynl_attr_validate(struct ynl_parse_arg *yarg, const struct nlattr *attr) "Invalid attribute (binary %s)", policy->name); return -1; case YNL_PT_NUL_STR: - if ((!policy->len || len <= policy->len) && !data[len - 1]) + if (len && (!policy->len || len <= policy->len) && !data[len - 1]) break; yerr(yarg->ys, YNL_ERROR_ATTR_INVALID, "Invalid attribute (string %s)", policy->name); diff --git a/tools/objtool/check.c b/tools/objtool/check.c index b87b75428ed1a..43550bc9eacd2 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -3287,7 +3287,7 @@ static int handle_insn_ops(struct instruction *insn, if (update_cfi_state(insn, next_insn, &state->cfi, op)) return 1; - if (!insn->alt_group) + if (!opts.uaccess || !insn->alt_group) continue; if (op->dest.type == OP_DEST_PUSHF) { @@ -3754,6 +3754,9 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, return 0; case INSN_STAC: + if (!opts.uaccess) + break; + if (state.uaccess) { WARN_INSN(insn, "recursive UACCESS enable"); return 1; @@ -3763,6 +3766,9 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, break; case INSN_CLAC: + if (!opts.uaccess) + break; + if (!state.uaccess && func) { WARN_INSN(insn, "redundant UACCESS disable"); return 1; @@ -4238,7 +4244,8 @@ static int validate_symbol(struct objtool_file *file, struct section *sec, if (!insn || insn->ignore || insn->visited) return 0; - state->uaccess = sym->uaccess_safe; + if (opts.uaccess) + state->uaccess = sym->uaccess_safe; ret = validate_branch(file, insn_func(insn), insn, *state); if (ret) @@ -4686,8 +4693,10 @@ int check(struct objtool_file *file) init_cfi_state(&force_undefined_cfi); force_undefined_cfi.force_undefined = true; - if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) + if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) { + ret = -1; goto out; + } cfi_hash_add(&init_cfi); cfi_hash_add(&func_cfi); @@ -4704,7 +4713,7 @@ int check(struct objtool_file *file) if (opts.retpoline) { ret = validate_retpoline(file); if (ret < 0) - return ret; + goto out; warnings += ret; } @@ -4740,7 +4749,7 @@ int check(struct objtool_file *file) */ ret = validate_unrets(file); if (ret < 0) - return ret; + goto out; warnings += ret; } @@ -4803,7 +4812,7 @@ int check(struct objtool_file *file) if (opts.prefix) { ret = add_prefix_symbols(file); if (ret < 0) - return ret; + goto out; warnings += ret; } diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore index f533e76fb4800..5a8617eb54195 100644 --- a/tools/perf/.gitignore +++ b/tools/perf/.gitignore @@ -45,7 +45,5 @@ libbpf/ libperf/ libsubcmd/ libsymbol/ -libtraceevent/ -libtraceevent_plugins/ fixdep Documentation/doc.dep diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index d66b52407e19c..9da9f878f50f7 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -554,6 +554,8 @@ ifndef NO_LIBELF ifeq ($(feature-libdebuginfod), 1) CFLAGS += -DHAVE_DEBUGINFOD_SUPPORT EXTLIBS += -ldebuginfod + else + $(warning No elfutils/debuginfod.h found, no debuginfo server support, please install libdebuginfod-dev/elfutils-debuginfod-client-devel or equivalent) endif endif diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index b97224a8a65b9..ff72a0d9dea0c 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -188,7 +188,32 @@ HOSTLD ?= ld HOSTAR ?= ar CLANG ?= clang -PKG_CONFIG = $(CROSS_COMPILE)pkg-config +# Some distros provide the command $(CROSS_COMPILE)pkg-config for +# searching packges installed with Multiarch. Use it for cross +# compilation if it is existed. +ifneq (, $(shell which $(CROSS_COMPILE)pkg-config)) + PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config +else + PKG_CONFIG ?= pkg-config + + # PKG_CONFIG_PATH or PKG_CONFIG_LIBDIR, alongside PKG_CONFIG_SYSROOT_DIR + # for modified system root, is required for the cross compilation. + # If these PKG_CONFIG environment variables are not set, Multiarch library + # paths are used instead. + ifdef CROSS_COMPILE + ifeq ($(PKG_CONFIG_LIBDIR)$(PKG_CONFIG_PATH)$(PKG_CONFIG_SYSROOT_DIR),) + CROSS_ARCH = $(shell $(CC) -dumpmachine) + PKG_CONFIG_LIBDIR := /usr/local/$(CROSS_ARCH)/lib/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/local/lib/$(CROSS_ARCH)/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/lib/$(CROSS_ARCH)/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/local/share/pkgconfig/ + PKG_CONFIG_LIBDIR := $(PKG_CONFIG_LIBDIR):/usr/share/pkgconfig/ + export PKG_CONFIG_LIBDIR + $(warning Missing PKG_CONFIG_LIBDIR, PKG_CONFIG_PATH and PKG_CONFIG_SYSROOT_DIR for cross compilation,) + $(warning set PKG_CONFIG_LIBDIR for using Multiarch libs.) + endif + endif +endif RM = rm -f LN = ln -f diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index b94ae33a343c2..81f77c0505fde 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -3427,7 +3427,7 @@ static struct option __record_options[] = { "sample selected machine registers on interrupt," " use '-I?' to list register names", parse_intr_regs), OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register", - "sample selected machine registers on interrupt," + "sample selected machine registers in user space," " use '--user-regs=?' to list register names", parse_user_regs), OPT_BOOLEAN(0, "running-time", &record.opts.running_time, "Record running/enabled time of read (:S) events"), diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index ac9d94dbbeefa..16cb7278fabaf 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -1125,6 +1125,21 @@ add_sched_in_event(struct work_atoms *atoms, u64 timestamp) atoms->nb_atoms++; } +static void free_work_atoms(struct work_atoms *atoms) +{ + struct work_atom *atom, *tmp; + + if (atoms == NULL) + return; + + list_for_each_entry_safe(atom, tmp, &atoms->work_list, list) { + list_del(&atom->list); + free(atom); + } + thread__zput(atoms->thread); + free(atoms); +} + static int latency_switch_event(struct perf_sched *sched, struct evsel *evsel, struct perf_sample *sample, @@ -1949,6 +1964,16 @@ static u64 evsel__get_time(struct evsel *evsel, u32 cpu) return r->last_time[cpu]; } +static void timehist__evsel_priv_destructor(void *priv) +{ + struct evsel_runtime *r = priv; + + if (r) { + free(r->last_time); + free(r); + } +} + static int comm_width = 30; static char *timehist_get_commstr(struct thread *thread) @@ -3080,6 +3105,8 @@ static int perf_sched__timehist(struct perf_sched *sched) setup_pager(); + evsel__set_priv_destructor(timehist__evsel_priv_destructor); + /* prefer sched_waking if it is captured */ if (evlist__find_tracepoint_by_name(session->evlist, "sched:sched_waking")) handlers[1].handler = timehist_sched_wakeup_ignore; @@ -3180,13 +3207,13 @@ static void __merge_work_atoms(struct rb_root_cached *root, struct work_atoms *d this->total_runtime += data->total_runtime; this->nb_atoms += data->nb_atoms; this->total_lat += data->total_lat; - list_splice(&data->work_list, &this->work_list); + list_splice_init(&data->work_list, &this->work_list); if (this->max_lat < data->max_lat) { this->max_lat = data->max_lat; this->max_lat_start = data->max_lat_start; this->max_lat_end = data->max_lat_end; } - zfree(&data); + free_work_atoms(data); return; } } @@ -3265,7 +3292,6 @@ static int perf_sched__lat(struct perf_sched *sched) work_list = rb_entry(next, struct work_atoms, node); output_lat_thread(sched, work_list); next = rb_next(next); - thread__zput(work_list->thread); } printf(" -----------------------------------------------------------------------------------------------------------------\n"); @@ -3279,6 +3305,13 @@ static int perf_sched__lat(struct perf_sched *sched) rc = 0; + while ((next = rb_first_cached(&sched->sorted_atom_root))) { + struct work_atoms *data; + + data = rb_entry(next, struct work_atoms, node); + rb_erase_cached(next, &sched->sorted_atom_root); + free_work_atoms(data); + } out_free_cpus_switch_event: free_cpus_switch_event(sched); return rc; @@ -3678,6 +3711,8 @@ int cmd_sched(int argc, const char **argv) if (!argc) usage_with_options(sched_usage, sched_options); + thread__set_priv_destructor(free); + /* * Aliased to 'perf script' for now: */ diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 9692ebdd7f11e..1512fedd90cf9 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -638,8 +638,7 @@ static enum counter_recovery stat_handle_error(struct evsel *counter) * (behavior changed with commit b0a873e). */ if (errno == EINVAL || errno == ENOSYS || - errno == ENOENT || errno == EOPNOTSUPP || - errno == ENXIO) { + errno == ENOENT || errno == ENXIO) { if (verbose > 0) ui__warning("%s event is not supported by the kernel.\n", evsel__name(counter)); @@ -657,7 +656,7 @@ static enum counter_recovery stat_handle_error(struct evsel *counter) if (verbose > 0) ui__warning("%s\n", msg); return COUNTER_RETRY; - } else if (target__has_per_thread(&target) && + } else if (target__has_per_thread(&target) && errno != EOPNOTSUPP && evsel_list->core.threads && evsel_list->core.threads->err_thread != -1) { /* @@ -678,6 +677,19 @@ static enum counter_recovery stat_handle_error(struct evsel *counter) return COUNTER_SKIP; } + if (errno == EOPNOTSUPP) { + if (verbose > 0) { + ui__warning("%s event is not supported by the kernel.\n", + evsel__name(counter)); + } + counter->supported = false; + counter->errored = true; + + if ((evsel__leader(counter) != counter) || + !(counter->core.leader->nr_members > 1)) + return COUNTER_SKIP; + } + evsel__open_strerror(counter, &target, errno, msg, sizeof(msg)); ui__error("%s\n", msg); diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index 12bdbf3ecc6ae..7ee3285af10c9 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -2586,8 +2586,8 @@ errno_print: { else if (sc->fmt->errpid) { struct thread *child = machine__find_thread(trace->host, ret, ret); + fprintf(trace->output, "%ld", ret); if (child != NULL) { - fprintf(trace->output, "%ld", ret); if (thread__comm_set(child)) fprintf(trace->output, " (%s)", thread__comm_str(child)); thread__put(child); @@ -3589,10 +3589,13 @@ static int trace__set_filter_loop_pids(struct trace *trace) if (!strcmp(thread__comm_str(parent), "sshd") || strstarts(thread__comm_str(parent), "gnome-terminal")) { pids[nr++] = thread__tid(parent); + thread__put(parent); break; } + thread__put(thread); thread = parent; } + thread__put(thread); err = evlist__append_tp_filter_pids(trace->evlist, nr, pids); if (!err && trace->filter_pids.map) diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py index 13f2d8a816109..99742013676b3 100755 --- a/tools/perf/scripts/python/exported-sql-viewer.py +++ b/tools/perf/scripts/python/exported-sql-viewer.py @@ -680,7 +680,10 @@ def FindSelect(self, value, pattern, query): s = value.replace("%", "\%") s = s.replace("_", "\_") # Translate * and ? into SQL LIKE pattern characters % and _ - trans = string.maketrans("*?", "%_") + if sys.version_info[0] == 3: + trans = str.maketrans("*?", "%_") + else: + trans = string.maketrans("*?", "%_") match = " LIKE '" + str(s).translate(trans) + "'" else: match = " GLOB '" + str(value) + "'" diff --git a/tools/perf/tests/bp_account.c b/tools/perf/tests/bp_account.c index 6f921db33cf90..855b81c3326c7 100644 --- a/tools/perf/tests/bp_account.c +++ b/tools/perf/tests/bp_account.c @@ -102,6 +102,7 @@ static int bp_accounting(int wp_cnt, int share) fd_wp = wp_event((void *)&the_var, &attr_new); TEST_ASSERT_VAL("failed to create max wp\n", fd_wp != -1); pr_debug("wp max created\n"); + close(fd_wp); } for (i = 0; i < wp_cnt; i++) diff --git a/tools/perf/tests/perf-record.c b/tools/perf/tests/perf-record.c index 1c4feec1adff1..6e7f053006b4f 100644 --- a/tools/perf/tests/perf-record.c +++ b/tools/perf/tests/perf-record.c @@ -115,6 +115,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest if (err < 0) { pr_debug("sched__get_first_possible_cpu: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); + evlist__cancel_workload(evlist); goto out_delete_evlist; } @@ -126,6 +127,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest if (sched_setaffinity(evlist->workload.pid, cpu_mask_size, &cpu_mask) < 0) { pr_debug("sched_setaffinity: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); + evlist__cancel_workload(evlist); goto out_delete_evlist; } @@ -137,6 +139,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); + evlist__cancel_workload(evlist); goto out_delete_evlist; } @@ -149,6 +152,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); + evlist__cancel_workload(evlist); goto out_delete_evlist; } diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh index 3f1e67795490a..62f13dfeae8e4 100755 --- a/tools/perf/tests/shell/stat.sh +++ b/tools/perf/tests/shell/stat.sh @@ -146,6 +146,34 @@ test_cputype() { echo "cputype test [Success]" } +test_hybrid() { + # Test the default stat command on hybrid devices opens one cycles event for + # each CPU type. + echo "hybrid test" + + # Count the number of core PMUs, assume minimum of 1 + pmus=$(ls /sys/bus/event_source/devices/*/cpus 2>/dev/null | wc -l) + if [ "$pmus" -lt 1 ] + then + pmus=1 + fi + + # Run default Perf stat + cycles_events=$(perf stat -- true 2>&1 | grep -E "/cycles/[uH]*| cycles[:uH]* " -c) + + # The expectation is that default output will have a cycles events on each + # hybrid PMU. In situations with no cycles PMU events, like virtualized, this + # can fall back to task-clock and so the end count may be 0. Fail if neither + # condition holds. + if [ "$pmus" -ne "$cycles_events" ] && [ "0" -ne "$cycles_events" ] + then + echo "hybrid test [Found $pmus PMUs but $cycles_events cycles events. Failed]" + err=1 + return + fi + echo "hybrid test [Success]" +} + test_default_stat test_stat_record_report test_stat_record_script @@ -153,4 +181,5 @@ test_stat_repeat_weak_groups test_topdown_groups test_topdown_weak_groups test_cputype +test_hybrid exit $err diff --git a/tools/perf/tests/switch-tracking.c b/tools/perf/tests/switch-tracking.c index e52b031bedc5a..2c7bdf4fd55ed 100644 --- a/tools/perf/tests/switch-tracking.c +++ b/tools/perf/tests/switch-tracking.c @@ -258,7 +258,7 @@ static int compar(const void *a, const void *b) const struct event_node *nodeb = b; s64 cmp = nodea->event_time - nodeb->event_time; - return cmp; + return cmp < 0 ? -1 : (cmp > 0 ? 1 : 0); } static int process_events(struct evlist *evlist, diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c index bb59d27642ccf..a53a0257a4bca 100644 --- a/tools/perf/ui/browsers/hists.c +++ b/tools/perf/ui/browsers/hists.c @@ -3239,10 +3239,10 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h /* * No need to set actions->dso here since * it's just to remove the current filter. - * Ditto for thread below. */ do_zoom_dso(browser, actions); } else if (top == &browser->hists->thread_filter) { + actions->thread = thread; do_zoom_thread(browser, actions); } else if (top == &browser->hists->socket_filter) { do_zoom_socket(browser, actions); diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h index 1443c28545a94..358c611eeddbb 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h @@ -56,15 +56,15 @@ enum arm_spe_op_type { ARM_SPE_OP_BR_INDIRECT = 1 << 17, }; -enum arm_spe_neoverse_data_source { - ARM_SPE_NV_L1D = 0x0, - ARM_SPE_NV_L2 = 0x8, - ARM_SPE_NV_PEER_CORE = 0x9, - ARM_SPE_NV_LOCAL_CLUSTER = 0xa, - ARM_SPE_NV_SYS_CACHE = 0xb, - ARM_SPE_NV_PEER_CLUSTER = 0xc, - ARM_SPE_NV_REMOTE = 0xd, - ARM_SPE_NV_DRAM = 0xe, +enum arm_spe_common_data_source { + ARM_SPE_COMMON_DS_L1D = 0x0, + ARM_SPE_COMMON_DS_L2 = 0x8, + ARM_SPE_COMMON_DS_PEER_CORE = 0x9, + ARM_SPE_COMMON_DS_LOCAL_CLUSTER = 0xa, + ARM_SPE_COMMON_DS_SYS_CACHE = 0xb, + ARM_SPE_COMMON_DS_PEER_CLUSTER = 0xc, + ARM_SPE_COMMON_DS_REMOTE = 0xd, + ARM_SPE_COMMON_DS_DRAM = 0xe, }; struct arm_spe_record { diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 9848310cee5f3..0302e6407b5a2 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -411,15 +411,15 @@ static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq, return arm_spe_deliver_synth_event(spe, speq, event, &sample); } -static const struct midr_range neoverse_spe[] = { +static const struct midr_range common_ds_encoding_cpus[] = { MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), {}, }; -static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *record, - union perf_mem_data_src *data_src) +static void arm_spe__synth_data_source_common(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) { /* * Even though four levels of cache hierarchy are possible, no known @@ -441,17 +441,17 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec } switch (record->source) { - case ARM_SPE_NV_L1D: + case ARM_SPE_COMMON_DS_L1D: data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; data_src->mem_snoop = PERF_MEM_SNOOP_NONE; break; - case ARM_SPE_NV_L2: + case ARM_SPE_COMMON_DS_L2: data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; data_src->mem_snoop = PERF_MEM_SNOOP_NONE; break; - case ARM_SPE_NV_PEER_CORE: + case ARM_SPE_COMMON_DS_PEER_CORE: data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; @@ -460,8 +460,8 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec * We don't know if this is L1, L2 but we do know it was a cache-2-cache * transfer, so set SNOOPX_PEER */ - case ARM_SPE_NV_LOCAL_CLUSTER: - case ARM_SPE_NV_PEER_CLUSTER: + case ARM_SPE_COMMON_DS_LOCAL_CLUSTER: + case ARM_SPE_COMMON_DS_PEER_CLUSTER: data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; @@ -469,7 +469,7 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec /* * System cache is assumed to be L3 */ - case ARM_SPE_NV_SYS_CACHE: + case ARM_SPE_COMMON_DS_SYS_CACHE: data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; data_src->mem_snoop = PERF_MEM_SNOOP_HIT; @@ -478,13 +478,13 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec * We don't know what level it hit in, except it came from the other * socket */ - case ARM_SPE_NV_REMOTE: - data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1; - data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE; + case ARM_SPE_COMMON_DS_REMOTE: + data_src->mem_lvl = PERF_MEM_LVL_NA; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; break; - case ARM_SPE_NV_DRAM: + case ARM_SPE_COMMON_DS_DRAM: data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; data_src->mem_snoop = PERF_MEM_SNOOP_NONE; @@ -514,13 +514,13 @@ static void arm_spe__synth_data_source_generic(const struct arm_spe_record *reco } if (record->type & ARM_SPE_REMOTE_ACCESS) - data_src->mem_lvl |= PERF_MEM_LVL_REM_CCE1; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; } static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 midr) { union perf_mem_data_src data_src = { .mem_op = PERF_MEM_OP_NA }; - bool is_neoverse = is_midr_in_range_list(midr, neoverse_spe); + bool is_common = is_midr_in_range_list(midr, common_ds_encoding_cpus); /* Only synthesize data source for LDST operations */ if (!is_ldst_op(record->op)) @@ -533,8 +533,8 @@ static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 m else return 0; - if (is_neoverse) - arm_spe__synth_data_source_neoverse(record, &data_src); + if (is_common) + arm_spe__synth_data_source_common(record, &data_src); else arm_spe__synth_data_source_generic(record, &data_src); diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index b94b4f16a60a5..1573d6b6478d2 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -289,9 +289,15 @@ static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, info_node->info_linear = info_linear; if (!perf_env__insert_bpf_prog_info(env, info_node)) { - free(info_linear); + /* + * Insert failed, likely because of a duplicate event + * made by the sideband thread. Ignore synthesizing the + * metadata. + */ free(info_node); + goto out; } + /* info_linear is now owned by info_node and shouldn't be freed below. */ info_linear = NULL; /* @@ -447,18 +453,18 @@ int perf_event__synthesize_bpf_events(struct perf_session *session, return err; } -static void perf_env__add_bpf_info(struct perf_env *env, u32 id) +static int perf_env__add_bpf_info(struct perf_env *env, u32 id) { struct bpf_prog_info_node *info_node; struct perf_bpil *info_linear; struct btf *btf = NULL; u64 arrays; u32 btf_id; - int fd; + int fd, err = 0; fd = bpf_prog_get_fd_by_id(id); if (fd < 0) - return; + return -EINVAL; arrays = 1UL << PERF_BPIL_JITED_KSYMS; arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; @@ -471,6 +477,7 @@ static void perf_env__add_bpf_info(struct perf_env *env, u32 id) info_linear = get_bpf_prog_info_linear(fd, arrays); if (IS_ERR_OR_NULL(info_linear)) { pr_debug("%s: failed to get BPF program info. aborting\n", __func__); + err = PTR_ERR(info_linear); goto out; } @@ -480,38 +487,46 @@ static void perf_env__add_bpf_info(struct perf_env *env, u32 id) if (info_node) { info_node->info_linear = info_linear; if (!perf_env__insert_bpf_prog_info(env, info_node)) { + pr_debug("%s: duplicate add bpf info request for id %u\n", + __func__, btf_id); free(info_linear); free(info_node); + goto out; } - } else + } else { free(info_linear); + err = -ENOMEM; + goto out; + } if (btf_id == 0) goto out; btf = btf__load_from_kernel_by_id(btf_id); - if (libbpf_get_error(btf)) { - pr_debug("%s: failed to get BTF of id %u, aborting\n", - __func__, btf_id); - goto out; + if (!btf) { + err = -errno; + pr_debug("%s: failed to get BTF of id %u %d\n", __func__, btf_id, err); + } else { + perf_env__fetch_btf(env, btf_id, btf); } - perf_env__fetch_btf(env, btf_id, btf); out: btf__free(btf); close(fd); + return err; } static int bpf_event__sb_cb(union perf_event *event, void *data) { struct perf_env *env = data; + int ret = 0; if (event->header.type != PERF_RECORD_BPF_EVENT) return -1; switch (event->bpf.type) { case PERF_BPF_EVENT_PROG_LOAD: - perf_env__add_bpf_info(env, event->bpf.id); + ret = perf_env__add_bpf_info(env, event->bpf.id); case PERF_BPF_EVENT_PROG_UNLOAD: /* @@ -525,7 +540,7 @@ static int bpf_event__sb_cb(union perf_event *event, void *data) break; } - return 0; + return ret; } int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 6d2b056232f6e..6d1327f8c6043 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -1460,6 +1460,15 @@ static void evsel__free_config_terms(struct evsel *evsel) free_config_terms(&evsel->config_terms); } +static void (*evsel__priv_destructor)(void *priv); + +void evsel__set_priv_destructor(void (*destructor)(void *priv)) +{ + assert(evsel__priv_destructor == NULL); + + evsel__priv_destructor = destructor; +} + void evsel__exit(struct evsel *evsel) { assert(list_empty(&evsel->core.node)); @@ -1485,6 +1494,8 @@ void evsel__exit(struct evsel *evsel) hashmap__free(evsel->per_pkg_mask); evsel->per_pkg_mask = NULL; zfree(&evsel->metric_events); + if (evsel__priv_destructor) + evsel__priv_destructor(evsel->priv); perf_evsel__object.fini(evsel); } @@ -2863,7 +2874,7 @@ bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize) /* If event has exclude user then don't exclude kernel. */ if (evsel->core.attr.exclude_user) - return false; + goto no_fallback; /* Is there already the separator in the name. */ if (strchr(name, '/') || @@ -2871,7 +2882,7 @@ bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize) sep = ""; if (asprintf(&new_name, "%s%su", name, sep) < 0) - return false; + goto no_fallback; free(evsel->name); evsel->name = new_name; @@ -2882,8 +2893,31 @@ bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize) evsel->core.attr.exclude_hv = 1; return true; - } + } else if (err == EOPNOTSUPP && !evsel->core.attr.exclude_guest && + !evsel->exclude_GH) { + const char *name = evsel__name(evsel); + char *new_name; + const char *sep = ":"; + /* Is there already the separator in the name. */ + if (strchr(name, '/') || + (strchr(name, ':') && !evsel->is_libpfm_event)) + sep = ""; + + if (asprintf(&new_name, "%s%sH", name, sep) < 0) + goto no_fallback; + + free(evsel->name); + evsel->name = new_name; + /* Apple M1 requires exclude_guest */ + scnprintf(msg, msgsize, "Trying to fall back to excluding guest samples"); + evsel->core.attr.exclude_guest = 1; + + return true; + } +no_fallback: + scnprintf(msg, msgsize, "No fallback found for '%s' for error %d", + evsel__name(evsel), err); return false; } @@ -3120,6 +3154,8 @@ bool evsel__is_hybrid(const struct evsel *evsel) struct evsel *evsel__leader(const struct evsel *evsel) { + if (evsel->core.leader == NULL) + return NULL; return container_of(evsel->core.leader, struct evsel, core); } diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 848534ec74fa0..ac396d6f95cb5 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -252,6 +252,8 @@ void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx); void evsel__exit(struct evsel *evsel); void evsel__delete(struct evsel *evsel); +void evsel__set_priv_destructor(void (*destructor)(void *priv)); + struct callchain_param; void evsel__config(struct evsel *evsel, struct record_opts *opts, diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index 4db9a098f5926..e9f97c0c33582 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -127,6 +127,7 @@ struct intel_pt { bool single_pebs; bool sample_pebs; + int pebs_data_src_fmt; struct evsel *pebs_evsel; u64 evt_sample_type; @@ -175,6 +176,7 @@ enum switch_state { struct intel_pt_pebs_event { struct evsel *evsel; u64 id; + int data_src_fmt; }; struct intel_pt_queue { @@ -2232,7 +2234,146 @@ static void intel_pt_add_lbrs(struct branch_stack *br_stack, } } -static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evsel *evsel, u64 id) +#define P(a, b) PERF_MEM_S(a, b) +#define OP_LH (P(OP, LOAD) | P(LVL, HIT)) +#define LEVEL(x) P(LVLNUM, x) +#define REM P(REMOTE, REMOTE) +#define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS)) + +#define PERF_PEBS_DATA_SOURCE_GRT_MAX 0x10 +#define PERF_PEBS_DATA_SOURCE_GRT_MASK (PERF_PEBS_DATA_SOURCE_GRT_MAX - 1) + +/* Based on kernel __intel_pmu_pebs_data_source_grt() and pebs_data_source */ +static const u64 pebs_data_source_grt[PERF_PEBS_DATA_SOURCE_GRT_MAX] = { + P(OP, LOAD) | P(LVL, MISS) | LEVEL(L3) | P(SNOOP, NA), /* L3 miss|SNP N/A */ + OP_LH | P(LVL, L1) | LEVEL(L1) | P(SNOOP, NONE), /* L1 hit|SNP None */ + OP_LH | P(LVL, LFB) | LEVEL(LFB) | P(SNOOP, NONE), /* LFB/MAB hit|SNP None */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* L2 hit|SNP None */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, NONE), /* L3 hit|SNP None */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT), /* L3 hit|SNP Hit */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM), /* L3 hit|SNP HitM */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM), /* L3 hit|SNP HitM */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD), /* L3 hit|SNP Fwd */ + OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HITM), /* Remote L3 hit|SNP HitM */ + OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | P(SNOOP, HIT), /* RAM hit|SNP Hit */ + OP_LH | P(LVL, REM_RAM1) | REM | LEVEL(L3) | P(SNOOP, HIT), /* Remote L3 hit|SNP Hit */ + OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | SNOOP_NONE_MISS, /* RAM hit|SNP None or Miss */ + OP_LH | P(LVL, REM_RAM1) | LEVEL(RAM) | REM | SNOOP_NONE_MISS, /* Remote RAM hit|SNP None or Miss */ + OP_LH | P(LVL, IO) | LEVEL(NA) | P(SNOOP, NONE), /* I/O hit|SNP None */ + OP_LH | P(LVL, UNC) | LEVEL(NA) | P(SNOOP, NONE), /* Uncached hit|SNP None */ +}; + +/* Based on kernel __intel_pmu_pebs_data_source_cmt() and pebs_data_source */ +static const u64 pebs_data_source_cmt[PERF_PEBS_DATA_SOURCE_GRT_MAX] = { + P(OP, LOAD) | P(LVL, MISS) | LEVEL(L3) | P(SNOOP, NA), /* L3 miss|SNP N/A */ + OP_LH | P(LVL, L1) | LEVEL(L1) | P(SNOOP, NONE), /* L1 hit|SNP None */ + OP_LH | P(LVL, LFB) | LEVEL(LFB) | P(SNOOP, NONE), /* LFB/MAB hit|SNP None */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* L2 hit|SNP None */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, NONE), /* L3 hit|SNP None */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, MISS), /* L3 hit|SNP Hit */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT), /* L3 hit|SNP HitM */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD), /* L3 hit|SNP HitM */ + OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM), /* L3 hit|SNP Fwd */ + OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HITM), /* Remote L3 hit|SNP HitM */ + OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | P(SNOOP, NONE), /* RAM hit|SNP Hit */ + OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE), /* Remote L3 hit|SNP Hit */ + OP_LH | LEVEL(RAM) | REM | P(SNOOPX, FWD), /* RAM hit|SNP None or Miss */ + OP_LH | LEVEL(RAM) | REM | P(SNOOP, HITM), /* Remote RAM hit|SNP None or Miss */ + OP_LH | P(LVL, IO) | LEVEL(NA) | P(SNOOP, NONE), /* I/O hit|SNP None */ + OP_LH | P(LVL, UNC) | LEVEL(NA) | P(SNOOP, NONE), /* Uncached hit|SNP None */ +}; + +/* Based on kernel pebs_set_tlb_lock() */ +static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock) +{ + /* + * TLB access + * 0 = did not miss 2nd level TLB + * 1 = missed 2nd level TLB + */ + if (tlb) + *val |= P(TLB, MISS) | P(TLB, L2); + else + *val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2); + + /* locked prefix */ + if (lock) + *val |= P(LOCK, LOCKED); +} + +/* Based on kernel __grt_latency_data() */ +static u64 intel_pt_grt_latency_data(u8 dse, bool tlb, bool lock, bool blk, + const u64 *pebs_data_source) +{ + u64 val; + + dse &= PERF_PEBS_DATA_SOURCE_GRT_MASK; + val = pebs_data_source[dse]; + + pebs_set_tlb_lock(&val, tlb, lock); + + if (blk) + val |= P(BLK, DATA); + else + val |= P(BLK, NA); + + return val; +} + +/* Default value for data source */ +#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\ + PERF_MEM_S(LVL, NA) |\ + PERF_MEM_S(SNOOP, NA) |\ + PERF_MEM_S(LOCK, NA) |\ + PERF_MEM_S(TLB, NA) |\ + PERF_MEM_S(LVLNUM, NA)) + +enum DATA_SRC_FORMAT { + DATA_SRC_FORMAT_ERR = -1, + DATA_SRC_FORMAT_NA = 0, + DATA_SRC_FORMAT_GRT = 1, + DATA_SRC_FORMAT_CMT = 2, +}; + +/* Based on kernel grt_latency_data() and cmt_latency_data */ +static u64 intel_pt_get_data_src(u64 mem_aux_info, int data_src_fmt) +{ + switch (data_src_fmt) { + case DATA_SRC_FORMAT_GRT: { + union { + u64 val; + struct { + unsigned int dse:4; + unsigned int locked:1; + unsigned int stlb_miss:1; + unsigned int fwd_blk:1; + unsigned int reserved:25; + }; + } x = {.val = mem_aux_info}; + return intel_pt_grt_latency_data(x.dse, x.stlb_miss, x.locked, x.fwd_blk, + pebs_data_source_grt); + } + case DATA_SRC_FORMAT_CMT: { + union { + u64 val; + struct { + unsigned int dse:5; + unsigned int locked:1; + unsigned int stlb_miss:1; + unsigned int fwd_blk:1; + unsigned int reserved:24; + }; + } x = {.val = mem_aux_info}; + return intel_pt_grt_latency_data(x.dse, x.stlb_miss, x.locked, x.fwd_blk, + pebs_data_source_cmt); + } + default: + return PERF_MEM_NA; + } +} + +static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evsel *evsel, + u64 id, int data_src_fmt) { const struct intel_pt_blk_items *items = &ptq->state->items; struct perf_sample sample = { .ip = 0, }; @@ -2350,6 +2491,18 @@ static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evse } } + if (sample_type & PERF_SAMPLE_DATA_SRC) { + if (items->has_mem_aux_info && data_src_fmt) { + if (data_src_fmt < 0) { + pr_err("Intel PT missing data_src info\n"); + return -1; + } + sample.data_src = intel_pt_get_data_src(items->mem_aux_info, data_src_fmt); + } else { + sample.data_src = PERF_MEM_NA; + } + } + if (sample_type & PERF_SAMPLE_TRANSACTION && items->has_tsx_aux_info) { u64 ax = items->has_rax ? items->rax : 0; /* Refer kernel's intel_hsw_transaction() */ @@ -2368,9 +2521,10 @@ static int intel_pt_synth_single_pebs_sample(struct intel_pt_queue *ptq) { struct intel_pt *pt = ptq->pt; struct evsel *evsel = pt->pebs_evsel; + int data_src_fmt = pt->pebs_data_src_fmt; u64 id = evsel->core.id[0]; - return intel_pt_do_synth_pebs_sample(ptq, evsel, id); + return intel_pt_do_synth_pebs_sample(ptq, evsel, id, data_src_fmt); } static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq) @@ -2395,7 +2549,7 @@ static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq) hw_id); return intel_pt_synth_single_pebs_sample(ptq); } - err = intel_pt_do_synth_pebs_sample(ptq, pe->evsel, pe->id); + err = intel_pt_do_synth_pebs_sample(ptq, pe->evsel, pe->id, pe->data_src_fmt); if (err) return err; } @@ -3355,6 +3509,49 @@ static int intel_pt_process_itrace_start(struct intel_pt *pt, event->itrace_start.tid); } +/* + * Events with data_src are identified by L1_Hit_Indication + * refer https://github.com/intel/perfmon + */ +static int intel_pt_data_src_fmt(struct intel_pt *pt, struct evsel *evsel) +{ + struct perf_env *env = pt->machine->env; + int fmt = DATA_SRC_FORMAT_NA; + + if (!env->cpuid) + return DATA_SRC_FORMAT_ERR; + + /* + * PEBS-via-PT is only supported on E-core non-hybrid. Of those only + * Gracemont and Crestmont have data_src. Check for: + * Alderlake N (Gracemont) + * Sierra Forest (Crestmont) + * Grand Ridge (Crestmont) + */ + + if (!strncmp(env->cpuid, "GenuineIntel,6,190,", 19)) + fmt = DATA_SRC_FORMAT_GRT; + + if (!strncmp(env->cpuid, "GenuineIntel,6,175,", 19) || + !strncmp(env->cpuid, "GenuineIntel,6,182,", 19)) + fmt = DATA_SRC_FORMAT_CMT; + + if (fmt == DATA_SRC_FORMAT_NA) + return fmt; + + /* + * Only data_src events are: + * mem-loads event=0xd0,umask=0x5 + * mem-stores event=0xd0,umask=0x6 + */ + if (evsel->core.attr.type == PERF_TYPE_RAW && + ((evsel->core.attr.config & 0xffff) == 0x5d0 || + (evsel->core.attr.config & 0xffff) == 0x6d0)) + return fmt; + + return DATA_SRC_FORMAT_NA; +} + static int intel_pt_process_aux_output_hw_id(struct intel_pt *pt, union perf_event *event, struct perf_sample *sample) @@ -3375,6 +3572,7 @@ static int intel_pt_process_aux_output_hw_id(struct intel_pt *pt, ptq->pebs[hw_id].evsel = evsel; ptq->pebs[hw_id].id = sample->id; + ptq->pebs[hw_id].data_src_fmt = intel_pt_data_src_fmt(pt, evsel); return 0; } @@ -3946,6 +4144,7 @@ static void intel_pt_setup_pebs_events(struct intel_pt *pt) } pt->single_pebs = true; pt->sample_pebs = true; + pt->pebs_data_src_fmt = intel_pt_data_src_fmt(pt, evsel); pt->pebs_evsel = evsel; } } diff --git a/tools/perf/util/lzma.c b/tools/perf/util/lzma.c index af9a97612f9df..f61574d1581e3 100644 --- a/tools/perf/util/lzma.c +++ b/tools/perf/util/lzma.c @@ -113,7 +113,7 @@ bool lzma_is_compressed(const char *input) ssize_t rc; if (fd < 0) - return -1; + return false; rc = read(fd, buf, sizeof(buf)); close(fd); diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c index 9bee082194d5e..fb11a967c450d 100644 --- a/tools/perf/util/print-events.c +++ b/tools/perf/util/print-events.c @@ -271,6 +271,7 @@ bool is_event_supported(u8 type, u64 config) ret = evsel__open(evsel, NULL, tmap) >= 0; } + evsel__close(evsel); evsel__delete(evsel); } diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index 277b2cbd51861..e4d57e7df7ae1 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -1662,7 +1662,7 @@ static s64 perf_session__process_user_event(struct perf_session *session, struct perf_tool *tool = session->tool; struct perf_sample sample = { .time = 0, }; int fd = perf_data__fd(session->data); - int err; + s64 err; if (event->header.type != PERF_RECORD_COMPRESSED || tool->compressed == perf_session__process_compressed_event_stub) diff --git a/tools/perf/util/setup.py b/tools/perf/util/setup.py index e837132d5031b..eecb462c021cd 100644 --- a/tools/perf/util/setup.py +++ b/tools/perf/util/setup.py @@ -1,6 +1,7 @@ from os import getenv, path from subprocess import Popen, PIPE from re import sub +import shlex cc = getenv("CC") @@ -16,7 +17,9 @@ src_feature_tests = getenv('srctree') + '/tools/build/feature' def clang_has_option(option): - cc_output = Popen([cc, cc_options + option, path.join(src_feature_tests, "test-hello.c") ], stderr=PIPE).stderr.readlines() + cmd = shlex.split(f"{cc} {cc_options} {option}") + cmd.append(path.join(src_feature_tests, "test-hello.c")) + cc_output = Popen(cmd, stderr=PIPE).stderr.readlines() return [o for o in cc_output if ((b"unknown argument" in o) or (b"is not supported" in o) or (b"unknown warning option" in o))] == [ ] if cc_is_clang: diff --git a/tools/perf/util/zlib.c b/tools/perf/util/zlib.c index 78d2297c1b674..1f7c065230599 100644 --- a/tools/perf/util/zlib.c +++ b/tools/perf/util/zlib.c @@ -88,7 +88,7 @@ bool gzip_is_compressed(const char *input) ssize_t rc; if (fd < 0) - return -1; + return false; rc = read(fd, buf, sizeof(buf)); close(fd); diff --git a/tools/power/cpupower/lib/cpuidle.c b/tools/power/cpupower/lib/cpuidle.c index 479c5971aa6da..c15d0de12357f 100644 --- a/tools/power/cpupower/lib/cpuidle.c +++ b/tools/power/cpupower/lib/cpuidle.c @@ -231,6 +231,7 @@ int cpuidle_state_disable(unsigned int cpu, { char value[SYSFS_PATH_MAX]; int bytes_written; + int len; if (cpuidle_state_count(cpu) <= idlestate) return -1; @@ -239,10 +240,10 @@ int cpuidle_state_disable(unsigned int cpu, idlestate_value_files[IDLESTATE_DISABLE])) return -2; - snprintf(value, SYSFS_PATH_MAX, "%u", disable); + len = snprintf(value, SYSFS_PATH_MAX, "%u", disable); bytes_written = cpuidle_state_write_file(cpu, idlestate, "disable", - value, sizeof(disable)); + value, len); if (bytes_written) return 0; return -3; diff --git a/tools/power/cpupower/lib/cpupower.c b/tools/power/cpupower/lib/cpupower.c index 7a2ef691b20e1..c2a7af89a67bb 100644 --- a/tools/power/cpupower/lib/cpupower.c +++ b/tools/power/cpupower/lib/cpupower.c @@ -55,7 +55,7 @@ unsigned int cpupower_write_sysfs(const char *path, char *buf, size_t buflen) if (numwritten < 1) { perror(path); close(fd); - return -1; + return 0; } close(fd); diff --git a/tools/power/cpupower/utils/cpupower-set.c b/tools/power/cpupower/utils/cpupower-set.c index 0677b58374abf..59ace394cf3ef 100644 --- a/tools/power/cpupower/utils/cpupower-set.c +++ b/tools/power/cpupower/utils/cpupower-set.c @@ -62,8 +62,8 @@ int cmd_set(int argc, char **argv) params.params = 0; /* parameter parsing */ - while ((ret = getopt_long(argc, argv, "b:e:m:", - set_opts, NULL)) != -1) { + while ((ret = getopt_long(argc, argv, "b:e:m:t:", + set_opts, NULL)) != -1) { switch (ret) { case 'b': if (params.perf_bias) diff --git a/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c b/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c index 08a399b0be286..6ab9139f16af9 100644 --- a/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c +++ b/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c @@ -240,9 +240,9 @@ static int mperf_stop(void) int cpu; for (cpu = 0; cpu < cpu_count; cpu++) { - mperf_measure_stats(cpu); - mperf_get_tsc(&tsc_at_measure_end[cpu]); clock_gettime(CLOCK_REALTIME, &time_end[cpu]); + mperf_get_tsc(&tsc_at_measure_end[cpu]); + mperf_measure_stats(cpu); } return 0; diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c index ebda9c366b2ba..891738116c8b2 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c @@ -62,6 +62,7 @@ unsigned char turbo_update_value; unsigned char update_hwp_epp; unsigned char update_hwp_min; unsigned char update_hwp_max; +unsigned char hwp_limits_done_via_sysfs; unsigned char update_hwp_desired; unsigned char update_hwp_window; unsigned char update_hwp_use_pkg; @@ -630,7 +631,7 @@ void cmdline(int argc, char **argv) */ FILE *fopen_or_die(const char *path, const char *mode) { - FILE *filep = fopen(path, "r"); + FILE *filep = fopen(path, mode); if (!filep) err(1, "%s: open failed", path); @@ -644,7 +645,7 @@ void err_on_hypervisor(void) char *buffer; /* On VMs /proc/cpuinfo contains a "flags" entry for hypervisor */ - cpuinfo = fopen_or_die("/proc/cpuinfo", "ro"); + cpuinfo = fopen_or_die("/proc/cpuinfo", "r"); buffer = malloc(4096); if (!buffer) { @@ -951,8 +952,10 @@ int ratio_2_sysfs_khz(int ratio) } /* * If HWP is enabled and cpufreq sysfs attribtes are present, - * then update sysfs, so that it will not become - * stale when we write to MSRs. + * then update via sysfs. The intel_pstate driver may modify (clip) + * this request, say, when HWP_CAP is outside of PLATFORM_INFO limits, + * and the driver-chosen value takes precidence. + * * (intel_pstate's max_perf_pct and min_perf_pct will follow cpufreq, * so we don't have to touch that.) */ @@ -1007,6 +1010,8 @@ int update_sysfs(int cpu) if (update_hwp_max) update_cpufreq_scaling_freq(1, cpu, req_update.hwp_max); + hwp_limits_done_via_sysfs = 1; + return 0; } @@ -1085,10 +1090,10 @@ int update_hwp_request(int cpu) if (debug) print_hwp_request(cpu, &req, "old: "); - if (update_hwp_min) + if (update_hwp_min && !hwp_limits_done_via_sysfs) req.hwp_min = req_update.hwp_min; - if (update_hwp_max) + if (update_hwp_max && !hwp_limits_done_via_sysfs) req.hwp_max = req_update.hwp_max; if (update_hwp_desired) @@ -1166,13 +1171,18 @@ int update_hwp_request_pkg(int pkg) int enable_hwp_on_cpu(int cpu) { - unsigned long long msr; + unsigned long long old_msr, new_msr; + + get_msr(cpu, MSR_PM_ENABLE, &old_msr); + + if (old_msr & 1) + return 0; /* already enabled */ - get_msr(cpu, MSR_PM_ENABLE, &msr); - put_msr(cpu, MSR_PM_ENABLE, 1); + new_msr = old_msr | 1; + put_msr(cpu, MSR_PM_ENABLE, new_msr); if (verbose) - printf("cpu%d: MSR_PM_ENABLE old: %d new: %d\n", cpu, (unsigned int) msr, 1); + printf("cpu%d: MSR_PM_ENABLE old: %llX new: %llX\n", cpu, old_msr, new_msr); return 0; } diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore new file mode 100644 index 0000000000000..a3240f9f7ebae --- /dev/null +++ b/tools/sched_ext/.gitignore @@ -0,0 +1,9 @@ +scx_example_simple +scx_example_qmap +scx_example_central +scx_example_pair +scx_example_flatcg +scx_example_userland +*.skel.h +*.subskel.h +/tools/ diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile new file mode 100644 index 0000000000000..73c43782837d4 --- /dev/null +++ b/tools/sched_ext/Makefile @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2022 Meta Platforms, Inc. and affiliates. +include ../build/Build.include +include ../scripts/Makefile.arch +include ../scripts/Makefile.include + +ifneq ($(LLVM),) +ifneq ($(filter %/,$(LLVM)),) +LLVM_PREFIX := $(LLVM) +else ifneq ($(filter -%,$(LLVM)),) +LLVM_SUFFIX := $(LLVM) +endif + +CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi +CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu +CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl +CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu +CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu +CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu +CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu +CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu +CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu +CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) + +ifeq ($(CROSS_COMPILE),) +ifeq ($(CLANG_TARGET_FLAGS),) +$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk +else +CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) +endif # CLANG_TARGET_FLAGS +else +CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) +endif # CROSS_COMPILE + +CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as +else +CC := $(CROSS_COMPILE)gcc +endif # LLVM + +CURDIR := $(abspath .) +TOOLSDIR := $(abspath ..) +LIBDIR := $(TOOLSDIR)/lib +BPFDIR := $(LIBDIR)/bpf +TOOLSINCDIR := $(TOOLSDIR)/include +BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool +APIDIR := $(TOOLSINCDIR)/uapi +GENDIR := $(abspath ../../include/generated) +GENHDR := $(GENDIR)/autoconf.h + +SCRATCH_DIR := $(CURDIR)/tools +BUILD_DIR := $(SCRATCH_DIR)/build +INCLUDE_DIR := $(SCRATCH_DIR)/include +BPFOBJ_DIR := $(BUILD_DIR)/libbpf +BPFOBJ := $(BPFOBJ_DIR)/libbpf.a +ifneq ($(CROSS_COMPILE),) +HOST_BUILD_DIR := $(BUILD_DIR)/host +HOST_SCRATCH_DIR := host-tools +HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include +else +HOST_BUILD_DIR := $(BUILD_DIR) +HOST_SCRATCH_DIR := $(SCRATCH_DIR) +HOST_INCLUDE_DIR := $(INCLUDE_DIR) +endif +HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a +RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids +DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool + +VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ + $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ + ../../vmlinux \ + /sys/kernel/btf/vmlinux \ + /boot/vmlinux-$(shell uname -r) +VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) +ifeq ($(VMLINUX_BTF),) +$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") +endif + +BPFTOOL ?= $(DEFAULT_BPFTOOL) + +ifneq ($(wildcard $(GENHDR)),) + GENFLAGS := -DHAVE_GENHDR +endif + +CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ + -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ + -I$(TOOLSINCDIR) -I$(APIDIR) + +CARGOFLAGS := --release + +# Silence some warnings when compiled with clang +ifneq ($(LLVM),) +CFLAGS += -Wno-unused-command-line-argument +endif + +LDFLAGS = -lelf -lz -lpthread + +IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - &1 \ + | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ +$(shell $(1) -dM -E - $@ +else + $(call msg,CP,,$@) + $(Q)cp "$(VMLINUX_H)" $@ +endif + +%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \ + | $(BPFOBJ) + $(call msg,CLNG-BPF,,$@) + $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ + +%.skel.h: %.bpf.o $(BPFTOOL) + $(call msg,GEN-SKEL,,$@) + $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< + $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) + $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) + $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) + $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@ + $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h) + +scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h + $(CC) $(CFLAGS) -c $< -o $@.o + $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) + +scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h + $(CC) $(CFLAGS) -c $< -o $@.o + $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) + +scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h + $(CC) $(CFLAGS) -c $< -o $@.o + $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) + +scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h + $(CC) $(CFLAGS) -c $< -o $@.o + $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) + +scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h + $(CC) $(CFLAGS) -c $< -o $@.o + $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) + +scx_example_userland: scx_example_userland.c scx_example_userland.skel.h \ + scx_example_userland_common.h user_exit_info.h + $(CC) $(CFLAGS) -c $< -o $@.o + $(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS) + +atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR) +atropos: export ATROPOS_CLANG = $(CLANG) +atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS) +atropos: $(INCLUDE_DIR)/vmlinux.h + cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS) + +clean: + cargo clean --manifest-path=atropos/Cargo.toml + rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) + rm -f *.o *.bpf.o *.skel.h *.subskel.h + rm -f scx_example_simple scx_example_qmap scx_example_central \ + scx_example_pair scx_example_flatcg scx_example_userland + +.PHONY: all atropos clean + +# delete failed targets +.DELETE_ON_ERROR: + +# keep intermediate (.skel.h, .bpf.o, etc) targets +.SECONDARY: diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore new file mode 100644 index 0000000000000..186dba259ec21 --- /dev/null +++ b/tools/sched_ext/atropos/.gitignore @@ -0,0 +1,3 @@ +src/bpf/.output +Cargo.lock +target diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml new file mode 100644 index 0000000000000..7462a836d53dd --- /dev/null +++ b/tools/sched_ext/atropos/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "scx_atropos" +version = "0.5.0" +authors = ["Dan Schatzberg ", "Meta"] +edition = "2021" +description = "Userspace scheduling with BPF" +license = "GPL-2.0-only" + +[dependencies] +anyhow = "1.0.65" +bitvec = { version = "1.0", features = ["serde"] } +clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] } +ctrlc = { version = "3.1", features = ["termination"] } +fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"} +hex = "0.4.3" +libbpf-rs = "0.19.1" +libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] } +libc = "0.2.137" +log = "0.4.17" +ordered-float = "3.4.0" +simplelog = "0.12.0" + +[build-dependencies] +bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false } +libbpf-cargo = "0.13.0" + +[features] +enable_backtrace = [] diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs new file mode 100644 index 0000000000000..26e792c5e17e9 --- /dev/null +++ b/tools/sched_ext/atropos/build.rs @@ -0,0 +1,70 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +extern crate bindgen; + +use std::env; +use std::fs::create_dir_all; +use std::path::Path; +use std::path::PathBuf; + +use libbpf_cargo::SkeletonBuilder; + +const HEADER_PATH: &str = "src/bpf/atropos.h"; + +fn bindgen_atropos() { + // Tell cargo to invalidate the built crate whenever the wrapper changes + println!("cargo:rerun-if-changed={}", HEADER_PATH); + + // The bindgen::Builder is the main entry point + // to bindgen, and lets you build up options for + // the resulting bindings. + let bindings = bindgen::Builder::default() + // The input header we would like to generate + // bindings for. + .header(HEADER_PATH) + // Tell cargo to invalidate the built crate whenever any of the + // included header files changed. + .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + // Finish the builder and generate the bindings. + .generate() + // Unwrap the Result and panic on failure. + .expect("Unable to generate bindings"); + + // Write the bindings to the $OUT_DIR/bindings.rs file. + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("atropos-sys.rs")) + .expect("Couldn't write bindings!"); +} + +fn gen_bpf_sched(name: &str) { + let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap(); + let clang = env::var("ATROPOS_CLANG").unwrap(); + eprintln!("{}", clang); + let outpath = format!("./src/bpf/.output/{}.skel.rs", name); + let skel = Path::new(&outpath); + let src = format!("./src/bpf/{}.bpf.c", name); + SkeletonBuilder::new() + .source(src.clone()) + .clang(clang) + .clang_args(bpf_cflags) + .build_and_generate(&skel) + .unwrap(); + println!("cargo:rerun-if-changed={}", src); +} + +fn main() { + bindgen_atropos(); + // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton. + // Reasons are because the generated skeleton contains compiler attributes + // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]` + // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside + // the path attribute either (see https://github.com/rust-lang/rust/pull/83366). + // + // However, there is hope! When the above feature stabilizes we can clean this + // all up. + create_dir_all("./src/bpf/.output").unwrap(); + gen_bpf_sched("atropos"); +} diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml new file mode 100644 index 0000000000000..b7258ed0a8d84 --- /dev/null +++ b/tools/sched_ext/atropos/rustfmt.toml @@ -0,0 +1,8 @@ +# Get help on options with `rustfmt --help=config` +# Please keep these in alphabetical order. +edition = "2021" +group_imports = "StdExternalCrate" +imports_granularity = "Item" +merge_derives = false +use_field_init_shorthand = true +version = "Two" diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs new file mode 100644 index 0000000000000..bbeaf856d40e8 --- /dev/null +++ b/tools/sched_ext/atropos/src/atropos_sys.rs @@ -0,0 +1,10 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(dead_code)] + +include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs")); diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c new file mode 100644 index 0000000000000..3905a403e9406 --- /dev/null +++ b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c @@ -0,0 +1,743 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ +/* + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2. + * + * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF + * part does simple round robin in each domain and the userspace part + * calculates the load factor of each domain and tells the BPF part how to load + * balance the domains. + * + * Every task has an entry in the task_data map which lists which domain the + * task belongs to. When a task first enters the system (atropos_prep_enable), + * they are round-robined to a domain. + * + * atropos_select_cpu is the primary scheduling logic, invoked when a task + * becomes runnable. The lb_data map is populated by userspace to inform the BPF + * scheduler that a task should be migrated to a new domain. Otherwise, the task + * is scheduled in priority order as follows: + * * The current core if the task was woken up synchronously and there are idle + * cpus in the system + * * The previous core, if idle + * * The pinned-to core if the task is pinned to a specific core + * * Any idle cpu in the domain + * + * If none of the above conditions are met, then the task is enqueued to a + * dispatch queue corresponding to the domain (atropos_enqueue). + * + * atropos_dispatch will attempt to consume a task from its domain's + * corresponding dispatch queue (this occurs after scheduling any tasks directly + * assigned to it due to the logic in atropos_select_cpu). If no task is found, + * then greedy load stealing will attempt to find a task on another dispatch + * queue to run. + * + * Load balancing is almost entirely handled by userspace. BPF populates the + * task weight, dom mask and current dom in the task_data map and executes the + * load balance based on userspace populating the lb_data map. + */ +#include "../../../scx_common.bpf.h" +#include "atropos.h" + +#include +#include +#include +#include +#include +#include + +char _license[] SEC("license") = "GPL"; + +/* + * const volatiles are set during initialization and treated as consts by the + * jit compiler. + */ + +/* + * Domains and cpus + */ +const volatile __u32 nr_doms = 32; /* !0 for veristat, set during init */ +const volatile __u32 nr_cpus = 64; /* !0 for veristat, set during init */ +const volatile __u32 cpu_dom_id_map[MAX_CPUS]; +const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; + +const volatile bool kthreads_local; +const volatile bool fifo_sched; +const volatile bool switch_partial; +const volatile __u32 greedy_threshold; + +/* base slice duration */ +const volatile __u64 slice_us = 20000; + +/* + * Exit info + */ +int exit_type = SCX_EXIT_NONE; +char exit_msg[SCX_EXIT_MSG_LEN]; + +struct pcpu_ctx { + __u32 dom_rr_cur; /* used when scanning other doms */ + + /* libbpf-rs does not respect the alignment, so pad out the struct explicitly */ + __u8 _padding[CACHELINE_SIZE - sizeof(u64)]; +} __attribute__((aligned(CACHELINE_SIZE))); + +struct pcpu_ctx pcpu_ctx[MAX_CPUS]; + +/* + * Domain context + */ +struct dom_ctx { + struct bpf_cpumask __kptr *cpumask; + u64 vtime_now; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, u32); + __type(value, struct dom_ctx); + __uint(max_entries, MAX_DOMS); + __uint(map_flags, 0); +} dom_ctx SEC(".maps"); + +/* + * Statistics + */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u64)); + __uint(max_entries, ATROPOS_NR_STATS); +} stats SEC(".maps"); + +static inline void stat_add(enum stat_idx idx, u64 addend) +{ + u32 idx_v = idx; + + u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); + if (cnt_p) + (*cnt_p) += addend; +} + +/* Map pid -> task_ctx */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, pid_t); + __type(value, struct task_ctx); + __uint(max_entries, 1000000); + __uint(map_flags, 0); +} task_data SEC(".maps"); + +/* + * This is populated from userspace to indicate which pids should be reassigned + * to new doms. + */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, pid_t); + __type(value, u32); + __uint(max_entries, 1000); + __uint(map_flags, 0); +} lb_data SEC(".maps"); + +static inline bool vtime_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p, + u32 new_dom_id) +{ + struct dom_ctx *old_domc, *new_domc; + struct bpf_cpumask *d_cpumask, *t_cpumask; + u32 old_dom_id = task_ctx->dom_id; + s64 vtime_delta; + + old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id); + if (!old_domc) { + scx_bpf_error("No dom%u", old_dom_id); + return false; + } + + vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now; + + new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id); + if (!new_domc) { + scx_bpf_error("No dom%u", new_dom_id); + return false; + } + + d_cpumask = new_domc->cpumask; + if (!d_cpumask) { + scx_bpf_error("Failed to get domain %u cpumask kptr", + new_dom_id); + return false; + } + + t_cpumask = task_ctx->cpumask; + if (!t_cpumask) { + scx_bpf_error("Failed to look up task cpumask"); + return false; + } + + /* + * set_cpumask might have happened between userspace requesting LB and + * here and @p might not be able to run in @dom_id anymore. Verify. + */ + if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask, + p->cpus_ptr)) { + p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta; + task_ctx->dom_id = new_dom_id; + bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask, + p->cpus_ptr); + } + + return task_ctx->dom_id == new_dom_id; +} + +s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu, + u32 wake_flags) +{ + s32 cpu; + pid_t pid = p->pid; + struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); + struct bpf_cpumask *p_cpumask; + + if (!task_ctx) + return -ENOENT; + + if (kthreads_local && + (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { + cpu = prev_cpu; + stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); + goto local; + } + + /* + * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the + * local dsq of the waker. + */ + if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) { + struct task_struct *current = (void *)bpf_get_current_task(); + + if (!(BPF_CORE_READ(current, flags) & PF_EXITING) && + task_ctx->dom_id < MAX_DOMS) { + struct dom_ctx *domc; + struct bpf_cpumask *d_cpumask; + const struct cpumask *idle_cpumask; + bool has_idle; + + domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id); + if (!domc) { + scx_bpf_error("Failed to find dom%u", + task_ctx->dom_id); + return prev_cpu; + } + d_cpumask = domc->cpumask; + if (!d_cpumask) { + scx_bpf_error("Failed to acquire domain %u cpumask kptr", + task_ctx->dom_id); + return prev_cpu; + } + + idle_cpumask = scx_bpf_get_idle_cpumask(); + + has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask, + idle_cpumask); + + scx_bpf_put_idle_cpumask(idle_cpumask); + + if (has_idle) { + cpu = bpf_get_smp_processor_id(); + if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { + stat_add(ATROPOS_STAT_WAKE_SYNC, 1); + goto local; + } + } + } + } + + /* if the previous CPU is idle, dispatch directly to it */ + if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { + stat_add(ATROPOS_STAT_PREV_IDLE, 1); + cpu = prev_cpu; + goto local; + } + + /* If only one core is allowed, dispatch */ + if (p->nr_cpus_allowed == 1) { + stat_add(ATROPOS_STAT_PINNED, 1); + cpu = prev_cpu; + goto local; + } + + p_cpumask = task_ctx->cpumask; + if (!p_cpumask) + return -ENOENT; + + /* If there is an eligible idle CPU, dispatch directly */ + cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); + if (cpu >= 0) { + stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1); + goto local; + } + + /* + * @prev_cpu may be in a different domain. Returning an out-of-domain + * CPU can lead to stalls as all in-domain CPUs may be idle by the time + * @p gets enqueued. + */ + if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask)) + cpu = prev_cpu; + else + cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask); + + return cpu; + +local: + task_ctx->dispatch_local = true; + return cpu; +} + +void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags) +{ + pid_t pid = p->pid; + struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); + u32 *new_dom; + + if (!task_ctx) { + scx_bpf_error("No task_ctx[%d]", pid); + return; + } + + new_dom = bpf_map_lookup_elem(&lb_data, &pid); + if (new_dom && *new_dom != task_ctx->dom_id && + task_set_dsq(task_ctx, p, *new_dom)) { + struct bpf_cpumask *p_cpumask; + s32 cpu; + + stat_add(ATROPOS_STAT_LOAD_BALANCE, 1); + + /* + * If dispatch_local is set, We own @p's idle state but we are + * not gonna put the task in the associated local dsq which can + * cause the CPU to stall. Kick it. + */ + if (task_ctx->dispatch_local) { + task_ctx->dispatch_local = false; + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0); + } + + p_cpumask = task_ctx->cpumask; + if (!p_cpumask) { + scx_bpf_error("Failed to get task_ctx->cpumask"); + return; + } + cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask); + + if (cpu >= 0) + scx_bpf_kick_cpu(cpu, 0); + } + + if (task_ctx->dispatch_local) { + task_ctx->dispatch_local = false; + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags); + return; + } + + if (fifo_sched) { + scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000, + enq_flags); + } else { + u64 vtime = p->scx.dsq_vtime; + u32 dom_id = task_ctx->dom_id; + struct dom_ctx *domc; + + domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); + if (!domc) { + scx_bpf_error("No dom[%u]", dom_id); + return; + } + + /* + * Limit the amount of budget that an idling task can accumulate + * to one slice. + */ + if (vtime_before(vtime, domc->vtime_now - slice_us * 1000)) + vtime = domc->vtime_now - slice_us * 1000; + + scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime, + enq_flags); + } +} + +static u32 cpu_to_dom_id(s32 cpu) +{ + const volatile u32 *dom_idp; + + if (nr_doms <= 1) + return 0; + + dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]); + if (!dom_idp) + return MAX_DOMS; + + return *dom_idp; +} + +static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) +{ + s32 cpu; + + if (dom_id >= MAX_DOMS) + return false; + + bpf_for(cpu, 0, nr_cpus) { + if (bpf_cpumask_test_cpu(cpu, cpumask) && + (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64)))) + return true; + } + return false; +} + +static u32 dom_rr_next(s32 cpu) +{ + struct pcpu_ctx *pcpuc; + u32 dom_id; + + pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]); + if (!pcpuc) + return 0; + + dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms; + + if (dom_id == cpu_to_dom_id(cpu)) + dom_id = (dom_id + 1) % nr_doms; + + pcpuc->dom_rr_cur = dom_id; + return dom_id; +} + +void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev) +{ + u32 dom = cpu_to_dom_id(cpu); + + if (scx_bpf_consume(dom)) { + stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1); + return; + } + + if (!greedy_threshold) + return; + + bpf_repeat(nr_doms - 1) { + u32 dom_id = dom_rr_next(cpu); + + if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold && + scx_bpf_consume(dom_id)) { + stat_add(ATROPOS_STAT_GREEDY, 1); + break; + } + } +} + +void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags) +{ + pid_t pid = p->pid; + struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); + + if (!task_ctx) { + scx_bpf_error("No task_ctx[%d]", pid); + return; + } + + task_ctx->runnable_at = bpf_ktime_get_ns(); +} + +void BPF_STRUCT_OPS(atropos_running, struct task_struct *p) +{ + struct task_ctx *taskc; + struct dom_ctx *domc; + pid_t pid = p->pid; + u32 dom_id; + + if (fifo_sched) + return; + + taskc = bpf_map_lookup_elem(&task_data, &pid); + if (!taskc) { + scx_bpf_error("No task_ctx[%d]", pid); + return; + } + dom_id = taskc->dom_id; + + domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); + if (!domc) { + scx_bpf_error("No dom[%u]", dom_id); + return; + } + + /* + * Global vtime always progresses forward as tasks start executing. The + * test and update can be performed concurrently from multiple CPUs and + * thus racy. Any error should be contained and temporary. Let's just + * live with it. + */ + if (vtime_before(domc->vtime_now, p->scx.dsq_vtime)) + domc->vtime_now = p->scx.dsq_vtime; +} + +void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable) +{ + if (fifo_sched) + return; + + /* scale the execution time by the inverse of the weight and charge */ + p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; +} + +void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags) +{ + pid_t pid = p->pid; + struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); + + if (!task_ctx) { + scx_bpf_error("No task_ctx[%d]", pid); + return; + } + + task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at; + task_ctx->runnable_at = 0; +} + +void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight) +{ + pid_t pid = p->pid; + struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); + + if (!task_ctx) { + scx_bpf_error("No task_ctx[%d]", pid); + return; + } + + task_ctx->weight = weight; +} + +struct pick_task_domain_loop_ctx { + struct task_struct *p; + const struct cpumask *cpumask; + u64 dom_mask; + u32 dom_rr_base; + u32 dom_id; +}; + +static int pick_task_domain_loopfn(u32 idx, void *data) +{ + struct pick_task_domain_loop_ctx *lctx = data; + u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms; + + if (dom_id >= MAX_DOMS) + return 1; + + if (cpumask_intersects_domain(lctx->cpumask, dom_id)) { + lctx->dom_mask |= 1LLU << dom_id; + if (lctx->dom_id == MAX_DOMS) + lctx->dom_id = dom_id; + } + return 0; +} + +static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p, + const struct cpumask *cpumask) +{ + struct pick_task_domain_loop_ctx lctx = { + .p = p, + .cpumask = cpumask, + .dom_id = MAX_DOMS, + }; + s32 cpu = bpf_get_smp_processor_id(); + + if (cpu < 0 || cpu >= MAX_CPUS) + return MAX_DOMS; + + lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur); + + bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0); + task_ctx->dom_mask = lctx.dom_mask; + + return lctx.dom_id; +} + +static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p, + const struct cpumask *cpumask) +{ + u32 dom_id = 0; + + if (nr_doms > 1) + dom_id = pick_task_domain(task_ctx, p, cpumask); + + if (!task_set_dsq(task_ctx, p, dom_id)) + scx_bpf_error("Failed to set domain %d for %s[%d]", + dom_id, p->comm, p->pid); +} + +void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p, + const struct cpumask *cpumask) +{ + pid_t pid = p->pid; + struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid); + if (!task_ctx) { + scx_bpf_error("No task_ctx[%d]", pid); + return; + } + + task_set_domain(task_ctx, p, cpumask); +} + +s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p, + struct scx_enable_args *args) +{ + struct bpf_cpumask *cpumask; + struct task_ctx task_ctx, *map_value; + long ret; + pid_t pid; + + memset(&task_ctx, 0, sizeof(task_ctx)); + + pid = p->pid; + ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST); + if (ret) { + stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); + return ret; + } + + /* + * Read the entry from the map immediately so we can add the cpumask + * with bpf_kptr_xchg(). + */ + map_value = bpf_map_lookup_elem(&task_data, &pid); + if (!map_value) + /* Should never happen -- it was just inserted above. */ + return -EINVAL; + + cpumask = bpf_cpumask_create(); + if (!cpumask) { + bpf_map_delete_elem(&task_data, &pid); + return -ENOMEM; + } + + cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask); + if (cpumask) { + /* Should never happen as we just inserted it above. */ + bpf_cpumask_release(cpumask); + bpf_map_delete_elem(&task_data, &pid); + return -EINVAL; + } + + task_set_domain(map_value, p, p->cpus_ptr); + + return 0; +} + +void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p) +{ + pid_t pid = p->pid; + long ret = bpf_map_delete_elem(&task_data, &pid); + if (ret) { + stat_add(ATROPOS_STAT_TASK_GET_ERR, 1); + return; + } +} + +static int create_dom_dsq(u32 idx, void *data) +{ + struct dom_ctx domc_init = {}, *domc; + struct bpf_cpumask *cpumask; + u32 cpu, dom_id = idx; + s32 ret; + + ret = scx_bpf_create_dsq(dom_id, -1); + if (ret < 0) { + scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret); + return 1; + } + + ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0); + if (ret) { + scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret); + return 1; + } + + domc = bpf_map_lookup_elem(&dom_ctx, &dom_id); + if (!domc) { + /* Should never happen, we just inserted it above. */ + scx_bpf_error("No dom%u", dom_id); + return 1; + } + + cpumask = bpf_cpumask_create(); + if (!cpumask) { + scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id); + return 1; + } + + for (cpu = 0; cpu < MAX_CPUS; cpu++) { + const volatile __u64 *dmask; + + dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]); + if (!dmask) { + scx_bpf_error("array index error"); + bpf_cpumask_release(cpumask); + return 1; + } + + if (*dmask & (1LLU << (cpu % 64))) + bpf_cpumask_set_cpu(cpu, cpumask); + } + + cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask); + if (cpumask) { + scx_bpf_error("Domain %u was already present", dom_id); + bpf_cpumask_release(cpumask); + return 1; + } + + return 0; +} + +int BPF_STRUCT_OPS_SLEEPABLE(atropos_init) +{ + if (!switch_partial) + scx_bpf_switch_all(); + + bpf_loop(nr_doms, create_dom_dsq, NULL, 0); + + for (u32 i = 0; i < nr_cpus; i++) + pcpu_ctx[i].dom_rr_cur = i; + + return 0; +} + +void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei) +{ + bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg); + exit_type = ei->type; +} + +SEC(".struct_ops") +struct sched_ext_ops atropos = { + .select_cpu = (void *)atropos_select_cpu, + .enqueue = (void *)atropos_enqueue, + .dispatch = (void *)atropos_dispatch, + .runnable = (void *)atropos_runnable, + .running = (void *)atropos_running, + .stopping = (void *)atropos_stopping, + .quiescent = (void *)atropos_quiescent, + .set_weight = (void *)atropos_set_weight, + .set_cpumask = (void *)atropos_set_cpumask, + .prep_enable = (void *)atropos_prep_enable, + .disable = (void *)atropos_disable, + .init = (void *)atropos_init, + .exit = (void *)atropos_exit, + .flags = 0, + .name = "atropos", +}; diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h new file mode 100644 index 0000000000000..addf29ca104a5 --- /dev/null +++ b/tools/sched_ext/atropos/src/bpf/atropos.h @@ -0,0 +1,44 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +#ifndef __ATROPOS_H +#define __ATROPOS_H + +#include +#ifndef __kptr +#ifdef __KERNEL__ +#error "__kptr_ref not defined in the kernel" +#endif +#define __kptr +#endif + +#define MAX_CPUS 512 +#define MAX_DOMS 64 /* limited to avoid complex bitmask ops */ +#define CACHELINE_SIZE 64 + +/* Statistics */ +enum stat_idx { + ATROPOS_STAT_TASK_GET_ERR, + ATROPOS_STAT_WAKE_SYNC, + ATROPOS_STAT_PREV_IDLE, + ATROPOS_STAT_PINNED, + ATROPOS_STAT_DIRECT_DISPATCH, + ATROPOS_STAT_DSQ_DISPATCH, + ATROPOS_STAT_GREEDY, + ATROPOS_STAT_LOAD_BALANCE, + ATROPOS_STAT_LAST_TASK, + ATROPOS_NR_STATS, +}; + +struct task_ctx { + unsigned long long dom_mask; /* the domains this task can run on */ + struct bpf_cpumask __kptr *cpumask; + unsigned int dom_id; + unsigned int weight; + unsigned long long runnable_at; + unsigned long long runnable_for; + bool dispatch_local; +}; + +#endif /* __ATROPOS_H */ diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs new file mode 100644 index 0000000000000..0d313662f7136 --- /dev/null +++ b/tools/sched_ext/atropos/src/main.rs @@ -0,0 +1,942 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2. +#[path = "bpf/.output/atropos.skel.rs"] +mod atropos; +pub use atropos::*; +pub mod atropos_sys; + +use std::cell::Cell; +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::CStr; +use std::ops::Bound::{Included, Unbounded}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use ::fb_procfs as procfs; +use anyhow::{anyhow, bail, Context, Result}; +use bitvec::prelude::*; +use clap::Parser; +use log::{info, trace, warn}; +use ordered_float::OrderedFloat; + +/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF +/// part does simple round robin in each domain and the userspace part +/// calculates the load factor of each domain and tells the BPF part how to load +/// balance the domains. + +/// This scheduler demonstrates dividing scheduling logic between BPF and +/// userspace and using rust to build the userspace part. An earlier variant of +/// this scheduler was used to balance across six domains, each representing a +/// chiplet in a six-chiplet AMD processor, and could match the performance of +/// production setup using CFS. +#[derive(Debug, Parser)] +struct Opts { + /// Scheduling slice duration in microseconds. + #[clap(short, long, default_value = "20000")] + slice_us: u64, + + /// Monitoring and load balance interval in seconds. + #[clap(short, long, default_value = "2.0")] + interval: f64, + + /// Build domains according to how CPUs are grouped at this cache level + /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id. + #[clap(short = 'c', long, default_value = "3")] + cache_level: u32, + + /// Instead of using cache locality, set the cpumask for each domain + /// manually, provide multiple --cpumasks, one for each domain. E.g. + /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with + /// the corresponding CPUs belonging to each domain. Each CPU must + /// belong to precisely one domain. + #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")] + cpumasks: Vec, + + /// When non-zero, enable greedy task stealing. When a domain is idle, a + /// cpu will attempt to steal tasks from a domain with at least + /// greedy_threshold tasks enqueued. These tasks aren't permanently + /// stolen from the domain. + #[clap(short, long, default_value = "4")] + greedy_threshold: u32, + + /// The load decay factor. Every interval, the existing load is decayed + /// by this factor and new load is added. Must be in the range [0.0, + /// 0.99]. The smaller the value, the more sensitive load calculation + /// is to recent changes. When 0.0, history is ignored and the load + /// value from the latest period is used directly. + #[clap(short, long, default_value = "0.5")] + load_decay_factor: f64, + + /// Disable load balancing. Unless disabled, periodically userspace will + /// calculate the load factor of each domain and instruct BPF which + /// processes to move. + #[clap(short, long, action = clap::ArgAction::SetTrue)] + no_load_balance: bool, + + /// Put per-cpu kthreads directly into local dsq's. + #[clap(short, long, action = clap::ArgAction::SetTrue)] + kthreads_local: bool, + + /// Use FIFO scheduling instead of weighted vtime scheduling. + #[clap(short, long, action = clap::ArgAction::SetTrue)] + fifo_sched: bool, + + /// If specified, only tasks which have their scheduling policy set to + /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all + /// tasks are switched. + #[clap(short, long, action = clap::ArgAction::SetTrue)] + partial: bool, + + /// Enable verbose output including libbpf details. Specify multiple + /// times to increase verbosity. + #[clap(short, long, action = clap::ArgAction::Count)] + verbose: u8, +} + +fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result { + Ok(reader + .read_stat() + .context("Failed to read procfs")? + .total_cpu + .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?) +} + +fn now_monotonic() -> u64 { + let mut time = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) }; + assert!(ret == 0); + time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 +} + +fn clear_map(map: &mut libbpf_rs::Map) { + // XXX: libbpf_rs has some design flaw that make it impossible to + // delete while iterating despite it being safe so we alias it here + let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) }; + for key in map.keys() { + let _ = deleter.delete(&key); + } +} + +#[derive(Debug)] +struct TaskLoad { + runnable_for: u64, + load: f64, +} + +#[derive(Debug)] +struct TaskInfo { + pid: i32, + dom_mask: u64, + migrated: Cell, +} + +struct LoadBalancer<'a, 'b, 'c> { + maps: AtroposMapsMut<'a>, + task_loads: &'b mut BTreeMap, + nr_doms: usize, + load_decay_factor: f64, + + tasks_by_load: Vec, TaskInfo>>, + load_avg: f64, + dom_loads: Vec, + + imbal: Vec, + doms_to_push: BTreeMap, u32>, + doms_to_pull: BTreeMap, u32>, + + nr_lb_data_errors: &'c mut u64, +} + +impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> { + const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10; + const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1; + const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50; + + fn new( + maps: AtroposMapsMut<'a>, + task_loads: &'b mut BTreeMap, + nr_doms: usize, + load_decay_factor: f64, + nr_lb_data_errors: &'c mut u64, + ) -> Self { + Self { + maps, + task_loads, + nr_doms, + load_decay_factor, + + tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(), + load_avg: 0f64, + dom_loads: vec![0.0; nr_doms], + + imbal: vec![0.0; nr_doms], + doms_to_pull: BTreeMap::new(), + doms_to_push: BTreeMap::new(), + + nr_lb_data_errors, + } + } + + fn read_task_loads(&mut self, period: Duration) -> Result<()> { + let now_mono = now_monotonic(); + let task_data = self.maps.task_data(); + let mut this_task_loads = BTreeMap::::new(); + let mut load_sum = 0.0f64; + self.dom_loads = vec![0f64; self.nr_doms]; + + for key in task_data.keys() { + if let Some(task_ctx_vec) = task_data + .lookup(&key, libbpf_rs::MapFlags::ANY) + .context("Failed to lookup task_data")? + { + let task_ctx = + unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) }; + let pid = i32::from_ne_bytes( + key.as_slice() + .try_into() + .context("Invalid key length in task_data map")?, + ); + + let (this_at, this_for, weight) = unsafe { + ( + std::ptr::read_volatile(&task_ctx.runnable_at as *const u64), + std::ptr::read_volatile(&task_ctx.runnable_for as *const u64), + std::ptr::read_volatile(&task_ctx.weight as *const u32), + ) + }; + + let (mut delta, prev_load) = match self.task_loads.get(&pid) { + Some(prev) => (this_for - prev.runnable_for, Some(prev.load)), + None => (this_for, None), + }; + + // Non-zero this_at indicates that the task is currently + // runnable. Note that we read runnable_at and runnable_for + // without any synchronization and there is a small window + // where we end up misaccounting. While this can cause + // temporary error, it's unlikely to cause any noticeable + // misbehavior especially given the load value clamping. + if this_at > 0 && this_at < now_mono { + delta += now_mono - this_at; + } + + delta = delta.min(period.as_nanos() as u64); + let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64) + .clamp(0.0, weight as f64); + + let this_load = match prev_load { + Some(prev_load) => { + prev_load * self.load_decay_factor + + this_load * (1.0 - self.load_decay_factor) + } + None => this_load, + }; + + this_task_loads.insert( + pid, + TaskLoad { + runnable_for: this_for, + load: this_load, + }, + ); + + load_sum += this_load; + self.dom_loads[task_ctx.dom_id as usize] += this_load; + // Only record pids that are eligible for load balancing + if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) { + continue; + } + self.tasks_by_load[task_ctx.dom_id as usize].insert( + OrderedFloat(this_load), + TaskInfo { + pid, + dom_mask: task_ctx.dom_mask, + migrated: Cell::new(false), + }, + ); + } + } + + self.load_avg = load_sum / self.nr_doms as f64; + *self.task_loads = this_task_loads; + Ok(()) + } + + // To balance dom loads we identify doms with lower and higher load than average + fn calculate_dom_load_balance(&mut self) -> Result<()> { + for (dom, dom_load) in self.dom_loads.iter().enumerate() { + let imbal = dom_load - self.load_avg; + if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO { + if imbal > 0f64 { + self.doms_to_push.insert(OrderedFloat(imbal), dom as u32); + } else { + self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32); + } + self.imbal[dom] = imbal; + } + } + Ok(()) + } + + // Find the first candidate pid which hasn't already been migrated and + // can run in @pull_dom. + fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)> + where + I: IntoIterator, &'d TaskInfo)>, + { + match tasks_by_load + .into_iter() + .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0) + .next() + { + Some((OrderedFloat(load), task)) => Some((*load, task)), + None => None, + } + } + + fn pick_victim( + &self, + (push_dom, to_push): (u32, f64), + (pull_dom, to_pull): (u32, f64), + ) -> Option<(&TaskInfo, f64)> { + let to_xfer = to_pull.min(to_push); + + trace!( + "considering dom {}@{:.2} -> {}@{:.2}", + push_dom, + to_push, + pull_dom, + to_pull + ); + + let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs(); + + trace!( + "to_xfer={:.2} tasks_by_load={:?}", + to_xfer, + &self.tasks_by_load[push_dom as usize] + ); + + // We want to pick a task to transfer from push_dom to pull_dom to + // maximize the reduction of load imbalance between the two. IOW, + // pick a task which has the closest load value to $to_xfer that can + // be migrated. Find such task by locating the first migratable task + // while scanning left from $to_xfer and the counterpart while + // scanning right and picking the better of the two. + let (load, task, new_imbal) = match ( + Self::find_first_candidate( + self.tasks_by_load[push_dom as usize] + .range((Unbounded, Included(&OrderedFloat(to_xfer)))) + .rev(), + pull_dom, + ), + Self::find_first_candidate( + self.tasks_by_load[push_dom as usize] + .range((Included(&OrderedFloat(to_xfer)), Unbounded)), + pull_dom, + ), + ) { + (None, None) => return None, + (Some((load, task)), None) | (None, Some((load, task))) => { + (load, task, calc_new_imbal(load)) + } + (Some((load0, task0)), Some((load1, task1))) => { + let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1)); + if new_imbal0 <= new_imbal1 { + (load0, task0, new_imbal0) + } else { + (load1, task1, new_imbal1) + } + } + }; + + // If the best candidate can't reduce the imbalance, there's nothing + // to do for this pair. + let old_imbal = to_push + to_pull; + if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal { + trace!( + "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}", + task.pid, + push_dom, + pull_dom, + old_imbal, + new_imbal + ); + return None; + } + + trace!( + "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}", + task.pid, + push_dom, + pull_dom, + old_imbal, + new_imbal, + ); + + Some((task, load)) + } + + // Actually execute the load balancing. Concretely this writes pid -> dom + // entries into the lb_data map for bpf side to consume. + fn load_balance(&mut self) -> Result<()> { + clear_map(self.maps.lb_data()); + + trace!("imbal={:?}", &self.imbal); + trace!("doms_to_push={:?}", &self.doms_to_push); + trace!("doms_to_pull={:?}", &self.doms_to_pull); + + // Push from the most imbalanced to least. + while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() { + let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO; + let mut pushed = 0f64; + + // Transfer tasks from push_dom to reduce imbalance. + loop { + let last_pushed = pushed; + + // Pull from the most imbalaned to least. + let mut doms_to_pull = BTreeMap::<_, _>::new(); + std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull); + let mut pull_doms = doms_to_pull.into_iter().rev().collect::>(); + + for (to_pull, pull_dom) in pull_doms.iter_mut() { + if let Some((task, load)) = + self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull))) + { + // Execute migration. + task.migrated.set(true); + to_push -= load; + *to_pull -= load; + pushed += load; + + // Ask BPF code to execute the migration. + let pid = task.pid; + let cpid = (pid as libc::pid_t).to_ne_bytes(); + if let Err(e) = self.maps.lb_data().update( + &cpid, + &pull_dom.to_ne_bytes(), + libbpf_rs::MapFlags::NO_EXIST, + ) { + warn!( + "Failed to update lb_data map for pid={} error={:?}", + pid, &e + ); + *self.nr_lb_data_errors += 1; + } + + // Always break after a successful migration so that + // the pulling domains are always considered in the + // descending imbalance order. + break; + } + } + + pull_doms + .into_iter() + .map(|(k, v)| self.doms_to_pull.insert(k, v)) + .count(); + + // Stop repeating if nothing got transferred or pushed enough. + if pushed == last_pushed || pushed >= push_max { + break; + } + } + } + Ok(()) + } +} + +struct Scheduler<'a> { + skel: AtroposSkel<'a>, + struct_ops: Option, + + nr_cpus: usize, + nr_doms: usize, + load_decay_factor: f64, + balance_load: bool, + + proc_reader: procfs::ProcReader, + + prev_at: SystemTime, + prev_total_cpu: procfs::CpuStat, + task_loads: BTreeMap, + + nr_lb_data_errors: u64, +} + +impl<'a> Scheduler<'a> { + // Returns Vec of cpuset for each dq and a vec of dq for each cpu + fn parse_cpusets( + cpumasks: &[String], + nr_cpus: usize, + ) -> Result<(Vec>, Vec)> { + if cpumasks.len() > atropos_sys::MAX_DOMS as usize { + bail!( + "Number of requested DSQs ({}) is greater than MAX_DOMS ({})", + cpumasks.len(), + atropos_sys::MAX_DOMS + ); + } + let mut cpus = vec![-1i32; nr_cpus]; + let mut cpusets = + vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()]; + for (dq, cpumask) in cpumasks.iter().enumerate() { + let hex_str = { + let mut tmp_str = cpumask + .strip_prefix("0x") + .unwrap_or(cpumask) + .replace('_', ""); + if tmp_str.len() % 2 != 0 { + tmp_str = "0".to_string() + &tmp_str; + } + tmp_str + }; + let byte_vec = hex::decode(&hex_str) + .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?; + + for (index, &val) in byte_vec.iter().rev().enumerate() { + let mut v = val; + while v != 0 { + let lsb = v.trailing_zeros() as usize; + v &= !(1 << lsb); + let cpu = index * 8 + lsb; + if cpu > nr_cpus { + bail!( + concat!( + "Found cpu ({}) in cpumask ({}) which is larger", + " than the number of cpus on the machine ({})" + ), + cpu, + cpumask, + nr_cpus + ); + } + if cpus[cpu] != -1 { + bail!( + "Found cpu ({}) with dq ({}) but also in cpumask ({})", + cpu, + cpus[cpu], + cpumask + ); + } + cpus[cpu] = dq as i32; + cpusets[dq].set(cpu, true); + } + } + cpusets[dq].set_uninitialized(false); + } + + for (cpu, &dq) in cpus.iter().enumerate() { + if dq < 0 { + bail!( + "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.", + cpu + ); + } + } + + Ok((cpusets, cpus)) + } + + // Returns Vec of cpuset for each dq and a vec of dq for each cpu + fn cpusets_from_cache( + level: u32, + nr_cpus: usize, + ) -> Result<(Vec>, Vec)> { + let mut cpu_to_cache = vec![]; // (cpu_id, cache_id) + let mut cache_ids = BTreeSet::::new(); + let mut nr_not_found = 0; + + // Build cpu -> cache ID mapping. + for cpu in 0..nr_cpus { + let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level); + let id = match std::fs::read_to_string(&path) { + Ok(val) => val + .trim() + .parse::() + .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + nr_not_found += 1; + 0 + } + Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)), + }; + + cpu_to_cache.push(id); + cache_ids.insert(id); + } + + if nr_not_found > 1 { + warn!( + "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0", + level, nr_not_found, nr_cpus + ); + } + + // Cache IDs may have holes. Assign consecutive domain IDs to + // existing cache IDs. + let mut cache_to_dom = BTreeMap::::new(); + let mut nr_doms = 0; + for cache_id in cache_ids.iter() { + cache_to_dom.insert(*cache_id, nr_doms); + nr_doms += 1; + } + + if nr_doms > atropos_sys::MAX_DOMS { + bail!( + "Total number of doms {} is greater than MAX_DOMS ({})", + nr_doms, + atropos_sys::MAX_DOMS + ); + } + + // Build and return dom -> cpumask and cpu -> dom mappings. + let mut cpusets = + vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize]; + let mut cpu_to_dom = vec![]; + + for cpu in 0..nr_cpus { + let dom_id = cache_to_dom[&cpu_to_cache[cpu]]; + cpusets[dom_id as usize].set(cpu, true); + cpu_to_dom.push(dom_id as i32); + } + + Ok((cpusets, cpu_to_dom)) + } + + fn init(opts: &Opts) -> Result { + // Open the BPF prog first for verification. + let mut skel_builder = AtroposSkelBuilder::default(); + skel_builder.obj_builder.debug(opts.verbose > 0); + let mut skel = skel_builder.open().context("Failed to open BPF program")?; + + let nr_cpus = libbpf_rs::num_possible_cpus().unwrap(); + if nr_cpus > atropos_sys::MAX_CPUS as usize { + bail!( + "nr_cpus ({}) is greater than MAX_CPUS ({})", + nr_cpus, + atropos_sys::MAX_CPUS + ); + } + + // Initialize skel according to @opts. + let (cpusets, cpus) = if opts.cpumasks.len() > 0 { + Self::parse_cpusets(&opts.cpumasks, nr_cpus)? + } else { + Self::cpusets_from_cache(opts.cache_level, nr_cpus)? + }; + let nr_doms = cpusets.len(); + skel.rodata().nr_doms = nr_doms as u32; + skel.rodata().nr_cpus = nr_cpus as u32; + + for (cpu, dom) in cpus.iter().enumerate() { + skel.rodata().cpu_dom_id_map[cpu] = *dom as u32; + } + + for (dom, cpuset) in cpusets.iter().enumerate() { + let raw_cpuset_slice = cpuset.as_raw_slice(); + let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom]; + let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len()); + left.clone_from_slice(cpuset.as_raw_slice()); + let cpumask_str = dom_cpumask_slice + .iter() + .take((nr_cpus + 63) / 64) + .rev() + .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x)); + info!( + "DOM[{:02}] cpumask{} ({} cpus)", + dom, + &cpumask_str, + cpuset.count_ones() + ); + } + + skel.rodata().slice_us = opts.slice_us; + skel.rodata().kthreads_local = opts.kthreads_local; + skel.rodata().fifo_sched = opts.fifo_sched; + skel.rodata().switch_partial = opts.partial; + skel.rodata().greedy_threshold = opts.greedy_threshold; + + // Attach. + let mut skel = skel.load().context("Failed to load BPF program")?; + skel.attach().context("Failed to attach BPF program")?; + let struct_ops = Some( + skel.maps_mut() + .atropos() + .attach_struct_ops() + .context("Failed to attach atropos struct ops")?, + ); + info!("Atropos Scheduler Attached"); + + // Other stuff. + let mut proc_reader = procfs::ProcReader::new(); + let prev_total_cpu = read_total_cpu(&mut proc_reader)?; + + Ok(Self { + skel, + struct_ops, // should be held to keep it attached + + nr_cpus, + nr_doms, + load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99), + balance_load: !opts.no_load_balance, + + proc_reader, + + prev_at: SystemTime::now(), + prev_total_cpu, + task_loads: BTreeMap::new(), + + nr_lb_data_errors: 0, + }) + } + + fn get_cpu_busy(&mut self) -> Result { + let total_cpu = read_total_cpu(&mut self.proc_reader)?; + let busy = match (&self.prev_total_cpu, &total_cpu) { + ( + procfs::CpuStat { + user_usec: Some(prev_user), + nice_usec: Some(prev_nice), + system_usec: Some(prev_system), + idle_usec: Some(prev_idle), + iowait_usec: Some(prev_iowait), + irq_usec: Some(prev_irq), + softirq_usec: Some(prev_softirq), + stolen_usec: Some(prev_stolen), + guest_usec: _, + guest_nice_usec: _, + }, + procfs::CpuStat { + user_usec: Some(curr_user), + nice_usec: Some(curr_nice), + system_usec: Some(curr_system), + idle_usec: Some(curr_idle), + iowait_usec: Some(curr_iowait), + irq_usec: Some(curr_irq), + softirq_usec: Some(curr_softirq), + stolen_usec: Some(curr_stolen), + guest_usec: _, + guest_nice_usec: _, + }, + ) => { + let idle_usec = curr_idle - prev_idle; + let iowait_usec = curr_iowait - prev_iowait; + let user_usec = curr_user - prev_user; + let system_usec = curr_system - prev_system; + let nice_usec = curr_nice - prev_nice; + let irq_usec = curr_irq - prev_irq; + let softirq_usec = curr_softirq - prev_softirq; + let stolen_usec = curr_stolen - prev_stolen; + + let busy_usec = + user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec; + let total_usec = idle_usec + busy_usec + iowait_usec; + busy_usec as f64 / total_usec as f64 + } + _ => { + bail!("Some procfs stats are not populated!"); + } + }; + + self.prev_total_cpu = total_cpu; + Ok(busy) + } + + fn read_bpf_stats(&mut self) -> Result> { + let mut maps = self.skel.maps_mut(); + let stats_map = maps.stats(); + let mut stats: Vec = Vec::new(); + let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus]; + + for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS { + let cpu_stat_vec = stats_map + .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("Failed to lookup stat {}", stat))? + .expect("per-cpu stat should exist"); + let sum = cpu_stat_vec + .iter() + .map(|val| { + u64::from_ne_bytes( + val.as_slice() + .try_into() + .expect("Invalid value length in stat map"), + ) + }) + .sum(); + stats_map + .update_percpu( + &(stat as u32).to_ne_bytes(), + &zero_vec, + libbpf_rs::MapFlags::ANY, + ) + .context("Failed to zero stat")?; + stats.push(sum); + } + Ok(stats) + } + + fn report( + &self, + stats: &Vec, + cpu_busy: f64, + processing_dur: Duration, + load_avg: f64, + dom_loads: &Vec, + imbal: &Vec, + ) { + let stat = |idx| stats[idx as usize]; + let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC) + + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE) + + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED) + + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH) + + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH) + + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY) + + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK); + + info!( + "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms", + cpu_busy * 100.0, + load_avg, + stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize], + stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize], + self.nr_lb_data_errors, + processing_dur.as_millis(), + ); + + let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0; + + info!( + "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}", + total, + stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC), + stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE), + stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED), + stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH), + stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH), + stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY), + ); + + for i in 0..self.nr_doms { + info!( + "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}", + i, + dom_loads[i], + if imbal[i] < 0.0 { -imbal[i] } else { 0.0 }, + if imbal[i] > 0.0 { imbal[i] } else { 0.0 }, + ); + } + } + + fn step(&mut self) -> Result<()> { + let started_at = std::time::SystemTime::now(); + let bpf_stats = self.read_bpf_stats()?; + let cpu_busy = self.get_cpu_busy()?; + + let mut lb = LoadBalancer::new( + self.skel.maps_mut(), + &mut self.task_loads, + self.nr_doms, + self.load_decay_factor, + &mut self.nr_lb_data_errors, + ); + + lb.read_task_loads(started_at.duration_since(self.prev_at)?)?; + lb.calculate_dom_load_balance()?; + + if self.balance_load { + lb.load_balance()?; + } + + // Extract fields needed for reporting and drop lb to release + // mutable borrows. + let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal); + + self.report( + &bpf_stats, + cpu_busy, + std::time::SystemTime::now().duration_since(started_at)?, + load_avg, + &dom_loads, + &imbal, + ); + + self.prev_at = started_at; + Ok(()) + } + + fn read_bpf_exit_type(&mut self) -> i32 { + unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) } + } + + fn report_bpf_exit_type(&mut self) -> Result<()> { + // Report msg if EXT_OPS_EXIT_ERROR. + match self.read_bpf_exit_type() { + 0 => Ok(()), + etype if etype == 2 => { + let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) }; + let msg = cstr + .to_str() + .context("Failed to convert exit msg to string") + .unwrap(); + bail!("BPF exit_type={} msg={}", etype, msg); + } + etype => { + info!("BPF exit_type={}", etype); + Ok(()) + } + } + } +} + +impl<'a> Drop for Scheduler<'a> { + fn drop(&mut self) { + if let Some(struct_ops) = self.struct_ops.take() { + drop(struct_ops); + } + } +} + +fn main() -> Result<()> { + let opts = Opts::parse(); + + let llv = match opts.verbose { + 0 => simplelog::LevelFilter::Info, + 1 => simplelog::LevelFilter::Debug, + _ => simplelog::LevelFilter::Trace, + }; + let mut lcfg = simplelog::ConfigBuilder::new(); + lcfg.set_time_level(simplelog::LevelFilter::Error) + .set_location_level(simplelog::LevelFilter::Off) + .set_target_level(simplelog::LevelFilter::Off) + .set_thread_level(simplelog::LevelFilter::Off); + simplelog::TermLogger::init( + llv, + lcfg.build(), + simplelog::TerminalMode::Stderr, + simplelog::ColorChoice::Auto, + )?; + + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_clone = shutdown.clone(); + ctrlc::set_handler(move || { + shutdown_clone.store(true, Ordering::Relaxed); + }) + .context("Error setting Ctrl-C handler")?; + + let mut sched = Scheduler::init(&opts)?; + + while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 { + std::thread::sleep(Duration::from_secs_f64(opts.interval)); + sched.step()?; + } + + sched.report_bpf_exit_type() +} diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h new file mode 100644 index 0000000000000..719225b166269 --- /dev/null +++ b/tools/sched_ext/gnu/stubs.h @@ -0,0 +1 @@ +/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h new file mode 100644 index 0000000000000..e56de9dc86f28 --- /dev/null +++ b/tools/sched_ext/scx_common.bpf.h @@ -0,0 +1,288 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#ifndef __SCHED_EXT_COMMON_BPF_H +#define __SCHED_EXT_COMMON_BPF_H + +#include "vmlinux.h" +#include +#include +#include +#include "user_exit_info.h" + +#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ +#define PF_EXITING 0x00000004 +#define CLOCK_MONOTONIC 1 + +/* + * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can + * lead to really confusing misbehaviors. Let's trigger a build failure. + */ +static inline void ___vmlinux_h_sanity_check___(void) +{ + _Static_assert(SCX_DSQ_FLAG_BUILTIN, + "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole"); +} + +void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; + +static inline __attribute__((format(printf, 1, 2))) +void ___scx_bpf_error_format_checker(const char *fmt, ...) {} + +/* + * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments + * instead of an array of u64. Note that __param[] must have at least one + * element to keep the verifier happy. + */ +#define scx_bpf_error(fmt, args...) \ +({ \ + static char ___fmt[] = fmt; \ + unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \ + \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + ___bpf_fill(___param, args); \ + _Pragma("GCC diagnostic pop") \ + \ + scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \ + \ + ___scx_bpf_error_format_checker(fmt, ##args); \ +}) + +void scx_bpf_switch_all(void) __ksym; +s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym; +bool scx_bpf_consume(u64 dsq_id) __ksym; +u32 scx_bpf_dispatch_nr_slots(void) __ksym; +void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym; +void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym; +void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym; +s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym; +bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym; +s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym; +const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym; +const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym; +void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym; +void scx_bpf_destroy_dsq(u64 dsq_id) __ksym; +bool scx_bpf_task_running(const struct task_struct *p) __ksym; +s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; +struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym; +u32 scx_bpf_reenqueue_local(void) __ksym; + +#define BPF_STRUCT_OPS(name, args...) \ +SEC("struct_ops/"#name) \ +BPF_PROG(name, ##args) + +#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \ +SEC("struct_ops.s/"#name) \ +BPF_PROG(name, ##args) + +/** + * MEMBER_VPTR - Obtain the verified pointer to a struct or array member + * @base: struct or array to index + * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...) + * + * The verifier often gets confused by the instruction sequence the compiler + * generates for indexing struct fields or arrays. This macro forces the + * compiler to generate a code sequence which first calculates the byte offset, + * checks it against the struct or array size and add that byte offset to + * generate the pointer to the member to help the verifier. + * + * Ideally, we want to abort if the calculated offset is out-of-bounds. However, + * BPF currently doesn't support abort, so evaluate to NULL instead. The caller + * must check for NULL and take appropriate action to appease the verifier. To + * avoid confusing the verifier, it's best to check for NULL and dereference + * immediately. + * + * vptr = MEMBER_VPTR(my_array, [i][j]); + * if (!vptr) + * return error; + * *vptr = new_value; + */ +#define MEMBER_VPTR(base, member) (typeof(base member) *)({ \ + u64 __base = (u64)base; \ + u64 __addr = (u64)&(base member) - __base; \ + asm volatile ( \ + "if %0 <= %[max] goto +2\n" \ + "%0 = 0\n" \ + "goto +1\n" \ + "%0 += %1\n" \ + : "+r"(__addr) \ + : "r"(__base), \ + [max]"i"(sizeof(base) - sizeof(base member))); \ + __addr; \ +}) + +/* + * BPF core and other generic helpers + */ + +/* list and rbtree */ +#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) +#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) + +void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; +void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; + +#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) +#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) + +void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; +void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym; +struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; +struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; +struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, + struct bpf_rb_node *node) __ksym; +void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, + bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym; +struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; + +/* task */ +struct task_struct *bpf_task_from_pid(s32 pid) __ksym; +struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; +void bpf_task_release(struct task_struct *p) __ksym; + +/* cgroup */ +struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; +void bpf_cgroup_release(struct cgroup *cgrp) __ksym; +struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; + +/* cpumask */ +struct bpf_cpumask *bpf_cpumask_create(void) __ksym; +struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; +void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym; +u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym; +u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym; +void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; +void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; +bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym; +bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; +bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym; +void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym; +void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym; +bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1, + const struct cpumask *src2) __ksym; +void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1, + const struct cpumask *src2) __ksym; +void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1, + const struct cpumask *src2) __ksym; +bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym; +bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym; +bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym; +bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym; +bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; +void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; +u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym; +u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym; + +/* rcu */ +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; + +/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */ +struct bpf_iter_num; + +extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym; +extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym; +extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym; + +#ifndef bpf_for_each +/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for + * using BPF open-coded iterators without having to write mundane explicit + * low-level loop logic. Instead, it provides for()-like generic construct + * that can be used pretty naturally. E.g., for some hypothetical cgroup + * iterator, you'd write: + * + * struct cgroup *cg, *parent_cg = <...>; + * + * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { + * bpf_printk("Child cgroup id = %d", cg->cgroup_id); + * if (cg->cgroup_id == 123) + * break; + * } + * + * I.e., it looks almost like high-level for each loop in other languages, + * supports continue/break, and is verifiable by BPF verifier. + * + * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) + * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to + * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int + * *`, not just `int`. So for integers bpf_for() is more convenient. + * + * Note: this macro relies on C99 feature of allowing to declare variables + * inside for() loop, bound to for() loop lifetime. It also utilizes GCC + * extension: __attribute__((cleanup())), supported by both GCC and + * Clang. + */ +#define bpf_for_each(type, cur, args...) for ( \ + /* initialize and define destructor */ \ + struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ + cleanup(bpf_iter_##type##_destroy))), \ + /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = ( \ + bpf_iter_##type##_new(&___it, ##args), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ + /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_##type##_destroy, (void *)0); \ + /* iteration and termination check */ \ + (((cur) = bpf_iter_##type##_next(&___it))); \ +) +#endif /* bpf_for_each */ + +#ifndef bpf_for +/* bpf_for(i, start, end) implements a for()-like looping construct that sets + * provided integer variable *i* to values starting from *start* through, + * but not including, *end*. It also proves to BPF verifier that *i* belongs + * to range [start, end), so this can be used for accessing arrays without + * extra checks. + * + * Note: *start* and *end* are assumed to be expressions with no side effects + * and whose values do not change throughout bpf_for() loop execution. They do + * not have to be statically known or constant, though. + * + * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() + * loop bound variables and cleanup attribute, supported by GCC and Clang. + */ +#define bpf_for(i, start, end) for ( \ + /* initialize and define destructor */ \ + struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ + cleanup(bpf_iter_num_destroy))), \ + /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = ( \ + bpf_iter_num_new(&___it, (start), (end)), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ + /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_num_destroy, (void *)0); \ + ({ \ + /* iteration step */ \ + int *___t = bpf_iter_num_next(&___it); \ + /* termination and bounds check */ \ + (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ + }); \ +) +#endif /* bpf_for */ + +#ifndef bpf_repeat +/* bpf_repeat(N) performs N iterations without exposing iteration number + * + * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() + * loop bound variables and cleanup attribute, supported by GCC and Clang. + */ +#define bpf_repeat(N) for ( \ + /* initialize and define destructor */ \ + struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ + cleanup(bpf_iter_num_destroy))), \ + /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = ( \ + bpf_iter_num_new(&___it, 0, (N)), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ + /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_num_destroy, (void *)0); \ + bpf_iter_num_next(&___it); \ + /* nothing here */ \ +) +#endif /* bpf_repeat */ + +#endif /* __SCHED_EXT_COMMON_BPF_H */ diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c new file mode 100644 index 0000000000000..4cec04b4c2ede --- /dev/null +++ b/tools/sched_ext/scx_example_central.bpf.c @@ -0,0 +1,334 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A central FIFO sched_ext scheduler which demonstrates the followings: + * + * a. Making all scheduling decisions from one CPU: + * + * The central CPU is the only one making scheduling decisions. All other + * CPUs kick the central CPU when they run out of tasks to run. + * + * There is one global BPF queue and the central CPU schedules all CPUs by + * dispatching from the global queue to each CPU's local dsq from dispatch(). + * This isn't the most straightforward. e.g. It'd be easier to bounce + * through per-CPU BPF queues. The current design is chosen to maximally + * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching. + * + * b. Tickless operation + * + * All tasks are dispatched with the infinite slice which allows stopping the + * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full + * parameter. The tickless operation can be observed through + * /proc/interrupts. + * + * Periodic switching is enforced by a periodic timer checking all CPUs and + * preempting them as necessary. Unfortunately, BPF timer currently doesn't + * have a way to pin to a specific CPU, so the periodic timer isn't pinned to + * the central CPU. + * + * c. Preemption + * + * Kthreads are unconditionally queued to the head of a matching local dsq + * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always + * prioritized over user threads, which is required for ensuring forward + * progress as e.g. the periodic timer may run on a ksoftirqd and if the + * ksoftirqd gets starved by a user thread, there may not be anything else to + * vacate that user thread. + * + * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the + * next tasks. + * + * This scheduler is designed to maximize usage of various SCX mechanisms. A + * more practical implementation would likely put the scheduling loop outside + * the central CPU's dispatch() path and add some form of priority mechanism. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#include "scx_common.bpf.h" + +char _license[] SEC("license") = "GPL"; + +enum { + FALLBACK_DSQ_ID = 0, + MAX_CPUS = 4096, + MS_TO_NS = 1000LLU * 1000, + TIMER_INTERVAL_NS = 1 * MS_TO_NS, +}; + +const volatile bool switch_partial; +const volatile s32 central_cpu; +const volatile u32 nr_cpu_ids = 64; /* !0 for veristat, set during init */ + +u64 nr_total, nr_locals, nr_queued, nr_lost_pids; +u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries; +u64 nr_overflows; + +struct user_exit_info uei; + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 4096); + __type(value, s32); +} central_q SEC(".maps"); + +/* can't use percpu map due to bad lookups */ +static bool cpu_gimme_task[MAX_CPUS]; +static u64 cpu_started_at[MAX_CPUS]; + +struct central_timer { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct central_timer); +} central_timer SEC(".maps"); + +static bool vtime_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, + s32 prev_cpu, u64 wake_flags) +{ + /* + * Steer wakeups to the central CPU as much as possible to avoid + * disturbing other CPUs. It's safe to blindly return the central cpu as + * select_cpu() is a hint and if @p can't be on it, the kernel will + * automatically pick a fallback CPU. + */ + return central_cpu; +} + +void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags) +{ + s32 pid = p->pid; + + __sync_fetch_and_add(&nr_total, 1); + + /* + * Push per-cpu kthreads at the head of local dsq's and preempt the + * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked + * behind other threads which is necessary for forward progress + * guarantee as we depend on the BPF timer which may run from ksoftirqd. + */ + if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) { + __sync_fetch_and_add(&nr_locals, 1); + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF, + enq_flags | SCX_ENQ_PREEMPT); + return; + } + + if (bpf_map_push_elem(¢ral_q, &pid, 0)) { + __sync_fetch_and_add(&nr_overflows, 1); + scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags); + return; + } + + __sync_fetch_and_add(&nr_queued, 1); + + if (!scx_bpf_task_running(p)) + scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); +} + +static bool dispatch_to_cpu(s32 cpu) +{ + struct task_struct *p; + s32 pid; + + bpf_repeat(BPF_MAX_LOOPS) { + if (bpf_map_pop_elem(¢ral_q, &pid)) + break; + + __sync_fetch_and_sub(&nr_queued, 1); + + p = bpf_task_from_pid(pid); + if (!p) { + __sync_fetch_and_add(&nr_lost_pids, 1); + continue; + } + + /* + * If we can't run the task at the top, do the dumb thing and + * bounce it to the fallback dsq. + */ + if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) { + __sync_fetch_and_add(&nr_mismatches, 1); + scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0); + bpf_task_release(p); + continue; + } + + /* dispatch to local and mark that @cpu doesn't need more */ + scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0); + + if (cpu != central_cpu) + scx_bpf_kick_cpu(cpu, 0); + + bpf_task_release(p); + return true; + } + + return false; +} + +void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev) +{ + if (cpu == central_cpu) { + /* dispatch for all other CPUs first */ + __sync_fetch_and_add(&nr_dispatches, 1); + + bpf_for(cpu, 0, nr_cpu_ids) { + bool *gimme; + + if (!scx_bpf_dispatch_nr_slots()) + break; + + /* central's gimme is never set */ + gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); + if (gimme && !*gimme) + continue; + + if (dispatch_to_cpu(cpu)) + *gimme = false; + } + + /* + * Retry if we ran out of dispatch buffer slots as we might have + * skipped some CPUs and also need to dispatch for self. The ext + * core automatically retries if the local dsq is empty but we + * can't rely on that as we're dispatching for other CPUs too. + * Kick self explicitly to retry. + */ + if (!scx_bpf_dispatch_nr_slots()) { + __sync_fetch_and_add(&nr_retries, 1); + scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); + return; + } + + /* look for a task to run on the central CPU */ + if (scx_bpf_consume(FALLBACK_DSQ_ID)) + return; + dispatch_to_cpu(central_cpu); + } else { + bool *gimme; + + if (scx_bpf_consume(FALLBACK_DSQ_ID)) + return; + + gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]); + if (gimme) + *gimme = true; + + /* + * Force dispatch on the scheduling CPU so that it finds a task + * to run for us. + */ + scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); + } +} + +void BPF_STRUCT_OPS(central_running, struct task_struct *p) +{ + s32 cpu = scx_bpf_task_cpu(p); + u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); + if (started_at) + *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ +} + +void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) +{ + s32 cpu = scx_bpf_task_cpu(p); + u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]); + if (started_at) + *started_at = 0; +} + +static int central_timerfn(void *map, int *key, struct bpf_timer *timer) +{ + u64 now = bpf_ktime_get_ns(); + u64 nr_to_kick = nr_queued; + s32 i; + + bpf_for(i, 0, nr_cpu_ids) { + s32 cpu = (nr_timers + i) % nr_cpu_ids; + u64 *started_at; + + if (cpu == central_cpu) + continue; + + /* kick iff the current one exhausted its slice */ + started_at = MEMBER_VPTR(cpu_started_at, [cpu]); + if (started_at && *started_at && + vtime_before(now, *started_at + SCX_SLICE_DFL)) + continue; + + /* and there's something pending */ + if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) || + scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu)) + ; + else if (nr_to_kick) + nr_to_kick--; + else + continue; + + scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); + } + + scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT); + + bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); + __sync_fetch_and_add(&nr_timers, 1); + return 0; +} + +int BPF_STRUCT_OPS_SLEEPABLE(central_init) +{ + u32 key = 0; + struct bpf_timer *timer; + int ret; + + if (!switch_partial) + scx_bpf_switch_all(); + + ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1); + if (ret) + return ret; + + timer = bpf_map_lookup_elem(¢ral_timer, &key); + if (!timer) + return -ESRCH; + + bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); + bpf_timer_set_callback(timer, central_timerfn); + ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0); + return ret; +} + +void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei) +{ + uei_record(&uei, ei); +} + +SEC(".struct_ops") +struct sched_ext_ops central_ops = { + /* + * We are offloading all scheduling decisions to the central CPU and + * thus being the last task on a given CPU doesn't mean anything + * special. Enqueue the last tasks like any other tasks. + */ + .flags = SCX_OPS_ENQ_LAST, + + .select_cpu = (void *)central_select_cpu, + .enqueue = (void *)central_enqueue, + .dispatch = (void *)central_dispatch, + .running = (void *)central_running, + .stopping = (void *)central_stopping, + .init = (void *)central_init, + .exit = (void *)central_exit, + .name = "central", +}; diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c new file mode 100644 index 0000000000000..7ad591cbdc65c --- /dev/null +++ b/tools/sched_ext/scx_example_central.c @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "user_exit_info.h" +#include "scx_example_central.skel.h" + +const char help_fmt[] = +"A central FIFO sched_ext scheduler.\n" +"\n" +"See the top-level comment in .bpf.c for more details.\n" +"\n" +"Usage: %s [-c CPU] [-p]\n" +"\n" +" -c CPU Override the central CPU (default: 0)\n" +" -p Switch only tasks on SCHED_EXT policy intead of all\n" +" -h Display this help and exit\n"; + +static volatile int exit_req; + +static void sigint_handler(int dummy) +{ + exit_req = 1; +} + +int main(int argc, char **argv) +{ + struct scx_example_central *skel; + struct bpf_link *link; + u64 seq = 0; + s32 opt; + + signal(SIGINT, sigint_handler); + signal(SIGTERM, sigint_handler); + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + + skel = scx_example_central__open(); + assert(skel); + + skel->rodata->central_cpu = 0; + skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); + + while ((opt = getopt(argc, argv, "c:ph")) != -1) { + switch (opt) { + case 'c': + skel->rodata->central_cpu = strtoul(optarg, NULL, 0); + break; + case 'p': + skel->rodata->switch_partial = true; + break; + default: + fprintf(stderr, help_fmt, basename(argv[0])); + return opt != 'h'; + } + } + + assert(!scx_example_central__load(skel)); + + link = bpf_map__attach_struct_ops(skel->maps.central_ops); + assert(link); + + while (!exit_req && !uei_exited(&skel->bss->uei)) { + printf("[SEQ %lu]\n", seq++); + printf("total :%10lu local:%10lu queued:%10lu lost:%10lu\n", + skel->bss->nr_total, + skel->bss->nr_locals, + skel->bss->nr_queued, + skel->bss->nr_lost_pids); + printf("timer :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n", + skel->bss->nr_timers, + skel->bss->nr_dispatches, + skel->bss->nr_mismatches, + skel->bss->nr_retries); + printf("overflow:%10lu\n", + skel->bss->nr_overflows); + fflush(stdout); + sleep(1); + } + + bpf_link__destroy(link); + uei_print(&skel->bss->uei); + scx_example_central__destroy(skel); + return 0; +} diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c new file mode 100644 index 0000000000000..f6078b9a681fe --- /dev/null +++ b/tools/sched_ext/scx_example_flatcg.bpf.c @@ -0,0 +1,872 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A demo sched_ext flattened cgroup hierarchy scheduler. It implements + * hierarchical weight-based cgroup CPU control by flattening the cgroup + * hierarchy into a single layer by compounding the active weight share at each + * level. Consider the following hierarchy with weights in parentheses: + * + * R + A (100) + B (100) + * | \ C (100) + * \ D (200) + * + * Ignoring the root and threaded cgroups, only B, C and D can contain tasks. + * Let's say all three have runnable tasks. The total share that each of these + * three cgroups is entitled to can be calculated by compounding its share at + * each level. + * + * For example, B is competing against C and in that competition its share is + * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's + * share in that competition is 200/(200+100) == 1/3. B's eventual share in the + * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's + * eventual shaer is the same at 1/6. D is only competing at the top level and + * its share is 200/(100+200) == 2/3. + * + * So, instead of hierarchically scheduling level-by-level, we can consider it + * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3 + * and keep updating the eventual shares as the cgroups' runnable states change. + * + * This flattening of hierarchy can bring a substantial performance gain when + * the cgroup hierarchy is nested multiple levels. in a simple benchmark using + * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it + * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two + * apache instances competing with 2:1 weight ratio nested four level deep. + * + * However, the gain comes at the cost of not being able to properly handle + * thundering herd of cgroups. For example, if many cgroups which are nested + * behind a low priority parent cgroup wake up around the same time, they may be + * able to consume more CPU cycles than they are entitled to. In many use cases, + * this isn't a real concern especially given the performance gain. Also, there + * are ways to mitigate the problem further by e.g. introducing an extra + * scheduling layer on cgroup delegation boundaries. + * + * The scheduler first picks the cgroup to run and then schedule the tasks + * within by using nested weighted vtime scheduling by default. The + * cgroup-internal scheduling can be switched to FIFO with the -f option. + */ +#include "scx_common.bpf.h" +#include "user_exit_info.h" +#include "scx_example_flatcg.h" + +char _license[] SEC("license") = "GPL"; + +const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */ +const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL; +const volatile bool fifo_sched; +const volatile bool switch_partial; + +u64 cvtime_now; +struct user_exit_info uei; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, u64); + __uint(max_entries, FCG_NR_STATS); +} stats SEC(".maps"); + +static void stat_inc(enum fcg_stat_idx idx) +{ + u32 idx_v = idx; + + u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); + if (cnt_p) + (*cnt_p)++; +} + +struct fcg_cpu_ctx { + u64 cur_cgid; + u64 cur_at; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, struct fcg_cpu_ctx); + __uint(max_entries, 1); +} cpu_ctx SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct fcg_cgrp_ctx); +} cgrp_ctx SEC(".maps"); + +struct cgv_node { + struct bpf_rb_node rb_node; + __u64 cvtime; + __u64 cgid; +}; + +private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock; +private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node); + +struct cgv_node_stash { + struct cgv_node __kptr *node; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 16384); + __type(key, __u64); + __type(value, struct cgv_node_stash); +} cgv_node_stash SEC(".maps"); + +struct fcg_task_ctx { + u64 bypassed_at; +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct fcg_task_ctx); +} task_ctx SEC(".maps"); + +/* gets inc'd on weight tree changes to expire the cached hweights */ +unsigned long hweight_gen = 1; + +static u64 div_round_up(u64 dividend, u64 divisor) +{ + return (dividend + divisor - 1) / divisor; +} + +static bool vtime_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct cgv_node *cgc_a, *cgc_b; + + cgc_a = container_of(a, struct cgv_node, rb_node); + cgc_b = container_of(b, struct cgv_node, rb_node); + + return cgc_a->cvtime < cgc_b->cvtime; +} + +static struct fcg_cpu_ctx *find_cpu_ctx(void) +{ + struct fcg_cpu_ctx *cpuc; + u32 idx = 0; + + cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx); + if (!cpuc) { + scx_bpf_error("cpu_ctx lookup failed"); + return NULL; + } + return cpuc; +} + +static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp) +{ + struct fcg_cgrp_ctx *cgc; + + cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); + if (!cgc) { + scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id); + return NULL; + } + return cgc; +} + +static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level) +{ + struct fcg_cgrp_ctx *cgc; + + cgrp = bpf_cgroup_ancestor(cgrp, level); + if (!cgrp) { + scx_bpf_error("ancestor cgroup lookup failed"); + return NULL; + } + + cgc = find_cgrp_ctx(cgrp); + if (!cgc) + scx_bpf_error("ancestor cgrp_ctx lookup failed"); + bpf_cgroup_release(cgrp); + return cgc; +} + +static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) +{ + int level; + + if (!cgc->nr_active) { + stat_inc(FCG_STAT_HWT_SKIP); + return; + } + + if (cgc->hweight_gen == hweight_gen) { + stat_inc(FCG_STAT_HWT_CACHE); + return; + } + + stat_inc(FCG_STAT_HWT_UPDATES); + bpf_for(level, 0, cgrp->level + 1) { + struct fcg_cgrp_ctx *cgc; + bool is_active; + + cgc = find_ancestor_cgrp_ctx(cgrp, level); + if (!cgc) + break; + + if (!level) { + cgc->hweight = FCG_HWEIGHT_ONE; + cgc->hweight_gen = hweight_gen; + } else { + struct fcg_cgrp_ctx *pcgc; + + pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); + if (!pcgc) + break; + + /* + * We can be oppotunistic here and not grab the + * cgv_tree_lock and deal with the occasional races. + * However, hweight updates are already cached and + * relatively low-frequency. Let's just do the + * straightforward thing. + */ + bpf_spin_lock(&cgv_tree_lock); + is_active = cgc->nr_active; + if (is_active) { + cgc->hweight_gen = pcgc->hweight_gen; + cgc->hweight = + div_round_up(pcgc->hweight * cgc->weight, + pcgc->child_weight_sum); + } + bpf_spin_unlock(&cgv_tree_lock); + + if (!is_active) { + stat_inc(FCG_STAT_HWT_RACE); + break; + } + } + } +} + +static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) +{ + u64 delta, cvtime, max_budget; + + /* + * A node which is on the rbtree can't be pointed to from elsewhere yet + * and thus can't be updated and repositioned. Instead, we collect the + * vtime deltas separately and apply it asynchronously here. + */ + delta = cgc->cvtime_delta; + __sync_fetch_and_sub(&cgc->cvtime_delta, delta); + cvtime = cgv_node->cvtime + delta; + + /* + * Allow a cgroup to carry the maximum budget proportional to its + * hweight such that a full-hweight cgroup can immediately take up half + * of the CPUs at the most while staying at the front of the rbtree. + */ + max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) / + (2 * FCG_HWEIGHT_ONE); + if (vtime_before(cvtime, cvtime_now - max_budget)) + cvtime = cvtime_now - max_budget; + + cgv_node->cvtime = cvtime; +} + +static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc) +{ + struct cgv_node_stash *stash; + struct cgv_node *cgv_node; + u64 cgid = cgrp->kn->id; + + /* paired with cmpxchg in try_pick_next_cgroup() */ + if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) { + stat_inc(FCG_STAT_ENQ_SKIP); + return; + } + + stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); + if (!stash) { + scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid); + return; + } + + /* NULL if the node is already on the rbtree */ + cgv_node = bpf_kptr_xchg(&stash->node, NULL); + if (!cgv_node) { + stat_inc(FCG_STAT_ENQ_RACE); + return; + } + + bpf_spin_lock(&cgv_tree_lock); + cgrp_cap_budget(cgv_node, cgc); + bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); + bpf_spin_unlock(&cgv_tree_lock); +} + +void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) +{ + struct fcg_task_ctx *taskc; + struct cgroup *cgrp; + struct fcg_cgrp_ctx *cgc; + + taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); + if (!taskc) { + scx_bpf_error("task_ctx lookup failed"); + return; + } + + /* + * If select_cpu_dfl() is recommending local enqueue, the target CPU is + * idle. Follow it and charge the cgroup later in fcg_stopping() after + * the fact. Use the same mechanism to deal with tasks with custom + * affinities so that we don't have to worry about per-cgroup dq's + * containing tasks that can't be executed from some CPUs. + */ + if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) { + /* + * Tell fcg_stopping() that this bypassed the regular scheduling + * path and should be force charged to the cgroup. 0 is used to + * indicate that the task isn't bypassing, so if the current + * runtime is 0, go back by one nanosecond. + */ + taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1; + + /* + * The global dq is deprioritized as we don't want to let tasks + * to boost themselves by constraining its cpumask. The + * deprioritization is rather severe, so let's not apply that to + * per-cpu kernel threads. This is ham-fisted. We probably wanna + * implement per-cgroup fallback dq's instead so that we have + * more control over when tasks with custom cpumask get issued. + */ + if ((enq_flags & SCX_ENQ_LOCAL) || + (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) { + stat_inc(FCG_STAT_LOCAL); + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); + } else { + stat_inc(FCG_STAT_GLOBAL); + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + } + return; + } + + cgrp = scx_bpf_task_cgroup(p); + cgc = find_cgrp_ctx(cgrp); + if (!cgc) + goto out_release; + + if (fifo_sched) { + scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags); + } else { + u64 tvtime = p->scx.dsq_vtime; + + /* + * Limit the amount of budget that an idling task can accumulate + * to one slice. + */ + if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) + tvtime = cgc->tvtime_now - SCX_SLICE_DFL; + + scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, + tvtime, enq_flags); + } + + cgrp_enqueued(cgrp, cgc); +out_release: + bpf_cgroup_release(cgrp); +} + +/* + * Walk the cgroup tree to update the active weight sums as tasks wake up and + * sleep. The weight sums are used as the base when calculating the proportion a + * given cgroup or task is entitled to at each level. + */ +static void update_active_weight_sums(struct cgroup *cgrp, bool runnable) +{ + struct fcg_cgrp_ctx *cgc; + bool updated = false; + int idx; + + cgc = find_cgrp_ctx(cgrp); + if (!cgc) + return; + + /* + * In most cases, a hot cgroup would have multiple threads going to + * sleep and waking up while the whole cgroup stays active. In leaf + * cgroups, ->nr_runnable which is updated with __sync operations gates + * ->nr_active updates, so that we don't have to grab the cgv_tree_lock + * repeatedly for a busy cgroup which is staying active. + */ + if (runnable) { + if (__sync_fetch_and_add(&cgc->nr_runnable, 1)) + return; + stat_inc(FCG_STAT_ACT); + } else { + if (__sync_sub_and_fetch(&cgc->nr_runnable, 1)) + return; + stat_inc(FCG_STAT_DEACT); + } + + /* + * If @cgrp is becoming runnable, its hweight should be refreshed after + * it's added to the weight tree so that enqueue has the up-to-date + * value. If @cgrp is becoming quiescent, the hweight should be + * refreshed before it's removed from the weight tree so that the usage + * charging which happens afterwards has access to the latest value. + */ + if (!runnable) + cgrp_refresh_hweight(cgrp, cgc); + + /* propagate upwards */ + bpf_for(idx, 0, cgrp->level) { + int level = cgrp->level - idx; + struct fcg_cgrp_ctx *cgc, *pcgc = NULL; + bool propagate = false; + + cgc = find_ancestor_cgrp_ctx(cgrp, level); + if (!cgc) + break; + if (level) { + pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1); + if (!pcgc) + break; + } + + /* + * We need the propagation protected by a lock to synchronize + * against weight changes. There's no reason to drop the lock at + * each level but bpf_spin_lock() doesn't want any function + * calls while locked. + */ + bpf_spin_lock(&cgv_tree_lock); + + if (runnable) { + if (!cgc->nr_active++) { + updated = true; + if (pcgc) { + propagate = true; + pcgc->child_weight_sum += cgc->weight; + } + } + } else { + if (!--cgc->nr_active) { + updated = true; + if (pcgc) { + propagate = true; + pcgc->child_weight_sum -= cgc->weight; + } + } + } + + bpf_spin_unlock(&cgv_tree_lock); + + if (!propagate) + break; + } + + if (updated) + __sync_fetch_and_add(&hweight_gen, 1); + + if (runnable) + cgrp_refresh_hweight(cgrp, cgc); +} + +void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) +{ + struct cgroup *cgrp; + + cgrp = scx_bpf_task_cgroup(p); + update_active_weight_sums(cgrp, true); + bpf_cgroup_release(cgrp); +} + +void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) +{ + struct cgroup *cgrp; + struct fcg_cgrp_ctx *cgc; + + if (fifo_sched) + return; + + cgrp = scx_bpf_task_cgroup(p); + cgc = find_cgrp_ctx(cgrp); + if (cgc) { + /* + * @cgc->tvtime_now always progresses forward as tasks start + * executing. The test and update can be performed concurrently + * from multiple CPUs and thus racy. Any error should be + * contained and temporary. Let's just live with it. + */ + if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) + cgc->tvtime_now = p->scx.dsq_vtime; + } + bpf_cgroup_release(cgrp); +} + +void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) +{ + struct fcg_task_ctx *taskc; + struct cgroup *cgrp; + struct fcg_cgrp_ctx *cgc; + + /* scale the execution time by the inverse of the weight and charge */ + if (!fifo_sched) + p->scx.dsq_vtime += + (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; + + taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); + if (!taskc) { + scx_bpf_error("task_ctx lookup failed"); + return; + } + + if (!taskc->bypassed_at) + return; + + cgrp = scx_bpf_task_cgroup(p); + cgc = find_cgrp_ctx(cgrp); + if (cgc) { + __sync_fetch_and_add(&cgc->cvtime_delta, + p->se.sum_exec_runtime - taskc->bypassed_at); + taskc->bypassed_at = 0; + } + bpf_cgroup_release(cgrp); +} + +void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) +{ + struct cgroup *cgrp; + + cgrp = scx_bpf_task_cgroup(p); + update_active_weight_sums(cgrp, false); + bpf_cgroup_release(cgrp); +} + +void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight) +{ + struct fcg_cgrp_ctx *cgc, *pcgc = NULL; + + cgc = find_cgrp_ctx(cgrp); + if (!cgc) + return; + + if (cgrp->level) { + pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1); + if (!pcgc) + return; + } + + bpf_spin_lock(&cgv_tree_lock); + if (pcgc && cgc->nr_active) + pcgc->child_weight_sum += (s64)weight - cgc->weight; + cgc->weight = weight; + bpf_spin_unlock(&cgv_tree_lock); +} + +static bool try_pick_next_cgroup(u64 *cgidp) +{ + struct bpf_rb_node *rb_node; + struct cgv_node_stash *stash; + struct cgv_node *cgv_node; + struct fcg_cgrp_ctx *cgc; + struct cgroup *cgrp; + u64 cgid; + + /* pop the front cgroup and wind cvtime_now accordingly */ + bpf_spin_lock(&cgv_tree_lock); + + rb_node = bpf_rbtree_first(&cgv_tree); + if (!rb_node) { + bpf_spin_unlock(&cgv_tree_lock); + stat_inc(FCG_STAT_PNC_NO_CGRP); + *cgidp = 0; + return true; + } + + rb_node = bpf_rbtree_remove(&cgv_tree, rb_node); + bpf_spin_unlock(&cgv_tree_lock); + + cgv_node = container_of(rb_node, struct cgv_node, rb_node); + cgid = cgv_node->cgid; + + if (vtime_before(cvtime_now, cgv_node->cvtime)) + cvtime_now = cgv_node->cvtime; + + /* + * If lookup fails, the cgroup's gone. Free and move on. See + * fcg_cgroup_exit(). + */ + cgrp = bpf_cgroup_from_id(cgid); + if (!cgrp) { + stat_inc(FCG_STAT_PNC_GONE); + goto out_free; + } + + cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); + if (!cgc) { + bpf_cgroup_release(cgrp); + stat_inc(FCG_STAT_PNC_GONE); + goto out_free; + } + + if (!scx_bpf_consume(cgid)) { + bpf_cgroup_release(cgrp); + stat_inc(FCG_STAT_PNC_EMPTY); + goto out_stash; + } + + /* + * Successfully consumed from the cgroup. This will be our current + * cgroup for the new slice. Refresh its hweight. + */ + cgrp_refresh_hweight(cgrp, cgc); + + bpf_cgroup_release(cgrp); + + /* + * As the cgroup may have more tasks, add it back to the rbtree. Note + * that here we charge the full slice upfront and then exact later + * according to the actual consumption. This prevents lowpri thundering + * herd from saturating the machine. + */ + bpf_spin_lock(&cgv_tree_lock); + cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1); + cgrp_cap_budget(cgv_node, cgc); + bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); + bpf_spin_unlock(&cgv_tree_lock); + + *cgidp = cgid; + stat_inc(FCG_STAT_PNC_NEXT); + return true; + +out_stash: + stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); + if (!stash) { + stat_inc(FCG_STAT_PNC_GONE); + goto out_free; + } + + /* + * Paired with cmpxchg in cgrp_enqueued(). If they see the following + * transition, they'll enqueue the cgroup. If they are earlier, we'll + * see their task in the dq below and requeue the cgroup. + */ + __sync_val_compare_and_swap(&cgc->queued, 1, 0); + + if (scx_bpf_dsq_nr_queued(cgid)) { + bpf_spin_lock(&cgv_tree_lock); + bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less); + bpf_spin_unlock(&cgv_tree_lock); + } else { + cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); + if (cgv_node) { + scx_bpf_error("unexpected !NULL cgv_node stash"); + goto out_free; + } + } + + return false; + +out_free: + bpf_obj_drop(cgv_node); + return false; +} + +void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) +{ + struct fcg_cpu_ctx *cpuc; + struct fcg_cgrp_ctx *cgc; + struct cgroup *cgrp; + u64 now = bpf_ktime_get_ns(); + + cpuc = find_cpu_ctx(); + if (!cpuc) + return; + + if (!cpuc->cur_cgid) + goto pick_next_cgroup; + + if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { + if (scx_bpf_consume(cpuc->cur_cgid)) { + stat_inc(FCG_STAT_CNS_KEEP); + return; + } + stat_inc(FCG_STAT_CNS_EMPTY); + } else { + stat_inc(FCG_STAT_CNS_EXPIRE); + } + + /* + * The current cgroup is expiring. It was already charged a full slice. + * Calculate the actual usage and accumulate the delta. + */ + cgrp = bpf_cgroup_from_id(cpuc->cur_cgid); + if (!cgrp) { + stat_inc(FCG_STAT_CNS_GONE); + goto pick_next_cgroup; + } + + cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0); + if (cgc) { + /* + * We want to update the vtime delta and then look for the next + * cgroup to execute but the latter needs to be done in a loop + * and we can't keep the lock held. Oh well... + */ + bpf_spin_lock(&cgv_tree_lock); + __sync_fetch_and_add(&cgc->cvtime_delta, + (cpuc->cur_at + cgrp_slice_ns - now) * + FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); + bpf_spin_unlock(&cgv_tree_lock); + } else { + stat_inc(FCG_STAT_CNS_GONE); + } + + bpf_cgroup_release(cgrp); + +pick_next_cgroup: + cpuc->cur_at = now; + + if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { + cpuc->cur_cgid = 0; + return; + } + + bpf_repeat(BPF_MAX_LOOPS) { + if (try_pick_next_cgroup(&cpuc->cur_cgid)) + break; + } +} + +s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p, + struct scx_enable_args *args) +{ + struct fcg_task_ctx *taskc; + + /* + * @p is new. Let's ensure that its task_ctx is available. We can sleep + * in this function and the following will automatically use GFP_KERNEL. + */ + taskc = bpf_task_storage_get(&task_ctx, p, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!taskc) + return -ENOMEM; + + taskc->bypassed_at = 0; + return 0; +} + +int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp, + struct scx_cgroup_init_args *args) +{ + struct fcg_cgrp_ctx *cgc; + struct cgv_node *cgv_node; + struct cgv_node_stash empty_stash = {}, *stash; + u64 cgid = cgrp->kn->id; + int ret; + + /* + * Technically incorrect as cgroup ID is full 64bit while dq ID is + * 63bit. Should not be a problem in practice and easy to spot in the + * unlikely case that it breaks. + */ + ret = scx_bpf_create_dsq(cgid, -1); + if (ret) + return ret; + + cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!cgc) { + ret = -ENOMEM; + goto err_destroy_dsq; + } + + cgc->weight = args->weight; + cgc->hweight = FCG_HWEIGHT_ONE; + + ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash, + BPF_NOEXIST); + if (ret) { + if (ret != -ENOMEM) + scx_bpf_error("unexpected stash creation error (%d)", + ret); + goto err_destroy_dsq; + } + + stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid); + if (!stash) { + scx_bpf_error("unexpected cgv_node stash lookup failure"); + ret = -ENOENT; + goto err_destroy_dsq; + } + + cgv_node = bpf_obj_new(struct cgv_node); + if (!cgv_node) { + ret = -ENOMEM; + goto err_del_cgv_node; + } + + cgv_node->cgid = cgid; + cgv_node->cvtime = cvtime_now; + + cgv_node = bpf_kptr_xchg(&stash->node, cgv_node); + if (cgv_node) { + scx_bpf_error("unexpected !NULL cgv_node stash"); + ret = -EBUSY; + goto err_drop; + } + + return 0; + +err_drop: + bpf_obj_drop(cgv_node); +err_del_cgv_node: + bpf_map_delete_elem(&cgv_node_stash, &cgid); +err_destroy_dsq: + scx_bpf_destroy_dsq(cgid); + return ret; +} + +void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp) +{ + u64 cgid = cgrp->kn->id; + + /* + * For now, there's no way find and remove the cgv_node if it's on the + * cgv_tree. Let's drain them in the dispatch path as they get popped + * off the front of the tree. + */ + bpf_map_delete_elem(&cgv_node_stash, &cgid); + scx_bpf_destroy_dsq(cgid); +} + +s32 BPF_STRUCT_OPS(fcg_init) +{ + if (!switch_partial) + scx_bpf_switch_all(); + return 0; +} + +void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) +{ + uei_record(&uei, ei); +} + +SEC(".struct_ops") +struct sched_ext_ops flatcg_ops = { + .enqueue = (void *)fcg_enqueue, + .dispatch = (void *)fcg_dispatch, + .runnable = (void *)fcg_runnable, + .running = (void *)fcg_running, + .stopping = (void *)fcg_stopping, + .quiescent = (void *)fcg_quiescent, + .prep_enable = (void *)fcg_prep_enable, + .cgroup_set_weight = (void *)fcg_cgroup_set_weight, + .cgroup_init = (void *)fcg_cgroup_init, + .cgroup_exit = (void *)fcg_cgroup_exit, + .init = (void *)fcg_init, + .exit = (void *)fcg_exit, + .flags = SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING, + .name = "flatcg", +}; diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c new file mode 100644 index 0000000000000..f9c8a5b84a703 --- /dev/null +++ b/tools/sched_ext/scx_example_flatcg.c @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2023 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2023 Tejun Heo + * Copyright (c) 2023 David Vernet + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "user_exit_info.h" +#include "scx_example_flatcg.h" +#include "scx_example_flatcg.skel.h" + +#ifndef FILEID_KERNFS +#define FILEID_KERNFS 0xfe +#endif + +const char help_fmt[] = +"A flattened cgroup hierarchy sched_ext scheduler.\n" +"\n" +"See the top-level comment in .bpf.c for more details.\n" +"\n" +"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n" +"\n" +" -s SLICE_US Override slice duration\n" +" -i INTERVAL Report interval\n" +" -f Use FIFO scheduling instead of weighted vtime scheduling\n" +" -p Switch only tasks on SCHED_EXT policy intead of all\n" +" -h Display this help and exit\n"; + +static volatile int exit_req; + +static void sigint_handler(int dummy) +{ + exit_req = 1; +} + +static float read_cpu_util(__u64 *last_sum, __u64 *last_idle) +{ + FILE *fp; + char buf[4096]; + char *line, *cur = NULL, *tok; + __u64 sum = 0, idle = 0; + __u64 delta_sum, delta_idle; + int idx; + + fp = fopen("/proc/stat", "r"); + if (!fp) { + perror("fopen(\"/proc/stat\")"); + return 0.0; + } + + if (!fgets(buf, sizeof(buf), fp)) { + perror("fgets(\"/proc/stat\")"); + fclose(fp); + return 0.0; + } + fclose(fp); + + line = buf; + for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) { + char *endp = NULL; + __u64 v; + + if (idx == 0) { + line = NULL; + continue; + } + v = strtoull(tok, &endp, 0); + if (!endp || *endp != '\0') { + fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n", + idx, tok); + continue; + } + sum += v; + if (idx == 4) + idle = v; + } + + delta_sum = sum - *last_sum; + delta_idle = idle - *last_idle; + *last_sum = sum; + *last_idle = idle; + + return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0; +} + +static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats) +{ + __u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus]; + __u32 idx; + + memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); + + for (idx = 0; idx < FCG_NR_STATS; idx++) { + int ret, cpu; + + ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), + &idx, cnts[idx]); + if (ret < 0) + continue; + for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++) + stats[idx] += cnts[idx][cpu]; + } +} + +int main(int argc, char **argv) +{ + struct scx_example_flatcg *skel; + struct bpf_link *link; + struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 }; + bool dump_cgrps = false; + __u64 last_cpu_sum = 0, last_cpu_idle = 0; + __u64 last_stats[FCG_NR_STATS] = {}; + unsigned long seq = 0; + s32 opt; + + signal(SIGINT, sigint_handler); + signal(SIGTERM, sigint_handler); + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + + skel = scx_example_flatcg__open(); + if (!skel) { + fprintf(stderr, "Failed to open: %s\n", strerror(errno)); + return 1; + } + + skel->rodata->nr_cpus = libbpf_num_possible_cpus(); + + while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) { + double v; + + switch (opt) { + case 's': + v = strtod(optarg, NULL); + skel->rodata->cgrp_slice_ns = v * 1000; + break; + case 'i': + v = strtod(optarg, NULL); + intv_ts.tv_sec = v; + intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000; + break; + case 'd': + dump_cgrps = true; + break; + case 'f': + skel->rodata->fifo_sched = true; + break; + case 'p': + skel->rodata->switch_partial = true; + break; + case 'h': + default: + fprintf(stderr, help_fmt, basename(argv[0])); + return opt != 'h'; + } + } + + printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d", + (double)skel->rodata->cgrp_slice_ns / 1000000.0, + (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0, + dump_cgrps); + + if (scx_example_flatcg__load(skel)) { + fprintf(stderr, "Failed to load: %s\n", strerror(errno)); + return 1; + } + + link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops); + if (!link) { + fprintf(stderr, "Failed to attach_struct_ops: %s\n", + strerror(errno)); + return 1; + } + + while (!exit_req && !uei_exited(&skel->bss->uei)) { + __u64 acc_stats[FCG_NR_STATS]; + __u64 stats[FCG_NR_STATS]; + float cpu_util; + int i; + + cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle); + + fcg_read_stats(skel, acc_stats); + for (i = 0; i < FCG_NR_STATS; i++) + stats[i] = acc_stats[i] - last_stats[i]; + + memcpy(last_stats, acc_stats, sizeof(acc_stats)); + + printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n", + seq++, cpu_util * 100.0, skel->data->hweight_gen); + printf(" act:%6llu deact:%6llu local:%6llu global:%6llu\n", + stats[FCG_STAT_ACT], + stats[FCG_STAT_DEACT], + stats[FCG_STAT_LOCAL], + stats[FCG_STAT_GLOBAL]); + printf("HWT skip:%6llu race:%6llu cache:%6llu update:%6llu\n", + stats[FCG_STAT_HWT_SKIP], + stats[FCG_STAT_HWT_RACE], + stats[FCG_STAT_HWT_CACHE], + stats[FCG_STAT_HWT_UPDATES]); + printf("ENQ skip:%6llu race:%6llu\n", + stats[FCG_STAT_ENQ_SKIP], + stats[FCG_STAT_ENQ_RACE]); + printf("CNS keep:%6llu expire:%6llu empty:%6llu gone:%6llu\n", + stats[FCG_STAT_CNS_KEEP], + stats[FCG_STAT_CNS_EXPIRE], + stats[FCG_STAT_CNS_EMPTY], + stats[FCG_STAT_CNS_GONE]); + printf("PNC nocgrp:%6llu next:%6llu empty:%6llu gone:%6llu\n", + stats[FCG_STAT_PNC_NO_CGRP], + stats[FCG_STAT_PNC_NEXT], + stats[FCG_STAT_PNC_EMPTY], + stats[FCG_STAT_PNC_GONE]); + printf("BAD remove:%6llu\n", + acc_stats[FCG_STAT_BAD_REMOVAL]); + + nanosleep(&intv_ts, NULL); + } + + bpf_link__destroy(link); + uei_print(&skel->bss->uei); + scx_example_flatcg__destroy(skel); + return 0; +} diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h new file mode 100644 index 0000000000000..490758ed41f0f --- /dev/null +++ b/tools/sched_ext/scx_example_flatcg.h @@ -0,0 +1,49 @@ +#ifndef __SCX_EXAMPLE_FLATCG_H +#define __SCX_EXAMPLE_FLATCG_H + +enum { + FCG_HWEIGHT_ONE = 1LLU << 16, +}; + +enum fcg_stat_idx { + FCG_STAT_ACT, + FCG_STAT_DEACT, + FCG_STAT_LOCAL, + FCG_STAT_GLOBAL, + + FCG_STAT_HWT_UPDATES, + FCG_STAT_HWT_CACHE, + FCG_STAT_HWT_SKIP, + FCG_STAT_HWT_RACE, + + FCG_STAT_ENQ_SKIP, + FCG_STAT_ENQ_RACE, + + FCG_STAT_CNS_KEEP, + FCG_STAT_CNS_EXPIRE, + FCG_STAT_CNS_EMPTY, + FCG_STAT_CNS_GONE, + + FCG_STAT_PNC_NO_CGRP, + FCG_STAT_PNC_NEXT, + FCG_STAT_PNC_EMPTY, + FCG_STAT_PNC_GONE, + + FCG_STAT_BAD_REMOVAL, + + FCG_NR_STATS, +}; + +struct fcg_cgrp_ctx { + u32 nr_active; + u32 nr_runnable; + u32 queued; + u32 weight; + u32 hweight; + u64 child_weight_sum; + u64 hweight_gen; + s64 cvtime_delta; + u64 tvtime_now; +}; + +#endif /* __SCX_EXAMPLE_FLATCG_H */ diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c new file mode 100644 index 0000000000000..279efe58b777b --- /dev/null +++ b/tools/sched_ext/scx_example_pair.bpf.c @@ -0,0 +1,627 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A demo sched_ext core-scheduler which always makes every sibling CPU pair + * execute from the same CPU cgroup. + * + * This scheduler is a minimal implementation and would need some form of + * priority handling both inside each cgroup and across the cgroups to be + * practically useful. + * + * Each CPU in the system is paired with exactly one other CPU, according to a + * "stride" value that can be specified when the BPF scheduler program is first + * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee + * that they will only ever schedule tasks that belong to the same CPU cgroup. + * + * Scheduler Initialization + * ------------------------ + * + * The scheduler BPF program is first initialized from user space, before it is + * enabled. During this initialization process, each CPU on the system is + * assigned several values that are constant throughout its runtime: + * + * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling + * decisions. Paired CPUs always schedule tasks from the same + * CPU cgroup, and synchronize with each other to guarantee + * that this constraint is not violated. + * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access + * a struct pair_ctx object that is shared between the pair. + * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the + * pair. Each struct pair_ctx has an active_mask field, + * which is a bitmap used to indicate whether each core + * in the pair currently has an actively running task. + * This index specifies which entry in the bitmap corresponds + * to each CPU in the pair. + * + * During this initialization, the CPUs are paired according to a "stride" that + * may be specified when invoking the user space program that initializes and + * loads the scheduler. By default, the stride is 1/2 the total number of CPUs. + * + * Tasks and cgroups + * ----------------- + * + * Every cgroup in the system is registered with the scheduler using the + * pair_cgroup_init() callback, and every task in the system is associated with + * exactly one cgroup. At a high level, the idea with the pair scheduler is to + * always schedule tasks from the same cgroup within a given CPU pair. When a + * task is enqueued (i.e. passed to the pair_enqueue() callback function), its + * cgroup ID is read from its task struct, and then a corresponding queue map + * is used to FIFO-enqueue the task for that cgroup. + * + * If you look through the implementation of the scheduler, you'll notice that + * there is quite a bit of complexity involved with looking up the per-cgroup + * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash + * BPF hash map that is used to map a cgroup ID to a globally unique ID that's + * allocated in the BPF program. This is done because we use separate maps to + * store the FIFO queue of tasks, and the length of that map, per cgroup. This + * complexity is only present because of current deficiencies in BPF that will + * soon be addressed. The main point to keep in mind is that newly enqueued + * tasks are added to their cgroup's FIFO queue. + * + * Dispatching tasks + * ----------------- + * + * This section will describe how enqueued tasks are dispatched and scheduled. + * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is + * as follows: + * + * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is + * the structure that's used to synchronize amongst the two pair CPUs in their + * scheduling decisions. After any of the following events have occurred: + * + * - The cgroup's slice run has expired, or + * - The cgroup becomes empty, or + * - Either CPU in the pair is preempted by a higher priority scheduling class + * + * The cgroup transitions to the draining state and stops executing new tasks + * from the cgroup. + * + * 2. If the pair is still executing a task, mark the pair_ctx as draining, and + * wait for the pair CPU to be preempted. + * + * 3. Otherwise, if the pair CPU is not running a task, we can move onto + * scheduling new tasks. Pop the next cgroup id from the top_q queue. + * + * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it. + * + * Note again that this scheduling behavior is simple, but the implementation + * is complex mostly because this it hits several BPF shortcomings and has to + * work around in often awkward ways. Most of the shortcomings are expected to + * be resolved in the near future which should allow greatly simplifying this + * scheduler. + * + * Dealing with preemption + * ----------------------- + * + * SCX is the lowest priority sched_class, and could be preempted by them at + * any time. To address this, the scheduler implements pair_cpu_release() and + * pair_cpu_acquire() callbacks which are invoked by the core scheduler when + * the scheduler loses and gains control of the CPU respectively. + * + * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and + * then invoke: + * + * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT); + * + * This preempts the pair CPU, and waits until it has re-entered the scheduler + * before returning. This is necessary to ensure that the higher priority + * sched_class that preempted our scheduler does not schedule a task + * concurrently with our pair CPU. + * + * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption + * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable + * pair scheduling. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#include "scx_common.bpf.h" +#include "scx_example_pair.h" + +char _license[] SEC("license") = "GPL"; + +const volatile bool switch_partial; + +/* !0 for veristat, set during init */ +const volatile u32 nr_cpu_ids = 64; + +/* a pair of CPUs stay on a cgroup for this duration */ +const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL; + +/* cpu ID -> pair cpu ID */ +const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 }; + +/* cpu ID -> pair_id */ +const volatile u32 pair_id[MAX_CPUS]; + +/* CPU ID -> CPU # in the pair (0 or 1) */ +const volatile u32 in_pair_idx[MAX_CPUS]; + +struct pair_ctx { + struct bpf_spin_lock lock; + + /* the cgroup the pair is currently executing */ + u64 cgid; + + /* the pair started executing the current cgroup at */ + u64 started_at; + + /* whether the current cgroup is draining */ + bool draining; + + /* the CPUs that are currently active on the cgroup */ + u32 active_mask; + + /* + * the CPUs that are currently preempted and running tasks in a + * different scheduler. + */ + u32 preempted_mask; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, MAX_CPUS / 2); + __type(key, u32); + __type(value, struct pair_ctx); +} pair_ctx SEC(".maps"); + +/* queue of cgrp_q's possibly with tasks on them */ +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + /* + * Because it's difficult to build strong synchronization encompassing + * multiple non-trivial operations in BPF, this queue is managed in an + * opportunistic way so that we guarantee that a cgroup w/ active tasks + * is always on it but possibly multiple times. Once we have more robust + * synchronization constructs and e.g. linked list, we should be able to + * do this in a prettier way but for now just size it big enough. + */ + __uint(max_entries, 4 * MAX_CGRPS); + __type(value, u64); +} top_q SEC(".maps"); + +/* per-cgroup q which FIFOs the tasks from the cgroup */ +struct cgrp_q { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, MAX_QUEUED); + __type(value, u32); +}; + +/* + * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local + * storage; however, a cgroup local storage can only be accessed from the BPF + * progs attached to the cgroup. For now, work around by allocating array of + * cgrp_q's and then allocating per-cgroup indices. + * + * Another caveat: It's difficult to populate a large array of maps statically + * or from BPF. Initialize it from userland. + */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, MAX_CGRPS); + __type(key, s32); + __array(values, struct cgrp_q); +} cgrp_q_arr SEC(".maps"); + +static u64 cgrp_q_len[MAX_CGRPS]; + +/* + * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be + * useful to have as a map type. + */ +static u32 cgrp_q_idx_cursor; +static u64 cgrp_q_idx_busy[MAX_CGRPS]; + +/* + * All added up, the following is what we do: + * + * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking + * for a free ID. If not found, fail cgroup creation with -EBUSY. + * + * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following + * cgrp_q_idx_hash. + * + * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from + * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr. + * + * This is sadly complicated for something pretty simple. Hopefully, we should + * be able to simplify in the future. + */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_CGRPS); + __uint(key_size, sizeof(u64)); /* cgrp ID */ + __uint(value_size, sizeof(s32)); /* cgrp_q idx */ +} cgrp_q_idx_hash SEC(".maps"); + +/* statistics */ +u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions; +u64 nr_exps, nr_exp_waits, nr_exp_empty; +u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty; + +struct user_exit_info uei; + +static bool time_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags) +{ + struct cgroup *cgrp; + struct cgrp_q *cgq; + s32 pid = p->pid; + u64 cgid; + u32 *q_idx; + u64 *cgq_len; + + __sync_fetch_and_add(&nr_total, 1); + + cgrp = scx_bpf_task_cgroup(p); + cgid = cgrp->kn->id; + bpf_cgroup_release(cgrp); + + /* find the cgroup's q and push @p into it */ + q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); + if (!q_idx) { + scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); + return; + } + + cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx); + if (!cgq) { + scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]", + cgid, *q_idx); + return; + } + + if (bpf_map_push_elem(cgq, &pid, 0)) { + scx_bpf_error("cgroup[%llu] queue overflow", cgid); + return; + } + + /* bump q len, if going 0 -> 1, queue cgroup into the top_q */ + cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); + if (!cgq_len) { + scx_bpf_error("MEMBER_VTPR malfunction"); + return; + } + + if (!__sync_fetch_and_add(cgq_len, 1) && + bpf_map_push_elem(&top_q, &cgid, 0)) { + scx_bpf_error("top_q overflow"); + return; + } +} + +static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask) +{ + u32 *vptr; + + vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]); + if (!vptr) + return -EINVAL; + + *pairc = bpf_map_lookup_elem(&pair_ctx, vptr); + if (!(*pairc)) + return -EINVAL; + + vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]); + if (!vptr) + return -EINVAL; + + *mask = 1U << *vptr; + + return 0; +} + +static int try_dispatch(s32 cpu) +{ + struct pair_ctx *pairc; + struct bpf_map *cgq_map; + struct task_struct *p; + u64 now = bpf_ktime_get_ns(); + bool kick_pair = false; + bool expired, pair_preempted; + u32 *vptr, in_pair_mask; + s32 pid, q_idx; + u64 cgid; + int ret; + + ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); + if (ret) { + scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]", + cpu); + return -ENOENT; + } + + bpf_spin_lock(&pairc->lock); + pairc->active_mask &= ~in_pair_mask; + + expired = time_before(pairc->started_at + pair_batch_dur_ns, now); + if (expired || pairc->draining) { + u64 new_cgid = 0; + + __sync_fetch_and_add(&nr_exps, 1); + + /* + * We're done with the current cgid. An obvious optimization + * would be not draining if the next cgroup is the current one. + * For now, be dumb and always expire. + */ + pairc->draining = true; + + pair_preempted = pairc->preempted_mask; + if (pairc->active_mask || pair_preempted) { + /* + * The other CPU is still active, or is no longer under + * our control due to e.g. being preempted by a higher + * priority sched_class. We want to wait until this + * cgroup expires, or until control of our pair CPU has + * been returned to us. + * + * If the pair controls its CPU, and the time already + * expired, kick. When the other CPU arrives at + * dispatch and clears its active mask, it'll push the + * pair to the next cgroup and kick this CPU. + */ + __sync_fetch_and_add(&nr_exp_waits, 1); + bpf_spin_unlock(&pairc->lock); + if (expired && !pair_preempted) + kick_pair = true; + goto out_maybe_kick; + } + + bpf_spin_unlock(&pairc->lock); + + /* + * Pick the next cgroup. It'd be easier / cleaner to not drop + * pairc->lock and use stronger synchronization here especially + * given that we'll be switching cgroups significantly less + * frequently than tasks. Unfortunately, bpf_spin_lock can't + * really protect anything non-trivial. Let's do opportunistic + * operations instead. + */ + bpf_repeat(BPF_MAX_LOOPS) { + u32 *q_idx; + u64 *cgq_len; + + if (bpf_map_pop_elem(&top_q, &new_cgid)) { + /* no active cgroup, go idle */ + __sync_fetch_and_add(&nr_exp_empty, 1); + return 0; + } + + q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid); + if (!q_idx) + continue; + + /* + * This is the only place where empty cgroups are taken + * off the top_q. + */ + cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]); + if (!cgq_len || !*cgq_len) + continue; + + /* + * If it has any tasks, requeue as we may race and not + * execute it. + */ + bpf_map_push_elem(&top_q, &new_cgid, 0); + break; + } + + bpf_spin_lock(&pairc->lock); + + /* + * The other CPU may already have started on a new cgroup while + * we dropped the lock. Make sure that we're still draining and + * start on the new cgroup. + */ + if (pairc->draining && !pairc->active_mask) { + __sync_fetch_and_add(&nr_cgrp_next, 1); + pairc->cgid = new_cgid; + pairc->started_at = now; + pairc->draining = false; + kick_pair = true; + } else { + __sync_fetch_and_add(&nr_cgrp_coll, 1); + } + } + + cgid = pairc->cgid; + pairc->active_mask |= in_pair_mask; + bpf_spin_unlock(&pairc->lock); + + /* again, it'd be better to do all these with the lock held, oh well */ + vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); + if (!vptr) { + scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid); + return -ENOENT; + } + q_idx = *vptr; + + /* claim one task from cgrp_q w/ q_idx */ + bpf_repeat(BPF_MAX_LOOPS) { + u64 *cgq_len, len; + + cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]); + if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) { + /* the cgroup must be empty, expire and repeat */ + __sync_fetch_and_add(&nr_cgrp_empty, 1); + bpf_spin_lock(&pairc->lock); + pairc->draining = true; + pairc->active_mask &= ~in_pair_mask; + bpf_spin_unlock(&pairc->lock); + return -EAGAIN; + } + + if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len) + continue; + + break; + } + + cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx); + if (!cgq_map) { + scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]", + cgid, q_idx); + return -ENOENT; + } + + if (bpf_map_pop_elem(cgq_map, &pid)) { + scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]", + cgid, q_idx); + return -ENOENT; + } + + p = bpf_task_from_pid(pid); + if (p) { + __sync_fetch_and_add(&nr_dispatched, 1); + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); + bpf_task_release(p); + } else { + /* we don't handle dequeues, retry on lost tasks */ + __sync_fetch_and_add(&nr_missing, 1); + return -EAGAIN; + } + +out_maybe_kick: + if (kick_pair) { + s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); + if (pair) { + __sync_fetch_and_add(&nr_kicks, 1); + scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); + } + } + return 0; +} + +void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev) +{ + bpf_repeat(BPF_MAX_LOOPS) { + if (try_dispatch(cpu) != -EAGAIN) + break; + } +} + +void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args) +{ + int ret; + u32 in_pair_mask; + struct pair_ctx *pairc; + bool kick_pair; + + ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); + if (ret) + return; + + bpf_spin_lock(&pairc->lock); + pairc->preempted_mask &= ~in_pair_mask; + /* Kick the pair CPU, unless it was also preempted. */ + kick_pair = !pairc->preempted_mask; + bpf_spin_unlock(&pairc->lock); + + if (kick_pair) { + s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); + + if (pair) { + __sync_fetch_and_add(&nr_kicks, 1); + scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT); + } + } +} + +void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args) +{ + int ret; + u32 in_pair_mask; + struct pair_ctx *pairc; + bool kick_pair; + + ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask); + if (ret) + return; + + bpf_spin_lock(&pairc->lock); + pairc->preempted_mask |= in_pair_mask; + pairc->active_mask &= ~in_pair_mask; + /* Kick the pair CPU if it's still running. */ + kick_pair = pairc->active_mask; + pairc->draining = true; + bpf_spin_unlock(&pairc->lock); + + if (kick_pair) { + s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]); + + if (pair) { + __sync_fetch_and_add(&nr_kicks, 1); + scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT); + } + } + __sync_fetch_and_add(&nr_preemptions, 1); +} + +s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp) +{ + u64 cgid = cgrp->kn->id; + s32 i, q_idx; + + bpf_for(i, 0, MAX_CGRPS) { + q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS; + if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1)) + break; + } + if (i == MAX_CGRPS) + return -EBUSY; + + if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) { + u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]); + if (busy) + *busy = 0; + return -EBUSY; + } + + return 0; +} + +void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp) +{ + u64 cgid = cgrp->kn->id; + s32 *q_idx; + + q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid); + if (q_idx) { + u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]); + if (busy) + *busy = 0; + bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid); + } +} + +s32 BPF_STRUCT_OPS(pair_init) +{ + if (!switch_partial) + scx_bpf_switch_all(); + return 0; +} + +void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei) +{ + uei_record(&uei, ei); +} + +SEC(".struct_ops") +struct sched_ext_ops pair_ops = { + .enqueue = (void *)pair_enqueue, + .dispatch = (void *)pair_dispatch, + .cpu_acquire = (void *)pair_cpu_acquire, + .cpu_release = (void *)pair_cpu_release, + .cgroup_init = (void *)pair_cgroup_init, + .cgroup_exit = (void *)pair_cgroup_exit, + .init = (void *)pair_init, + .exit = (void *)pair_exit, + .name = "pair", +}; diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c new file mode 100644 index 0000000000000..18e032bbc173b --- /dev/null +++ b/tools/sched_ext/scx_example_pair.c @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "user_exit_info.h" +#include "scx_example_pair.h" +#include "scx_example_pair.skel.h" + +const char help_fmt[] = +"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n" +"execute from the same CPU cgroup.\n" +"\n" +"See the top-level comment in .bpf.c for more details.\n" +"\n" +"Usage: %s [-S STRIDE] [-p]\n" +"\n" +" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n" +" -p Switch only tasks on SCHED_EXT policy intead of all\n" +" -h Display this help and exit\n"; + +static volatile int exit_req; + +static void sigint_handler(int dummy) +{ + exit_req = 1; +} + +int main(int argc, char **argv) +{ + struct scx_example_pair *skel; + struct bpf_link *link; + u64 seq = 0; + s32 stride, i, opt, outer_fd; + + signal(SIGINT, sigint_handler); + signal(SIGTERM, sigint_handler); + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + + skel = scx_example_pair__open(); + assert(skel); + + skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus(); + + /* pair up the earlier half to the latter by default, override with -s */ + stride = skel->rodata->nr_cpu_ids / 2; + + while ((opt = getopt(argc, argv, "S:ph")) != -1) { + switch (opt) { + case 'S': + stride = strtoul(optarg, NULL, 0); + break; + case 'p': + skel->rodata->switch_partial = true; + break; + default: + fprintf(stderr, help_fmt, basename(argv[0])); + return opt != 'h'; + } + } + + for (i = 0; i < skel->rodata->nr_cpu_ids; i++) { + if (skel->rodata->pair_cpu[i] < 0) { + skel->rodata->pair_cpu[i] = i + stride; + skel->rodata->pair_cpu[i + stride] = i; + skel->rodata->pair_id[i] = i; + skel->rodata->pair_id[i + stride] = i; + skel->rodata->in_pair_idx[i] = 0; + skel->rodata->in_pair_idx[i + stride] = 1; + } + } + + assert(!scx_example_pair__load(skel)); + + /* + * Populate the cgrp_q_arr map which is an array containing per-cgroup + * queues. It'd probably be better to do this from BPF but there are too + * many to initialize statically and there's no way to dynamically + * populate from BPF. + */ + outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr); + assert(outer_fd >= 0); + + printf("Initializing"); + for (i = 0; i < MAX_CGRPS; i++) { + s32 inner_fd; + + if (exit_req) + break; + + inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, + sizeof(u32), MAX_QUEUED, NULL); + assert(inner_fd >= 0); + assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY)); + close(inner_fd); + + if (!(i % 10)) + printf("."); + fflush(stdout); + } + printf("\n"); + + /* + * Fully initialized, attach and run. + */ + link = bpf_map__attach_struct_ops(skel->maps.pair_ops); + assert(link); + + while (!exit_req && !uei_exited(&skel->bss->uei)) { + printf("[SEQ %lu]\n", seq++); + printf(" total:%10lu dispatch:%10lu missing:%10lu\n", + skel->bss->nr_total, + skel->bss->nr_dispatched, + skel->bss->nr_missing); + printf(" kicks:%10lu preemptions:%7lu\n", + skel->bss->nr_kicks, + skel->bss->nr_preemptions); + printf(" exp:%10lu exp_wait:%10lu exp_empty:%10lu\n", + skel->bss->nr_exps, + skel->bss->nr_exp_waits, + skel->bss->nr_exp_empty); + printf("cgnext:%10lu cgcoll:%10lu cgempty:%10lu\n", + skel->bss->nr_cgrp_next, + skel->bss->nr_cgrp_coll, + skel->bss->nr_cgrp_empty); + fflush(stdout); + sleep(1); + } + + bpf_link__destroy(link); + uei_print(&skel->bss->uei); + scx_example_pair__destroy(skel); + return 0; +} diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h new file mode 100644 index 0000000000000..f60b824272f75 --- /dev/null +++ b/tools/sched_ext/scx_example_pair.h @@ -0,0 +1,10 @@ +#ifndef __SCX_EXAMPLE_PAIR_H +#define __SCX_EXAMPLE_PAIR_H + +enum { + MAX_CPUS = 4096, + MAX_QUEUED = 4096, + MAX_CGRPS = 4096, +}; + +#endif /* __SCX_EXAMPLE_PAIR_H */ diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c new file mode 100644 index 0000000000000..579ab21ae4036 --- /dev/null +++ b/tools/sched_ext/scx_example_qmap.bpf.c @@ -0,0 +1,401 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A simple five-level FIFO queue scheduler. + * + * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets + * assigned to one depending on its compound weight. Each CPU round robins + * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from + * queue0, 2 from queue1, 4 from queue2 and so on. + * + * This scheduler demonstrates: + * + * - BPF-side queueing using PIDs. + * - Sleepable per-task storage allocation using ops.prep_enable(). + * - Using ops.cpu_release() to handle a higher priority scheduling class taking + * the CPU away. + * - Core-sched support. + * + * This scheduler is primarily for demonstration and testing of sched_ext + * features and unlikely to be useful for actual workloads. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#include "scx_common.bpf.h" +#include + +char _license[] SEC("license") = "GPL"; + +const volatile u64 slice_ns = SCX_SLICE_DFL; +const volatile bool switch_partial; +const volatile u32 stall_user_nth; +const volatile u32 stall_kernel_nth; +const volatile u32 dsp_inf_loop_after; +const volatile s32 disallow_tgid; + +u32 test_error_cnt; + +struct user_exit_info uei; + +struct qmap { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 4096); + __type(value, u32); +} queue0 SEC(".maps"), + queue1 SEC(".maps"), + queue2 SEC(".maps"), + queue3 SEC(".maps"), + queue4 SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 5); + __type(key, int); + __array(values, struct qmap); +} queue_arr SEC(".maps") = { + .values = { + [0] = &queue0, + [1] = &queue1, + [2] = &queue2, + [3] = &queue3, + [4] = &queue4, + }, +}; + +/* + * Per-queue sequence numbers to implement core-sched ordering. + * + * Tail seq is assigned to each queued task and incremented. Head seq tracks the + * sequence number of the latest dispatched task. The distance between the a + * task's seq and the associated queue's head seq is called the queue distance + * and used when comparing two tasks for ordering. See qmap_core_sched_before(). + */ +static u64 core_sched_head_seqs[5]; +static u64 core_sched_tail_seqs[5]; + +/* Per-task scheduling context */ +struct task_ctx { + bool force_local; /* Dispatch directly to local_dsq */ + u64 core_sched_seq; +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct task_ctx); +} task_ctx_stor SEC(".maps"); + +/* Per-cpu dispatch index and remaining count */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 2); + __type(key, u32); + __type(value, u64); +} dispatch_idx_cnt SEC(".maps"); + +/* Statistics */ +unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued; +unsigned long nr_core_sched_execed; + +s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, + s32 prev_cpu, u64 wake_flags) +{ + struct task_ctx *tctx; + s32 cpu; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) { + scx_bpf_error("task_ctx lookup failed"); + return -ESRCH; + } + + if (p->nr_cpus_allowed == 1 || + scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { + tctx->force_local = true; + return prev_cpu; + } + + cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); + if (cpu >= 0) + return cpu; + + return prev_cpu; +} + +static int weight_to_idx(u32 weight) +{ + /* Coarsely map the compound weight to a FIFO. */ + if (weight <= 25) + return 0; + else if (weight <= 50) + return 1; + else if (weight < 200) + return 2; + else if (weight < 400) + return 3; + else + return 4; +} + +void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) +{ + static u32 user_cnt, kernel_cnt; + struct task_ctx *tctx; + u32 pid = p->pid; + int idx = weight_to_idx(p->scx.weight); + void *ring; + + if (p->flags & PF_KTHREAD) { + if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth)) + return; + } else { + if (stall_user_nth && !(++user_cnt % stall_user_nth)) + return; + } + + if (test_error_cnt && !--test_error_cnt) + scx_bpf_error("test triggering error"); + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) { + scx_bpf_error("task_ctx lookup failed"); + return; + } + + /* + * All enqueued tasks must have their core_sched_seq updated for correct + * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in + * qmap_ops.flags. + */ + tctx->core_sched_seq = core_sched_tail_seqs[idx]++; + + /* + * If qmap_select_cpu() is telling us to or this is the last runnable + * task on the CPU, enqueue locally. + */ + if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) { + tctx->force_local = false; + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags); + return; + } + + /* + * If the task was re-enqueued due to the CPU being preempted by a + * higher priority scheduling class, just re-enqueue the task directly + * on the global DSQ. As we want another CPU to pick it up, find and + * kick an idle CPU. + */ + if (enq_flags & SCX_ENQ_REENQ) { + s32 cpu; + + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags); + cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); + if (cpu >= 0) + scx_bpf_kick_cpu(cpu, 0); + return; + } + + ring = bpf_map_lookup_elem(&queue_arr, &idx); + if (!ring) { + scx_bpf_error("failed to find ring %d", idx); + return; + } + + /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */ + if (bpf_map_push_elem(ring, &pid, 0)) { + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags); + return; + } + + __sync_fetch_and_add(&nr_enqueued, 1); +} + +/* + * The BPF queue map doesn't support removal and sched_ext can handle spurious + * dispatches. qmap_dequeue() is only used to collect statistics. + */ +void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags) +{ + __sync_fetch_and_add(&nr_dequeued, 1); + if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) + __sync_fetch_and_add(&nr_core_sched_execed, 1); +} + +static void update_core_sched_head_seq(struct task_struct *p) +{ + struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + int idx = weight_to_idx(p->scx.weight); + + if (tctx) + core_sched_head_seqs[idx] = tctx->core_sched_seq; + else + scx_bpf_error("task_ctx lookup failed"); +} + +void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) +{ + u32 zero = 0, one = 1; + u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero); + u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one); + void *fifo; + s32 pid; + int i; + + if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) { + struct task_struct *p; + + /* + * PID 2 should be kthreadd which should mostly be idle and off + * the scheduler. Let's keep dispatching it to force the kernel + * to call this function over and over again. + */ + p = bpf_task_from_pid(2); + if (p) { + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); + bpf_task_release(p); + return; + } + } + + if (!idx || !cnt) { + scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt); + return; + } + + for (i = 0; i < 5; i++) { + /* Advance the dispatch cursor and pick the fifo. */ + if (!*cnt) { + *idx = (*idx + 1) % 5; + *cnt = 1 << *idx; + } + (*cnt)--; + + fifo = bpf_map_lookup_elem(&queue_arr, idx); + if (!fifo) { + scx_bpf_error("failed to find ring %llu", *idx); + return; + } + + /* Dispatch or advance. */ + if (!bpf_map_pop_elem(fifo, &pid)) { + struct task_struct *p; + + p = bpf_task_from_pid(pid); + if (p) { + update_core_sched_head_seq(p); + __sync_fetch_and_add(&nr_dispatched, 1); + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0); + bpf_task_release(p); + return; + } + } + + *cnt = 0; + } +} + +/* + * The distance from the head of the queue scaled by the weight of the queue. + * The lower the number, the older the task and the higher the priority. + */ +static s64 task_qdist(struct task_struct *p) +{ + int idx = weight_to_idx(p->scx.weight); + struct task_ctx *tctx; + s64 qdist; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) { + scx_bpf_error("task_ctx lookup failed"); + return 0; + } + + qdist = tctx->core_sched_seq - core_sched_head_seqs[idx]; + + /* + * As queue index increments, the priority doubles. The queue w/ index 3 + * is dispatched twice more frequently than 2. Reflect the difference by + * scaling qdists accordingly. Note that the shift amount needs to be + * flipped depending on the sign to avoid flipping priority direction. + */ + if (qdist >= 0) + return qdist << (4 - idx); + else + return qdist << idx; +} + +/* + * This is called to determine the task ordering when core-sched is picking + * tasks to execute on SMT siblings and should encode about the same ordering as + * the regular scheduling path. Use the priority-scaled distances from the head + * of the queues to compare the two tasks which should be consistent with the + * dispatch path behavior. + */ +bool BPF_STRUCT_OPS(qmap_core_sched_before, + struct task_struct *a, struct task_struct *b) +{ + return task_qdist(a) > task_qdist(b); +} + +void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) +{ + u32 cnt; + + /* + * Called when @cpu is taken by a higher priority scheduling class. This + * makes @cpu no longer available for executing sched_ext tasks. As we + * don't want the tasks in @cpu's local dsq to sit there until @cpu + * becomes available again, re-enqueue them into the global dsq. See + * %SCX_ENQ_REENQ handling in qmap_enqueue(). + */ + cnt = scx_bpf_reenqueue_local(); + if (cnt) + __sync_fetch_and_add(&nr_reenqueued, cnt); +} + +s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p, + struct scx_enable_args *args) +{ + if (p->tgid == disallow_tgid) + p->scx.disallow = true; + + /* + * @p is new. Let's ensure that its task_ctx is available. We can sleep + * in this function and the following will automatically use GFP_KERNEL. + */ + if (bpf_task_storage_get(&task_ctx_stor, p, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE)) + return 0; + else + return -ENOMEM; +} + +s32 BPF_STRUCT_OPS(qmap_init) +{ + if (!switch_partial) + scx_bpf_switch_all(); + return 0; +} + +void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) +{ + uei_record(&uei, ei); +} + +SEC(".struct_ops") +struct sched_ext_ops qmap_ops = { + .select_cpu = (void *)qmap_select_cpu, + .enqueue = (void *)qmap_enqueue, + .dequeue = (void *)qmap_dequeue, + .dispatch = (void *)qmap_dispatch, + .core_sched_before = (void *)qmap_core_sched_before, + .cpu_release = (void *)qmap_cpu_release, + .prep_enable = (void *)qmap_prep_enable, + .init = (void *)qmap_init, + .exit = (void *)qmap_exit, + .flags = SCX_OPS_ENQ_LAST, + .timeout_ms = 5000U, + .name = "qmap", +}; diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c new file mode 100644 index 0000000000000..ccb4814ee61ba --- /dev/null +++ b/tools/sched_ext/scx_example_qmap.c @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "user_exit_info.h" +#include "scx_example_qmap.skel.h" + +const char help_fmt[] = +"A simple five-level FIFO queue sched_ext scheduler.\n" +"\n" +"See the top-level comment in .bpf.c for more details.\n" +"\n" +"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n" +"\n" +" -s SLICE_US Override slice duration\n" +" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n" +" -t COUNT Stall every COUNT'th user thread\n" +" -T COUNT Stall every COUNT'th kernel thread\n" +" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n" +" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n" +" -p Switch only tasks on SCHED_EXT policy intead of all\n" +" -h Display this help and exit\n"; + +static volatile int exit_req; + +static void sigint_handler(int dummy) +{ + exit_req = 1; +} + +int main(int argc, char **argv) +{ + struct scx_example_qmap *skel; + struct bpf_link *link; + int opt; + + signal(SIGINT, sigint_handler); + signal(SIGTERM, sigint_handler); + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + + skel = scx_example_qmap__open(); + assert(skel); + + while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) { + switch (opt) { + case 's': + skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; + break; + case 'e': + skel->bss->test_error_cnt = strtoul(optarg, NULL, 0); + break; + case 't': + skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0); + break; + case 'T': + skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0); + break; + case 'l': + skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0); + break; + case 'd': + skel->rodata->disallow_tgid = strtol(optarg, NULL, 0); + if (skel->rodata->disallow_tgid < 0) + skel->rodata->disallow_tgid = getpid(); + break; + case 'p': + skel->rodata->switch_partial = true; + break; + default: + fprintf(stderr, help_fmt, basename(argv[0])); + return opt != 'h'; + } + } + + assert(!scx_example_qmap__load(skel)); + + link = bpf_map__attach_struct_ops(skel->maps.qmap_ops); + assert(link); + + while (!exit_req && !uei_exited(&skel->bss->uei)) { + long nr_enqueued = skel->bss->nr_enqueued; + long nr_dispatched = skel->bss->nr_dispatched; + + printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n", + nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched, + skel->bss->nr_reenqueued, skel->bss->nr_dequeued, + skel->bss->nr_core_sched_execed); + fflush(stdout); + sleep(1); + } + + bpf_link__destroy(link); + uei_print(&skel->bss->uei); + scx_example_qmap__destroy(skel); + return 0; +} diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c new file mode 100644 index 0000000000000..4bccca3e20470 --- /dev/null +++ b/tools/sched_ext/scx_example_simple.bpf.c @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A simple scheduler. + * + * By default, it operates as a simple global weighted vtime scheduler and can + * be switched to FIFO scheduling. It also demonstrates the following niceties. + * + * - Statistics tracking how many tasks are queued to local and global dsq's. + * - Termination notification for userspace. + * + * While very simple, this scheduler should work reasonably well on CPUs with a + * uniform L3 cache topology. While preemption is not implemented, the fact that + * the scheduling queue is shared across all CPUs means that whatever is at the + * front of the queue is likely to be executed fairly quickly given enough + * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads + * but comes with the usual problems with FIFO scheduling where saturating + * threads can easily drown out interactive ones. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#include "scx_common.bpf.h" + +char _license[] SEC("license") = "GPL"; + +const volatile bool fifo_sched; +const volatile bool switch_partial; + +static u64 vtime_now; +struct user_exit_info uei; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u64)); + __uint(max_entries, 2); /* [local, global] */ +} stats SEC(".maps"); + +static void stat_inc(u32 idx) +{ + u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); + if (cnt_p) + (*cnt_p)++; +} + +static inline bool vtime_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags) +{ + /* + * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that + * running @p on its CPU directly shouldn't affect fairness. Just queue + * it on the local FIFO. + */ + if (enq_flags & SCX_ENQ_LOCAL) { + stat_inc(0); /* count local queueing */ + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); + return; + } + + stat_inc(1); /* count global queueing */ + + if (fifo_sched) { + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + } else { + u64 vtime = p->scx.dsq_vtime; + + /* + * Limit the amount of budget that an idling task can accumulate + * to one slice. + */ + if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) + vtime = vtime_now - SCX_SLICE_DFL; + + scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime, + enq_flags); + } +} + +void BPF_STRUCT_OPS(simple_running, struct task_struct *p) +{ + if (fifo_sched) + return; + + /* + * Global vtime always progresses forward as tasks start executing. The + * test and update can be performed concurrently from multiple CPUs and + * thus racy. Any error should be contained and temporary. Let's just + * live with it. + */ + if (vtime_before(vtime_now, p->scx.dsq_vtime)) + vtime_now = p->scx.dsq_vtime; +} + +void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) +{ + if (fifo_sched) + return; + + /* scale the execution time by the inverse of the weight and charge */ + p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; +} + +s32 BPF_STRUCT_OPS(simple_init) +{ + if (!switch_partial) + scx_bpf_switch_all(); + return 0; +} + +void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) +{ + uei_record(&uei, ei); +} + +SEC(".struct_ops") +struct sched_ext_ops simple_ops = { + .enqueue = (void *)simple_enqueue, + .running = (void *)simple_running, + .stopping = (void *)simple_stopping, + .init = (void *)simple_init, + .exit = (void *)simple_exit, + .name = "simple", +}; diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c new file mode 100644 index 0000000000000..486b401f7c951 --- /dev/null +++ b/tools/sched_ext/scx_example_simple.c @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "user_exit_info.h" +#include "scx_example_simple.skel.h" + +const char help_fmt[] = +"A simple sched_ext scheduler.\n" +"\n" +"See the top-level comment in .bpf.c for more details.\n" +"\n" +"Usage: %s [-f] [-p]\n" +"\n" +" -f Use FIFO scheduling instead of weighted vtime scheduling\n" +" -p Switch only tasks on SCHED_EXT policy intead of all\n" +" -h Display this help and exit\n"; + +static volatile int exit_req; + +static void sigint_handler(int simple) +{ + exit_req = 1; +} + +static void read_stats(struct scx_example_simple *skel, u64 *stats) +{ + int nr_cpus = libbpf_num_possible_cpus(); + u64 cnts[2][nr_cpus]; + u32 idx; + + memset(stats, 0, sizeof(stats[0]) * 2); + + for (idx = 0; idx < 2; idx++) { + int ret, cpu; + + ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats), + &idx, cnts[idx]); + if (ret < 0) + continue; + for (cpu = 0; cpu < nr_cpus; cpu++) + stats[idx] += cnts[idx][cpu]; + } +} + +int main(int argc, char **argv) +{ + struct scx_example_simple *skel; + struct bpf_link *link; + u32 opt; + + signal(SIGINT, sigint_handler); + signal(SIGTERM, sigint_handler); + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + + skel = scx_example_simple__open(); + assert(skel); + + while ((opt = getopt(argc, argv, "fph")) != -1) { + switch (opt) { + case 'f': + skel->rodata->fifo_sched = true; + break; + case 'p': + skel->rodata->switch_partial = true; + break; + default: + fprintf(stderr, help_fmt, basename(argv[0])); + return opt != 'h'; + } + } + + assert(!scx_example_simple__load(skel)); + + link = bpf_map__attach_struct_ops(skel->maps.simple_ops); + assert(link); + + while (!exit_req && !uei_exited(&skel->bss->uei)) { + u64 stats[2]; + + read_stats(skel, stats); + printf("local=%lu global=%lu\n", stats[0], stats[1]); + fflush(stdout); + sleep(1); + } + + bpf_link__destroy(link); + uei_print(&skel->bss->uei); + scx_example_simple__destroy(skel); + return 0; +} diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c new file mode 100644 index 0000000000000..a089bc6bbe868 --- /dev/null +++ b/tools/sched_ext/scx_example_userland.bpf.c @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A minimal userland scheduler. + * + * In terms of scheduling, this provides two different types of behaviors: + * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity. + * All such tasks are direct-dispatched from the kernel, and are never + * enqueued in user space. + * 2. A primitive vruntime scheduler that is implemented in user space, for all + * other tasks. + * + * Some parts of this example user space scheduler could be implemented more + * efficiently using more complex and sophisticated data structures. For + * example, rather than using BPF_MAP_TYPE_QUEUE's, + * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between + * user space and kernel space. Similarly, we use a simple vruntime-sorted list + * in user space, but an rbtree could be used instead. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#include +#include "scx_common.bpf.h" +#include "scx_example_userland_common.h" + +char _license[] SEC("license") = "GPL"; + +const volatile bool switch_partial; +const volatile s32 usersched_pid; + +/* !0 for veristat, set during init */ +const volatile u32 num_possible_cpus = 64; + +/* Stats that are printed by user space. */ +u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues; + +struct user_exit_info uei; + +/* + * Whether the user space scheduler needs to be scheduled due to a task being + * enqueued in user space. + */ +static bool usersched_needed; + +/* + * The map containing tasks that are enqueued in user space from the kernel. + * + * This map is drained by the user space scheduler. + */ +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, USERLAND_MAX_TASKS); + __type(value, struct scx_userland_enqueued_task); +} enqueued SEC(".maps"); + +/* + * The map containing tasks that are dispatched to the kernel from user space. + * + * Drained by the kernel in userland_dispatch(). + */ +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, USERLAND_MAX_TASKS); + __type(value, s32); +} dispatched SEC(".maps"); + +/* Per-task scheduling context */ +struct task_ctx { + bool force_local; /* Dispatch directly to local DSQ */ +}; + +/* Map that contains task-local storage. */ +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct task_ctx); +} task_ctx_stor SEC(".maps"); + +static bool is_usersched_task(const struct task_struct *p) +{ + return p->pid == usersched_pid; +} + +static bool keep_in_kernel(const struct task_struct *p) +{ + return p->nr_cpus_allowed < num_possible_cpus; +} + +static struct task_struct *usersched_task(void) +{ + struct task_struct *p; + + p = bpf_task_from_pid(usersched_pid); + /* + * Should never happen -- the usersched task should always be managed + * by sched_ext. + */ + if (!p) { + scx_bpf_error("Failed to find usersched task %d", usersched_pid); + /* + * We should never hit this path, and we error out of the + * scheduler above just in case, so the scheduler will soon be + * be evicted regardless. So as to simplify the logic in the + * caller to not have to check for NULL, return an acquired + * reference to the current task here rather than NULL. + */ + return bpf_task_acquire(bpf_get_current_task_btf()); + } + + return p; +} + +s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p, + s32 prev_cpu, u64 wake_flags) +{ + if (keep_in_kernel(p)) { + s32 cpu; + struct task_ctx *tctx; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) { + scx_bpf_error("Failed to look up task-local storage for %s", p->comm); + return -ESRCH; + } + + if (p->nr_cpus_allowed == 1 || + scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { + tctx->force_local = true; + return prev_cpu; + } + + cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr); + if (cpu >= 0) { + tctx->force_local = true; + return cpu; + } + } + + return prev_cpu; +} + +static void dispatch_user_scheduler(void) +{ + struct task_struct *p; + + usersched_needed = false; + p = usersched_task(); + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); + bpf_task_release(p); +} + +static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags) +{ + struct scx_userland_enqueued_task task; + + memset(&task, 0, sizeof(task)); + task.pid = p->pid; + task.sum_exec_runtime = p->se.sum_exec_runtime; + task.weight = p->scx.weight; + + if (bpf_map_push_elem(&enqueued, &task, 0)) { + /* + * If we fail to enqueue the task in user space, put it + * directly on the global DSQ. + */ + __sync_fetch_and_add(&nr_failed_enqueues, 1); + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + } else { + __sync_fetch_and_add(&nr_user_enqueues, 1); + usersched_needed = true; + } +} + +void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags) +{ + if (keep_in_kernel(p)) { + u64 dsq_id = SCX_DSQ_GLOBAL; + struct task_ctx *tctx; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) { + scx_bpf_error("Failed to lookup task ctx for %s", p->comm); + return; + } + + if (tctx->force_local) + dsq_id = SCX_DSQ_LOCAL; + tctx->force_local = false; + scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags); + __sync_fetch_and_add(&nr_kernel_enqueues, 1); + return; + } else if (!is_usersched_task(p)) { + enqueue_task_in_user_space(p, enq_flags); + } +} + +void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev) +{ + if (usersched_needed) + dispatch_user_scheduler(); + + bpf_repeat(4096) { + s32 pid; + struct task_struct *p; + + if (bpf_map_pop_elem(&dispatched, &pid)) + break; + + /* + * The task could have exited by the time we get around to + * dispatching it. Treat this as a normal occurrence, and simply + * move onto the next iteration. + */ + p = bpf_task_from_pid(pid); + if (!p) + continue; + + scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); + bpf_task_release(p); + } +} + +s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p, + struct scx_enable_args *args) +{ + if (bpf_task_storage_get(&task_ctx_stor, p, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE)) + return 0; + else + return -ENOMEM; +} + +s32 BPF_STRUCT_OPS(userland_init) +{ + if (num_possible_cpus == 0) { + scx_bpf_error("User scheduler # CPUs uninitialized (%d)", + num_possible_cpus); + return -EINVAL; + } + + if (usersched_pid <= 0) { + scx_bpf_error("User scheduler pid uninitialized (%d)", + usersched_pid); + return -EINVAL; + } + + if (!switch_partial) + scx_bpf_switch_all(); + return 0; +} + +void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei) +{ + uei_record(&uei, ei); +} + +SEC(".struct_ops") +struct sched_ext_ops userland_ops = { + .select_cpu = (void *)userland_select_cpu, + .enqueue = (void *)userland_enqueue, + .dispatch = (void *)userland_dispatch, + .prep_enable = (void *)userland_prep_enable, + .init = (void *)userland_init, + .exit = (void *)userland_exit, + .timeout_ms = 3000, + .name = "userland", +}; diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c new file mode 100644 index 0000000000000..4152b1e65fe1a --- /dev/null +++ b/tools/sched_ext/scx_example_userland.c @@ -0,0 +1,402 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A demo sched_ext user space scheduler which provides vruntime semantics + * using a simple ordered-list implementation. + * + * Each CPU in the system resides in a single, global domain. This precludes + * the need to do any load balancing between domains. The scheduler could + * easily be extended to support multiple domains, with load balancing + * happening in user space. + * + * Any task which has any CPU affinity is scheduled entirely in BPF. This + * program only schedules tasks which may run on any CPU. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "user_exit_info.h" +#include "scx_example_userland_common.h" +#include "scx_example_userland.skel.h" + +const char help_fmt[] = +"A minimal userland sched_ext scheduler.\n" +"\n" +"See the top-level comment in .bpf.c for more details.\n" +"\n" +"Usage: %s [-b BATCH] [-p]\n" +"\n" +" -b BATCH The number of tasks to batch when dispatching (default: 8)\n" +" -p Don't switch all, switch only tasks on SCHED_EXT policy\n" +" -h Display this help and exit\n"; + +/* Defined in UAPI */ +#define SCHED_EXT 7 + +/* Number of tasks to batch when dispatching to user space. */ +static __u32 batch_size = 8; + +static volatile int exit_req; +static int enqueued_fd, dispatched_fd; + +static struct scx_example_userland *skel; +static struct bpf_link *ops_link; + +/* Stats collected in user space. */ +static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches; + +/* The data structure containing tasks that are enqueued in user space. */ +struct enqueued_task { + LIST_ENTRY(enqueued_task) entries; + __u64 sum_exec_runtime; + double vruntime; +}; + +/* + * Use a vruntime-sorted list to store tasks. This could easily be extended to + * a more optimal data structure, such as an rbtree as is done in CFS. We + * currently elect to use a sorted list to simplify the example for + * illustrative purposes. + */ +LIST_HEAD(listhead, enqueued_task); + +/* + * A vruntime-sorted list of tasks. The head of the list contains the task with + * the lowest vruntime. That is, the task that has the "highest" claim to be + * scheduled. + */ +static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head); + +/* + * The statically allocated array of tasks. We use a statically allocated list + * here to avoid having to allocate on the enqueue path, which could cause a + * deadlock. A more substantive user space scheduler could e.g. provide a hook + * for newly enabled tasks that are passed to the scheduler from the + * .prep_enable() callback to allows the scheduler to allocate on safe paths. + */ +struct enqueued_task tasks[USERLAND_MAX_TASKS]; + +static double min_vruntime; + +static void sigint_handler(int userland) +{ + exit_req = 1; +} + +static __u32 task_pid(const struct enqueued_task *task) +{ + return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task); +} + +static int dispatch_task(s32 pid) +{ + int err; + + err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0); + if (err) { + fprintf(stderr, "Failed to dispatch task %d\n", pid); + exit_req = 1; + } else { + nr_vruntime_dispatches++; + } + + return err; +} + +static struct enqueued_task *get_enqueued_task(__s32 pid) +{ + if (pid >= USERLAND_MAX_TASKS) + return NULL; + + return &tasks[pid]; +} + +static double calc_vruntime_delta(__u64 weight, __u64 delta) +{ + double weight_f = (double)weight / 100.0; + double delta_f = (double)delta; + + return delta_f / weight_f; +} + +static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task) +{ + __u64 delta; + + delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime; + + enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta); + if (min_vruntime > enqueued->vruntime) + enqueued->vruntime = min_vruntime; + enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime; +} + +static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task) +{ + struct enqueued_task *curr, *enqueued, *prev; + + curr = get_enqueued_task(bpf_task->pid); + if (!curr) + return ENOENT; + + update_enqueued(curr, bpf_task); + nr_vruntime_enqueues++; + + /* + * Enqueue the task in a vruntime-sorted list. A more optimal data + * structure such as an rbtree could easily be used as well. We elect + * to use a list here simply because it's less code, and thus the + * example is less convoluted and better serves to illustrate what a + * user space scheduler could look like. + */ + + if (LIST_EMPTY(&vruntime_head)) { + LIST_INSERT_HEAD(&vruntime_head, curr, entries); + return 0; + } + + LIST_FOREACH(enqueued, &vruntime_head, entries) { + if (curr->vruntime <= enqueued->vruntime) { + LIST_INSERT_BEFORE(enqueued, curr, entries); + return 0; + } + prev = enqueued; + } + + LIST_INSERT_AFTER(prev, curr, entries); + + return 0; +} + +static void drain_enqueued_map(void) +{ + while (1) { + struct scx_userland_enqueued_task task; + int err; + + if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) + return; + + err = vruntime_enqueue(&task); + if (err) { + fprintf(stderr, "Failed to enqueue task %d: %s\n", + task.pid, strerror(err)); + exit_req = 1; + return; + } + } +} + +static void dispatch_batch(void) +{ + __u32 i; + + for (i = 0; i < batch_size; i++) { + struct enqueued_task *task; + int err; + __s32 pid; + + task = LIST_FIRST(&vruntime_head); + if (!task) + return; + + min_vruntime = task->vruntime; + pid = task_pid(task); + LIST_REMOVE(task, entries); + err = dispatch_task(pid); + if (err) { + fprintf(stderr, "Failed to dispatch task %d in %u\n", + pid, i); + return; + } + } +} + +static void *run_stats_printer(void *arg) +{ + while (!exit_req) { + __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total; + + nr_failed_enqueues = skel->bss->nr_failed_enqueues; + nr_kernel_enqueues = skel->bss->nr_kernel_enqueues; + nr_user_enqueues = skel->bss->nr_user_enqueues; + total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues; + + printf("o-----------------------o\n"); + printf("| BPF ENQUEUES |\n"); + printf("|-----------------------|\n"); + printf("| kern: %10llu |\n", nr_kernel_enqueues); + printf("| user: %10llu |\n", nr_user_enqueues); + printf("| failed: %10llu |\n", nr_failed_enqueues); + printf("| -------------------- |\n"); + printf("| total: %10llu |\n", total); + printf("| |\n"); + printf("|-----------------------|\n"); + printf("| VRUNTIME / USER |\n"); + printf("|-----------------------|\n"); + printf("| enq: %10llu |\n", nr_vruntime_enqueues); + printf("| disp: %10llu |\n", nr_vruntime_dispatches); + printf("o-----------------------o\n"); + printf("\n\n"); + sleep(1); + } + + return NULL; +} + +static int spawn_stats_thread(void) +{ + pthread_t stats_printer; + + return pthread_create(&stats_printer, NULL, run_stats_printer, NULL); +} + +static int bootstrap(int argc, char **argv) +{ + int err; + __u32 opt; + struct sched_param sched_param = { + .sched_priority = sched_get_priority_max(SCHED_EXT), + }; + bool switch_partial = false; + + signal(SIGINT, sigint_handler); + signal(SIGTERM, sigint_handler); + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + + /* + * Enforce that the user scheduler task is managed by sched_ext. The + * task eagerly drains the list of enqueued tasks in its main work + * loop, and then yields the CPU. The BPF scheduler only schedules the + * user space scheduler task when at least one other task in the system + * needs to be scheduled. + */ + err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param); + if (err) { + fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err)); + return err; + } + + while ((opt = getopt(argc, argv, "b:ph")) != -1) { + switch (opt) { + case 'b': + batch_size = strtoul(optarg, NULL, 0); + break; + case 'p': + switch_partial = true; + break; + default: + fprintf(stderr, help_fmt, basename(argv[0])); + exit(opt != 'h'); + } + } + + /* + * It's not always safe to allocate in a user space scheduler, as an + * enqueued task could hold a lock that we require in order to be able + * to allocate. + */ + err = mlockall(MCL_CURRENT | MCL_FUTURE); + if (err) { + fprintf(stderr, "Failed to prefault and lock address space: %s\n", + strerror(err)); + return err; + } + + skel = scx_example_userland__open(); + if (!skel) { + fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno)); + return errno; + } + skel->rodata->num_possible_cpus = libbpf_num_possible_cpus(); + assert(skel->rodata->num_possible_cpus > 0); + skel->rodata->usersched_pid = getpid(); + assert(skel->rodata->usersched_pid > 0); + skel->rodata->switch_partial = switch_partial; + + err = scx_example_userland__load(skel); + if (err) { + fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err)); + goto destroy_skel; + } + + enqueued_fd = bpf_map__fd(skel->maps.enqueued); + dispatched_fd = bpf_map__fd(skel->maps.dispatched); + assert(enqueued_fd > 0); + assert(dispatched_fd > 0); + + err = spawn_stats_thread(); + if (err) { + fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err)); + goto destroy_skel; + } + + ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops); + if (!ops_link) { + fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno)); + err = errno; + goto destroy_skel; + } + + return 0; + +destroy_skel: + scx_example_userland__destroy(skel); + exit_req = 1; + return err; +} + +static void sched_main_loop(void) +{ + while (!exit_req) { + /* + * Perform the following work in the main user space scheduler + * loop: + * + * 1. Drain all tasks from the enqueued map, and enqueue them + * to the vruntime sorted list. + * + * 2. Dispatch a batch of tasks from the vruntime sorted list + * down to the kernel. + * + * 3. Yield the CPU back to the system. The BPF scheduler will + * reschedule the user space scheduler once another task has + * been enqueued to user space. + */ + drain_enqueued_map(); + dispatch_batch(); + sched_yield(); + } +} + +int main(int argc, char **argv) +{ + int err; + + err = bootstrap(argc, argv); + if (err) { + fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err)); + return err; + } + + sched_main_loop(); + + exit_req = 1; + bpf_link__destroy(ops_link); + uei_print(&skel->bss->uei); + scx_example_userland__destroy(skel); + return 0; +} diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h new file mode 100644 index 0000000000000..639c6809c5ffe --- /dev/null +++ b/tools/sched_ext/scx_example_userland_common.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta, Inc */ + +#ifndef __SCX_USERLAND_COMMON_H +#define __SCX_USERLAND_COMMON_H + +#define USERLAND_MAX_TASKS 8192 + +/* + * An instance of a task that has been enqueued by the kernel for consumption + * by a user space global scheduler thread. + */ +struct scx_userland_enqueued_task { + __s32 pid; + u64 sum_exec_runtime; + u64 weight; +}; + +#endif // __SCX_USERLAND_COMMON_H diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h new file mode 100644 index 0000000000000..e701ef0e0b86c --- /dev/null +++ b/tools/sched_ext/user_exit_info.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Define struct user_exit_info which is shared between BPF and userspace parts + * to communicate exit status and other information. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#ifndef __USER_EXIT_INFO_H +#define __USER_EXIT_INFO_H + +struct user_exit_info { + int type; + char reason[128]; + char msg[1024]; +}; + +#ifdef __bpf__ + +#include "vmlinux.h" +#include + +static inline void uei_record(struct user_exit_info *uei, + const struct scx_exit_info *ei) +{ + bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason); + bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg); + /* use __sync to force memory barrier */ + __sync_val_compare_and_swap(&uei->type, uei->type, ei->type); +} + +#else /* !__bpf__ */ + +static inline bool uei_exited(struct user_exit_info *uei) +{ + /* use __sync to force memory barrier */ + return __sync_val_compare_and_swap(&uei->type, -1, -1); +} + +static inline void uei_print(const struct user_exit_info *uei) +{ + fprintf(stderr, "EXIT: %s", uei->reason); + if (uei->msg[0] != '\0') + fprintf(stderr, " (%s)", uei->msg); + fputs("\n", stderr); +} + +#endif /* __bpf__ */ +#endif /* __USER_EXIT_INFO_H */ diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index ff527ac065cf8..c006e72b4f434 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -98,7 +98,9 @@ else ifneq ($(CROSS_COMPILE),) # Allow userspace to override CLANG_CROSS_FLAGS to specify their own # sysroots and flags or to avoid the GCC call in pure Clang builds. ifeq ($(CLANG_CROSS_FLAGS),) -CLANG_CROSS_FLAGS := --target=$(notdir $(CROSS_COMPILE:%-=%)) +CLANG_TARGET := $(notdir $(CROSS_COMPILE:%-=%)) +CLANG_TARGET := $(subst s390-linux,s390x-linux,$(CLANG_TARGET)) +CLANG_CROSS_FLAGS := --target=$(CLANG_TARGET) GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE)gcc 2>/dev/null)) ifneq ($(GCC_TOOLCHAIN_DIR),) CLANG_CROSS_FLAGS += --prefix=$(GCC_TOOLCHAIN_DIR)$(notdir $(CROSS_COMPILE)) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 3316015757433..a8979280b5055 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1358,7 +1358,10 @@ sub __eval_option { # If a variable contains itself, use the default var if (($var eq $name) && defined($opt{$var})) { $o = $opt{$var}; - $retval = "$retval$o"; + # Only append if the default doesn't contain itself + if ($o !~ m/\$\{$var\}/) { + $retval = "$retval$o"; + } } elsif (defined($opt{$o})) { $o = $opt{$o}; $retval = "$retval$o"; diff --git a/tools/testing/kunit/qemu_configs/x86_64.py b/tools/testing/kunit/qemu_configs/x86_64.py index dc79490768630..4a6bf4e048f5b 100644 --- a/tools/testing/kunit/qemu_configs/x86_64.py +++ b/tools/testing/kunit/qemu_configs/x86_64.py @@ -7,4 +7,6 @@ qemu_arch='x86_64', kernel_path='arch/x86/boot/bzImage', kernel_command_line='console=ttyS0', - extra_qemu_params=[]) + # qboot is faster than SeaBIOS and doesn't mess up + # the terminal. + extra_qemu_params=['-bios', 'qboot.rom']) diff --git a/tools/testing/nvdimm/test/ndtest.c b/tools/testing/nvdimm/test/ndtest.c index 3eba10c1e3e8f..e711a2584143f 100644 --- a/tools/testing/nvdimm/test/ndtest.c +++ b/tools/testing/nvdimm/test/ndtest.c @@ -845,11 +845,22 @@ static int ndtest_probe(struct platform_device *pdev) p->dcr_dma = devm_kcalloc(&p->pdev.dev, NUM_DCR, sizeof(dma_addr_t), GFP_KERNEL); + if (!p->dcr_dma) { + rc = -ENOMEM; + goto err; + } p->label_dma = devm_kcalloc(&p->pdev.dev, NUM_DCR, sizeof(dma_addr_t), GFP_KERNEL); + if (!p->label_dma) { + rc = -ENOMEM; + goto err; + } p->dimm_dma = devm_kcalloc(&p->pdev.dev, NUM_DCR, sizeof(dma_addr_t), GFP_KERNEL); - + if (!p->dimm_dma) { + rc = -ENOMEM; + goto err; + } rc = ndtest_nvdimm_init(p); if (rc) goto err; diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 5b61b8bb29f84..a670d5de302b9 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -269,7 +269,7 @@ gen_tar: install @echo "Created ${TAR_PATH}" clean: - @for TARGET in $(TARGETS); do \ + @for TARGET in $(TARGETS) $(INSTALL_DEP_TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ $(MAKE) OUTPUT=$$BUILD_TARGET -C $$TARGET clean;\ done; diff --git a/tools/testing/selftests/arm64/fp/sve-ptrace.c b/tools/testing/selftests/arm64/fp/sve-ptrace.c index 6d61992fe8a01..408fb1c5c2f85 100644 --- a/tools/testing/selftests/arm64/fp/sve-ptrace.c +++ b/tools/testing/selftests/arm64/fp/sve-ptrace.c @@ -168,7 +168,7 @@ static void ptrace_set_get_inherit(pid_t child, const struct vec_type *type) memset(&sve, 0, sizeof(sve)); sve.size = sizeof(sve); sve.vl = sve_vl_from_vq(SVE_VQ_MIN); - sve.flags = SVE_PT_VL_INHERIT; + sve.flags = SVE_PT_VL_INHERIT | SVE_PT_REGS_SVE; ret = set_sve(child, type, &sve); if (ret != 0) { ksft_test_result_fail("Failed to set %s SVE_PT_VL_INHERIT\n", @@ -233,6 +233,7 @@ static void ptrace_set_get_vl(pid_t child, const struct vec_type *type, /* Set the VL by doing a set with no register payload */ memset(&sve, 0, sizeof(sve)); sve.size = sizeof(sve); + sve.flags = SVE_PT_REGS_SVE; sve.vl = vl; ret = set_sve(child, type, &sve); if (ret != 0) { @@ -251,7 +252,7 @@ static void ptrace_set_get_vl(pid_t child, const struct vec_type *type, return; } - ksft_test_result(new_sve->vl = prctl_vl, "Set %s VL %u\n", + ksft_test_result(new_sve->vl == prctl_vl, "Set %s VL %u\n", type->name, vl); free(new_sve); diff --git a/tools/testing/selftests/arm64/pauth/exec_target.c b/tools/testing/selftests/arm64/pauth/exec_target.c index 4435600ca400d..e597861b26d6b 100644 --- a/tools/testing/selftests/arm64/pauth/exec_target.c +++ b/tools/testing/selftests/arm64/pauth/exec_target.c @@ -13,7 +13,12 @@ int main(void) unsigned long hwcaps; size_t val; - fread(&val, sizeof(size_t), 1, stdin); + size_t size = fread(&val, sizeof(size_t), 1, stdin); + + if (size != 1) { + fprintf(stderr, "Could not read input from stdin\n"); + return EXIT_FAILURE; + } /* don't try to execute illegal (unimplemented) instructions) caller * should have checked this and keep worker simple diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index 4494eaa9937e5..0276d917467a6 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -162,4 +162,9 @@ extern void bpf_percpu_obj_drop_impl(void *kptr, void *meta) __ksym; /* Convenience macro to wrap over bpf_obj_drop_impl */ #define bpf_percpu_obj_drop(kptr) bpf_percpu_obj_drop_impl(kptr, NULL) +struct bpf_iter_dmabuf; +extern int bpf_iter_dmabuf_new(struct bpf_iter_dmabuf *it) __weak __ksym; +extern struct dma_buf *bpf_iter_dmabuf_next(struct bpf_iter_dmabuf *it) __weak __ksym; +extern void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) __weak __ksym; + #endif diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index 5751614aef6a5..aff7a0fa325d9 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -17,6 +17,8 @@ CONFIG_CRYPTO_USER_API_HASH=y CONFIG_DEBUG_INFO=y CONFIG_DEBUG_INFO_BTF=y CONFIG_DEBUG_INFO_DWARF4=y +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_DUMMY=y CONFIG_DYNAMIC_FTRACE=y CONFIG_FPROBE=y @@ -83,6 +85,7 @@ CONFIG_RC_CORE=y CONFIG_SECURITY=y CONFIG_SECURITYFS=y CONFIG_TEST_BPF=m +CONFIG_UDMABUF=y CONFIG_USERFAULTFD=y CONFIG_VXLAN=y CONFIG_XDP_SOCKETS=y diff --git a/tools/testing/selftests/bpf/prog_tests/arg_parsing.c b/tools/testing/selftests/bpf/prog_tests/arg_parsing.c index bb143de68875c..e27d66b75fb1f 100644 --- a/tools/testing/selftests/bpf/prog_tests/arg_parsing.c +++ b/tools/testing/selftests/bpf/prog_tests/arg_parsing.c @@ -144,11 +144,17 @@ static void test_parse_test_list_file(void) if (!ASSERT_OK(ferror(fp), "prepare tmp")) goto out_fclose; + if (!ASSERT_OK(fsync(fileno(fp)), "fsync tmp")) + goto out_fclose; + init_test_filter_set(&set); - ASSERT_OK(parse_test_list_file(tmpfile, &set, true), "parse file"); + if (!ASSERT_OK(parse_test_list_file(tmpfile, &set, true), "parse file")) + goto out_fclose; + + if (!ASSERT_EQ(set.cnt, 4, "test count")) + goto out_free_set; - ASSERT_EQ(set.cnt, 4, "test count"); ASSERT_OK(strcmp("test_with_spaces", set.tests[0].name), "test 0 name"); ASSERT_EQ(set.tests[0].subtest_cnt, 0, "test 0 subtest count"); ASSERT_OK(strcmp("testA", set.tests[1].name), "test 1 name"); @@ -158,8 +164,8 @@ static void test_parse_test_list_file(void) ASSERT_OK(strcmp("testB", set.tests[2].name), "test 2 name"); ASSERT_OK(strcmp("testC_no_eof_newline", set.tests[3].name), "test 3 name"); +out_free_set: free_test_filter_set(&set); - out_fclose: fclose(fp); out_remove: diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index b30ff6b3b81ae..f80660c00a1a7 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -63,6 +63,12 @@ static void test_bpf_nf_ct(int mode) .repeat = 1, ); + if (SYS_NOFAIL("iptables-legacy --version")) { + fprintf(stdout, "Missing required iptables-legacy tool\n"); + test__skip(); + return; + } + skel = test_bpf_nf__open_and_load(); if (!ASSERT_OK_PTR(skel, "test_bpf_nf__open_and_load")) return; diff --git a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c new file mode 100644 index 0000000000000..e234eb05f45d6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Google */ + +#define _GNU_SOURCE +#include +#include +#include +#include "dmabuf_iter.skel.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static int udmabuf = -1; +static const char udmabuf_test_buffer_name[DMA_BUF_NAME_LEN] = "udmabuf_test_buffer_for_iter"; +static size_t udmabuf_test_buffer_size; +static int sysheap_dmabuf = -1; +static const char sysheap_test_buffer_name[DMA_BUF_NAME_LEN] = "sysheap_test_buffer_for_iter"; +static size_t sysheap_test_buffer_size; + +static int create_udmabuf(void) +{ + struct udmabuf_create create; + int dev_udmabuf, memfd, local_udmabuf; + + udmabuf_test_buffer_size = 10 * getpagesize(); + + if (!ASSERT_LE(sizeof(udmabuf_test_buffer_name), DMA_BUF_NAME_LEN, "NAMETOOLONG")) + return -1; + + memfd = memfd_create("memfd_test", MFD_ALLOW_SEALING); + if (!ASSERT_OK_FD(memfd, "memfd_create")) + return -1; + + if (!ASSERT_OK(ftruncate(memfd, udmabuf_test_buffer_size), "ftruncate")) + goto close_memfd; + + if (!ASSERT_OK(fcntl(memfd, F_ADD_SEALS, F_SEAL_SHRINK), "seal")) + goto close_memfd; + + dev_udmabuf = open("/dev/udmabuf", O_RDONLY); + if (!ASSERT_OK_FD(dev_udmabuf, "open udmabuf")) + goto close_memfd; + + memset(&create, 0, sizeof(create)); + create.memfd = memfd; + create.flags = UDMABUF_FLAGS_CLOEXEC; + create.offset = 0; + create.size = udmabuf_test_buffer_size; + + local_udmabuf = ioctl(dev_udmabuf, UDMABUF_CREATE, &create); + close(dev_udmabuf); + if (!ASSERT_OK_FD(local_udmabuf, "udmabuf_create")) + goto close_memfd; + + if (!ASSERT_OK(ioctl(local_udmabuf, DMA_BUF_SET_NAME_B, udmabuf_test_buffer_name), "name")) + goto close_udmabuf; + + return local_udmabuf; + +close_udmabuf: + close(local_udmabuf); +close_memfd: + close(memfd); + return -1; +} + +static int create_sys_heap_dmabuf(size_t bytes) +{ + struct dma_heap_allocation_data data = { + .len = bytes, + .fd = 0, + .fd_flags = O_RDWR | O_CLOEXEC, + .heap_flags = 0, + }; + int heap_fd, ret; + + if (!ASSERT_LE(sizeof(sysheap_test_buffer_name), DMA_BUF_NAME_LEN, "NAMETOOLONG")) + return -1; + + heap_fd = open("/dev/dma_heap/system", O_RDONLY); + if (!ASSERT_OK_FD(heap_fd, "open dma heap")) + return -1; + + ret = ioctl(heap_fd, DMA_HEAP_IOCTL_ALLOC, &data); + close(heap_fd); + if (!ASSERT_OK(ret, "syheap alloc")) + return -1; + + if (!ASSERT_OK(ioctl(data.fd, DMA_BUF_SET_NAME_B, sysheap_test_buffer_name), "name")) + goto close_sysheap_dmabuf; + + return data.fd; + +close_sysheap_dmabuf: + close(data.fd); + return -1; +} + +static int create_test_buffers(void) +{ + udmabuf = create_udmabuf(); + + sysheap_test_buffer_size = 20 * getpagesize(); + sysheap_dmabuf = create_sys_heap_dmabuf(sysheap_test_buffer_size); + + if (udmabuf < 0 || sysheap_dmabuf < 0) + return -1; + + return 0; +} + +static void destroy_test_buffers(void) +{ + close(udmabuf); + udmabuf = -1; + + close(sysheap_dmabuf); + sysheap_dmabuf = -1; +} + +enum Fields { INODE, SIZE, NAME, EXPORTER, FIELD_COUNT }; +struct DmabufInfo { + unsigned long inode; + unsigned long size; + char name[DMA_BUF_NAME_LEN]; + char exporter[32]; +}; + +static bool check_dmabuf_info(const struct DmabufInfo *bufinfo, + unsigned long size, + const char *name, const char *exporter) +{ + return size == bufinfo->size && + !strcmp(name, bufinfo->name) && + !strcmp(exporter, bufinfo->exporter); +} + +static void subtest_dmabuf_iter_check_no_infinite_reads(struct dmabuf_iter *skel) +{ + int iter_fd; + char buf[256]; + + iter_fd = bpf_iter_create(bpf_link__fd(skel->links.dmabuf_collector)); + if (!ASSERT_OK_FD(iter_fd, "iter_create")) + return; + + while (read(iter_fd, buf, sizeof(buf)) > 0) + ; /* Read out all contents */ + + /* Next reads should return 0 */ + ASSERT_EQ(read(iter_fd, buf, sizeof(buf)), 0, "read"); + + close(iter_fd); +} + +static void subtest_dmabuf_iter_check_default_iter(struct dmabuf_iter *skel) +{ + bool found_test_sysheap_dmabuf = false; + bool found_test_udmabuf = false; + struct DmabufInfo bufinfo; + size_t linesize = 0; + char *line = NULL; + FILE *iter_file; + int iter_fd, f = INODE; + + iter_fd = bpf_iter_create(bpf_link__fd(skel->links.dmabuf_collector)); + if (!ASSERT_OK_FD(iter_fd, "iter_create")) + return; + + iter_file = fdopen(iter_fd, "r"); + if (!ASSERT_OK_PTR(iter_file, "fdopen")) + goto close_iter_fd; + + while (getline(&line, &linesize, iter_file) != -1) { + if (f % FIELD_COUNT == INODE) { + ASSERT_EQ(sscanf(line, "%ld", &bufinfo.inode), 1, + "read inode"); + } else if (f % FIELD_COUNT == SIZE) { + ASSERT_EQ(sscanf(line, "%ld", &bufinfo.size), 1, + "read size"); + } else if (f % FIELD_COUNT == NAME) { + ASSERT_EQ(sscanf(line, "%s", bufinfo.name), 1, + "read name"); + } else if (f % FIELD_COUNT == EXPORTER) { + ASSERT_EQ(sscanf(line, "%31s", bufinfo.exporter), 1, + "read exporter"); + + if (check_dmabuf_info(&bufinfo, + sysheap_test_buffer_size, + sysheap_test_buffer_name, + "system")) + found_test_sysheap_dmabuf = true; + else if (check_dmabuf_info(&bufinfo, + udmabuf_test_buffer_size, + udmabuf_test_buffer_name, + "udmabuf")) + found_test_udmabuf = true; + } + ++f; + } + + ASSERT_EQ(f % FIELD_COUNT, INODE, "number of fields"); + + ASSERT_TRUE(found_test_sysheap_dmabuf, "found_test_sysheap_dmabuf"); + ASSERT_TRUE(found_test_udmabuf, "found_test_udmabuf"); + + free(line); + fclose(iter_file); +close_iter_fd: + close(iter_fd); +} + +static void subtest_dmabuf_iter_check_lots_of_buffers(struct dmabuf_iter *skel) +{ + int iter_fd; + char buf[1024]; + size_t total_bytes_read = 0; + ssize_t bytes_read; + + iter_fd = bpf_iter_create(bpf_link__fd(skel->links.dmabuf_collector)); + if (!ASSERT_OK_FD(iter_fd, "iter_create")) + return; + + while ((bytes_read = read(iter_fd, buf, sizeof(buf))) > 0) + total_bytes_read += bytes_read; + + ASSERT_GT(total_bytes_read, getpagesize(), "total_bytes_read"); + + close(iter_fd); +} + + +static void subtest_dmabuf_iter_check_open_coded(struct dmabuf_iter *skel, int map_fd) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char key[DMA_BUF_NAME_LEN]; + int err, fd; + bool found; + + /* No need to attach it, just run it directly */ + fd = bpf_program__fd(skel->progs.iter_dmabuf_for_each); + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run_opts err")) + return; + if (!ASSERT_OK(topts.retval, "test_run_opts retval")) + return; + + if (!ASSERT_OK(bpf_map_get_next_key(map_fd, NULL, key), "get next key")) + return; + + do { + ASSERT_OK(bpf_map_lookup_elem(map_fd, key, &found), "lookup"); + ASSERT_TRUE(found, "found test buffer"); + } while (bpf_map_get_next_key(map_fd, key, key)); +} + +void test_dmabuf_iter(void) +{ + struct dmabuf_iter *skel = NULL; + int map_fd; + const bool f = false; + + skel = dmabuf_iter__open_and_load(); + if (!ASSERT_OK_PTR(skel, "dmabuf_iter__open_and_load")) + return; + + map_fd = bpf_map__fd(skel->maps.testbuf_hash); + if (!ASSERT_OK_FD(map_fd, "map_fd")) + goto destroy_skel; + + if (!ASSERT_OK(bpf_map_update_elem(map_fd, udmabuf_test_buffer_name, &f, BPF_ANY), + "insert udmabuf")) + goto destroy_skel; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, sysheap_test_buffer_name, &f, BPF_ANY), + "insert sysheap buffer")) + goto destroy_skel; + + if (!ASSERT_OK(create_test_buffers(), "create_test_buffers")) + goto destroy; + + if (!ASSERT_OK(dmabuf_iter__attach(skel), "skel_attach")) + goto destroy; + + if (test__start_subtest("no_infinite_reads")) + subtest_dmabuf_iter_check_no_infinite_reads(skel); + if (test__start_subtest("default_iter")) + subtest_dmabuf_iter_check_default_iter(skel); + if (test__start_subtest("lots_of_buffers")) { + size_t NUM_BUFS = 100; + int buffers[NUM_BUFS]; + int i; + + for (i = 0; i < NUM_BUFS; ++i) { + buffers[i] = create_sys_heap_dmabuf(getpagesize()); + if (!ASSERT_OK_FD(buffers[i], "dmabuf_fd")) + goto cleanup_bufs; + } + + subtest_dmabuf_iter_check_lots_of_buffers(skel); + +cleanup_bufs: + for (--i; i >= 0; --i) + close(buffers[i]); + } + if (test__start_subtest("open_coded")) + subtest_dmabuf_iter_check_open_coded(skel, map_fd); + +destroy: + destroy_test_buffers(); +destroy_skel: + dmabuf_iter__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c b/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c index d3d94596ab79c..dd926c00f4146 100644 --- a/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c +++ b/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c @@ -147,31 +147,6 @@ static void test_dummy_sleepable(void) dummy_st_ops_success__destroy(skel); } -/* dummy_st_ops.test_sleepable() parameter is not marked as nullable, - * thus bpf_prog_test_run_opts() below should be rejected as it tries - * to pass NULL for this parameter. - */ -static void test_dummy_sleepable_reject_null(void) -{ - __u64 args[1] = {0}; - LIBBPF_OPTS(bpf_test_run_opts, attr, - .ctx_in = args, - .ctx_size_in = sizeof(args), - ); - struct dummy_st_ops_success *skel; - int fd, err; - - skel = dummy_st_ops_success__open_and_load(); - if (!ASSERT_OK_PTR(skel, "dummy_st_ops_load")) - return; - - fd = bpf_program__fd(skel->progs.test_sleepable); - err = bpf_prog_test_run_opts(fd, &attr); - ASSERT_EQ(err, -EINVAL, "test_run"); - - dummy_st_ops_success__destroy(skel); -} - void test_dummy_st_ops(void) { if (test__start_subtest("dummy_st_ops_attach")) @@ -184,8 +159,6 @@ void test_dummy_st_ops(void) test_dummy_multiple_args(); if (test__start_subtest("dummy_sleepable")) test_dummy_sleepable(); - if (test__start_subtest("dummy_sleepable_reject_null")) - test_dummy_sleepable_reject_null(); RUN_TESTS(dummy_st_ops_fail); } diff --git a/tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c b/tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c index 2c57ceede095e..a84c41862ff8c 100644 --- a/tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c +++ b/tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c @@ -12,7 +12,6 @@ #include #include #include -#include "network_helpers.h" #define STACK_SIZE (1024 * 1024) static char child_stack[STACK_SIZE]; @@ -75,50 +74,6 @@ static int test_current_pid_tgid_tp(void *args) return ret; } -static int test_current_pid_tgid_cgrp(void *args) -{ - struct test_ns_current_pid_tgid__bss *bss; - struct test_ns_current_pid_tgid *skel; - int server_fd = -1, ret = -1, err; - int cgroup_fd = *(int *)args; - pid_t tgid, pid; - - skel = test_ns_current_pid_tgid__open(); - if (!ASSERT_OK_PTR(skel, "test_ns_current_pid_tgid__open")) - return ret; - - bpf_program__set_autoload(skel->progs.cgroup_bind4, true); - - err = test_ns_current_pid_tgid__load(skel); - if (!ASSERT_OK(err, "test_ns_current_pid_tgid__load")) - goto cleanup; - - bss = skel->bss; - if (get_pid_tgid(&pid, &tgid, bss)) - goto cleanup; - - skel->links.cgroup_bind4 = bpf_program__attach_cgroup( - skel->progs.cgroup_bind4, cgroup_fd); - if (!ASSERT_OK_PTR(skel->links.cgroup_bind4, "bpf_program__attach_cgroup")) - goto cleanup; - - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); - if (!ASSERT_GE(server_fd, 0, "start_server")) - goto cleanup; - - if (!ASSERT_EQ(bss->user_pid, pid, "pid")) - goto cleanup; - if (!ASSERT_EQ(bss->user_tgid, tgid, "tgid")) - goto cleanup; - ret = 0; - -cleanup: - if (server_fd >= 0) - close(server_fd); - test_ns_current_pid_tgid__destroy(skel); - return ret; -} - static void test_ns_current_pid_tgid_new_ns(int (*fn)(void *), void *arg) { int wstatus; @@ -140,25 +95,6 @@ static void test_ns_current_pid_tgid_new_ns(int (*fn)(void *), void *arg) return; } -static void test_in_netns(int (*fn)(void *), void *arg) -{ - struct nstoken *nstoken = NULL; - - SYS(cleanup, "ip netns add ns_current_pid_tgid"); - SYS(cleanup, "ip -net ns_current_pid_tgid link set dev lo up"); - - nstoken = open_netns("ns_current_pid_tgid"); - if (!ASSERT_OK_PTR(nstoken, "open_netns")) - goto cleanup; - - test_ns_current_pid_tgid_new_ns(fn, arg); - -cleanup: - if (nstoken) - close_netns(nstoken); - SYS_NOFAIL("ip netns del ns_current_pid_tgid"); -} - /* TODO: use a different tracepoint */ void serial_test_ns_current_pid_tgid(void) { @@ -166,13 +102,4 @@ void serial_test_ns_current_pid_tgid(void) test_current_pid_tgid_tp(NULL); if (test__start_subtest("new_ns_tp")) test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_tp, NULL); - if (test__start_subtest("new_ns_cgrp")) { - int cgroup_fd = -1; - - cgroup_fd = test__join_cgroup("/sock_addr"); - if (ASSERT_GE(cgroup_fd, 0, "join_cgroup")) { - test_in_netns(test_current_pid_tgid_cgrp, &cgroup_fd); - close(cgroup_fd); - } - } } diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c index 2d0796314862a..0a99fd404f6dc 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c @@ -68,7 +68,6 @@ static void test_sockmap_ktls_disconnect_after_delete(int family, int map) goto close_cli; err = disconnect(cli); - ASSERT_OK(err, "disconnect"); close_cli: close(cli); diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index 84d59419e4eb5..a6d8aa8c2a9ac 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -894,6 +894,8 @@ static void redir_partial(int family, int sotype, int sock_map, int parser_map) goto close; n = xsend(c1, buf, sizeof(buf), 0); + if (n == -1) + goto close; if (n < sizeof(buf)) FAIL("incomplete write"); diff --git a/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c b/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c index dfff6feac12c3..7e9a508c15718 100644 --- a/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c +++ b/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c @@ -21,8 +21,7 @@ #include "../progs/test_user_ringbuf.h" static const long c_sample_size = sizeof(struct sample) + BPF_RINGBUF_HDR_SZ; -static const long c_ringbuf_size = 1 << 12; /* 1 small page */ -static const long c_max_entries = c_ringbuf_size / c_sample_size; +static long c_ringbuf_size, c_max_entries; static void drain_current_samples(void) { @@ -424,7 +423,9 @@ static void test_user_ringbuf_loop(void) uint32_t remaining_samples = total_samples; int err; - BUILD_BUG_ON(total_samples <= c_max_entries); + if (!ASSERT_LT(c_max_entries, total_samples, "compare_c_max_entries")) + return; + err = load_skel_create_user_ringbuf(&skel, &ringbuf); if (err) return; @@ -686,6 +687,9 @@ void test_user_ringbuf(void) { int i; + c_ringbuf_size = getpagesize(); /* 1 page */ + c_max_entries = c_ringbuf_size / c_sample_size; + for (i = 0; i < ARRAY_SIZE(success_tests); i++) { if (!test__start_subtest(success_tests[i].test_name)) continue; diff --git a/tools/testing/selftests/bpf/progs/dmabuf_iter.c b/tools/testing/selftests/bpf/progs/dmabuf_iter.c new file mode 100644 index 0000000000000..3dfdc96080744 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/dmabuf_iter.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Google LLC */ +#include +#include +#include +#include "bpf_experimental.h" + +/* From uapi/linux/dma-buf.h */ +#define DMA_BUF_NAME_LEN 32 + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, DMA_BUF_NAME_LEN); + __type(value, bool); + __uint(max_entries, 5); +} testbuf_hash SEC(".maps"); + +/* + * Fields output by this iterator are delimited by newlines. Convert any + * newlines in user-provided printed strings to spaces. + */ +static void sanitize_string(char *src, size_t size) +{ + for (char *c = src; (size_t)(c - src) < size && *c; ++c) + if (*c == '\n') + *c = ' '; +} + +SEC("iter/dmabuf") +int dmabuf_collector(struct bpf_iter__dmabuf *ctx) +{ + const struct dma_buf *dmabuf = ctx->dmabuf; + struct seq_file *seq = ctx->meta->seq; + unsigned long inode = 0; + size_t size; + const char *pname, *exporter; + char name[DMA_BUF_NAME_LEN] = {'\0'}; + + if (!dmabuf) + return 0; + + if (BPF_CORE_READ_INTO(&inode, dmabuf, file, f_inode, i_ino) || + bpf_core_read(&size, sizeof(size), &dmabuf->size) || + bpf_core_read(&pname, sizeof(pname), &dmabuf->name) || + bpf_core_read(&exporter, sizeof(exporter), &dmabuf->exp_name)) + return 1; + + /* Buffers are not required to be named */ + if (pname) { + if (bpf_probe_read_kernel(name, sizeof(name), pname)) + return 1; + + /* Name strings can be provided by userspace */ + sanitize_string(name, sizeof(name)); + } + + BPF_SEQ_PRINTF(seq, "%lu\n%llu\n%s\n%s\n", inode, size, name, exporter); + return 0; +} + +SEC("syscall") +int iter_dmabuf_for_each(const void *ctx) +{ + struct dma_buf *d; + + bpf_for_each(dmabuf, d) { + char name[DMA_BUF_NAME_LEN]; + const char *pname; + bool *found; + long len; + int i; + + if (bpf_core_read(&pname, sizeof(pname), &d->name)) + return 1; + + /* Buffers are not required to be named */ + if (!pname) + continue; + + len = bpf_probe_read_kernel_str(name, sizeof(name), pname); + if (len < 0) + return 1; + + /* + * The entire name buffer is used as a map key. + * Zeroize any uninitialized trailing bytes after the NUL. + */ + bpf_for(i, 0, DMA_BUF_NAME_LEN) + if (i >= len) + name[i] = 0; + + found = bpf_map_lookup_elem(&testbuf_hash, name); + if (found) { + bool t = true; + + bpf_map_update_elem(&testbuf_hash, name, &t, BPF_EXIST); + } + } + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/dummy_st_ops_success.c b/tools/testing/selftests/bpf/progs/dummy_st_ops_success.c index ec0c595d47af8..151e3a3ea27fb 100644 --- a/tools/testing/selftests/bpf/progs/dummy_st_ops_success.c +++ b/tools/testing/selftests/bpf/progs/dummy_st_ops_success.c @@ -11,17 +11,8 @@ int BPF_PROG(test_1, struct bpf_dummy_ops_state *state) { int ret; - /* Check that 'state' nullable status is detected correctly. - * If 'state' argument would be assumed non-null by verifier - * the code below would be deleted as dead (which it shouldn't). - * Hide it from the compiler behind 'asm' block to avoid - * unnecessary optimizations. - */ - asm volatile ( - "if %[state] != 0 goto +2;" - "r0 = 0xf2f3f4f5;" - "exit;" - ::[state]"p"(state)); + if (!state) + return 0xf2f3f4f5; ret = state->val; state->val = 0x5a; diff --git a/tools/testing/selftests/bpf/progs/test_global_map_resize.c b/tools/testing/selftests/bpf/progs/test_global_map_resize.c index 1fbb73d3e5d5a..9be0e32cfeeea 100644 --- a/tools/testing/selftests/bpf/progs/test_global_map_resize.c +++ b/tools/testing/selftests/bpf/progs/test_global_map_resize.c @@ -31,6 +31,16 @@ int my_int_last SEC(".data.array_not_last"); int percpu_arr[1] SEC(".data.percpu_arr"); +/* at least one extern is included, to ensure that a specific + * regression is tested whereby resizing resulted in a free-after-use + * bug after type information is invalidated by the resize operation. + * + * There isn't a particularly good API to test for this specific condition, + * but by having externs for the resizing tests it will cover this path. + */ +extern int LINUX_KERNEL_VERSION __kconfig; +long version_sink; + SEC("tp/syscalls/sys_enter_getpid") int bss_array_sum(void *ctx) { @@ -43,6 +53,9 @@ int bss_array_sum(void *ctx) for (size_t i = 0; i < bss_array_len; ++i) sum += array[i]; + /* see above; ensure this is not optimized out */ + version_sink = LINUX_KERNEL_VERSION; + return 0; } @@ -58,5 +71,8 @@ int data_array_sum(void *ctx) for (size_t i = 0; i < data_array_len; ++i) sum += my_array[i]; + /* see above; ensure this is not optimized out */ + version_sink = LINUX_KERNEL_VERSION; + return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c b/tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c index d0010e698f668..aa3ec7ca16d9b 100644 --- a/tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c +++ b/tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c @@ -28,11 +28,4 @@ int tp_handler(const void *ctx) return 0; } -SEC("?cgroup/bind4") -int cgroup_bind4(struct bpf_sock_addr *ctx) -{ - get_pid_tgid(); - return 1; -} - char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tcpnotify_kern.c b/tools/testing/selftests/bpf/progs/test_tcpnotify_kern.c index 540181c115a85..ef00d38b0a8d2 100644 --- a/tools/testing/selftests/bpf/progs/test_tcpnotify_kern.c +++ b/tools/testing/selftests/bpf/progs/test_tcpnotify_kern.c @@ -23,7 +23,6 @@ struct { struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); - __uint(max_entries, 2); __type(key, int); __type(value, __u32); } perf_event_map SEC(".maps"); diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 7ea535bfbacd3..e4ef82a6ee38c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -619,7 +619,7 @@ __naked void pass_pointer_to_tail_call(void) SEC("socket") __description("unpriv: cmp map pointer with zero") -__success __failure_unpriv __msg_unpriv("R1 pointer comparison") +__success __success_unpriv __retval(0) __naked void cmp_map_pointer_with_zero(void) { diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c index 4694422aa76c3..88e4aeab21b7b 100644 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ b/tools/testing/selftests/bpf/test_lirc_mode2_user.c @@ -74,7 +74,7 @@ int main(int argc, char **argv) /* Let's try detach it before it was ever attached */ ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret != -1 || errno != ENOENT) { + if (ret != -ENOENT) { printf("bpf_prog_detach2 not attached should fail: %m\n"); return 1; } diff --git a/tools/testing/selftests/bpf/test_lru_map.c b/tools/testing/selftests/bpf/test_lru_map.c index fda7589c50236..0921939532c6c 100644 --- a/tools/testing/selftests/bpf/test_lru_map.c +++ b/tools/testing/selftests/bpf/test_lru_map.c @@ -138,6 +138,18 @@ static int sched_next_online(int pid, int *next_to_try) return ret; } +/* Derive target_free from map_size, same as bpf_common_lru_populate */ +static unsigned int __tgt_size(unsigned int map_size) +{ + return (map_size / nr_cpus) / 2; +} + +/* Inverse of how bpf_common_lru_populate derives target_free from map_size. */ +static unsigned int __map_size(unsigned int tgt_free) +{ + return tgt_free * nr_cpus * 2; +} + /* Size of the LRU map is 2 * Add key=1 (+1 key) * Add key=2 (+1 key) @@ -231,11 +243,11 @@ static void test_lru_sanity0(int map_type, int map_flags) printf("Pass\n"); } -/* Size of the LRU map is 1.5*tgt_free - * Insert 1 to tgt_free (+tgt_free keys) - * Lookup 1 to tgt_free/2 - * Insert 1+tgt_free to 2*tgt_free (+tgt_free keys) - * => 1+tgt_free/2 to LOCALFREE_TARGET will be removed by LRU +/* Verify that unreferenced elements are recycled before referenced ones. + * Insert elements. + * Reference a subset of these. + * Insert more, enough to trigger recycling. + * Verify that unreferenced are recycled. */ static void test_lru_sanity1(int map_type, int map_flags, unsigned int tgt_free) { @@ -257,7 +269,7 @@ static void test_lru_sanity1(int map_type, int map_flags, unsigned int tgt_free) batch_size = tgt_free / 2; assert(batch_size * 2 == tgt_free); - map_size = tgt_free + batch_size; + map_size = __map_size(tgt_free) + batch_size; lru_map_fd = create_map(map_type, map_flags, map_size); assert(lru_map_fd != -1); @@ -266,13 +278,13 @@ static void test_lru_sanity1(int map_type, int map_flags, unsigned int tgt_free) value[0] = 1234; - /* Insert 1 to tgt_free (+tgt_free keys) */ - end_key = 1 + tgt_free; + /* Insert map_size - batch_size keys */ + end_key = 1 + __map_size(tgt_free); for (key = 1; key < end_key; key++) assert(!bpf_map_update_elem(lru_map_fd, &key, value, BPF_NOEXIST)); - /* Lookup 1 to tgt_free/2 */ + /* Lookup 1 to batch_size */ end_key = 1 + batch_size; for (key = 1; key < end_key; key++) { assert(!bpf_map_lookup_elem_with_ref_bit(lru_map_fd, key, value)); @@ -280,12 +292,13 @@ static void test_lru_sanity1(int map_type, int map_flags, unsigned int tgt_free) BPF_NOEXIST)); } - /* Insert 1+tgt_free to 2*tgt_free - * => 1+tgt_free/2 to LOCALFREE_TARGET will be + /* Insert another map_size - batch_size keys + * Map will contain 1 to batch_size plus these latest, i.e., + * => previous 1+batch_size to map_size - batch_size will have been * removed by LRU */ - key = 1 + tgt_free; - end_key = key + tgt_free; + key = 1 + __map_size(tgt_free); + end_key = key + __map_size(tgt_free); for (; key < end_key; key++) { assert(!bpf_map_update_elem(lru_map_fd, &key, value, BPF_NOEXIST)); @@ -301,17 +314,8 @@ static void test_lru_sanity1(int map_type, int map_flags, unsigned int tgt_free) printf("Pass\n"); } -/* Size of the LRU map 1.5 * tgt_free - * Insert 1 to tgt_free (+tgt_free keys) - * Update 1 to tgt_free/2 - * => The original 1 to tgt_free/2 will be removed due to - * the LRU shrink process - * Re-insert 1 to tgt_free/2 again and do a lookup immeidately - * Insert 1+tgt_free to tgt_free*3/2 - * Insert 1+tgt_free*3/2 to tgt_free*5/2 - * => Key 1+tgt_free to tgt_free*3/2 - * will be removed from LRU because it has never - * been lookup and ref bit is not set +/* Verify that insertions exceeding map size will recycle the oldest. + * Verify that unreferenced elements are recycled before referenced. */ static void test_lru_sanity2(int map_type, int map_flags, unsigned int tgt_free) { @@ -334,7 +338,7 @@ static void test_lru_sanity2(int map_type, int map_flags, unsigned int tgt_free) batch_size = tgt_free / 2; assert(batch_size * 2 == tgt_free); - map_size = tgt_free + batch_size; + map_size = __map_size(tgt_free) + batch_size; lru_map_fd = create_map(map_type, map_flags, map_size); assert(lru_map_fd != -1); @@ -343,8 +347,8 @@ static void test_lru_sanity2(int map_type, int map_flags, unsigned int tgt_free) value[0] = 1234; - /* Insert 1 to tgt_free (+tgt_free keys) */ - end_key = 1 + tgt_free; + /* Insert map_size - batch_size keys */ + end_key = 1 + __map_size(tgt_free); for (key = 1; key < end_key; key++) assert(!bpf_map_update_elem(lru_map_fd, &key, value, BPF_NOEXIST)); @@ -357,8 +361,7 @@ static void test_lru_sanity2(int map_type, int map_flags, unsigned int tgt_free) * shrink the inactive list to get tgt_free * number of free nodes. * - * Hence, the oldest key 1 to tgt_free/2 - * are removed from the LRU list. + * Hence, the oldest key is removed from the LRU list. */ key = 1; if (map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { @@ -370,8 +373,7 @@ static void test_lru_sanity2(int map_type, int map_flags, unsigned int tgt_free) BPF_EXIST)); } - /* Re-insert 1 to tgt_free/2 again and do a lookup - * immeidately. + /* Re-insert 1 to batch_size again and do a lookup immediately. */ end_key = 1 + batch_size; value[0] = 4321; @@ -387,17 +389,18 @@ static void test_lru_sanity2(int map_type, int map_flags, unsigned int tgt_free) value[0] = 1234; - /* Insert 1+tgt_free to tgt_free*3/2 */ - end_key = 1 + tgt_free + batch_size; - for (key = 1 + tgt_free; key < end_key; key++) + /* Insert batch_size new elements */ + key = 1 + __map_size(tgt_free); + end_key = key + batch_size; + for (; key < end_key; key++) /* These newly added but not referenced keys will be * gone during the next LRU shrink. */ assert(!bpf_map_update_elem(lru_map_fd, &key, value, BPF_NOEXIST)); - /* Insert 1+tgt_free*3/2 to tgt_free*5/2 */ - end_key = key + tgt_free; + /* Insert map_size - batch_size elements */ + end_key += __map_size(tgt_free); for (; key < end_key; key++) { assert(!bpf_map_update_elem(lru_map_fd, &key, value, BPF_NOEXIST)); @@ -413,12 +416,12 @@ static void test_lru_sanity2(int map_type, int map_flags, unsigned int tgt_free) printf("Pass\n"); } -/* Size of the LRU map is 2*tgt_free - * It is to test the active/inactive list rotation - * Insert 1 to 2*tgt_free (+2*tgt_free keys) - * Lookup key 1 to tgt_free*3/2 - * Add 1+2*tgt_free to tgt_free*5/2 (+tgt_free/2 keys) - * => key 1+tgt_free*3/2 to 2*tgt_free are removed from LRU +/* Test the active/inactive list rotation + * + * Fill the whole map, deplete the free list. + * Reference all except the last lru->target_free elements. + * Insert lru->target_free new elements. This triggers one shrink. + * Verify that the non-referenced elements are replaced. */ static void test_lru_sanity3(int map_type, int map_flags, unsigned int tgt_free) { @@ -437,8 +440,7 @@ static void test_lru_sanity3(int map_type, int map_flags, unsigned int tgt_free) assert(sched_next_online(0, &next_cpu) != -1); - batch_size = tgt_free / 2; - assert(batch_size * 2 == tgt_free); + batch_size = __tgt_size(tgt_free); map_size = tgt_free * 2; lru_map_fd = create_map(map_type, map_flags, map_size); @@ -449,23 +451,21 @@ static void test_lru_sanity3(int map_type, int map_flags, unsigned int tgt_free) value[0] = 1234; - /* Insert 1 to 2*tgt_free (+2*tgt_free keys) */ - end_key = 1 + (2 * tgt_free); + /* Fill the map */ + end_key = 1 + map_size; for (key = 1; key < end_key; key++) assert(!bpf_map_update_elem(lru_map_fd, &key, value, BPF_NOEXIST)); - /* Lookup key 1 to tgt_free*3/2 */ - end_key = tgt_free + batch_size; + /* Reference all but the last batch_size */ + end_key = 1 + map_size - batch_size; for (key = 1; key < end_key; key++) { assert(!bpf_map_lookup_elem_with_ref_bit(lru_map_fd, key, value)); assert(!bpf_map_update_elem(expected_map_fd, &key, value, BPF_NOEXIST)); } - /* Add 1+2*tgt_free to tgt_free*5/2 - * (+tgt_free/2 keys) - */ + /* Insert new batch_size: replaces the non-referenced elements */ key = 2 * tgt_free + 1; end_key = key + batch_size; for (; key < end_key; key++) { @@ -500,7 +500,8 @@ static void test_lru_sanity4(int map_type, int map_flags, unsigned int tgt_free) lru_map_fd = create_map(map_type, map_flags, 3 * tgt_free * nr_cpus); else - lru_map_fd = create_map(map_type, map_flags, 3 * tgt_free); + lru_map_fd = create_map(map_type, map_flags, + 3 * __map_size(tgt_free)); assert(lru_map_fd != -1); expected_map_fd = create_map(BPF_MAP_TYPE_HASH, 0, diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index 2f9f6f250f171..baf11b905c4e2 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -377,6 +377,15 @@ int test__join_cgroup(const char *path); ___ok; \ }) +#define ASSERT_OK_FD(fd, name) ({ \ + static int duration = 0; \ + int ___fd = (fd); \ + bool ___ok = ___fd >= 0; \ + CHECK(!___ok, (name), "unexpected fd: %d (errno %d)\n", \ + ___fd, errno); \ + ___ok; \ +}) + #define SYS(goto_label, fmt, ...) \ ({ \ char cmd[1024]; \ diff --git a/tools/testing/selftests/bpf/test_tcpnotify_user.c b/tools/testing/selftests/bpf/test_tcpnotify_user.c index 595194453ff8f..35b4893ccdf8a 100644 --- a/tools/testing/selftests/bpf/test_tcpnotify_user.c +++ b/tools/testing/selftests/bpf/test_tcpnotify_user.c @@ -15,20 +15,18 @@ #include #include #include -#include #include -#include -#include "bpf_util.h" #include "cgroup_helpers.h" #include "test_tcpnotify.h" -#include "trace_helpers.h" #include "testing_helpers.h" #define SOCKET_BUFFER_SIZE (getpagesize() < 8192L ? getpagesize() : 8192L) pthread_t tid; +static bool exit_thread; + int rx_callbacks; static void dummyfn(void *ctx, int cpu, void *data, __u32 size) @@ -45,7 +43,7 @@ void tcp_notifier_poller(struct perf_buffer *pb) { int err; - while (1) { + while (!exit_thread) { err = perf_buffer__poll(pb, 100); if (err < 0 && err != -EINTR) { printf("failed perf_buffer__poll: %d\n", err); @@ -78,15 +76,10 @@ int main(int argc, char **argv) int error = EXIT_FAILURE; struct bpf_object *obj; char test_script[80]; - cpu_set_t cpuset; __u32 key = 0; libbpf_set_strict_mode(LIBBPF_STRICT_ALL); - CPU_ZERO(&cpuset); - CPU_SET(0, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); - cg_fd = cgroup_setup_and_join(cg_path); if (cg_fd < 0) goto err; @@ -151,6 +144,13 @@ int main(int argc, char **argv) sleep(10); + exit_thread = true; + int ret = pthread_join(tid, NULL); + if (ret) { + printf("FAILED: pthread_join\n"); + goto err; + } + if (verify_result(&g)) { printf("FAILED: Wrong stats Expected %d calls, got %d\n", g.ncalls, rx_callbacks); diff --git a/tools/testing/selftests/bpf/test_xsk.sh b/tools/testing/selftests/bpf/test_xsk.sh index 2aa5a3445056a..f4205823175a9 100755 --- a/tools/testing/selftests/bpf/test_xsk.sh +++ b/tools/testing/selftests/bpf/test_xsk.sh @@ -203,4 +203,6 @@ done if [ $failures -eq 0 ]; then echo "All tests successful!" +else + exit 1 fi diff --git a/tools/testing/selftests/drivers/net/netdevsim/Makefile b/tools/testing/selftests/drivers/net/netdevsim/Makefile new file mode 100644 index 0000000000000..50932e13cb5a8 --- /dev/null +++ b/tools/testing/selftests/drivers/net/netdevsim/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0+ OR MIT + +TEST_PROGS = devlink.sh \ + devlink_in_netns.sh \ + devlink_trap.sh \ + ethtool-coalesce.sh \ + ethtool-fec.sh \ + ethtool-pause.sh \ + ethtool-ring.sh \ + fib.sh \ + hw_stats_l3.sh \ + nexthop.sh \ + psample.sh \ + tc-mq-visibility.sh \ + udp_tunnel_nic.sh \ + +TEST_FILES := \ + ethtool-common.sh +# end of TEST_FILES + +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/net/netdevsim/settings b/tools/testing/selftests/drivers/net/netdevsim/settings new file mode 100644 index 0000000000000..a62d2fa1275c6 --- /dev/null +++ b/tools/testing/selftests/drivers/net/netdevsim/settings @@ -0,0 +1 @@ +timeout=600 diff --git a/tools/testing/selftests/exec/Makefile b/tools/testing/selftests/exec/Makefile index a0b8688b08369..a705493c04bb7 100644 --- a/tools/testing/selftests/exec/Makefile +++ b/tools/testing/selftests/exec/Makefile @@ -3,8 +3,13 @@ CFLAGS = -Wall CFLAGS += -Wno-nonnull CFLAGS += -D_GNU_SOURCE +ALIGNS := 0x1000 0x200000 0x1000000 +ALIGN_PIES := $(patsubst %,load_address.%,$(ALIGNS)) +ALIGN_STATIC_PIES := $(patsubst %,load_address.static.%,$(ALIGNS)) +ALIGNMENT_TESTS := $(ALIGN_PIES) $(ALIGN_STATIC_PIES) + TEST_PROGS := binfmt_script.py -TEST_GEN_PROGS := execveat load_address_4096 load_address_2097152 load_address_16777216 non-regular +TEST_GEN_PROGS := execveat non-regular $(ALIGNMENT_TESTS) TEST_GEN_FILES := execveat.symlink execveat.denatured script subdir # Makefile is a run-time dependency, since it's accessed by the execveat test TEST_FILES := Makefile @@ -28,9 +33,9 @@ $(OUTPUT)/execveat.symlink: $(OUTPUT)/execveat $(OUTPUT)/execveat.denatured: $(OUTPUT)/execveat cp $< $@ chmod -x $@ -$(OUTPUT)/load_address_4096: load_address.c - $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x1000 -pie -static $< -o $@ -$(OUTPUT)/load_address_2097152: load_address.c - $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x200000 -pie -static $< -o $@ -$(OUTPUT)/load_address_16777216: load_address.c - $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x1000000 -pie -static $< -o $@ +$(OUTPUT)/load_address.0x%: load_address.c + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=$(lastword $(subst ., ,$@)) \ + -fPIE -pie $< -o $@ +$(OUTPUT)/load_address.static.0x%: load_address.c + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=$(lastword $(subst ., ,$@)) \ + -fPIE -static-pie $< -o $@ diff --git a/tools/testing/selftests/exec/load_address.c b/tools/testing/selftests/exec/load_address.c index d487c2f6a6150..8257fddba8c8d 100644 --- a/tools/testing/selftests/exec/load_address.c +++ b/tools/testing/selftests/exec/load_address.c @@ -5,10 +5,13 @@ #include #include #include +#include +#include "../kselftest.h" struct Statistics { unsigned long long load_address; unsigned long long alignment; + bool interp; }; int ExtractStatistics(struct dl_phdr_info *info, size_t size, void *data) @@ -25,11 +28,20 @@ int ExtractStatistics(struct dl_phdr_info *info, size_t size, void *data) stats->alignment = 0; for (i = 0; i < info->dlpi_phnum; i++) { + unsigned long long align; + + if (info->dlpi_phdr[i].p_type == PT_INTERP) { + stats->interp = true; + continue; + } + if (info->dlpi_phdr[i].p_type != PT_LOAD) continue; - if (info->dlpi_phdr[i].p_align > stats->alignment) - stats->alignment = info->dlpi_phdr[i].p_align; + align = info->dlpi_phdr[i].p_align; + + if (align > stats->alignment) + stats->alignment = align; } return 1; // Terminate dl_iterate_phdr. @@ -37,32 +49,57 @@ int ExtractStatistics(struct dl_phdr_info *info, size_t size, void *data) int main(int argc, char **argv) { - struct Statistics extracted; - unsigned long long misalign; + struct Statistics extracted = { }; + unsigned long long misalign, pow2; + bool interp_needed; + char buf[1024]; + FILE *maps; int ret; - ret = dl_iterate_phdr(ExtractStatistics, &extracted); - if (ret != 1) { - fprintf(stderr, "FAILED\n"); - return 1; - } + ksft_print_header(); + ksft_set_plan(4); - if (extracted.alignment == 0) { - fprintf(stderr, "No alignment found\n"); - return 1; - } else if (extracted.alignment & (extracted.alignment - 1)) { - fprintf(stderr, "Alignment is not a power of 2\n"); - return 1; + /* Dump maps file for debugging reference. */ + maps = fopen("/proc/self/maps", "r"); + if (!maps) + ksft_exit_fail_msg("FAILED: /proc/self/maps: %s\n", strerror(errno)); + while (fgets(buf, sizeof(buf), maps)) { + ksft_print_msg("%s", buf); } + fclose(maps); + + /* Walk the program headers. */ + ret = dl_iterate_phdr(ExtractStatistics, &extracted); + if (ret != 1) + ksft_exit_fail_msg("FAILED: dl_iterate_phdr\n"); + + /* Report our findings. */ + ksft_print_msg("load_address=%#llx alignment=%#llx\n", + extracted.load_address, extracted.alignment); + + /* If we're named with ".static." we expect no INTERP. */ + interp_needed = strstr(argv[0], ".static.") == NULL; + + /* Were we built as expected? */ + ksft_test_result(interp_needed == extracted.interp, + "%s INTERP program header %s\n", + interp_needed ? "Wanted" : "Unwanted", + extracted.interp ? "seen" : "missing"); + /* Did we find an alignment? */ + ksft_test_result(extracted.alignment != 0, + "Alignment%s found\n", extracted.alignment ? "" : " NOT"); + + /* Is the alignment sane? */ + pow2 = extracted.alignment & (extracted.alignment - 1); + ksft_test_result(pow2 == 0, + "Alignment is%s a power of 2: %#llx\n", + pow2 == 0 ? "" : " NOT", extracted.alignment); + + /* Is the load address aligned? */ misalign = extracted.load_address & (extracted.alignment - 1); - if (misalign) { - printf("alignment = %llu, load_address = %llu\n", - extracted.alignment, extracted.load_address); - fprintf(stderr, "FAILED\n"); - return 1; - } + ksft_test_result(misalign == 0, "Load Address is %saligned (%#llx)\n", + misalign ? "MIS" : "", misalign); - fprintf(stderr, "PASS\n"); - return 0; + ksft_finished(); } diff --git a/tools/testing/selftests/filesystems/fuse/fuse_test.c b/tools/testing/selftests/filesystems/fuse/fuse_test.c index 06305ea9f6b16..359ae9e889b11 100644 --- a/tools/testing/selftests/filesystems/fuse/fuse_test.c +++ b/tools/testing/selftests/filesystems/fuse/fuse_test.c @@ -255,7 +255,7 @@ static int bpf_test_partial(const char *mount_dir) TEST(src_fd = open(ft_src, O_DIRECTORY | O_RDONLY | O_CLOEXEC), src_fd != -1); TESTEQUAL(create_file(src_fd, s(test_name), 1, 2), 0); - TESTEQUAL(install_elf_bpf("test_bpf.bpf", "test_trace", + TESTEQUAL(install_elf_bpf("test_bpf.bpf", "test_partial", &bpf_fd, NULL, NULL), 0); TESTEQUAL(mount_fuse(mount_dir, bpf_fd, src_fd, &fuse_dev), 0); @@ -363,7 +363,7 @@ static int bpf_test_readdir(const char *mount_dir) src_fd != -1); TESTEQUAL(create_file(src_fd, s(names[0]), 1, 2), 0); TESTEQUAL(create_file(src_fd, s(names[1]), 1, 2), 0); - TESTEQUAL(install_elf_bpf("test_bpf.bpf", "test_trace", + TESTEQUAL(install_elf_bpf("test_bpf.bpf", "test_partial", &bpf_fd, NULL, NULL), 0); TESTEQUAL(mount_fuse(mount_dir, bpf_fd, src_fd, &fuse_dev), 0); @@ -1490,6 +1490,8 @@ static int bpf_test_statfs(const char *mount_dir) static int bpf_test_lseek(const char *mount_dir) { const char *file = "real"; + const char *sparse_file = "sparse"; + const off_t sparse_length = 0x100000000u; const char *test_data = "data"; int result = TEST_FAILURE; int src_fd = -1; @@ -1504,6 +1506,12 @@ static int bpf_test_lseek(const char *mount_dir) TESTEQUAL(write(fd, test_data, strlen(test_data)), strlen(test_data)); TESTSYSCALL(close(fd)); fd = -1; + TEST(fd = openat(src_fd, sparse_file, O_CREAT | O_RDWR | O_CLOEXEC, + 0777), + fd != -1); + TESTSYSCALL(ftruncate(fd, sparse_length)); + TESTSYSCALL(close(fd)); + fd = -1; TESTEQUAL(install_elf_bpf("test_bpf.bpf", "test_trace", &bpf_fd, NULL, NULL), 0); TESTEQUAL(mount_fuse(mount_dir, bpf_fd, src_fd, &fuse_dev), 0); @@ -1518,6 +1526,18 @@ static int bpf_test_lseek(const char *mount_dir) TESTEQUAL(bpf_test_trace("lseek"), 0); TESTEQUAL(lseek(fd, 1, SEEK_DATA), 1); TESTEQUAL(bpf_test_trace("lseek"), 0); + TESTSYSCALL(close(fd)); + fd = -1; + + TEST(fd = s_open(s_path(s(mount_dir), s(sparse_file)), + O_RDONLY | O_CLOEXEC), + fd != -1); + TESTEQUAL(lseek(fd, -256, SEEK_END), sparse_length - 256); + TESTEQUAL(lseek(fd, 0, SEEK_CUR), sparse_length - 256); + + TESTSYSCALL(close(fd)); + fd = -1; + result = TEST_SUCCESS; out: close(fd); diff --git a/tools/testing/selftests/filesystems/fuse/test_bpf.c b/tools/testing/selftests/filesystems/fuse/test_bpf.c index a014b915c059b..be5f59ad83439 100644 --- a/tools/testing/selftests/filesystems/fuse/test_bpf.c +++ b/tools/testing/selftests/filesystems/fuse/test_bpf.c @@ -28,9 +28,9 @@ int readdir_test(struct fuse_bpf_args *fa) } } -SEC("test_trace") +SEC("test_partial") /* return FUSE_BPF_BACKING to use backing fs, 0 to pass to usermode */ -int trace_test(struct fuse_bpf_args *fa) +int partial_test(struct fuse_bpf_args *fa) { switch (fa->opcode) { case FUSE_LOOKUP | FUSE_PREFILTER: { @@ -329,6 +329,195 @@ int trace_test(struct fuse_bpf_args *fa) } } +SEC("test_trace") +/* return FUSE_BPF_BACKING to use backing fs, 0 to pass to usermode */ +int trace_test(struct fuse_bpf_args *fa) +{ + switch (fa->opcode) { + case FUSE_LOOKUP | FUSE_PREFILTER: { + /* real and partial use backing file */ + const char *name = fa->in_args[0].value; + + bpf_printk("lookup %s", name); + return FUSE_BPF_BACKING; + } + + case FUSE_ACCESS | FUSE_PREFILTER: { + bpf_printk("Access: %d", fa->nodeid); + return FUSE_BPF_BACKING; + } + + case FUSE_CREATE | FUSE_PREFILTER: + bpf_printk("Create: %d", fa->nodeid); + return FUSE_BPF_BACKING; + + case FUSE_MKNOD | FUSE_PREFILTER: { + const struct fuse_mknod_in *fmi = fa->in_args[0].value; + const char *name = fa->in_args[1].value; + + bpf_printk("mknod %s %x %x", name, fmi->rdev | fmi->mode, fmi->umask); + return FUSE_BPF_BACKING; + } + + case FUSE_MKDIR | FUSE_PREFILTER: { + const struct fuse_mkdir_in *fmi = fa->in_args[0].value; + const char *name = fa->in_args[1].value; + + bpf_printk("mkdir %s %x %x", name, fmi->mode, fmi->umask); + return FUSE_BPF_BACKING; + } + + case FUSE_RMDIR | FUSE_PREFILTER: { + const char *name = fa->in_args[0].value; + + bpf_printk("rmdir %s", name); + return FUSE_BPF_BACKING; + } + + case FUSE_RENAME | FUSE_PREFILTER: { + const char *oldname = fa->in_args[1].value; + const char *newname = fa->in_args[2].value; + + bpf_printk("rename from %s", oldname); + bpf_printk("rename to %s", newname); + return FUSE_BPF_BACKING; + } + + case FUSE_RENAME2 | FUSE_PREFILTER: { + const struct fuse_rename2_in *fri = fa->in_args[0].value; + uint32_t flags = fri->flags; + const char *oldname = fa->in_args[1].value; + const char *newname = fa->in_args[2].value; + + bpf_printk("rename(%x) from %s", flags, oldname); + bpf_printk("rename to %s", newname); + return FUSE_BPF_BACKING; + } + + case FUSE_UNLINK | FUSE_PREFILTER: { + const char *name = fa->in_args[0].value; + + bpf_printk("unlink %s", name); + return FUSE_BPF_BACKING; + } + + case FUSE_LINK | FUSE_PREFILTER: { + const struct fuse_link_in *fli = fa->in_args[0].value; + const char *link_name = fa->in_args[1].value; + + bpf_printk("link %d %s", fli->oldnodeid, link_name); + return FUSE_BPF_BACKING; + } + + case FUSE_SYMLINK | FUSE_PREFILTER: { + const char *link_name = fa->in_args[0].value; + const char *link_dest = fa->in_args[1].value; + + bpf_printk("symlink from %s", link_name); + bpf_printk("symlink to %s", link_dest); + return FUSE_BPF_BACKING; + } + + case FUSE_READLINK | FUSE_PREFILTER: { + const char *link_name = fa->in_args[0].value; + + bpf_printk("readlink from", link_name); + return FUSE_BPF_BACKING; + } + + case FUSE_OPEN | FUSE_PREFILTER: { + bpf_printk("open"); + return FUSE_BPF_BACKING; + } + + case FUSE_OPEN | FUSE_POSTFILTER: + bpf_printk("open postfilter"); + return FUSE_BPF_USER_FILTER; + + case FUSE_READ | FUSE_PREFILTER: { + const struct fuse_read_in *fri = fa->in_args[0].value; + + bpf_printk("read %llu", fri->offset); + return FUSE_BPF_BACKING; + } + + case FUSE_GETATTR | FUSE_PREFILTER: { + bpf_printk("getattr"); + return FUSE_BPF_BACKING; + } + + case FUSE_SETATTR | FUSE_PREFILTER: { + bpf_printk("setattr"); + return FUSE_BPF_BACKING; + } + + case FUSE_OPENDIR | FUSE_PREFILTER: { + bpf_printk("opendir"); + return FUSE_BPF_BACKING; + } + + case FUSE_READDIR | FUSE_PREFILTER: { + bpf_printk("readdir"); + return FUSE_BPF_BACKING; + } + + case FUSE_FLUSH | FUSE_PREFILTER: { + bpf_printk("Flush"); + return FUSE_BPF_BACKING; + } + + case FUSE_GETXATTR | FUSE_PREFILTER: { + const char *name = fa->in_args[1].value; + + bpf_printk("getxattr %s", name); + return FUSE_BPF_BACKING; + } + + case FUSE_LISTXATTR | FUSE_PREFILTER: { + const char *name = fa->in_args[1].value; + + bpf_printk("listxattr %s", name); + return FUSE_BPF_BACKING; + } + + case FUSE_SETXATTR | FUSE_PREFILTER: { + const char *name = fa->in_args[1].value; + unsigned int size = fa->in_args[2].size; + + bpf_printk("setxattr %s %u", name, size); + return FUSE_BPF_BACKING; + } + + case FUSE_REMOVEXATTR | FUSE_PREFILTER: { + const char *name = fa->in_args[0].value; + + bpf_printk("removexattr %s", name); + return FUSE_BPF_BACKING; + } + + case FUSE_CANONICAL_PATH | FUSE_PREFILTER: { + bpf_printk("canonical_path"); + return FUSE_BPF_BACKING; + } + + case FUSE_STATFS | FUSE_PREFILTER: { + bpf_printk("statfs"); + return FUSE_BPF_BACKING; + } + + case FUSE_LSEEK | FUSE_PREFILTER: { + const struct fuse_lseek_in *fli = fa->in_args[0].value; + + bpf_printk("lseek type:%d, offset:%lld", fli->whence, fli->offset); + return FUSE_BPF_BACKING; + } + + default: + bpf_printk("Unknown opcode %d", fa->opcode); + return FUSE_BPF_BACKING; + } +} + SEC("test_hidden") int trace_hidden(struct fuse_bpf_args *fa) { diff --git a/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc b/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc index b7c8f29c09a97..65916bb55dfbb 100644 --- a/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc +++ b/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc @@ -14,11 +14,35 @@ fail() { #msg exit_fail } +# As reading trace can last forever, simply look for 3 different +# events then exit out of reading the file. If there's not 3 different +# events, then the test has failed. +check_unique() { + cat trace | grep -v '^#' | awk ' + BEGIN { cnt = 0; } + { + for (i = 0; i < cnt; i++) { + if (event[i] == $5) { + break; + } + } + if (i == cnt) { + event[cnt++] = $5; + if (cnt > 2) { + exit; + } + } + } + END { + printf "%d", cnt; + }' +} + echo 'sched:*' > set_event yield -count=`head -n 100 trace | grep -v ^# | awk '{ print $5 }' | sort -u | wc -l` +count=`check_unique` if [ $count -lt 3 ]; then fail "at least fork, exec and exit events should be recorded" fi @@ -29,7 +53,7 @@ echo 1 > events/sched/enable yield -count=`head -n 100 trace | grep -v ^# | awk '{ print $5 }' | sort -u | wc -l` +count=`check_unique` if [ $count -lt 3 ]; then fail "at least fork, exec and exit events should be recorded" fi diff --git a/tools/testing/selftests/ftrace/test.d/filter/event-filter-function.tc b/tools/testing/selftests/ftrace/test.d/filter/event-filter-function.tc index 118247b8dd84d..ed791b995a43f 100644 --- a/tools/testing/selftests/ftrace/test.d/filter/event-filter-function.tc +++ b/tools/testing/selftests/ftrace/test.d/filter/event-filter-function.tc @@ -20,6 +20,10 @@ sample_events() { echo 0 > tracing_on echo 0 > events/enable +# Clear functions caused by page cache; run sample_events twice +sample_events +sample_events + echo "Get the most frequently calling function" echo > trace sample_events diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-glob.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-glob.tc index 4b994b6df5ac3..ed81eaf2afd6d 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-glob.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-glob.tc @@ -29,7 +29,7 @@ ftrace_filter_check 'schedule*' '^schedule.*$' ftrace_filter_check '*pin*lock' '.*pin.*lock$' # filter by start*mid* -ftrace_filter_check 'mutex*try*' '^mutex.*try.*' +ftrace_filter_check 'mutex*unl*' '^mutex.*unl.*' # Advanced full-glob matching feature is recently supported. # Skip the tests if we are sure the kernel does not support it. diff --git a/tools/testing/selftests/futex/include/futextest.h b/tools/testing/selftests/futex/include/futextest.h index ddbcfc9b7bac4..7a5fd1d5355e7 100644 --- a/tools/testing/selftests/futex/include/futextest.h +++ b/tools/testing/selftests/futex/include/futextest.h @@ -47,6 +47,17 @@ typedef volatile u_int32_t futex_t; FUTEX_PRIVATE_FLAG) #endif +/* + * SYS_futex is expected from system C library, in glibc some 32-bit + * architectures (e.g. RV32) are using 64-bit time_t, therefore it doesn't have + * SYS_futex defined but just SYS_futex_time64. Define SYS_futex as + * SYS_futex_time64 in this situation to ensure the compilation and the + * compatibility. + */ +#if !defined(SYS_futex) && defined(SYS_futex_time64) +#define SYS_futex SYS_futex_time64 +#endif + /** * futex() - SYS_futex syscall wrapper * @uaddr: address of first futex diff --git a/tools/testing/selftests/iommu/iommufd.c b/tools/testing/selftests/iommu/iommufd.c index 890a81f4ff618..67fcc99fbd6d3 100644 --- a/tools/testing/selftests/iommu/iommufd.c +++ b/tools/testing/selftests/iommu/iommufd.c @@ -1728,6 +1728,8 @@ TEST_F(vfio_compat_mock_domain, map) ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size); + /* Unmap of empty is success */ + ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); /* UNMAP_FLAG_ALL requres 0 iova/size */ ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); diff --git a/tools/testing/selftests/memfd/memfd_test.c b/tools/testing/selftests/memfd/memfd_test.c index e92b60eecb7d5..9c9c82fd18a7e 100644 --- a/tools/testing/selftests/memfd/memfd_test.c +++ b/tools/testing/selftests/memfd/memfd_test.c @@ -285,6 +285,24 @@ static void *mfd_assert_mmap_shared(int fd) return p; } +static void *mfd_assert_mmap_read_shared(int fd) +{ + void *p; + + p = mmap(NULL, + mfd_def_size, + PROT_READ, + MAP_SHARED, + fd, + 0); + if (p == MAP_FAILED) { + printf("mmap() failed: %m\n"); + abort(); + } + + return p; +} + static void *mfd_assert_mmap_private(int fd) { void *p; @@ -986,6 +1004,30 @@ static void test_seal_future_write(void) close(fd); } +static void test_seal_write_map_read_shared(void) +{ + int fd; + void *p; + + printf("%s SEAL-WRITE-MAP-READ\n", memfd_str); + + fd = mfd_assert_new("kern_memfd_seal_write_map_read", + mfd_def_size, + MFD_CLOEXEC | MFD_ALLOW_SEALING); + + mfd_assert_add_seals(fd, F_SEAL_WRITE); + mfd_assert_has_seals(fd, F_SEAL_WRITE); + + p = mfd_assert_mmap_read_shared(fd); + + mfd_assert_read(fd); + mfd_assert_read_shared(fd); + mfd_fail_write(fd); + + munmap(p, mfd_def_size); + close(fd); +} + /* * Test SEAL_SHRINK * Test whether SEAL_SHRINK actually prevents shrinking @@ -1603,6 +1645,7 @@ int main(int argc, char **argv) test_seal_write(); test_seal_future_write(); + test_seal_write_map_read_shared(); test_seal_shrink(); test_seal_grow(); test_seal_resize(); diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 309b3750e57e1..38fec412206b9 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -89,6 +89,8 @@ int check_compaction(unsigned long mem_free, unsigned long hugepage_size) int compaction_index = 0; char initial_nr_hugepages[20] = {0}; char nr_hugepages[20] = {0}; + char target_nr_hugepages[24] = {0}; + int slen; /* We want to test with 80% of available memory. Else, OOM killer comes in to play */ @@ -119,11 +121,18 @@ int check_compaction(unsigned long mem_free, unsigned long hugepage_size) lseek(fd, 0, SEEK_SET); - /* Request a large number of huge pages. The Kernel will allocate - as much as it can */ - if (write(fd, "100000", (6*sizeof(char))) != (6*sizeof(char))) { - ksft_print_msg("Failed to write 100000 to /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); + /* + * Request huge pages for about half of the free memory. The Kernel + * will allocate as much as it can, and we expect it will get at least 1/3 + */ + nr_hugepages_ul = mem_free / hugepage_size / 2; + snprintf(target_nr_hugepages, sizeof(target_nr_hugepages), + "%lu", nr_hugepages_ul); + + slen = strlen(target_nr_hugepages); + if (write(fd, target_nr_hugepages, slen) != slen) { + ksft_print_msg("Failed to write %lu to /proc/sys/vm/nr_hugepages: %s\n", + nr_hugepages_ul, strerror(errno)); goto close_fd; } diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c index 17bcb07f19f34..7278623acf8c5 100644 --- a/tools/testing/selftests/mm/madv_populate.c +++ b/tools/testing/selftests/mm/madv_populate.c @@ -264,23 +264,6 @@ static void test_softdirty(void) munmap(addr, SIZE); } -static int system_has_softdirty(void) -{ - /* - * There is no way to check if the kernel supports soft-dirty, other - * than by writing to a page and seeing if the bit was set. But the - * tests are intended to check that the bit gets set when it should, so - * doing that check would turn a potentially legitimate fail into a - * skip. Fortunately, we know for sure that arm64 does not support - * soft-dirty. So for now, let's just use the arch as a corse guide. - */ -#if defined(__aarch64__) - return 0; -#else - return 1; -#endif -} - int main(int argc, char **argv) { int nr_tests = 16; @@ -288,7 +271,7 @@ int main(int argc, char **argv) pagesize = getpagesize(); - if (system_has_softdirty()) + if (softdirty_supported()) nr_tests += 5; ksft_print_header(); @@ -300,7 +283,7 @@ int main(int argc, char **argv) test_holes(); test_populate_read(); test_populate_write(); - if (system_has_softdirty()) + if (softdirty_supported()) test_softdirty(); err = ksft_get_fail_cnt(); diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 5c3773de9f0f7..944645f9d2b3d 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -22,7 +22,9 @@ #define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */ #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ +#ifndef MIN #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#endif struct config { unsigned long long src_alignment; diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index 7dbfa53d93a05..7095b95d19ae2 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -193,8 +193,11 @@ int main(int argc, char **argv) int pagesize; ksft_print_header(); - ksft_set_plan(15); + if (!softdirty_supported()) + ksft_exit_skip("soft-dirty is not support\n"); + + ksft_set_plan(15); pagemap_fd = open(PAGEMAP_FILE_PATH, O_RDONLY); if (pagemap_fd < 0) ksft_exit_fail_msg("Failed to open %s\n", PAGEMAP_FILE_PATH); diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 646b0c4e0244d..dff3be23488b4 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -88,76 +88,6 @@ static void write_debugfs(const char *fmt, ...) } } -static char *allocate_zero_filled_hugepage(size_t len) -{ - char *result; - size_t i; - - result = memalign(pmd_pagesize, len); - if (!result) { - printf("Fail to allocate memory\n"); - exit(EXIT_FAILURE); - } - - madvise(result, len, MADV_HUGEPAGE); - - for (i = 0; i < len; i++) - result[i] = (char)0; - - return result; -} - -static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len) -{ - unsigned long rss_anon_before, rss_anon_after; - size_t i; - - if (!check_huge_anon(one_page, 4, pmd_pagesize)) { - printf("No THP is allocated\n"); - exit(EXIT_FAILURE); - } - - rss_anon_before = rss_anon(); - if (!rss_anon_before) { - printf("No RssAnon is allocated before split\n"); - exit(EXIT_FAILURE); - } - - /* split all THPs */ - write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, - (uint64_t)one_page + len, 0); - - for (i = 0; i < len; i++) - if (one_page[i] != (char)0) { - printf("%ld byte corrupted\n", i); - exit(EXIT_FAILURE); - } - - if (!check_huge_anon(one_page, 0, pmd_pagesize)) { - printf("Still AnonHugePages not split\n"); - exit(EXIT_FAILURE); - } - - rss_anon_after = rss_anon(); - if (rss_anon_after >= rss_anon_before) { - printf("Incorrect RssAnon value. Before: %ld After: %ld\n", - rss_anon_before, rss_anon_after); - exit(EXIT_FAILURE); - } -} - -void split_pmd_zero_pages(void) -{ - char *one_page; - int nr_hpages = 4; - size_t len = nr_hpages * pmd_pagesize; - - one_page = allocate_zero_filled_hugepage(len); - verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len); - printf("Split zero filled huge pages successful\n"); - free(one_page); -} - void split_pmd_thp(void) { char *one_page; @@ -375,7 +305,6 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - split_pmd_zero_pages(); split_pmd_thp(); split_pte_mapped_thp(); split_file_backed_thp(); diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 598826ebce8f9..b2af70d75711a 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -11,7 +11,6 @@ #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" #define SMAP_FILE_PATH "/proc/self/smaps" -#define STATUS_FILE_PATH "/proc/self/status" #define MAX_LINE_LENGTH 500 unsigned int __page_size; @@ -98,25 +97,40 @@ uint64_t read_pmd_pagesize(void) return strtoul(buf, NULL, 10); } -unsigned long rss_anon(void) +char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) { - unsigned long rss_anon = 0; + int ret; FILE *fp; - char buffer[MAX_LINE_LENGTH]; + char *entry = NULL; + char addr_pattern[MAX_LINE_LENGTH]; + + ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", + (unsigned long)addr); + if (ret >= MAX_LINE_LENGTH) + ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); - fp = fopen(STATUS_FILE_PATH, "r"); + fp = fopen(SMAP_FILE_PATH, "r"); if (!fp) - ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); + ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, + SMAP_FILE_PATH); + + if (!check_for_pattern(fp, addr_pattern, buf, len)) + goto err_out; - if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer))) + /* Fetch the pattern in the same block */ + if (!check_for_pattern(fp, pattern, buf, len)) goto err_out; - if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1) - ksft_exit_fail_msg("Reading status error\n"); + /* Trim trailing newline */ + entry = strchr(buf, '\n'); + if (entry) + *entry = '\0'; + + entry = buf + strlen(pattern); err_out: fclose(fp); - return rss_anon; + return entry; } bool __check_huge(void *addr, char *pattern, int nr_hpages, @@ -291,3 +305,44 @@ int uffd_unregister(int uffd, void *addr, uint64_t len) return ret; } + +static bool check_vmflag(void *addr, const char *flag) +{ + char buffer[MAX_LINE_LENGTH]; + const char *flags; + size_t flaglen; + + flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer)); + if (!flags) + ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr); + + while (true) { + flags += strspn(flags, " "); + + flaglen = strcspn(flags, " "); + if (!flaglen) + return false; + + if (flaglen == strlen(flag) && !memcmp(flags, flag, flaglen)) + return true; + + flags += flaglen; + } +} + +bool softdirty_supported(void) +{ + char *addr; + bool supported = false; + const size_t pagesize = getpagesize(); + + /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */ + addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); + if (!addr) + ksft_exit_fail_msg("mmap failed\n"); + + supported = check_vmflag(addr, "sd"); + munmap(addr, pagesize); + return supported; +} diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 65bee2e5624e9..9816d6e9bce6c 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -39,7 +39,6 @@ unsigned long pagemap_get_pfn(int fd, char *start); void clear_softdirty(void); bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len); uint64_t read_pmd_pagesize(void); -unsigned long rss_anon(void); bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size); bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size); bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size); @@ -52,6 +51,7 @@ int uffd_register(int uffd, void *addr, uint64_t len, int uffd_unregister(int uffd, void *addr, uint64_t len); int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor, uint64_t *ioctls); +bool softdirty_supported(void); /* * On ppc64 this will only work with radix 2M hugepage size diff --git a/tools/testing/selftests/net/bareudp.sh b/tools/testing/selftests/net/bareudp.sh index f366cadbc5e86..ff4308b48e65d 100755 --- a/tools/testing/selftests/net/bareudp.sh +++ b/tools/testing/selftests/net/bareudp.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # SPDX-License-Identifier: GPL-2.0 # Test various bareudp tunnel configurations. diff --git a/tools/testing/selftests/net/bind_bhash.c b/tools/testing/selftests/net/bind_bhash.c index 57ff67a3751eb..da04b0b19b73c 100644 --- a/tools/testing/selftests/net/bind_bhash.c +++ b/tools/testing/selftests/net/bind_bhash.c @@ -75,7 +75,7 @@ static void *setup(void *arg) int *array = (int *)arg; for (i = 0; i < MAX_CONNECTIONS; i++) { - sock_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, setup_addr); + sock_fd = bind_socket(SO_REUSEPORT, setup_addr); if (sock_fd < 0) { ret = sock_fd; pthread_exit(&ret); @@ -103,7 +103,7 @@ int main(int argc, const char *argv[]) setup_addr = use_v6 ? setup_addr_v6 : setup_addr_v4; - listener_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, setup_addr); + listener_fd = bind_socket(SO_REUSEPORT, setup_addr); if (listen(listener_fd, 100) < 0) { perror("listen failed"); return -1; diff --git a/tools/testing/selftests/net/fcnal-test.sh b/tools/testing/selftests/net/fcnal-test.sh index d32a14ba069ac..5d8f50cd38b7e 100755 --- a/tools/testing/selftests/net/fcnal-test.sh +++ b/tools/testing/selftests/net/fcnal-test.sh @@ -194,7 +194,7 @@ show_hint() kill_procs() { killall nettest ping ping6 >/dev/null 2>&1 - sleep 1 + slowwait 2 sh -c 'test -z "$(pgrep '"'^(nettest|ping|ping6)$'"')"' } do_run_cmd() @@ -423,6 +423,8 @@ create_ns() ip netns exec ${ns} sysctl -qw net.ipv6.conf.all.keep_addr_on_down=1 ip netns exec ${ns} sysctl -qw net.ipv6.conf.all.forwarding=1 ip netns exec ${ns} sysctl -qw net.ipv6.conf.default.forwarding=1 + ip netns exec ${ns} sysctl -qw net.ipv6.conf.default.accept_dad=0 + ip netns exec ${ns} sysctl -qw net.ipv6.conf.all.accept_dad=0 } # create veth pair to connect namespaces and apply addresses. @@ -865,7 +867,7 @@ ipv4_tcp_md5_novrf() # basic use case log_start run_cmd nettest -s -M ${MD5_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: Single address config" @@ -873,7 +875,7 @@ ipv4_tcp_md5_novrf() log_start show_hint "Should timeout due to MD5 mismatch" run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: Server no config, client uses password" @@ -881,7 +883,7 @@ ipv4_tcp_md5_novrf() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -s -M ${MD5_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: Client uses wrong password" @@ -889,7 +891,7 @@ ipv4_tcp_md5_novrf() log_start show_hint "Should timeout due to MD5 mismatch" run_cmd nettest -s -M ${MD5_PW} -m ${NSB_LO_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: Client address does not match address configured with password" @@ -900,7 +902,7 @@ ipv4_tcp_md5_novrf() # client in prefix log_start run_cmd nettest -s -M ${MD5_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: Prefix config" @@ -908,7 +910,7 @@ ipv4_tcp_md5_novrf() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -s -M ${MD5_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: Prefix config, client uses wrong password" @@ -916,7 +918,7 @@ ipv4_tcp_md5_novrf() log_start show_hint "Should timeout due to MD5 mismatch" run_cmd nettest -s -M ${MD5_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -c ${NSB_LO_IP} -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: Prefix config, client address not in configured prefix" } @@ -933,7 +935,7 @@ ipv4_tcp_md5() # basic use case log_start run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Single address config" @@ -941,7 +943,7 @@ ipv4_tcp_md5() log_start show_hint "Should timeout since server does not have MD5 auth" run_cmd nettest -s -I ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Server no config, client uses password" @@ -949,7 +951,7 @@ ipv4_tcp_md5() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Client uses wrong password" @@ -957,7 +959,7 @@ ipv4_tcp_md5() log_start show_hint "Should timeout since server config differs from client" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NSB_LO_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Client address does not match address configured with password" @@ -968,7 +970,7 @@ ipv4_tcp_md5() # client in prefix log_start run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Prefix config" @@ -976,7 +978,7 @@ ipv4_tcp_md5() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Prefix config, client uses wrong password" @@ -984,7 +986,7 @@ ipv4_tcp_md5() log_start show_hint "Should timeout since client address is outside of prefix" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -c ${NSB_LO_IP} -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Prefix config, client address not in configured prefix" @@ -995,14 +997,14 @@ ipv4_tcp_md5() log_start run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Single address config in default VRF and VRF, conn in VRF" log_start run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 0 "MD5: VRF: Single address config in default VRF and VRF, conn in default VRF" @@ -1010,7 +1012,7 @@ ipv4_tcp_md5() show_hint "Should timeout since client in default VRF uses VRF password" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Single address config in default VRF and VRF, conn in default VRF with VRF pw" @@ -1018,21 +1020,21 @@ ipv4_tcp_md5() show_hint "Should timeout since client in VRF uses default VRF password" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NSB_IP} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Single address config in default VRF and VRF, conn in VRF with default VRF pw" log_start run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Prefix config in default VRF and VRF, conn in VRF" log_start run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 0 "MD5: VRF: Prefix config in default VRF and VRF, conn in default VRF" @@ -1040,7 +1042,7 @@ ipv4_tcp_md5() show_hint "Should timeout since client in default VRF uses VRF password" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Prefix config in default VRF and VRF, conn in default VRF with VRF pw" @@ -1048,7 +1050,7 @@ ipv4_tcp_md5() show_hint "Should timeout since client in VRF uses default VRF password" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} & run_cmd nettest -s -M ${MD5_WRONG_PW} -m ${NS_NET} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Prefix config in default VRF and VRF, conn in VRF with default VRF pw" @@ -1072,14 +1074,14 @@ test_ipv4_md5_vrf__vrf_server__no_bind_ifindex() log_start show_hint "Simulates applications using VRF without TCP_MD5SIG_FLAG_IFINDEX" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} --no-bind-key-ifindex & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: VRF-bound server, unbound key accepts connection" log_start show_hint "Binding both the socket and the key is not required but it works" run_cmd nettest -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET} --force-bind-key-ifindex & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: VRF-bound server, bound key accepts connection" } @@ -1093,25 +1095,25 @@ test_ipv4_md5_vrf__global_server__bind_ifindex0() log_start run_cmd nettest -s -M ${MD5_PW} -m ${NS_NET} --force-bind-key-ifindex & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Global server, Key bound to ifindex=0 rejects VRF connection" log_start run_cmd nettest -s -M ${MD5_PW} -m ${NS_NET} --force-bind-key-ifindex & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Global server, key bound to ifindex=0 accepts non-VRF connection" log_start run_cmd nettest -s -M ${MD5_PW} -m ${NS_NET} --no-bind-key-ifindex & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Global server, key not bound to ifindex accepts VRF connection" log_start run_cmd nettest -s -M ${MD5_PW} -m ${NS_NET} --no-bind-key-ifindex & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -r ${NSA_IP} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Global server, key not bound to ifindex accepts non-VRF connection" @@ -1183,7 +1185,7 @@ ipv4_tcp_novrf() do log_start run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 0 "Global server" done @@ -1191,7 +1193,7 @@ ipv4_tcp_novrf() a=${NSA_IP} log_start run_cmd nettest -s -I ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 0 "Device server" @@ -1211,13 +1213,13 @@ ipv4_tcp_novrf() do log_start run_cmd_nsb nettest -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -r ${a} -0 ${NSA_IP} log_test_addr ${a} $? 0 "Client" log_start run_cmd_nsb nettest -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 0 "Client, device bind" @@ -1239,7 +1241,7 @@ ipv4_tcp_novrf() do log_start run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -0 ${a} -1 ${a} log_test_addr ${a} $? 0 "Global server, local connection" done @@ -1247,7 +1249,7 @@ ipv4_tcp_novrf() a=${NSA_IP} log_start run_cmd nettest -s -I ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -0 ${a} log_test_addr ${a} $? 0 "Device server, unbound client, local connection" @@ -1256,7 +1258,7 @@ ipv4_tcp_novrf() log_start show_hint "Should fail 'Connection refused' since addresses on loopback are out of device scope" run_cmd nettest -s -I ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} log_test_addr ${a} $? 1 "Device server, unbound client, local connection" done @@ -1264,7 +1266,7 @@ ipv4_tcp_novrf() a=${NSA_IP} log_start run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -0 ${a} -d ${NSA_DEV} log_test_addr ${a} $? 0 "Global server, device client, local connection" @@ -1273,7 +1275,7 @@ ipv4_tcp_novrf() log_start show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope" run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 1 "Global server, device client, local connection" done @@ -1281,7 +1283,7 @@ ipv4_tcp_novrf() a=${NSA_IP} log_start run_cmd nettest -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -d ${NSA_DEV} -r ${a} -0 ${a} log_test_addr ${a} $? 0 "Device server, device client, local connection" @@ -1313,19 +1315,19 @@ ipv4_tcp_vrf() log_start show_hint "Should fail 'Connection refused' since global server with VRF is disabled" run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 1 "Global server" log_start run_cmd nettest -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 0 "VRF server" log_start run_cmd nettest -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 0 "Device server" @@ -1342,7 +1344,7 @@ ipv4_tcp_vrf() log_start show_hint "Should fail 'Connection refused' since global server with VRF is disabled" run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 1 "Global server, local connection" @@ -1364,14 +1366,14 @@ ipv4_tcp_vrf() log_start show_hint "client socket should be bound to VRF" run_cmd nettest -s -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 0 "Global server" log_start show_hint "client socket should be bound to VRF" run_cmd nettest -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 0 "VRF server" @@ -1386,7 +1388,7 @@ ipv4_tcp_vrf() log_start show_hint "client socket should be bound to device" run_cmd nettest -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 0 "Device server" @@ -1396,7 +1398,7 @@ ipv4_tcp_vrf() log_start show_hint "Should fail 'Connection refused' since client is not bound to VRF" run_cmd nettest -s -I ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} log_test_addr ${a} $? 1 "Global server, local connection" done @@ -1408,13 +1410,13 @@ ipv4_tcp_vrf() do log_start run_cmd_nsb nettest -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -r ${a} -d ${VRF} log_test_addr ${a} $? 0 "Client, VRF bind" log_start run_cmd_nsb nettest -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 0 "Client, device bind" @@ -1433,7 +1435,7 @@ ipv4_tcp_vrf() do log_start run_cmd nettest -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -d ${VRF} -0 ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local connection" done @@ -1441,26 +1443,26 @@ ipv4_tcp_vrf() a=${NSA_IP} log_start run_cmd nettest -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -d ${NSA_DEV} -0 ${a} log_test_addr ${a} $? 0 "VRF server, device client, local connection" log_start show_hint "Should fail 'No route to host' since client is out of VRF scope" run_cmd nettest -s -I ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} log_test_addr ${a} $? 1 "VRF server, unbound client, local connection" log_start run_cmd nettest -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -d ${VRF} -0 ${a} log_test_addr ${a} $? 0 "Device server, VRF client, local connection" log_start run_cmd nettest -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -r ${a} -d ${NSA_DEV} -0 ${a} log_test_addr ${a} $? 0 "Device server, device client, local connection" } @@ -1499,7 +1501,7 @@ ipv4_udp_novrf() do log_start run_cmd nettest -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 0 "Global server" @@ -1512,7 +1514,7 @@ ipv4_udp_novrf() a=${NSA_IP} log_start run_cmd nettest -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 0 "Device server" @@ -1523,31 +1525,31 @@ ipv4_udp_novrf() do log_start run_cmd_nsb nettest -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -D -r ${a} -0 ${NSA_IP} log_test_addr ${a} $? 0 "Client" log_start run_cmd_nsb nettest -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} -0 ${NSA_IP} log_test_addr ${a} $? 0 "Client, device bind" log_start run_cmd_nsb nettest -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} -C -0 ${NSA_IP} log_test_addr ${a} $? 0 "Client, device send via cmsg" log_start run_cmd_nsb nettest -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -0 ${NSA_IP} log_test_addr ${a} $? 0 "Client, device bind via IP_UNICAST_IF" log_start run_cmd_nsb nettest -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -0 ${NSA_IP} -U log_test_addr ${a} $? 0 "Client, device bind via IP_UNICAST_IF, with connect()" @@ -1570,7 +1572,7 @@ ipv4_udp_novrf() do log_start run_cmd nettest -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -r ${a} -0 ${a} -1 ${a} log_test_addr ${a} $? 0 "Global server, local connection" done @@ -1578,7 +1580,7 @@ ipv4_udp_novrf() a=${NSA_IP} log_start run_cmd nettest -s -D -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -r ${a} log_test_addr ${a} $? 0 "Device server, unbound client, local connection" @@ -1587,7 +1589,7 @@ ipv4_udp_novrf() log_start show_hint "Should fail 'Connection refused' since address is out of device scope" run_cmd nettest -s -D -I ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -r ${a} log_test_addr ${a} $? 1 "Device server, unbound client, local connection" done @@ -1595,25 +1597,25 @@ ipv4_udp_novrf() a=${NSA_IP} log_start run_cmd nettest -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Global server, device client, local connection" log_start run_cmd nettest -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -C -r ${a} log_test_addr ${a} $? 0 "Global server, device send via cmsg, local connection" log_start run_cmd nettest -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -S -r ${a} log_test_addr ${a} $? 0 "Global server, device client via IP_UNICAST_IF, local connection" log_start run_cmd nettest -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -S -r ${a} -U log_test_addr ${a} $? 0 "Global server, device client via IP_UNICAST_IF, local connection, with connect()" @@ -1626,28 +1628,28 @@ ipv4_udp_novrf() log_start show_hint "Should fail since addresses on loopback are out of device scope" run_cmd nettest -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 2 "Global server, device client, local connection" log_start show_hint "Should fail since addresses on loopback are out of device scope" run_cmd nettest -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} -C log_test_addr ${a} $? 1 "Global server, device send via cmsg, local connection" log_start show_hint "Should fail since addresses on loopback are out of device scope" run_cmd nettest -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection" log_start show_hint "Should fail since addresses on loopback are out of device scope" run_cmd nettest -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -U log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection, with connect()" @@ -1657,7 +1659,7 @@ ipv4_udp_novrf() a=${NSA_IP} log_start run_cmd nettest -D -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -r ${a} -0 ${a} log_test_addr ${a} $? 0 "Device server, device client, local conn" @@ -1699,19 +1701,19 @@ ipv4_udp_vrf() log_start show_hint "Fails because ingress is in a VRF and global server is disabled" run_cmd nettest -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 1 "Global server" log_start run_cmd nettest -D -I ${VRF} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 0 "VRF server" log_start run_cmd nettest -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 0 "Enslaved device server" @@ -1723,7 +1725,7 @@ ipv4_udp_vrf() log_start show_hint "Should fail 'Connection refused' since global server is out of scope" run_cmd nettest -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 1 "Global server, VRF client, local connection" done @@ -1731,26 +1733,26 @@ ipv4_udp_vrf() a=${NSA_IP} log_start run_cmd nettest -s -D -I ${VRF} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local conn" log_start run_cmd nettest -s -D -I ${VRF} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "VRF server, enslaved device client, local connection" a=${NSA_IP} log_start run_cmd nettest -s -D -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Enslaved device server, VRF client, local conn" log_start run_cmd nettest -s -D -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Enslaved device server, device client, local conn" @@ -1765,19 +1767,19 @@ ipv4_udp_vrf() do log_start run_cmd nettest -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 0 "Global server" log_start run_cmd nettest -D -I ${VRF} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 0 "VRF server" log_start run_cmd nettest -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -D -r ${a} log_test_addr ${a} $? 0 "Enslaved device server" @@ -1792,13 +1794,13 @@ ipv4_udp_vrf() # log_start run_cmd_nsb nettest -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -d ${VRF} -D -r ${NSB_IP} -1 ${NSA_IP} log_test $? 0 "VRF client" log_start run_cmd_nsb nettest -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -d ${NSA_DEV} -D -r ${NSB_IP} -1 ${NSA_IP} log_test $? 0 "Enslaved device client" @@ -1819,31 +1821,31 @@ ipv4_udp_vrf() a=${NSA_IP} log_start run_cmd nettest -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Global server, VRF client, local conn" log_start run_cmd nettest -s -D -I ${VRF} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local conn" log_start run_cmd nettest -s -D -I ${VRF} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "VRF server, device client, local conn" log_start run_cmd nettest -s -D -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Enslaved device server, VRF client, local conn" log_start run_cmd nettest -s -D -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Enslaved device server, device client, local conn" @@ -1851,7 +1853,7 @@ ipv4_udp_vrf() do log_start run_cmd nettest -D -s -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Global server, VRF client, local conn" done @@ -1860,7 +1862,7 @@ ipv4_udp_vrf() do log_start run_cmd nettest -s -D -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local conn" done @@ -2083,7 +2085,7 @@ ipv4_rt() do log_start run_cmd nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest ${varg} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2097,7 +2099,7 @@ ipv4_rt() do log_start run_cmd nettest ${varg} -s -I ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest ${varg} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2110,7 +2112,7 @@ ipv4_rt() a=${NSA_IP} log_start run_cmd nettest ${varg} -s -I ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest ${varg} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2124,7 +2126,7 @@ ipv4_rt() # log_start run_cmd_nsb nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest ${varg} -d ${VRF} -r ${NSB_IP} & sleep 3 run_cmd ip link del ${VRF} @@ -2135,7 +2137,7 @@ ipv4_rt() log_start run_cmd_nsb nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${NSB_IP} & sleep 3 run_cmd ip link del ${VRF} @@ -2151,7 +2153,7 @@ ipv4_rt() do log_start run_cmd nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${VRF} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2165,7 +2167,7 @@ ipv4_rt() do log_start run_cmd nettest ${varg} -I ${VRF} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${VRF} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2179,7 +2181,7 @@ ipv4_rt() log_start run_cmd nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2190,7 +2192,7 @@ ipv4_rt() log_start run_cmd nettest ${varg} -I ${VRF} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2201,7 +2203,7 @@ ipv4_rt() log_start run_cmd nettest ${varg} -I ${NSA_DEV} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -2551,7 +2553,7 @@ ipv6_tcp_md5_novrf() # basic use case log_start run_cmd nettest -6 -s -M ${MD5_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 0 "MD5: Single address config" @@ -2559,7 +2561,7 @@ ipv6_tcp_md5_novrf() log_start show_hint "Should timeout due to MD5 mismatch" run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: Server no config, client uses password" @@ -2567,7 +2569,7 @@ ipv6_tcp_md5_novrf() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -6 -s -M ${MD5_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: Client uses wrong password" @@ -2575,7 +2577,7 @@ ipv6_tcp_md5_novrf() log_start show_hint "Should timeout due to MD5 mismatch" run_cmd nettest -6 -s -M ${MD5_PW} -m ${NSB_LO_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: Client address does not match address configured with password" @@ -2586,7 +2588,7 @@ ipv6_tcp_md5_novrf() # client in prefix log_start run_cmd nettest -6 -s -M ${MD5_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 0 "MD5: Prefix config" @@ -2594,7 +2596,7 @@ ipv6_tcp_md5_novrf() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -6 -s -M ${MD5_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: Prefix config, client uses wrong password" @@ -2602,7 +2604,7 @@ ipv6_tcp_md5_novrf() log_start show_hint "Should timeout due to MD5 mismatch" run_cmd nettest -6 -s -M ${MD5_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -c ${NSB_LO_IP6} -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: Prefix config, client address not in configured prefix" } @@ -2619,7 +2621,7 @@ ipv6_tcp_md5() # basic use case log_start run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Single address config" @@ -2627,7 +2629,7 @@ ipv6_tcp_md5() log_start show_hint "Should timeout since server does not have MD5 auth" run_cmd nettest -6 -s -I ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Server no config, client uses password" @@ -2635,7 +2637,7 @@ ipv6_tcp_md5() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Client uses wrong password" @@ -2643,7 +2645,7 @@ ipv6_tcp_md5() log_start show_hint "Should timeout since server config differs from client" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NSB_LO_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Client address does not match address configured with password" @@ -2654,7 +2656,7 @@ ipv6_tcp_md5() # client in prefix log_start run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Prefix config" @@ -2662,7 +2664,7 @@ ipv6_tcp_md5() log_start show_hint "Should timeout since client uses wrong password" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Prefix config, client uses wrong password" @@ -2670,7 +2672,7 @@ ipv6_tcp_md5() log_start show_hint "Should timeout since client address is outside of prefix" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -c ${NSB_LO_IP6} -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Prefix config, client address not in configured prefix" @@ -2681,14 +2683,14 @@ ipv6_tcp_md5() log_start run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Single address config in default VRF and VRF, conn in VRF" log_start run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 0 "MD5: VRF: Single address config in default VRF and VRF, conn in default VRF" @@ -2696,7 +2698,7 @@ ipv6_tcp_md5() show_hint "Should timeout since client in default VRF uses VRF password" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Single address config in default VRF and VRF, conn in default VRF with VRF pw" @@ -2704,21 +2706,21 @@ ipv6_tcp_md5() show_hint "Should timeout since client in VRF uses default VRF password" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NSB_IP6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NSB_IP6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Single address config in default VRF and VRF, conn in VRF with default VRF pw" log_start run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 0 "MD5: VRF: Prefix config in default VRF and VRF, conn in VRF" log_start run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 0 "MD5: VRF: Prefix config in default VRF and VRF, conn in default VRF" @@ -2726,7 +2728,7 @@ ipv6_tcp_md5() show_hint "Should timeout since client in default VRF uses VRF password" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsc nettest -6 -r ${NSA_IP6} -X ${MD5_PW} log_test $? 2 "MD5: VRF: Prefix config in default VRF and VRF, conn in default VRF with VRF pw" @@ -2734,7 +2736,7 @@ ipv6_tcp_md5() show_hint "Should timeout since client in VRF uses default VRF password" run_cmd nettest -6 -s -I ${VRF} -M ${MD5_PW} -m ${NS_NET6} & run_cmd nettest -6 -s -M ${MD5_WRONG_PW} -m ${NS_NET6} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${NSA_IP6} -X ${MD5_WRONG_PW} log_test $? 2 "MD5: VRF: Prefix config in default VRF and VRF, conn in VRF with default VRF pw" @@ -2762,7 +2764,7 @@ ipv6_tcp_novrf() do log_start run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "Global server" done @@ -2783,7 +2785,7 @@ ipv6_tcp_novrf() do log_start run_cmd_nsb nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -6 -r ${a} log_test_addr ${a} $? 0 "Client" done @@ -2792,7 +2794,7 @@ ipv6_tcp_novrf() do log_start run_cmd_nsb nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -6 -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 0 "Client, device bind" done @@ -2812,7 +2814,7 @@ ipv6_tcp_novrf() do log_start run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} log_test_addr ${a} $? 0 "Global server, local connection" done @@ -2820,7 +2822,7 @@ ipv6_tcp_novrf() a=${NSA_IP6} log_start run_cmd nettest -6 -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -0 ${a} log_test_addr ${a} $? 0 "Device server, unbound client, local connection" @@ -2829,7 +2831,7 @@ ipv6_tcp_novrf() log_start show_hint "Should fail 'Connection refused' since addresses on loopback are out of device scope" run_cmd nettest -6 -s -I ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} log_test_addr ${a} $? 1 "Device server, unbound client, local connection" done @@ -2837,7 +2839,7 @@ ipv6_tcp_novrf() a=${NSA_IP6} log_start run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -d ${NSA_DEV} -0 ${a} log_test_addr ${a} $? 0 "Global server, device client, local connection" @@ -2846,7 +2848,7 @@ ipv6_tcp_novrf() log_start show_hint "Should fail 'Connection refused' since addresses on loopback are out of device scope" run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 1 "Global server, device client, local connection" done @@ -2855,7 +2857,7 @@ ipv6_tcp_novrf() do log_start run_cmd nettest -6 -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Device server, device client, local conn" done @@ -2888,7 +2890,7 @@ ipv6_tcp_vrf() log_start show_hint "Should fail 'Connection refused' since global server with VRF is disabled" run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 1 "Global server" done @@ -2897,7 +2899,7 @@ ipv6_tcp_vrf() do log_start run_cmd nettest -6 -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "VRF server" done @@ -2906,7 +2908,7 @@ ipv6_tcp_vrf() a=${NSA_LINKIP6}%${NSB_DEV} log_start run_cmd nettest -6 -s -I ${VRF} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "VRF server" @@ -2914,7 +2916,7 @@ ipv6_tcp_vrf() do log_start run_cmd nettest -6 -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "Device server" done @@ -2933,7 +2935,7 @@ ipv6_tcp_vrf() log_start show_hint "Should fail 'Connection refused' since global server with VRF is disabled" run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 1 "Global server, local connection" @@ -2954,7 +2956,7 @@ ipv6_tcp_vrf() do log_start run_cmd nettest -6 -s -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "Global server" done @@ -2963,7 +2965,7 @@ ipv6_tcp_vrf() do log_start run_cmd nettest -6 -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "VRF server" done @@ -2972,13 +2974,13 @@ ipv6_tcp_vrf() a=${NSA_LINKIP6}%${NSB_DEV} log_start run_cmd nettest -6 -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "Global server" log_start run_cmd nettest -6 -s -I ${VRF} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "VRF server" @@ -2986,7 +2988,7 @@ ipv6_tcp_vrf() do log_start run_cmd nettest -6 -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 0 "Device server" done @@ -3006,7 +3008,7 @@ ipv6_tcp_vrf() log_start show_hint "Fails 'Connection refused' since client is not in VRF" run_cmd nettest -6 -s -I ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} log_test_addr ${a} $? 1 "Global server, local connection" done @@ -3019,7 +3021,7 @@ ipv6_tcp_vrf() do log_start run_cmd_nsb nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -6 -r ${a} -d ${VRF} log_test_addr ${a} $? 0 "Client, VRF bind" done @@ -3028,7 +3030,7 @@ ipv6_tcp_vrf() log_start show_hint "Fails since VRF device does not allow linklocal addresses" run_cmd_nsb nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -6 -r ${a} -d ${VRF} log_test_addr ${a} $? 1 "Client, VRF bind" @@ -3036,7 +3038,7 @@ ipv6_tcp_vrf() do log_start run_cmd_nsb nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest -6 -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 0 "Client, device bind" done @@ -3061,7 +3063,7 @@ ipv6_tcp_vrf() do log_start run_cmd nettest -6 -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -d ${VRF} -0 ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local connection" done @@ -3069,7 +3071,7 @@ ipv6_tcp_vrf() a=${NSA_IP6} log_start run_cmd nettest -6 -s -I ${VRF} -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -d ${NSA_DEV} -0 ${a} log_test_addr ${a} $? 0 "VRF server, device client, local connection" @@ -3077,13 +3079,13 @@ ipv6_tcp_vrf() log_start show_hint "Should fail since unbound client is out of VRF scope" run_cmd nettest -6 -s -I ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} log_test_addr ${a} $? 1 "VRF server, unbound client, local connection" log_start run_cmd nettest -6 -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -d ${VRF} -0 ${a} log_test_addr ${a} $? 0 "Device server, VRF client, local connection" @@ -3091,7 +3093,7 @@ ipv6_tcp_vrf() do log_start run_cmd nettest -6 -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest -6 -r ${a} -d ${NSA_DEV} -0 ${a} log_test_addr ${a} $? 0 "Device server, device client, local connection" done @@ -3131,13 +3133,13 @@ ipv6_udp_novrf() do log_start run_cmd nettest -6 -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "Global server" log_start run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "Device server" done @@ -3145,7 +3147,7 @@ ipv6_udp_novrf() a=${NSA_LO_IP6} log_start run_cmd nettest -6 -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "Global server" @@ -3155,7 +3157,7 @@ ipv6_udp_novrf() #log_start #show_hint "Should fail since loopback address is out of scope" #run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - #sleep 1 + wait_local_port_listen ${NSA} 12345 udp #run_cmd_nsb nettest -6 -D -r ${a} #log_test_addr ${a} $? 1 "Device server" @@ -3175,25 +3177,25 @@ ipv6_udp_novrf() do log_start run_cmd_nsb nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -6 -D -r ${a} -0 ${NSA_IP6} log_test_addr ${a} $? 0 "Client" log_start run_cmd_nsb nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -0 ${NSA_IP6} log_test_addr ${a} $? 0 "Client, device bind" log_start run_cmd_nsb nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -C -0 ${NSA_IP6} log_test_addr ${a} $? 0 "Client, device send via cmsg" log_start run_cmd_nsb nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -S -0 ${NSA_IP6} log_test_addr ${a} $? 0 "Client, device bind via IPV6_UNICAST_IF" @@ -3215,7 +3217,7 @@ ipv6_udp_novrf() do log_start run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -r ${a} -0 ${a} -1 ${a} log_test_addr ${a} $? 0 "Global server, local connection" done @@ -3223,7 +3225,7 @@ ipv6_udp_novrf() a=${NSA_IP6} log_start run_cmd nettest -6 -s -D -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "Device server, unbound client, local connection" @@ -3232,7 +3234,7 @@ ipv6_udp_novrf() log_start show_hint "Should fail 'Connection refused' since address is out of device scope" run_cmd nettest -6 -s -D -I ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -r ${a} log_test_addr ${a} $? 1 "Device server, local connection" done @@ -3240,19 +3242,19 @@ ipv6_udp_novrf() a=${NSA_IP6} log_start run_cmd nettest -6 -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Global server, device client, local connection" log_start run_cmd nettest -6 -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -C -r ${a} log_test_addr ${a} $? 0 "Global server, device send via cmsg, local connection" log_start run_cmd nettest -6 -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -S -r ${a} log_test_addr ${a} $? 0 "Global server, device client via IPV6_UNICAST_IF, local connection" @@ -3261,28 +3263,28 @@ ipv6_udp_novrf() log_start show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope" run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} log_test_addr ${a} $? 1 "Global server, device client, local connection" log_start show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope" run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -C log_test_addr ${a} $? 1 "Global server, device send via cmsg, local connection" log_start show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope" run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -S log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection" log_start show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope" run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -S -U log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection, with connect()" done @@ -3290,7 +3292,7 @@ ipv6_udp_novrf() a=${NSA_IP6} log_start run_cmd nettest -6 -D -s -I ${NSA_DEV} -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} -0 ${a} log_test_addr ${a} $? 0 "Device server, device client, local conn" @@ -3304,7 +3306,7 @@ ipv6_udp_novrf() run_cmd_nsb ip -6 ro add ${NSA_IP6}/128 dev ${NSB_DEV} log_start run_cmd nettest -6 -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${NSA_IP6} log_test $? 0 "UDP in - LLA to GUA" @@ -3328,7 +3330,7 @@ ipv6_udp_vrf() log_start show_hint "Should fail 'Connection refused' since global server is disabled" run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 1 "Global server" done @@ -3337,7 +3339,7 @@ ipv6_udp_vrf() do log_start run_cmd nettest -6 -D -I ${VRF} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "VRF server" done @@ -3346,7 +3348,7 @@ ipv6_udp_vrf() do log_start run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "Enslaved device server" done @@ -3368,7 +3370,7 @@ ipv6_udp_vrf() log_start show_hint "Should fail 'Connection refused' since global server is disabled" run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 1 "Global server, VRF client, local conn" done @@ -3377,7 +3379,7 @@ ipv6_udp_vrf() do log_start run_cmd nettest -6 -D -I ${VRF} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local conn" done @@ -3386,25 +3388,25 @@ ipv6_udp_vrf() log_start show_hint "Should fail 'Connection refused' since global server is disabled" run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 1 "Global server, device client, local conn" log_start run_cmd nettest -6 -D -I ${VRF} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "VRF server, device client, local conn" log_start run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Enslaved device server, VRF client, local conn" log_start run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Enslaved device server, device client, local conn" @@ -3419,7 +3421,7 @@ ipv6_udp_vrf() do log_start run_cmd nettest -6 -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "Global server" done @@ -3428,7 +3430,7 @@ ipv6_udp_vrf() do log_start run_cmd nettest -6 -D -I ${VRF} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "VRF server" done @@ -3437,7 +3439,7 @@ ipv6_udp_vrf() do log_start run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${a} log_test_addr ${a} $? 0 "Enslaved device server" done @@ -3455,7 +3457,7 @@ ipv6_udp_vrf() # log_start run_cmd_nsb nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${NSB_IP6} log_test $? 0 "VRF client" @@ -3466,7 +3468,7 @@ ipv6_udp_vrf() log_start run_cmd_nsb nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${NSB_IP6} log_test $? 0 "Enslaved device client" @@ -3481,13 +3483,13 @@ ipv6_udp_vrf() a=${NSA_IP6} log_start run_cmd nettest -6 -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Global server, VRF client, local conn" #log_start run_cmd nettest -6 -D -I ${VRF} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local conn" @@ -3495,13 +3497,13 @@ ipv6_udp_vrf() a=${VRF_IP6} log_start run_cmd nettest -6 -D -s -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Global server, VRF client, local conn" log_start run_cmd nettest -6 -D -I ${VRF} -s -3 ${VRF} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "VRF server, VRF client, local conn" @@ -3517,25 +3519,25 @@ ipv6_udp_vrf() a=${NSA_IP6} log_start run_cmd nettest -6 -D -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Global server, device client, local conn" log_start run_cmd nettest -6 -D -I ${VRF} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "VRF server, device client, local conn" log_start run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${VRF} -r ${a} log_test_addr ${a} $? 0 "Device server, VRF client, local conn" log_start run_cmd nettest -6 -D -I ${NSA_DEV} -s -3 ${NSA_DEV} & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${a} log_test_addr ${a} $? 0 "Device server, device client, local conn" @@ -3547,7 +3549,7 @@ ipv6_udp_vrf() # link local addresses log_start run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -d ${NSB_DEV} -r ${NSA_LINKIP6} log_test $? 0 "Global server, linklocal IP" @@ -3558,7 +3560,7 @@ ipv6_udp_vrf() log_start run_cmd_nsb nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${NSB_LINKIP6} log_test $? 0 "Enslaved device client, linklocal IP" @@ -3569,7 +3571,7 @@ ipv6_udp_vrf() log_start run_cmd nettest -6 -D -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd nettest -6 -D -d ${NSA_DEV} -r ${NSA_LINKIP6} log_test $? 0 "Enslaved device client, local conn - linklocal IP" @@ -3582,7 +3584,7 @@ ipv6_udp_vrf() run_cmd_nsb ip -6 ro add ${NSA_IP6}/128 dev ${NSB_DEV} log_start run_cmd nettest -6 -s -D & - sleep 1 + wait_local_port_listen ${NSA} 12345 udp run_cmd_nsb nettest -6 -D -r ${NSA_IP6} log_test $? 0 "UDP in - LLA to GUA" @@ -3761,7 +3763,7 @@ ipv6_rt() do log_start run_cmd nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest ${varg} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3775,7 +3777,7 @@ ipv6_rt() do log_start run_cmd nettest ${varg} -I ${VRF} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest ${varg} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3789,7 +3791,7 @@ ipv6_rt() do log_start run_cmd nettest ${varg} -I ${NSA_DEV} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest ${varg} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3804,7 +3806,7 @@ ipv6_rt() # log_start run_cmd_nsb nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest ${varg} -d ${VRF} -r ${NSB_IP6} & sleep 3 run_cmd ip link del ${VRF} @@ -3815,7 +3817,7 @@ ipv6_rt() log_start run_cmd_nsb nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSB} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${NSB_IP6} & sleep 3 run_cmd ip link del ${VRF} @@ -3832,7 +3834,7 @@ ipv6_rt() do log_start run_cmd nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${VRF} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3846,7 +3848,7 @@ ipv6_rt() do log_start run_cmd nettest ${varg} -I ${VRF} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${VRF} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3859,7 +3861,7 @@ ipv6_rt() a=${NSA_IP6} log_start run_cmd nettest ${varg} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3870,7 +3872,7 @@ ipv6_rt() log_start run_cmd nettest ${varg} -I ${VRF} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3881,7 +3883,7 @@ ipv6_rt() log_start run_cmd nettest ${varg} -I ${NSA_DEV} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd nettest ${varg} -d ${NSA_DEV} -r ${a} & sleep 3 run_cmd ip link del ${VRF} @@ -3940,7 +3942,7 @@ netfilter_tcp_reset() do log_start run_cmd nettest -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -r ${a} log_test_addr ${a} $? 1 "Global server, reject with TCP-reset on Rx" done @@ -3958,7 +3960,7 @@ netfilter_icmp() do log_start run_cmd nettest ${arg} -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest ${arg} -r ${a} log_test_addr ${a} $? 1 "Global ${stype} server, Rx reject icmp-port-unreach" done @@ -3997,7 +3999,7 @@ netfilter_tcp6_reset() do log_start run_cmd nettest -6 -s & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 -r ${a} log_test_addr ${a} $? 1 "Global server, reject with TCP-reset on Rx" done @@ -4015,7 +4017,7 @@ netfilter_icmp6() do log_start run_cmd nettest -6 -s ${arg} & - sleep 1 + wait_local_port_listen ${NSA} 12345 tcp run_cmd_nsb nettest -6 ${arg} -r ${a} log_test_addr ${a} $? 1 "Global ${stype} server, Rx reject icmp-port-unreach" done @@ -4211,12 +4213,12 @@ use_case_snat_on_vrf() run_cmd ip6tables -t nat -A POSTROUTING -p tcp -m tcp --dport ${port} -j SNAT --to-source ${NSA_LO_IP6} -o ${VRF} run_cmd_nsb nettest -s -l ${NSB_IP} -p ${port} & - sleep 1 + wait_local_port_listen ${NSB} ${port} tcp run_cmd nettest -d ${VRF} -r ${NSB_IP} -p ${port} log_test $? 0 "IPv4 TCP connection over VRF with SNAT" run_cmd_nsb nettest -6 -s -l ${NSB_IP6} -p ${port} & - sleep 1 + wait_local_port_listen ${NSB} ${port} tcp run_cmd nettest -6 -d ${VRF} -r ${NSB_IP6} -p ${port} log_test $? 0 "IPv6 TCP connection over VRF with SNAT" diff --git a/tools/testing/selftests/net/fib_nexthops.sh b/tools/testing/selftests/net/fib_nexthops.sh index a6f2c0b9555d1..e2e4fffd87e39 100755 --- a/tools/testing/selftests/net/fib_nexthops.sh +++ b/tools/testing/selftests/net/fib_nexthops.sh @@ -464,8 +464,8 @@ ipv6_fdb_grp_fcnal() log_test $? 0 "Get Fdb nexthop group by id" # fdb nexthop group can only contain fdb nexthops - run_cmd "$IP nexthop add id 63 via 2001:db8:91::4" - run_cmd "$IP nexthop add id 64 via 2001:db8:91::5" + run_cmd "$IP nexthop add id 63 via 2001:db8:91::4 dev veth1" + run_cmd "$IP nexthop add id 64 via 2001:db8:91::5 dev veth1" run_cmd "$IP nexthop add id 103 group 63/64 fdb" log_test $? 2 "Fdb Nexthop group with non-fdb nexthops" @@ -544,15 +544,15 @@ ipv4_fdb_grp_fcnal() log_test $? 0 "Get Fdb nexthop group by id" # fdb nexthop group can only contain fdb nexthops - run_cmd "$IP nexthop add id 14 via 172.16.1.2" - run_cmd "$IP nexthop add id 15 via 172.16.1.3" + run_cmd "$IP nexthop add id 14 via 172.16.1.2 dev veth1" + run_cmd "$IP nexthop add id 15 via 172.16.1.3 dev veth1" run_cmd "$IP nexthop add id 103 group 14/15 fdb" log_test $? 2 "Fdb Nexthop group with non-fdb nexthops" # Non fdb nexthop group can not contain fdb nexthops run_cmd "$IP nexthop add id 16 via 172.16.1.2 fdb" run_cmd "$IP nexthop add id 17 via 172.16.1.3 fdb" - run_cmd "$IP nexthop add id 104 group 14/15" + run_cmd "$IP nexthop add id 104 group 16/17" log_test $? 2 "Non-Fdb Nexthop group with fdb nexthops" # fdb nexthop cannot have blackhole @@ -579,7 +579,7 @@ ipv4_fdb_grp_fcnal() run_cmd "$BRIDGE fdb add 02:02:00:00:00:14 dev vx10 nhid 12 self" log_test $? 255 "Fdb mac add with nexthop" - run_cmd "$IP ro add 172.16.0.0/22 nhid 15" + run_cmd "$IP ro add 172.16.0.0/22 nhid 16" log_test $? 2 "Route add with fdb nexthop" run_cmd "$IP ro add 172.16.0.0/22 nhid 103" diff --git a/tools/testing/selftests/net/forwarding/bridge_mdb.sh b/tools/testing/selftests/net/forwarding/bridge_mdb.sh index a3678dfe5848a..c151374ddf040 100755 --- a/tools/testing/selftests/net/forwarding/bridge_mdb.sh +++ b/tools/testing/selftests/net/forwarding/bridge_mdb.sh @@ -149,7 +149,7 @@ cfg_test_host_common() check_err $? "Failed to add $name host entry" bridge mdb replace dev br0 port br0 grp $grp $state vid 10 &> /dev/null - check_fail $? "Managed to replace $name host entry" + check_err $? "Failed to replace $name host entry" bridge mdb del dev br0 port br0 grp $grp $state vid 10 bridge mdb get dev br0 grp $grp vid 10 &> /dev/null diff --git a/tools/testing/selftests/net/forwarding/local_termination.sh b/tools/testing/selftests/net/forwarding/local_termination.sh index 9b5a63519b949..6cde61f10fd0e 100755 --- a/tools/testing/selftests/net/forwarding/local_termination.sh +++ b/tools/testing/selftests/net/forwarding/local_termination.sh @@ -108,6 +108,8 @@ run_test() local smac=$(mac_get $h1) local rcv_dmac=$(mac_get $rcv_if_name) + setup_wait + tcpdump_start $rcv_if_name mc_route_prepare $h1 diff --git a/tools/testing/selftests/net/gro.c b/tools/testing/selftests/net/gro.c index 30024d0ed3739..8dd6857e52cb5 100644 --- a/tools/testing/selftests/net/gro.c +++ b/tools/testing/selftests/net/gro.c @@ -71,6 +71,12 @@ #define MAX_PAYLOAD (IP_MAXPACKET - sizeof(struct tcphdr) - sizeof(struct ipv6hdr)) #define NUM_LARGE_PKT (MAX_PAYLOAD / MSS) #define MAX_HDR_LEN (ETH_HLEN + sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) +#define MIN_EXTHDR_SIZE 8 +#define EXT_PAYLOAD_1 "\x00\x00\x00\x00\x00\x00" +#define EXT_PAYLOAD_2 "\x11\x11\x11\x11\x11\x11" + +#define ipv6_optlen(p) (((p)->hdrlen+1) << 3) /* calculate IPv6 extension header len */ +#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) static const char *addr6_src = "fdaa::2"; static const char *addr6_dst = "fdaa::1"; @@ -104,7 +110,7 @@ static void setup_sock_filter(int fd) const int dport_off = tcp_offset + offsetof(struct tcphdr, dest); const int ethproto_off = offsetof(struct ethhdr, h_proto); int optlen = 0; - int ipproto_off; + int ipproto_off, opt_ipproto_off; int next_off; if (proto == PF_INET) @@ -116,14 +122,30 @@ static void setup_sock_filter(int fd) if (strcmp(testname, "ip") == 0) { if (proto == PF_INET) optlen = sizeof(struct ip_timestamp); - else - optlen = sizeof(struct ip6_frag); + else { + BUILD_BUG_ON(sizeof(struct ip6_hbh) > MIN_EXTHDR_SIZE); + BUILD_BUG_ON(sizeof(struct ip6_dest) > MIN_EXTHDR_SIZE); + BUILD_BUG_ON(sizeof(struct ip6_frag) > MIN_EXTHDR_SIZE); + + /* same size for HBH and Fragment extension header types */ + optlen = MIN_EXTHDR_SIZE; + opt_ipproto_off = ETH_HLEN + sizeof(struct ipv6hdr) + + offsetof(struct ip6_ext, ip6e_nxt); + } } + /* this filter validates the following: + * - packet is IPv4/IPv6 according to the running test. + * - packet is TCP. Also handles the case of one extension header and then TCP. + * - checks the packet tcp dport equals to DPORT. Also handles the case of one + * extension header and then TCP. + */ struct sock_filter filter[] = { BPF_STMT(BPF_LD + BPF_H + BPF_ABS, ethproto_off), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, ntohs(ethhdr_proto), 0, 7), + BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, ntohs(ethhdr_proto), 0, 9), BPF_STMT(BPF_LD + BPF_B + BPF_ABS, ipproto_off), + BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, IPPROTO_TCP, 2, 0), + BPF_STMT(BPF_LD + BPF_B + BPF_ABS, opt_ipproto_off), BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, IPPROTO_TCP, 0, 5), BPF_STMT(BPF_LD + BPF_H + BPF_ABS, dport_off), BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, DPORT, 2, 0), @@ -576,6 +598,39 @@ static void add_ipv4_ts_option(void *buf, void *optpkt) iph->check = checksum_fold(iph, sizeof(struct iphdr) + optlen, 0); } +static void add_ipv6_exthdr(void *buf, void *optpkt, __u8 exthdr_type, char *ext_payload) +{ + struct ipv6_opt_hdr *exthdr = (struct ipv6_opt_hdr *)(optpkt + tcp_offset); + struct ipv6hdr *iph = (struct ipv6hdr *)(optpkt + ETH_HLEN); + char *exthdr_payload_start = (char *)(exthdr + 1); + + exthdr->hdrlen = 0; + exthdr->nexthdr = IPPROTO_TCP; + + memcpy(exthdr_payload_start, ext_payload, MIN_EXTHDR_SIZE - sizeof(*exthdr)); + + memcpy(optpkt, buf, tcp_offset); + memcpy(optpkt + tcp_offset + MIN_EXTHDR_SIZE, buf + tcp_offset, + sizeof(struct tcphdr) + PAYLOAD_LEN); + + iph->nexthdr = exthdr_type; + iph->payload_len = htons(ntohs(iph->payload_len) + MIN_EXTHDR_SIZE); +} + +static void send_ipv6_exthdr(int fd, struct sockaddr_ll *daddr, char *ext_data1, char *ext_data2) +{ + static char buf[MAX_HDR_LEN + PAYLOAD_LEN]; + static char exthdr_pck[sizeof(buf) + MIN_EXTHDR_SIZE]; + + create_packet(buf, 0, 0, PAYLOAD_LEN, 0); + add_ipv6_exthdr(buf, exthdr_pck, IPPROTO_DSTOPTS, ext_data1); + write_packet(fd, exthdr_pck, total_hdr_len + PAYLOAD_LEN + MIN_EXTHDR_SIZE, daddr); + + create_packet(buf, PAYLOAD_LEN * 1, 0, PAYLOAD_LEN, 0); + add_ipv6_exthdr(buf, exthdr_pck, IPPROTO_DSTOPTS, ext_data2); + write_packet(fd, exthdr_pck, total_hdr_len + PAYLOAD_LEN + MIN_EXTHDR_SIZE, daddr); +} + /* IPv4 options shouldn't coalesce */ static void send_ip_options(int fd, struct sockaddr_ll *daddr) { @@ -697,7 +752,7 @@ static void send_fragment6(int fd, struct sockaddr_ll *daddr) create_packet(buf, PAYLOAD_LEN * i, 0, PAYLOAD_LEN, 0); write_packet(fd, buf, bufpkt_len, daddr); } - + sleep(1); create_packet(buf, PAYLOAD_LEN * 2, 0, PAYLOAD_LEN, 0); memset(extpkt, 0, extpkt_len); @@ -760,6 +815,7 @@ static void check_recv_pkts(int fd, int *correct_payload, vlog("}, Total %d packets\nReceived {", correct_num_pkts); while (1) { + ip_ext_len = 0; pkt_size = recv(fd, buffer, IP_MAXPACKET + ETH_HLEN + 1, 0); if (pkt_size < 0) error(1, errno, "could not receive"); @@ -767,7 +823,7 @@ static void check_recv_pkts(int fd, int *correct_payload, if (iph->version == 4) ip_ext_len = (iph->ihl - 5) * 4; else if (ip6h->version == 6 && ip6h->nexthdr != IPPROTO_TCP) - ip_ext_len = sizeof(struct ip6_frag); + ip_ext_len = MIN_EXTHDR_SIZE; tcph = (struct tcphdr *)(buffer + tcp_offset + ip_ext_len); @@ -802,6 +858,7 @@ static void check_recv_pkts(int fd, int *correct_payload, static void gro_sender(void) { + const int fin_delay_us = 100 * 1000; static char fin_pkt[MAX_HDR_LEN]; struct sockaddr_ll daddr = {}; int txfd = -1; @@ -845,15 +902,22 @@ static void gro_sender(void) write_packet(txfd, fin_pkt, total_hdr_len, &daddr); } else if (strcmp(testname, "tcp") == 0) { send_changed_checksum(txfd, &daddr); + /* Adding sleep before sending FIN so that it is not + * received prior to other packets. + */ + usleep(fin_delay_us); write_packet(txfd, fin_pkt, total_hdr_len, &daddr); send_changed_seq(txfd, &daddr); + usleep(fin_delay_us); write_packet(txfd, fin_pkt, total_hdr_len, &daddr); send_changed_ts(txfd, &daddr); + usleep(fin_delay_us); write_packet(txfd, fin_pkt, total_hdr_len, &daddr); send_diff_opt(txfd, &daddr); + usleep(fin_delay_us); write_packet(txfd, fin_pkt, total_hdr_len, &daddr); } else if (strcmp(testname, "ip") == 0) { send_changed_ECN(txfd, &daddr); @@ -880,7 +944,21 @@ static void gro_sender(void) sleep(1); write_packet(txfd, fin_pkt, total_hdr_len, &daddr); } else if (proto == PF_INET6) { + sleep(1); send_fragment6(txfd, &daddr); + sleep(1); + write_packet(txfd, fin_pkt, total_hdr_len, &daddr); + + sleep(1); + /* send IPv6 packets with ext header with same payload */ + send_ipv6_exthdr(txfd, &daddr, EXT_PAYLOAD_1, EXT_PAYLOAD_1); + sleep(1); + write_packet(txfd, fin_pkt, total_hdr_len, &daddr); + + sleep(1); + /* send IPv6 packets with ext header with different payload */ + send_ipv6_exthdr(txfd, &daddr, EXT_PAYLOAD_1, EXT_PAYLOAD_2); + sleep(1); write_packet(txfd, fin_pkt, total_hdr_len, &daddr); } } else if (strcmp(testname, "large") == 0) { @@ -997,6 +1075,17 @@ static void gro_receiver(void) */ printf("fragmented ip6 doesn't coalesce: "); correct_payload[0] = PAYLOAD_LEN * 2; + correct_payload[1] = PAYLOAD_LEN; + correct_payload[2] = PAYLOAD_LEN; + check_recv_pkts(rxfd, correct_payload, 3); + + printf("ipv6 with ext header does coalesce: "); + correct_payload[0] = PAYLOAD_LEN * 2; + check_recv_pkts(rxfd, correct_payload, 1); + + printf("ipv6 with ext header with different payloads doesn't coalesce: "); + correct_payload[0] = PAYLOAD_LEN; + correct_payload[1] = PAYLOAD_LEN; check_recv_pkts(rxfd, correct_payload, 2); } } else if (strcmp(testname, "large") == 0) { diff --git a/tools/testing/selftests/net/gro.sh b/tools/testing/selftests/net/gro.sh index 342ad27f631b1..e771f5f7faa26 100755 --- a/tools/testing/selftests/net/gro.sh +++ b/tools/testing/selftests/net/gro.sh @@ -95,5 +95,6 @@ trap cleanup EXIT if [[ "${test}" == "all" ]]; then run_all_tests else - run_test "${proto}" "${test}" + exit_code=$(run_test "${proto}" "${test}") + exit $exit_code fi; diff --git a/tools/testing/selftests/net/mptcp/Makefile b/tools/testing/selftests/net/mptcp/Makefile index 7b936a9268594..3c2fb9efb0b14 100644 --- a/tools/testing/selftests/net/mptcp/Makefile +++ b/tools/testing/selftests/net/mptcp/Makefile @@ -4,7 +4,8 @@ top_srcdir = ../../../../.. CFLAGS = -Wall -Wl,--no-as-needed -O2 -g -I$(top_srcdir)/usr/include $(KHDR_INCLUDES) -TEST_PROGS := mptcp_connect.sh pm_netlink.sh mptcp_join.sh diag.sh \ +TEST_PROGS := mptcp_connect.sh mptcp_connect_mmap.sh mptcp_connect_sendfile.sh \ + mptcp_connect_checksum.sh pm_netlink.sh mptcp_join.sh diag.sh \ simult_flows.sh mptcp_sockopt.sh userspace_pm.sh TEST_GEN_FILES = mptcp_connect pm_nl_ctl mptcp_sockopt mptcp_inq diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.c b/tools/testing/selftests/net/mptcp/mptcp_connect.c index c83a8b47bbdfa..20c29324b814f 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.c +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.c @@ -696,8 +696,14 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bw = do_rnd_write(peerfd, winfo->buf + winfo->off, winfo->len); if (bw < 0) { - if (cfg_rcv_trunc) - return 0; + /* expected reset, continue to read */ + if (cfg_rcv_trunc && + (errno == ECONNRESET || + errno == EPIPE)) { + fds.events &= ~POLLOUT; + continue; + } + perror("write"); return 111; } @@ -723,8 +729,10 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, } if (fds.revents & (POLLERR | POLLNVAL)) { - if (cfg_rcv_trunc) - return 0; + if (cfg_rcv_trunc) { + fds.events &= ~(POLLERR | POLLNVAL); + continue; + } fprintf(stderr, "Unexpected revents: " "POLLERR/POLLNVAL(%x)\n", fds.revents); return 5; @@ -1079,6 +1087,7 @@ int main_loop_s(int listensock) struct pollfd polls; socklen_t salen; int remotesock; + int err = 0; int fd = 0; again: @@ -1111,7 +1120,7 @@ int main_loop_s(int listensock) SOCK_TEST_TCPULP(remotesock, 0); memset(&winfo, 0, sizeof(winfo)); - copyfd_io(fd, remotesock, 1, true, &winfo); + err = copyfd_io(fd, remotesock, 1, true, &winfo); } else { perror("accept"); return 1; @@ -1120,10 +1129,10 @@ int main_loop_s(int listensock) if (cfg_input) close(fd); - if (--cfg_repeat > 0) + if (!err && --cfg_repeat > 0) goto again; - return 0; + return err; } static void init_rng(void) @@ -1233,7 +1242,7 @@ void xdisconnect(int fd) else xerror("bad family"); - strcpy(cmd, "ss -M | grep -q "); + strcpy(cmd, "ss -Mnt | grep -q "); cmdlen = strlen(cmd); if (!inet_ntop(addr.ss_family, raw_addr, &cmd[cmdlen], sizeof(cmd) - cmdlen)) @@ -1243,7 +1252,7 @@ void xdisconnect(int fd) /* * wait until the pending data is completely flushed and all - * the MPTCP sockets reached the closed status. + * the sockets reached the closed status. * disconnect will bypass/ignore/drop any pending data. */ for (i = 0; ; i += msec_sleep) { @@ -1418,7 +1427,7 @@ static void parse_opts(int argc, char **argv) */ if (cfg_truncate < 0) { cfg_rcv_trunc = true; - signal(SIGPIPE, handle_signal); + signal(SIGPIPE, SIG_IGN); } break; case 'j': diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh index 3763ffa214d53..f475b27185dec 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh @@ -516,7 +516,7 @@ do_transfer() "${stat_synrx_now_l}" "${expect_synrx}" 1>&2 retc=1 fi - if [ ${stat_ackrx_now_l} -lt ${expect_ackrx} ] && [ ${stat_ooo_now} -eq 0 ]; then + if [ ${stat_ackrx_now_l} -lt ${expect_ackrx} ]; then if [ ${stat_ooo_now} -eq 0 ]; then printf "[ FAIL ] lower MPC ACK rx (%d) than expected (%d)\n" \ "${stat_ackrx_now_l}" "${expect_ackrx}" 1>&2 diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect_checksum.sh b/tools/testing/selftests/net/mptcp/mptcp_connect_checksum.sh new file mode 100644 index 0000000000000..ce93ec2f107fb --- /dev/null +++ b/tools/testing/selftests/net/mptcp/mptcp_connect_checksum.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +MPTCP_LIB_KSFT_TEST="$(basename "${0}" .sh)" \ + "$(dirname "${0}")/mptcp_connect.sh" -C "${@}" diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect_mmap.sh b/tools/testing/selftests/net/mptcp/mptcp_connect_mmap.sh new file mode 100644 index 0000000000000..5dd30f9394af6 --- /dev/null +++ b/tools/testing/selftests/net/mptcp/mptcp_connect_mmap.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +MPTCP_LIB_KSFT_TEST="$(basename "${0}" .sh)" \ + "$(dirname "${0}")/mptcp_connect.sh" -m mmap "${@}" diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect_sendfile.sh b/tools/testing/selftests/net/mptcp/mptcp_connect_sendfile.sh new file mode 100644 index 0000000000000..1d16fb1cc9bb6 --- /dev/null +++ b/tools/testing/selftests/net/mptcp/mptcp_connect_sendfile.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +MPTCP_LIB_KSFT_TEST="$(basename "${0}" .sh)" \ + "$(dirname "${0}")/mptcp_connect.sh" -m sendfile "${@}" diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index 442b7220468af..ef16edce4759e 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -2361,7 +2361,7 @@ remove_tests() if reset "remove single subflow"; then pm_nl_set_limits $ns1 0 1 pm_nl_set_limits $ns2 0 1 - pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow + pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup addr_nr_ns2=-1 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 @@ -2374,8 +2374,8 @@ remove_tests() if reset "remove multiple subflows"; then pm_nl_set_limits $ns1 0 2 pm_nl_set_limits $ns2 0 2 - pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow - pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow + pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup + pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup addr_nr_ns2=-2 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 @@ -2386,7 +2386,7 @@ remove_tests() # single address, remove if reset "remove single address"; then pm_nl_set_limits $ns1 0 1 - pm_nl_add_endpoint $ns1 10.0.2.1 flags signal + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,backup pm_nl_set_limits $ns2 1 1 addr_nr_ns1=-1 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 @@ -2399,9 +2399,9 @@ remove_tests() # subflow and signal, remove if reset "remove subflow and signal"; then pm_nl_set_limits $ns1 0 2 - pm_nl_add_endpoint $ns1 10.0.2.1 flags signal + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,backup pm_nl_set_limits $ns2 1 2 - pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow + pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup addr_nr_ns1=-1 addr_nr_ns2=-1 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 2 2 2 @@ -2413,10 +2413,10 @@ remove_tests() # subflows and signal, remove if reset "remove subflows and signal"; then pm_nl_set_limits $ns1 0 3 - pm_nl_add_endpoint $ns1 10.0.2.1 flags signal + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,backup pm_nl_set_limits $ns2 1 3 - pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow + pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup + pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow,backup addr_nr_ns1=-1 addr_nr_ns2=-2 speed=10 \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 @@ -2428,9 +2428,9 @@ remove_tests() # addresses remove if reset "remove addresses"; then pm_nl_set_limits $ns1 3 3 - pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250 - pm_nl_add_endpoint $ns1 10.0.3.1 flags signal - pm_nl_add_endpoint $ns1 10.0.4.1 flags signal + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,backup id 250 + pm_nl_add_endpoint $ns1 10.0.3.1 flags signal,backup + pm_nl_add_endpoint $ns1 10.0.4.1 flags signal,backup pm_nl_set_limits $ns2 3 3 addr_nr_ns1=-3 speed=10 \ run_tests $ns1 $ns2 10.0.1.1 @@ -2443,10 +2443,10 @@ remove_tests() # invalid addresses remove if reset "remove invalid addresses"; then pm_nl_set_limits $ns1 3 3 - pm_nl_add_endpoint $ns1 10.0.12.1 flags signal + pm_nl_add_endpoint $ns1 10.0.12.1 flags signal,backup # broadcast IP: no packet for this address will be received on ns1 - pm_nl_add_endpoint $ns1 224.0.0.1 flags signal - pm_nl_add_endpoint $ns1 10.0.3.1 flags signal + pm_nl_add_endpoint $ns1 224.0.0.1 flags signal,backup + pm_nl_add_endpoint $ns1 10.0.3.1 flags signal,backup pm_nl_set_limits $ns2 2 2 addr_nr_ns1=-3 speed=10 \ run_tests $ns1 $ns2 10.0.1.1 @@ -2459,10 +2459,10 @@ remove_tests() # subflows and signal, flush if reset "flush subflows and signal"; then pm_nl_set_limits $ns1 0 3 - pm_nl_add_endpoint $ns1 10.0.2.1 flags signal + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,backup pm_nl_set_limits $ns2 1 3 - pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow + pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup + pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow,backup addr_nr_ns1=-8 addr_nr_ns2=-8 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 @@ -2475,9 +2475,9 @@ remove_tests() if reset "flush subflows"; then pm_nl_set_limits $ns1 3 3 pm_nl_set_limits $ns2 3 3 - pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow id 150 - pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow - pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow + pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup id 150 + pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup + pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow,backup addr_nr_ns1=-8 addr_nr_ns2=-8 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 3 3 3 @@ -2494,9 +2494,9 @@ remove_tests() # addresses flush if reset "flush addresses"; then pm_nl_set_limits $ns1 3 3 - pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250 - pm_nl_add_endpoint $ns1 10.0.3.1 flags signal - pm_nl_add_endpoint $ns1 10.0.4.1 flags signal + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,backup id 250 + pm_nl_add_endpoint $ns1 10.0.3.1 flags signal,backup + pm_nl_add_endpoint $ns1 10.0.4.1 flags signal,backup pm_nl_set_limits $ns2 3 3 addr_nr_ns1=-8 addr_nr_ns2=-8 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 @@ -2509,9 +2509,9 @@ remove_tests() # invalid addresses flush if reset "flush invalid addresses"; then pm_nl_set_limits $ns1 3 3 - pm_nl_add_endpoint $ns1 10.0.12.1 flags signal - pm_nl_add_endpoint $ns1 10.0.3.1 flags signal - pm_nl_add_endpoint $ns1 10.0.14.1 flags signal + pm_nl_add_endpoint $ns1 10.0.12.1 flags signal,backup + pm_nl_add_endpoint $ns1 10.0.3.1 flags signal,backup + pm_nl_add_endpoint $ns1 10.0.14.1 flags signal,backup pm_nl_set_limits $ns2 3 3 addr_nr_ns1=-8 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 @@ -3234,6 +3234,17 @@ deny_join_id0_tests() run_tests $ns1 $ns2 10.0.1.1 chk_join_nr 1 1 1 fi + + # default limits, server deny join id 0 + signal + if reset_with_allow_join_id0 "default limits, server deny join id 0" 0 1; then + pm_nl_set_limits $ns1 0 2 + pm_nl_set_limits $ns2 0 2 + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal + pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow + pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow + run_tests $ns1 $ns2 10.0.1.1 + chk_join_nr 2 2 2 + fi } fullmesh_tests() @@ -3726,11 +3737,11 @@ endpoint_tests() # subflow_rebuild_header is needed to support the implicit flag # userspace pm type prevents add_addr if reset "implicit EP" && - mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then + continue_if mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then pm_nl_set_limits $ns1 2 2 pm_nl_set_limits $ns2 2 2 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal - speed=slow \ + test_linkfail=128 speed=slow \ run_tests $ns1 $ns2 10.0.1.1 & local tests_pid=$! @@ -3751,13 +3762,13 @@ endpoint_tests() fi if reset_with_tcp_filter "delete and re-add" ns2 10.0.3.2 REJECT OUTPUT && - mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then + continue_if mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then start_events pm_nl_set_limits $ns1 0 3 pm_nl_set_limits $ns2 0 3 pm_nl_add_endpoint $ns2 10.0.1.2 id 1 dev ns2eth1 flags subflow pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow - test_linkfail=4 speed=5 \ + test_linkfail=128 speed=5 \ run_tests $ns1 $ns2 10.0.1.1 & local tests_pid=$! @@ -3826,14 +3837,15 @@ endpoint_tests() # remove and re-add if reset_with_events "delete re-add signal" && - mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then + continue_if mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then + ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=0 pm_nl_set_limits $ns1 0 3 pm_nl_set_limits $ns2 3 3 pm_nl_add_endpoint $ns1 10.0.2.1 id 1 flags signal # broadcast IP: no packet for this address will be received on ns1 pm_nl_add_endpoint $ns1 224.0.0.1 id 2 flags signal pm_nl_add_endpoint $ns1 10.0.1.1 id 42 flags signal - test_linkfail=4 speed=5 \ + test_linkfail=128 speed=5 \ run_tests $ns1 $ns2 10.0.1.1 & local tests_pid=$! @@ -3899,13 +3911,13 @@ endpoint_tests() # flush and re-add if reset_with_tcp_filter "flush re-add" ns2 10.0.3.2 REJECT OUTPUT && - mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then + continue_if mptcp_lib_kallsyms_has "subflow_rebuild_header$"; then pm_nl_set_limits $ns1 0 2 pm_nl_set_limits $ns2 1 2 # broadcast IP: no packet for this address will be received on ns1 pm_nl_add_endpoint $ns1 224.0.0.1 id 2 flags signal pm_nl_add_endpoint $ns2 10.0.3.2 id 3 flags subflow - test_linkfail=4 speed=20 \ + test_linkfail=128 speed=20 \ run_tests $ns1 $ns2 10.0.1.1 & local tests_pid=$! diff --git a/tools/testing/selftests/net/mptcp/mptcp_sockopt.c b/tools/testing/selftests/net/mptcp/mptcp_sockopt.c index 926b0be87c990..1dc2bd6ee4a50 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_sockopt.c +++ b/tools/testing/selftests/net/mptcp/mptcp_sockopt.c @@ -658,22 +658,26 @@ static void process_one_client(int fd, int pipefd) do_getsockopts(&s, fd, ret, ret2); if (s.mptcpi_rcv_delta != (uint64_t)ret + 1) - xerror("mptcpi_rcv_delta %" PRIu64 ", expect %" PRIu64, s.mptcpi_rcv_delta, ret + 1, s.mptcpi_rcv_delta - ret); + xerror("mptcpi_rcv_delta %" PRIu64 ", expect %" PRIu64 ", diff %" PRId64, + s.mptcpi_rcv_delta, ret + 1, s.mptcpi_rcv_delta - (ret + 1)); /* be nice when running on top of older kernel */ if (s.pkt_stats_avail) { if (s.last_sample.mptcpi_bytes_sent != ret2) - xerror("mptcpi_bytes_sent %" PRIu64 ", expect %" PRIu64, + xerror("mptcpi_bytes_sent %" PRIu64 ", expect %" PRIu64 + ", diff %" PRId64, s.last_sample.mptcpi_bytes_sent, ret2, s.last_sample.mptcpi_bytes_sent - ret2); if (s.last_sample.mptcpi_bytes_received != ret) - xerror("mptcpi_bytes_received %" PRIu64 ", expect %" PRIu64, + xerror("mptcpi_bytes_received %" PRIu64 ", expect %" PRIu64 + ", diff %" PRId64, s.last_sample.mptcpi_bytes_received, ret, s.last_sample.mptcpi_bytes_received - ret); if (s.last_sample.mptcpi_bytes_acked != ret) - xerror("mptcpi_bytes_acked %" PRIu64 ", expect %" PRIu64, - s.last_sample.mptcpi_bytes_acked, ret2, - s.last_sample.mptcpi_bytes_acked - ret2); + xerror("mptcpi_bytes_acked %" PRIu64 ", expect %" PRIu64 + ", diff %" PRId64, + s.last_sample.mptcpi_bytes_acked, ret, + s.last_sample.mptcpi_bytes_acked - ret); } close(fd); diff --git a/tools/testing/selftests/net/mptcp/pm_netlink.sh b/tools/testing/selftests/net/mptcp/pm_netlink.sh index 71899a3ffa7a9..3528e730e4d37 100755 --- a/tools/testing/selftests/net/mptcp/pm_netlink.sh +++ b/tools/testing/selftests/net/mptcp/pm_netlink.sh @@ -134,6 +134,7 @@ ip netns exec $ns1 ./pm_nl_ctl limits 1 9 2>/dev/null check "ip netns exec $ns1 ./pm_nl_ctl limits" "$default_limits" "subflows above hard limit" ip netns exec $ns1 ./pm_nl_ctl limits 8 8 +ip netns exec $ns1 ./pm_nl_ctl flush check "ip netns exec $ns1 ./pm_nl_ctl limits" "accept 8 subflows 8" "set limits" diff --git a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c index 763402dd17742..234c267dd2aad 100644 --- a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c +++ b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c @@ -194,6 +194,13 @@ static int capture_events(int fd, int event_group) fprintf(stderr, ",error:%u", *(__u8 *)RTA_DATA(attrs)); else if (attrs->rta_type == MPTCP_ATTR_SERVER_SIDE) fprintf(stderr, ",server_side:%u", *(__u8 *)RTA_DATA(attrs)); + else if (attrs->rta_type == MPTCP_ATTR_FLAGS) { + __u16 flags = *(__u16 *)RTA_DATA(attrs); + + /* only print when present, easier */ + if (flags & MPTCP_PM_EV_FLAG_DENY_JOIN_ID0) + fprintf(stderr, ",deny_join_id0:1"); + } attrs = RTA_NEXT(attrs, msg_len); } diff --git a/tools/testing/selftests/net/mptcp/userspace_pm.sh b/tools/testing/selftests/net/mptcp/userspace_pm.sh index c5d7af8e8efde..4e966a9e37383 100755 --- a/tools/testing/selftests/net/mptcp/userspace_pm.sh +++ b/tools/testing/selftests/net/mptcp/userspace_pm.sh @@ -196,6 +196,9 @@ make_connection() is_v6="v4" fi + # set this on the client side only: will not affect the rest + ip netns exec "$ns2" sysctl -q net.mptcp.allow_join_initial_addr_port=0 + # Capture netlink events over the two network namespaces running # the MPTCP client and server if [ -z "$client_evts" ]; then @@ -227,23 +230,28 @@ make_connection() local client_token local client_port local client_serverside + local client_nojoin local server_token local server_serverside + local server_nojoin client_token=$(mptcp_lib_evts_get_info token "$client_evts") client_port=$(mptcp_lib_evts_get_info sport "$client_evts") client_serverside=$(mptcp_lib_evts_get_info server_side "$client_evts") + client_nojoin=$(mptcp_lib_evts_get_info deny_join_id0 "$client_evts") server_token=$(mptcp_lib_evts_get_info token "$server_evts") server_serverside=$(mptcp_lib_evts_get_info server_side "$server_evts") + server_nojoin=$(mptcp_lib_evts_get_info deny_join_id0 "$server_evts") print_test "Established IP${is_v6} MPTCP Connection ns2 => ns1" - if [ "$client_token" != "" ] && [ "$server_token" != "" ] && [ "$client_serverside" = 0 ] && - [ "$server_serverside" = 1 ] + if [ "${client_token}" != "" ] && [ "${server_token}" != "" ] && + [ "${client_serverside}" = 0 ] && [ "${server_serverside}" = 1 ] && + [ "${client_nojoin:-0}" = 0 ] && [ "${server_nojoin:-0}" = 1 ] then test_pass print_title "Connection info: ${client_addr}:${client_port} -> ${connect_addr}:${app_port}" else - test_fail "Expected tokens (c:${client_token} - s:${server_token}) and server (c:${client_serverside} - s:${server_serverside})" + test_fail "Expected tokens (c:${client_token} - s:${server_token}), server (c:${client_serverside} - s:${server_serverside}), nojoin (c:${client_nojoin} - s:${server_nojoin})" mptcp_lib_result_print_all_tap exit 1 fi diff --git a/tools/testing/selftests/net/psock_tpacket.c b/tools/testing/selftests/net/psock_tpacket.c index e5a4c53076d2a..ca0d9a5a9e08c 100644 --- a/tools/testing/selftests/net/psock_tpacket.c +++ b/tools/testing/selftests/net/psock_tpacket.c @@ -22,6 +22,7 @@ * - TPACKET_V3: RX_RING */ +#undef NDEBUG #include #include #include @@ -33,9 +34,6 @@ #include #include #include -#ifndef __ANDROID__ -#include -#endif #include #include #include @@ -787,7 +785,7 @@ static int test_kernel_bit_width(void) static int test_user_bit_width(void) { - return __WORDSIZE; + return sizeof(long) * 8; } static const char *tpacket_str[] = { diff --git a/tools/testing/selftests/net/rtnetlink.sh b/tools/testing/selftests/net/rtnetlink.sh index 855505c40ed8e..c9ba4d269c382 100755 --- a/tools/testing/selftests/net/rtnetlink.sh +++ b/tools/testing/selftests/net/rtnetlink.sh @@ -854,6 +854,11 @@ kci_test_ipsec_offload() sysfsf=$sysfsd/ipsec sysfsnet=/sys/bus/netdevsim/devices/netdevsim0/net/ probed=false + esp4_offload_probed_default=false + + if lsmod | grep -q esp4_offload; then + esp4_offload_probed_default=true + fi # setup netdevsim since dummydev doesn't have offload support if [ ! -w /sys/bus/netdevsim/new_device ] ; then @@ -943,6 +948,7 @@ EOF fi # clean up any leftovers + ! "$esp4_offload_probed_default" && lsmod | grep -q esp4_offload && rmmod esp4_offload echo 0 > /sys/bus/netdevsim/del_device $probed && rmmod netdevsim diff --git a/tools/testing/selftests/net/sctp_hello.c b/tools/testing/selftests/net/sctp_hello.c index f02f1f95d2275..a04dac0b8027d 100644 --- a/tools/testing/selftests/net/sctp_hello.c +++ b/tools/testing/selftests/net/sctp_hello.c @@ -29,7 +29,6 @@ static void set_addr(struct sockaddr_storage *ss, char *ip, char *port, int *len static int do_client(int argc, char *argv[]) { struct sockaddr_storage ss; - char buf[] = "hello"; int csk, ret, len; if (argc < 5) { @@ -56,16 +55,10 @@ static int do_client(int argc, char *argv[]) set_addr(&ss, argv[3], argv[4], &len); ret = connect(csk, (struct sockaddr *)&ss, len); - if (ret < 0) { - printf("failed to connect to peer\n"); + if (ret < 0) return -1; - } - ret = send(csk, buf, strlen(buf) + 1, 0); - if (ret < 0) { - printf("failed to send msg %d\n", ret); - return -1; - } + recv(csk, NULL, 0, 0); close(csk); return 0; @@ -75,7 +68,6 @@ int main(int argc, char *argv[]) { struct sockaddr_storage ss; int lsk, csk, ret, len; - char buf[20]; if (argc < 2 || (strcmp(argv[1], "server") && strcmp(argv[1], "client"))) { printf("%s server|client ...\n", argv[0]); @@ -125,11 +117,6 @@ int main(int argc, char *argv[]) return -1; } - ret = recv(csk, buf, sizeof(buf), 0); - if (ret <= 0) { - printf("failed to recv msg %d\n", ret); - return -1; - } close(csk); close(lsk); diff --git a/tools/testing/selftests/net/sctp_vrf.sh b/tools/testing/selftests/net/sctp_vrf.sh index c721e952e5f30..667b211aa8a11 100755 --- a/tools/testing/selftests/net/sctp_vrf.sh +++ b/tools/testing/selftests/net/sctp_vrf.sh @@ -6,13 +6,11 @@ # SERVER_NS # CLIENT_NS2 (veth1) <---> (veth2) -> vrf_s2 -CLIENT_NS1="client-ns1" -CLIENT_NS2="client-ns2" +source lib.sh CLIENT_IP4="10.0.0.1" CLIENT_IP6="2000::1" CLIENT_PORT=1234 -SERVER_NS="server-ns" SERVER_IP4="10.0.0.2" SERVER_IP6="2000::2" SERVER_PORT=1234 @@ -20,13 +18,11 @@ SERVER_PORT=1234 setup() { modprobe sctp modprobe sctp_diag - ip netns add $CLIENT_NS1 - ip netns add $CLIENT_NS2 - ip netns add $SERVER_NS + setup_ns CLIENT_NS1 CLIENT_NS2 SERVER_NS - ip net exec $CLIENT_NS1 sysctl -w net.ipv6.conf.default.accept_dad=0 2>&1 >/dev/null - ip net exec $CLIENT_NS2 sysctl -w net.ipv6.conf.default.accept_dad=0 2>&1 >/dev/null - ip net exec $SERVER_NS sysctl -w net.ipv6.conf.default.accept_dad=0 2>&1 >/dev/null + ip net exec $CLIENT_NS1 sysctl -wq net.ipv6.conf.default.accept_dad=0 + ip net exec $CLIENT_NS2 sysctl -wq net.ipv6.conf.default.accept_dad=0 + ip net exec $SERVER_NS sysctl -wq net.ipv6.conf.default.accept_dad=0 ip -n $SERVER_NS link add veth1 type veth peer name veth1 netns $CLIENT_NS1 ip -n $SERVER_NS link add veth2 type veth peer name veth1 netns $CLIENT_NS2 @@ -66,19 +62,40 @@ setup() { } cleanup() { - ip netns exec $SERVER_NS pkill sctp_hello 2>&1 >/dev/null - ip netns del "$CLIENT_NS1" - ip netns del "$CLIENT_NS2" - ip netns del "$SERVER_NS" + wait_client $CLIENT_NS1 + wait_client $CLIENT_NS2 + stop_server + cleanup_ns $CLIENT_NS1 $CLIENT_NS2 $SERVER_NS } -wait_server() { +start_server() { local IFACE=$1 local CNT=0 - until ip netns exec $SERVER_NS ss -lS src $SERVER_IP:$SERVER_PORT | \ - grep LISTEN | grep "$IFACE" 2>&1 >/dev/null; do - [ $((CNT++)) = "20" ] && { RET=3; return $RET; } + ip netns exec $SERVER_NS ./sctp_hello server $AF $SERVER_IP $SERVER_PORT $IFACE & + disown + until ip netns exec $SERVER_NS ss -SlH | grep -q "$IFACE"; do + [ $((CNT++)) -eq 30 ] && { RET=3; return $RET; } + sleep 0.1 + done +} + +stop_server() { + local CNT=0 + + ip netns exec $SERVER_NS pkill sctp_hello + while ip netns exec $SERVER_NS ss -SaH | grep -q .; do + [ $((CNT++)) -eq 30 ] && break + sleep 0.1 + done +} + +wait_client() { + local CLIENT_NS=$1 + local CNT=0 + + while ip netns exec $CLIENT_NS ss -SaH | grep -q .; do + [ $((CNT++)) -eq 30 ] && break sleep 0.1 done } @@ -87,14 +104,12 @@ do_test() { local CLIENT_NS=$1 local IFACE=$2 - ip netns exec $SERVER_NS pkill sctp_hello 2>&1 >/dev/null - ip netns exec $SERVER_NS ./sctp_hello server $AF $SERVER_IP \ - $SERVER_PORT $IFACE 2>&1 >/dev/null & - disown - wait_server $IFACE || return $RET + start_server $IFACE || return $RET timeout 3 ip netns exec $CLIENT_NS ./sctp_hello client $AF \ - $SERVER_IP $SERVER_PORT $CLIENT_IP $CLIENT_PORT 2>&1 >/dev/null + $SERVER_IP $SERVER_PORT $CLIENT_IP $CLIENT_PORT RET=$? + wait_client $CLIENT_NS + stop_server return $RET } @@ -102,25 +117,21 @@ do_testx() { local IFACE1=$1 local IFACE2=$2 - ip netns exec $SERVER_NS pkill sctp_hello 2>&1 >/dev/null - ip netns exec $SERVER_NS ./sctp_hello server $AF $SERVER_IP \ - $SERVER_PORT $IFACE1 2>&1 >/dev/null & - disown - wait_server $IFACE1 || return $RET - ip netns exec $SERVER_NS ./sctp_hello server $AF $SERVER_IP \ - $SERVER_PORT $IFACE2 2>&1 >/dev/null & - disown - wait_server $IFACE2 || return $RET + start_server $IFACE1 || return $RET + start_server $IFACE2 || return $RET timeout 3 ip netns exec $CLIENT_NS1 ./sctp_hello client $AF \ - $SERVER_IP $SERVER_PORT $CLIENT_IP $CLIENT_PORT 2>&1 >/dev/null && \ + $SERVER_IP $SERVER_PORT $CLIENT_IP $CLIENT_PORT && \ timeout 3 ip netns exec $CLIENT_NS2 ./sctp_hello client $AF \ - $SERVER_IP $SERVER_PORT $CLIENT_IP $CLIENT_PORT 2>&1 >/dev/null + $SERVER_IP $SERVER_PORT $CLIENT_IP $CLIENT_PORT RET=$? + wait_client $CLIENT_NS1 + wait_client $CLIENT_NS2 + stop_server return $RET } testup() { - ip netns exec $SERVER_NS sysctl -w net.sctp.l3mdev_accept=1 2>&1 >/dev/null + ip netns exec $SERVER_NS sysctl -wq net.sctp.l3mdev_accept=1 echo -n "TEST 01: nobind, connect from client 1, l3mdev_accept=1, Y " do_test $CLIENT_NS1 || { echo "[FAIL]"; return $RET; } echo "[PASS]" @@ -129,7 +140,7 @@ testup() { do_test $CLIENT_NS2 && { echo "[FAIL]"; return $RET; } echo "[PASS]" - ip netns exec $SERVER_NS sysctl -w net.sctp.l3mdev_accept=0 2>&1 >/dev/null + ip netns exec $SERVER_NS sysctl -wq net.sctp.l3mdev_accept=0 echo -n "TEST 03: nobind, connect from client 1, l3mdev_accept=0, N " do_test $CLIENT_NS1 && { echo "[FAIL]"; return $RET; } echo "[PASS]" @@ -166,7 +177,7 @@ testup() { do_testx vrf-1 vrf-2 || { echo "[FAIL]"; return $RET; } echo "[PASS]" - echo -n "TEST 12: bind vrf-2 & 1 in server, connect from client 1 & 2, N " + echo -n "TEST 12: bind vrf-2 & 1 in server, connect from client 1 & 2, Y " do_testx vrf-2 vrf-1 || { echo "[FAIL]"; return $RET; } echo "[PASS]" } diff --git a/tools/testing/selftests/net/traceroute.sh b/tools/testing/selftests/net/traceroute.sh index de9ca97abc306..9cb5e96e64333 100755 --- a/tools/testing/selftests/net/traceroute.sh +++ b/tools/testing/selftests/net/traceroute.sh @@ -209,11 +209,6 @@ setup_traceroute6() run_traceroute6() { - if [ ! -x "$(command -v traceroute6)" ]; then - echo "SKIP: Could not run IPV6 test without traceroute6" - return - fi - setup_traceroute6 # traceroute6 host-2 from host-1 (expects 2000:102::2) @@ -278,11 +273,6 @@ setup_traceroute() run_traceroute() { - if [ ! -x "$(command -v traceroute)" ]; then - echo "SKIP: Could not run IPV4 test without traceroute" - return - fi - setup_traceroute # traceroute host-2 from host-1 (expects 1.0.1.1). Takes a while. @@ -316,6 +306,9 @@ do esac done +require_command traceroute6 +require_command traceroute + run_tests printf "\nTests passed: %3d\n" ${nsuccess} diff --git a/tools/testing/selftests/net/udpgro.sh b/tools/testing/selftests/net/udpgro.sh index 53341c8135e88..b65cf09f9914e 100755 --- a/tools/testing/selftests/net/udpgro.sh +++ b/tools/testing/selftests/net/udpgro.sh @@ -50,7 +50,7 @@ run_one() { cfg_veth - ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 10 ${rx_args} & + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 100 ${rx_args} & local PID1=$! wait_local_port_listen ${PEER_NS} 8000 udp @@ -97,7 +97,7 @@ run_one_nat() { # will land on the 'plain' one ip netns exec "${PEER_NS}" ./udpgso_bench_rx -G ${family} -b ${addr1} -n 0 & local PID1=$! - ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 10 ${family} -b ${addr2%/*} ${rx_args} & + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 100 ${family} -b ${addr2%/*} ${rx_args} & local PID2=$! wait_local_port_listen "${PEER_NS}" 8000 udp @@ -119,9 +119,9 @@ run_one_2sock() { cfg_veth - ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 10 ${rx_args} -p 12345 & + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 100 ${rx_args} -p 12345 & local PID1=$! - ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 2000 -R 10 ${rx_args} & + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 2000 -R 100 ${rx_args} & local PID2=$! wait_local_port_listen "${PEER_NS}" 12345 udp diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index 4aaafbfc2f973..a019de995e5df 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -176,8 +176,8 @@ int expect_zr(int expr, int llen) } -#define EXPECT_NZ(cond, expr, val) \ - do { if (!(cond)) result(llen, SKIPPED); else ret += expect_nz(expr, llen; } while (0) +#define EXPECT_NZ(cond, expr) \ + do { if (!(cond)) result(llen, SKIPPED); else ret += expect_nz(expr, llen); } while (0) static __attribute__((unused)) int expect_nz(int expr, int llen) diff --git a/tools/testing/selftests/perf_events/.gitignore b/tools/testing/selftests/perf_events/.gitignore index 790c47001e77e..4858977dd55b5 100644 --- a/tools/testing/selftests/perf_events/.gitignore +++ b/tools/testing/selftests/perf_events/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only sigtrap_threads remove_on_exec +mmap diff --git a/tools/testing/selftests/perf_events/Makefile b/tools/testing/selftests/perf_events/Makefile index db93c4ff081a4..913854914ae49 100644 --- a/tools/testing/selftests/perf_events/Makefile +++ b/tools/testing/selftests/perf_events/Makefile @@ -2,5 +2,5 @@ CFLAGS += -Wl,-no-as-needed -Wall $(KHDR_INCLUDES) LDFLAGS += -lpthread -TEST_GEN_PROGS := sigtrap_threads remove_on_exec +TEST_GEN_PROGS := sigtrap_threads remove_on_exec mmap include ../lib.mk diff --git a/tools/testing/selftests/perf_events/mmap.c b/tools/testing/selftests/perf_events/mmap.c new file mode 100644 index 0000000000000..ea0427aac1f98 --- /dev/null +++ b/tools/testing/selftests/perf_events/mmap.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "../kselftest_harness.h" + +#define RB_SIZE 0x3000 +#define AUX_SIZE 0x10000 +#define AUX_OFFS 0x4000 + +#define HOLE_SIZE 0x1000 + +/* Reserve space for rb, aux with space for shrink-beyond-vma testing. */ +#define REGION_SIZE (2 * RB_SIZE + 2 * AUX_SIZE) +#define REGION_AUX_OFFS (2 * RB_SIZE) + +#define MAP_BASE 1 +#define MAP_AUX 2 + +#define EVENT_SRC_DIR "/sys/bus/event_source/devices" + +FIXTURE(perf_mmap) +{ + int fd; + void *ptr; + void *region; +}; + +FIXTURE_VARIANT(perf_mmap) +{ + bool aux; + unsigned long ptr_size; +}; + +FIXTURE_VARIANT_ADD(perf_mmap, rb) +{ + .aux = false, + .ptr_size = RB_SIZE, +}; + +FIXTURE_VARIANT_ADD(perf_mmap, aux) +{ + .aux = true, + .ptr_size = AUX_SIZE, +}; + +static bool read_event_type(struct dirent *dent, __u32 *type) +{ + char typefn[512]; + FILE *fp; + int res; + + snprintf(typefn, sizeof(typefn), "%s/%s/type", EVENT_SRC_DIR, dent->d_name); + fp = fopen(typefn, "r"); + if (!fp) + return false; + + res = fscanf(fp, "%u", type); + fclose(fp); + return res > 0; +} + +FIXTURE_SETUP(perf_mmap) +{ + struct perf_event_attr attr = { + .size = sizeof(attr), + .disabled = 1, + .exclude_kernel = 1, + .exclude_hv = 1, + }; + struct perf_event_attr attr_ok = {}; + unsigned int eacces = 0, map = 0; + struct perf_event_mmap_page *rb; + struct dirent *dent; + void *aux, *region; + DIR *dir; + + self->ptr = NULL; + + dir = opendir(EVENT_SRC_DIR); + if (!dir) + SKIP(return, "perf not available."); + + region = mmap(NULL, REGION_SIZE, PROT_NONE, MAP_ANON | MAP_PRIVATE, -1, 0); + ASSERT_NE(region, MAP_FAILED); + self->region = region; + + // Try to find a suitable event on this system + while ((dent = readdir(dir))) { + int fd; + + if (!read_event_type(dent, &attr.type)) + continue; + + fd = syscall(SYS_perf_event_open, &attr, 0, -1, -1, 0); + if (fd < 0) { + if (errno == EACCES) + eacces++; + continue; + } + + // Check whether the event supports mmap() + rb = mmap(region, RB_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0); + if (rb == MAP_FAILED) { + close(fd); + continue; + } + + if (!map) { + // Save the event in case that no AUX capable event is found + attr_ok = attr; + map = MAP_BASE; + } + + if (!variant->aux) + continue; + + rb->aux_offset = AUX_OFFS; + rb->aux_size = AUX_SIZE; + + // Check whether it supports a AUX buffer + aux = mmap(region + REGION_AUX_OFFS, AUX_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, AUX_OFFS); + if (aux == MAP_FAILED) { + munmap(rb, RB_SIZE); + close(fd); + continue; + } + + attr_ok = attr; + map = MAP_AUX; + munmap(aux, AUX_SIZE); + munmap(rb, RB_SIZE); + close(fd); + break; + } + closedir(dir); + + if (!map) { + if (!eacces) + SKIP(return, "No mappable perf event found."); + else + SKIP(return, "No permissions for perf_event_open()"); + } + + self->fd = syscall(SYS_perf_event_open, &attr_ok, 0, -1, -1, 0); + ASSERT_NE(self->fd, -1); + + rb = mmap(region, RB_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, self->fd, 0); + ASSERT_NE(rb, MAP_FAILED); + + if (!variant->aux) { + self->ptr = rb; + return; + } + + if (map != MAP_AUX) + SKIP(return, "No AUX event found."); + + rb->aux_offset = AUX_OFFS; + rb->aux_size = AUX_SIZE; + aux = mmap(region + REGION_AUX_OFFS, AUX_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, self->fd, AUX_OFFS); + ASSERT_NE(aux, MAP_FAILED); + self->ptr = aux; +} + +FIXTURE_TEARDOWN(perf_mmap) +{ + ASSERT_EQ(munmap(self->region, REGION_SIZE), 0); + if (self->fd != -1) + ASSERT_EQ(close(self->fd), 0); +} + +TEST_F(perf_mmap, remap) +{ + void *tmp, *ptr = self->ptr; + unsigned long size = variant->ptr_size; + + // Test the invalid remaps + ASSERT_EQ(mremap(ptr, size, HOLE_SIZE, MREMAP_MAYMOVE), MAP_FAILED); + ASSERT_EQ(mremap(ptr + HOLE_SIZE, size, HOLE_SIZE, MREMAP_MAYMOVE), MAP_FAILED); + ASSERT_EQ(mremap(ptr + size - HOLE_SIZE, HOLE_SIZE, size, MREMAP_MAYMOVE), MAP_FAILED); + // Shrink the end of the mapping such that we only unmap past end of the VMA, + // which should succeed and poke a hole into the PROT_NONE region + ASSERT_NE(mremap(ptr + size - HOLE_SIZE, size, HOLE_SIZE, MREMAP_MAYMOVE), MAP_FAILED); + + // Remap the whole buffer to a new address + tmp = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(tmp, MAP_FAILED); + + // Try splitting offset 1 hole size into VMA, this should fail + ASSERT_EQ(mremap(ptr + HOLE_SIZE, size - HOLE_SIZE, size - HOLE_SIZE, + MREMAP_MAYMOVE | MREMAP_FIXED, tmp), MAP_FAILED); + // Remapping the whole thing should succeed fine + ptr = mremap(ptr, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, tmp); + ASSERT_EQ(ptr, tmp); + ASSERT_EQ(munmap(tmp, size), 0); +} + +TEST_F(perf_mmap, unmap) +{ + unsigned long size = variant->ptr_size; + + // Try to poke holes into the mappings + ASSERT_NE(munmap(self->ptr, HOLE_SIZE), 0); + ASSERT_NE(munmap(self->ptr + HOLE_SIZE, HOLE_SIZE), 0); + ASSERT_NE(munmap(self->ptr + size - HOLE_SIZE, HOLE_SIZE), 0); +} + +TEST_F(perf_mmap, map) +{ + unsigned long size = variant->ptr_size; + + // Try to poke holes into the mappings by mapping anonymous memory over it + ASSERT_EQ(mmap(self->ptr, HOLE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0), MAP_FAILED); + ASSERT_EQ(mmap(self->ptr + HOLE_SIZE, HOLE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0), MAP_FAILED); + ASSERT_EQ(mmap(self->ptr + size - HOLE_SIZE, HOLE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0), MAP_FAILED); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c index f6156790c3b4d..05dc77fd527b3 100644 --- a/tools/testing/selftests/rseq/rseq.c +++ b/tools/testing/selftests/rseq/rseq.c @@ -40,9 +40,9 @@ * Define weak versions to play nice with binaries that are statically linked * against a libc that doesn't support registering its own rseq. */ -__weak ptrdiff_t __rseq_offset; -__weak unsigned int __rseq_size; -__weak unsigned int __rseq_flags; +extern __weak ptrdiff_t __rseq_offset; +extern __weak unsigned int __rseq_size; +extern __weak unsigned int __rseq_flags; static const ptrdiff_t *libc_rseq_offset_p = &__rseq_offset; static const unsigned int *libc_rseq_size_p = &__rseq_size; @@ -198,7 +198,7 @@ void rseq_init(void) * libc not having registered a restartable sequence. Try to find the * symbols if that's the case. */ - if (!*libc_rseq_size_p) { + if (!libc_rseq_size_p || !*libc_rseq_size_p) { libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset"); libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size"); libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags"); diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index cacf6507f6905..4390965c84c42 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -60,7 +60,9 @@ #define SKIP(s, ...) XFAIL(s, ##__VA_ARGS__) #endif +#ifndef MIN #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#endif #ifndef PR_SET_PTRACER # define PR_SET_PTRACER 0x59616d61 @@ -3154,12 +3156,15 @@ TEST(syscall_restart) ret = get_syscall(_metadata, child_pid); #if defined(__arm__) /* - * FIXME: * - native ARM registers do NOT expose true syscall. * - compat ARM registers on ARM64 DO expose true syscall. + * - values of utsbuf.machine include 'armv8l' or 'armb8b' + * for ARM64 running in compat mode. */ ASSERT_EQ(0, uname(&utsbuf)); - if (strncmp(utsbuf.machine, "arm", 3) == 0) { + if ((strncmp(utsbuf.machine, "arm", 3) == 0) && + (strncmp(utsbuf.machine, "armv8l", 6) != 0) && + (strncmp(utsbuf.machine, "armv8b", 6) != 0)) { EXPECT_EQ(__NR_nanosleep, ret); } else #endif diff --git a/tools/testing/selftests/syscall_user_dispatch/sud_test.c b/tools/testing/selftests/syscall_user_dispatch/sud_test.c index d975a67673299..48cf01aeec3e7 100644 --- a/tools/testing/selftests/syscall_user_dispatch/sud_test.c +++ b/tools/testing/selftests/syscall_user_dispatch/sud_test.c @@ -79,6 +79,21 @@ TEST_SIGNAL(dispatch_trigger_sigsys, SIGSYS) } } +static void prctl_valid(struct __test_metadata *_metadata, + unsigned long op, unsigned long off, + unsigned long size, void *sel) +{ + EXPECT_EQ(0, prctl(PR_SET_SYSCALL_USER_DISPATCH, op, off, size, sel)); +} + +static void prctl_invalid(struct __test_metadata *_metadata, + unsigned long op, unsigned long off, + unsigned long size, void *sel, int err) +{ + EXPECT_EQ(-1, prctl(PR_SET_SYSCALL_USER_DISPATCH, op, off, size, sel)); + EXPECT_EQ(err, errno); +} + TEST(bad_prctl_param) { char sel = SYSCALL_DISPATCH_FILTER_ALLOW; @@ -86,57 +101,42 @@ TEST(bad_prctl_param) /* Invalid op */ op = -1; - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, 0, 0, &sel); - ASSERT_EQ(EINVAL, errno); + prctl_invalid(_metadata, op, 0, 0, &sel, EINVAL); /* PR_SYS_DISPATCH_OFF */ op = PR_SYS_DISPATCH_OFF; /* offset != 0 */ - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, 0x1, 0x0, 0); - EXPECT_EQ(EINVAL, errno); + prctl_invalid(_metadata, op, 0x1, 0x0, 0, EINVAL); /* len != 0 */ - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, 0x0, 0xff, 0); - EXPECT_EQ(EINVAL, errno); + prctl_invalid(_metadata, op, 0x0, 0xff, 0, EINVAL); /* sel != NULL */ - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, 0x0, 0x0, &sel); - EXPECT_EQ(EINVAL, errno); + prctl_invalid(_metadata, op, 0x0, 0x0, &sel, EINVAL); /* Valid parameter */ - errno = 0; - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, 0x0, 0x0, 0x0); - EXPECT_EQ(0, errno); + prctl_valid(_metadata, op, 0x0, 0x0, 0x0); /* PR_SYS_DISPATCH_ON */ op = PR_SYS_DISPATCH_ON; /* Dispatcher region is bad (offset > 0 && len == 0) */ - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, 0x1, 0x0, &sel); - EXPECT_EQ(EINVAL, errno); - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, -1L, 0x0, &sel); - EXPECT_EQ(EINVAL, errno); + prctl_invalid(_metadata, op, 0x1, 0x0, &sel, EINVAL); + prctl_invalid(_metadata, op, -1L, 0x0, &sel, EINVAL); /* Invalid selector */ - prctl(PR_SET_SYSCALL_USER_DISPATCH, op, 0x0, 0x1, (void *) -1); - ASSERT_EQ(EFAULT, errno); + prctl_invalid(_metadata, op, 0x0, 0x1, (void *) -1, EFAULT); /* * Dispatcher range overflows unsigned long */ - prctl(PR_SET_SYSCALL_USER_DISPATCH, PR_SYS_DISPATCH_ON, 1, -1L, &sel); - ASSERT_EQ(EINVAL, errno) { - TH_LOG("Should reject bad syscall range"); - } + prctl_invalid(_metadata, PR_SYS_DISPATCH_ON, 1, -1L, &sel, EINVAL); /* * Allowed range overflows usigned long */ - prctl(PR_SET_SYSCALL_USER_DISPATCH, PR_SYS_DISPATCH_ON, -1L, 0x1, &sel); - ASSERT_EQ(EINVAL, errno) { - TH_LOG("Should reject bad syscall range"); - } + prctl_invalid(_metadata, PR_SYS_DISPATCH_ON, -1L, 0x1, &sel, EINVAL); } /* diff --git a/tools/testing/selftests/user_events/perf_test.c b/tools/testing/selftests/user_events/perf_test.c index 5288e768b207a..68625362add28 100644 --- a/tools/testing/selftests/user_events/perf_test.c +++ b/tools/testing/selftests/user_events/perf_test.c @@ -236,7 +236,7 @@ TEST_F(user, perf_empty_events) { ASSERT_EQ(1 << reg.enable_bit, self->check); /* Ensure write shows up at correct offset */ - ASSERT_NE(-1, write(self->data_fd, ®.write_index, + ASSERT_NE(-1, write(self->data_fd, (void *)®.write_index, sizeof(reg.write_index))); val = (void *)(((char *)perf_page) + perf_page->data_offset); ASSERT_EQ(PERF_RECORD_SAMPLE, *val); diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c index a1f506ba55786..4f09c5db0c7f3 100644 --- a/tools/testing/selftests/watchdog/watchdog-test.c +++ b/tools/testing/selftests/watchdog/watchdog-test.c @@ -332,6 +332,12 @@ int main(int argc, char *argv[]) if (oneshot) goto end; + /* Check if WDIOF_KEEPALIVEPING is supported */ + if (!(info.options & WDIOF_KEEPALIVEPING)) { + printf("WDIOC_KEEPALIVE not supported by this device\n"); + goto end; + } + printf("Watchdog Ticking Away!\n"); /* diff --git a/tools/testing/selftests/x86/Makefile b/tools/testing/selftests/x86/Makefile index 7e8c937627dd9..13ce9a53d5dec 100644 --- a/tools/testing/selftests/x86/Makefile +++ b/tools/testing/selftests/x86/Makefile @@ -12,7 +12,7 @@ CAN_BUILD_WITH_NOPIE := $(shell ./check_cc.sh "$(CC)" trivial_program.c -no-pie) TARGETS_C_BOTHBITS := single_step_syscall sysret_ss_attrs syscall_nt test_mremap_vdso \ check_initial_reg_state sigreturn iopl ioperm \ - test_vsyscall mov_ss_trap \ + test_vsyscall mov_ss_trap sigtrap_loop \ syscall_arg_fault fsgsbase_restore sigaltstack TARGETS_C_32BIT_ONLY := entry_from_vm86 test_syscall_vdso unwind_vdso \ test_FCMOV test_FCOMI test_FISTTP \ diff --git a/tools/testing/selftests/x86/sigtrap_loop.c b/tools/testing/selftests/x86/sigtrap_loop.c new file mode 100644 index 0000000000000..9d065479e89f9 --- /dev/null +++ b/tools/testing/selftests/x86/sigtrap_loop.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 Intel Corporation + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#ifdef __x86_64__ +# define REG_IP REG_RIP +#else +# define REG_IP REG_EIP +#endif + +static void sethandler(int sig, void (*handler)(int, siginfo_t *, void *), int flags) +{ + struct sigaction sa; + + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = handler; + sa.sa_flags = SA_SIGINFO | flags; + sigemptyset(&sa.sa_mask); + + if (sigaction(sig, &sa, 0)) + err(1, "sigaction"); + + return; +} + +static void sigtrap(int sig, siginfo_t *info, void *ctx_void) +{ + ucontext_t *ctx = (ucontext_t *)ctx_void; + static unsigned int loop_count_on_same_ip; + static unsigned long last_trap_ip; + + if (last_trap_ip == ctx->uc_mcontext.gregs[REG_IP]) { + printf("\tTrapped at %016lx\n", last_trap_ip); + + /* + * If the same IP is hit more than 10 times in a row, it is + * _considered_ an infinite loop. + */ + if (++loop_count_on_same_ip > 10) { + printf("[FAIL]\tDetected SIGTRAP infinite loop\n"); + exit(1); + } + + return; + } + + loop_count_on_same_ip = 0; + last_trap_ip = ctx->uc_mcontext.gregs[REG_IP]; + printf("\tTrapped at %016lx\n", last_trap_ip); +} + +int main(int argc, char *argv[]) +{ + sethandler(SIGTRAP, sigtrap, 0); + + /* + * Set the Trap Flag (TF) to single-step the test code, therefore to + * trigger a SIGTRAP signal after each instruction until the TF is + * cleared. + * + * Because the arithmetic flags are not significant here, the TF is + * set by pushing 0x302 onto the stack and then popping it into the + * flags register. + * + * Four instructions in the following asm code are executed with the + * TF set, thus the SIGTRAP handler is expected to run four times. + */ + printf("[RUN]\tSIGTRAP infinite loop detection\n"); + asm volatile( +#ifdef __x86_64__ + /* + * Avoid clobbering the redzone + * + * Equivalent to "sub $128, %rsp", however -128 can be encoded + * in a single byte immediate while 128 uses 4 bytes. + */ + "add $-128, %rsp\n\t" +#endif + "push $0x302\n\t" + "popf\n\t" + "nop\n\t" + "nop\n\t" + "push $0x202\n\t" + "popf\n\t" +#ifdef __x86_64__ + "sub $-128, %rsp\n\t" +#endif + ); + + printf("[OK]\tNo SIGTRAP infinite loop detected\n"); + return 0; +} diff --git a/tools/tracing/latency/latency-collector.c b/tools/tracing/latency/latency-collector.c index cf263fe9deaf4..ef97916e3873a 100644 --- a/tools/tracing/latency/latency-collector.c +++ b/tools/tracing/latency/latency-collector.c @@ -1725,7 +1725,7 @@ static void show_usage(void) "-n, --notrace\t\tIf latency is detected, do not print out the content of\n" "\t\t\tthe trace file to standard output\n\n" -"-t, --threads NRTHR\tRun NRTHR threads for printing. Default is %d.\n\n" +"-e, --threads NRTHR\tRun NRTHR threads for printing. Default is %d.\n\n" "-r, --random\t\tArbitrarily sleep a certain amount of time, default\n" "\t\t\t%ld ms, before reading the trace file. The\n" diff --git a/tools/verification/rv/src/in_kernel.c b/tools/verification/rv/src/in_kernel.c index f04479ecc96c0..ced72950cb1ee 100644 --- a/tools/verification/rv/src/in_kernel.c +++ b/tools/verification/rv/src/in_kernel.c @@ -353,7 +353,7 @@ ikm_event_handler(struct trace_seq *s, struct tep_record *record, if (config_has_id && (config_my_pid == id)) return 0; - else if (config_my_pid && (config_my_pid == pid)) + else if (config_my_pid == pid) return 0; tep_print_event(trace_event->tep, s, record, "%16s-%-8d ", TEP_PRINT_COMM, TEP_PRINT_PID); @@ -595,7 +595,7 @@ static int parse_arguments(char *monitor_name, int argc, char **argv) config_reactor = optarg; break; case 's': - config_my_pid = 0; + config_my_pid = -1; break; case 't': config_trace = 1; diff --git a/usr/include/headers_check.pl b/usr/include/headers_check.pl index b6aec5e4365f9..682980781eb37 100755 --- a/usr/include/headers_check.pl +++ b/usr/include/headers_check.pl @@ -160,6 +160,8 @@ sub check_sizetypes if (my $included = ($line =~ /^\s*#\s*include\s+[<"](\S+)[>"]/)[0]) { check_include_typesh($included); } + # strip single-line comments, as types may be referenced within them + $line =~ s@/\*.*?\*/@@; if ($line =~ m/__[us](8|16|32|64)\b/) { printf STDERR "$filename:$lineno: " . "found __[us]{8,16,32,64} type " .