########## GPU 机恢复取证 20260928_174251 ##########
===== 1. 当前状态 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 hostname; uptime; echo "--- who ---"; who; echo "--- last reboot ---"; last -x reboot 2>/dev/null | head -5
zyw@192.168.31.31's password:
zywpc
17:42:51 up 1 min, 2 users, load average: 2.81, 0.89, 0.31
--- who ---
zyw tty1 Sep 28 17:42
--- last reboot ---
reboot system boot 6.8.0-142-generi Mon Sep 28 17:41 still running
reboot system boot 6.8.0-142-generi Mon Sep 28 11:58 still running
reboot system boot 6.8.0-142-generi Mon Sep 28 10:47 still running
reboot system boot 6.8.0-142-generi Mon Sep 28 02:52 still running
reboot system boot 6.8.0-142-generi Sun Sep 27 20:38 still running
===== 2. 上一次启动的内核日志（找 Xid / GSP / 掉总线 / OOM）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b -1 --no-pager 2>/dev/null | tail -200
zyw@192.168.31.31's password:
Sep 28 11:58:16 zywpc systemd[1]: Listening on lvm2-lvmpolld.socket - LVM2 poll daemon socket.
Sep 28 11:58:16 zywpc systemd[1]: Listening on multipathd.socket - multipathd control socket.
Sep 28 11:58:16 zywpc systemd[1]: Listening on syslog.socket - Syslog Socket.
Sep 28 11:58:16 zywpc systemd[1]: Listening on systemd-fsckd.socket - fsck to fsckd communication Socket.
Sep 28 11:58:16 zywpc systemd[1]: Listening on systemd-initctl.socket - initctl Compatibility Named Pipe.
Sep 28 11:58:16 zywpc systemd[1]: Listening on systemd-journald-dev-log.socket - Journal Socket (/dev/log).
Sep 28 11:58:16 zywpc systemd[1]: Listening on systemd-journald.socket - Journal Socket.
Sep 28 11:58:16 zywpc systemd[1]: Listening on systemd-networkd.socket - Network Service Netlink Socket.
Sep 28 11:58:16 zywpc systemd[1]: systemd-pcrextend.socket - TPM2 PCR Extension (Varlink) was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Sep 28 11:58:16 zywpc systemd[1]: Listening on systemd-udevd-control.socket - udev Control Socket.
Sep 28 11:58:16 zywpc systemd[1]: Listening on systemd-udevd-kernel.socket - udev Kernel Socket.
Sep 28 11:58:16 zywpc systemd[1]: Mounting dev-hugepages.mount - Huge Pages File System...
Sep 28 11:58:16 zywpc systemd[1]: Mounting dev-mqueue.mount - POSIX Message Queue File System...
Sep 28 11:58:16 zywpc systemd[1]: Mounting sys-kernel-debug.mount - Kernel Debug File System...
Sep 28 11:58:16 zywpc systemd[1]: Mounting sys-kernel-tracing.mount - Kernel Trace File System...
Sep 28 11:58:16 zywpc systemd[1]: Starting systemd-journald.service - Journal Service...
Sep 28 11:58:16 zywpc systemd[1]: Starting keyboard-setup.service - Set the console keyboard layout...
Sep 28 11:58:16 zywpc systemd[1]: Starting kmod-static-nodes.service - Create List of Static Device Nodes...
Sep 28 11:58:16 zywpc systemd[1]: Starting lvm2-monitor.service - Monitoring of LVM2 mirrors, snapshots etc. using dmeventd or progress polling...
Sep 28 11:58:16 zywpc systemd[1]: Starting modprobe@configfs.service - Load Kernel Module configfs...
Sep 28 11:58:16 zywpc systemd[1]: Starting modprobe@dm_mod.service - Load Kernel Module dm_mod...
Sep 28 11:58:16 zywpc systemd[1]: Starting modprobe@drm.service - Load Kernel Module drm...
Sep 28 11:58:16 zywpc systemd[1]: Starting modprobe@efi_pstore.service - Load Kernel Module efi_pstore...
Sep 28 11:58:16 zywpc systemd[1]: Starting modprobe@fuse.service - Load Kernel Module fuse...
Sep 28 11:58:16 zywpc systemd[1]: Starting modprobe@loop.service - Load Kernel Module loop...
Sep 28 11:58:16 zywpc systemd[1]: netplan-ovs-cleanup.service - OpenVSwitch configuration for cleanup was skipped because of an unmet condition check (ConditionFileIsExecutable=/usr/bin/ovs-vsctl).
Sep 28 11:58:16 zywpc systemd[1]: systemd-fsck-root.service - File System Check on Root Device was skipped because of an unmet condition check (ConditionPathExists=!/run/initramfs/fsck-root).
Sep 28 11:58:16 zywpc systemd[1]: Starting systemd-modules-load.service - Load Kernel Modules...
Sep 28 11:58:16 zywpc systemd[1]: systemd-pcrmachine.service - TPM2 PCR Machine ID Measurement was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Sep 28 11:58:16 zywpc systemd[1]: Starting systemd-remount-fs.service - Remount Root and Kernel File Systems...
Sep 28 11:58:16 zywpc systemd[1]: systemd-tpm2-setup-early.service - TPM2 SRK Setup (Early) was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Sep 28 11:58:16 zywpc kernel: pstore: Using crash dump compression: deflate
Sep 28 11:58:16 zywpc systemd[1]: Starting systemd-udev-trigger.service - Coldplug All udev Devices...
Sep 28 11:58:16 zywpc systemd[1]: Mounted dev-hugepages.mount - Huge Pages File System.
Sep 28 11:58:16 zywpc kernel: pstore: Registered efi_pstore as persistent store backend
Sep 28 11:58:16 zywpc systemd[1]: Mounted dev-mqueue.mount - POSIX Message Queue File System.
Sep 28 11:58:16 zywpc systemd[1]: Mounted sys-kernel-debug.mount - Kernel Debug File System.
Sep 28 11:58:16 zywpc systemd[1]: Mounted sys-kernel-tracing.mount - Kernel Trace File System.
Sep 28 11:58:16 zywpc systemd[1]: Finished kmod-static-nodes.service - Create List of Static Device Nodes.
Sep 28 11:58:16 zywpc systemd[1]: modprobe@configfs.service: Deactivated successfully.
Sep 28 11:58:16 zywpc systemd[1]: Finished modprobe@configfs.service - Load Kernel Module configfs.
Sep 28 11:58:16 zywpc systemd[1]: modprobe@dm_mod.service: Deactivated successfully.
Sep 28 11:58:16 zywpc systemd[1]: Finished modprobe@dm_mod.service - Load Kernel Module dm_mod.
Sep 28 11:58:16 zywpc systemd[1]: modprobe@drm.service: Deactivated successfully.
Sep 28 11:58:16 zywpc systemd[1]: Finished modprobe@drm.service - Load Kernel Module drm.
Sep 28 11:58:16 zywpc systemd[1]: modprobe@efi_pstore.service: Deactivated successfully.
Sep 28 11:58:16 zywpc systemd[1]: Finished modprobe@efi_pstore.service - Load Kernel Module efi_pstore.
Sep 28 11:58:16 zywpc systemd[1]: modprobe@fuse.service: Deactivated successfully.
Sep 28 11:58:16 zywpc systemd[1]: Finished modprobe@fuse.service - Load Kernel Module fuse.
Sep 28 11:58:16 zywpc systemd[1]: modprobe@loop.service: Deactivated successfully.
Sep 28 11:58:16 zywpc systemd[1]: Finished modprobe@loop.service - Load Kernel Module loop.
Sep 28 11:58:16 zywpc systemd[1]: Mounting sys-fs-fuse-connections.mount - FUSE Control File System...
Sep 28 11:58:16 zywpc systemd[1]: Mounting sys-kernel-config.mount - Kernel Configuration File System...
Sep 28 11:58:16 zywpc systemd[1]: systemd-repart.service - Repartition Root Disk was skipped because no trigger condition checks were met.
Sep 28 11:58:16 zywpc systemd[1]: Starting systemd-tmpfiles-setup-dev-early.service - Create Static Device Nodes in /dev gracefully...
Sep 28 11:58:16 zywpc systemd-journald[464]: Collecting audit messages is disabled.
Sep 28 11:58:16 zywpc systemd[1]: Mounted sys-fs-fuse-connections.mount - FUSE Control File System.
Sep 28 11:58:16 zywpc systemd[1]: Mounted sys-kernel-config.mount - Kernel Configuration File System.
Sep 28 11:58:16 zywpc kernel: EXT4-fs (sda5): re-mounted 5cea3463-6633-42ac-9011-cee5516972f1 r/w.
Sep 28 11:58:16 zywpc systemd[1]: Finished systemd-remount-fs.service - Remount Root and Kernel File Systems.
Sep 28 11:58:16 zywpc systemd[1]: Started systemd-journald.service - Journal Service.
Sep 28 11:58:16 zywpc systemd-journald[464]: Received client request to flush runtime journal.
Sep 28 11:58:16 zywpc kernel: Adding 8388604k swap on /swap.img. Priority:-2 extents:67 across:340049920k SS
Sep 28 11:58:16 zywpc systemd-journald[464]: File /var/log/journal/cab04c94c96f4a58b6b86fcc2420b27e/system.journal corrupted or uncleanly shut down, renaming and replacing.
Sep 28 11:58:16 zywpc kernel: Adding 67108860k swap on /swap2.img. Priority:-3 extents:40 across:666304512k SS
Sep 28 11:58:16 zywpc kernel: nvidia: loading out-of-tree module taints kernel.
Sep 28 11:58:16 zywpc kernel: nvidia: module verification failed: signature and/or required key missing - tainting kernel
Sep 28 11:58:16 zywpc kernel: nvidia-nvlink: Nvlink Core is being initialized, major device number 238
Sep 28 11:58:16 zywpc kernel:
Sep 28 11:58:16 zywpc kernel: nvidia 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=none:owns=none
Sep 28 11:58:16 zywpc kernel: NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:24:27 UTC 2026
Sep 28 11:58:16 zywpc kernel: intel_pmc_core INT33A1:00: initialized
Sep 28 11:58:16 zywpc kernel: i801_smbus 0000:00:1f.4: SPD Write Disable is set
Sep 28 11:58:16 zywpc kernel: i801_smbus 0000:00:1f.4: SMBus using PCI interrupt
Sep 28 11:58:16 zywpc kernel: nvidia-modeset: Loading NVIDIA UNIX Open Kernel Mode Setting Driver for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:12:42 UTC 2026
Sep 28 11:58:16 zywpc kernel: mei_me 0000:00:16.0: enabling device (0000 -> 0002)
Sep 28 11:58:16 zywpc kernel: i2c i2c-0: 1/4 memory slots populated (from DMI)
Sep 28 11:58:16 zywpc kernel: i2c i2c-0: Successfully instantiated SPD at 0x52
Sep 28 11:58:16 zywpc kernel: ee1004 0-0052: 512 byte EE1004-compliant SPD EEPROM, read-only
Sep 28 11:58:16 zywpc kernel: [drm] [nvidia-drm] [GPU ID 0x00000100] Loading driver
Sep 28 11:58:16 zywpc kernel: RAPL PMU: API unit is 2^-32 Joules, 3 fixed counters, 655360 ms ovfl timer
Sep 28 11:58:16 zywpc kernel: RAPL PMU: hw unit of domain pp0-core 2^-14 Joules
Sep 28 11:58:16 zywpc kernel: RAPL PMU: hw unit of domain package 2^-14 Joules
Sep 28 11:58:16 zywpc kernel: RAPL PMU: hw unit of domain pp1-gpu 2^-14 Joules
Sep 28 11:58:16 zywpc kernel: Creating 1 MTD partitions on "0000:00:1f.5":
Sep 28 11:58:16 zywpc kernel: 0x000000000000-0x000001000000 : "BIOS"
Sep 28 11:58:16 zywpc kernel: snd_hda_intel 0000:00:1f.3: enabling device (0000 -> 0002)
Sep 28 11:58:16 zywpc kernel: snd_hda_intel 0000:01:00.1: Disabling MSI
Sep 28 11:58:16 zywpc kernel: snd_hda_intel 0000:01:00.1: Handle vga_switcheroo audio client
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: enabling device (0006 -> 0007)
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] VT-d active for gfx access
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] Using Transparent Hugepages
Sep 28 11:58:16 zywpc kernel: mei_hdcp 0000:00:16.0-b638ab7e-94e2-4ea2-a552-d1c54b627f04: bound 0000:00:02.0 (ops i915_hdcp_ops [i915])
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] Finished loading DMC firmware i915/adls_dmc_ver2_01.bin (v2.1)
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] GT0: GuC firmware i915/tgl_guc_70.bin version 70.36.0
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] GT0: HuC firmware i915/tgl_huc.bin version 7.9.3
Sep 28 11:58:16 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=3 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input15
Sep 28 11:58:16 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=7 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input16
Sep 28 11:58:16 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=8 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input17
Sep 28 11:58:16 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=9 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input18
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] GT0: HuC: authenticated for all workloads
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] GT0: GUC: submission disabled
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] GT0: GUC: SLPC disabled
Sep 28 11:58:16 zywpc kernel: mei_pxp 0000:00:16.0-fbf6fcf1-96cf-4e2e-a6a6-1bab8cbe36b1: bound 0000:00:02.0 (ops i915_pxp_tee_component_ops [i915])
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] Protected Xe Path (PXP) protected content support initialized
Sep 28 11:58:16 zywpc kernel: [drm] Initialized i915 1.6.0 20230929 for 0000:00:02.0 on minor 1
Sep 28 11:58:16 zywpc kernel: i915 display info: display version: 12
Sep 28 11:58:16 zywpc kernel: i915 display info: cursor_needs_physical: no
Sep 28 11:58:16 zywpc kernel: i915 display info: has_cdclk_crawl: no
Sep 28 11:58:16 zywpc kernel: i915 display info: has_cdclk_squash: no
Sep 28 11:58:16 zywpc kernel: i915 display info: has_ddi: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_dp_mst: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_dsb: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_fpga_dbg: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_gmch: no
Sep 28 11:58:16 zywpc kernel: i915 display info: has_hotplug: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_hti: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_ipc: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_overlay: no
Sep 28 11:58:16 zywpc kernel: i915 display info: has_psr: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_psr_hw_tracking: no
Sep 28 11:58:16 zywpc kernel: i915 display info: overlay_needs_physical: no
Sep 28 11:58:16 zywpc kernel: i915 display info: supports_tv: no
Sep 28 11:58:16 zywpc kernel: i915 display info: has_hdcp: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_dmc: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: has_dsc: yes
Sep 28 11:58:16 zywpc kernel: i915 display info: rawclk rate: 19200 kHz
Sep 28 11:58:16 zywpc kernel: snd_hda_intel 0000:00:1f.3: bound 0000:00:02.0 (ops i915_audio_component_bind_ops [i915])
Sep 28 11:58:16 zywpc kernel: Console: switching to colour frame buffer device 160x45
Sep 28 11:58:16 zywpc kernel: i915 0000:00:02.0: [drm] fb0: i915drmfb frame buffer device
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: autoconfig for ALC897: line_outs=1 (0x14/0x0/0x0/0x0/0x0) type:line
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: speaker_outs=0 (0x0/0x0/0x0/0x0/0x0)
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: hp_outs=1 (0x1b/0x0/0x0/0x0/0x0)
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: mono: mono_out=0x0
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: inputs:
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: Rear Mic=0x18
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: Front Mic=0x19
Sep 28 11:58:16 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: Line=0x1a
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH Rear Mic as /devices/pci0000:00/0000:00:1f.3/sound/card1/input19
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH Front Mic as /devices/pci0000:00/0000:00:1f.3/sound/card1/input20
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH Line as /devices/pci0000:00/0000:00:1f.3/sound/card1/input21
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH Line Out as /devices/pci0000:00/0000:00:1f.3/sound/card1/input22
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH Front Headphone as /devices/pci0000:00/0000:00:1f.3/sound/card1/input23
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=3 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input24
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=7 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input25
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=8 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input26
Sep 28 11:58:17 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=9 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input27
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.133:2): apparmor="STATUS" operation="profile_load" profile="unconfined" name="brave" pid=778 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:3): apparmor="STATUS" operation="profile_load" profile="unconfined" name="buildah" pid=779 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:4): apparmor="STATUS" operation="profile_load" profile="unconfined" name="QtWebEngineProcess" pid=776 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:5): apparmor="STATUS" operation="profile_load" profile="unconfined" name="vscode" pid=785 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:6): apparmor="STATUS" operation="profile_load" profile="unconfined" name="Discord" pid=774 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:7): apparmor="STATUS" operation="profile_load" profile="unconfined" name="element-desktop" pid=788 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:8): apparmor="STATUS" operation="profile_load" profile="unconfined" name="busybox" pid=780 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:9): apparmor="STATUS" operation="profile_load" profile="unconfined" name="ch-run" pid=783 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.134:10): apparmor="STATUS" operation="profile_load" profile="unconfined" name="devhelp" pid=787 comm="apparmor_parser"
Sep 28 11:58:17 zywpc kernel: cfg80211: Loading compiled-in X.509 certificates for regulatory database
Sep 28 11:58:17 zywpc kernel: Loaded X.509 cert 'sforshee: 00b28ddf47aef9cea7'
Sep 28 11:58:17 zywpc kernel: Loaded X.509 cert 'wens: 61c038651aabdcf94bd0ac7ff06c7248db18c600'
Sep 28 11:58:17 zywpc kernel: RTL8226B_RTL8221B 2.5Gbps PHY r8169-0-200:00: attached PHY driver (mii_bus:phy_addr=r8169-0-200:00, irq=MAC)
Sep 28 11:58:17 zywpc kernel: r8169 0000:02:00.0 enp2s0: Link is Down
Sep 28 11:58:17 zywpc kernel: kauditd_printk_skb: 109 callbacks suppressed
Sep 28 11:58:17 zywpc kernel: audit: type=1400 audit(1790567897.571:120): apparmor="STATUS" operation="profile_replace" info="same as current profile, skipping" profile="unconfined" name="rsyslogd" pid=924 comm="apparmor_parser"
Sep 28 11:58:18 zywpc kernel: loop0: detected capacity change from 0 to 8
Sep 28 11:58:18 zywpc kernel: NET: Registered PF_QIPCRTR protocol family
Sep 28 11:58:18 zywpc kernel: intel_tcc_cooling: TCC Offset locked
Sep 28 11:58:18 zywpc kernel: audit: type=1400 audit(1790567898.117:121): apparmor="STATUS" operation="profile_replace" info="same as current profile, skipping" profile="unconfined" name="/usr/lib/snapd/snap-confine" pid=1065 comm="apparmor_parser"
Sep 28 11:58:18 zywpc kernel: intel_rapl_msr: PL4 support detected.
Sep 28 11:58:18 zywpc kernel: intel_rapl_common: Found RAPL domain package
Sep 28 11:58:18 zywpc kernel: intel_rapl_common: Found RAPL domain core
Sep 28 11:58:18 zywpc kernel: intel_rapl_common: Found RAPL domain uncore
Sep 28 11:58:18 zywpc kernel: [drm] Initialized nvidia-drm 0.0.0 20160202 for 0000:01:00.0 on minor 0
Sep 28 11:58:18 zywpc kernel: nvidia 0000:01:00.0: vgaarb: deactivate vga console
Sep 28 11:58:18 zywpc kernel: nvidia 0000:01:00.0: [drm] Cannot find any crtc or sizes
Sep 28 11:58:19 zywpc kernel: r8169 0000:02:00.0 enp2s0: Link is Up - 1Gbps/Full - flow control rx/tx
Sep 28 11:58:21 zywpc systemd-journald[464]: /var/log/journal/cab04c94c96f4a58b6b86fcc2420b27e/user-1000.journal: Journal file uses a different sequence number ID, rotating.
Sep 28 11:58:22 zywpc kernel: audit: type=1400 audit(1790567902.121:122): apparmor="STATUS" operation="profile_load" profile="unconfined" name="docker-default" pid=1513 comm="apparmor_parser"
Sep 28 11:58:22 zywpc kernel: Initializing XFRM netlink socket
Sep 28 11:58:22 zywpc kernel: bridge: filtering via arp/ip/ip6tables is no longer available by default. Update your scripts to load br_netfilter if you need this.
Sep 28 11:58:23 zywpc kernel: br-b03fdbdfcf4b: port 1(vetha5a57b1) entered blocking state
Sep 28 11:58:23 zywpc kernel: br-b03fdbdfcf4b: port 1(vetha5a57b1) entered disabled state
Sep 28 11:58:23 zywpc kernel: vetha5a57b1: entered allmulticast mode
Sep 28 11:58:23 zywpc kernel: vetha5a57b1: entered promiscuous mode
Sep 28 11:58:23 zywpc kernel: br-b03fdbdfcf4b: port 2(veth034dab9) entered blocking state
Sep 28 11:58:23 zywpc kernel: br-b03fdbdfcf4b: port 2(veth034dab9) entered disabled state
Sep 28 11:58:23 zywpc kernel: veth034dab9: entered allmulticast mode
Sep 28 11:58:23 zywpc kernel: veth034dab9: entered promiscuous mode
Sep 28 11:58:23 zywpc kernel: eth0: renamed from vetha158d02
Sep 28 11:58:24 zywpc kernel: br-b03fdbdfcf4b: port 1(vetha5a57b1) entered blocking state
Sep 28 11:58:24 zywpc kernel: br-b03fdbdfcf4b: port 1(vetha5a57b1) entered forwarding state
Sep 28 11:58:24 zywpc kernel: eth0: renamed from vethe1f9570
Sep 28 11:58:24 zywpc kernel: br-b03fdbdfcf4b: port 2(veth034dab9) entered blocking state
Sep 28 11:58:24 zywpc kernel: br-b03fdbdfcf4b: port 2(veth034dab9) entered forwarding state
Sep 28 11:58:24 zywpc kernel: evm: overlay not supported
Sep 28 11:58:26 zywpc kernel: wireguard: WireGuard 1.0.0 loaded. See www.wireguard.com for information.
Sep 28 11:58:26 zywpc kernel: wireguard: Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
Sep 28 12:34:28 zywpc kernel: perf: interrupt took too long (2653 > 2500), lowering kernel.perf_event_max_sample_rate to 75000
Sep 28 13:04:59 zywpc kernel: perf: interrupt took too long (3574 > 3316), lowering kernel.perf_event_max_sample_rate to 55000
Sep 28 13:27:35 zywpc kernel: perf: interrupt took too long (4522 > 4467), lowering kernel.perf_event_max_sample_rate to 44000
Sep 28 14:34:18 zywpc kernel: perf: interrupt took too long (5666 > 5652), lowering kernel.perf_event_max_sample_rate to 35000
===== 3. 上次启动日志里的关键错误 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b -1 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|out of memory|hung_task|watchdog|BUG:|panic" | tail -60
zyw@192.168.31.31's password:
Sep 28 11:58:16 zywpc kernel: NMI watchdog: Enabled. Permanently consumes one hw-PMU counter.
Sep 28 11:58:16 zywpc kernel: ACPI FADT declares the system doesn't support PCIe ASPM, so disable it
Sep 28 11:58:16 zywpc kernel: acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug SHPCHotplug PME AER PCIeCapability LTR DPC]
Sep 28 11:58:16 zywpc kernel: pci 0000:00:01.0: [8086:460d] type 01 class 0x060400 PCIe Root Port
Sep 28 11:58:16 zywpc kernel: pci 0000:00:02.0: [8086:4688] type 00 class 0x038000 PCIe Root Complex Integrated Endpoint
Sep 28 11:58:16 zywpc kernel: pci 0000:00:0a.0: [8086:467d] type 00 class 0x118000 PCIe Root Complex Integrated Endpoint
Sep 28 11:58:16 zywpc kernel: pci 0000:00:1d.0: [8086:7ab7] type 01 class 0x060400 PCIe Root Port
Sep 28 11:58:16 zywpc kernel: pci 0000:01:00.0: [10de:2d04] type 00 class 0x030000 PCIe Legacy Endpoint
Sep 28 11:58:16 zywpc kernel: pci 0000:01:00.0: 252.056 Gb/s available PCIe bandwidth, limited by 32.0 GT/s PCIe x8 link at 0000:00:01.0 (capable of 504.112 Gb/s with 32.0 GT/s PCIe x16 link)
Sep 28 11:58:16 zywpc kernel: pci 0000:01:00.1: [10de:22eb] type 00 class 0x040300 PCIe Endpoint
Sep 28 11:58:16 zywpc kernel: pci 0000:02:00.0: [10ec:8125] type 00 class 0x020000 PCIe Endpoint
Sep 28 11:58:16 zywpc kernel: pcieport 0000:00:01.0: PME: Signaling with IRQ 122
Sep 28 11:58:16 zywpc kernel: pcieport 0000:00:1d.0: PME: Signaling with IRQ 123
Sep 28 11:58:16 zywpc kernel: pcieport 0000:00:1d.0: AER: enabled with IRQ 123
Sep 28 11:58:16 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
Sep 28 11:58:16 zywpc kernel: NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:24:27 UTC 2026
===== 4. 当前启动的内核错误 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b 0 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|hung_task|BUG:|panic" | tail -40
zyw@192.168.31.31's password:
Sep 28 17:41:49 zywpc kernel: ACPI FADT declares the system doesn't support PCIe ASPM, so disable it
Sep 28 17:41:49 zywpc kernel: acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug SHPCHotplug PME AER PCIeCapability LTR DPC]
Sep 28 17:41:49 zywpc kernel: pci 0000:00:01.0: [8086:460d] type 01 class 0x060400 PCIe Root Port
Sep 28 17:41:49 zywpc kernel: pci 0000:00:02.0: [8086:4688] type 00 class 0x038000 PCIe Root Complex Integrated Endpoint
Sep 28 17:41:49 zywpc kernel: pci 0000:00:0a.0: [8086:467d] type 00 class 0x118000 PCIe Root Complex Integrated Endpoint
Sep 28 17:41:49 zywpc kernel: pci 0000:00:1d.0: [8086:7ab7] type 01 class 0x060400 PCIe Root Port
Sep 28 17:41:49 zywpc kernel: pci 0000:01:00.0: [10de:2d04] type 00 class 0x030000 PCIe Legacy Endpoint
Sep 28 17:41:49 zywpc kernel: pci 0000:01:00.0: 252.056 Gb/s available PCIe bandwidth, limited by 32.0 GT/s PCIe x8 link at 0000:00:01.0 (capable of 504.112 Gb/s with 32.0 GT/s PCIe x16 link)
Sep 28 17:41:49 zywpc kernel: pci 0000:01:00.1: [10de:22eb] type 00 class 0x040300 PCIe Endpoint
Sep 28 17:41:49 zywpc kernel: pci 0000:02:00.0: [10ec:8125] type 00 class 0x020000 PCIe Endpoint
Sep 28 17:41:49 zywpc kernel: pcieport 0000:00:01.0: PME: Signaling with IRQ 122
Sep 28 17:41:49 zywpc kernel: pcieport 0000:00:1d.0: PME: Signaling with IRQ 123
Sep 28 17:41:49 zywpc kernel: pcieport 0000:00:1d.0: AER: enabled with IRQ 123
Sep 28 17:41:49 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
Sep 28 17:41:51 zywpc kernel: NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:24:27 UTC 2026
===== 5. journald 是否持久化（-b -1 能否用）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 ls -la /var/log/journal 2>/dev/null | head -3; grep -E "^\s*Storage=" /etc/systemd/journald.conf 2>/dev/null; journalctl --list-boots --no-pager 2>/dev/null | tail -5
zyw@192.168.31.31's password:
total 28
drwxr-sr-x+ 4 root systemd-journal 4096 Sep 27 20:56 .
drwxrwxr-x 11 root syslog 4096 Sep 28 17:41 ..
-4 2bb558e59c3e4b2c8865ff7b0fb759d0 Sun 2026-09-27 20:38:51 CST Mon 2026-09-28 02:26:57 CST
-3 5a19a7c6874a4816b70028da41af13d7 Mon 2026-09-28 02:52:05 CST Mon 2026-09-28 05:04:39 CST
-2 e62b9594a0e7488c9fdd9009d20fa9a7 Mon 2026-09-28 10:47:42 CST Mon 2026-09-28 11:32:07 CST
-1 88e82b95a06f4a95bebe653df5686a9a Mon 2026-09-28 11:58:16 CST Mon 2026-09-28 15:44:10 CST
0 6dbd09333b3640f18ecc0d64adac2411 Mon 2026-09-28 17:41:49 CST Mon 2026-09-28 17:43:08 CST
===== 6. GPU 现状 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 nvidia-smi -q 2>/dev/null | grep -iE "Product Name|Driver Version|CUDA Version|Power Limit|Temperature|GPU Current Temp|Clocks Throttle|Bus Id|PCIe Generation|Retired|Remapped|Xid|ECC" | head -30
zyw@192.168.31.31's password:
Driver Version : 610.43.02 [Deprecated; will be removed in CUDA 14.0. Use KMD Version instead]
CUDA Version : 13.3 [Deprecated; will be removed in CUDA 14.0. Use CUDA UMD Version instead]
Product Name : NVIDIA GeForce RTX 5060 Ti
ECC Object : N/A
Bus Id : 00000000:01:00.0
PCIe Generation
ECC Mode
ECC Errors
Retired Pages
Single Bit ECC : N/A
Double Bit ECC : N/A
Remapped Rows : N/A
Temperature
GPU Current Temp : 43 C
GPU Target Temperature Specification : N/A
GPU Ceiling Power Limit
Current Power Limit : 180.00 W
Requested Power Limit : 180.00 W
Default Power Limit : 180.00 W
Min Power Limit : 150.00 W
Max Power Limit : 180.00 W
Current Power Limit : N/A
Requested Power Limit : N/A
Default Power Limit : N/A
Min Power Limit : N/A
Max Power Limit : N/A
===== 7. NVreg 参数与内核 panic 设置 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 cat /sys/module/nvidia/parameters/NVreg_DynamicPowerManagement 2>/dev/null; echo "--- sysctl ---"; sysctl kernel.panic kernel.panic_on_oops kernel.hung_task_panic kernel.hung_task_timeout_secs 2>/dev/null; echo "--- mem ---"; free -g
zyw@192.168.31.31's password:
--- sysctl ---
kernel.panic = 10
kernel.panic_on_oops = 0
kernel.hung_task_panic = 1
kernel.hung_task_timeout_secs = 300
--- mem ---
total used free shared buff/cache available
Mem: 30 2 27 0 2 28
Swap: 71 0 71
===== 8. ComfyUI 日志尾部（死亡时的最后动作）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 ls -la ~/comfyui_8189_opt.log* 2>/dev/null; echo "---"; tail -c 3000 ~/comfyui_8189_opt.log 2>/dev/null
zyw@192.168.31.31's password:
-rw-rw-r-- 1 zyw zyw 53237 Sep 28 15:42 /home/zyw/comfyui_8189_opt.log
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:55 /home/zyw/comfyui_8189_opt.log.bak-0915-1955
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:56 /home/zyw/comfyui_8189_opt.log.bak-0915-1956
-rw-rw-r-- 1 zyw zyw 22941 Sep 16 04:21 /home/zyw/comfyui_8189_opt.log.bak-0915-2021
-rw-rw-r-- 1 zyw zyw 27263 Sep 16 04:39 /home/zyw/comfyui_8189_opt.log.bak-0915-2039
-rw-rw-r-- 1 zyw zyw 24377 Sep 16 04:42 /home/zyw/comfyui_8189_opt.log.bak-0915-2042
-rw-rw-r-- 1 zyw zyw 31368 Sep 16 12:56 /home/zyw/comfyui_8189_opt.log.bak-0916-0456
-rw-rw-r-- 1 zyw zyw 43775 Sep 16 14:48 /home/zyw/comfyui_8189_opt.log.bak-0916-0648
-rw-rw-r-- 1 zyw zyw 115390 Sep 17 18:44 /home/zyw/comfyui_8189_opt.log.bak-0917-1044
-rw-rw-r-- 1 zyw zyw 49507 Sep 17 20:31 /home/zyw/comfyui_8189_opt.log.bak-0917-1231
-rw-rw-r-- 1 zyw zyw 53620 Sep 18 01:46 /home/zyw/comfyui_8189_opt.log.bak-0917-1746
-rw-rw-r-- 1 zyw zyw 4253 Sep 18 01:47 /home/zyw/comfyui_8189_opt.log.bak-0917-1747
-rw-rw-r-- 1 zyw zyw 183939 Sep 19 15:38 /home/zyw/comfyui_8189_opt.log.bak-0919-0738
-rw-rw-r-- 1 zyw zyw 43176 Sep 19 18:10 /home/zyw/comfyui_8189_opt.log.bak-0919-1010
-rw-rw-r-- 1 zyw zyw 38378 Sep 19 19:51 /home/zyw/comfyui_8189_opt.log.bak-0919-1151
-rw-rw-r-- 1 zyw zyw 69814 Sep 20 03:15 /home/zyw/comfyui_8189_opt.log.bak-0919-1915
-rw-rw-r-- 1 zyw zyw 81821 Sep 21 12:59 /home/zyw/comfyui_8189_opt.log.bak-0921-0459
-rw-rw-r-- 1 zyw zyw 8084 Sep 21 13:00 /home/zyw/comfyui_8189_opt.log.bak-0921-0500
-rw-rw-r-- 1 zyw zyw 82281 Sep 21 20:25 /home/zyw/comfyui_8189_opt.log.bak-0921-1225
-rw-rw-r-- 1 zyw zyw 35146 Sep 21 21:12 /home/zyw/comfyui_8189_opt.log.bak-0921-1312
-rw-rw-r-- 1 zyw zyw 26036 Sep 21 21:14 /home/zyw/comfyui_8189_opt.log.bak-0921-1314
-rw-rw-r-- 1 zyw zyw 61654 Sep 21 22:02 /home/zyw/comfyui_8189_opt.log.bak-0921-1402
-rw-rw-r-- 1 zyw zyw 122254 Sep 22 02:35 /home/zyw/comfyui_8189_opt.log.bak-0921-1835
-rw-rw-r-- 1 zyw zyw 168741 Sep 24 21:29 /home/zyw/comfyui_8189_opt.log.bak-0924-1329
-rw-rw-r-- 1 zyw zyw 251562 Sep 25 22:41 /home/zyw/comfyui_8189_opt.log.bak-0925-1441
-rw-rw-r-- 1 zyw zyw 47281 Sep 26 23:41 /home/zyw/comfyui_8189_opt.log.bak-0926-1541
-rw-rw-r-- 1 zyw zyw 89836 Sep 27 02:13 /home/zyw/comfyui_8189_opt.log.bak-0926-1813
-rw-rw-r-- 1 zyw zyw 22278 Sep 27 02:14 /home/zyw/comfyui_8189_opt.log.bak-0926-1814
-rw-rw-r-- 1 zyw zyw 41488 Sep 27 03:19 /home/zyw/comfyui_8189_opt.log.bak-0926-1919
-rw-rw-r-- 1 zyw zyw 33794 Sep 27 03:36 /home/zyw/comfyui_8189_opt.log.bak-0926-1936
-rw-rw-r-- 1 zyw zyw 45395 Sep 27 21:53 /home/zyw/comfyui_8189_opt.log.bak-0927-1353
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom
---
m Model MiniMaxH3AudioVAE prepared for dynamic VRAM loading. 576MB Staged. 0 patches attached. Force pre-loaded 401 weights: 539 KB.
[32m[INFO][0m Comfy model compiler graph breaks: 1, rogues: 0
[32m[INFO][0m [32mPrompt executed in 00:27:42[0m
[32m[INFO][0m got prompt
[32m[INFO][0m Requested to load MiniMaxH3VideoVAE
[32m[INFO][0m Model MiniMaxH3VideoVAE prepared for dynamic VRAM loading. 2677MB Staged. 0 patches attached. Force pre-loaded 128 weights: 697 KB.
[32m[INFO][0m Requested to load MiniMaxH3TEModel_
[32m[INFO][0m Model MiniMaxH3TEModel_ prepared for dynamic VRAM loading. 14956MB Staged. 0 patches attached. Force pre-loaded 410 weights: 4572 KB.
[32m[INFO][0m Requested to load MiniMaxH3
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3 prepared for dynamic VRAM loading. 19995MB Staged. 208 patches attached. Force pre-loaded 210 weights: 1175 KB.
[MiniMax H3 Aicg] 渲染结束已清理（卸载模型）：内存 9.8G，显存 0.0G(保留 0.1G) -> 内存 9.8G，显存 0.0G(保留 0.1G)
0%| | 0/3 [00:00<?, ?it/s]
0%| | 0/3 [00:00<?, ?it/s, Model Initializing ... ]
0%| | 0/3 [01:44<?, ?it/s, Model Initialization complete! ]
33%|███▎ | 1/3 [01:44<03:29, 104.78s/it, Model Initialization complete! ]
33%|███▎ | 1/3 [02:27<03:29, 104.78s/it]
67%|██████▋ | 2/3 [02:27<01:13, 73.51s/it]
100%|██████████| 3/3 [03:40<00:00, 73.57s/it]
100%|██████████| 3/3 [03:40<00:00, 73.53s/it]
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3VideoVAE prepared for dynamic VRAM loading. 2677MB Staged. 0 patches attached. Force pre-loaded 128 weights: 697 KB.
[32m[INFO][0m Requested to load MiniMaxH3AudioVAE
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3AudioVAE prepared for dynamic VRAM loading. 576MB Staged. 0 patches attached. Force pre-loaded 401 weights: 539 KB.
[32m[INFO][0m Comfy model compiler graph breaks: 2, rogues: 0
[32m[INFO][0m Requested to load MiniMaxH3
[32m[INFO][0m Model MiniMaxH3 prepared for dynamic VRAM loading. 19995MB Staged. 208 patches attached. Force pre-loaded 210 weights: 1175 KB.
[MiniMax H3 Aicg] 渲染结束已清理（卸载模型）：内存 6.9G，显存 0.0G(保留 0.1G) -> 内存 6.9G，显存 0.0G(保留 0.1G)
[MiniMax H3 Aicg] 二采放大 1.2x：(1, 24, 37, 86, 48) -> (1, 24, 37, 104, 58)
0%| | 0/8 [00:00<?, ?it/s]
0%| | 0/8 [00:00<?, ?it/s, Model Initializing ... ]
0%| | 0/8 [02:47<?, ?it/s, Model Initialization complete! ]
12%|█▎ | 1/8 [02:47<19:32, 167.44s/it, Model Initialization complete! ]
12%|█▎ | 1/8 [04:34<19:32, 167.44s/it]
25%|██▌ | 2/8 [04:34<13:43, 137.20s/it]
38%|███▊ | 3/8 [06:51<11:26, 137.29s/it]
50%|█████ | 4/8 [09:08<09:08, 137.23s/it]
===== 9. 是否有 .bak 轮转 / ram_guard 痕迹 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 ls -la ~/comfyui_8189_opt.log* ~/ram_guard* 2>/dev/null; echo "--- ram_guard 日志 ---"; tail -20 ~/ram_guard.log 2>/dev/null
zyw@192.168.31.31's password:
-rw-rw-r-- 1 zyw zyw 53237 Sep 28 15:42 /home/zyw/comfyui_8189_opt.log
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:55 /home/zyw/comfyui_8189_opt.log.bak-0915-1955
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:56 /home/zyw/comfyui_8189_opt.log.bak-0915-1956
-rw-rw-r-- 1 zyw zyw 22941 Sep 16 04:21 /home/zyw/comfyui_8189_opt.log.bak-0915-2021
-rw-rw-r-- 1 zyw zyw 27263 Sep 16 04:39 /home/zyw/comfyui_8189_opt.log.bak-0915-2039
-rw-rw-r-- 1 zyw zyw 24377 Sep 16 04:42 /home/zyw/comfyui_8189_opt.log.bak-0915-2042
-rw-rw-r-- 1 zyw zyw 31368 Sep 16 12:56 /home/zyw/comfyui_8189_opt.log.bak-0916-0456
-rw-rw-r-- 1 zyw zyw 43775 Sep 16 14:48 /home/zyw/comfyui_8189_opt.log.bak-0916-0648
-rw-rw-r-- 1 zyw zyw 115390 Sep 17 18:44 /home/zyw/comfyui_8189_opt.log.bak-0917-1044
-rw-rw-r-- 1 zyw zyw 49507 Sep 17 20:31 /home/zyw/comfyui_8189_opt.log.bak-0917-1231
-rw-rw-r-- 1 zyw zyw 53620 Sep 18 01:46 /home/zyw/comfyui_8189_opt.log.bak-0917-1746
-rw-rw-r-- 1 zyw zyw 4253 Sep 18 01:47 /home/zyw/comfyui_8189_opt.log.bak-0917-1747
-rw-rw-r-- 1 zyw zyw 183939 Sep 19 15:38 /home/zyw/comfyui_8189_opt.log.bak-0919-0738
-rw-rw-r-- 1 zyw zyw 43176 Sep 19 18:10 /home/zyw/comfyui_8189_opt.log.bak-0919-1010
-rw-rw-r-- 1 zyw zyw 38378 Sep 19 19:51 /home/zyw/comfyui_8189_opt.log.bak-0919-1151
-rw-rw-r-- 1 zyw zyw 69814 Sep 20 03:15 /home/zyw/comfyui_8189_opt.log.bak-0919-1915
-rw-rw-r-- 1 zyw zyw 81821 Sep 21 12:59 /home/zyw/comfyui_8189_opt.log.bak-0921-0459
-rw-rw-r-- 1 zyw zyw 8084 Sep 21 13:00 /home/zyw/comfyui_8189_opt.log.bak-0921-0500
-rw-rw-r-- 1 zyw zyw 82281 Sep 21 20:25 /home/zyw/comfyui_8189_opt.log.bak-0921-1225
-rw-rw-r-- 1 zyw zyw 35146 Sep 21 21:12 /home/zyw/comfyui_8189_opt.log.bak-0921-1312
-rw-rw-r-- 1 zyw zyw 26036 Sep 21 21:14 /home/zyw/comfyui_8189_opt.log.bak-0921-1314
-rw-rw-r-- 1 zyw zyw 61654 Sep 21 22:02 /home/zyw/comfyui_8189_opt.log.bak-0921-1402
-rw-rw-r-- 1 zyw zyw 122254 Sep 22 02:35 /home/zyw/comfyui_8189_opt.log.bak-0921-1835
-rw-rw-r-- 1 zyw zyw 168741 Sep 24 21:29 /home/zyw/comfyui_8189_opt.log.bak-0924-1329
-rw-rw-r-- 1 zyw zyw 251562 Sep 25 22:41 /home/zyw/comfyui_8189_opt.log.bak-0925-1441
-rw-rw-r-- 1 zyw zyw 47281 Sep 26 23:41 /home/zyw/comfyui_8189_opt.log.bak-0926-1541
-rw-rw-r-- 1 zyw zyw 89836 Sep 27 02:13 /home/zyw/comfyui_8189_opt.log.bak-0926-1813
-rw-rw-r-- 1 zyw zyw 22278 Sep 27 02:14 /home/zyw/comfyui_8189_opt.log.bak-0926-1814
-rw-rw-r-- 1 zyw zyw 41488 Sep 27 03:19 /home/zyw/comfyui_8189_opt.log.bak-0926-1919
-rw-rw-r-- 1 zyw zyw 33794 Sep 27 03:36 /home/zyw/comfyui_8189_opt.log.bak-0926-1936
-rw-rw-r-- 1 zyw zyw 45395 Sep 27 21:53 /home/zyw/comfyui_8189_opt.log.bak-0927-1353
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom
-rw-rw-r-- 1 zyw zyw 193806 Sep 28 11:30 /home/zyw/ram_guard.log
-rwxrwxr-x 1 zyw zyw 854 Sep 17 20:14 /home/zyw/ram_guard.sh
--- ram_guard 日志 ---
2026-09-28 10:52:55 HB avail=27914MB swapfree=73727MB
2026-09-28 10:54:55 HB avail=25016MB swapfree=73727MB
2026-09-28 10:56:55 HB avail=24497MB swapfree=73727MB
2026-09-28 10:58:55 HB avail=24472MB swapfree=73727MB
2026-09-28 11:00:55 HB avail=24422MB swapfree=73727MB
2026-09-28 11:02:55 HB avail=21638MB swapfree=73727MB
2026-09-28 11:04:55 HB avail=21613MB swapfree=73727MB
2026-09-28 11:06:55 HB avail=21627MB swapfree=73727MB
2026-09-28 11:08:55 HB avail=21583MB swapfree=73727MB
2026-09-28 11:10:55 HB avail=21624MB swapfree=73727MB
2026-09-28 11:12:55 HB avail=21625MB swapfree=73727MB
2026-09-28 11:14:55 HB avail=21559MB swapfree=73727MB
2026-09-28 11:16:55 HB avail=21559MB swapfree=73727MB
2026-09-28 11:18:55 HB avail=21611MB swapfree=73727MB
2026-09-28 11:20:55 HB avail=21554MB swapfree=73727MB
2026-09-28 11:22:55 HB avail=20976MB swapfree=73714MB
2026-09-28 11:24:56 HB avail=20543MB swapfree=72497MB
2026-09-28 11:26:56 HB avail=20553MB swapfree=72497MB
2026-09-28 11:28:56 HB avail=24271MB swapfree=72582MB
2026-09-28 11:30:56 HB avail=23915MB swapfree=72582MB
########## 摘要（自动提取）##########
===== 2. 上一次启动的内核日志（找 Xid / GSP / 掉总线 / OOM）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b -1 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|out of memory|hung_task|watchdog|BUG:|panic" | tail -60
Sep 28 11:58:16 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b 0 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|hung_task|BUG:|panic" | tail -40
Sep 28 17:41:49 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 nvidia-smi -q 2>/dev/null | grep -iE "Product Name|Driver Version|CUDA Version|Power Limit|Temperature|GPU Current Temp|Clocks Throttle|Bus Id|PCIe Generation|Retired|Remapped|Xid|ECC" | head -30
===== 7. NVreg 参数与内核 panic 设置 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 cat /sys/module/nvidia/parameters/NVreg_DynamicPowerManagement 2>/dev/null; echo "--- sysctl ---"; sysctl kernel.panic kernel.panic_on_oops kernel.hung_task_panic kernel.hung_task_timeout_secs 2>/dev/null; echo "--- mem ---"; free -g
kernel.panic = 10
kernel.panic_on_oops = 0
kernel.hung_task_panic = 1
kernel.hung_task_timeout_secs = 300
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom