########## GPU 机恢复取证 20260928_104833 ##########
===== 1. 当前状态 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 hostname; uptime; echo "--- who ---"; who; echo "--- last reboot ---"; last -x reboot 2>/dev/null | head -5
zyw@192.168.31.31's password:
zywpc
10:48:34 up 1 min, 2 users, load average: 5.70, 1.60, 0.55
--- who ---
zyw tty1 Sep 28 10:48
--- last reboot ---
reboot system boot 6.8.0-142-generi Mon Sep 28 10:47 still running
reboot system boot 6.8.0-142-generi Mon Sep 28 02:52 still running
reboot system boot 6.8.0-142-generi Sun Sep 27 20:38 still running
reboot system boot 6.8.0-142-generi Sun Sep 27 03:35 still running
reboot system boot 6.8.0-142-generi Fri Sep 25 18:57 still running
===== 2. 上一次启动的内核日志（找 Xid / GSP / 掉总线 / OOM）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b -1 --no-pager 2>/dev/null | tail -200
zyw@192.168.31.31's password:
Sep 28 02:52:05 zywpc systemd[1]: Reached target snapd.mounts.target - Mounted snaps.
Sep 28 02:52:05 zywpc systemd[1]: Reached target veritysetup.target - Local Verity Protected Volumes.
Sep 28 02:52:05 zywpc systemd[1]: Listening on dm-event.socket - Device-mapper event daemon FIFOs.
Sep 28 02:52:05 zywpc systemd[1]: Listening on lvm2-lvmpolld.socket - LVM2 poll daemon socket.
Sep 28 02:52:05 zywpc systemd[1]: Listening on multipathd.socket - multipathd control socket.
Sep 28 02:52:05 zywpc systemd[1]: Listening on syslog.socket - Syslog Socket.
Sep 28 02:52:05 zywpc systemd[1]: Listening on systemd-fsckd.socket - fsck to fsckd communication Socket.
Sep 28 02:52:05 zywpc systemd[1]: Listening on systemd-initctl.socket - initctl Compatibility Named Pipe.
Sep 28 02:52:05 zywpc systemd[1]: Listening on systemd-journald-dev-log.socket - Journal Socket (/dev/log).
Sep 28 02:52:05 zywpc systemd[1]: Listening on systemd-journald.socket - Journal Socket.
Sep 28 02:52:05 zywpc systemd[1]: Listening on systemd-networkd.socket - Network Service Netlink Socket.
Sep 28 02:52:05 zywpc systemd[1]: systemd-pcrextend.socket - TPM2 PCR Extension (Varlink) was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Sep 28 02:52:05 zywpc systemd[1]: Listening on systemd-udevd-control.socket - udev Control Socket.
Sep 28 02:52:05 zywpc systemd[1]: Listening on systemd-udevd-kernel.socket - udev Kernel Socket.
Sep 28 02:52:05 zywpc systemd[1]: Mounting dev-hugepages.mount - Huge Pages File System...
Sep 28 02:52:05 zywpc systemd[1]: Mounting dev-mqueue.mount - POSIX Message Queue File System...
Sep 28 02:52:05 zywpc systemd[1]: Mounting sys-kernel-debug.mount - Kernel Debug File System...
Sep 28 02:52:05 zywpc systemd[1]: Mounting sys-kernel-tracing.mount - Kernel Trace File System...
Sep 28 02:52:05 zywpc systemd[1]: Starting systemd-journald.service - Journal Service...
Sep 28 02:52:05 zywpc systemd[1]: Starting keyboard-setup.service - Set the console keyboard layout...
Sep 28 02:52:05 zywpc systemd[1]: Starting kmod-static-nodes.service - Create List of Static Device Nodes...
Sep 28 02:52:05 zywpc systemd[1]: Starting lvm2-monitor.service - Monitoring of LVM2 mirrors, snapshots etc. using dmeventd or progress polling...
Sep 28 02:52:05 zywpc systemd[1]: Starting modprobe@configfs.service - Load Kernel Module configfs...
Sep 28 02:52:05 zywpc systemd[1]: Starting modprobe@dm_mod.service - Load Kernel Module dm_mod...
Sep 28 02:52:05 zywpc systemd[1]: Starting modprobe@drm.service - Load Kernel Module drm...
Sep 28 02:52:05 zywpc systemd[1]: Starting modprobe@efi_pstore.service - Load Kernel Module efi_pstore...
Sep 28 02:52:05 zywpc systemd[1]: Starting modprobe@fuse.service - Load Kernel Module fuse...
Sep 28 02:52:05 zywpc systemd[1]: Starting modprobe@loop.service - Load Kernel Module loop...
Sep 28 02:52:05 zywpc systemd[1]: netplan-ovs-cleanup.service - OpenVSwitch configuration for cleanup was skipped because of an unmet condition check (ConditionFileIsExecutable=/usr/bin/ovs-vsctl).
Sep 28 02:52:05 zywpc systemd[1]: systemd-fsck-root.service - File System Check on Root Device was skipped because of an unmet condition check (ConditionPathExists=!/run/initramfs/fsck-root).
Sep 28 02:52:05 zywpc systemd[1]: Starting systemd-modules-load.service - Load Kernel Modules...
Sep 28 02:52:05 zywpc systemd[1]: systemd-pcrmachine.service - TPM2 PCR Machine ID Measurement was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Sep 28 02:52:05 zywpc systemd[1]: Starting systemd-remount-fs.service - Remount Root and Kernel File Systems...
Sep 28 02:52:05 zywpc systemd[1]: systemd-tpm2-setup-early.service - TPM2 SRK Setup (Early) was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Sep 28 02:52:05 zywpc systemd[1]: Starting systemd-udev-trigger.service - Coldplug All udev Devices...
Sep 28 02:52:05 zywpc systemd[1]: Mounted dev-hugepages.mount - Huge Pages File System.
Sep 28 02:52:05 zywpc systemd[1]: Mounted dev-mqueue.mount - POSIX Message Queue File System.
Sep 28 02:52:05 zywpc systemd[1]: Mounted sys-kernel-debug.mount - Kernel Debug File System.
Sep 28 02:52:05 zywpc systemd[1]: Mounted sys-kernel-tracing.mount - Kernel Trace File System.
Sep 28 02:52:05 zywpc systemd[1]: Finished kmod-static-nodes.service - Create List of Static Device Nodes.
Sep 28 02:52:05 zywpc systemd[1]: modprobe@configfs.service: Deactivated successfully.
Sep 28 02:52:05 zywpc systemd[1]: Finished modprobe@configfs.service - Load Kernel Module configfs.
Sep 28 02:52:05 zywpc systemd[1]: modprobe@dm_mod.service: Deactivated successfully.
Sep 28 02:52:05 zywpc systemd[1]: Finished modprobe@dm_mod.service - Load Kernel Module dm_mod.
Sep 28 02:52:05 zywpc systemd[1]: modprobe@drm.service: Deactivated successfully.
Sep 28 02:52:05 zywpc systemd[1]: Finished modprobe@drm.service - Load Kernel Module drm.
Sep 28 02:52:05 zywpc systemd[1]: modprobe@fuse.service: Deactivated successfully.
Sep 28 02:52:05 zywpc systemd[1]: Finished modprobe@fuse.service - Load Kernel Module fuse.
Sep 28 02:52:05 zywpc systemd[1]: modprobe@loop.service: Deactivated successfully.
Sep 28 02:52:05 zywpc systemd[1]: Finished modprobe@loop.service - Load Kernel Module loop.
Sep 28 02:52:05 zywpc systemd[1]: Mounting sys-fs-fuse-connections.mount - FUSE Control File System...
Sep 28 02:52:05 zywpc kernel: pstore: Using crash dump compression: deflate
Sep 28 02:52:05 zywpc systemd[1]: Mounting sys-kernel-config.mount - Kernel Configuration File System...
Sep 28 02:52:05 zywpc systemd[1]: systemd-repart.service - Repartition Root Disk was skipped because no trigger condition checks were met.
Sep 28 02:52:05 zywpc systemd[1]: Starting systemd-tmpfiles-setup-dev-early.service - Create Static Device Nodes in /dev gracefully...
Sep 28 02:52:05 zywpc systemd-journald[480]: Collecting audit messages is disabled.
Sep 28 02:52:05 zywpc systemd[1]: Mounted sys-fs-fuse-connections.mount - FUSE Control File System.
Sep 28 02:52:05 zywpc kernel: pstore: Registered efi_pstore as persistent store backend
Sep 28 02:52:05 zywpc systemd[1]: modprobe@efi_pstore.service: Deactivated successfully.
Sep 28 02:52:05 zywpc systemd[1]: Finished modprobe@efi_pstore.service - Load Kernel Module efi_pstore.
Sep 28 02:52:05 zywpc systemd[1]: Mounted sys-kernel-config.mount - Kernel Configuration File System.
Sep 28 02:52:05 zywpc systemd[1]: Started systemd-journald.service - Journal Service.
Sep 28 02:52:05 zywpc kernel: EXT4-fs (sda5): re-mounted 5cea3463-6633-42ac-9011-cee5516972f1 r/w.
Sep 28 02:52:06 zywpc systemd-journald[480]: Received client request to flush runtime journal.
Sep 28 02:52:06 zywpc systemd-journald[480]: File /var/log/journal/cab04c94c96f4a58b6b86fcc2420b27e/system.journal corrupted or uncleanly shut down, renaming and replacing.
Sep 28 02:52:06 zywpc kernel: Adding 8388604k swap on /swap.img. Priority:-2 extents:67 across:340049920k SS
Sep 28 02:52:06 zywpc kernel: Adding 67108860k swap on /swap2.img. Priority:-3 extents:40 across:666304512k SS
Sep 28 02:52:06 zywpc kernel: nvidia: loading out-of-tree module taints kernel.
Sep 28 02:52:06 zywpc kernel: nvidia: module verification failed: signature and/or required key missing - tainting kernel
Sep 28 02:52:06 zywpc kernel: nvidia-nvlink: Nvlink Core is being initialized, major device number 238
Sep 28 02:52:06 zywpc kernel:
Sep 28 02:52:06 zywpc kernel: nvidia 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=none:owns=none
Sep 28 02:52:06 zywpc kernel: NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:24:27 UTC 2026
Sep 28 02:52:06 zywpc kernel: intel_pmc_core INT33A1:00: initialized
Sep 28 02:52:06 zywpc kernel: nvidia-modeset: Loading NVIDIA UNIX Open Kernel Mode Setting Driver for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:12:42 UTC 2026
Sep 28 02:52:06 zywpc kernel: i801_smbus 0000:00:1f.4: SPD Write Disable is set
Sep 28 02:52:06 zywpc kernel: i801_smbus 0000:00:1f.4: SMBus using PCI interrupt
Sep 28 02:52:06 zywpc kernel: [drm] [nvidia-drm] [GPU ID 0x00000100] Loading driver
Sep 28 02:52:06 zywpc kernel: mei_me 0000:00:16.0: enabling device (0000 -> 0002)
Sep 28 02:52:06 zywpc kernel: RAPL PMU: API unit is 2^-32 Joules, 3 fixed counters, 655360 ms ovfl timer
Sep 28 02:52:06 zywpc kernel: RAPL PMU: hw unit of domain pp0-core 2^-14 Joules
Sep 28 02:52:06 zywpc kernel: RAPL PMU: hw unit of domain package 2^-14 Joules
Sep 28 02:52:06 zywpc kernel: RAPL PMU: hw unit of domain pp1-gpu 2^-14 Joules
Sep 28 02:52:06 zywpc kernel: i2c i2c-0: 1/4 memory slots populated (from DMI)
Sep 28 02:52:06 zywpc kernel: i2c i2c-0: Successfully instantiated SPD at 0x52
Sep 28 02:52:06 zywpc kernel: ee1004 0-0052: 512 byte EE1004-compliant SPD EEPROM, read-only
Sep 28 02:52:06 zywpc kernel: Creating 1 MTD partitions on "0000:00:1f.5":
Sep 28 02:52:06 zywpc kernel: 0x000000000000-0x000001000000 : "BIOS"
Sep 28 02:52:06 zywpc kernel: snd_hda_intel 0000:00:1f.3: enabling device (0000 -> 0002)
Sep 28 02:52:06 zywpc kernel: snd_hda_intel 0000:01:00.1: Disabling MSI
Sep 28 02:52:06 zywpc kernel: snd_hda_intel 0000:01:00.1: Handle vga_switcheroo audio client
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: enabling device (0006 -> 0007)
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] VT-d active for gfx access
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] Using Transparent Hugepages
Sep 28 02:52:06 zywpc kernel: mei_hdcp 0000:00:16.0-b638ab7e-94e2-4ea2-a552-d1c54b627f04: bound 0000:00:02.0 (ops i915_hdcp_ops [i915])
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] Finished loading DMC firmware i915/adls_dmc_ver2_01.bin (v2.1)
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] GT0: GuC firmware i915/tgl_guc_70.bin version 70.36.0
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] GT0: HuC firmware i915/tgl_huc.bin version 7.9.3
Sep 28 02:52:06 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=3 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input15
Sep 28 02:52:06 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=7 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input16
Sep 28 02:52:06 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=8 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input17
Sep 28 02:52:06 zywpc kernel: input: HDA NVidia HDMI/DP,pcm=9 as /devices/pci0000:00/0000:00:01.0/0000:01:00.1/sound/card0/input18
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] GT0: HuC: authenticated for all workloads
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] GT0: GUC: submission disabled
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] GT0: GUC: SLPC disabled
Sep 28 02:52:06 zywpc kernel: mei_pxp 0000:00:16.0-fbf6fcf1-96cf-4e2e-a6a6-1bab8cbe36b1: bound 0000:00:02.0 (ops i915_pxp_tee_component_ops [i915])
Sep 28 02:52:06 zywpc kernel: i915 0000:00:02.0: [drm] Protected Xe Path (PXP) protected content support initialized
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:2): apparmor="STATUS" operation="profile_load" profile="unconfined" name="1password" pid=749 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:3): apparmor="STATUS" operation="profile_load" profile="unconfined" name="balena-etcher" pid=753 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:4): apparmor="STATUS" operation="profile_load" profile="unconfined" name="Discord" pid=750 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:5): apparmor="STATUS" operation="profile_load" profile="unconfined" name="vscode" pid=761 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:6): apparmor="STATUS" operation="profile_load" profile="unconfined" name="devhelp" pid=763 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:7): apparmor="STATUS" operation="profile_load" profile="unconfined" name="element-desktop" pid=764 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:8): apparmor="STATUS" operation="profile_load" profile="unconfined" name="busybox" pid=756 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:9): apparmor="STATUS" operation="profile_load" profile="unconfined" name="crun" pid=762 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:10): apparmor="STATUS" operation="profile_load" profile="unconfined" name="brave" pid=754 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: audit: type=1400 audit(1790535126.628:11): apparmor="STATUS" operation="profile_load" profile="unconfined" name="ch-run" pid=759 comm="apparmor_parser"
Sep 28 02:52:06 zywpc kernel: cfg80211: Loading compiled-in X.509 certificates for regulatory database
Sep 28 02:52:06 zywpc kernel: Loaded X.509 cert 'sforshee: 00b28ddf47aef9cea7'
Sep 28 02:52:06 zywpc kernel: Loaded X.509 cert 'wens: 61c038651aabdcf94bd0ac7ff06c7248db18c600'
Sep 28 02:52:06 zywpc kernel: RTL8226B_RTL8221B 2.5Gbps PHY r8169-0-200:00: attached PHY driver (mii_bus:phy_addr=r8169-0-200:00, irq=MAC)
Sep 28 02:52:06 zywpc kernel: r8169 0000:02:00.0 enp2s0: Link is Down
Sep 28 02:52:07 zywpc kernel: [drm] Initialized i915 1.6.0 20230929 for 0000:00:02.0 on minor 1
Sep 28 02:52:07 zywpc kernel: i915 display info: display version: 12
Sep 28 02:52:07 zywpc kernel: i915 display info: cursor_needs_physical: no
Sep 28 02:52:07 zywpc kernel: i915 display info: has_cdclk_crawl: no
Sep 28 02:52:07 zywpc kernel: i915 display info: has_cdclk_squash: no
Sep 28 02:52:07 zywpc kernel: i915 display info: has_ddi: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_dp_mst: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_dsb: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_fpga_dbg: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_gmch: no
Sep 28 02:52:07 zywpc kernel: i915 display info: has_hotplug: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_hti: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_ipc: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_overlay: no
Sep 28 02:52:07 zywpc kernel: i915 display info: has_psr: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_psr_hw_tracking: no
Sep 28 02:52:07 zywpc kernel: i915 display info: overlay_needs_physical: no
Sep 28 02:52:07 zywpc kernel: i915 display info: supports_tv: no
Sep 28 02:52:07 zywpc kernel: i915 display info: has_hdcp: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_dmc: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: has_dsc: yes
Sep 28 02:52:07 zywpc kernel: i915 display info: rawclk rate: 19200 kHz
Sep 28 02:52:07 zywpc kernel: snd_hda_intel 0000:00:1f.3: bound 0000:00:02.0 (ops i915_audio_component_bind_ops [i915])
Sep 28 02:52:07 zywpc kernel: Console: switching to colour frame buffer device 160x45
Sep 28 02:52:07 zywpc kernel: i915 0000:00:02.0: [drm] fb0: i915drmfb frame buffer device
Sep 28 02:52:07 zywpc kernel: intel_tcc_cooling: TCC Offset locked
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: autoconfig for ALC897: line_outs=1 (0x14/0x0/0x0/0x0/0x0) type:line
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: speaker_outs=0 (0x0/0x0/0x0/0x0/0x0)
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: hp_outs=1 (0x1b/0x0/0x0/0x0/0x0)
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: mono: mono_out=0x0
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: inputs:
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: Rear Mic=0x18
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: Front Mic=0x19
Sep 28 02:52:07 zywpc kernel: snd_hda_codec_realtek hdaudioC1D0: Line=0x1a
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH Rear Mic as /devices/pci0000:00/0000:00:1f.3/sound/card1/input19
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH Front Mic as /devices/pci0000:00/0000:00:1f.3/sound/card1/input20
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH Line as /devices/pci0000:00/0000:00:1f.3/sound/card1/input21
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH Line Out as /devices/pci0000:00/0000:00:1f.3/sound/card1/input22
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH Front Headphone as /devices/pci0000:00/0000:00:1f.3/sound/card1/input23
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=3 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input24
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=7 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input25
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=8 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input26
Sep 28 02:52:07 zywpc kernel: input: HDA Intel PCH HDMI/DP,pcm=9 as /devices/pci0000:00/0000:00:1f.3/sound/card1/input27
Sep 28 02:52:07 zywpc kernel: intel_rapl_msr: PL4 support detected.
Sep 28 02:52:07 zywpc kernel: intel_rapl_common: Found RAPL domain package
Sep 28 02:52:07 zywpc kernel: intel_rapl_common: Found RAPL domain core
Sep 28 02:52:07 zywpc kernel: intel_rapl_common: Found RAPL domain uncore
Sep 28 02:52:07 zywpc kernel: [drm] Initialized nvidia-drm 0.0.0 20160202 for 0000:01:00.0 on minor 0
Sep 28 02:52:07 zywpc kernel: nvidia 0000:01:00.0: vgaarb: deactivate vga console
Sep 28 02:52:07 zywpc kernel: nvidia 0000:01:00.0: [drm] Cannot find any crtc or sizes
Sep 28 02:52:08 zywpc kernel: NET: Registered PF_QIPCRTR protocol family
Sep 28 02:52:08 zywpc kernel: loop0: detected capacity change from 0 to 8
Sep 28 02:52:09 zywpc kernel: r8169 0000:02:00.0 enp2s0: Link is Up - 1Gbps/Full - flow control rx/tx
Sep 28 02:52:11 zywpc systemd-journald[480]: File /var/log/journal/cab04c94c96f4a58b6b86fcc2420b27e/user-1000.journal corrupted or uncleanly shut down, renaming and replacing.
Sep 28 02:52:11 zywpc kernel: kauditd_printk_skb: 110 callbacks suppressed
Sep 28 02:52:11 zywpc kernel: audit: type=1400 audit(1790535131.736:122): apparmor="STATUS" operation="profile_load" profile="unconfined" name="docker-default" pid=1534 comm="apparmor_parser"
Sep 28 02:52:12 zywpc kernel: Initializing XFRM netlink socket
Sep 28 02:52:12 zywpc kernel: bridge: filtering via arp/ip/ip6tables is no longer available by default. Update your scripts to load br_netfilter if you need this.
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 1(veth01fad75) entered blocking state
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 1(veth01fad75) entered disabled state
Sep 28 02:52:13 zywpc kernel: veth01fad75: entered allmulticast mode
Sep 28 02:52:13 zywpc kernel: veth01fad75: entered promiscuous mode
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 2(veth7194878) entered blocking state
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 2(veth7194878) entered disabled state
Sep 28 02:52:13 zywpc kernel: veth7194878: entered allmulticast mode
Sep 28 02:52:13 zywpc kernel: veth7194878: entered promiscuous mode
Sep 28 02:52:13 zywpc kernel: eth0: renamed from veth27d01be
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 1(veth01fad75) entered blocking state
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 1(veth01fad75) entered forwarding state
Sep 28 02:52:13 zywpc kernel: eth0: renamed from veth379a1fe
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 2(veth7194878) entered blocking state
Sep 28 02:52:13 zywpc kernel: br-b03fdbdfcf4b: port 2(veth7194878) entered forwarding state
Sep 28 02:52:13 zywpc kernel: evm: overlay not supported
Sep 28 02:52:16 zywpc kernel: wireguard: WireGuard 1.0.0 loaded. See www.wireguard.com for information.
Sep 28 02:52:16 zywpc kernel: wireguard: Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
Sep 28 02:54:13 zywpc kernel: nvidia 0000:01:00.0: Using 39-bit DMA addresses
Sep 28 04:19:31 zywpc systemd-journald[480]: Under memory pressure, flushing caches.
Sep 28 04:55:24 zywpc kernel: perf: interrupt took too long (2512 > 2500), lowering kernel.perf_event_max_sample_rate to 79000
===== 3. 上次启动日志里的关键错误 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b -1 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|out of memory|hung_task|watchdog|BUG:|panic" | tail -60
zyw@192.168.31.31's password:
Sep 28 02:52:05 zywpc kernel: NMI watchdog: Enabled. Permanently consumes one hw-PMU counter.
Sep 28 02:52:05 zywpc kernel: ACPI FADT declares the system doesn't support PCIe ASPM, so disable it
Sep 28 02:52:05 zywpc kernel: acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug SHPCHotplug PME AER PCIeCapability LTR DPC]
Sep 28 02:52:05 zywpc kernel: pci 0000:00:01.0: [8086:460d] type 01 class 0x060400 PCIe Root Port
Sep 28 02:52:05 zywpc kernel: pci 0000:00:02.0: [8086:4688] type 00 class 0x038000 PCIe Root Complex Integrated Endpoint
Sep 28 02:52:05 zywpc kernel: pci 0000:00:0a.0: [8086:467d] type 00 class 0x118000 PCIe Root Complex Integrated Endpoint
Sep 28 02:52:05 zywpc kernel: pci 0000:00:1d.0: [8086:7ab7] type 01 class 0x060400 PCIe Root Port
Sep 28 02:52:05 zywpc kernel: pci 0000:01:00.0: [10de:2d04] type 00 class 0x030000 PCIe Legacy Endpoint
Sep 28 02:52:05 zywpc kernel: pci 0000:01:00.0: 252.056 Gb/s available PCIe bandwidth, limited by 32.0 GT/s PCIe x8 link at 0000:00:01.0 (capable of 504.112 Gb/s with 32.0 GT/s PCIe x16 link)
Sep 28 02:52:05 zywpc kernel: pci 0000:01:00.1: [10de:22eb] type 00 class 0x040300 PCIe Endpoint
Sep 28 02:52:05 zywpc kernel: pci 0000:02:00.0: [10ec:8125] type 00 class 0x020000 PCIe Endpoint
Sep 28 02:52:05 zywpc kernel: pcieport 0000:00:01.0: PME: Signaling with IRQ 122
Sep 28 02:52:05 zywpc kernel: pcieport 0000:00:1d.0: PME: Signaling with IRQ 123
Sep 28 02:52:05 zywpc kernel: pcieport 0000:00:1d.0: AER: enabled with IRQ 123
Sep 28 02:52:05 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
Sep 28 02:52:06 zywpc kernel: NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:24:27 UTC 2026
===== 4. 当前启动的内核错误 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b 0 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|hung_task|BUG:|panic" | tail -40
zyw@192.168.31.31's password:
Sep 28 10:47:42 zywpc kernel: ACPI FADT declares the system doesn't support PCIe ASPM, so disable it
Sep 28 10:47:42 zywpc kernel: acpi PNP0A08:00: _OSC: OS now controls [PCIeHotplug SHPCHotplug PME AER PCIeCapability LTR DPC]
Sep 28 10:47:42 zywpc kernel: pci 0000:00:01.0: [8086:460d] type 01 class 0x060400 PCIe Root Port
Sep 28 10:47:43 zywpc kernel: pci 0000:00:02.0: [8086:4688] type 00 class 0x038000 PCIe Root Complex Integrated Endpoint
Sep 28 10:47:43 zywpc kernel: pci 0000:00:0a.0: [8086:467d] type 00 class 0x118000 PCIe Root Complex Integrated Endpoint
Sep 28 10:47:43 zywpc kernel: pci 0000:00:1d.0: [8086:7ab7] type 01 class 0x060400 PCIe Root Port
Sep 28 10:47:43 zywpc kernel: pci 0000:01:00.0: [10de:2d04] type 00 class 0x030000 PCIe Legacy Endpoint
Sep 28 10:47:43 zywpc kernel: pci 0000:01:00.0: 126.024 Gb/s available PCIe bandwidth, limited by 16.0 GT/s PCIe x8 link at 0000:00:01.0 (capable of 504.112 Gb/s with 32.0 GT/s PCIe x16 link)
Sep 28 10:47:43 zywpc kernel: pci 0000:01:00.1: [10de:22eb] type 00 class 0x040300 PCIe Endpoint
Sep 28 10:47:43 zywpc kernel: pci 0000:02:00.0: [10ec:8125] type 00 class 0x020000 PCIe Endpoint
Sep 28 10:47:43 zywpc kernel: pcieport 0000:00:01.0: PME: Signaling with IRQ 122
Sep 28 10:47:43 zywpc kernel: pcieport 0000:00:1d.0: PME: Signaling with IRQ 123
Sep 28 10:47:43 zywpc kernel: pcieport 0000:00:1d.0: AER: enabled with IRQ 123
Sep 28 10:47:43 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
Sep 28 10:47:46 zywpc kernel: NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64 610.43.02 Release Build (dvs-builder@U22-I3-H05-01-2) Tue May 19 11:24:27 UTC 2026
===== 5. journald 是否持久化（-b -1 能否用）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 ls -la /var/log/journal 2>/dev/null | head -3; grep -E "^\s*Storage=" /etc/systemd/journald.conf 2>/dev/null; journalctl --list-boots --no-pager 2>/dev/null | tail -5
zyw@192.168.31.31's password:
total 28
drwxr-sr-x+ 4 root systemd-journal 4096 Sep 27 20:56 .
drwxrwxr-x 11 root syslog 4096 Sep 28 10:47 ..
-4 464721c5273d470f8e83c8af0e4d1b75 Fri 2026-09-25 18:57:47 CST Sun 2026-09-27 03:32:30 CST
-3 e2b989b00b404389b612387ead995053 Sun 2026-09-27 03:35:35 CST Sun 2026-09-27 20:35:32 CST
-2 2bb558e59c3e4b2c8865ff7b0fb759d0 Sun 2026-09-27 20:38:51 CST Mon 2026-09-28 02:26:57 CST
-1 5a19a7c6874a4816b70028da41af13d7 Mon 2026-09-28 02:52:05 CST Mon 2026-09-28 05:04:39 CST
0 e62b9594a0e7488c9fdd9009d20fa9a7 Mon 2026-09-28 10:47:42 CST Mon 2026-09-28 10:49:09 CST
===== 6. GPU 现状 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 nvidia-smi -q 2>/dev/null | grep -iE "Product Name|Driver Version|CUDA Version|Power Limit|Temperature|GPU Current Temp|Clocks Throttle|Bus Id|PCIe Generation|Retired|Remapped|Xid|ECC" | head -30
zyw@192.168.31.31's password:
Driver Version : 610.43.02 [Deprecated; will be removed in CUDA 14.0. Use KMD Version instead]
CUDA Version : 13.3 [Deprecated; will be removed in CUDA 14.0. Use CUDA UMD Version instead]
Product Name : NVIDIA GeForce RTX 5060 Ti
ECC Object : N/A
Bus Id : 00000000:01:00.0
PCIe Generation
ECC Mode
ECC Errors
Retired Pages
Single Bit ECC : N/A
Double Bit ECC : N/A
Remapped Rows : N/A
Temperature
GPU Current Temp : 42 C
GPU Target Temperature Specification : N/A
GPU Ceiling Power Limit
Current Power Limit : 180.00 W
Requested Power Limit : 180.00 W
Default Power Limit : 180.00 W
Min Power Limit : 150.00 W
Max Power Limit : 180.00 W
Current Power Limit : N/A
Requested Power Limit : N/A
Default Power Limit : N/A
Min Power Limit : N/A
Max Power Limit : N/A
===== 7. NVreg 参数与内核 panic 设置 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 cat /sys/module/nvidia/parameters/NVreg_DynamicPowerManagement 2>/dev/null; echo "--- sysctl ---"; sysctl kernel.panic kernel.panic_on_oops kernel.hung_task_panic kernel.hung_task_timeout_secs 2>/dev/null; echo "--- mem ---"; free -g
zyw@192.168.31.31's password:
--- sysctl ---
kernel.panic = 10
kernel.panic_on_oops = 0
kernel.hung_task_panic = 1
kernel.hung_task_timeout_secs = 300
--- mem ---
total used free shared buff/cache available
Mem: 30 1 28 0 1 29
Swap: 71 0 71
===== 8. ComfyUI 日志尾部（死亡时的最后动作）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 ls -la ~/comfyui_8189_opt.log* 2>/dev/null; echo "---"; tail -c 3000 ~/comfyui_8189_opt.log 2>/dev/null
zyw@192.168.31.31's password:
-rw-rw-r-- 1 zyw zyw 48683 Sep 28 05:02 /home/zyw/comfyui_8189_opt.log
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:55 /home/zyw/comfyui_8189_opt.log.bak-0915-1955
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:56 /home/zyw/comfyui_8189_opt.log.bak-0915-1956
-rw-rw-r-- 1 zyw zyw 22941 Sep 16 04:21 /home/zyw/comfyui_8189_opt.log.bak-0915-2021
-rw-rw-r-- 1 zyw zyw 27263 Sep 16 04:39 /home/zyw/comfyui_8189_opt.log.bak-0915-2039
-rw-rw-r-- 1 zyw zyw 24377 Sep 16 04:42 /home/zyw/comfyui_8189_opt.log.bak-0915-2042
-rw-rw-r-- 1 zyw zyw 31368 Sep 16 12:56 /home/zyw/comfyui_8189_opt.log.bak-0916-0456
-rw-rw-r-- 1 zyw zyw 43775 Sep 16 14:48 /home/zyw/comfyui_8189_opt.log.bak-0916-0648
-rw-rw-r-- 1 zyw zyw 115390 Sep 17 18:44 /home/zyw/comfyui_8189_opt.log.bak-0917-1044
-rw-rw-r-- 1 zyw zyw 49507 Sep 17 20:31 /home/zyw/comfyui_8189_opt.log.bak-0917-1231
-rw-rw-r-- 1 zyw zyw 53620 Sep 18 01:46 /home/zyw/comfyui_8189_opt.log.bak-0917-1746
-rw-rw-r-- 1 zyw zyw 4253 Sep 18 01:47 /home/zyw/comfyui_8189_opt.log.bak-0917-1747
-rw-rw-r-- 1 zyw zyw 183939 Sep 19 15:38 /home/zyw/comfyui_8189_opt.log.bak-0919-0738
-rw-rw-r-- 1 zyw zyw 43176 Sep 19 18:10 /home/zyw/comfyui_8189_opt.log.bak-0919-1010
-rw-rw-r-- 1 zyw zyw 38378 Sep 19 19:51 /home/zyw/comfyui_8189_opt.log.bak-0919-1151
-rw-rw-r-- 1 zyw zyw 69814 Sep 20 03:15 /home/zyw/comfyui_8189_opt.log.bak-0919-1915
-rw-rw-r-- 1 zyw zyw 81821 Sep 21 12:59 /home/zyw/comfyui_8189_opt.log.bak-0921-0459
-rw-rw-r-- 1 zyw zyw 8084 Sep 21 13:00 /home/zyw/comfyui_8189_opt.log.bak-0921-0500
-rw-rw-r-- 1 zyw zyw 82281 Sep 21 20:25 /home/zyw/comfyui_8189_opt.log.bak-0921-1225
-rw-rw-r-- 1 zyw zyw 35146 Sep 21 21:12 /home/zyw/comfyui_8189_opt.log.bak-0921-1312
-rw-rw-r-- 1 zyw zyw 26036 Sep 21 21:14 /home/zyw/comfyui_8189_opt.log.bak-0921-1314
-rw-rw-r-- 1 zyw zyw 61654 Sep 21 22:02 /home/zyw/comfyui_8189_opt.log.bak-0921-1402
-rw-rw-r-- 1 zyw zyw 122254 Sep 22 02:35 /home/zyw/comfyui_8189_opt.log.bak-0921-1835
-rw-rw-r-- 1 zyw zyw 168741 Sep 24 21:29 /home/zyw/comfyui_8189_opt.log.bak-0924-1329
-rw-rw-r-- 1 zyw zyw 251562 Sep 25 22:41 /home/zyw/comfyui_8189_opt.log.bak-0925-1441
-rw-rw-r-- 1 zyw zyw 47281 Sep 26 23:41 /home/zyw/comfyui_8189_opt.log.bak-0926-1541
-rw-rw-r-- 1 zyw zyw 89836 Sep 27 02:13 /home/zyw/comfyui_8189_opt.log.bak-0926-1813
-rw-rw-r-- 1 zyw zyw 22278 Sep 27 02:14 /home/zyw/comfyui_8189_opt.log.bak-0926-1814
-rw-rw-r-- 1 zyw zyw 41488 Sep 27 03:19 /home/zyw/comfyui_8189_opt.log.bak-0926-1919
-rw-rw-r-- 1 zyw zyw 33794 Sep 27 03:36 /home/zyw/comfyui_8189_opt.log.bak-0926-1936
-rw-rw-r-- 1 zyw zyw 45395 Sep 27 21:53 /home/zyw/comfyui_8189_opt.log.bak-0927-1353
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom
---
4:36, 138.27s/it]
88%|████████▊ | 7/8 [16:09<02:18, 138.32s/it]
100%|██████████| 8/8 [18:27<00:00, 138.31s/it]
100%|██████████| 8/8 [18:27<00:00, 138.44s/it]
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3VideoVAE prepared for dynamic VRAM loading. 2677MB Staged. 0 patches attached. Force pre-loaded 128 weights: 697 KB.
[32m[INFO][0m Model MiniMaxH3AudioVAE prepared for dynamic VRAM loading. 576MB Staged. 0 patches attached. Force pre-loaded 401 weights: 539 KB.
[32m[INFO][0m Comfy model compiler graph breaks: 1, rogues: 0
[32m[INFO][0m [32mPrompt executed in 00:27:17[0m
[32m[INFO][0m got prompt
[32m[INFO][0m Model MiniMaxH3VideoVAE prepared for dynamic VRAM loading. 2677MB Staged. 0 patches attached. Force pre-loaded 128 weights: 697 KB.
[32m[INFO][0m Model MiniMaxH3VideoVAE prepared for dynamic VRAM loading. 2677MB Staged. 0 patches attached. Force pre-loaded 128 weights: 697 KB.
[32m[INFO][0m Model MiniMaxH3TEModel_ prepared for dynamic VRAM loading. 14956MB Staged. 0 patches attached. Force pre-loaded 410 weights: 4572 KB.
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3 prepared for dynamic VRAM loading. 19995MB Staged. 208 patches attached. Force pre-loaded 210 weights: 1175 KB.
0%| | 0/3 [00:00<?, ?it/s]
0%| | 0/3 [00:00<?, ?it/s, Model Initializing ... ]
0%| | 0/3 [01:37<?, ?it/s, Model Initialization complete! ]
33%|███▎ | 1/3 [01:37<03:15, 97.52s/it, Model Initialization complete! ]
33%|███▎ | 1/3 [02:39<03:15, 97.52s/it]
67%|██████▋ | 2/3 [02:39<01:19, 79.83s/it]
100%|██████████| 3/3 [03:59<00:00, 79.84s/it]
100%|██████████| 3/3 [03:59<00:00, 79.84s/it]
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3VideoVAE prepared for dynamic VRAM loading. 2677MB Staged. 0 patches attached. Force pre-loaded 128 weights: 697 KB.
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3AudioVAE prepared for dynamic VRAM loading. 576MB Staged. 0 patches attached. Force pre-loaded 401 weights: 539 KB.
[32m[INFO][0m Comfy model compiler graph breaks: 2, rogues: 0
[32m[INFO][0m Requested to load MiniMaxH3
[32m[INFO][0m 0 models unloaded.
[32m[INFO][0m Model MiniMaxH3 prepared for dynamic VRAM loading. 19995MB Staged. 208 patches attached. Force pre-loaded 210 weights: 1175 KB.
[MiniMax H3 Aicg] 二采放大 1.2x：(1, 24, 37, 86, 48) -> (1, 24, 37, 104, 58)
0%| | 0/8 [00:00<?, ?it/s]
0%| | 0/8 [00:00<?, ?it/s, Model Initializing ... ]
0%| | 0/8 [03:00<?, ?it/s, Model Initialization complete! ]
12%|█▎ | 1/8 [03:00<21:06, 180.90s/it, Model Initialization complete! ]
12%|█▎ | 1/8 [04:53<21:06, 180.90s/it]
25%|██▌ | 2/8 [04:53<14:40, 146.76s/it]
===== 9. 是否有 .bak 轮转 / ram_guard 痕迹 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 ls -la ~/comfyui_8189_opt.log* ~/ram_guard* 2>/dev/null; echo "--- ram_guard 日志 ---"; tail -20 ~/ram_guard.log 2>/dev/null
zyw@192.168.31.31's password:
-rw-rw-r-- 1 zyw zyw 48683 Sep 28 05:02 /home/zyw/comfyui_8189_opt.log
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:55 /home/zyw/comfyui_8189_opt.log.bak-0915-1955
-rw-rw-r-- 1 zyw zyw 39190 Sep 16 03:56 /home/zyw/comfyui_8189_opt.log.bak-0915-1956
-rw-rw-r-- 1 zyw zyw 22941 Sep 16 04:21 /home/zyw/comfyui_8189_opt.log.bak-0915-2021
-rw-rw-r-- 1 zyw zyw 27263 Sep 16 04:39 /home/zyw/comfyui_8189_opt.log.bak-0915-2039
-rw-rw-r-- 1 zyw zyw 24377 Sep 16 04:42 /home/zyw/comfyui_8189_opt.log.bak-0915-2042
-rw-rw-r-- 1 zyw zyw 31368 Sep 16 12:56 /home/zyw/comfyui_8189_opt.log.bak-0916-0456
-rw-rw-r-- 1 zyw zyw 43775 Sep 16 14:48 /home/zyw/comfyui_8189_opt.log.bak-0916-0648
-rw-rw-r-- 1 zyw zyw 115390 Sep 17 18:44 /home/zyw/comfyui_8189_opt.log.bak-0917-1044
-rw-rw-r-- 1 zyw zyw 49507 Sep 17 20:31 /home/zyw/comfyui_8189_opt.log.bak-0917-1231
-rw-rw-r-- 1 zyw zyw 53620 Sep 18 01:46 /home/zyw/comfyui_8189_opt.log.bak-0917-1746
-rw-rw-r-- 1 zyw zyw 4253 Sep 18 01:47 /home/zyw/comfyui_8189_opt.log.bak-0917-1747
-rw-rw-r-- 1 zyw zyw 183939 Sep 19 15:38 /home/zyw/comfyui_8189_opt.log.bak-0919-0738
-rw-rw-r-- 1 zyw zyw 43176 Sep 19 18:10 /home/zyw/comfyui_8189_opt.log.bak-0919-1010
-rw-rw-r-- 1 zyw zyw 38378 Sep 19 19:51 /home/zyw/comfyui_8189_opt.log.bak-0919-1151
-rw-rw-r-- 1 zyw zyw 69814 Sep 20 03:15 /home/zyw/comfyui_8189_opt.log.bak-0919-1915
-rw-rw-r-- 1 zyw zyw 81821 Sep 21 12:59 /home/zyw/comfyui_8189_opt.log.bak-0921-0459
-rw-rw-r-- 1 zyw zyw 8084 Sep 21 13:00 /home/zyw/comfyui_8189_opt.log.bak-0921-0500
-rw-rw-r-- 1 zyw zyw 82281 Sep 21 20:25 /home/zyw/comfyui_8189_opt.log.bak-0921-1225
-rw-rw-r-- 1 zyw zyw 35146 Sep 21 21:12 /home/zyw/comfyui_8189_opt.log.bak-0921-1312
-rw-rw-r-- 1 zyw zyw 26036 Sep 21 21:14 /home/zyw/comfyui_8189_opt.log.bak-0921-1314
-rw-rw-r-- 1 zyw zyw 61654 Sep 21 22:02 /home/zyw/comfyui_8189_opt.log.bak-0921-1402
-rw-rw-r-- 1 zyw zyw 122254 Sep 22 02:35 /home/zyw/comfyui_8189_opt.log.bak-0921-1835
-rw-rw-r-- 1 zyw zyw 168741 Sep 24 21:29 /home/zyw/comfyui_8189_opt.log.bak-0924-1329
-rw-rw-r-- 1 zyw zyw 251562 Sep 25 22:41 /home/zyw/comfyui_8189_opt.log.bak-0925-1441
-rw-rw-r-- 1 zyw zyw 47281 Sep 26 23:41 /home/zyw/comfyui_8189_opt.log.bak-0926-1541
-rw-rw-r-- 1 zyw zyw 89836 Sep 27 02:13 /home/zyw/comfyui_8189_opt.log.bak-0926-1813
-rw-rw-r-- 1 zyw zyw 22278 Sep 27 02:14 /home/zyw/comfyui_8189_opt.log.bak-0926-1814
-rw-rw-r-- 1 zyw zyw 41488 Sep 27 03:19 /home/zyw/comfyui_8189_opt.log.bak-0926-1919
-rw-rw-r-- 1 zyw zyw 33794 Sep 27 03:36 /home/zyw/comfyui_8189_opt.log.bak-0926-1936
-rw-rw-r-- 1 zyw zyw 45395 Sep 27 21:53 /home/zyw/comfyui_8189_opt.log.bak-0927-1353
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom
-rw-rw-r-- 1 zyw zyw 192672 Sep 27 03:31 /home/zyw/ram_guard.log
-rwxrwxr-x 1 zyw zyw 854 Sep 17 20:14 /home/zyw/ram_guard.sh
--- ram_guard 日志 ---
2026-09-26 18:53:07 HB avail=24312MB swapfree=72902MB
2026-09-26 18:55:07 HB avail=24329MB swapfree=72902MB
2026-09-26 18:57:07 HB avail=24377MB swapfree=72902MB
2026-09-26 18:59:07 HB avail=24321MB swapfree=72902MB
2026-09-26 19:01:07 HB avail=24373MB swapfree=72902MB
2026-09-26 19:03:07 HB avail=24349MB swapfree=72902MB
2026-09-26 19:05:07 HB avail=24362MB swapfree=72902MB
2026-09-26 19:07:07 HB avail=24333MB swapfree=72902MB
2026-09-26 19:09:07 HB avail=24271MB swapfree=72902MB
2026-09-26 19:11:07 HB avail=24299MB swapfree=72902MB
2026-09-26 19:13:07 HB avail=24230MB swapfree=72902MB
2026-09-26 19:15:07 HB avail=24306MB swapfree=72902MB
2026-09-26 19:17:07 HB avail=24260MB swapfree=72902MB
2026-09-26 19:19:07 HB avail=29880MB swapfree=72902MB
2026-09-26 19:21:08 HB avail=26851MB swapfree=72902MB
2026-09-26 19:23:08 HB avail=26117MB swapfree=72898MB
2026-09-26 19:25:08 HB avail=26079MB swapfree=72898MB
2026-09-26 19:27:08 HB avail=24445MB swapfree=72898MB
2026-09-26 19:29:08 HB avail=24435MB swapfree=72898MB
2026-09-26 19:31:08 HB avail=26059MB swapfree=72898MB
########## 摘要（自动提取）##########
===== 2. 上一次启动的内核日志（找 Xid / GSP / 掉总线 / OOM）=====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b -1 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|out of memory|hung_task|watchdog|BUG:|panic" | tail -60
Sep 28 02:52:05 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 journalctl -k -b 0 --no-pager 2>/dev/null | grep -iE "xid|nvrm|gsp|fell off|pcie|aer|oom|hung_task|BUG:|panic" | tail -40
Sep 28 10:47:43 zywpc kernel: r8169 0000:02:00.0 eth0: RTL8125B, 00:e0:4c:d3:03:a4, XID 641, IRQ 133
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 nvidia-smi -q 2>/dev/null | grep -iE "Product Name|Driver Version|CUDA Version|Power Limit|Temperature|GPU Current Temp|Clocks Throttle|Bus Id|PCIe Generation|Retired|Remapped|Xid|ECC" | head -30
===== 7. NVreg 参数与内核 panic 设置 =====
spawn ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 zyw@192.168.31.31 cat /sys/module/nvidia/parameters/NVreg_DynamicPowerManagement 2>/dev/null; echo "--- sysctl ---"; sysctl kernel.panic kernel.panic_on_oops kernel.hung_task_panic kernel.hung_task_timeout_secs 2>/dev/null; echo "--- mem ---"; free -g
kernel.panic = 10
kernel.panic_on_oops = 0
kernel.hung_task_panic = 1
kernel.hung_task_timeout_secs = 300
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom
-rw-rw-r-- 1 zyw zyw 46954 Sep 14 22:39 /home/zyw/comfyui_8189_opt.log.bak-oom