Skip to main content

ksoftirqd consuming high cpu 100%

This happens to be a post from an experience on a Ubuntu machine... although the machine was practically unused, an entire CPU was consumed (by ksoftirqd)

Resolution:
# date -s "`date`"

root@wwwsrv01:~#  uname -a
Linux wwwsrv01 2.6.32-32-server #62-Ubuntu SMP Wed Apr 20 22:07:43 UTC 2011 x86_64 GNU/Linux

root@wwwsrv01:~# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 10.04.2 LTS
Release: 10.04
Codename: lucid

root@wwwsrv01:~# top

 top - 23:02:11 up 491 days, 12:35,  1 user,  load average: 1.45, 1.37, 1.30
Tasks: 206 total,   2 running, 204 sleeping,   0 stopped,   0 zombie
Cpu(s):  6.3%us, 12.0%sy,  0.0%ni, 81.6%id,  0.0%wa,  0.0%hi,  0.1%si,  0.0%st
Mem:   8253648k total,  7464608k used,   789040k free,   448916k buffers
Swap: 19771384k total,     3140k used, 19768244k free,  5471676k cached

   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                                 
    7 root      20   0     0    0    0 R   12  0.0  22251:58 ksoftirqd/1                                                             
   13 root      20   0     0    0    0 S   12  0.0  23523:10 ksoftirqd/3                                                             
    4 root      20   0     0    0    0 S   11  0.0  20969:03 ksoftirqd/0                                                             
   10 root      20   0     0    0    0 S   10  0.0  20349:29 ksoftirqd/2 

root@wwwsrv01:~# cat /proc/interrupts 
            CPU0       CPU1       CPU2       CPU3       CPU4       CPU5       CPU6       CPU7       
   0:     141593          0          0          0          0          0          0          0   IO-APIC-edge      timer
   1:          2          0          0          0          0          0          0          0   IO-APIC-edge      i8042
   7:          2          0          0          0          0          0          0          0   IO-APIC-edge    
   8:          1          0          0          0          0          0          0          0   IO-APIC-edge      rtc0
   9:         75          0          0          0          0          0          0          0   IO-APIC-fasteoi   acpi
  10:         91          0          0          0          0          0          0          0   IO-APIC-fasteoi   sata_svw
  11:        123          0          0          0          0          0          0          0   IO-APIC-fasteoi   ehci_hcd:usb1, ohci_hcd:usb2, ohci_hcd:usb3
  12:          4          0          0          0          0          0          0          0   IO-APIC-edge      i8042
  37:      18129  156077862      26272          0          0          0          0          0   IO-APIC-fasteoi   megasas
  80:          0          0          0          0          0          0          0          0   PCI-MSI-edge      aerdrv
  81:          0          0          0          0          0          0          0          0   PCI-MSI-edge      aerdrv
  82:          0          0          0          0          0          0          0          0   PCI-MSI-edge      aerdrv
  83:          0          0          0          0          0          0          0          0   PCI-MSI-edge      aerdrv
  84:          0          0          0          0          0          0          0          0   PCI-MSI-edge      aerdrv
  89:  709933902          0          0          0          0          0          0          0   PCI-MSI-edge      eth0
 NMI:          0          0          0          0          0          0          0          0   Non-maskable interrupts
 LOC: 2256128861 2589012639 3627874637 3826038395 3949501889 2439241460   26457107 4261066274   Local timer interrupts
 SPU:          0          0          0          0          0          0          0          0   Spurious interrupts
 PMI:          0          0          0          0          0          0          0          0   Performance monitoring interrupts
 PND:          0          0          0          0          0          0          0          0   Performance pending work
 RES: 1621900349  345575279 2713531435 3786193932 3684814249 1064081969   47543173   28060846   Rescheduling interrupts
 CAL:  161318244       5250        190        234        211        221        229        231   Function call interrupts
 TLB:    1207114    2695308    3909688   14876996    1122807    1581252     684022     551056   TLB shootdowns
 TRM:          0          0          0          0          0          0          0          0   Thermal event interrupts
 THR:          0          0          0          0          0          0          0          0   Threshold APIC interrupts
 MCE:          0          0          0          0          0          0          0          0   Machine check exceptions
 MCP:     141549     141549     141549     141549     141549     141549     141549     141549   Machine check polls
 ERR:          2
 MIS:          0

root@wwwsrv01:~# date -s "`date`"

root@wwwsrv01:~# top

 top - 23:15:04 up 491 days, 12:48,  1 user,  load average: 0.21, 0.90, 1.13
Tasks: 205 total,   1 running, 204 sleeping,   0 stopped,   0 zombie
Cpu(s):  0.0%us,  0.1%sy,  0.0%ni, 99.8%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Mem:   8253648k total,  7467608k used,   786040k free,   448928k buffers
Swap: 19771384k total,     3140k used, 19768244k free,  5472860k cached

   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                                 
16436 www-data  20   0 22464 3852 1912 S    1  0.0   0:00.02 getuploadworksv                                                         
   27 root      20   0     0    0    0 S    0  0.0 116:26.99 events/0                                                                
16432 root      20   0 42152 1280  912 S    0  0.0   0:00.01 cron                                                                    
16437 www-data  20   0 37208 2160 1700 S    0  0.0   0:00.01 sendmail                                                                
16440 www-data  20   0 37196 2152 1688 S    0  0.0   0:00.01 postdrop                                                                
    1 root      20   0 23712 1832 1196 S    0  0.0   0:01.88 init                                                                    
    2 root      20   0     0    0    0 S    0  0.0   0:00.01 kthreadd                                                                
    3 root      RT   0     0    0    0 S    0  0.0   0:01.20 migration/0                                                             
    4 root      20   0     0    0    0 S    0  0.0  20970:11 ksoftirqd/0   
Labels:

Comments

Post a Comment

Popular posts from this blog

P2V using dd for KVM-QEMU guest

Preface: I have certainly not exhaustively tested this process.  I had a specific need and found a specific solution that worked. Situation:  I was issued a shiny new laptop running Red Hat Enterprise Linux 7 (with Corp VPN, certs, Authentication configuration, etc...)  The image was great, but I needed more flexibility on my bare metal.  So, my goal was to P2V the corporate image so I could just run it as a VM. * Remove corporate drive and install new SSD * install corp drive in external USB-3 case * Install RHEL 7 on new SSD * dd old drive to a disk-image file in a temp location which will be an image which is the same size as your actual drive (unless you have enough space in your destination to contain a temp and converted image) * convert the raw disk-image to a qcow file while pushing it to the final location - this step should reduce the disk size - however, I believe it will only reduce/collapse zero-byte blocks (not just free space - i.e. if you de...
1 comment
Read more

Sun USS 7100 foo

TIP: put ALL of your LUNs into a designated TARGET and INITIATOR group when you create them.  If you leave them in the "default" group, then everything that does an discovery against the array will find them :-( I'm struggling to recognize a reason that a default should even be present on the array. Also - who, exactly, is Sun trying to kid.  The USS is simply a box.. running Solaris .. with IPMP and ZFS.  Great.  If you have ever attempted to "break-in" or "p0wn" your IBM HMC, you know that there are people out there that can harden a box - then.. there's Sun.  After a recent meltdown at the office I had to get quite intimate with my USS 7110 and learned quite a bit.  Namely: there's a shell ;-) My current irritation is how they attempt to "warn you" away from using the shell (my coverage expired a long time ago to worry about that) and then how they try to hide things, poorly. I was curious as to what version of SunOS it ...
1 comment
Read more

Extending SNMP to run arbitrary shell script

Why are we here... This is not likely something I would have pursued under normal circumstances.  I happen to be working for a customer/client who is not afforded a lot of flexibility to accomplish their goals.  In this case, the rigor is justified.  They have to sometimes be fairly creative with how they solve problems. In this case they would like to utilize an existing snmp implementation to execute a command (or shell script) on a remote system.  They came to me with the idea of using Net-SNMP extend. https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/sect-System_Monitoring_Tools-Net-SNMP-Extending.html NOTE:  This is NOT a good implementation strategy in the "real world"  it will simply allow you to test the functionality.  There are a TON of security implications which would need to be taken in to consideration. Implementation Steps: [root@rh7tst01 ~]# yum -y install net-snmp net-snmp-utils ...
Post a Comment
Read more