Page MenuHomeVyOS Platform

snmpd messages in log with nightly "vyos-999.201709032137-amd64.iso"
Closed, ResolvedPublicBUG

Description

I get every 5min the following 2messages:

Sep  4 12:39:42 router snmpd[3239]: ioctl 35123 returned -1
Sep  4 12:39:42 router snmpd[3239]: message repeated 2 times: [ ioctl 35123 returned -1]

It's "vyos-999.201709032137" on AWS

If you need more info, please ask

thanks
Line2

Details

Difficulty level
Unknown (require assessment)
Version
vyos-999.201709032137
Why the issue appeared?
Will be filled on close

Event Timeline

no problem:

firewall {
    all-ping enable
    broadcast-ping disable
    config-trap enable
    group {
        network-group XXXXX_AWS-VPC1-LAN1_NET {
            description "XXXXX AWS-VPC1 LAN1 private"
            network xxx.xxx.100.0/24
        }
        network-group XXXXX_AWS-VPC1-Public_NET {
            description "XXXXX AWS-VPC1 public"
            network xxx.xxx.101.0/27
        }
        network-group XXXXX_HQ_SUPERNET {
            description "XXXXX Site XY"
            network xxx.xxx.8.0/21
        }
        network-group RFC1918 {
            description "private networks"
            network xxx.xxx.0.0/16
            network xxx.xxx.0.0/12
            network xxx.xxx.0.0/8
        }
    }
    ipv6-receive-redirects disable
    ipv6-src-route disable
    ip-src-route disable
    log-martians enable
    name lan1-local {
        default-action drop
        description "LAN1 to local"
        enable-default-log
        rule 50 {
            action accept
            description "ICMP echo allowed"
            icmp {
                type-name echo-request
            }
            log disable
            protocol icmp
        }
        rule 60 {
            action accept
            description DNS
            destination {
                port 53
            }
            log disable
            protocol udp
        }
    }
    name public-in {
        default-action drop
        description "public ingoing"
        enable-default-log
        rule 100 {
            action accept
            description "IPSec traffic allowed"
            ipsec {
                match-ipsec
            }
        }
    }
    name public-local {
        default-action drop
        description "public to local"
        enable-default-log
        rule 40 {
            action accept
            description "ESP allow"
            disable
            protocol esp
        }
        rule 42 {
            action accept
            description "IKE allow"
            destination {
                port 500
            }
            protocol udp
        }
        rule 44 {
            action accept
            description "NATT allow"
            destination {
                port 4500
            }
            protocol udp
        }
        rule 50 {
            action accept
            description "IPSec allow"
            ipsec {
                match-ipsec
            }
        }
        rule 58 {
            action drop
            description "only 3 new connections in 30sec"
            destination {
                port 22
            }
            protocol tcp
            recent {
                count 3
                time 30
            }
            state {
                new enable
            }
        }
        rule 60 {
            action accept
            description "SSH, allowed source set in AWS security group"
            destination {
                port 22
            }
            protocol tcp
        }
        rule 70 {
            action accept
            description "ICMP, allowed source set in AWS security group"
            icmp {
                type-name echo-request
            }
            protocol icmp
        }
    }
    name public-out {
        default-action drop
        description "public outgoing"
        enable-default-log
        rule 100 {
            action accept
            description "not to LAN1"
            destination {
                address !xxx.xxx.100.0/24
            }
        }
    }
    receive-redirects disable
    send-redirects enable
    source-validation disable
    state-policy {
        established {
            action accept
        }
        invalid {
            action drop
        }
        related {
            action accept
        }
    }
    syn-cookies enable
    twa-hazards-protection disable
}
interfaces {
    ethernet eth0 {
        address dhcp
        description public
        duplex auto
        firewall {
            in {
                name public-in
            }
            local {
                name public-local
            }
            out {
                name public-out
            }
        }
        hw-id XX:XX:XX:5a:e2:d1
        smp-affinity auto
        speed auto
    }
    ethernet eth1 {
        address xxx.xxx.100.10/24
        description lan1
        duplex auto
        firewall {
            local {
                name lan1-local
            }
        }
        hw-id XX:XX:XX:8a:8d:e7
        smp-affinity auto
        speed auto
    }
    loopback lo {
        address xxx.xxx.0.100/32
    }
}
nat {
    source {
        rule 5000 {
            destination {
                address xxx.xxx.0.0/16
            }
            exclude
            outbound-interface eth0
            source {
                address xxx.xxx.100.0/23
            }
        }
        rule 5002 {
            outbound-interface eth0
            translation {
                address masquerade
            }
        }
    }
}
protocols {
    static {
        route xxx.xxx.0.0/8 {
            blackhole {
                distance 254
            }
        }
        route xxx.xxx.0.0/12 {
            blackhole {
                distance 254
            }
        }
        route xxx.xxx.0.0/16 {
            blackhole {
                distance 254
            }
        }
    }
}
service {
    dns {
        forwarding {
            cache-size 150
            dhcp eth0
            domain XXXXX.tld {
                server xxxxx.tld
            }
            listen-on eth1
            listen-on lo
        }
    }
    lldp {
        interface eth1 {
        }
        snmp {
            enable
        }
    }
    snmp {
        contact "XX"
        description XX
        location xxxxxx 
        trap-target xxx.xxx.10.45 {
        }
        v3 {
            engineid 0x8100af8980f286be1718314c5800002000
            group nmsreadonly {
                mode ro
                seclevel priv
                view all
            }
            user xxxxxx {
                auth {
                    encrypted-key ****************
                    type sha
                }
                engineid 0x8100af8980f286be1718314c5800002000
                group nmsreadonly
                mode ro
                privacy {
                    encrypted-key ****************
                    type aes
                }
            }
            view all {
                oid 1 {
                }
            }
        }
    }
    ssh {
        disable-password-authentication
        port 22
    }
}
system {
    config-management {
        commit-revisions 20
    }
    console {
        device ttyS0 {
            speed 9600
        }
    }
    domain-name xxxxxx
    gateway-address xxx.xxx.101.1
    host-name xxxxxx
    login {
        banner {
            post-login "XX"
        }
        user xxxxxx {
            authentication {
                encrypted-password xxxxxx
                plaintext-password xxxxxx
                public-keys [email protected] {
                    key xxxxxx
                    type ssh-rsa
                }
            }
            level admin
        }
    }
    name-server xxx.xxx.0.1
    ntp {
        server xxxxx.tld {
        }
        server xxxxx.tld {
        }
        server xxxxx.tld {
        }
        server xxxxx.tld {
            prefer
        }
    }
    package {
        auto-sync 1
        repository community {
            components main
            distribution helium
            password xxxxxx
            url http://packages.vyos.net/vyos
            username xxxxxx
        }
    }
    syslog {
        global {
            facility all {
                level notice
            }
            facility protocols {
                level debug
            }
        }
        host xxx.xxx.10.45 {
            facility all {
                level info
            }
        }
    }
    time-zone Europe/Zurich
}
vpn {
    ipsec {
        auto-update 60
        esp-group esp1 {
            compression disable
            lifetime 3600
            mode tunnel
            pfs dh-group5
            proposal 1 {
                encryption 3des
                hash sha256
            }
        }
        ike-group ike1 {
            dead-peer-detection {
                action restart
                interval 30
                timeout 120
            }
            ikev2-reauth no
            key-exchange ikev2
            lifetime 28800
            proposal 1 {
                dh-group 5
                encryption 3des
                hash sha256
            }
        }
        ipsec-interfaces {
            interface eth0
        }
        site-to-site {
            peer xxxxx.tld {
                authentication {
                    id @x2.tld
                    mode x509
                    remote-id @x1.tld
                    x509 {
                        ca-cert-file xxxxxx
                        cert-file xxxxxx
                        key xxxxxx
                            file /config/auth/xx.key
                        }
                    }
                }
                connection-type respond
                default-esp-group esp1
                description "Tunnel to XXXXX-HQ"
                ike-group ike1
                ikev2-reauth inherit
                local-address xxx.xxx.101.10
                tunnel 1 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    local {
                        prefix xxx.xxx.100.0/23
                    }
                    remote {
                        prefix xxx.xxx.8.0/21
                    }
                }
                tunnel 5 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    local {
                        prefix xxx.xxx.100.0/23
                    }
                    remote {
                        prefix xxx.xxx.50.0/23
                    }
                }
            }
        }
    }
}
syncer triaged this task as Normal priority.Dec 21 2017, 9:50 PM

I am seeing similar messages in 1.1.8.

Jan 12 16:30:32 vyos snmpd[9958]: ioctl 35123 returned -1
Jan 12 16:30:32 vyos snmpd[9958]: ioctl 35123 returned -1
Jan 12 16:30:32 vyos snmpd[9958]: IfIndex of an interface changed. Such interfaces will appear multiple times in IF-MIB.
Jan 12 16:30:32 vyos snmpd[9958]: ioctl 35111 returned -1
Jan 12 16:30:32 vyos snmpd[9958]: ioctl 35091 returned -1
Jan 12 16:30:32 vyos snmpd[9958]: ioctl 35105 returned -1
Jan 12 16:30:49 vyos snmpd[9958]: ioctl 35123 returned -1
Jan 12 16:30:49 vyos snmpd[9958]: ioctl 35123 returned -1
Jan 12 16:30:49 vyos snmpd[9958]: ioctl 35111 returned -1
Jan 12 16:30:49 vyos snmpd[9958]: ioctl 35091 returned -1
Jan 12 16:30:49 vyos snmpd[9958]: ioctl 35105 returned -1
Jan 12 16:30:49 vyos snmpd[9958]: Name of an interface changed. Such interfaces will keep its old name in IF-MIB.

I have another issue (most likely unrelated but I figure I would mention it) where if I run show interfaces it takes ~30 seconds to complete and uses about 40% CPU. If I kill snmpd the command completes with no issues. This is similar to the issue described here.

I am actually currently running almost same config on 1.1.7 in production with no issues and was rebuilding the config in 1.1.8 on new hardware when I discovered this behavior.

I see the same messages in EdgeOS 1.10.3 if that helps

syncer changed the task status from Open to On hold.Oct 13 2018, 7:19 PM

requires testing on the latest rolling

I retested on 1.2.0-rolling-201810240337, the same log:

Oct 24 16:07:05 vyos10 snmpd[3485]: ioctl 35123 returned -1
Oct 24 16:07:05 vyos10 snmpd[3485]: message repeated 2 times: [ ioctl 35123 returned -1]
Oct 24 16:12:10 vyos10 snmpd[3485]: ioctl 35123 returned -1
Oct 24 16:12:10 vyos10 snmpd[3485]: message repeated 2 times: [ ioctl 35123 returned -1]
Oct 24 16:17:15 vyos10 snmpd[3485]: ioctl 35123 returned -1
Oct 24 16:17:15 vyos10 snmpd[3485]: message repeated 2 times: [ ioctl 35123 returned -1]
Oct 24 16:22:20 vyos10 snmpd[3485]: ioctl 35123 returned -1

I saw a few problems like this in net-snmp but all years ago, there was a problem with interface names longer than 8 characters.

I made a few further tests. These errors occure in log every 5min. That's the routing-table refresh intervall of the network monitoring system. I deactivated encryption of snmp and captured the traffic in these moments. Now I can reproduce this by a snmpwalk to OID .iso.org.dod.internet.mgmt.mib-2.ip.ipRouteTable.ipRouteEntry.ipRouteDest (.1.3.6.1.2.1.4.21.1.1)

I just retested on VyOS 1.2.0-rolling+201811240337. Problem is gone.