Page MenuHomeVyOS Platform

Reduce telegraf binary size
Closed, ResolvedPublic

Description

Relating to T4358

Telegraf binary in debian package is 144MB as it includes all available plugins. I think we should build our own Telegraf binary with only a select amount of plugins to reduce the footprint on images.

https://community.influxdata.com/t/bigger-size-of-the-telegraf-binary/22299/3
https://github.com/influxdata/telegraf/blob/master/plugins/inputs/all/all.go
https://github.com/influxdata/telegraf/blob/master/plugins/outputs/all/all.go

Details

Difficulty level
Unknown (require assessment)
Version
-
Why the issue appeared?
Will be filled on close
Is it a breaking change?
Perfectly compatible
Issue type
Improvement (missing useful functionality)

Event Timeline

sdev triaged this task as Wishlist priority.Jul 7 2022, 11:01 AM
sdev created this task.
sdev created this object in space S1 VyOS Public.

Output plugins that we use in 1.4

"github.com/influxdata/telegraf/plugins/outputs/azure_data_explorer"
"github.com/influxdata/telegraf/plugins/outputs/influxdb_v2"
"github.com/influxdata/telegraf/plugins/outputs/prometheus_client"
"github.com/influxdata/telegraf/plugins/outputs/http"
(07:27) vyos_bld 8214ca94b6bc:/vyos/telegraf # ls -alh telegraf
-rwxr-xr-x 1 vyos_bld vyos_bld 157M Jul  9 07:08 telegraf

(07:31) vyos_bld 8214ca94b6bc:/vyos/telegraf # vim plugins/outputs/all/all.go

(07:32) vyos_bld 8214ca94b6bc:/vyos/telegraf # LDFLAGS=-w make
go mod download -x
env -u GOOS -u GOARCH -u GOARM -- go build -o ./tools/readme_config_includer/generator ./tools/readme_config_includer/generator.go
go generate -run="readme_config_includer/generator$" ./plugins/inputs/...
go generate -run="readme_config_includer/generator$" ./plugins/outputs/...
go generate -run="readme_config_includer/generator$" ./plugins/processors/...
go generate -run="readme_config_includer/generator$" ./plugins/aggregators/...
go build -ldflags " -X main.commit=686717fe -X main.branch=HEAD -X main.goos=linux -X main.goarch=amd64 -X main.version=1.23.1" ./cmd/telegraf

(07:32) vyos_bld 8214ca94b6bc:/vyos/telegraf # ls -alh telegraf
-rwxr-xr-x 1 vyos_bld vyos_bld 52M Jul  9 07:32 telegraf
diff --git i/plugins/inputs/all/all.go w/plugins/inputs/all/all.go
index 549397d24..7b7c45b91 100644
--- i/plugins/inputs/all/all.go
+++ w/plugins/inputs/all/all.go
@@ -2,39 +2,15 @@ package all

 import (
        //Blank imports for plugins to register themselves
-       _ "github.com/influxdata/telegraf/plugins/inputs/activemq"
-       _ "github.com/influxdata/telegraf/plugins/inputs/aerospike"
-       _ "github.com/influxdata/telegraf/plugins/inputs/aliyuncms"
-       _ "github.com/influxdata/telegraf/plugins/inputs/amd_rocm_smi"
-       _ "github.com/influxdata/telegraf/plugins/inputs/amqp_consumer"
-       _ "github.com/influxdata/telegraf/plugins/inputs/apache"
        _ "github.com/influxdata/telegraf/plugins/inputs/apcupsd"
-       _ "github.com/influxdata/telegraf/plugins/inputs/aurora"
        _ "github.com/influxdata/telegraf/plugins/inputs/azure_storage_queue"
        _ "github.com/influxdata/telegraf/plugins/inputs/bcache"
-       _ "github.com/influxdata/telegraf/plugins/inputs/beanstalkd"
        _ "github.com/influxdata/telegraf/plugins/inputs/beat"
-       _ "github.com/influxdata/telegraf/plugins/inputs/bind"
        _ "github.com/influxdata/telegraf/plugins/inputs/bond"
-       _ "github.com/influxdata/telegraf/plugins/inputs/burrow"
-       _ "github.com/influxdata/telegraf/plugins/inputs/cassandra"
-       _ "github.com/influxdata/telegraf/plugins/inputs/ceph"
        _ "github.com/influxdata/telegraf/plugins/inputs/cgroup"
        _ "github.com/influxdata/telegraf/plugins/inputs/chrony"
-       _ "github.com/influxdata/telegraf/plugins/inputs/cisco_telemetry_mdt"
-       _ "github.com/influxdata/telegraf/plugins/inputs/clickhouse"
-       _ "github.com/influxdata/telegraf/plugins/inputs/cloud_pubsub"
-       _ "github.com/influxdata/telegraf/plugins/inputs/cloud_pubsub_push"
-       _ "github.com/influxdata/telegraf/plugins/inputs/cloudwatch"
        _ "github.com/influxdata/telegraf/plugins/inputs/conntrack"
-       _ "github.com/influxdata/telegraf/plugins/inputs/consul"
-       _ "github.com/influxdata/telegraf/plugins/inputs/consul_agent"
-       _ "github.com/influxdata/telegraf/plugins/inputs/couchbase"
-       _ "github.com/influxdata/telegraf/plugins/inputs/couchdb"
        _ "github.com/influxdata/telegraf/plugins/inputs/cpu"
-       _ "github.com/influxdata/telegraf/plugins/inputs/csgo"
-       _ "github.com/influxdata/telegraf/plugins/inputs/dcos"
-       _ "github.com/influxdata/telegraf/plugins/inputs/directory_monitor"
        _ "github.com/influxdata/telegraf/plugins/inputs/disk"
        _ "github.com/influxdata/telegraf/plugins/inputs/diskio"
        _ "github.com/influxdata/telegraf/plugins/inputs/disque"
@@ -42,34 +18,19 @@ import (
        _ "github.com/influxdata/telegraf/plugins/inputs/dns_query"
        _ "github.com/influxdata/telegraf/plugins/inputs/docker"
        _ "github.com/influxdata/telegraf/plugins/inputs/docker_log"
-       _ "github.com/influxdata/telegraf/plugins/inputs/dovecot"
-       _ "github.com/influxdata/telegraf/plugins/inputs/dpdk"
-       _ "github.com/influxdata/telegraf/plugins/inputs/ecs"
-       _ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch"
-       _ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch_query"
        _ "github.com/influxdata/telegraf/plugins/inputs/ethtool"
-       _ "github.com/influxdata/telegraf/plugins/inputs/eventhub_consumer"
        _ "github.com/influxdata/telegraf/plugins/inputs/exec"
        _ "github.com/influxdata/telegraf/plugins/inputs/execd"
-       _ "github.com/influxdata/telegraf/plugins/inputs/fail2ban"
-       _ "github.com/influxdata/telegraf/plugins/inputs/fibaro"
        _ "github.com/influxdata/telegraf/plugins/inputs/file"
        _ "github.com/influxdata/telegraf/plugins/inputs/filecount"
        _ "github.com/influxdata/telegraf/plugins/inputs/filestat"
        _ "github.com/influxdata/telegraf/plugins/inputs/fireboard"
-       _ "github.com/influxdata/telegraf/plugins/inputs/fluentd"
-       _ "github.com/influxdata/telegraf/plugins/inputs/github"
-       _ "github.com/influxdata/telegraf/plugins/inputs/gnmi"
-       _ "github.com/influxdata/telegraf/plugins/inputs/graylog"
-       _ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
        _ "github.com/influxdata/telegraf/plugins/inputs/hddtemp"
        _ "github.com/influxdata/telegraf/plugins/inputs/http"
        _ "github.com/influxdata/telegraf/plugins/inputs/http_listener_v2"
        _ "github.com/influxdata/telegraf/plugins/inputs/http_response"
        _ "github.com/influxdata/telegraf/plugins/inputs/httpjson"
        _ "github.com/influxdata/telegraf/plugins/inputs/hugepages"
-       _ "github.com/influxdata/telegraf/plugins/inputs/icinga2"
-       _ "github.com/influxdata/telegraf/plugins/inputs/infiniband"
        _ "github.com/influxdata/telegraf/plugins/inputs/influxdb"
        _ "github.com/influxdata/telegraf/plugins/inputs/influxdb_listener"
        _ "github.com/influxdata/telegraf/plugins/inputs/influxdb_v2_listener"
@@ -83,139 +44,38 @@ import (
        _ "github.com/influxdata/telegraf/plugins/inputs/ipset"
        _ "github.com/influxdata/telegraf/plugins/inputs/iptables"
        _ "github.com/influxdata/telegraf/plugins/inputs/ipvs"
-       _ "github.com/influxdata/telegraf/plugins/inputs/jenkins"
-       _ "github.com/influxdata/telegraf/plugins/inputs/jolokia"
-       _ "github.com/influxdata/telegraf/plugins/inputs/jolokia2"
-       _ "github.com/influxdata/telegraf/plugins/inputs/jti_openconfig_telemetry"
-       _ "github.com/influxdata/telegraf/plugins/inputs/kafka_consumer"
-       _ "github.com/influxdata/telegraf/plugins/inputs/kafka_consumer_legacy"
-       _ "github.com/influxdata/telegraf/plugins/inputs/kapacitor"
        _ "github.com/influxdata/telegraf/plugins/inputs/kernel"
        _ "github.com/influxdata/telegraf/plugins/inputs/kernel_vmstat"
-       _ "github.com/influxdata/telegraf/plugins/inputs/kibana"
-       _ "github.com/influxdata/telegraf/plugins/inputs/kinesis_consumer"
-       _ "github.com/influxdata/telegraf/plugins/inputs/knx_listener"
-       _ "github.com/influxdata/telegraf/plugins/inputs/kube_inventory"
-       _ "github.com/influxdata/telegraf/plugins/inputs/kubernetes"
-       _ "github.com/influxdata/telegraf/plugins/inputs/lanz"
-       _ "github.com/influxdata/telegraf/plugins/inputs/leofs"
        _ "github.com/influxdata/telegraf/plugins/inputs/linux_sysctl_fs"
        _ "github.com/influxdata/telegraf/plugins/inputs/logparser"
-       _ "github.com/influxdata/telegraf/plugins/inputs/logstash"
-       _ "github.com/influxdata/telegraf/plugins/inputs/lustre2"
-       _ "github.com/influxdata/telegraf/plugins/inputs/lvm"
-       _ "github.com/influxdata/telegraf/plugins/inputs/mailchimp"
-       _ "github.com/influxdata/telegraf/plugins/inputs/marklogic"
-       _ "github.com/influxdata/telegraf/plugins/inputs/mcrouter"
        _ "github.com/influxdata/telegraf/plugins/inputs/mdstat"
        _ "github.com/influxdata/telegraf/plugins/inputs/mem"
-       _ "github.com/influxdata/telegraf/plugins/inputs/memcached"
-       _ "github.com/influxdata/telegraf/plugins/inputs/mesos"
-       _ "github.com/influxdata/telegraf/plugins/inputs/minecraft"
-       _ "github.com/influxdata/telegraf/plugins/inputs/mock"
-       _ "github.com/influxdata/telegraf/plugins/inputs/modbus"
-       _ "github.com/influxdata/telegraf/plugins/inputs/mongodb"
-       _ "github.com/influxdata/telegraf/plugins/inputs/monit"
-       _ "github.com/influxdata/telegraf/plugins/inputs/mqtt_consumer"
-       _ "github.com/influxdata/telegraf/plugins/inputs/multifile"
-       _ "github.com/influxdata/telegraf/plugins/inputs/mysql"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nats"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nats_consumer"
-       _ "github.com/influxdata/telegraf/plugins/inputs/neptune_apex"
        _ "github.com/influxdata/telegraf/plugins/inputs/net"
        _ "github.com/influxdata/telegraf/plugins/inputs/net_response"
        _ "github.com/influxdata/telegraf/plugins/inputs/netstat"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nfsclient"
        _ "github.com/influxdata/telegraf/plugins/inputs/nginx"
        _ "github.com/influxdata/telegraf/plugins/inputs/nginx_plus"
        _ "github.com/influxdata/telegraf/plugins/inputs/nginx_plus_api"
        _ "github.com/influxdata/telegraf/plugins/inputs/nginx_sts"
        _ "github.com/influxdata/telegraf/plugins/inputs/nginx_upstream_check"
        _ "github.com/influxdata/telegraf/plugins/inputs/nginx_vts"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nomad"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nsd"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nsq"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nsq_consumer"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nstat"
        _ "github.com/influxdata/telegraf/plugins/inputs/ntpq"
-       _ "github.com/influxdata/telegraf/plugins/inputs/nvidia_smi"
-       _ "github.com/influxdata/telegraf/plugins/inputs/opcua"
-       _ "github.com/influxdata/telegraf/plugins/inputs/openldap"
-       _ "github.com/influxdata/telegraf/plugins/inputs/openntpd"
-       _ "github.com/influxdata/telegraf/plugins/inputs/opensmtpd"
-       _ "github.com/influxdata/telegraf/plugins/inputs/openstack"
-       _ "github.com/influxdata/telegraf/plugins/inputs/opentelemetry"
-       _ "github.com/influxdata/telegraf/plugins/inputs/openweathermap"
-       _ "github.com/influxdata/telegraf/plugins/inputs/passenger"
-       _ "github.com/influxdata/telegraf/plugins/inputs/pf"
-       _ "github.com/influxdata/telegraf/plugins/inputs/pgbouncer"
-       _ "github.com/influxdata/telegraf/plugins/inputs/phpfpm"
        _ "github.com/influxdata/telegraf/plugins/inputs/ping"
-       _ "github.com/influxdata/telegraf/plugins/inputs/postfix"
-       _ "github.com/influxdata/telegraf/plugins/inputs/postgresql"
-       _ "github.com/influxdata/telegraf/plugins/inputs/postgresql_extensible"
-       _ "github.com/influxdata/telegraf/plugins/inputs/powerdns"
        _ "github.com/influxdata/telegraf/plugins/inputs/powerdns_recursor"
        _ "github.com/influxdata/telegraf/plugins/inputs/processes"
        _ "github.com/influxdata/telegraf/plugins/inputs/procstat"
-       _ "github.com/influxdata/telegraf/plugins/inputs/prometheus"
-       _ "github.com/influxdata/telegraf/plugins/inputs/proxmox"
-       _ "github.com/influxdata/telegraf/plugins/inputs/puppetagent"
-       _ "github.com/influxdata/telegraf/plugins/inputs/rabbitmq"
-       _ "github.com/influxdata/telegraf/plugins/inputs/raindrops"
-       _ "github.com/influxdata/telegraf/plugins/inputs/ras"
-       _ "github.com/influxdata/telegraf/plugins/inputs/ravendb"
-       _ "github.com/influxdata/telegraf/plugins/inputs/redfish"
-       _ "github.com/influxdata/telegraf/plugins/inputs/redis"
-       _ "github.com/influxdata/telegraf/plugins/inputs/redis_sentinel"
-       _ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb"
-       _ "github.com/influxdata/telegraf/plugins/inputs/riak"
-       _ "github.com/influxdata/telegraf/plugins/inputs/riemann_listener"
-       _ "github.com/influxdata/telegraf/plugins/inputs/salesforce"
        _ "github.com/influxdata/telegraf/plugins/inputs/sensors"
        _ "github.com/influxdata/telegraf/plugins/inputs/sflow"
-       _ "github.com/influxdata/telegraf/plugins/inputs/slab"
        _ "github.com/influxdata/telegraf/plugins/inputs/smart"
-       _ "github.com/influxdata/telegraf/plugins/inputs/snmp"
-       _ "github.com/influxdata/telegraf/plugins/inputs/snmp_legacy"
-       _ "github.com/influxdata/telegraf/plugins/inputs/snmp_trap"
        _ "github.com/influxdata/telegraf/plugins/inputs/socket_listener"
        _ "github.com/influxdata/telegraf/plugins/inputs/socketstat"
-       _ "github.com/influxdata/telegraf/plugins/inputs/solr"
-       _ "github.com/influxdata/telegraf/plugins/inputs/sql"
-       _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver"
-       _ "github.com/influxdata/telegraf/plugins/inputs/stackdriver"
        _ "github.com/influxdata/telegraf/plugins/inputs/statsd"
-       _ "github.com/influxdata/telegraf/plugins/inputs/suricata"
-       _ "github.com/influxdata/telegraf/plugins/inputs/swap"
-       _ "github.com/influxdata/telegraf/plugins/inputs/synproxy"
        _ "github.com/influxdata/telegraf/plugins/inputs/syslog"
        _ "github.com/influxdata/telegraf/plugins/inputs/sysstat"
        _ "github.com/influxdata/telegraf/plugins/inputs/system"
        _ "github.com/influxdata/telegraf/plugins/inputs/systemd_units"
        _ "github.com/influxdata/telegraf/plugins/inputs/tail"
        _ "github.com/influxdata/telegraf/plugins/inputs/tcp_listener"
-       _ "github.com/influxdata/telegraf/plugins/inputs/teamspeak"
        _ "github.com/influxdata/telegraf/plugins/inputs/temp"
-       _ "github.com/influxdata/telegraf/plugins/inputs/tengine"
-       _ "github.com/influxdata/telegraf/plugins/inputs/tomcat"
-       _ "github.com/influxdata/telegraf/plugins/inputs/trig"
-       _ "github.com/influxdata/telegraf/plugins/inputs/twemproxy"
        _ "github.com/influxdata/telegraf/plugins/inputs/udp_listener"
-       _ "github.com/influxdata/telegraf/plugins/inputs/unbound"
-       _ "github.com/influxdata/telegraf/plugins/inputs/uwsgi"
-       _ "github.com/influxdata/telegraf/plugins/inputs/varnish"
-       _ "github.com/influxdata/telegraf/plugins/inputs/vault"
-       _ "github.com/influxdata/telegraf/plugins/inputs/vsphere"
-       _ "github.com/influxdata/telegraf/plugins/inputs/webhooks"
-       _ "github.com/influxdata/telegraf/plugins/inputs/win_eventlog"
-       _ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters"
-       _ "github.com/influxdata/telegraf/plugins/inputs/win_services"
-       _ "github.com/influxdata/telegraf/plugins/inputs/wireguard"
-       _ "github.com/influxdata/telegraf/plugins/inputs/wireless"
-       _ "github.com/influxdata/telegraf/plugins/inputs/x509_cert"
-       _ "github.com/influxdata/telegraf/plugins/inputs/xtremio"
-       _ "github.com/influxdata/telegraf/plugins/inputs/zfs"
-       _ "github.com/influxdata/telegraf/plugins/inputs/zipkin"
-       _ "github.com/influxdata/telegraf/plugins/inputs/zookeeper"
 )
diff --git i/plugins/outputs/all/all.go w/plugins/outputs/all/all.go
index d6cad6524..49f7e63d4 100644
--- i/plugins/outputs/all/all.go
+++ w/plugins/outputs/all/all.go
@@ -2,57 +2,8 @@ package all

 import (
        //Blank imports for plugins to register themselves
-       _ "github.com/influxdata/telegraf/plugins/outputs/amon"
-       _ "github.com/influxdata/telegraf/plugins/outputs/amqp"
-       _ "github.com/influxdata/telegraf/plugins/outputs/application_insights"
        _ "github.com/influxdata/telegraf/plugins/outputs/azure_data_explorer"
-       _ "github.com/influxdata/telegraf/plugins/outputs/azure_monitor"
-       _ "github.com/influxdata/telegraf/plugins/outputs/bigquery"
-       _ "github.com/influxdata/telegraf/plugins/outputs/cloud_pubsub"
-       _ "github.com/influxdata/telegraf/plugins/outputs/cloudwatch"
-       _ "github.com/influxdata/telegraf/plugins/outputs/cloudwatch_logs"
-       _ "github.com/influxdata/telegraf/plugins/outputs/cratedb"
-       _ "github.com/influxdata/telegraf/plugins/outputs/datadog"
-       _ "github.com/influxdata/telegraf/plugins/outputs/discard"
-       _ "github.com/influxdata/telegraf/plugins/outputs/dynatrace"
-       _ "github.com/influxdata/telegraf/plugins/outputs/elasticsearch"
-       _ "github.com/influxdata/telegraf/plugins/outputs/event_hubs"
-       _ "github.com/influxdata/telegraf/plugins/outputs/exec"
-       _ "github.com/influxdata/telegraf/plugins/outputs/execd"
-       _ "github.com/influxdata/telegraf/plugins/outputs/file"
-       _ "github.com/influxdata/telegraf/plugins/outputs/graphite"
-       _ "github.com/influxdata/telegraf/plugins/outputs/graylog"
-       _ "github.com/influxdata/telegraf/plugins/outputs/groundwork"
-       _ "github.com/influxdata/telegraf/plugins/outputs/health"
        _ "github.com/influxdata/telegraf/plugins/outputs/http"
-       _ "github.com/influxdata/telegraf/plugins/outputs/influxdb"
        _ "github.com/influxdata/telegraf/plugins/outputs/influxdb_v2"
-       _ "github.com/influxdata/telegraf/plugins/outputs/instrumental"
-       _ "github.com/influxdata/telegraf/plugins/outputs/kafka"
-       _ "github.com/influxdata/telegraf/plugins/outputs/kinesis"
-       _ "github.com/influxdata/telegraf/plugins/outputs/librato"
-       _ "github.com/influxdata/telegraf/plugins/outputs/logzio"
-       _ "github.com/influxdata/telegraf/plugins/outputs/loki"
-       _ "github.com/influxdata/telegraf/plugins/outputs/mongodb"
-       _ "github.com/influxdata/telegraf/plugins/outputs/mqtt"
-       _ "github.com/influxdata/telegraf/plugins/outputs/nats"
-       _ "github.com/influxdata/telegraf/plugins/outputs/newrelic"
-       _ "github.com/influxdata/telegraf/plugins/outputs/nsq"
-       _ "github.com/influxdata/telegraf/plugins/outputs/opentelemetry"
-       _ "github.com/influxdata/telegraf/plugins/outputs/opentsdb"
        _ "github.com/influxdata/telegraf/plugins/outputs/prometheus_client"
-       _ "github.com/influxdata/telegraf/plugins/outputs/riemann"
-       _ "github.com/influxdata/telegraf/plugins/outputs/riemann_legacy"
-       _ "github.com/influxdata/telegraf/plugins/outputs/sensu"
-       _ "github.com/influxdata/telegraf/plugins/outputs/signalfx"
-       _ "github.com/influxdata/telegraf/plugins/outputs/socket_writer"
-       _ "github.com/influxdata/telegraf/plugins/outputs/sql"
-       _ "github.com/influxdata/telegraf/plugins/outputs/stackdriver"
-       _ "github.com/influxdata/telegraf/plugins/outputs/sumologic"
-       _ "github.com/influxdata/telegraf/plugins/outputs/syslog"
-       _ "github.com/influxdata/telegraf/plugins/outputs/timestream"
-       _ "github.com/influxdata/telegraf/plugins/outputs/warp10"
-       _ "github.com/influxdata/telegraf/plugins/outputs/wavefront"
-       _ "github.com/influxdata/telegraf/plugins/outputs/websocket"
-       _ "github.com/influxdata/telegraf/plugins/outputs/yandex_cloud_monitoring"
 )

This change currently removes the nstat plugin which is used in the configuration (https://github.com/vyos/vyos-1x/blob/current/data/templates/monitoring/telegraf.j2#L108).
This results in telegraf crashing on startup. Adding the plugin back to the https://github.com/vyos/vyos-build/blob/current/packages/telegraf/plugins/inputs/all/all.go file fixes this (Tested by compiling a patched package and installing it on a broken install).
As far as I can tell this is the only missing plugin.

Will be fixed in the next rolling release. Thanks!