fluentd eat disk

[zh7606559342012]

Describe the bug

fluentd eat disk, systemctl restart td-agent can free the eat disk

To Reproduce

Expected behavior

Your Environment

- Fluentd version:0.0.0
- TD Agent version:4.4.2
- Operating system:centos7.9
- Kernel version:3.10

Your Configuration

####
## Output descriptions:
##

# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
<match td.*.*>
  @type tdlog
  @id output_td
  apikey YOUR_API_KEY

  auto_create_table
  <buffer>
    @type file
    path /var/log/td-agent/buffer/td
  </buffer>

  <secondary>
    @type file
    path /var/log/td-agent/failed_records
  </secondary>
</match>

## match tag=debug.** and dump to console
<match debug.**>
  @type stdout
  @id output_stdout
</match>

####
## Source descriptions:
##

## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
#<source>
#  @type forward
#  @id input_forward
#</source>

## built-in UNIX socket input
#<source>
#  type unix
#</source>

# HTTP input
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
#<source>
#  @type http
#  @id input_http
#  port 8888
#</source>

## live debugging agent
<source>
  @type debug_agent
  @id input_debug_agent
  bind 127.0.0.1
  port 24230
</source>

####
## Examples:
##

## File input
## read apache logs continuously and tags td.apache.access
#<source>
#  @type tail
#  @id input_tail
#  <parse>
#    @type apache2
#  </parse>
#  path /var/log/httpd-access.log
#  tag td.apache.access
#</source>

## File output
## match tag=local.** and write to file
#<match local.**>
#  @type file
#  @id output_file
#  path /var/log/td-agent/access
#</match>

## Forwarding
## match tag=system.** and forward to another td-agent server
#<match system.**>
#  @type forward
#  @id output_system_forward
#
#  <server>
#    host 192.168.0.11
#  </server>
#  # secondary host is optional
#  <secondary>
#    <server>
#      host 192.168.0.12
#    </server>
#  </secondary>
#</match>

## Multiple output
## match tag=td.*.* and output to Treasure Data AND file
#<match td.*.*>
#  @type copy
#  @id output_copy
#  <store>
#    @type tdlog
#    apikey API_KEY
#    auto_create_table
#    <buffer>
#      @type file
#      path /var/log/td-agent/buffer/td
#    </buffer>
#  </store>
#  <store>
#    @type file
#    path /var/log/td-agent/td-%Y-%m-%d/%H.log
#  </store>
#</match>

<source>
  @type tail
  @id cert_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/cert_fail.log.pos
  tag cert_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter cert_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /no issuer certificate found for/
  </regexp>
</filter>
<match cert_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id psk_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/psk_fail.log.pos
  tag psk_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter psk_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /shared key for/
  </regexp>
</filter>
<match psk_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id rsa_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/rsa_fail.log.pos
  tag rsa_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter rsa_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /no matching proposal found, trying alternative config/
  </regexp>
</filter>
<match rsa_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id whitelist_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/whitelist_fail.log.pos
  tag whitelist_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter whitelist_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /not whitelisted/
  </regexp>
</filter>
<match whitelist_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id eap_aka_undo_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/eap_aka_undo_fail.log.pos
  tag eap_aka_undo_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter eap_aka_undo_parse>
  @type grep
  <regexp>
    key content
    pattern /failed to map pseudonym/
  </regexp>
</filter>
<match eap_aka_undo_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id eap_aka_done_err_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/eap_aka_done_err_fail.log.pos
  tag eap_aka_done_err_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+ [^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter eap_aka_done_err_parse>
  @type grep
  <regexp>
    key content
    pattern /received AKA_AUTHENTICATION_REJECT, authentication failed/
  </regexp>
</filter>
<match eap_aka_done_err_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/test
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id child_sa_fail_parse
  path /var/log/strongswan/charon.log
  pos_file /var/log/td-agent/child_sa_fail.log.pos
  tag child_sa_fail_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^\s]+)(?:\s+)(?<content>.+)$
  </parse>
</source>
<filter child_sa_fail_parse>
  @type grep
  <regexp>
    key content
    pattern /CHILD_SA rekeying failed/
  </regexp>
</filter>
<match child_sa_fail_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/tdalarm
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id datagw_alarm_parse
  path /var/log/datagw/datagw.log
  pos_file /var/log/td-agent/datagw_alarm.log.pos
  tag datagw_alarm_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^:]+:.*?) ([^:]+:.*?)(?<content>.+)$
  </parse>
</source>
<filter datagw_alarm_parse>
  @type grep
  <regexp>
    key content
    pattern /t receive arp reply, generate alarm/
  </regexp>
</filter>
<match datagw_alarm_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/tdalarm
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

<source>
  @type tail
  @id datagw_alarm_recv_parse
  path /var/log/datagw/datagw.log
  pos_file /var/log/td-agent/datagw_alarm_recv.log.pos
  tag datagw_alarm_recv_parse
  <parse>
    @type regexp
    expression ^(?<logtime>[A-Za-z]+\s+\d+\s+\d+:\d+:\d+) (?<compnent>[^:]+:.*?) ([^:]+:.*?)(?<content>.+)$
  </parse>
</source>
<filter datagw_alarm_recv_parse>
  @type grep
  <regexp>
    key content
    pattern /received arp reply, clear alarm/
  </regexp>
</filter>
<match datagw_alarm_recv_parse>
  @type http
  endpoint http://127.0.0.1:10000/fluentd_agent/v1/tdalarm
  open_timeout 2
  http_method post
  content_type application/json
  <buffer>
    chunk_limit_records 1
  </buffer>
</match>

Your Error Log

none

Additional context

No response

[zh7606559342012]

i saw the question #480

the same question i meet now, i wonder if it was caused by the logrotate.d conf

cat << EOF > /etc/logrotate.d/td-agent
/var/log/td-agent/td-agent.log {
missingok
daily
minsize 10M
create 0600 root
rotate 7
notifempty
compress
postrotate
systemctl restart rsyslog
endscript
}
EOF

need add copytruncate can avoid the question?
thanks for reply!
Astin ^_^

[daipom]

Thanks for your report, but with this information of To Reproduce, we can't judge if this is a bug of Fluentd or not.
So, I have transferred this to Discussions.

[daipom]

Looks like the size of Fluentd's log file (td-agent.log) is about 11MB.
Do you need to make this smaller?

[zh7606559342012]

Looks like the size of Fluentd's log file (td-agent.log) is about 11MB is one of the our test env.
business env log file is 1Gb so crazy

[zh7606559342012]

i guess the question should is logrotate cause, we add copytruncate conf into logrotato.conf to watch if it will appear again.
thank for you reply
Astin

[daipom]

business env log file is 1Gb so crazy

the question should is logrotate cause

If the size of Fluentd's log file is too large, then yes, you should adjust the logrotate conf.

It depends on your requirements, but for example, you could set size to limit the maximum file size.
(It means to stop rotating in periods such as daily.)

[daipom]

Please see man logrotate for details.

[zh7606559342012]

okkkkk lol thank for your answer.