From 3b2d90c922894663f3a2cfecaf4d35a7639accfa Mon Sep 17 00:00:00 2001 From: Srinivas Chippagiri Date: Sat, 31 Oct 2026 12:13:48 -0700 Subject: [PATCH 1/3] feat(fluentbit): add the mkdir option to the file output Add mkdir to the File output so Fluent Bit creates the output directory recursively, instead of every path having to be mounted into the Fluent Bit pods. Fixes #1618 Signed-off-by: Srinivas Chippagiri --- .../v1alpha2/plugins/output/file_types.go | 7 +++ .../plugins/output/file_types_test.go | 49 +++++++++++++++++++ .../plugins/output/zz_generated.deepcopy.go | 5 ++ .../fluentbit.fluent.io_clusteroutputs.yaml | 4 ++ .../fluentbit.fluent.io_outputs.yaml | 4 ++ .../fluentbit.fluent.io_clusteroutputs.yaml | 4 ++ .../crds/fluentbit.fluent.io_outputs.yaml | 4 ++ .../fluentbit.fluent.io_clusteroutputs.yaml | 4 ++ .../bases/fluentbit.fluent.io_outputs.yaml | 4 ++ docs/plugins/fluentbit/output/file.md | 1 + manifests/setup/setup.yaml | 8 +++ 11 files changed, 94 insertions(+) create mode 100644 apis/fluentbit/v1alpha2/plugins/output/file_types_test.go diff --git a/apis/fluentbit/v1alpha2/plugins/output/file_types.go b/apis/fluentbit/v1alpha2/plugins/output/file_types.go index bcf97db1e..4e84f5a24 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/file_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/file_types.go @@ -1,6 +1,8 @@ package output import ( + "fmt" + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params" ) @@ -23,6 +25,8 @@ type File struct { LabelDelimiter string `json:"labelDelimiter,omitempty"` // The format string. Applicable only if format is template. Template string `json:"template,omitempty"` + // Recursively create the output directory if it does not exist. The directories are created with permissions 0755. + Mkdir *bool `json:"mkdir,omitempty"` } func (*File) Name() string { @@ -38,6 +42,9 @@ func (f *File) Params(_ plugins.SecretLoader) (*params.KVs, error) { plugins.InsertKVString(kvs, "Delimiter", f.Delimiter) plugins.InsertKVString(kvs, "Label_Delimiter", f.LabelDelimiter) plugins.InsertKVString(kvs, "Template", f.Template) + if f.Mkdir != nil { + kvs.Insert("Mkdir", fmt.Sprint(*f.Mkdir)) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/output/file_types_test.go b/apis/fluentbit/v1alpha2/plugins/output/file_types_test.go new file mode 100644 index 000000000..b7441cc38 --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/output/file_types_test.go @@ -0,0 +1,49 @@ +package output + +import ( + "testing" + + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params" + . "github.com/onsi/gomega" +) + +func TestOutput_File_Params(t *testing.T) { + g := NewWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + enabled := true + + file := File{ + Path: "/var/log/out/app", + File: "records.log", + Mkdir: &enabled, + } + + expected := params.NewKVs() + expected.Insert("Path", "/var/log/out/app") + expected.Insert("File", "records.log") + expected.Insert("Mkdir", "true") + + kvs, err := file.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) +} + +func TestOutput_File_ParamsMkdirUnsetAndFalse(t *testing.T) { + g := NewWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + disabled := false + + kvs, err := (&File{Path: "/out"}).Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + expected := params.NewKVs() + expected.Insert("Path", "/out") + g.Expect(kvs).To(Equal(expected)) + + kvs, err = (&File{Path: "/out", Mkdir: &disabled}).Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + expected.Insert("Mkdir", "false") + g.Expect(kvs).To(Equal(expected)) +} diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go index b5b514a34..89ffd940d 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go @@ -242,6 +242,11 @@ func (in *Elasticsearch) DeepCopy() *Elasticsearch { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *File) DeepCopyInto(out *File) { *out = *in + if in.Mkdir != nil { + in, out := &in.Mkdir, &out.Mkdir + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new File. diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusteroutputs.yaml index ebd9eced6..24e367570 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusteroutputs.yaml @@ -1045,6 +1045,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_outputs.yaml index 002cff96d..dbd82cee8 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_outputs.yaml @@ -1045,6 +1045,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml index 5fd112752..c27c9f254 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -1043,6 +1043,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_outputs.yaml index 3e12af2cd..dc736c0ba 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_outputs.yaml @@ -1043,6 +1043,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index 10dcc1d84..765a31f57 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -1044,6 +1044,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index d6fe5d8d8..c410c0623 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -1044,6 +1044,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. diff --git a/docs/plugins/fluentbit/output/file.md b/docs/plugins/fluentbit/output/file.md index fd4004372..7391df19f 100644 --- a/docs/plugins/fluentbit/output/file.md +++ b/docs/plugins/fluentbit/output/file.md @@ -11,3 +11,4 @@ The file output plugin allows to write the data received through the input plugi | delimiter | The character to separate each pair. Applicable only if format is csv or ltsv. | string | | labelDelimiter | The character to separate each pair. Applicable only if format is ltsv. | string | | template | The format string. Applicable only if format is template. | string | +| mkdir | Recursively create the output directory if it does not exist. The directories are created with permissions 0755. | *bool | diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 4656af914..372dd3b60 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -5270,6 +5270,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. @@ -36875,6 +36879,10 @@ spec: description: The character to separate each pair. Applicable only if format is ltsv. type: string + mkdir: + description: Recursively create the output directory if it does + not exist. The directories are created with permissions 0755. + type: boolean path: description: Absolute directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. From d9493fecacba8eae194fc69c8465543df269e8ef Mon Sep 17 00:00:00 2001 From: Srinivas Chippagiri Date: Sat, 31 Oct 2026 12:18:38 -0700 Subject: [PATCH 2/3] fix(fluentbit): deepcopy the file output now that it has a pointer field Signed-off-by: Srinivas Chippagiri --- apis/fluentbit/v1alpha2/zz_generated.deepcopy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index 2eef2059c..efcddf2e4 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1491,7 +1491,7 @@ func (in *OutputSpec) DeepCopyInto(out *OutputSpec) { if in.File != nil { in, out := &in.File, &out.File *out = new(output.File) - **out = **in + (*in).DeepCopyInto(*out) } if in.Forward != nil { in, out := &in.Forward, &out.Forward From a5b23da5f13d6a1ca10e025e47d45c0027b191a0 Mon Sep 17 00:00:00 2001 From: Srinivas Chippagiri Date: Mon, 2 Nov 2026 00:24:32 -0800 Subject: [PATCH 3/3] refactor(fluentbit): use plugins.InsertKVField for the file output mkdir option Signed-off-by: Srinivas Chippagiri --- apis/fluentbit/v1alpha2/plugins/output/file_types.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apis/fluentbit/v1alpha2/plugins/output/file_types.go b/apis/fluentbit/v1alpha2/plugins/output/file_types.go index 4e84f5a24..a6868bb9a 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/file_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/file_types.go @@ -1,8 +1,6 @@ package output import ( - "fmt" - "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params" ) @@ -42,9 +40,7 @@ func (f *File) Params(_ plugins.SecretLoader) (*params.KVs, error) { plugins.InsertKVString(kvs, "Delimiter", f.Delimiter) plugins.InsertKVString(kvs, "Label_Delimiter", f.LabelDelimiter) plugins.InsertKVString(kvs, "Template", f.Template) - if f.Mkdir != nil { - kvs.Insert("Mkdir", fmt.Sprint(*f.Mkdir)) - } + plugins.InsertKVField(kvs, "Mkdir", f.Mkdir) return kvs, nil }