WIP: New design

.dockerignore

@@ -0,0 +1,7 @@
+*
+!Makefile
+!go.mod
+!go.sum
+!main.go
+!internal
+!hack

Design.md

@@ -0,0 +1,80 @@
+
+### Logwatcher Design
+
+Logwatcher uses Linux inotify[1] to add watches to directories and files generated in "/var/log/pods". Different
+set of watch flags are used for directories and files.
+
+Watches for folders:
+ - `IN_CREATE`: watch is triggered when a file or direcory is created in this directory.
+ - `IN_DELETE`: watch is triggered when a file or direcory is deleted in this direcory.
+
+Watches for files:
+ - `IN_MODIFY`: watch is triggered when file is written.
+ - `IN_CLOSE_WRITE`: watch is triggered when file is closed for writing.
+ - `IN_ONESHOT`: When ONESHOT flag is used with other set of flags, when the condition for watch is fulfilled, 
+   and a message is sent, the watch is deleted after sending the message. Application needs to set the watch again
+   if needed.
+
+When some activity on directories and files, watches are triggered, and a message is sent on watch file descriptor. 
+Since watches on files are not kept permanently and deleted as soon as the watch is triggered because of ONESHOT flag,
+messages do not accumulate in the inotify queue. Without ONESHOT flag, with logs continuosely getting writen, queue can
+get full very easily and messages start getting dropped.
+
+### Watch Structure
+
+
+    /var/log/pods/  (IN_CREATE|IN_DELETE)
+      │
+      │
+      ├───── openshift-kube-controller-manager_kube-controller-manager-crc-pbwlw-master-0_738a9f84e9aed99070694fd38123a679/  (IN_CREATE|IN_DELETE)
+      │        │
+      │        ├────────── cluster-policy-controller/  (IN_CREATE|IN_DELETE)
+      │        │            │
+      │        │            │       0.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │        │            │       4.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │        │            │       5.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │        │            └────── 6.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │        │                    7.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │        │                    8.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │        │
+      │        │
+      │        └────────── kube-controller-manager/  (IN_CREATE|IN_DELETE)
+      │                     │
+      │                     │       0.log  (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │                     │       10.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │                     │       11.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │                     └────── 12.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │                             13.log (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │                             9.log  (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+      │
+      │
+      └────── openshift-cluster-version_cluster-version-operator-8b9c98bfd-8mj5d_60452d84-5db1-4e5f-815c-245aaa76cbb9/  (IN_CREATE|IN_DELETE)
+                │
+                │
+                └───────── cluster-version-operator/  (IN_CREATE|IN_DELETE)
+                            │
+                            │       0.log                    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                            │       1.log                    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                            │       1.log.20230102-180708    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                            │       2.log                    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                            │       2.log.20230104-094208.gz (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                            │       2.log.20230104-182347.gz (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                            │       2.log.20230105-030537.gz (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                            └────── 2.log.20230105-114647    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                                    3.log                    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                                    4.log                    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                                    4.log.20230111-190053    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+                                    5.log                    (IN_MODIFY|IN_CLOSE_WRITE|IN_ONESHOT)
+
+### Read Loop
+A goroutine reads the inotify file descriptor and reads the raw events posted to the inotify file descriptor, and sends an
+event to the Event Loop for handeling over a buffered channel.
+
+### Event Loop
+A goroutine reads the buffered channel, and processes the NotifyEvent.
+
+
+
+### References
+ [1] [Filesystem notification, part 2: A deeper investigation of inotify](https://lwn.net/Articles/605128/)
+

Dockerfile

@@ -1,42 +1,19 @@
-### This is a generated file from Dockerfile.in ###
-#@follow_tag(registry-proxy.engineering.redhat.com/rh-osbs/openshift-golang-builder:rhel_8_golang_1.18)
 FROM registry.access.redhat.com/ubi8/go-toolset AS builder
 
-ENV BUILD_VERSION=1.1.0
-ENV OS_GIT_MAJOR=1
-ENV OS_GIT_MINOR=1
-ENV OS_GIT_PATCH=0
-ENV SOURCE_GIT_COMMIT=${CI_LOG_FILE_METRIC_EXPORTER_UPSTREAM_COMMIT}
-ENV SOURCE_GIT_URL=${CI_LOG_FILE_METRIC_EXPORTER_UPSTREAM_URL}
-ENV REMOTE_SOURCE=${REMOTE_SOURCE:-.}
-
-
+WORKDIR /go/src/github.com/ViaQ/logwatcher
+COPY . .
 USER 0
-WORKDIR  /go/src/github.com/log-file-metric-exporter
-COPY ${REMOTE_SOURCE} .
-
+RUN go mod download
 RUN make build
 
-#@follow_tag(registry.redhat.io/ubi8:latest)
-FROM registry.access.redhat.com/ubi8
-COPY --from=builder /go/src/github.com/log-file-metric-exporter/bin/log-file-metric-exporter  /usr/local/bin/.
-COPY --from=builder /go/src/github.com/log-file-metric-exporter/hack/log-file-metric-exporter.sh  /usr/local/bin/.
-
-RUN chmod +x /usr/local/bin/log-file-metric-exporter
-RUN chmod +x /usr/local/bin/log-file-metric-exporter.sh
-
-LABEL \
-        io.k8s.display-name="OpenShift LogFileMetric Exporter" \
-        io.k8s.description="OpenShift LogFileMetric Exporter component of OpenShift Cluster Logging" \
-        License="Apache-2.0" \
-        name="openshift-logging/log-file-metric-exporter-rhel8" \
-        com.redhat.component="log-file-metric-exporter-container" \
-        io.openshift.maintainer.product="OpenShift Container Platform" \
-        io.openshift.maintainer.component="Logging" \
-        io.openshift.build.commit.id=${CI_LOG_FILE_METRIC_EXPORTER_UPSTREAM_COMMIT} \
-        io.openshift.build.source-location=${CI_LOG_FILE_METRIC_EXPORTER_UPSTREAM_URL} \
-        io.openshift.build.commit.url=${CI_LOG_FILE_METRIC_EXPORTER_UPSTREAM_URL}/commit/${CI_LOG_FILE_METRIC_EXPORTER_UPSTREAM_COMMIT} \
-        version=v1.1.0
-
-CMD ["sh", "-c", "/usr/local/bin/log-file-metric-exporter.sh"]
-
+#FROM registry.access.redhat.com/ubi8
+FROM centos:centos8
+USER 0
+RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+#RUN echo 'kernel.yama.ptrace_scope = 0' >> /etc/sysctl.conf
+RUN yum -y install strace
+RUN yum -y install trace-cmd
+COPY --from=builder /go/src/github.com/ViaQ/logwatcher/bin/logwatcher  /usr/local/bin/
+COPY --from=builder /go/src/github.com/ViaQ/logwatcher/hack/list-watches.sh  /usr/local/bin/
+CMD ["sh", "-c", "/usr/local/bin/logwatcher", "-watch_dir=/var/log/pods", "-v=0", "-logtostderr=true"]

ISSUES.md

@@ -0,0 +1,18 @@
+
+
+#### Watch descriptor reused by inotify
+
+eventLoop:
+
+ 1. program adds a watch on a file, gets a watch fd (say `a0`)
+ 2. file gets modified, program gets an event on watch fd `a0`
+ 3. (since `IN_ONESHOT` was used, the watch is auto deleted by inotify now.)
+ 4. program handles the modify (`0x2`) event and does its logic
+ 5. program adds a watch again on the same file, gets a new watch fd (say `a1`) (this step is same as step 1.)
+
+While testing it is observed that after some iterations of the above loop, the watch fd returned in the step 5 is same 
+as one returned in step 1. Once this happens, there are no more events received for file changes. the loop essentially halts.
+
+#### Workaround
+Save the watch desctiptor for files, and compare the new watch descriptor with earlier one, if same add watch again will the returned watch descriptor is different.
+https://github.com/vimalk78/logwatcher/blob/main/internal/inotify/watch.go#L49-L64

Makefile

@@ -1,73 +1,12 @@
-export GOROOT=$(shell go env GOROOT)
-export GOFLAGS=
-export GO111MODULE=on
 
-ARTIFACT_DIR?=./tmp
-CURPATH=$(PWD)
-GOFLAGS?=
-CLO_RELEASE_VERSION?=5.4
-BIN_NAME=log-file-metric-exporter
-IMAGE_REPOSITORY_NAME=quay.io/openshift-logging/origin-${BIN_NAME}:${CLO_RELEASE_VERSION}
-LOCAL_IMAGE_TAG=127.0.0.1:5000/openshift/origin-${BIN_NAME}:${CLO_RELEASE_VERSION}
-#just for testing purpose pushing it to docker.io
-MAIN_PKG=cmd/main.go
-TARGET_DIR=$(CURPATH)/_output
-TARGET=$(CURPATH)/bin/$(BIN_NAME)
-BUILD_GOPATH=$(TARGET_DIR)
+IMAGE=viaq/logwatcher:v0.0.1
+TARGET=bin/logwatcher
+MAIN_PKG=main.go
 
-#inputs to 'run' which may need to change
-TLS_CERTS_BASEDIR=_output
-NAMESPACE ?= "openshift-logging"
-ES_CERTS_DIR ?= ""
-CACHE_EXPIRY ?= "5s"
-
-PKGS=$(shell go list ./...)
-TEST_OPTIONS?=
-
-
-
-all: fmt build image deploy-image
-.PHONY: all
-
-artifactdir:
-	@mkdir -p $(ARTIFACT_DIR)
-
-
-fmt:
-	@gofmt -l -w cmd && \
-	gofmt -l -w pkg
-.PHONY: fmt
-
-build: fmt
-	go build $(LDFLAGS) -o $(TARGET) $(MAIN_PKG)
 .PHONY: build
+build:
+	go build $(LDFLAGS) -o $(TARGET) $(MAIN_PKG)
 
-image:
-	podman build -f Dockerfile -t $(LOCAL_IMAGE_TAG) .
-	podman tag ${LOCAL_IMAGE_TAG} ${IMAGE_REPOSITORY_NAME}
 .PHONY: image
-
-deploy-image: image
-	IMAGE_TAG=$(LOCAL_IMAGE_TAG) hack/deploy-image.sh
-	IMAGE_TAG=$(IMAGE_REPOSITORY_NAME) hack/deploy-image.sh
-.PHONY: deploy-image
-
-clean:
-	rm -rf $(TARGET_DIR)
-.PHONY: clean
-
-COVERAGE_DIR=$(ARTIFACT_DIR)/coverage
-test: artifactdir
-	@mkdir -p $(COVERAGE_DIR)
-	@go test -race -coverprofile=$(COVERAGE_DIR)/test-unit.cov ./pkg/...
-	@go test -v ./cmd
-	@go tool cover -html=$(COVERAGE_DIR)/test-unit.cov -o $(COVERAGE_DIR)/test-unit-coverage.html
-	@go tool cover -func=$(COVERAGE_DIR)/test-unit.cov | tail -n 1
-.PHONY: test
-
-lint:
-	@hack/run-linter
-.PHONY: lint
-gen-dockerfiles:
-	./hack/generate-dockerfile-from-midstream > Dockerfile
-.PHONY: gen-dockerfiles
+image:
+	podman build -f Dockerfile . -t $(IMAGE)

README.md

@@ -1,6 +1,6 @@
-# log-file-metric-exporter
 
-Exporter to collect metrics about container logs being produced in a kubernetes environment
-It publishes log_logged_bytes_total metric in prometheus. This metric allows one to see total data bytes actually logged vs. what collector (fluentd) is able to collect during runtime.
-This implementation is based on Golang and it uses fsnotify package to watch out for new data written to log files residing in the Watcher path.
+### LogWatcher 
+Monitors logging root folder "/var/log/pods" for log activity by containers, and generates metrics about amount of logs produced by each container.
 
+### Design
+Please refer to [Design.md](./Design.md)

TODO.md

@@ -0,0 +1,5 @@
+
+
+1. Add metrics
+2. Add Unit tests
+3. Fix the problem identified in ISSUES.md