Refactor: Improve incremental log indexing logic (#1460)

Co-authored-by: Cursor Agent <cursoragent@cursor.com>

Author: 0xJacky

internal/cron/incremental_indexing.go

@@ -7,10 +7,16 @@ import (
 
 "github.com/0xJacky/Nginx-UI/internal/nginx_log"
 "github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
+"github.com/0xJacky/Nginx-UI/model"
 "github.com/go-co-op/gocron/v2"
 "github.com/uozi-tech/cosy/logger"
 )
 
+// logIndexProvider provides access to stored per-file index metadata.
+type logIndexProvider interface {
+GetLogIndex(path string) (*model.NginxLogIndex, error)
+}
+
 // setupIncrementalIndexingJob sets up the periodic incremental log indexing job
 func setupIncrementalIndexingJob(s gocron.Scheduler) (gocron.Job, error) {
 logger.Info("Setting up incremental log indexing job")
@@ -42,6 +48,12 @@ func performIncrementalIndexing() {
 return
 }
 
+persistence := logFileManager.GetPersistence()
+if persistence == nil {
+logger.Warn("Persistence manager not available for incremental indexing")
+return
+}
+
 // Get modern indexer
 modernIndexer := nginx_log.GetIndexer()
 if modernIndexer == nil {
@@ -64,7 +76,7 @@ func performIncrementalIndexing() {
 changedCount := 0
 for _, log := range allLogs {
 // Check if file needs incremental indexing
-if needsIncrementalIndexing(log) {
+if needsIncrementalIndexing(log, persistence) {
 if err := queueIncrementalIndexing(log.Path, modernIndexer, logFileManager); err != nil {
 logger.Errorf("Failed to queue incremental indexing for %s: %v", log.Path, err)
 } else {
@@ -81,7 +93,7 @@ func performIncrementalIndexing() {
 }
 
 // needsIncrementalIndexing checks if a log file needs incremental indexing
-func needsIncrementalIndexing(log *nginx_log.NginxLogWithIndex) bool {
+func needsIncrementalIndexing(log *nginx_log.NginxLogWithIndex, persistence logIndexProvider) bool {
 // Skip if already indexing or queued
 if log.IndexStatus == string(indexer.IndexStatusIndexing) ||
 log.IndexStatus == string(indexer.IndexStatusQueued) {
@@ -99,22 +111,43 @@ func needsIncrementalIndexing(log *nginx_log.NginxLogWithIndex) bool {
 return false
 }
 
-// Check if file has been modified since last index
 fileModTime := fileInfo.ModTime()
 fileSize := fileInfo.Size()
+
+if persistence != nil {
+if logIndex, err := persistence.GetLogIndex(log.Path); err == nil {
+if logIndex.NeedsIndexing(fileModTime, fileSize) {
+logger.Debugf("File %s needs incremental indexing based on persisted metadata", log.Path)
+return true
+}
+return false
+} else {
+logger.Debugf("Could not load persisted metadata for %s: %v", log.Path, err)
+}
+}
+
+// Fallback: use aggregated data cautiously by clamping the stored size so grouped entries
+// do not trigger false positives when rotation files are aggregated together.
 lastModified := time.Unix(log.LastModified, 0)
+lastSize := log.LastSize
+if lastSize == 0 || lastSize > fileSize {
+lastSize = fileSize
+}
+
+// If the file was never indexed, queue it once.
+if log.LastIndexed == 0 {
+return true
+}
 
-// File was modified after last index and size increased
-if fileModTime.After(lastModified) && fileSize > log.LastSize {
-logger.Debugf("File %s needs incremental indexing: mod_time=%s, size=%d",
+if fileModTime.After(lastModified) && fileSize > lastSize {
+logger.Debugf("File %s needs incremental indexing (fallback path): mod_time=%s, size=%d",
 log.Path, fileModTime.Format("2006-01-02 15:04:05"), fileSize)
 return true
 }
 
-// File size decreased - might be file rotation
-if fileSize < log.LastSize {
-logger.Debugf("File %s needs full re-indexing due to size decrease: old_size=%d, new_size=%d",
-log.Path, log.LastSize, fileSize)
+if fileSize < lastSize {
+logger.Debugf("File %s needs full re-indexing (fallback path) due to size decrease: old_size=%d, new_size=%d",
+log.Path, lastSize, fileSize)
 return true
 }
 

internal/cron/incremental_indexing_test.go

@@ -0,0 +1,108 @@
+package cron
+
+import (
+"os"
+"path/filepath"
+"testing"
+"time"
+
+"github.com/0xJacky/Nginx-UI/internal/nginx_log"
+"github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
+"github.com/0xJacky/Nginx-UI/model"
+)
+
+type stubLogIndexProvider struct {
+idx *model.NginxLogIndex
+err error
+}
+
+func (s stubLogIndexProvider) GetLogIndex(path string) (*model.NginxLogIndex, error) {
+if s.err != nil {
+return nil, s.err
+}
+if s.idx != nil {
+s.idx.Path = path
+}
+return s.idx, nil
+}
+
+func TestNeedsIncrementalIndexingSkipsWhenUnchanged(t *testing.T) {
+dir := t.TempDir()
+logPath := filepath.Join(dir, "access.log")
+if err := os.WriteFile(logPath, []byte("initial\n"), 0o644); err != nil {
+t.Fatalf("write temp log: %v", err)
+}
+
+info, err := os.Stat(logPath)
+if err != nil {
+t.Fatalf("stat temp log: %v", err)
+}
+
+persisted := &model.NginxLogIndex{
+Path: logPath,
+LastModified: info.ModTime(),
+LastSize: info.Size(),
+LastIndexed: time.Now(),
+}
+
+logData := &nginx_log.NginxLogWithIndex{
+Path: logPath,
+Type: "access",
+IndexStatus: string(indexer.IndexStatusIndexed),
+LastModified: info.ModTime().Unix(),
+LastSize: info.Size() * 10, // simulate grouped size inflation
+LastIndexed: time.Now().Unix(),
+}
+
+if needsIncrementalIndexing(logData, stubLogIndexProvider{idx: persisted}) {
+t.Fatalf("expected no incremental indexing when file metadata is unchanged")
+}
+}
+
+func TestNeedsIncrementalIndexingDetectsGrowth(t *testing.T) {
+dir := t.TempDir()
+logPath := filepath.Join(dir, "access.log")
+if err := os.WriteFile(logPath, []byte("initial\n"), 0o644); err != nil {
+t.Fatalf("write temp log: %v", err)
+}
+
+initialInfo, err := os.Stat(logPath)
+if err != nil {
+t.Fatalf("stat temp log: %v", err)
+}
+
+persisted := &model.NginxLogIndex{
+Path: logPath,
+LastModified: initialInfo.ModTime().Add(-time.Minute),
+LastSize: initialInfo.Size(),
+LastIndexed: time.Now().Add(-time.Minute),
+}
+
+f, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0)
+if err != nil {
+t.Fatalf("open temp log: %v", err)
+}
+if _, err := f.WriteString("more data\n"); err != nil {
+f.Close()
+t.Fatalf("append temp log: %v", err)
+}
+_ = f.Close()
+
+finalInfo, err := os.Stat(logPath)
+if err != nil {
+t.Fatalf("restat temp log: %v", err)
+}
+
+logData := &nginx_log.NginxLogWithIndex{
+Path: logPath,
+Type: "access",
+IndexStatus: string(indexer.IndexStatusIndexed),
+LastModified: finalInfo.ModTime().Unix(),
+LastSize: initialInfo.Size(),
+LastIndexed: time.Now().Unix(),
+}
+
+if !needsIncrementalIndexing(logData, stubLogIndexProvider{idx: persisted}) {
+t.Fatalf("expected incremental indexing when file grew")
+}
+}