mchus/logpile
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Add pluggable live collectors and simplify API connect form

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-02-04 19:00:03 +03:00
parent 60c52b18b1
commit c89ee0118f
15 changed files with 939 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/server/collect_handlers_test.go
View File

@@ -14,7 +14,8 @@ import (
func newCollectTestServer() (*Server, *httptest.Server) {
s := &Server{
jobManager: NewJobManager(),
jobManager: NewJobManager(),
collectors: testCollectorRegistry(),
}
mux := http.NewServeMux()
mux.HandleFunc("POST /api/collect", s.handleCollectStart)

63
internal/server/collect_test_helpers_test.go Normal file
View File

@@ -0,0 +1,63 @@
package server
import (
"context"
"strings"
"time"
"git.mchus.pro/mchus/logpile/internal/collector"
"git.mchus.pro/mchus/logpile/internal/models"
)
type mockConnector struct {
protocol string
}
func (c *mockConnector) Protocol() string {
return c.protocol
}
func (c *mockConnector) Collect(ctx context.Context, req collector.Request, emit collector.ProgressFn) (*models.AnalysisResult, error) {
steps := []collector.Progress{
{Status: CollectStatusRunning, Progress: 20, Message: "Подключение..."},
{Status: CollectStatusRunning, Progress: 50, Message: "Сбор инвентаря..."},
{Status: CollectStatusRunning, Progress: 80, Message: "Нормализация..."},
}
for _, step := range steps {
if !collectorSleep(ctx, 100*time.Millisecond) {
return nil, ctx.Err()
}
if emit != nil {
emit(step)
}
}
if strings.Contains(strings.ToLower(req.Host), "fail") {
return nil, context.DeadlineExceeded
}
return &models.AnalysisResult{
Events: make([]models.Event, 0),
FRU: make([]models.FRUInfo, 0),
Sensors: make([]models.SensorReading, 0),
Hardware: &models.HardwareConfig{},
}, nil
}
func testCollectorRegistry() *collector.Registry {
r := collector.NewRegistry()
r.Register(&mockConnector{protocol: "redfish"})
r.Register(&mockConnector{protocol: "ipmi"})
return r
}
func collectorSleep(ctx context.Context, d time.Duration) bool {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}

104
internal/server/handlers.go
View File

@@ -13,6 +13,7 @@ import (
"strings"
"time"
"git.mchus.pro/mchus/logpile/internal/collector"
"git.mchus.pro/mchus/logpile/internal/exporter"
"git.mchus.pro/mchus/logpile/internal/models"
"git.mchus.pro/mchus/logpile/internal/parser"
@@ -592,7 +593,7 @@ func (s *Server) handleCollectStart(w http.ResponseWriter, r *http.Request) {
}
job := s.jobManager.CreateJob(req)
s.startMockCollectionJob(job.ID, req)
s.startCollectionJob(job.ID, req)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
@@ -631,7 +632,7 @@ func (s *Server) handleCollectCancel(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, job.toStatusResponse())
}
func (s *Server) startMockCollectionJob(jobID string, req CollectRequest) {
func (s *Server) startCollectionJob(jobID string, req CollectRequest) {
ctx, cancel := context.WithCancel(context.Background())
if attached := s.jobManager.AttachJobCancel(jobID, cancel); !attached {
cancel()
@@ -639,31 +640,37 @@ func (s *Server) startMockCollectionJob(jobID string, req CollectRequest) {
}
go func() {
steps := []struct {
delay time.Duration
status string
progress int
log string
}{
{delay: 250 * time.Millisecond, status: CollectStatusRunning, progress: 20, log: "Подключение..."},
{delay: 250 * time.Millisecond, status: CollectStatusRunning, progress: 50, log: "Сбор инвентаря..."},
{delay: 250 * time.Millisecond, status: CollectStatusRunning, progress: 80, log: "Нормализация..."},
connector, ok := s.getCollector(req.Protocol)
if !ok {
s.jobManager.UpdateJobStatus(jobID, CollectStatusFailed, 100, "Коннектор для протокола не зарегистрирован")
s.jobManager.AppendJobLog(jobID, "Сбор завершен с ошибкой")
return
}
for _, step := range steps {
if !waitWithCancel(ctx, step.delay) {
return
}
emitProgress := func(update collector.Progress) {
if job, ok := s.jobManager.GetJob(jobID); !ok || isTerminalCollectStatus(job.Status) {
return
}
s.jobManager.UpdateJobStatus(jobID, step.status, step.progress, "")
s.jobManager.AppendJobLog(jobID, step.log)
status := update.Status
if status == "" {
status = CollectStatusRunning
}
s.jobManager.UpdateJobStatus(jobID, status, update.Progress, "")
if update.Message != "" {
s.jobManager.AppendJobLog(jobID, update.Message)
}
}
if !waitWithCancel(ctx, 250*time.Millisecond) {
result, err := connector.Collect(ctx, toCollectorRequest(req), emitProgress)
if err != nil {
if ctx.Err() != nil {
return
}
if job, ok := s.jobManager.GetJob(jobID); !ok || isTerminalCollectStatus(job.Status) {
return
}
s.jobManager.UpdateJobStatus(jobID, CollectStatusFailed, 100, err.Error())
s.jobManager.AppendJobLog(jobID, "Сбор завершен с ошибкой")
return
}
@@ -671,31 +678,14 @@ func (s *Server) startMockCollectionJob(jobID string, req CollectRequest) {
return
}
if strings.Contains(strings.ToLower(req.Host), "fail") {
s.jobManager.UpdateJobStatus(jobID, CollectStatusFailed, 100, "Mock: не удалось завершить сбор")
s.jobManager.AppendJobLog(jobID, "Сбор завершен с ошибкой")
return
}
applyCollectSourceMetadata(result, req)
s.jobManager.UpdateJobStatus(jobID, CollectStatusSuccess, 100, "")
s.jobManager.AppendJobLog(jobID, "Сбор завершен")
s.SetResult(newAPIResult(req))
s.SetResult(result)
s.SetDetectedVendor("")
}()
}
func waitWithCancel(ctx context.Context, d time.Duration) bool {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
func validateCollectRequest(req CollectRequest) error {
if strings.TrimSpace(req.Host) == "" {
return fmt.Errorf("field 'host' is required")
@@ -756,16 +746,34 @@ func applyArchiveSourceMetadata(result *models.AnalysisResult) {
result.CollectedAt = time.Now().UTC()
}
func newAPIResult(req CollectRequest) *models.AnalysisResult {
return &models.AnalysisResult{
SourceType: models.SourceTypeAPI,
Protocol: req.Protocol,
TargetHost: req.Host,
CollectedAt: time.Now().UTC(),
Events: make([]models.Event, 0),
FRU: make([]models.FRUInfo, 0),
Sensors: make([]models.SensorReading, 0),
func applyCollectSourceMetadata(result *models.AnalysisResult, req CollectRequest) {
if result == nil {
return
}
result.SourceType = models.SourceTypeAPI
result.Protocol = req.Protocol
result.TargetHost = req.Host
result.CollectedAt = time.Now().UTC()
}
func toCollectorRequest(req CollectRequest) collector.Request {
return collector.Request{
Host: req.Host,
Protocol: req.Protocol,
Port: req.Port,
Username: req.Username,
AuthType: req.AuthType,
Password: req.Password,
Token: req.Token,
TLSMode: req.TLSMode,
}
}
func (s *Server) getCollector(protocol string) (collector.Connector, bool) {
if s.collectors == nil {
s.collectors = collector.NewDefaultRegistry()
}
return s.collectors.Get(protocol)
}
func jsonResponse(w http.ResponseWriter, data interface{}) {

5
internal/server/server.go
View File

@@ -9,6 +9,7 @@ import (
"sync"
"time"
"git.mchus.pro/mchus/logpile/internal/collector"
"git.mchus.pro/mchus/logpile/internal/models"
)
@@ -29,7 +30,8 @@ type Server struct {
result *models.AnalysisResult
detectedVendor string
jobManager *JobManager
jobManager *JobManager
collectors *collector.Registry
}
func New(cfg Config) *Server {
@@ -37,6 +39,7 @@ func New(cfg Config) *Server {
config: cfg,
mux: http.NewServeMux(),
jobManager: NewJobManager(),
collectors: collector.NewDefaultRegistry(),
}
s.setupRoutes()
return s

9
internal/server/source_metadata_test.go
View File

@@ -30,7 +30,7 @@ func TestApplyArchiveSourceMetadata(t *testing.T) {
}
}
func TestNewAPIResultMetadata(t *testing.T) {
func TestApplyCollectSourceMetadata(t *testing.T) {
req := CollectRequest{
Host: "bmc-api.local",
Protocol: "redfish",
@@ -41,7 +41,12 @@ func TestNewAPIResultMetadata(t *testing.T) {
TLSMode: "strict",
}
result := newAPIResult(req)
result := &models.AnalysisResult{
Events: make([]models.Event, 0),
FRU: make([]models.FRUInfo, 0),
Sensors: make([]models.SensorReading, 0),
}
applyCollectSourceMetadata(result, req)
if result.SourceType != models.SourceTypeAPI {
t.Fatalf("expected source type %q, got %q", models.SourceTypeAPI, result.SourceType)

3
internal/server/upload_live_smoke_test.go
View File

@@ -15,7 +15,8 @@ import (
func newFlowTestServer() (*Server, *httptest.Server) {
s := &Server{
jobManager: NewJobManager(),
jobManager: NewJobManager(),
collectors: testCollectorRegistry(),
}
mux := http.NewServeMux()
mux.HandleFunc("POST /api/upload", s.handleUpload)