Browse Source
plugin worker: support job type categories (all, default, heavy) (#8547)
plugin worker: support job type categories (all, default, heavy) (#8547)
* plugin worker: add handler registry with job categories
Introduce a self-registration pattern for plugin worker job handlers.
Each handler can register itself via init() with a HandlerFactory that
declares its job type, category (default/heavy), CLI aliases, and a
builder function.
ResolveHandlerFactories accepts a mix of category names ("all",
"default", "heavy") and explicit job type names/aliases, returning the
matching factories. This enables workers to be configured by resource
profile rather than requiring explicit job type enumeration.
* plugin worker: register all handlers via init()
Each job handler now self-registers into the global handler registry
with its canonical job type, category, CLI aliases, and build function:
- vacuum: category=default
- volume_balance: category=default
- admin_script: category=default
- erasure_coding: category=heavy
- iceberg_maintenance: category=heavy
Adding a new job type now only requires adding the init() call in the
handler file itself — no other files need to be touched.
* plugin worker: replace hardcoded job type switch with registry
Remove buildPluginWorkerHandler, parsePluginWorkerJobTypes, and
canonicalPluginWorkerJobType from worker_runtime.go. The simplified
buildPluginWorkerHandlers now delegates to
pluginworker.ResolveHandlerFactories, which resolves category names
("all", "default", "heavy") and explicit job type names/aliases.
The default job type is changed from an explicit list to "all", so new
handlers registered via init() are automatically picked up.
Update all tests to use the new API.
* plugin worker: update CLI help text for job categories
Update the -jobType flag description and command examples to document
category support (all, default, heavy) alongside explicit job type names.
* plugin worker: address review feedback
- Add CategoryAll constant; use typed constants in tokenAsCategory
- Pre-allocate result slice in ResolveHandlerFactories
- Add vacuum aliases (vol.vacuum, volume.vacuum)
- List alias examples (ec, balance, iceberg) in -jobType flag help
- Create handlers aggregator package for subpackage blank imports so
new handler subpackages only need to be added in one place
- Make category tests relationship-based (subset/union checks) instead
of asserting exact handler counts
- Add clarifying comments to worker_test.go and mini_plugin_test.go
listing expected handler names next to count assertions
---------
Co-authored-by: Copilot <copilot@github.com>
master
Chris Lu
9 hours ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 349 additions and 175 deletions
-
18weed/command/mini_plugin_test.go
-
168weed/command/plugin_worker_test.go
-
21weed/command/worker.go
-
105weed/command/worker_runtime.go
-
18weed/command/worker_test.go
-
11weed/plugin/worker/admin_script_handler.go
-
11weed/plugin/worker/erasure_coding_handler.go
-
130weed/plugin/worker/handler_registry.go
-
9weed/plugin/worker/handlers/handlers.go
-
11weed/plugin/worker/iceberg/handler.go
-
11weed/plugin/worker/vacuum_handler.go
-
11weed/plugin/worker/volume_balance_handler.go
@ -1,13 +1,21 @@ |
|||
package command |
|||
|
|||
import "testing" |
|||
import ( |
|||
"testing" |
|||
|
|||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker" |
|||
"google.golang.org/grpc" |
|||
"google.golang.org/grpc/credentials/insecure" |
|||
) |
|||
|
|||
func TestMiniDefaultPluginJobTypes(t *testing.T) { |
|||
jobTypes, err := parsePluginWorkerJobTypes(defaultMiniPluginJobTypes) |
|||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) |
|||
// defaultMiniPluginJobTypes is an explicit list: "vacuum,volume_balance,erasure_coding,admin_script"
|
|||
handlers, err := buildPluginWorkerHandlers(defaultMiniPluginJobTypes, dialOption, int(pluginworker.DefaultMaxExecutionConcurrency), "") |
|||
if err != nil { |
|||
t.Fatalf("parsePluginWorkerJobTypes(mini default) err = %v", err) |
|||
t.Fatalf("buildPluginWorkerHandlers(mini default) err = %v", err) |
|||
} |
|||
if len(jobTypes) != 4 { |
|||
t.Fatalf("expected mini default job types to include 4 handlers, got %v", jobTypes) |
|||
if len(handlers) != 4 { |
|||
t.Fatalf("expected mini default job types to include 4 handlers, got %d", len(handlers)) |
|||
} |
|||
} |
|||
@ -1,13 +1,21 @@ |
|||
package command |
|||
|
|||
import "testing" |
|||
import ( |
|||
"testing" |
|||
|
|||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker" |
|||
"google.golang.org/grpc" |
|||
"google.golang.org/grpc/credentials/insecure" |
|||
) |
|||
|
|||
func TestWorkerDefaultJobTypes(t *testing.T) { |
|||
jobTypes, err := parsePluginWorkerJobTypes(*workerJobType) |
|||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) |
|||
handlers, err := buildPluginWorkerHandlers(*workerJobType, dialOption, int(pluginworker.DefaultMaxExecutionConcurrency), "") |
|||
if err != nil { |
|||
t.Fatalf("parsePluginWorkerJobTypes(default worker flag) err = %v", err) |
|||
t.Fatalf("buildPluginWorkerHandlers(default worker flag) err = %v", err) |
|||
} |
|||
if len(jobTypes) != 5 { |
|||
t.Fatalf("expected default worker job types to include 5 handlers, got %v", jobTypes) |
|||
// Expected: vacuum, volume_balance, admin_script, erasure_coding, iceberg_maintenance
|
|||
if len(handlers) != 5 { |
|||
t.Fatalf("expected default worker job types to include 5 handlers, got %d", len(handlers)) |
|||
} |
|||
} |
|||
@ -0,0 +1,130 @@ |
|||
package pluginworker |
|||
|
|||
import ( |
|||
"fmt" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
// JobCategory groups job types by resource profile so that workers can be
|
|||
// configured with a category name instead of an explicit list of job types.
|
|||
type JobCategory string |
|||
|
|||
const ( |
|||
CategoryAll JobCategory = "all" // pseudo-category matching every handler
|
|||
CategoryDefault JobCategory = "default" // lightweight, safe for any worker
|
|||
CategoryHeavy JobCategory = "heavy" // resource-intensive jobs
|
|||
) |
|||
|
|||
// HandlerFactory describes how to build a JobHandler for a single job type.
|
|||
type HandlerFactory struct { |
|||
// JobType is the canonical job type string (e.g. "vacuum").
|
|||
JobType string |
|||
// Category controls which category label selects this handler.
|
|||
Category JobCategory |
|||
// Aliases are alternative CLI names that resolve to this job type
|
|||
// (e.g. "ec" for "erasure_coding").
|
|||
Aliases []string |
|||
// Build constructs the JobHandler.
|
|||
Build func(opts HandlerBuildOptions) (JobHandler, error) |
|||
} |
|||
|
|||
// HandlerBuildOptions carries parameters forwarded from the CLI to handler
|
|||
// constructors.
|
|||
type HandlerBuildOptions struct { |
|||
GrpcDialOption grpc.DialOption |
|||
MaxExecute int |
|||
WorkingDir string |
|||
} |
|||
|
|||
var ( |
|||
registryMu sync.Mutex |
|||
registry []HandlerFactory |
|||
) |
|||
|
|||
// RegisterHandler adds a handler factory to the global registry.
|
|||
// It is intended to be called from handler init() functions.
|
|||
func RegisterHandler(f HandlerFactory) { |
|||
registryMu.Lock() |
|||
defer registryMu.Unlock() |
|||
registry = append(registry, f) |
|||
} |
|||
|
|||
// ResolveHandlerFactories takes a comma-separated token list that can contain
|
|||
// category names ("all", "default", "heavy") and/or explicit job type names
|
|||
// (including aliases). It returns a deduplicated, ordered slice of factories.
|
|||
func ResolveHandlerFactories(tokens string) ([]HandlerFactory, error) { |
|||
registryMu.Lock() |
|||
snapshot := make([]HandlerFactory, len(registry)) |
|||
copy(snapshot, registry) |
|||
registryMu.Unlock() |
|||
|
|||
parts := strings.Split(tokens, ",") |
|||
result := make([]HandlerFactory, 0, len(snapshot)) |
|||
seen := make(map[string]bool) |
|||
|
|||
for _, raw := range parts { |
|||
tok := strings.ToLower(strings.TrimSpace(raw)) |
|||
if tok == "" { |
|||
continue |
|||
} |
|||
|
|||
if cat, ok := tokenAsCategory(tok); ok { |
|||
for _, f := range snapshot { |
|||
if cat == CategoryAll || f.Category == cat { |
|||
if !seen[f.JobType] { |
|||
seen[f.JobType] = true |
|||
result = append(result, f) |
|||
} |
|||
} |
|||
} |
|||
continue |
|||
} |
|||
|
|||
f, err := findFactory(snapshot, tok) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
if !seen[f.JobType] { |
|||
seen[f.JobType] = true |
|||
result = append(result, f) |
|||
} |
|||
} |
|||
|
|||
if len(result) == 0 { |
|||
return nil, fmt.Errorf("no job types resolved from %q", tokens) |
|||
} |
|||
return result, nil |
|||
} |
|||
|
|||
// tokenAsCategory returns the category and true when tok is a known category
|
|||
// keyword. "all" is treated as a special pseudo-category that matches every
|
|||
// registered handler.
|
|||
func tokenAsCategory(tok string) (JobCategory, bool) { |
|||
switch tok { |
|||
case string(CategoryAll): |
|||
return CategoryAll, true |
|||
case string(CategoryDefault): |
|||
return CategoryDefault, true |
|||
case string(CategoryHeavy): |
|||
return CategoryHeavy, true |
|||
default: |
|||
return "", false |
|||
} |
|||
} |
|||
|
|||
func findFactory(factories []HandlerFactory, tok string) (HandlerFactory, error) { |
|||
for _, f := range factories { |
|||
if strings.EqualFold(f.JobType, tok) { |
|||
return f, nil |
|||
} |
|||
for _, alias := range f.Aliases { |
|||
if strings.EqualFold(alias, tok) { |
|||
return f, nil |
|||
} |
|||
} |
|||
} |
|||
return HandlerFactory{}, fmt.Errorf("unknown job type %q", tok) |
|||
} |
|||
@ -0,0 +1,9 @@ |
|||
// Package handlers is an aggregator that blank-imports every plugin worker
|
|||
// handler subpackage so their init() functions register with the handler
|
|||
// registry. Import this package instead of individual subpackages when you
|
|||
// need all handlers available.
|
|||
package handlers |
|||
|
|||
import ( |
|||
_ "github.com/seaweedfs/seaweedfs/weed/plugin/worker/iceberg" // register iceberg_maintenance handler
|
|||
) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue