From 4bd9dd67349f0ef1109a967745cd899f6905ace2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 21 Sep 2026 17:27:08 +0100 Subject: [PATCH] rc: allow _logs to be an object to leave out the unattributed logs _logs can now be set to an object, e.g. {"level": "INFO", "unattributed": false} Setting unattributed to false leaves out the logs which aren't attributed to a job, so a client can have just the logs of its own job. It defaults to true which is what _logs did before. --- docs/content/rc.md | 14 ++++++- fs/rc/jobs/job.go | 86 +++++++++++++++++++++++++++--------------- fs/rc/jobs/job_test.go | 26 +++++++++++++ 3 files changed, 95 insertions(+), 31 deletions(-) diff --git a/docs/content/rc.md b/docs/content/rc.md index 1e67696bb..508c860ef 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -591,6 +591,17 @@ otherwise the call will return an error. (e.g. `"INFO"` or `"ERROR"`) to return only the logs at that level or more severe. +It can also be set to an object to control what is returned: + +- `level` - only return the logs at this level or more severe - optional +- `unattributed` - set this to `false` to leave out the logs which + aren't attributed to a job - optional, defaults to `true` + +```console +$ rclone rc sync/copy srcFs=/tmp/src dstFs=/tmp/dst \ + --json '{"_logs": {"level": "INFO", "unattributed": false}}' +``` + The logs are returned as a `_logs` key in the result of the call, or in the error response if the call failed. It has the following keys: @@ -646,7 +657,8 @@ Note that: - `_logs` returns the entries for the job and the unattributed entries made **while** the job was running. The unattributed entries may have come from other jobs, or other rclone activity, running at the same - time. Entries attributed to other jobs are left out. + time - set `unattributed` to `false` to leave them out. Entries + attributed to other jobs are always left out. - The log buffer only contains logs at the current `--log-level`. This is currently a global setting so it can't be changed for a call with `_config`. Use [options/set](#options-set) to change it. diff --git a/fs/rc/jobs/job.go b/fs/rc/jobs/job.go index 84ef96d60..750b3cfb8 100644 --- a/fs/rc/jobs/job.go +++ b/fs/rc/jobs/job.go @@ -56,10 +56,9 @@ type Job struct { // string error message. realErr error - wantLogs bool // set if _logs was passed in - logLevel slog.Level // return logs of this level or more severe - logStart int64 // sequence number in log.Recent when the job started - logEnd int64 // sequence number in log.Recent when the job finished + logsOpt logsOpt // how the logs were asked for with _logs + logStart int64 // sequence number in log.Recent when the job started + logEnd int64 // sequence number in log.Recent when the job finished } // logs returns the logs made while the job was running so far or nil @@ -76,7 +75,7 @@ type Job struct { // // Call with job.mu held. func (job *Job) logs(since int64, limit int) rc.Params { - if !job.wantLogs { + if !job.logsOpt.wanted { return nil } // Never read from before the job started, so since 0 means @@ -86,8 +85,9 @@ func (job *Job) logs(since int64, limit int) rc.Params { if job.Finished { logEnd = job.logEnd } - // Return the job's own logs and any which aren't attributed - filter := log.Filter{Level: job.logLevel, JobID: job.ID, Unattributed: true} + // Return the job's own logs, and any which aren't attributed + // unless they were turned off with _logs unattributed + filter := log.Filter{Level: job.logsOpt.level, JobID: job.ID, Unattributed: job.logsOpt.unattributed} entries, next, lost := log.Recent.Get(since, logEnd, filter, limit) return rc.Params{ "entries": entries, @@ -311,36 +311,62 @@ func LogsRequested(in rc.Params) bool { return err != nil || wantLogs } -// See if _logs is set returning whether the logs are wanted and the -// minimum level of logs to return. +// logsOpt is how the logs of a job were asked for with _logs. +type logsOpt struct { + wanted bool + level slog.Level + unattributed bool // return the logs which aren't attributed to a job too +} + +// logsOptJSON is the object form of the _logs parameter. +type logsOptJSON struct { + Level string `json:"level"` + Unattributed *bool `json:"unattributed"` +} + +// See if _logs is set returning how the logs were asked for. // -// _logs can be a boolean or a log level, e.g. "INFO". -func getLogs(in rc.Params) (wantLogs bool, level slog.Level, err error) { - level = slog.LevelDebug - wantLogs = LogsRequested(in) +// _logs can be a boolean, a log level, e.g. "INFO", or an object, +// e.g. {"level": "INFO", "unattributed": false}. +func getLogs(in rc.Params) (opt logsOpt, err error) { + opt = logsOpt{ + wanted: LogsRequested(in), + level: slog.LevelDebug, + unattributed: true, + } _, boolErr := in.GetBool("_logs") levelString, levelErr := in.GetString("_logs") + var optJSON logsOptJSON + structErr := in.GetStruct("_logs", &optJSON) delete(in, "_logs") // remove the logs parameter after parsing - if !wantLogs { - return false, level, nil + if !opt.wanted { + return logsOpt{}, nil } if !log.Recent.Enabled() { - return false, level, log.ErrBufferDisabled + return logsOpt{}, log.ErrBufferDisabled } - // _logs is true or a level here + // _logs is true, a level or an object here if boolErr != nil { - if levelErr != nil { - return false, level, levelErr + if structErr == nil && (optJSON.Level != "" || optJSON.Unattributed != nil) { + levelString = optJSON.Level + if optJSON.Unattributed != nil { + opt.unattributed = *optJSON.Unattributed + } + if levelString == "" { + return opt, nil + } + } else if levelErr != nil { + return logsOpt{}, levelErr } - level, err = log.ParseLevel(levelString) + opt.level, err = log.ParseLevel(levelString) if err != nil { - return false, level, rc.NewErrParamInvalid(fmt.Errorf("_logs must be a boolean or a log level: %w", err)) + return logsOpt{}, rc.NewErrParamInvalid(fmt.Errorf("_logs must be a boolean, a log level or an object: %w", err)) } - if level >= fs.SlogLevelOff { - return false, level, rc.NewErrParamInvalid(fmt.Errorf("_logs level can't be %q", levelString)) + if opt.level >= fs.SlogLevelOff { + return logsOpt{}, rc.NewErrParamInvalid(fmt.Errorf("_logs level can't be %q", levelString)) } } - return wantLogs, level, nil + return opt, nil } // NewJob creates a Job and executes it, possibly in the background if _async is set @@ -369,7 +395,7 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo return nil, nil, err } - wantLogs, logLevel, err := getLogs(in) + logsOpt, err := getLogs(in) if err != nil { return nil, nil, err } @@ -386,8 +412,7 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo Group: group, StartTime: time.Now(), Stop: stop, - wantLogs: wantLogs, - logLevel: logLevel, + logsOpt: logsOpt, logStart: logStart, } @@ -411,7 +436,7 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo job.run(ctx, fn, in) out = job.Output err = job.realErr - if wantLogs { + if logsOpt.wanted { job.mu.Lock() logs := job.logs(-1, 0) job.mu.Unlock() @@ -484,8 +509,9 @@ Results: - success - boolean - true for success false otherwise - output - output of the job as would have been returned if called synchronously - _logs - the logs of the job if it was started with _logs - - entries - array of log entries, oldest first - the job's own logs and + - entries - array of log entries, oldest first - the job's own logs, and the logs made while it was running which aren't attributed to a job + unless they were turned off with the _logs unattributed parameter - next - pass this as since to carry on reading the logs from where this call finished - lost - the number of log entries which had already been dropped from the log buffer - progress - output of the progress related to the underlying job @@ -510,7 +536,7 @@ func rcJobStatus(ctx context.Context, in rc.Params) (out rc.Params, err error) { if err != nil { return nil, fmt.Errorf("reshape failed in job status: %w", err) } - if job.wantLogs { + if job.logsOpt.wanted { since, err := in.GetInt64("since") if rc.IsErrParamNotFound(err) { since = -1 diff --git a/fs/rc/jobs/job_test.go b/fs/rc/jobs/job_test.go index 9dfe4e585..c1ffdd2f5 100644 --- a/fs/rc/jobs/job_test.go +++ b/fs/rc/jobs/job_test.go @@ -1184,6 +1184,32 @@ func TestExecuteJobWithLogs(t *testing.T) { require.Error(t, err) assert.True(t, rc.IsErrParamInvalid(err), err) + // The object form gives control over what is returned + _, out, err = NewJob(ctx, jobFn, rc.Params{"_logs": rc.Params{"level": "ERROR"}}) + require.NoError(t, err) + assert.Equal(t, []string{"error from job"}, logMessages(t, out["_logs"])) + + // Unattributed logs can be left out + unattributedFn := func(ctx context.Context, in rc.Params) (rc.Params, error) { + fs.LogfCtx(ctx, nil, "attributed") + fs.Logf(nil, "unattributed") + return nil, nil + } + _, out, err = NewJob(ctx, unattributedFn, rc.Params{"_logs": true}) + require.NoError(t, err) + assert.Equal(t, []string{"attributed", "unattributed"}, logMessages(t, out["_logs"])) + _, out, err = NewJob(ctx, unattributedFn, rc.Params{"_logs": rc.Params{"unattributed": false}}) + require.NoError(t, err) + assert.Equal(t, []string{"attributed"}, logMessages(t, out["_logs"])) + _, out, err = NewJob(ctx, unattributedFn, rc.Params{"_logs": rc.Params{"level": "NOTICE", "unattributed": true}}) + require.NoError(t, err) + assert.Equal(t, []string{"attributed", "unattributed"}, logMessages(t, out["_logs"])) + + // A bad level in the object form is an error + _, _, err = NewJob(ctx, jobFn, rc.Params{"_logs": rc.Params{"level": "potato"}}) + require.Error(t, err) + assert.True(t, rc.IsErrParamInvalid(err), err) + // Logs made while parsing the request are included _, out, err = NewJob(ctx, jobFn, rc.Params{ "_logs": "ERROR",