mirror of
https://github.com/KDE/kde-linux.git
synced 2026-04-18 13:40:01 -04:00
add upload-vacuum-v3
cleans up the s3 and prepares for the repo v3 (which accounts for editions in the repo path) Closes #590
This commit is contained in:
Harald Sitter
11
upload-vacuum-v3/artifact.go
Normal file
11
upload-vacuum-v3/artifact.go
Normal file
@@ -0,0 +1,11 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
// SPDX-FileCopyrightText: 2024 Harald Sitter <sitter@kde.org>
|
||||
|
||||
package main
|
||||
|
||||
type Artifact interface {
|
||||
Path() string
|
||||
// A SHA256 string with filename. Separated by two spaces. Never includes dirname!
|
||||
SHA256() string
|
||||
Delete() error
|
||||
}
|
||||
35
upload-vacuum-v3/awsconfig.go
Normal file
35
upload-vacuum-v3/awsconfig.go
Normal file
@@ -0,0 +1,35 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
// SPDX-FileCopyrightText: 2025 Harald Sitter <sitter@kde.org>
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
type AWSSection struct {
|
||||
AccessKeyId string `ini:"aws_access_key_id"`
|
||||
SecretKey string `ini:"aws_secret_access_key"`
|
||||
SessionToken string `ini:"aws_session_token"`
|
||||
}
|
||||
|
||||
func readConfigAWS(section string) (AWSSection, error) {
|
||||
awsSection := AWSSection{}
|
||||
|
||||
awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials")
|
||||
cfg, err := ini.Load(awsConfigPath)
|
||||
if err != nil {
|
||||
return awsSection, errors.New("failed to load AWS credentials file: " + err.Error())
|
||||
}
|
||||
|
||||
err = cfg.Section(section).MapTo(&awsSection)
|
||||
if err != nil {
|
||||
return awsSection, errors.New("failed to map AWS credentials section: " + err.Error())
|
||||
}
|
||||
|
||||
return awsSection, nil
|
||||
}
|
||||
31
upload-vacuum-v3/go.mod
Normal file
31
upload-vacuum-v3/go.mod
Normal file
@@ -0,0 +1,31 @@
|
||||
module invent.kde.org/sitter/kde-linux/upload-vacuum-v3
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/minio/minio-go/v7 v7.0.100
|
||||
github.com/pkg/sftp v1.13.6
|
||||
golang.org/x/crypto v0.50.0
|
||||
gopkg.in/ini.v1 v1.67.1
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
)
|
||||
2
upload-vacuum-v3/go.mod.license
Normal file
2
upload-vacuum-v3/go.mod.license
Normal file
@@ -0,0 +1,2 @@
|
||||
SPDX-License-Identifier: CC0-1.0
|
||||
SPDX-FileCopyrightText: none
|
||||
117
upload-vacuum-v3/go.sum
Normal file
117
upload-vacuum-v3/go.sum
Normal file
@@ -0,0 +1,117 @@
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=
|
||||
github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
|
||||
github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
|
||||
github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8=
|
||||
github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
|
||||
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
|
||||
github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
|
||||
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
|
||||
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
|
||||
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
|
||||
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
|
||||
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k=
|
||||
gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
2
upload-vacuum-v3/go.sum.license
Normal file
2
upload-vacuum-v3/go.sum.license
Normal file
@@ -0,0 +1,2 @@
|
||||
SPDX-License-Identifier: CC0-1.0
|
||||
SPDX-FileCopyrightText: none
|
||||
336
upload-vacuum-v3/main.go
Normal file
336
upload-vacuum-v3/main.go
Normal file
@@ -0,0 +1,336 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
// SPDX-FileCopyrightText: 2024-2026 Harald Sitter <sitter@kde.org>
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type release struct {
|
||||
artifacts []Artifact
|
||||
}
|
||||
|
||||
func connectToMinIO() *minio.Client {
|
||||
endpoint := "storage.kde.org"
|
||||
awsSection, err := readConfigAWS("default")
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to read AWS config:", err)
|
||||
}
|
||||
accessKeyID := awsSection.AccessKeyId
|
||||
secretAccessKey := awsSection.SecretKey
|
||||
useSSL := true
|
||||
|
||||
// Initialize minio client object.
|
||||
minioClient, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
TrailingHeaders: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
buckets, err := minioClient.ListBuckets(context.Background())
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
for _, bucket := range buckets {
|
||||
log.Println(bucket)
|
||||
}
|
||||
|
||||
return minioClient
|
||||
}
|
||||
|
||||
func loadReleasesMinIO(client *minio.Client, dir string, config *config) (releases map[string]release, err error) {
|
||||
releases = make(map[string]release)
|
||||
bucketName := "kde-linux"
|
||||
ctx := context.Background()
|
||||
|
||||
log.Println("Loading releases from MinIO bucket", bucketName)
|
||||
|
||||
objects := client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
|
||||
Prefix: dir,
|
||||
Recursive: false,
|
||||
WithMetadata: true,
|
||||
})
|
||||
for object := range objects {
|
||||
if object.Err != nil {
|
||||
log.Fatalln(object.Err)
|
||||
}
|
||||
|
||||
err = appendRelease(&releases, S3Artifact{
|
||||
client: client,
|
||||
bucket: bucketName,
|
||||
path: object.Key,
|
||||
sha256Sum: object.UserMetadata["X-Amz-Meta-X-Kde-Sha256"],
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
log.Println(releases)
|
||||
return
|
||||
}
|
||||
|
||||
func downloadCaibxFiles(client *minio.Client) (caibxFiles []string, err error) {
|
||||
bucketName := "kde-linux"
|
||||
ctx := context.Background()
|
||||
|
||||
log.Println("Downloading caibx files from", bucketName)
|
||||
|
||||
os.RemoveAll("caibx-files")
|
||||
objects := client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
|
||||
Recursive: true,
|
||||
})
|
||||
for object := range objects {
|
||||
if object.Err != nil {
|
||||
log.Fatalln(object.Err)
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(object.Key, ".caibx") {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Println("Downloading caibx", object.Key)
|
||||
path := filepath.Join("caibx-files", object.Key)
|
||||
err := client.FGetObject(ctx, bucketName, object.Key, path, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
log.Fatalln(errors.New("Failed to download caibx " + object.Key + ": " + err.Error()))
|
||||
}
|
||||
caibxFiles = append(caibxFiles, path)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func writeSHA256s(path string, sha256s []string) {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
for _, sha256 := range sha256s {
|
||||
_, err := file.WriteString(sha256 + "\n")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type config struct {
|
||||
TombstoneImages []string `yaml:"tombstone_images"`
|
||||
GoldenImages []string `yaml:"golden_images"`
|
||||
}
|
||||
|
||||
func readConfig(client *minio.Client) (*config, error) {
|
||||
configFile, err := client.GetObject(context.Background(), "kde-linux", "vacuum.yaml", minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer configFile.Close()
|
||||
|
||||
data, err := io.ReadAll(configFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var config config
|
||||
err = yaml.UnmarshalStrict(data, &config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
func getReleaseFrom(name string) (string, error) {
|
||||
name = strings.TrimPrefix(name, "kdeos_")
|
||||
name = strings.TrimPrefix(name, "kde-linux_")
|
||||
name = strings.SplitN(name, ".", 2)[0]
|
||||
name = strings.SplitN(name, "_", 2)[0]
|
||||
|
||||
_, err := strconv.Atoi(name)
|
||||
if err != nil {
|
||||
return "", errors.New("Failed to parse release number: " + name)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
func appendRelease(releases *map[string]release, artifact Artifact) error {
|
||||
// NOTE: we want to keep the legacy kdeos_ prefix for as long as we have relevant tombstones around. Which is possibly forever.
|
||||
basename := filepath.Base(artifact.Path())
|
||||
if !strings.HasPrefix(basename, "kdeos_") && !strings.HasPrefix(basename, "kde-linux_") {
|
||||
return nil
|
||||
}
|
||||
|
||||
name, err := getReleaseFrom(basename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, ok := (*releases)[name]; !ok {
|
||||
(*releases)[name] = release{}
|
||||
}
|
||||
release := (*releases)[name]
|
||||
release.artifacts = append(release.artifacts, artifact)
|
||||
(*releases)[name] = release
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildDeletionSlice(releases map[string]release, toProtect []string) (toKeep, toDelete []string) {
|
||||
if len(releases) == 0 {
|
||||
log.Println("No releases found")
|
||||
return
|
||||
}
|
||||
|
||||
// Sort releases by key
|
||||
for key := range releases {
|
||||
toKeep = append(toKeep, key)
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(toKeep)))
|
||||
|
||||
for len(toKeep) > 4 {
|
||||
release := toKeep[len(toKeep)-1]
|
||||
// Protect certain releases from deletion
|
||||
if !slices.Contains(toProtect, release) {
|
||||
log.Println("Marking for deletion (unless protected)", release)
|
||||
toDelete = append(toDelete, release)
|
||||
}
|
||||
toKeep = toKeep[:len(toKeep)-1]
|
||||
}
|
||||
// always keep protected version, only appending here for logging reasons. The actual protection is above!
|
||||
toKeep = append(toKeep, toProtect...)
|
||||
return
|
||||
}
|
||||
|
||||
func deleteReleases(releases map[string]release, toKeep, toDelete []string) {
|
||||
for _, key := range toDelete {
|
||||
log.Println("Deleting", key)
|
||||
for _, artifact := range releases[key].artifacts {
|
||||
log.Println("Deleting", artifact.Path())
|
||||
if os.Getenv("VACUUM_REALLY_DELETE") == "1" {
|
||||
err := artifact.Delete()
|
||||
if err != nil {
|
||||
log.Println("Failed to delete", artifact, err)
|
||||
}
|
||||
} else {
|
||||
log.Println("... not really deleting")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range toKeep {
|
||||
log.Println("Keeping", key)
|
||||
}
|
||||
}
|
||||
|
||||
func generateSHA256s(releases map[string]release, toKeep []string, dir string) {
|
||||
sha256s := []string{}
|
||||
for _, key := range toKeep {
|
||||
for _, artifact := range releases[key].artifacts {
|
||||
sha256 := artifact.SHA256()
|
||||
if sha256 != "" {
|
||||
sha256s = append(sha256s, sha256)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(sha256s) == 0 {
|
||||
log.Fatalln("No SHA256s found for", dir)
|
||||
}
|
||||
|
||||
sumsDir := filepath.Join("upload-tree", dir)
|
||||
os.MkdirAll(sumsDir, 0o700)
|
||||
writeSHA256s(filepath.Join(sumsDir, "SHA256SUMS"), sha256s)
|
||||
}
|
||||
|
||||
func main() {
|
||||
minioClient := connectToMinIO()
|
||||
os.Chdir("../") // We get started inside the vacuum dir, move to the root.
|
||||
|
||||
os.RemoveAll("upload-tree") // will be populated by generateSHA256s
|
||||
|
||||
config, err := readConfig(minioClient)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
var toProtect []string
|
||||
for _, release := range config.TombstoneImages {
|
||||
toProtect = append(toProtect, release)
|
||||
}
|
||||
for _, release := range config.GoldenImages {
|
||||
toProtect = append(toProtect, release)
|
||||
}
|
||||
|
||||
// Clean up the sysupdate directories
|
||||
for _, dir := range []string{"testing/sysupdate/v2/"} {
|
||||
releases, err := loadReleasesMinIO(minioClient, dir, config)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
toKeep, toDelete := buildDeletionSlice(releases, toProtect)
|
||||
deleteReleases(releases, toKeep, toDelete)
|
||||
|
||||
generateSHA256s(releases, toKeep, dir)
|
||||
}
|
||||
|
||||
// Clean up the images (.raw and .torrent for download)
|
||||
for _, dir := range []string{"testing/"} {
|
||||
releases, err := loadReleasesMinIO(minioClient, dir, config)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
toKeep, toDelete := buildDeletionSlice(releases, toProtect)
|
||||
deleteReleases(releases, toKeep, toDelete)
|
||||
}
|
||||
|
||||
if os.Getenv("VACUUM_REALLY_DELETE") != "1" {
|
||||
log.Println("Not pruning desync store. set VACUUM_REALLY_DELETE=1 to actually delete")
|
||||
return
|
||||
}
|
||||
|
||||
// Clean up the desync store
|
||||
// TODO move this into its own thing, we only need to run this weekly or so, it is a bit expensive
|
||||
{
|
||||
caibxFiles, err := downloadCaibxFiles(minioClient)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
args := []string{"prune", "--yes", "--store", "s3+https://storage.kde.org/kde-linux/sysupdate/store"}
|
||||
args = append(args, caibxFiles...)
|
||||
cmd := exec.Command("desync", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
log.Fatal("desync prune failed: ", err)
|
||||
}
|
||||
|
||||
log.Println("Ran", cmd.Args)
|
||||
if cmd.ProcessState.ExitCode() != 0 {
|
||||
log.Fatal("desync prune failed. This is a critical problem. Get someone on this immediately!")
|
||||
}
|
||||
}
|
||||
}
|
||||
38
upload-vacuum-v3/s3artifact.go
Normal file
38
upload-vacuum-v3/s3artifact.go
Normal file
@@ -0,0 +1,38 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
// SPDX-FileCopyrightText: 2024 Harald Sitter <sitter@kde.org>
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
type S3Artifact struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
path string
|
||||
sha256Sum string
|
||||
}
|
||||
|
||||
func (a S3Artifact) SHA256() string {
|
||||
if strings.HasSuffix(a.path, ".erofs.caibx") {
|
||||
// Never put the .erofs.caibx files into the SHA256SUMS it triggers a bug.
|
||||
// https://github.com/systemd/systemd/issues/38605
|
||||
return ""
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s %s", a.sha256Sum, filepath.Base(a.path))
|
||||
}
|
||||
|
||||
func (a S3Artifact) Delete() error {
|
||||
return a.client.RemoveObject(context.Background(), a.bucket, a.path, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (a S3Artifact) Path() string {
|
||||
return a.path
|
||||
}
|
||||
16
upload.sh
16
upload.sh
@@ -66,11 +66,23 @@ go -C ../token-redeemer/ run .
|
||||
## Prepare the image upload tree
|
||||
cd ..
|
||||
rm -rf upload-tree
|
||||
mkdir -p upload-tree/sysupdate/v2
|
||||
V2_TREE="upload-tree/sysupdate/v2"
|
||||
mkdir -p "$V2_TREE"
|
||||
|
||||
mv "$OUTDIR"/*.raw "$OUTDIR"/*.torrent upload-tree/
|
||||
mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$OUTDIR"/SHA256SUMS "$OUTDIR"/SHA256SUMS.gpg upload-tree/sysupdate/v2/
|
||||
mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$V2_TREE/"
|
||||
|
||||
### Upload
|
||||
go -C ./token-redeemer/ run .
|
||||
go -C ./uploader/ run . --remote "$S3_TARGET"
|
||||
|
||||
### Clean up previous images (this replaces the upload-tree with only SHA256SUMS!)
|
||||
go -C ./upload-vacuum-v3/ build -o upload-vacuum-v3 .
|
||||
./upload-vacuum-v3/upload-vacuum-v3
|
||||
|
||||
#### Sign SHA256SUMS
|
||||
gpg --homedir="$GNUPGHOME" --output "$V2_TREE/SHA256SUMS.gpg" --detach-sign "$V2_TREE/SHA256SUMS"
|
||||
|
||||
#### Upload SHA256SUMS
|
||||
go -C ./token-redeemer/ run .
|
||||
go -C ./uploader/ run . --remote "$S3_TARGET"
|
||||
|
||||
Reference in New Issue
Block a user