diff --git a/upload-vacuum-v3/artifact.go b/upload-vacuum-v3/artifact.go new file mode 100644 index 0000000..10bd1d1 --- /dev/null +++ b/upload-vacuum-v3/artifact.go @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2024 Harald Sitter + +package main + +type Artifact interface { + Path() string + // A SHA256 string with filename. Separated by two spaces. Never includes dirname! + SHA256() string + Delete() error +} diff --git a/upload-vacuum-v3/awsconfig.go b/upload-vacuum-v3/awsconfig.go new file mode 100644 index 0000000..74a797b --- /dev/null +++ b/upload-vacuum-v3/awsconfig.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2025 Harald Sitter + +package main + +import ( + "errors" + "os" + "path/filepath" + + "gopkg.in/ini.v1" +) + +type AWSSection struct { + AccessKeyId string `ini:"aws_access_key_id"` + SecretKey string `ini:"aws_secret_access_key"` + SessionToken string `ini:"aws_session_token"` +} + +func readConfigAWS(section string) (AWSSection, error) { + awsSection := AWSSection{} + + awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials") + cfg, err := ini.Load(awsConfigPath) + if err != nil { + return awsSection, errors.New("failed to load AWS credentials file: " + err.Error()) + } + + err = cfg.Section(section).MapTo(&awsSection) + if err != nil { + return awsSection, errors.New("failed to map AWS credentials section: " + err.Error()) + } + + return awsSection, nil +} diff --git a/upload-vacuum-v3/go.mod b/upload-vacuum-v3/go.mod new file mode 100644 index 0000000..1f6bc1c --- /dev/null +++ b/upload-vacuum-v3/go.mod @@ -0,0 +1,31 @@ +module invent.kde.org/sitter/kde-linux/upload-vacuum-v3 + +go 1.25.0 + +require ( + github.com/minio/minio-go/v7 v7.0.100 + github.com/pkg/sftp v1.13.6 + golang.org/x/crypto v0.50.0 + gopkg.in/ini.v1 v1.67.1 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect +) diff --git a/upload-vacuum-v3/go.mod.license b/upload-vacuum-v3/go.mod.license new file mode 100644 index 0000000..23c1849 --- /dev/null +++ b/upload-vacuum-v3/go.mod.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: none diff --git a/upload-vacuum-v3/go.sum b/upload-vacuum-v3/go.sum new file mode 100644 index 0000000..0acd8c2 --- /dev/null +++ b/upload-vacuum-v3/go.sum @@ -0,0 +1,117 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8= +github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= +github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= +gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/upload-vacuum-v3/go.sum.license b/upload-vacuum-v3/go.sum.license new file mode 100644 index 0000000..23c1849 --- /dev/null +++ b/upload-vacuum-v3/go.sum.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: none diff --git a/upload-vacuum-v3/main.go b/upload-vacuum-v3/main.go new file mode 100644 index 0000000..31b66e6 --- /dev/null +++ b/upload-vacuum-v3/main.go @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2024-2026 Harald Sitter + +package main + +import ( + "context" + "errors" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + + "gopkg.in/yaml.v2" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type release struct { + artifacts []Artifact +} + +func connectToMinIO() *minio.Client { + endpoint := "storage.kde.org" + awsSection, err := readConfigAWS("default") + if err != nil { + log.Fatalln("Failed to read AWS config:", err) + } + accessKeyID := awsSection.AccessKeyId + secretAccessKey := awsSection.SecretKey + useSSL := true + + // Initialize minio client object. + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + TrailingHeaders: true, + }) + if err != nil { + log.Fatalln(err) + } + + buckets, err := minioClient.ListBuckets(context.Background()) + if err != nil { + log.Fatalln(err) + } + for _, bucket := range buckets { + log.Println(bucket) + } + + return minioClient +} + +func loadReleasesMinIO(client *minio.Client, dir string, config *config) (releases map[string]release, err error) { + releases = make(map[string]release) + bucketName := "kde-linux" + ctx := context.Background() + + log.Println("Loading releases from MinIO bucket", bucketName) + + objects := client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ + Prefix: dir, + Recursive: false, + WithMetadata: true, + }) + for object := range objects { + if object.Err != nil { + log.Fatalln(object.Err) + } + + err = appendRelease(&releases, S3Artifact{ + client: client, + bucket: bucketName, + path: object.Key, + sha256Sum: object.UserMetadata["X-Amz-Meta-X-Kde-Sha256"], + }) + if err != nil { + return + } + } + + log.Println(releases) + return +} + +func downloadCaibxFiles(client *minio.Client) (caibxFiles []string, err error) { + bucketName := "kde-linux" + ctx := context.Background() + + log.Println("Downloading caibx files from", bucketName) + + os.RemoveAll("caibx-files") + objects := client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ + Recursive: true, + }) + for object := range objects { + if object.Err != nil { + log.Fatalln(object.Err) + } + + if !strings.HasSuffix(object.Key, ".caibx") { + continue + } + + log.Println("Downloading caibx", object.Key) + path := filepath.Join("caibx-files", object.Key) + err := client.FGetObject(ctx, bucketName, object.Key, path, minio.GetObjectOptions{}) + if err != nil { + log.Fatalln(errors.New("Failed to download caibx " + object.Key + ": " + err.Error())) + } + caibxFiles = append(caibxFiles, path) + } + + return +} + +func writeSHA256s(path string, sha256s []string) { + file, err := os.Create(path) + if err != nil { + log.Fatal(err) + } + defer file.Close() + for _, sha256 := range sha256s { + _, err := file.WriteString(sha256 + "\n") + if err != nil { + log.Fatal(err) + } + } +} + +type config struct { + TombstoneImages []string `yaml:"tombstone_images"` + GoldenImages []string `yaml:"golden_images"` +} + +func readConfig(client *minio.Client) (*config, error) { + configFile, err := client.GetObject(context.Background(), "kde-linux", "vacuum.yaml", minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer configFile.Close() + + data, err := io.ReadAll(configFile) + if err != nil { + return nil, err + } + + var config config + err = yaml.UnmarshalStrict(data, &config) + if err != nil { + return nil, err + } + + return &config, nil +} + +func getReleaseFrom(name string) (string, error) { + name = strings.TrimPrefix(name, "kdeos_") + name = strings.TrimPrefix(name, "kde-linux_") + name = strings.SplitN(name, ".", 2)[0] + name = strings.SplitN(name, "_", 2)[0] + + _, err := strconv.Atoi(name) + if err != nil { + return "", errors.New("Failed to parse release number: " + name) + } + return name, nil +} + +func appendRelease(releases *map[string]release, artifact Artifact) error { + // NOTE: we want to keep the legacy kdeos_ prefix for as long as we have relevant tombstones around. Which is possibly forever. + basename := filepath.Base(artifact.Path()) + if !strings.HasPrefix(basename, "kdeos_") && !strings.HasPrefix(basename, "kde-linux_") { + return nil + } + + name, err := getReleaseFrom(basename) + if err != nil { + return err + } + + if _, ok := (*releases)[name]; !ok { + (*releases)[name] = release{} + } + release := (*releases)[name] + release.artifacts = append(release.artifacts, artifact) + (*releases)[name] = release + return nil +} + +func buildDeletionSlice(releases map[string]release, toProtect []string) (toKeep, toDelete []string) { + if len(releases) == 0 { + log.Println("No releases found") + return + } + + // Sort releases by key + for key := range releases { + toKeep = append(toKeep, key) + } + sort.Sort(sort.Reverse(sort.StringSlice(toKeep))) + + for len(toKeep) > 4 { + release := toKeep[len(toKeep)-1] + // Protect certain releases from deletion + if !slices.Contains(toProtect, release) { + log.Println("Marking for deletion (unless protected)", release) + toDelete = append(toDelete, release) + } + toKeep = toKeep[:len(toKeep)-1] + } + // always keep protected version, only appending here for logging reasons. The actual protection is above! + toKeep = append(toKeep, toProtect...) + return +} + +func deleteReleases(releases map[string]release, toKeep, toDelete []string) { + for _, key := range toDelete { + log.Println("Deleting", key) + for _, artifact := range releases[key].artifacts { + log.Println("Deleting", artifact.Path()) + if os.Getenv("VACUUM_REALLY_DELETE") == "1" { + err := artifact.Delete() + if err != nil { + log.Println("Failed to delete", artifact, err) + } + } else { + log.Println("... not really deleting") + } + } + } + + for _, key := range toKeep { + log.Println("Keeping", key) + } +} + +func generateSHA256s(releases map[string]release, toKeep []string, dir string) { + sha256s := []string{} + for _, key := range toKeep { + for _, artifact := range releases[key].artifacts { + sha256 := artifact.SHA256() + if sha256 != "" { + sha256s = append(sha256s, sha256) + } + } + } + + if len(sha256s) == 0 { + log.Fatalln("No SHA256s found for", dir) + } + + sumsDir := filepath.Join("upload-tree", dir) + os.MkdirAll(sumsDir, 0o700) + writeSHA256s(filepath.Join(sumsDir, "SHA256SUMS"), sha256s) +} + +func main() { + minioClient := connectToMinIO() + os.Chdir("../") // We get started inside the vacuum dir, move to the root. + + os.RemoveAll("upload-tree") // will be populated by generateSHA256s + + config, err := readConfig(minioClient) + if err != nil { + log.Fatal(err) + } + + var toProtect []string + for _, release := range config.TombstoneImages { + toProtect = append(toProtect, release) + } + for _, release := range config.GoldenImages { + toProtect = append(toProtect, release) + } + + // Clean up the sysupdate directories + for _, dir := range []string{"testing/sysupdate/v2/"} { + releases, err := loadReleasesMinIO(minioClient, dir, config) + if err != nil { + log.Fatal(err) + } + + toKeep, toDelete := buildDeletionSlice(releases, toProtect) + deleteReleases(releases, toKeep, toDelete) + + generateSHA256s(releases, toKeep, dir) + } + + // Clean up the images (.raw and .torrent for download) + for _, dir := range []string{"testing/"} { + releases, err := loadReleasesMinIO(minioClient, dir, config) + if err != nil { + log.Fatal(err) + } + + toKeep, toDelete := buildDeletionSlice(releases, toProtect) + deleteReleases(releases, toKeep, toDelete) + } + + if os.Getenv("VACUUM_REALLY_DELETE") != "1" { + log.Println("Not pruning desync store. set VACUUM_REALLY_DELETE=1 to actually delete") + return + } + + // Clean up the desync store + // TODO move this into its own thing, we only need to run this weekly or so, it is a bit expensive + { + caibxFiles, err := downloadCaibxFiles(minioClient) + if err != nil { + log.Fatal(err) + } + + args := []string{"prune", "--yes", "--store", "s3+https://storage.kde.org/kde-linux/sysupdate/store"} + args = append(args, caibxFiles...) + cmd := exec.Command("desync", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err = cmd.Run() + if err != nil { + log.Fatal("desync prune failed: ", err) + } + + log.Println("Ran", cmd.Args) + if cmd.ProcessState.ExitCode() != 0 { + log.Fatal("desync prune failed. This is a critical problem. Get someone on this immediately!") + } + } +} diff --git a/upload-vacuum-v3/s3artifact.go b/upload-vacuum-v3/s3artifact.go new file mode 100644 index 0000000..f139035 --- /dev/null +++ b/upload-vacuum-v3/s3artifact.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2024 Harald Sitter + +package main + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/minio/minio-go/v7" +) + +type S3Artifact struct { + client *minio.Client + bucket string + path string + sha256Sum string +} + +func (a S3Artifact) SHA256() string { + if strings.HasSuffix(a.path, ".erofs.caibx") { + // Never put the .erofs.caibx files into the SHA256SUMS it triggers a bug. + // https://github.com/systemd/systemd/issues/38605 + return "" + } + + return fmt.Sprintf("%s %s", a.sha256Sum, filepath.Base(a.path)) +} + +func (a S3Artifact) Delete() error { + return a.client.RemoveObject(context.Background(), a.bucket, a.path, minio.RemoveObjectOptions{}) +} + +func (a S3Artifact) Path() string { + return a.path +} diff --git a/upload.sh b/upload.sh index 29f80ce..05a5314 100755 --- a/upload.sh +++ b/upload.sh @@ -66,11 +66,23 @@ go -C ../token-redeemer/ run . ## Prepare the image upload tree cd .. rm -rf upload-tree -mkdir -p upload-tree/sysupdate/v2 +V2_TREE="upload-tree/sysupdate/v2" +mkdir -p "$V2_TREE" mv "$OUTDIR"/*.raw "$OUTDIR"/*.torrent upload-tree/ -mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$OUTDIR"/SHA256SUMS "$OUTDIR"/SHA256SUMS.gpg upload-tree/sysupdate/v2/ +mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$V2_TREE/" ### Upload go -C ./token-redeemer/ run . go -C ./uploader/ run . --remote "$S3_TARGET" + +### Clean up previous images (this replaces the upload-tree with only SHA256SUMS!) +go -C ./upload-vacuum-v3/ build -o upload-vacuum-v3 . +./upload-vacuum-v3/upload-vacuum-v3 + +#### Sign SHA256SUMS +gpg --homedir="$GNUPGHOME" --output "$V2_TREE/SHA256SUMS.gpg" --detach-sign "$V2_TREE/SHA256SUMS" + +#### Upload SHA256SUMS +go -C ./token-redeemer/ run . +go -C ./uploader/ run . --remote "$S3_TARGET"