mirror/mirror-to-gitea
1
0
Fork 0
mirror of https://github.com/amalgamated-tools/mirror-to-gitea.git synced 2025-12-23 22:18:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

add support for mirroring a single repository and to a specific Gitea organization

Browse Source
This commit is contained in:
Arunavo Ray
2025-04-02 13:57:54 +05:30
parent b8f6c36fad
commit 936246b9a7
4 changed files with 352 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
README.md
View File

@@ -25,6 +25,8 @@ Additionally, you can now mirror:
- Issues from GitHub repositories (including labels)
- Starred repositories from your GitHub account
- Repositories from organizations you belong to
- A single repository instead of all repositories
- Repositories to a specific Gitea organization
## Prerequisites
@@ -44,13 +46,16 @@ All configuration is performed through environment variables. Flags are consider
| GITHUB_USERNAME | yes | string | - | The name of the GitHub user or organisation to mirror. |
| GITEA_URL | yes | string | - | The url of your Gitea server. |
| GITEA_TOKEN | yes | string | - | The token for your gitea user (Settings -> Applications -> Generate New Token). **Attention: if this is set, the token will be transmitted to your specified Gitea instance!** |
| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`, `MIRROR_ISSUES`, `MIRROR_STARRED`, or `MIRROR_ORGANIZATIONS`. |
| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`, `MIRROR_ISSUES`, `MIRROR_STARRED`, `MIRROR_ORGANIZATIONS`, or `SINGLE_REPO`. |
| MIRROR_PRIVATE_REPOSITORIES | no | bool | FALSE | If set to `true` your private GitHub Repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_ISSUES | no | bool | FALSE | If set to `true` the issues of your GitHub repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_STARRED | no | bool | FALSE | If set to `true` repositories you've starred on GitHub will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_ORGANIZATIONS | no | bool | FALSE | If set to `true` repositories from organizations you belong to will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| SINGLE_REPO | no | string | - | URL of a single GitHub repository to mirror (e.g., https://github.com/username/repo or username/repo). When specified, only this repository will be mirrored. Requires `GITHUB_TOKEN`. |
| GITEA_ORGANIZATION | no | string | - | Name of a Gitea organization to mirror repositories to. If doesn't exist, will be created. |
| GITEA_ORG_VISIBILITY | no | string | public | Visibility of the Gitea organization to create. Can be "public" or "private". |
| SKIP_FORKS | no | bool | FALSE | If set to `true` will disable the mirroring of forks from your GitHub User / Organisation. |
| DELAY | no | int | 3600 | Number of seconds between program executions. Setting this will only affect how soon after a new repo was created a mirror may appar on Gitea, but has no affect on the ongoing replication. |
| DELAY | no | int | 3600 | Number of seconds between program executions. Setting this will only affect how soon after a new repo was created a mirror may appear on Gitea, but has no affect on the ongoing replication. |
| DRY_RUN | no | bool | FALSE | If set to `true` will perform no writing changes to your Gitea instance, but log the planned actions. |
| INCLUDE | no | string | "*" | Name based repository filter (include): If any filter matches, the repository will be mirrored. It supports glob format, multiple filters can be separated with commas (`,`) |
| EXCLUDE | no | string | "" | Name based repository filter (exclude). If any filter matches, the repository will not be mirrored. It supports glob format, multiple filters can be separated with commas (`,`). `EXCLUDE` filters are applied after `INCLUDE` ones.
@@ -72,8 +77,34 @@ docker container run \
jaedle/mirror-to-gitea:latest
```
This will a spin up a docker container which will run forever, mirroring all your repositories once every hour to your
gitea server.
### Mirror a Single Repository
```sh
docker container run \
-d \
--restart always \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e SINGLE_REPO=https://github.com/organization/repository \
jaedle/mirror-to-gitea:latest
```
### Mirror to a Gitea Organization
```sh
docker container run \
-d \
--restart always \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e GITEA_ORGANIZATION=my-organization \
-e GITEA_ORG_VISIBILITY=private \
jaedle/mirror-to-gitea:latest
```
### Docker Compose
@@ -92,6 +123,9 @@ services:
- MIRROR_ISSUES=true
- MIRROR_STARRED=true
- MIRROR_ORGANIZATIONS=true
# - SINGLE_REPO=https://github.com/organization/repository
# - GITEA_ORGANIZATION=my-organization
# - GITEA_ORG_VISIBILITY=public
```
## Development
@@ -120,6 +154,9 @@ export GITEA_TOKEN='...'
export MIRROR_ISSUES='true'
export MIRROR_STARRED='true'
export MIRROR_ORGANIZATIONS='true'
# export SINGLE_REPO='https://github.com/user/repo'
# export GITEA_ORGANIZATION='my-organization'
# export GITEA_ORG_VISIBILITY='public'
```
Execute the script in foreground:

7
src/configuration.mjs
View File

@@ -38,10 +38,13 @@ export function configuration() {
mirrorIssues: readBoolean("MIRROR_ISSUES"),
mirrorStarred: readBoolean("MIRROR_STARRED"),
mirrorOrganizations: readBoolean("MIRROR_ORGANIZATIONS"),
singleRepo: readEnv("SINGLE_REPO"),
},
gitea: {
url: mustReadEnv("GITEA_URL"),
token: mustReadEnv("GITEA_TOKEN"),
organization: readEnv("GITEA_ORGANIZATION"),
visibility: readEnv("GITEA_ORG_VISIBILITY") || "public",
},
dryRun: readBoolean("DRY_RUN"),
delay: readInt("DELAY") ?? defaultDelay,
@@ -61,10 +64,10 @@ export function configuration() {
}
// GitHub token is required for mirroring issues, starred repos, and orgs
if ((config.github.mirrorIssues || config.github.mirrorStarred || config.github.mirrorOrganizations)
if ((config.github.mirrorIssues || config.github.mirrorStarred || config.github.mirrorOrganizations || config.github.singleRepo)
&& config.github.token === undefined) {
throw new Error(
"invalid configuration, mirroring issues, starred repositories, or organizations requires setting GITHUB_TOKEN",
"invalid configuration, mirroring issues, starred repositories, organizations, or a single repo requires setting GITHUB_TOKEN",
);
}

133
src/get-github-repositories.mjs
View File

@@ -1,31 +1,81 @@
async function getRepositories(octokit, mirrorOptions) {
const publicRepositories = await fetchPublicRepositories(
octokit,
mirrorOptions.username,
);
const privateRepos = mirrorOptions.privateRepositories
? await fetchPrivateRepositories(octokit)
: [];
let repositories = [];
// Fetch starred repos if the option is enabled
const starredRepos = mirrorOptions.mirrorStarred
? await fetchStarredRepositories(octokit)
: [];
// Fetch organization repos if the option is enabled
const orgRepos = mirrorOptions.mirrorOrganizations
? await fetchOrganizationRepositories(octokit)
: [];
// Combine all repositories and filter duplicates
const repos = filterDuplicates([
...publicRepositories,
...privateRepos,
...starredRepos,
...orgRepos
]);
// Check if we're mirroring a single repo
if (mirrorOptions.singleRepo) {
const singleRepo = await fetchSingleRepository(octokit, mirrorOptions.singleRepo);
if (singleRepo) {
repositories.push(singleRepo);
}
} else {
// Standard mirroring logic
const publicRepositories = await fetchPublicRepositories(
octokit,
mirrorOptions.username,
);
const privateRepos = mirrorOptions.privateRepositories
? await fetchPrivateRepositories(octokit)
: [];
// Fetch starred repos if the option is enabled
const starredRepos = mirrorOptions.mirrorStarred
? await fetchStarredRepositories(octokit)
: [];
// Fetch organization repos if the option is enabled
const orgRepos = mirrorOptions.mirrorOrganizations
? await fetchOrganizationRepositories(octokit)
: [];
// Combine all repositories and filter duplicates
repositories = filterDuplicates([
...publicRepositories,
...privateRepos,
...starredRepos,
...orgRepos
]);
}
return mirrorOptions.skipForks ? withoutForks(repos) : repos;
return mirrorOptions.skipForks ? withoutForks(repositories) : repositories;
}
async function fetchSingleRepository(octokit, repoUrl) {
try {
// Remove URL prefix if present and clean up
let repoPath = repoUrl;
if (repoPath.startsWith('https://github.com/')) {
repoPath = repoPath.replace('https://github.com/', '');
}
if (repoPath.endsWith('.git')) {
repoPath = repoPath.slice(0, -4);
}
// Split into owner and repo
const [owner, repo] = repoPath.split('/');
if (!owner || !repo) {
console.error(`Invalid repository URL format: ${repoUrl}`);
return null;
}
// Fetch the repository details
const response = await octokit.rest.repos.get({
owner,
repo
});
return {
name: response.data.name,
url: response.data.clone_url,
private: response.data.private,
fork: response.data.fork,
owner: response.data.owner.login,
full_name: response.data.full_name,
has_issues: response.data.has_issues,
};
} catch (error) {
console.error(`Error fetching single repository ${repoUrl}:`, error.message);
return null;
}
}
async function fetchPublicRepositories(octokit, username) {
@@ -50,20 +100,25 @@ async function fetchStarredRepositories(octokit) {
}
async function fetchOrganizationRepositories(octokit) {
// First get all organizations the user belongs to
const orgs = await octokit.paginate("GET /user/orgs");
// Then fetch repositories for each organization
const orgRepoPromises = orgs.map(org =>
octokit.paginate("GET /orgs/{org}/repos", { org: org.login })
);
// Wait for all requests to complete and flatten the results
const orgRepos = await Promise.all(orgRepoPromises)
.then(repoArrays => repoArrays.flat())
.then(toRepositoryList);
return orgRepos;
try {
// First get all organizations the user belongs to
const orgs = await octokit.paginate("GET /user/orgs");
// Then fetch repositories for each organization
const orgRepoPromises = orgs.map(org =>
octokit.paginate("GET /orgs/{org}/repos", { org: org.login })
);
// Wait for all requests to complete and flatten the results
const orgRepos = await Promise.all(orgRepoPromises)
.then(repoArrays => repoArrays.flat())
.then(toRepositoryList);
return orgRepos;
} catch (error) {
console.error("Error fetching organization repositories:", error.message);
return [];
}
}
function withoutForks(repositories) {

340
src/index.mjs
View File

@@ -6,36 +6,205 @@ import { configuration } from "./configuration.mjs";
import { Logger } from "./logger.js";
import getGithubRepositories from "./get-github-repositories.mjs";
async function getGithubRepositories(
username,
token,
mirrorPrivateRepositories,
mirrorForks,
mirrorStarred,
mirrorOrganizations,
include,
exclude,
) {
async function main() {
let config;
try {
config = configuration();
} catch (e) {
console.error("invalid configuration", e);
process.exit(1);
}
const logger = new Logger();
logger.showConfig(config);
// Create Gitea organization if specified
if (config.gitea.organization) {
await createGiteaOrganization(
{
url: config.gitea.url,
token: config.gitea.token,
},
config.gitea.organization,
config.gitea.visibility,
config.dryRun
);
}
const octokit = new Octokit({
auth: token || null,
auth: config.github.token || null,
});
const repositories = await getGithubRepositories(octokit, {
username,
privateRepositories: mirrorPrivateRepositories,
skipForks: !mirrorForks,
mirrorStarred,
mirrorOrganizations,
// Get user or organization repositories
const githubRepositories = await getGithubRepositories(octokit, {
username: config.github.username,
privateRepositories: config.github.privateRepositories,
skipForks: config.github.skipForks,
mirrorStarred: config.github.mirrorStarred,
mirrorOrganizations: config.github.mirrorOrganizations,
singleRepo: config.github.singleRepo,
});
return repositories.filter(
// Apply include/exclude filters
const filteredRepositories = githubRepositories.filter(
(repository) =>
include.some((f) => minimatch(repository.name, f)) &&
!exclude.some((f) => minimatch(repository.name, f)),
config.include.some((f) => minimatch(repository.name, f)) &&
!config.exclude.some((f) => minimatch(repository.name, f)),
);
console.log(`Found ${filteredRepositories.length} repositories to mirror`);
const gitea = {
url: config.gitea.url,
token: config.gitea.token,
};
// Get Gitea user or organization ID
const giteaTarget = config.gitea.organization
? await getGiteaOrganization(gitea, config.gitea.organization)
: await getGiteaUser(gitea);
if (!giteaTarget) {
console.error("Failed to get Gitea user or organization. Exiting.");
process.exit(1);
}
// Mirror repositories
const queue = new PQueue({ concurrency: 4 });
await queue.addAll(
filteredRepositories.map((repository) => {
return async () => {
await mirror(
repository,
gitea,
giteaTarget,
config.github.token,
config.github.mirrorIssues,
config.dryRun,
);
};
}),
);
}
// Fetch issues for a given repository
// Get Gitea user information
async function getGiteaUser(gitea) {
try {
const response = await request
.get(`${gitea.url}/api/v1/user`)
.set("Authorization", `token ${gitea.token}`);
return {
id: response.body.id,
name: response.body.username,
type: "user"
};
} catch (error) {
console.error("Error fetching Gitea user:", error.message);
return null;
}
}
// Get Gitea organization information
async function getGiteaOrganization(gitea, orgName) {
try {
const response = await request
.get(`${gitea.url}/api/v1/orgs/${orgName}`)
.set("Authorization", `token ${gitea.token}`);
return {
id: response.body.id,
name: orgName,
type: "organization"
};
} catch (error) {
console.error(`Error fetching Gitea organization ${orgName}:`, error.message);
return null;
}
}
// Create a Gitea organization
async function createGiteaOrganization(gitea, orgName, visibility, dryRun) {
if (dryRun) {
console.log(`DRY RUN: Would create Gitea organization: ${orgName} (${visibility})`);
return true;
}
try {
// First check if organization already exists
try {
const existingOrg = await request
.get(`${gitea.url}/api/v1/orgs/${orgName}`)
.set("Authorization", `token ${gitea.token}`);
console.log(`Organization ${orgName} already exists`);
return true;
} catch (checkError) {
// Organization doesn't exist, continue to create it
}
const response = await request
.post(`${gitea.url}/api/v1/orgs`)
.set("Authorization", `token ${gitea.token}`)
.send({
username: orgName,
visibility: visibility || "public",
});
console.log(`Created organization: ${orgName}`);
return true;
} catch (error) {
// 422 error typically means the organization already exists
if (error.status === 422) {
console.log(`Organization ${orgName} already exists`);
return true;
}
console.error(`Error creating Gitea organization ${orgName}:`, error.message);
return false;
}
}
// Check if repository is already mirrored
async function isAlreadyMirroredOnGitea(repository, gitea, giteaTarget) {
const repoName = repository.name;
const ownerName = giteaTarget.name;
const requestUrl = `${gitea.url}/api/v1/repos/${ownerName}/${repoName}`;
try {
await request
.get(requestUrl)
.set("Authorization", `token ${gitea.token}`);
return true;
} catch (error) {
return false;
}
}
// Mirror repository to Gitea
async function mirrorOnGitea(repository, gitea, giteaTarget, githubToken) {
try {
const response = await request
.post(`${gitea.url}/api/v1/repos/migrate`)
.set("Authorization", `token ${gitea.token}`)
.send({
auth_token: githubToken || null,
clone_addr: repository.url,
mirror: true,
repo_name: repository.name,
uid: giteaTarget.id,
private: repository.private,
});
console.log(`Successfully mirrored: ${repository.name}`);
return response.body;
} catch (error) {
console.error(`Failed to mirror ${repository.name}:`, error.message);
throw error;
}
}
// Fetch issues for a repository
async function getGithubIssues(octokit, owner, repo) {
try {
const issues = await octokit.paginate("GET /repos/{owner}/{repo}/issues", {
@@ -62,54 +231,11 @@ async function getGithubIssues(octokit, owner, repo) {
}
}
async function getGiteaUser(gitea) {
return request
.get(`${gitea.url}/api/v1/user`)
.set("Authorization", `token ${gitea.token}`)
.then((response) => {
return { id: response.body.id, name: response.body.username };
});
}
function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) {
const repoName = repository.name;
const ownerName = giteaUser.name;
const requestUrl = `${gitea.url}/api/v1/repos/${ownerName}/${repoName}`;
return request
.get(requestUrl)
.set("Authorization", `token ${gitea.token}`)
.then(() => true)
.catch(() => false);
}
function mirrorOnGitea(repository, gitea, giteaUser, githubToken) {
return request
.post(`${gitea.url}/api/v1/repos/migrate`)
.set("Authorization", `token ${gitea.token}`)
.send({
auth_token: githubToken || null,
clone_addr: repository.url,
mirror: true,
repo_name: repository.name,
uid: giteaUser.id,
private: repository.private,
})
.then((response) => {
console.log(`Successfully mirrored: ${repository.name}`);
return response.body;
})
.catch((err) => {
console.log(`Failed to mirror ${repository.name}:`, err.message);
throw err;
});
}
// Create an issue in a Gitea repository
async function createGiteaIssue(issue, repository, gitea, giteaUser) {
async function createGiteaIssue(issue, repository, gitea, giteaTarget) {
try {
const response = await request
.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues`)
.post(`${gitea.url}/api/v1/repos/${giteaTarget.name}/${repository.name}/issues`)
.set("Authorization", `token ${gitea.token}`)
.send({
title: issue.title,
@@ -126,7 +252,7 @@ async function createGiteaIssue(issue, repository, gitea, giteaUser) {
try {
// First try to create the label if it doesn't exist
await request
.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/labels`)
.post(`${gitea.url}/api/v1/repos/${giteaTarget.name}/${repository.name}/labels`)
.set("Authorization", `token ${gitea.token}`)
.send({
name: label,
@@ -138,7 +264,7 @@ async function createGiteaIssue(issue, repository, gitea, giteaUser) {
// Then add the label to the issue
await request
.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues/${response.body.number}/labels`)
.post(`${gitea.url}/api/v1/repos/${giteaTarget.name}/${repository.name}/issues/${response.body.number}/labels`)
.set("Authorization", `token ${gitea.token}`)
.send({
labels: [label]
@@ -156,7 +282,8 @@ async function createGiteaIssue(issue, repository, gitea, giteaUser) {
}
}
async function mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun) {
// Mirror issues for a repository
async function mirrorIssues(repository, gitea, giteaTarget, githubToken, dryRun) {
if (!repository.has_issues) {
console.log(`Repository ${repository.name} doesn't have issues enabled. Skipping issues mirroring.`);
return;
@@ -176,7 +303,7 @@ async function mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun) {
// Create issues one by one to maintain order
for (const issue of issues) {
await createGiteaIssue(issue, repository, gitea, giteaUser);
await createGiteaIssue(issue, repository, gitea, giteaTarget);
}
console.log(`Completed mirroring issues for ${repository.name}`);
@@ -185,77 +312,34 @@ async function mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun) {
}
}
async function mirror(repository, gitea, giteaUser, githubToken, mirrorIssues, dryRun) {
if (await isAlreadyMirroredOnGitea(repository, gitea, giteaUser)) {
// Mirror a repository
async function mirror(repository, gitea, giteaTarget, githubToken, mirrorIssuesFlag, dryRun) {
if (await isAlreadyMirroredOnGitea(repository, gitea, giteaTarget)) {
console.log(
"Repository is already mirrored; doing nothing: ",
repository.name,
`Repository ${repository.name} is already mirrored; doing nothing.`
);
return;
}
if (dryRun) {
console.log("DRY RUN: Would mirror repository to gitea: ", repository);
console.log(`DRY RUN: Would mirror repository to gitea: ${repository.name}`);
return;
}
console.log("Mirroring repository to gitea: ", repository.name);
console.log(`Mirroring repository to gitea: ${repository.name}`);
try {
await mirrorOnGitea(repository, gitea, giteaUser, githubToken);
await mirrorOnGitea(repository, gitea, giteaTarget, githubToken);
// Mirror issues if requested and not in dry run mode
if (mirrorIssues && !dryRun) {
await mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun);
if (mirrorIssuesFlag && !dryRun) {
await mirrorIssues(repository, gitea, giteaTarget, githubToken, dryRun);
}
} catch (error) {
console.error(`Error during mirroring of ${repository.name}:`, error.message);
}
}
async function main() {
let config;
try {
config = configuration();
} catch (e) {
console.error("invalid configuration", e);
process.exit(1);
}
const logger = new Logger();
logger.showConfig(config);
const githubRepositories = await getGithubRepositories(
config.github.username,
config.github.token,
config.github.privateRepositories,
!config.github.skipForks,
config.github.mirrorStarred,
config.github.mirrorOrganizations,
config.include,
config.exclude,
);
console.log(`Found ${githubRepositories.length} repositories on github`);
const gitea = {
url: config.gitea.url,
token: config.gitea.token,
};
const giteaUser = await getGiteaUser(gitea);
const queue = new PQueue({ concurrency: 4 });
await queue.addAll(
githubRepositories.map((repository) => {
return async () => {
await mirror(
repository,
gitea,
giteaUser,
config.github.token,
config.github.mirrorIssues,
config.dryRun,
);
};
}),
);
}
main();
main().catch(error => {
console.error("Application error:", error);
process.exit(1);
});