mirror/WowUp
1
0
Fork 0
mirror of https://github.com/WowUp/WowUp.git synced 2026-04-20 05:48:01 -04:00
Code Issues Packages Projects Releases Wiki Activity

Curse scanning working on windows.

Browse Source
This commit is contained in:
jliddev
2020-09-16 22:23:20 -05:00
parent 031964a287
commit bcc39d626e
47 changed files with 870 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
wowup-electron/.gitignore vendored
View File

@@ -7,6 +7,7 @@
/app-builds
/release
main.js
ipc-events.js
src/**/*.js
!src/karma.conf.js
*.js.map

34
wowup-electron/.vscode/launch.json vendored
View File

@@ -1,17 +1,19 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Debug Main Process",
"type": "node",
"request": "launch",
"cwd": "${workspaceFolder}",
"runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron",
"windows": {
"runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron.cmd"
},
"args" : ["."],
"outputCapture": "std"
}
]
}
"version": "0.2.0",
"configurations": [
{
"name": "Debug Main Process",
"type": "node",
"request": "launch",
"cwd": "${workspaceFolder}",
"runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron",
"windows": {
"runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron.cmd"
},
"args": [
"."
],
"outputCapture": "std"
}
]
}

47
wowup-electron/ipc-events.ts Normal file
View File

@@ -0,0 +1,47 @@
import { ipcMain } from "electron";
import * as fs from 'fs';
import { CURSE_HASH_FILE_CHANNEL } from './src/common/constants';
import { CurseHashFileRequest } from './src/common/models/curse-hash-file-request';
import { CurseHashFileResponse } from './src/common/models/curse-hash-file-response';
const nativeAddon = require('./build/Debug/addon.node');
let _win: Electron.BrowserWindow;
export function setIpcEventsWindow(window: Electron.BrowserWindow) {
_win = window;
}
ipcMain.on(CURSE_HASH_FILE_CHANNEL, async (evt, arg: CurseHashFileRequest) => {
// console.log(CURSE_HASH_FILE_CHANNEL, arg);
const response: CurseHashFileResponse = {
fingerprint: 0
};
try {
if (arg.targetString !== undefined) {
const strBuffer = Buffer.from(arg.targetString, arg.targetStringEncoding || 'ascii');
const hash = nativeAddon.computeHash(strBuffer, strBuffer.length);
response.fingerprint = hash;
_win?.webContents?.send(arg.responseKey, response);
} else {
fs.readFile(arg.filePath, (err, buffer) => {
if (err) {
response.error = err;
_win?.webContents?.send(arg.responseKey, response);
return;
}
const hash = nativeAddon.computeHash(buffer, buffer.length);
response.fingerprint = hash;
_win?.webContents?.send(arg.responseKey, response);
});
}
} catch (err) {
console.error(err);
console.log(arg);
response.error = err;
_win?.webContents?.send(arg.responseKey, response);
}
});

56
wowup-electron/main.ts
View File

@@ -9,7 +9,7 @@ import { DownloadRequest } from './src/common/models/download-request';
import { DownloadStatus } from './src/common/models/download-status';
import { DownloadStatusType } from './src/common/models/download-status-type';
import { UnzipStatus } from './src/common/models/unzip-status';
import { DOWNLOAD_FILE_CHANNEL, UNZIP_FILE_CHANNEL, COPY_FILE_CHANNEL, COPY_DIRECTORY_CHANNEL, DELETE_DIRECTORY_CHANNEL, RENAME_DIRECTORY_CHANNEL, READ_FILE_CHANNEL } from './src/common/constants';
import { DOWNLOAD_FILE_CHANNEL, UNZIP_FILE_CHANNEL, COPY_FILE_CHANNEL, COPY_DIRECTORY_CHANNEL, DELETE_DIRECTORY_CHANNEL, RENAME_DIRECTORY_CHANNEL, READ_FILE_CHANNEL, STAT_DIRECTORY_CHANNEL, LIST_FILES_CHANNEL } from './src/common/constants';
import { UnzipStatusType } from './src/common/models/unzip-status-type';
import { UnzipRequest } from './src/common/models/unzip-request';
import { CopyFileRequest } from './src/common/models/copy-file-request';
@@ -17,8 +17,17 @@ import { CopyDirectoryRequest } from './src/common/models/copy-directory-request
import { DeleteDirectoryRequest } from './src/common/models/delete-directory-request';
import { ReadFileRequest } from './src/common/models/read-file-request';
import { ReadFileResponse } from './src/common/models/read-file-response';
import { ListFilesRequest } from './src/common/models/list-files-request';
import { ListFilesResponse } from './src/common/models/list-files-response';
import { ncp } from 'ncp';
import * as rimraf from 'rimraf';
import { setIpcEventsWindow } from './ipc-events';
const nativeAddon = require('./build/Debug/addon.node');
const b1 = fs.readFileSync('c:\\program files (x86)\\world of warcraft\\_retail_\\interface\\addons\\aap-core\\banners.lua')
console.log(b1.length);
console.log('addon', nativeAddon.computeHash(b1, b1.length));
app.commandLine.appendSwitch('disable-features', 'OutOfBlinkCors');
electronDl();
@@ -121,6 +130,8 @@ function createWindow(): BrowserWindow {
// win.show();
// });
setIpcEventsWindow(win);
return win;
}
@@ -235,7 +246,7 @@ ipcMain.on(RENAME_DIRECTORY_CHANNEL, async (evt, arg: CopyDirectoryRequest) => {
});
ipcMain.on(READ_FILE_CHANNEL, async (evt, arg: ReadFileRequest) => {
console.log('Read File', arg);
// console.log('Read File', arg);
fs.readFile(arg.sourcePath, { encoding: 'utf-8' }, (err, data) => {
const response: ReadFileResponse = {
data: data,
@@ -243,4 +254,43 @@ ipcMain.on(READ_FILE_CHANNEL, async (evt, arg: ReadFileRequest) => {
}
win.webContents.send(arg.sourcePath, response);
});
});
});
ipcMain.on(LIST_FILES_CHANNEL, async (evt, arg: ListFilesRequest) => {
console.log('list files', arg);
const response: ListFilesResponse = {
files: []
};
try {
response.files = await readDirRecursive(arg.sourcePath);
} catch (err) {
response.error = err;
}
win.webContents.send(arg.sourcePath, response);
});
async function readDirRecursive(sourcePath: string): Promise<string[]> {
return new Promise((resolve, reject) => {
const dirFiles: string[] = [];
fs.readdir(sourcePath, { withFileTypes: true }, async (err, files) => {
if (err) {
return reject(err);
}
for (let file of files) {
const filePath = path.join(sourcePath, file.name);
if (file.isDirectory()) {
const nestedFiles = await readDirRecursive(filePath);
dirFiles.push(...nestedFiles);
} else {
dirFiles.push(filePath)
}
}
resolve(dirFiles);
});
});
}

144
wowup-electron/native/curse.cc Normal file
View File

@@ -0,0 +1,144 @@
#include "curse.h"
#include <fstream>
std::string curse::hello()
{
return "Hello World";
}
uint32_t curse::add(int a, int b)
{
return (uint32_t)1767121699 * (uint32_t)1540483477;
// return a + b;
}
Napi::String curse::HelloWrapped(const Napi::CallbackInfo &info)
{
Napi::Env env = info.Env();
Napi::String returnValue = Napi::String::New(env, curse::hello());
return returnValue;
}
Napi::Number curse::AddWrapped(const Napi::CallbackInfo &info)
{
Napi::Env env = info.Env();
if (info.Length() < 2 || !info[0].IsNumber() || !info[1].IsNumber())
{
Napi::TypeError::New(env, "Number expected").ThrowAsJavaScriptException();
}
Napi::Number first = info[0].As<Napi::Number>();
Napi::Number second = info[1].As<Napi::Number>();
uint32_t returnValue = curse::add(first.Int32Value(), second.Int32Value());
return Napi::Number::New(env, returnValue);
}
bool curse::isWhitespaceCharacter(char b)
{
return b == 9 || b == 10 || b == 13 || b == 32;
}
uint32_t curse::computeNormalizedLength(char *input, int length)
{
int32_t bufferSize = 65536;
int32_t num1 = 0;
int32_t pos = 0;
int32_t chunkSize = 0;
// do
// {
// chunkSize = std::min(length - pos, bufferSize);
// if (chunkSize == 0)
// {
// return num1;
// }
for (int32_t index = 0; index < length; ++index)
{
if (!curse::isWhitespaceCharacter(input[index]))
{
++num1;
}
}
return num1;
// pos += num2;
// } while (true);
}
uint32_t curse::computeHash(char *buffer, int length)
{
// std::ofstream myfile("hash.txt", std::ios::out | std::ios::binary);
uint32_t bufferSize = 65536;
uint32_t multiplex = 1540483477;
uint32_t num1 = length;
uint32_t pos = 0;
if (true)
{
num1 = curse::computeNormalizedLength(buffer, length);
}
uint32_t num2 = (uint32_t)1 ^ num1;
uint32_t num3 = 0;
uint32_t num4 = 0;
for (int index = 0; index < length; ++index)
{
unsigned char b = buffer[index];
if (!curse::isWhitespaceCharacter(b))
{
num3 |= (uint32_t)b << num4;
num4 += 8;
if (num4 == 32)
{
uint32_t num6 = num3 * multiplex;
uint32_t num7 = (num6 ^ num6 >> 24) * multiplex;
num2 = num2 * multiplex ^ num7;
num3 = 0;
num4 = 0;
}
}
}
if (num4 > 0)
{
num2 = (num2 ^ num3) * multiplex;
}
uint32_t num6 = (num2 ^ num2 >> 13) * multiplex;
return num6 ^ num6 >> 15;
}
Napi::Number curse::ComputeHashWrapped(const Napi::CallbackInfo &info)
{
Napi::Env env = info.Env();
if (info.Length() < 2 || !info[0].IsBuffer() || !info[1].IsNumber())
{
Napi::TypeError::New(env, "Buffer, Number expected").ThrowAsJavaScriptException();
}
Napi::Buffer<char> buffer = info[0].As<Napi::Buffer<char>>();
Napi::Number length = info[1].As<Napi::Number>();
uint32_t returnValue = curse::computeHash(buffer.Data(), length.Int32Value());
return Napi::Number::New(env, returnValue);
}
Napi::Object curse::Init(Napi::Env env, Napi::Object exports)
{
exports.Set("computeHash", Napi::Function::New(env, curse::ComputeHashWrapped));
exports.Set("hello", Napi::Function::New(env, curse::HelloWrapped));
exports.Set("add", Napi::Function::New(env, curse::AddWrapped));
return exports;
}

17
wowup-electron/native/curse.h Normal file
View File

@@ -0,0 +1,17 @@
#include <napi.h>
namespace curse
{
std::string hello();
Napi::String HelloWrapped(const Napi::CallbackInfo &info);
uint32_t add(int a, int b);
Napi::Number AddWrapped(const Napi::CallbackInfo &info);
bool isWhitespaceCharacter(char b);
uint32_t computeNormalizedLength(char *input, int length);
uint32_t computeHash(char *buffer, int length);
Napi::Number ComputeHashWrapped(const Napi::CallbackInfo &info);
Napi::Object Init(Napi::Env env, Napi::Object exports);
} // namespace curse

10
wowup-electron/native/hello.cc Normal file
View File

@@ -0,0 +1,10 @@
#include <napi.h>
#include "curse.h"
Napi::Object InitAll(Napi::Env env, Napi::Object exports) {
curse::Init(env, exports);
return exports;
}
NODE_API_MODULE(NODE_GYP_MODULE_NAME, InitAll)

3
wowup-electron/package.json
View File

@@ -7,6 +7,7 @@
"name": "Maxime GRIS",
"email": "maxime.gris@gmail.com"
},
"gypfile": true,
"keywords": [
"angular",
"angular 10",
@@ -78,6 +79,7 @@
"karma-jasmine": "~4.0.0",
"karma-jasmine-html-reporter": "^1.5.0",
"mocha": "7.2.0",
"node-gyp": "7.1.0",
"npm-run-all": "4.1.5",
"protractor": "~7.0.0",
"rxjs": "6.6.2",
@@ -116,6 +118,7 @@
"globrex": "0.1.2",
"lodash": "4.17.19",
"ncp": "2.0.0",
"node-addon-api": "3.0.0",
"node-disk-info": "1.1.0",
"protobufjs": "6.10.1",
"rimraf": "3.0.2",

16
wowup-electron/serve/out-tsc/preload.js Normal file
View File

@@ -0,0 +1,16 @@
// import { Titlebar, Color } from 'custom-electron-titlebar'
// window.addEventListener('DOMContentLoaded', () => {
// new Titlebar({
// backgroundColor: Color.fromHex('#ECECEC'),
// menu: null,
// icon: './assets/wowup_logo_512np.png'
// });
// const replaceText = (selector, text) => {
// const element = document.getElementById(selector)
// if (element) element.innerText = text
// }
// for (const type of ['chrome', 'node', 'electron']) {
// replaceText(`${type}-version`, process.versions[type])
// }
// });
//# sourceMappingURL=preload.js.map

14
wowup-electron/serve/out-tsc/src/common/constants.js Normal file
View File

@@ -0,0 +1,14 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.CURSE_HASH_FILE_CHANNEL = exports.COPY_FILE_CHANNEL = exports.UNZIP_FILE_CHANNEL = exports.READ_FILE_CHANNEL = exports.LIST_FILES_CHANNEL = exports.STAT_DIRECTORY_CHANNEL = exports.RENAME_DIRECTORY_CHANNEL = exports.DELETE_DIRECTORY_CHANNEL = exports.COPY_DIRECTORY_CHANNEL = exports.DOWNLOAD_FILE_CHANNEL = void 0;
exports.DOWNLOAD_FILE_CHANNEL = 'download-file';
exports.COPY_DIRECTORY_CHANNEL = 'copy-directory';
exports.DELETE_DIRECTORY_CHANNEL = 'delete-directory';
exports.RENAME_DIRECTORY_CHANNEL = 'rename-directory';
exports.STAT_DIRECTORY_CHANNEL = 'stat-directory';
exports.LIST_FILES_CHANNEL = 'list-files';
exports.READ_FILE_CHANNEL = 'read-file';
exports.UNZIP_FILE_CHANNEL = 'unzip-file';
exports.COPY_FILE_CHANNEL = 'copy-file';
exports.CURSE_HASH_FILE_CHANNEL = 'curse-hash-file';
//# sourceMappingURL=constants.js.map

3
wowup-electron/serve/out-tsc/src/common/models/copy-directory-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=copy-directory-request.js.map

3
wowup-electron/serve/out-tsc/src/common/models/copy-file-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=copy-file-request.js.map

3
wowup-electron/serve/out-tsc/src/common/models/curse-hash-file-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=curse-hash-file-request.js.map

3
wowup-electron/serve/out-tsc/src/common/models/curse-hash-file-response.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=curse-hash-file-response.js.map

3
wowup-electron/serve/out-tsc/src/common/models/delete-directory-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=delete-directory-request.js.map

3
wowup-electron/serve/out-tsc/src/common/models/download-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=download-request.js.map

10
wowup-electron/serve/out-tsc/src/common/models/download-status-type.js Normal file
View File

@@ -0,0 +1,10 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DownloadStatusType = void 0;
var DownloadStatusType;
(function (DownloadStatusType) {
DownloadStatusType[DownloadStatusType["Progress"] = 0] = "Progress";
DownloadStatusType[DownloadStatusType["Complete"] = 1] = "Complete";
DownloadStatusType[DownloadStatusType["Error"] = 2] = "Error";
})(DownloadStatusType = exports.DownloadStatusType || (exports.DownloadStatusType = {}));
//# sourceMappingURL=download-status-type.js.map

3
wowup-electron/serve/out-tsc/src/common/models/download-status.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=download-status.js.map

3
wowup-electron/serve/out-tsc/src/common/models/list-files-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=list-files-request.js.map

3
wowup-electron/serve/out-tsc/src/common/models/list-files-response.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=list-files-response.js.map

3
wowup-electron/serve/out-tsc/src/common/models/read-file-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=read-file-request.js.map

3
wowup-electron/serve/out-tsc/src/common/models/read-file-response.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=read-file-response.js.map

3
wowup-electron/serve/out-tsc/src/common/models/unzip-request.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=unzip-request.js.map

9
wowup-electron/serve/out-tsc/src/common/models/unzip-status-type.js Normal file
View File

@@ -0,0 +1,9 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.UnzipStatusType = void 0;
var UnzipStatusType;
(function (UnzipStatusType) {
UnzipStatusType[UnzipStatusType["Complete"] = 0] = "Complete";
UnzipStatusType[UnzipStatusType["Error"] = 1] = "Error";
})(UnzipStatusType = exports.UnzipStatusType || (exports.UnzipStatusType = {}));
//# sourceMappingURL=unzip-status-type.js.map

3
wowup-electron/serve/out-tsc/src/common/models/unzip-status.js Normal file
View File

@@ -0,0 +1,3 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
//# sourceMappingURL=unzip-status.js.map

4
wowup-electron/src/app/addon-providers/addon-provider.ts
View File

@@ -3,6 +3,8 @@ import { Addon } from "../entities/addon";
import { PotentialAddon } from "../models/wowup/potential-addon";
import { AddonSearchResult } from "../models/wowup/addon-search-result";
import { Observable } from "rxjs";
import { AddonFolder } from "app/models/wowup/addon-folder";
import { AddonChannelType } from "app/models/wowup/addon-channel-type";
export interface AddonProvider {
@@ -23,4 +25,6 @@ export interface AddonProvider {
isValidAddonUri(addonUri: URL): boolean;
onPostInstall(addon: Addon): void;
scan(clientType: WowClientType, addonChannelType: AddonChannelType, addonFolders: AddonFolder[]): Promise<void>;
}

154
wowup-electron/src/app/addon-providers/curse-addon-provider.ts
View File

@@ -3,17 +3,27 @@ import { WowClientType } from "../models/warcraft/wow-client-type";
import { Addon } from "../entities/addon";
import { HttpClient } from "@angular/common/http";
import { CurseSearchResult } from "../models/curse/curse-search-result";
import { map } from "rxjs/operators";
import { map, mergeMap } from "rxjs/operators";
import { CurseFile } from "../models/curse/curse-file";
import * as _ from 'lodash';
import * as fp from 'lodash/fp';
import * as path from 'path';
import { AddonSearchResult } from "../models/wowup/addon-search-result";
import { Observable, of } from "rxjs";
import { from, Observable, of } from "rxjs";
import { AddonSearchResultFile } from "../models/wowup/addon-search-result-file";
import { CurseReleaseType } from "../models/curse/curse-release-type";
import { AddonChannelType } from "../models/wowup/addon-channel-type";
import { PotentialAddon } from "../models/wowup/potential-addon";
import { CurseGetFeaturedResponse } from "../models/curse/curse-get-featured-response";
import { CachingService } from "app/services/caching/caching-service";
import { AddonFolder } from "app/models/wowup/addon-folder";
import { FileService } from "app/services/files/file.service";
import { CurseFolderScanner } from "./curse/curse-folder-scanner";
import { ElectronService } from "app/services";
import { CurseScanResult } from "../models/curse/curse-scan-result";
import { CurseFingerprintsResponse } from "app/models/curse/curse-fingerprint-response";
import { CurseMatch } from "app/models/curse/curse-match";
import { v4 as uuidv4 } from 'uuid';
const API_URL = "https://addons-ecs.forgesvc.net/api/v2";
@@ -25,11 +35,113 @@ export class CurseAddonProvider implements AddonProvider {
constructor(
httpClient: HttpClient,
private _cachingService: CachingService
private _cachingService: CachingService,
private _electronService: ElectronService,
private _fileService: FileService
) {
this._httpClient = httpClient;
}
async scan(clientType: WowClientType, addonChannelType: AddonChannelType, addonFolders: AddonFolder[]): Promise<void> {
const addonDirectory = path.dirname(addonFolders[0].path);
const scanResults = await this.getScanResults(addonFolders);
await this.mapAddonFolders(scanResults, clientType);
const addonIds = fp.flow(
fp.filter((sr: CurseScanResult) => !!sr.exactMatch),
fp.map((sr: CurseScanResult) => sr.exactMatch.id),
fp.uniq
)(scanResults);
// console.log(_.sortBy(addonIds).join('\n'));
var addonResults = await this.getAllIds(addonIds).toPromise();
for (let addonFolder of addonFolders) {
var scanResult = scanResults.find(sr => sr.addonFolder.name === addonFolder.name);
if (!scanResult.exactMatch) {
continue;
}
scanResult.searchResult = addonResults.find(addonResult => addonResult.id === scanResult.exactMatch.id);
if (!scanResult.searchResult) {
continue;
}
try {
addonFolder.matchingAddon = this.getAddon(clientType, addonChannelType, scanResult);
} catch (err) {
// TODO
// _analyticsService.Track(ex, $"Failed to create addon for result {scanResult.FolderScanner.Fingerprint}");
}
}
}
private async mapAddonFolders(scanResults: CurseScanResult[], clientType: WowClientType) {
const fingerprintResponse = await this.getAddonsByFingerprints(scanResults.map(result => result.fingerprint)).toPromise();
console.log(fingerprintResponse);
for (let scanResult of scanResults) {
// Curse can deliver the wrong result sometimes, ensure the result matches the client type
scanResult.exactMatch = fingerprintResponse.exactMatches
.find(exactMatch =>
this.isGameVersionFlavor(exactMatch.file.gameVersionFlavor, clientType) &&
this.hasMatchingFingerprint(scanResult, exactMatch));
// If the addon does not have an exact match, check the partial matches.
if (!scanResult.exactMatch) {
scanResult.exactMatch = fingerprintResponse.partialMatches
.find(partialMatch => partialMatch.file.modules.some(module => module.fingerprint === scanResult.fingerprint));
}
}
}
private hasMatchingFingerprint(scanResult: CurseScanResult, exactMatch: CurseMatch) {
return exactMatch.file.modules.some(m => m.fingerprint === scanResult.fingerprint);
}
private isGameVersionFlavor(gameVersionFlavor: string, clientType: WowClientType) {
return gameVersionFlavor === this.getGameVersionFlavor(clientType);
}
private getAddonsByFingerprints(fingerprints: number[]): Observable<CurseFingerprintsResponse> {
const url = `${API_URL}/fingerprint`;
return this._httpClient.post<CurseFingerprintsResponse>(url, fingerprints);
}
private getAllIds(addonIds: number[]): Observable<CurseSearchResult[]> {
const url = `${API_URL}/addon`;
return this._httpClient.post<CurseSearchResult[]>(url, addonIds);
}
private async getScanResults(addonFolders: AddonFolder[]): Promise<CurseScanResult[]> {
const scanResults: CurseScanResult[] = [];
const t1 = Date.now();
// Scan addon folders in parallel for speed!?
await from(addonFolders.map(folder => {
return new CurseFolderScanner(this._electronService, this._fileService).scanFolder(folder);
}))
.pipe(
mergeMap(obs => obs, 3),
map(res => scanResults.push(res))
)
.toPromise();
console.log('scan delta', Date.now() - t1);
// const str = _.orderBy(scanResults, sr => sr.folderName.toLowerCase())
// .map(sr => `${sr.fingerprint} ${sr.folderName}`).join('\n');
// console.log(str);
return scanResults;
}
getAll(clientType: WowClientType, addonIds: string[]): Promise<import("../models/wowup/addon-search-result").AddonSearchResult[]> {
throw new Error("Method not implemented.");
}
@@ -47,7 +159,7 @@ export class CurseAddonProvider implements AddonProvider {
}
private filterFeaturedAddons(results: CurseSearchResult[], clientType: WowClientType) {
const clientTypeStr = this.getClientTypeString(clientType);
const clientTypeStr = this.getGameVersionFlavor(clientType);
return results.filter(r => r.latestFiles.some(lf => this.isClientType(lf, clientTypeStr)));
}
@@ -198,12 +310,12 @@ export class CurseAddonProvider implements AddonProvider {
}
private getThumbnailUrl(result: CurseSearchResult): string {
const attachment = _.find(result.attachments, f => f.isDefault && !!f.thumbnailUrl);
const attachment = result.attachments.find(f => f.isDefault && !!f.thumbnailUrl);
return attachment?.thumbnailUrl;
}
private getLatestFiles(result: CurseSearchResult, clientType: WowClientType): CurseFile[] {
const clientTypeStr = this.getClientTypeString(clientType);
const clientTypeStr = this.getGameVersionFlavor(clientType);
return _.flow(
_.filter((f: CurseFile) => f.isAlternate == false && f.gameVersionFlavor == clientTypeStr),
@@ -212,7 +324,7 @@ export class CurseAddonProvider implements AddonProvider {
)(result.latestFiles) as CurseFile[];
}
private getClientTypeString(clientType: WowClientType): string {
private getGameVersionFlavor(clientType: WowClientType): string {
switch (clientType) {
case WowClientType.Classic:
case WowClientType.ClassicPtr:
@@ -225,4 +337,32 @@ export class CurseAddonProvider implements AddonProvider {
}
}
private getAddon(clientType: WowClientType, addonChannelType: AddonChannelType, scanResult: CurseScanResult): Addon {
const currentVersion = scanResult.exactMatch.file;
const authors = scanResult.searchResult.authors.map(author => author.name).join(', ');
const folderList = scanResult.exactMatch.file.modules.map(module => module.foldername).join(',');
const latestVersion = this.getLatestFiles(scanResult.searchResult, clientType)[0];
return {
id: uuidv4(),
author: authors,
name: scanResult.searchResult.name,
channelType: addonChannelType,
autoUpdateEnabled: false,
clientType: clientType,
downloadUrl: latestVersion.downloadUrl,
externalUrl: scanResult.searchResult.websiteUrl,
externalId: scanResult.searchResult.id.toString(),
folderName: scanResult.addonFolder.name,
gameVersion: currentVersion.gameVersion[0],
installedAt: new Date(),
installedFolders: folderList,
installedVersion: currentVersion.displayName,
isIgnored: false,
latestVersion: latestVersion.displayName,
providerName: this.name,
thumbnailUrl: this.getThumbnailUrl(scanResult.searchResult)
};
}
}

209
wowup-electron/src/app/addon-providers/curse/curse-folder-scanner.ts Normal file
View File

@@ -0,0 +1,209 @@
import { FileService } from "app/services/files/file.service";
import * as path from 'path';
import * as fs from 'fs';
import * as _ from 'lodash';
import { AddonFolder } from "app/models/wowup/addon-folder";
import { ElectronService } from "app/services";
import { CurseHashFileResponse } from "common/models/curse-hash-file-response";
import { CurseHashFileRequest } from "common/models/curse-hash-file-request";
import { CURSE_HASH_FILE_CHANNEL } from "common/constants";
import { v4 as uuidv4 } from 'uuid';
import { CurseScanResult } from "../../models/curse/curse-scan-result";
import { from } from "rxjs";
import { mergeMap } from "rxjs/operators";
export class CurseFolderScanner {
constructor(
private _electronService: ElectronService,
private _fileService: FileService
) { }
private get tocFileCommentsRegex() {
return /\s*#.*$/mg;
}
private get tocFileIncludesRegex() {
return /^\s*((?:(?<!\.\.).)+\.(?:xml|lua))\s*$/mig;
}
private get tocFileRegex() {
return /^([^\/]+)[\\\/]\1\.toc$/i;
}
private get bindingsXmlRegex() {
return /^[^\/\\]+[\/\\]Bindings\.xml$/i;
}
private get bindingsXmlIncludesRegex() {
return /<(?:Include|Script)\s+file=[\""\""']((?:(?<!\.\.).)+)[\""\""']\s*\/>/ig;
}
private get bindingsXmlCommentsRegex() {
return /<!--.*?-->/gs;
}
async scanFolder(addonFolder: AddonFolder): Promise<CurseScanResult> {
const folderPath = addonFolder.path;
const files = await this._fileService.listAllFiles(folderPath);
let matchingFiles = await this.getMatchingFiles(folderPath, files);
matchingFiles = _.sortBy(matchingFiles, f => f.toLowerCase());
const fst = matchingFiles.map(f => f.toLowerCase()).join('\n');
const individualFingerprints: number[] = [];
for (let path of matchingFiles) {
const normalizedFileHash = await this.computeNormalizedFileHash(path);
individualFingerprints.push(normalizedFileHash);
}
const hashConcat = _.orderBy(individualFingerprints).join('');
const fingerprint = await this.computeStringHash(hashConcat);
return {
directory: folderPath,
fileCount: matchingFiles.length,
fingerprint,
folderName: path.basename(folderPath),
individualFingerprints,
addonFolder
};
}
private async getMatchingFiles(folderPath: string, filePaths: string[]): Promise<string[]> {
const parentDir = path.dirname(folderPath) + path.sep;
const matchingFileList: string[] = [];
const fileInfoList: string[] = [];
for (let filePath of filePaths) {
const input = filePath.toLowerCase().replace(parentDir.toLowerCase(), '');
if (this.tocFileRegex.test(input)) {
fileInfoList.push(filePath);
} else if (this.bindingsXmlRegex.test(input)) {
matchingFileList.push(filePath);
}
}
for (let fileInfo of fileInfoList) {
await this.processIncludeFile(matchingFileList, fileInfo);
}
return matchingFileList;
}
private async processIncludeFile(matchingFileList: string[], fileInfo: string) {
if (!fs.existsSync(fileInfo) || matchingFileList.indexOf(fileInfo) !== -1) {
return;
}
matchingFileList.push(fileInfo);
let input = await this._fileService.readFile(fileInfo);
input = this.removeComments(fileInfo, input);
const inclusions = this.getFileInclusionMatches(fileInfo, input);
if (!inclusions || !inclusions.length) {
return;
}
for (let include of inclusions) {
const fileName = path.join(path.dirname(fileInfo), include);
await this.processIncludeFile(matchingFileList, fileName);
}
}
private getFileInclusionMatches(fileInfo: string, fileContent: string): string[] | null {
const ext = path.extname(fileInfo);
switch (ext) {
case '.xml':
return this.matchAll(fileContent, this.bindingsXmlIncludesRegex);
case '.toc':
return this.matchAll(fileContent, this.tocFileIncludesRegex);
default:
return null;
}
}
private removeComments(fileInfo: string, fileContent: string): string {
const ext = path.extname(fileInfo);
switch (ext) {
case '.xml':
return fileContent.replace(this.bindingsXmlCommentsRegex, '');
case '.toc':
return fileContent.replace(this.tocFileCommentsRegex, '');
default:
return fileContent;
}
}
private matchAll(str: string, regex: RegExp): string[] {
const matches: string[] = [];
let currentMatch: RegExpExecArray;
do {
currentMatch = regex.exec(str);
if (currentMatch) {
matches.push(currentMatch[1]);
}
} while (currentMatch);
return matches;
}
private computeNormalizedFileHash(filePath: string) {
return this.computeFileHash(filePath, true);
}
private computeFileHash(filePath: string, normalizeWhitespace: boolean) {
return this.computeHash(filePath, 0, normalizeWhitespace);
}
private computeStringHash(str: string): Promise<number> {
return new Promise((resolve, reject) => {
const eventHandler = (_evt: any, arg: CurseHashFileResponse) => {
if (arg.error) {
return reject(arg.error);
}
resolve(arg.fingerprint);
};
const request: CurseHashFileRequest = {
targetString: str,
targetStringEncoding: 'ascii',
responseKey: uuidv4(),
normalizeWhitespace: false,
precomputedLength: 0
};
this._electronService.ipcRenderer.once(request.responseKey, eventHandler);
this._electronService.ipcRenderer.send(CURSE_HASH_FILE_CHANNEL, request);
});
}
private computeHash(
filePath: string,
precomputedLength: number = 0,
normalizeWhitespace: boolean = false
): Promise<number> {
return new Promise((resolve, reject) => {
const eventHandler = (_evt: any, arg: CurseHashFileResponse) => {
if (arg.error) {
return reject(arg.error);
}
resolve(arg.fingerprint);
};
const request: CurseHashFileRequest = {
responseKey: uuidv4(),
filePath,
normalizeWhitespace,
precomputedLength
};
this._electronService.ipcRenderer.once(request.responseKey, eventHandler);
this._electronService.ipcRenderer.send(CURSE_HASH_FILE_CHANNEL, request);
});
}
}

4
wowup-electron/src/app/components/addon-status-column/addon-status-column.component.html
View File

@@ -1 +1,3 @@
<p>Up to Date</p>
<div class="container">
<p>Up to Date</p>
</div>

6
wowup-electron/src/app/components/addon-status-column/addon-status-column.component.scss
View File

@@ -0,0 +1,6 @@
.container {
display: flex;
align-items: center;
height: 100%;
justify-content: center;
}

4
wowup-electron/src/app/components/addon-table-column/addon-table-column.component.html
View File

@@ -1,6 +1,6 @@
<div class="addon-column row align-items-center">
<div class="addon-logo-container">
<img class="addon-logo" [src]="addon.thumbnailUrl" />
<div class="addon-logo-container" [style.backgroundImage]="'url(' + addon.thumbnailUrl + ')'">
<!-- <img class="addon-logo" [src]="addon.thumbnailUrl" /> -->
</div>
<div>
<div class="addon-title">{{addon.name}}</div>

4
wowup-electron/src/app/components/addon-table-column/addon-table-column.component.scss
View File

@@ -13,9 +13,11 @@
justify-content: center;
margin-right: 11px;
flex-shrink: 0;
background-size: contain;
background-repeat: none;
.addon-logo {
width: 100%;
height: 100%;
}
}

11
wowup-electron/src/app/models/curse/curse-fingerprint-response.ts Normal file
View File

@@ -0,0 +1,11 @@
import { CurseMatch } from "./curse-match";
export interface CurseFingerprintsResponse {
isCacheBuild: boolean;
exactMatches: CurseMatch[];
exactFingerprints: number[];
partialMatches: CurseMatch[];
partialMatchFingerprints: { [key: string]: number[] };
installedFingerprints: number[];
unmatchedFingerprints: number[];
}

7
wowup-electron/src/app/models/curse/curse-match.ts Normal file
View File

@@ -0,0 +1,7 @@
import { CurseFile } from "./curse-file";
export interface CurseMatch {
id: number;
file: CurseFile;
latestFiles: CurseFile[];
}

15
wowup-electron/src/app/models/curse/curse-scan-result.ts Normal file
View File

@@ -0,0 +1,15 @@
import { AddonFolder } from "app/models/wowup/addon-folder";
import { CurseMatch } from "./curse-match";
import { CurseSearchResult } from "./curse-search-result";
export interface CurseScanResult {
fileCount: number;
fileDateHash?: number;
fingerprint: number;
folderName: string;
individualFingerprints: number[];
directory: string;
addonFolder?: AddonFolder;
exactMatch?: CurseMatch;
searchResult?: CurseSearchResult;
}

2
wowup-electron/src/app/models/wowup/addon-folder.ts
View File

@@ -1,3 +1,4 @@
import { Addon } from "app/entities/addon";
import { Toc } from "./toc";
export interface AddonFolder {
@@ -8,4 +9,5 @@ export interface AddonFolder {
latestVersion?: string;
toc: Toc;
tocMetaData: string[];
matchingAddon?: Addon;
}

4
wowup-electron/src/app/pages/get-addons/get-addons.component.ts
View File

@@ -45,8 +45,8 @@ export class GetAddonsComponent implements OnInit {
minWidth: 200,
flex: 1
},
{ headerName: 'Author', field: 'author', cellClass: 'cell-wrap-text', flex: 1 },
{ headerName: 'Provider', field: 'providerName', cellClass: 'cell-wrap-text', width: 100, suppressSizeToFit: true },
{ headerName: 'Author', field: 'author', cellClass: 'cell-center-text', flex: 1 },
{ headerName: 'Provider', field: 'providerName', cellClass: 'cell-center-text', width: 100, suppressSizeToFit: true },
{
headerName: 'Status',
field: 'value',

7
wowup-electron/src/app/pages/my-addons/my-addons.component.html
View File

@@ -15,13 +15,6 @@
<button mat-flat-button color="primary" [disabled]="busy === true">Update All</button>
<button mat-flat-button color="primary" [disabled]="busy === true">Refresh</button>
<button mat-flat-button color="primary" [disabled]="busy === true" (click)="onReScan()">Re-Scan</button>
<mat-form-field class="example-form-field">
<mat-label>Search</mat-label>
<input matInput type="text" [(ngModel)]="query">
<button mat-button color="accent" *ngIf="query" matSuffix mat-icon-button aria-label="Clear" (click)="query=''">
<mat-icon>close</mat-icon>
</button>
</mat-form-field>
</div>
</div>

25
wowup-electron/src/app/services/addons/addon.service.ts
View File

@@ -24,6 +24,7 @@ import { DownloadSevice } from "../download/download.service";
import { WowUpService } from "../wowup/wowup.service";
import { FileService } from "../files/file.service";
import { TocService } from "../toc/toc.service";
import { ElectronService } from "../electron/electron.service";
@Injectable({
providedIn: 'root'
@@ -39,12 +40,13 @@ export class AddonService {
private _wowUpService: WowUpService,
private _wowupApiService: WowUpApiService,
private _downloadService: DownloadSevice,
private _electronService: ElectronService,
private _fileService: FileService,
private _tocService: TocService,
httpClient: HttpClient
) {
this._addonProviders = [
new CurseAddonProvider(httpClient, this._cachingService)
new CurseAddonProvider(httpClient, this._cachingService, this._electronService, this._fileService)
];
}
@@ -204,10 +206,9 @@ export class AddonService {
public async getAddons(clientType: WowClientType, rescan = false): Promise<Addon[]> {
let addons = this._addonStorage.getAllForClientType(clientType);
console.log('addons', addons.length)
if (rescan || !addons.length) {
this._addonStorage.removeForClientType(clientType);
addons = await this.getLocalAddons(clientType);
addons = await this.scanAddons(clientType);
this._addonStorage.setAll(addons);
}
// RemoveAddons(clientType);
@@ -221,6 +222,24 @@ export class AddonService {
return addons;
}
private async scanAddons(clientType: WowClientType): Promise<Addon[]> {
const addonFolders = await this._warcraftService.listAddons(clientType);
for (let provider of this._addonProviders) {
try {
const validFolders = addonFolders.filter(af => !af.matchingAddon && af.toc)
await provider.scan(clientType, AddonChannelType.Stable, validFolders);
} catch (err) {
console.log(err);
}
}
const matchedAddonFolders = addonFolders.filter(addonFolder => !!addonFolder.matchingAddon);
const matchedGroups = _.groupBy(matchedAddonFolders, addonFolder => `${addonFolder.matchingAddon.providerName}${addonFolder.matchingAddon.externalId}`);
console.log(matchedGroups);
return Object.values(matchedGroups).map(value => value[0].matchingAddon);
}
public getFeaturedAddons(clientType: WowClientType): Observable<PotentialAddon[]> {
return forkJoin(this._addonProviders.map(p => p.getFeaturedAddons(clientType)))
.pipe(

31
wowup-electron/src/app/services/files/file.service.ts
View File

@@ -1,5 +1,5 @@
import { Injectable } from "@angular/core";
import { COPY_DIRECTORY_CHANNEL, DELETE_DIRECTORY_CHANNEL, READ_FILE_CHANNEL, RENAME_DIRECTORY_CHANNEL } from "common/constants";
import { COPY_DIRECTORY_CHANNEL, DELETE_DIRECTORY_CHANNEL, LIST_FILES_CHANNEL, READ_FILE_CHANNEL, RENAME_DIRECTORY_CHANNEL } from "common/constants";
import { CopyDirectoryRequest } from "common/models/copy-directory-request";
import { DeleteDirectoryRequest } from "common/models/delete-directory-request";
import { ElectronService } from "../electron/electron.service";
@@ -7,6 +7,8 @@ import * as fs from 'fs';
import * as globrex from 'globrex';
import { ReadFileResponse } from "common/models/read-file-response";
import { ReadFileRequest } from "common/models/read-file-request";
import { ListFilesResponse } from "common/models/list-files-response";
import { ListFilesRequest } from "common/models/list-files-request";
@Injectable({
providedIn: 'root'
@@ -28,7 +30,7 @@ export class FileService {
const request: DeleteDirectoryRequest = { sourcePath };
this._electronService.ipcRenderer.on(sourcePath, eventHandler);
this._electronService.ipcRenderer.once(sourcePath, eventHandler);
this._electronService.ipcRenderer.send(DELETE_DIRECTORY_CHANNEL, request);
})
}
@@ -45,7 +47,7 @@ export class FileService {
const request: CopyDirectoryRequest = { sourcePath, destinationPath };
this._electronService.ipcRenderer.on(destinationPath, eventHandler);
this._electronService.ipcRenderer.once(destinationPath, eventHandler);
this._electronService.ipcRenderer.send(COPY_DIRECTORY_CHANNEL, request);
})
}
@@ -62,9 +64,9 @@ export class FileService {
const request: CopyDirectoryRequest = { sourcePath, destinationPath };
this._electronService.ipcRenderer.on(destinationPath, eventHandler);
this._electronService.ipcRenderer.once(destinationPath, eventHandler);
this._electronService.ipcRenderer.send(RENAME_DIRECTORY_CHANNEL, request);
})
});
}
public readFile(sourcePath: string): Promise<string> {
@@ -79,7 +81,7 @@ export class FileService {
const request: ReadFileRequest = { sourcePath };
this._electronService.ipcRenderer.on(sourcePath, eventHandler);
this._electronService.ipcRenderer.once(sourcePath, eventHandler);
this._electronService.ipcRenderer.send(READ_FILE_CHANNEL, request);
})
}
@@ -97,4 +99,21 @@ export class FileService {
.filter(entry => !!globFilter.regex.test(entry.name))
.map(entry => entry.name);
}
public listAllFiles(sourcePath: string, recursive: boolean = true): Promise<string[]> {
return new Promise((resolve, reject) => {
const eventHandler = (_evt: any, arg: ListFilesResponse) => {
if (arg.error) {
return reject(arg.error);
}
resolve(arg.files);
};
const request: ListFilesRequest = { sourcePath, recursive };
this._electronService.ipcRenderer.once(sourcePath, eventHandler);
this._electronService.ipcRenderer.send(LIST_FILES_CHANNEL, request);
})
}
}

5
wowup-electron/src/common/constants.ts
View File

@@ -2,6 +2,9 @@ export const DOWNLOAD_FILE_CHANNEL = 'download-file';
export const COPY_DIRECTORY_CHANNEL = 'copy-directory';
export const DELETE_DIRECTORY_CHANNEL = 'delete-directory';
export const RENAME_DIRECTORY_CHANNEL = 'rename-directory';
export const STAT_DIRECTORY_CHANNEL = 'stat-directory';
export const LIST_FILES_CHANNEL = 'list-files';
export const READ_FILE_CHANNEL = 'read-file';
export const UNZIP_FILE_CHANNEL = 'unzip-file';
export const COPY_FILE_CHANNEL = 'copy-file';
export const COPY_FILE_CHANNEL = 'copy-file';
export const CURSE_HASH_FILE_CHANNEL = 'curse-hash-file';

9
wowup-electron/src/common/models/curse-hash-file-request.ts Normal file
View File

@@ -0,0 +1,9 @@
import { IpcResponse } from "./ipc-response";
export interface CurseHashFileRequest extends IpcResponse {
filePath?: string;
targetString?: string;
targetStringEncoding?: BufferEncoding;
precomputedLength: number;
normalizeWhitespace: boolean;
}

4
wowup-electron/src/common/models/curse-hash-file-response.ts Normal file
View File

@@ -0,0 +1,4 @@
export interface CurseHashFileResponse {
fingerprint: number;
error?: Error;
}

3
wowup-electron/src/common/models/ipc-response.ts Normal file
View File

@@ -0,0 +1,3 @@
export interface IpcResponse {
responseKey: string;
}

4
wowup-electron/src/common/models/list-files-request.ts Normal file
View File

@@ -0,0 +1,4 @@
export interface ListFilesRequest {
sourcePath: string;
recursive: boolean;
}

4
wowup-electron/src/common/models/list-files-response.ts Normal file
View File

@@ -0,0 +1,4 @@
export interface ListFilesResponse {
files: string[];
error?: Error;
}

4
wowup-electron/src/environments/environment.dev.ts
View File

@@ -5,5 +5,7 @@
export const AppConfig = {
production: false,
environment: 'DEV'
environment: 'DEV',
appVersion: require('../../package.json').version,
wowUpApiUrl: 'https://4g2nuwcupj.execute-api.us-east-1.amazonaws.com/production'
};