From 13b52f9e6885ce344268642d4872d9ceba4cdbd4 Mon Sep 17 00:00:00 2001 From: jliddev Date: Tue, 13 Oct 2020 16:04:11 -0500 Subject: [PATCH] Play with the wowup provider --- wowup-electron/ipc-events.ts | 33 +++ wowup-electron/package.json | 2 +- .../src/app/addon-providers/addon-provider.ts | 2 +- .../addon-providers/wowup-addon-provider.ts | 261 ++++++++++++++++++ .../business-objects/wowup-folder-scanner.ts | 183 ++++++++++++ .../components/footer/footer.component.html | 6 +- .../components/footer/footer.component.scss | 19 +- .../my-addons-addon-cell.component.html | 20 +- .../my-addons-addon-cell.component.scss | 15 + wowup-electron/src/app/entities/addon.ts | 4 + .../get-addons-by-fingerprint.response.ts | 5 + .../src/app/models/wowup-api/wow-game-type.ts | 4 + ...wup-addon-release-folder.representation.ts | 10 + .../wowup-addon-release.representation.ts | 17 ++ .../wowup-api/wowup-addon.representation.ts | 20 ++ .../app/models/wowup/app-wowup-scan-result.ts | 6 + .../pages/my-addons/my-addons.component.html | 21 +- .../pages/my-addons/my-addons.component.scss | 14 + .../services/addons/addon.provider.factory.ts | 5 + .../src/app/services/addons/addon.service.ts | 7 +- .../images/custom_funding_logo_small.png | Bin 0 -> 1233 bytes .../src/assets/images/discord_logo_small.png | Bin 0 -> 1559 bytes .../src/assets/images/github_logo_small.png | Bin 0 -> 4044 bytes .../src/assets/images/patreon_logo_small.png | Bin 0 -> 60896 bytes wowup-electron/src/common/constants.ts | 1 + .../src/common/wowup/wowup-folder-scanner.ts | 196 +++++++++++++ .../wowup/wowup-get-scan-results-request.ts | 5 + .../wowup/wowup-get-scan-results-response.ts | 6 + .../src/common/wowup/wowup-scan-result.ts | 7 + .../src/environments/environment.dev.ts | 3 +- .../src/environments/environment.prod.ts | 5 +- .../src/environments/environment.ts | 3 +- .../src/environments/environment.web.ts | 4 +- 33 files changed, 865 insertions(+), 19 deletions(-) create mode 100644 wowup-electron/src/app/addon-providers/wowup-addon-provider.ts create mode 100644 wowup-electron/src/app/business-objects/wowup-folder-scanner.ts create mode 100644 wowup-electron/src/app/models/wowup-api/get-addons-by-fingerprint.response.ts create mode 100644 wowup-electron/src/app/models/wowup-api/wow-game-type.ts create mode 100644 wowup-electron/src/app/models/wowup-api/wowup-addon-release-folder.representation.ts create mode 100644 wowup-electron/src/app/models/wowup-api/wowup-addon-release.representation.ts create mode 100644 wowup-electron/src/app/models/wowup-api/wowup-addon.representation.ts create mode 100644 wowup-electron/src/app/models/wowup/app-wowup-scan-result.ts create mode 100644 wowup-electron/src/assets/images/custom_funding_logo_small.png create mode 100644 wowup-electron/src/assets/images/discord_logo_small.png create mode 100644 wowup-electron/src/assets/images/github_logo_small.png create mode 100644 wowup-electron/src/assets/images/patreon_logo_small.png create mode 100644 wowup-electron/src/common/wowup/wowup-folder-scanner.ts create mode 100644 wowup-electron/src/common/wowup/wowup-get-scan-results-request.ts create mode 100644 wowup-electron/src/common/wowup/wowup-get-scan-results-response.ts create mode 100644 wowup-electron/src/common/wowup/wowup-scan-result.ts diff --git a/wowup-electron/ipc-events.ts b/wowup-electron/ipc-events.ts index a724440a..fa80a978 100644 --- a/wowup-electron/ipc-events.ts +++ b/wowup-electron/ipc-events.ts @@ -10,6 +10,7 @@ import { SHOW_DIRECTORY, PATH_EXISTS_CHANNEL, CURSE_GET_SCAN_RESULTS, + WOWUP_GET_SCAN_RESULTS, } from "./src/common/constants"; import { CurseGetScanResultsRequest } from "./src/common/curse/curse-get-scan-results-request"; import { CurseGetScanResultsResponse } from "./src/common/curse/curse-get-scan-results-response"; @@ -23,6 +24,11 @@ import { ValueResponse } from "./src/common/models/value-response"; import { CurseScanResult } from "./src/common/curse/curse-scan-result"; import { CurseFolderScanner } from "./src/common/curse/curse-folder-scanner"; +import { WowUpGetScanResultsRequest } from "./src/common/wowup/wowup-get-scan-results-request"; +import { WowUpGetScanResultsResponse } from "./src/common/wowup/wowup-get-scan-results-response"; +import { WowUpFolderScanner } from "./src/common/wowup/wowup-folder-scanner"; +import { WowUpScanResult } from "./src/common/wowup/wowup-scan-result"; + const nativeAddon = require("./build/Release/addon.node"); ipcMain.on(SHOW_DIRECTORY, async (evt, arg: ShowDirectoryRequest) => { @@ -142,3 +148,30 @@ ipcMain.on( evt.reply(arg.responseKey, response); } ); + +ipcMain.on( + WOWUP_GET_SCAN_RESULTS, + async (evt, arg: WowUpGetScanResultsRequest) => { + const response: WowUpGetScanResultsResponse = { + scanResults: [], + }; + + try { + const scanResults = await async.mapLimit( + arg.filePaths, + 2, + async (folder, callback) => { + const scanResult = await new WowUpFolderScanner(folder).scanFolder(); + + callback(undefined, scanResult); + } + ); + + response.scanResults = scanResults; + } catch (err) { + response.error = err; + } + + evt.reply(arg.responseKey, response); + } +); diff --git a/wowup-electron/package.json b/wowup-electron/package.json index c69a0bfb..db2a3ccf 100644 --- a/wowup-electron/package.json +++ b/wowup-electron/package.json @@ -1,7 +1,7 @@ { "name": "wowup", "productName": "WowUp", - "version": "2.0.0-alpha.7", + "version": "2.0.0-alpha.8", "description": "Word of Warcraft addon updater", "homepage": "https://github.com/maximegris/angular-electron", "author": { diff --git a/wowup-electron/src/app/addon-providers/addon-provider.ts b/wowup-electron/src/app/addon-providers/addon-provider.ts index 5f8c2150..da3d649f 100644 --- a/wowup-electron/src/app/addon-providers/addon-provider.ts +++ b/wowup-electron/src/app/addon-providers/addon-provider.ts @@ -29,4 +29,4 @@ export interface AddonProvider { scan(clientType: WowClientType, addonChannelType: AddonChannelType, addonFolders: AddonFolder[]): Promise; } -export type AddonProviderType = 'Curse' | 'GitHub' | 'TukUI' | 'WowInterface'; +export type AddonProviderType = 'Curse' | 'GitHub' | 'TukUI' | 'WowInterface' | 'WowUp'; diff --git a/wowup-electron/src/app/addon-providers/wowup-addon-provider.ts b/wowup-electron/src/app/addon-providers/wowup-addon-provider.ts new file mode 100644 index 00000000..2c74d530 --- /dev/null +++ b/wowup-electron/src/app/addon-providers/wowup-addon-provider.ts @@ -0,0 +1,261 @@ +import { HttpClient } from "@angular/common/http"; +import { AppConfig } from "environments/environment"; +import { Observable, of } from "rxjs"; +import { v4 as uuidv4 } from "uuid"; +import { Addon } from "../entities/addon"; +import { WowClientType } from "../models/warcraft/wow-client-type"; +import { AddonSearchResult } from "../models/wowup/addon-search-result"; +import { PotentialAddon } from "../models/wowup/potential-addon"; +import { AddonProvider, AddonProviderType } from "./addon-provider"; +import { WowUpAddonRepresentation } from "../models/wowup-api/wowup-addon.representation"; +import { AddonFolder } from "app/models/wowup/addon-folder"; +import { WowUpGetScanResultsRequest } from "common/wowup/wowup-get-scan-results-request"; +import { WowUpGetScanResultsResponse } from "common/wowup/wowup-get-scan-results-response"; +import { ElectronService } from "app/services"; +import { WOWUP_GET_SCAN_RESULTS } from "common/constants"; +import { WowUpScanResult } from "common/wowup/wowup-scan-result"; +import { GetAddonsByFingerprintResponse } from "app/models/wowup-api/get-addons-by-fingerprint.response"; +import { WowUpAddonReleaseRepresentation } from "app/models/wowup-api/wowup-addon-release.representation"; +import { WowGameType } from "app/models/wowup-api/wow-game-type"; +import { AddonChannelType } from "app/models/wowup/addon-channel-type"; +import { AppWowUpScanResult } from "app/models/wowup/app-wowup-scan-result"; + +const API_URL = AppConfig.wowUpHubUrl; + +export class WowUpAddonProvider implements AddonProvider { + public readonly name = "WowUp"; + + constructor( + private _httpClient: HttpClient, + private _electronService: ElectronService + ) {} + + async getAll( + clientType: WowClientType, + addonIds: string[] + ): Promise { + const url = `${API_URL}/addons`; + const addons = await this._httpClient + .get(url.toString()) + .toPromise(); + + throw new Error("Method not implemented."); + } + + getFeaturedAddons(clientType: WowClientType): Observable { + return of([]); + } + + async searchByQuery( + query: string, + clientType: WowClientType + ): Promise { + return []; + } + + searchByUrl( + addonUri: URL, + clientType: WowClientType + ): Promise { + throw new Error("Method not implemented."); + } + + searchByName( + addonName: string, + folderName: string, + clientType: WowClientType, + nameOverride?: string + ): Promise { + throw new Error("Method not implemented."); + } + + getById( + addonId: string, + clientType: WowClientType + ): Observable { + throw new Error("Method not implemented."); + } + + isValidAddonUri(addonUri: URL): boolean { + throw new Error("Method not implemented."); + } + + onPostInstall(addon: Addon): void { + throw new Error("Method not implemented."); + } + + async scan( + clientType: WowClientType, + addonChannelType: any, + addonFolders: AddonFolder[] + ): Promise { + // const url = `${API_URL}/addons`; + // const addons = await this._httpClient + // .get(url.toString()) + // .toPromise(); + + const scanResults = await this.getScanResults(addonFolders); + + console.log("ScanResults", scanResults.length); + + const fingerprintResponse = await this.getAddonsByFingerprints( + scanResults.map((result) => result.fingerprint) + ).toPromise(); + + console.log("fingerprintResponse", fingerprintResponse); + + for (let scanResult of scanResults) { + // Curse can deliver the wrong result sometimes, ensure the result matches the client type + scanResult.exactMatch = fingerprintResponse.exactMatches.find( + (exactMatch) => + this.isGameType(exactMatch.matched_release, clientType) && + this.hasMatchingFingerprint(scanResult, exactMatch.matched_release) + ); + + // If the addon does not have an exact match, check the partial matches. + // if (!scanResult.exactMatch) { + // scanResult.exactMatch = fingerprintResponse.partialMatches.find( + // (partialMatch) => + // partialMatch.file?.modules?.some( + // (module) => module.fingerprint === scanResult.fingerprint + // ) + // ); + // } + } + + const matchedScanResults = scanResults.filter((sr) => !!sr.exactMatch); + const matchedScanResultIds = matchedScanResults.map( + (sr) => sr.exactMatch.id + ); + + for (let addonFolder of addonFolders) { + var scanResult = scanResults.find((sr) => sr.path === addonFolder.path); + if (!scanResult.exactMatch) { + console.log("No search result match", scanResult.path); + continue; + } + + try { + const newAddon = this.getAddon( + clientType, + addonChannelType, + scanResult + ); + + addonFolder.matchingAddon = newAddon; + } catch (err) { + console.error(scanResult); + console.error(err); + // TODO + // _analyticsService.Track(ex, $"Failed to create addon for result {scanResult.FolderScanner.Fingerprint}"); + } + } + } + + private hasMatchingFingerprint( + scanResult: WowUpScanResult, + release: WowUpAddonReleaseRepresentation + ) { + return release.addonFolders.some( + (addonFolder) => addonFolder.fingerprint == scanResult.fingerprint + ); + } + + private isGameType( + release: WowUpAddonReleaseRepresentation, + clientType: WowClientType + ) { + return release.game_type === this.getWowGameType(clientType); + } + + private getWowGameType(clientType: WowClientType): string { + switch (clientType) { + case WowClientType.Classic: + case WowClientType.ClassicPtr: + return WowGameType.Classic; + case WowClientType.Retail: + case WowClientType.RetailPtr: + case WowClientType.Beta: + default: + return WowGameType.Retail; + } + } + + private getAddonsByFingerprints( + fingerprints: string[] + ): Observable { + const url = `${API_URL}/addons/fingerprint`; + + return this._httpClient.post(url, { + fingerprints, + }); + } + + private getScanResults = async ( + addonFolders: AddonFolder[] + ): Promise => { + const t1 = Date.now(); + + return new Promise((resolve, reject) => { + const eventHandler = (_evt: any, arg: WowUpGetScanResultsResponse) => { + if (arg.error) { + return reject(arg.error); + } + + console.log("scan delta", Date.now() - t1); + console.log("WowUpGetScanResultsResponse", arg); + resolve(arg.scanResults); + }; + + const request: WowUpGetScanResultsRequest = { + filePaths: addonFolders.map((addonFolder) => addonFolder.path), + responseKey: uuidv4(), + }; + + this._electronService.ipcRenderer.once(request.responseKey, eventHandler); + this._electronService.ipcRenderer.send(WOWUP_GET_SCAN_RESULTS, request); + }); + }; + + private getAddon( + clientType: WowClientType, + addonChannelType: AddonChannelType, + scanResult: AppWowUpScanResult + ): Addon { + const primaryAddonFolder = scanResult.exactMatch.matched_release.addonFolders.find( + (af) => af.load_on_demand === false + ); + const authors = scanResult.exactMatch.owner_name; + const folderList = scanResult.exactMatch.matched_release.addonFolders + .map((af) => af.folder_name) + .join(", "); + + let channelType = addonChannelType; + let latestVersion = primaryAddonFolder.version; + + return { + id: uuidv4(), + author: authors, + name: scanResult.exactMatch.repository_name, + channelType, + autoUpdateEnabled: false, + clientType, + downloadUrl: scanResult.exactMatch.matched_release.download_url, + externalUrl: scanResult.exactMatch.repository, + externalId: scanResult.exactMatch.external_id, + folderName: primaryAddonFolder.folder_name, + gameVersion: scanResult.exactMatch.matched_release.game_version, + installedAt: new Date(), + installedFolders: folderList, + installedVersion: scanResult.exactMatch.matched_release.tagName, + isIgnored: false, + latestVersion: scanResult.exactMatch.matched_release.tagName, + providerName: this.name, + providerSource: scanResult.exactMatch.source, + thumbnailUrl: scanResult.exactMatch.image_url, + patreonFundingLink: scanResult.exactMatch.patreon_funding_link, + customFundingLink: scanResult.exactMatch.custom_funding_link, + githubFundingLink: scanResult.exactMatch.github_funding_link + }; + } +} diff --git a/wowup-electron/src/app/business-objects/wowup-folder-scanner.ts b/wowup-electron/src/app/business-objects/wowup-folder-scanner.ts new file mode 100644 index 00000000..e3373f97 --- /dev/null +++ b/wowup-electron/src/app/business-objects/wowup-folder-scanner.ts @@ -0,0 +1,183 @@ +export interface ScanResult { + path: string; + fileFingerprints: { [path: string]: string }; + fingerprint: string; + gameVersion: string; + addonTitle: string; + addonAuthors: string; + loadOnDemand: boolean; + version: string; + } + + export class WowUpFolderScanner { + private _folderPath = ""; + + constructor(folderPath: string) { + this._folderPath = folderPath; + } + + private get tocFileCommentsRegex() { + return /\s*#.*$/gm; + } + + private get tocFileIncludesRegex() { + return /^\s*((?:(?/gi; + } + + private get bindingsXmlCommentsRegex() { + return //gs; + } + + private async _scanFolder(folderPath: string): Promise { + const files = await readDirRecursive(folderPath); + console.log("listAllFiles", folderPath, files.length); + + let matchingFiles = await this.getMatchingFiles(folderPath, files); + matchingFiles = _.sortBy(matchingFiles, (f) => f.toLowerCase()); + + const tocFile = this.getTocFile(folderPath); + const toc = await this.parseToc(tocFile); + + let fileFingerprints: { [path: string]: string } = {}; + for (let file of matchingFiles) { + const fileHash = await hashFile(file); + fileFingerprints[file] = fileHash; + } + + const hashConcat = _.orderBy(Object.values(fileFingerprints)).join(""); + const fingerprint = hashString(hashConcat); + + const result: ScanResult = { + fileFingerprints, + fingerprint, + path: folderPath, + addonAuthors: toc?.author ?? '', + addonTitle: toc?.title ?? '', + gameVersion: toc?.interface ?? '', + loadOnDemand: toc?.loadOnDemand === '1', + version: toc?.version ?? '' + }; + + return result; + } + + private getTocFile(directory: string) { + const baseFiles = fs.readdirSync(directory); + const tocFile = baseFiles.find(file => path.extname(file) === '.toc'); + if (!tocFile) { + console.warn('No toc file: ' + directory); + return ''; + } + return path.join(directory, tocFile); + } + + private async parseToc(filePath: string) { + if (!filePath) { + return undefined; + } + const tocText = await readFile(filePath); + return parseToc(tocText); + } + + private async getMatchingFiles( + folderPath: string, + filePaths: string[] + ): Promise { + const parentDir = path.dirname(folderPath) + path.sep; + const matchingFileList: string[] = []; + const fileInfoList: string[] = []; + for (let filePath of filePaths) { + const input = filePath.toLowerCase().replace(parentDir.toLowerCase(), ""); + + if (this.tocFileRegex.test(input)) { + fileInfoList.push(filePath); + } else if (this.bindingsXmlRegex.test(input)) { + matchingFileList.push(filePath); + } + } + + // console.log('fileInfoList', fileInfoList.length) + for (let fileInfo of fileInfoList) { + await this.processIncludeFile(matchingFileList, fileInfo); + } + + return matchingFileList; + } + + private async processIncludeFile( + matchingFileList: string[], + fileInfo: string + ) { + if (!fs.existsSync(fileInfo) || matchingFileList.indexOf(fileInfo) !== -1) { + return; + } + + matchingFileList.push(fileInfo); + + let input = await readFile(fileInfo); + input = this.removeComments(fileInfo, input); + + const inclusions = this.getFileInclusionMatches(fileInfo, input); + if (!inclusions || !inclusions.length) { + return; + } + + const dirname = path.dirname(fileInfo); + for (let include of inclusions) { + const fileName = path.join(dirname, include.replace(/\\/g, path.sep)); + await this.processIncludeFile(matchingFileList, fileName); + } + } + + private removeComments(fileInfo: string, fileContent: string): string { + const ext = path.extname(fileInfo); + switch (ext) { + case ".xml": + return fileContent.replace(this.bindingsXmlCommentsRegex, ""); + case ".toc": + return fileContent.replace(this.tocFileCommentsRegex, ""); + default: + return fileContent; + } + } + + private getFileInclusionMatches( + fileInfo: string, + fileContent: string + ): string[] | null { + const ext = path.extname(fileInfo); + switch (ext) { + case ".xml": + return this.matchAll(fileContent, this.bindingsXmlIncludesRegex); + case ".toc": + return this.matchAll(fileContent, this.tocFileIncludesRegex); + default: + return null; + } + } + + private matchAll(str: string, regex: RegExp): string[] { + const matches: string[] = []; + let currentMatch: RegExpExecArray; + do { + currentMatch = regex.exec(str); + if (currentMatch) { + matches.push(currentMatch[1]); + } + } while (currentMatch); + + return matches; + } + } \ No newline at end of file diff --git a/wowup-electron/src/app/components/footer/footer.component.html b/wowup-electron/src/app/components/footer/footer.component.html index 888a881c..1a4f8342 100644 --- a/wowup-electron/src/app/components/footer/footer.component.html +++ b/wowup-electron/src/app/components/footer/footer.component.html @@ -1,7 +1,11 @@