Skip to content

Support new interfaces #9474

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
raych1 merged 4 commits into from
Dec 12, 2024
Merged

Support new interfaces #9474

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
060e288
Support new interfaces
raych1 Dec 4, 2024
7e5e722
Supported output contract and added file logger
raych1 Dec 5, 2024
4aae48d
Use local cloned repo for generation
raych1 Dec 7, 2024
ed54ca1
Set the right path to use in generation
raych1 Dec 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Support new interfaces
  • Loading branch information
@raych1
raych1 committed Dec 4, 2024
commit 060e288d6a1c55b35e8f67d4ab415b0513d1734d
826 changes: 97 additions & 729 deletions tools/spec-gen-sdk/package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 0 additions & 2 deletions tools/spec-gen-sdk/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
"node": ">=20.10.0"
},
"dependencies": {
"@azure/identity": "^4.2.0",
"@azure/storage-blob": "~12.17.0",
"@octokit/auth-app": "^2.4.5",
"@octokit/rest": "18.0.3",
"ajv": "^6.12.6",
Expand Down
230 changes: 39 additions & 191 deletions tools/spec-gen-sdk/src/automation/entrypoint.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,40 +1,33 @@
import { RestEndpointMethodTypes } from '@octokit/rest';
import { Octokit } from '@octokit/rest';
import {
BlobServiceClient,
ContainerClient,
} from '@azure/storage-blob';
import * as winston from 'winston';
import { getAuthenticatedOctokit, RepoKey } from '../utils/githubUtils';
import {
FailureType,
setFailureType,
WorkflowContext,
workflowFilterSdkMain,
workflowInit,
workflowMain
} from './workflow';
import { TriggerType } from '../types/TriggerType';
import {
getBlobName,
loggerConsoleTransport,
loggerDevOpsTransport,
loggerStorageAccountTransport,
loggerTestTransport,
loggerWaitToFinish,
sdkAutoLogLevels
} from './logging';
import { sdkAutoReportStatus } from './reportStatus';
import { SDKAutomationState } from './sdkAutomationState';
import { DefaultAzureCredential } from '@azure/identity';
import * as pkginfo from 'pkginfo';
//import { sdkAutoReportStatus } from './reportStatus';

interface SdkAutoOptions {
specRepo: RepoKey;
pullNumber: number;
sdkName: string;
branchPrefix: string;
filterSwaggerToSdk?: boolean;
localSpecRepoPath: string;
localSdkRepoPath: string;
tspConfigPath?: string;
readmePath?: string;
pullNumber?: number;
specCommitSha?: string;
specRepoHttpsUrl?: string;

github: {
token?: string;
Expand All @@ -43,78 +36,25 @@ interface SdkAutoOptions {
commentAuthorName?: string;
};

storage: {
name: string;
prefix: string;
downloadCommand: string;
isPublic: boolean;
};

buildID?: string;
runEnv: 'local' | 'azureDevOps' | 'test';
}

type SpecPrInfo = {
head: {owner: string; repo: string};
base: {owner: string; repo: string};
}

export type SdkAutoContext = {
config: SdkAutoOptions;
octokit: Octokit;
getGithubAccessToken: (owner: string) => Promise<string>;
logger: winston.Logger;
useMergedRoutine: boolean;
trigger: TriggerType;
specCommitSha: string;
specHeadRef: string;
specHtmlUrl: string;
specIsPrivate: boolean;
specPrInfo: {head: {owner: string; repo: string}, base: {owner: string; repo: string}};
specPrBaseBranch: string;
specPrHeadBranch: string;
specPrTitle: string;
specPrHtmlUrl: string;
specPrInfo: SpecPrInfo | undefined;
specPrBaseBranch: string | undefined;
specPrHeadBranch: string | undefined;
workingFolder: string;
blobContainerClient: ContainerClient;
logsBlobUrl?: string;
version: string;
autorestConfig?: string;
};

const getAutorestConfigFromPRComment = async (
octokit: Octokit,
owner: string,
repo: string,
pullNumber: number,
sdkName: string,
logger: winston.Logger
) => {
let comments: (string | undefined)[] | undefined = undefined;
try {
const res = await octokit.issues.listComments({
owner,
repo,
issue_number: pullNumber
});
if (res.status !== 200) {
throw new Error(`Error: Get autorest configuration from PR https://github.com/${owner}/${repo}/pull/${pullNumber} failed with status code: ${res.status}. If autorest config is requried, please re-run the failed job in pipeline run or emit '/azp run' in the PR comment to trigger the re-run if the status code is retryable.`);
}
comments = res.data.map(e => e.body).map(e => {
if (!e.includes('\r\n') && e.includes('\n')) {
return e.replace(/\n/g, '\r\n');
}
return e;
});
} catch (e) {
logger.warn(e.message);
}

const regexToFilterComment = new RegExp(`#+ *${sdkName}`);
let autorestConfigComment: string|undefined = undefined;
for (const comment of comments!) {
if (!comment) { continue; }
if (comment.match(regexToFilterComment)) {
autorestConfigComment = comment;
}
}
return autorestConfigComment;
};

export const getSdkAutoContext = async (options: SdkAutoOptions): Promise<SdkAutoContext> => {
const logger = winston.createLogger({
Expand All @@ -129,63 +69,14 @@ export const getSdkAutoContext = async (options: SdkAutoOptions): Promise<SdkAut
logger.add(loggerTestTransport());
}

logger.info(
`Working on https://github.com/${options.specRepo.owner}/${options.specRepo.name}/pull/${options.pullNumber}`
);

const [{ octokit, getGithubAccessToken, specPR }, { blobContainerClient, logsBlobName }] = await Promise.all([
const [{ octokit, getGithubAccessToken, specPR }] = await Promise.all([
getGithubContext(options, logger),
getStorageContext(options, logger)
]);

const version = pkginfo.read(module).package.version;
logger.info(`SDK Automation ${version}`);

logger.info(
`Getting autorest configuration from PR https://github.com/${options.specRepo.owner}/${options.specRepo.name}/pull/${options.pullNumber}`
);

const autorestConfig = await getAutorestConfigFromPRComment(
octokit,
options.specRepo.owner,
options.specRepo.name,
options.pullNumber,
options.sdkName,
logger
);

const useMergedRoutine =
specPR.state === 'closed' && specPR.merged ? true : specPR.state === 'open' ? false : undefined;
if (useMergedRoutine === undefined) {
throw new Error(`TriggerError: PR ${options.pullNumber} is closed and isn't merged. Please re-open the PR to trigger the SDK generation.`);
}
if (useMergedRoutine && specPR.base.ref !== 'main') {
throw new Error(`TriggerError: PR ${options.pullNumber} is not merged to main branch. This PR state isn't supported by the SDK Automation. Please ensure the PR is open or is merged to main branch.`);
}
const trigger: TriggerType = useMergedRoutine ? 'continuousIntegration' : 'pullRequest';
logger.info(`Trigger type: ${trigger}`);

const specCommitSha = specPR.merge_commit_sha;
if (!specCommitSha) {
throw new Error(`TriggerError: PR ${options.pullNumber} doesn't have merge_commit_sha. Maybe there's a conflict. Please ensure the PR is merged to main correctly in order to trigger the SDK Automation.`);
}

const workingFolder = '.';

const logsBlobUrl = `${logsBlobName}`;

return {
config: options,
logger,
octokit,
getGithubAccessToken,
useMergedRoutine,
trigger,
specCommitSha,
workingFolder,
specIsPrivate: specPR.base.repo.private,
specHtmlUrl: specPR.base.repo.html_url,
specPrInfo: {
let specPrInfo: SpecPrInfo | undefined
if (specPR) {
specPrInfo = {
head: {
owner: specPR.head.repo.owner.login,
repo: specPR.head.repo.name
Expand All @@ -194,49 +85,26 @@ export const getSdkAutoContext = async (options: SdkAutoOptions): Promise<SdkAut
owner: specPR.base.repo.owner.login,
repo: specPR.base.repo.name
}
},
specPrBaseBranch: specPR.base.ref,
specPrHeadBranch: specPR.head.ref,
specPrTitle: specPR.title,
specPrHtmlUrl: specPR.html_url,
specHeadRef: useMergedRoutine ? specPR.base.ref : `refs/pull/${specPR.number}/merge`,
blobContainerClient,
logsBlobUrl,
version,
autorestConfig
};
};

const getStorageContext = async (options: SdkAutoOptions, logger: winston.Logger) => {
const credential = new DefaultAzureCredential({
loggingOptions: { allowLoggingAccountIdentifiers: true },
});
const serviceClient = new BlobServiceClient(
`https://${options.storage.name}.blob.core.windows.net`,
credential
);
const blobContainerClient = serviceClient.getContainerClient(options.storage.prefix);

if (options.filterSwaggerToSdk || options.runEnv === 'test') {
return { blobContainerClient };
};
}

logger.info(`Ensure blob container exists: ${blobContainerClient.url}`);
await blobContainerClient.createIfNotExists();

const blobLogger = await loggerStorageAccountTransport(
{ blobContainerClient, config: options },
getBlobName({ config: options }, 'logs.txt')
);
logger.add(blobLogger.blobTransport);
const logsBlobName = `${blobLogger.blobName}`;
logger.info(`Log to ${logsBlobName}`);
return { blobContainerClient, logsBlobName };
return {
config: options,
octokit,
getGithubAccessToken,
logger,
specPrInfo,
specPrBaseBranch: specPR?.base.ref,
specPrHeadBranch: specPR?.head.ref,
workingFolder,
};
};

const getGithubContext = async (options: SdkAutoOptions, logger: winston.Logger) => {
const {octokit, getToken: getGithubAccessToken} = getAuthenticatedOctokit(options.github, logger);
const {octokit, getToken: getGithubAccessToken} = getAuthenticatedOctokit(options.github, logger);

if (!options.pullNumber) {
return { octokit, getGithubAccessToken, specPR: undefined };
}
let specPR: RestEndpointMethodTypes['pulls']['get']['response']['data'];
do {
const rsp = await octokit.pulls.get({
Expand All @@ -246,24 +114,16 @@ const getGithubContext = async (options: SdkAutoOptions, logger: winston.Logger)
});
specPR = rsp.data;
} while (specPR.mergeable === null && !specPR.merged);

return { octokit, getGithubAccessToken, specPR };
};

export const sdkAutoMain = async (options: SdkAutoOptions) => {
const sdkContext = await getSdkAutoContext(options);
let workflowContext: WorkflowContext | undefined = undefined;
let workflowFilterSdkMainStatus: SDKAutomationState | undefined = undefined;
// identify the sdkAutoMain whether to run the sdk filter or SdkGen
const runSdkFilter = options.filterSwaggerToSdk;

try {
if (runSdkFilter) {
await workflowFilterSdkMain(sdkContext);
} else {
workflowContext = await workflowInit(sdkContext);
await workflowMain(workflowContext);
}
workflowContext = await workflowInit(sdkContext);
await workflowMain(workflowContext);
} catch (e) {
if (workflowContext) {
sdkContext.logger.error(`FatalError: ${e.message}. Please refer to the inner logs for details or report this issue through https://aka.ms/azsdk/support/specreview-channel.`);
Expand All @@ -274,23 +134,11 @@ export const sdkAutoMain = async (options: SdkAutoOptions) => {
if (e.stack) {
sdkContext.logger.error(`ErrorStack: ${e.stack}.`);
}
if (runSdkFilter) {
workflowFilterSdkMainStatus = 'failed';
console.log(`##vso[task.setVariable variable=SkipAll;isOutput=true]true`);
// hardcode the skipped job name when the runSdkFilter failed. That can help customer can easily jump to devops pipeline info
console.log(`##vso[task.setVariable variable=SkippedJobs]azure-sdk-for-go`);
console.log(`##vso[task.complete result=Failed;]`);
}
}
if (workflowContext) {
await sdkAutoReportStatus(workflowContext);
}
await loggerWaitToFinish(sdkContext.logger);
if (runSdkFilter) {
return workflowFilterSdkMainStatus;
} else {
return workflowContext?.status;
//await sdkAutoReportStatus(workflowContext);
}
return workflowContext?.status;
};

export const getLanguageByRepoName = (repoName: string) => {
Expand Down
Loading