compileProtos.js
11.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
#!/usr/bin/env node
"use strict";
/**
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.main = exports.generateRootName = void 0;
const fs = require("fs");
const path = require("path");
const util = require("util");
const pbjs = require("protobufjs/cli/pbjs");
const pbts = require("protobufjs/cli/pbts");
const readdir = util.promisify(fs.readdir);
const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);
const stat = util.promisify(fs.stat);
const pbjsMain = util.promisify(pbjs.main);
const pbtsMain = util.promisify(pbts.main);
const PROTO_LIST_REGEX = /_proto_list\.json$/;
const apacheLicense = `// Copyright ${new Date().getFullYear()} Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
`;
/**
* Recursively scans directories starting from `directory` and finds all files
* matching `PROTO_LIST_REGEX`.
*
* @param {string} directory Path to start the scan from.
* @return {Promise<string[]} Resolves to an array of strings, each element is a full path to a matching file.
*/
async function findProtoJsonFiles(directory) {
const result = [];
const files = await readdir(directory);
for (const file of files) {
const fullPath = path.join(directory, file);
const fileStat = await stat(fullPath);
if (fileStat.isFile() && file.match(PROTO_LIST_REGEX)) {
result.push(fullPath);
}
else if (fileStat.isDirectory()) {
const nested = await findProtoJsonFiles(fullPath);
result.push(...nested);
}
}
return result;
}
/**
* Normalizes the Linux path for the current operating system.
*
* @param {string} filePath Linux-style path (with forward slashes)
* @return {string} Normalized path.
*/
function normalizePath(filePath) {
return path.join(...filePath.split('/'));
}
function getAllEnums(dts) {
const result = new Set();
const lines = dts.split('\n');
const nestedIds = [];
let currentEnum = undefined;
for (const line of lines) {
const match = line.match(/^\s*(?:export )?(namespace|class|interface|enum) (\w+) .*{/);
if (match) {
const [, keyword, id] = match;
nestedIds.push(id);
if (keyword === 'enum') {
currentEnum = nestedIds.join('.');
result.add(currentEnum);
}
continue;
}
if (line.match(/^\s*}/)) {
nestedIds.pop();
currentEnum = undefined;
continue;
}
}
return result;
}
function updateDtsTypes(dts, enums) {
const lines = dts.split('\n');
const result = [];
for (const line of lines) {
let typeName = undefined;
// Enums can be used in interfaces and in classes.
// For simplicity, we'll check these two cases independently.
// encoding?: (google.cloud.speech.v1p1beta1.RecognitionConfig.AudioEncoding|null);
const interfaceMatch = line.match(/"?\w+"?\?: \(([\w.]+)\|null\);/);
if (interfaceMatch) {
typeName = interfaceMatch[1];
}
// public encoding: google.cloud.speech.v1p1beta1.RecognitionConfig.AudioEncoding;
const classMatch = line.match(/public \w+: ([\w.]+);/);
if (classMatch) {
typeName = classMatch[1];
}
if (line.match(/\(number\|Long(?:\|null)?\)/)) {
typeName = 'Long';
}
let replaced = line;
if (typeName && enums.has(typeName)) {
// enum: E => E|keyof typeof E to allow all string values
replaced = replaced.replace(typeName, `${typeName}|keyof typeof ${typeName}`);
}
else if (typeName === 'Uint8Array') {
// bytes: Uint8Array => Uint8Array|string to allow base64-encoded strings
replaced = replaced.replace(typeName, `${typeName}|string`);
}
else if (typeName === 'Long') {
// Longs can be passed as strings :(
// number|Long => number|Long|string
replaced = replaced.replace('number|Long', 'number|Long|string');
}
// add brackets if we have added a |
replaced = replaced.replace(/: ([\w.]+\|[ \w.|]+);/, ': ($1);');
result.push(replaced);
}
return result.join('\n');
}
function fixJsFile(js) {
// 1. fix protobufjs require: we don't want the libraries to
// depend on protobufjs, so we re-export it from google-gax
js = js.replace('require("protobufjs/minimal")', 'require("google-gax").protobufMinimal');
// 2. add Apache license to the generated .js file
js = apacheLicense + js;
return js;
}
function fixDtsFile(dts) {
// 1. fix for pbts output: the corresponding protobufjs PR
// https://github.com/protobufjs/protobuf.js/pull/1166
// is merged but not yet released.
if (!dts.match(/import \* as Long/)) {
dts = 'import * as Long from "long";\n' + dts;
}
// 2. fix protobufjs import: we don't want the libraries to
// depend on protobufjs, so we re-export it from google-gax
dts = dts.replace('import * as $protobuf from "protobufjs"', 'import {protobuf as $protobuf} from "google-gax"');
// 3. add Apache license to the generated .d.ts file
dts = apacheLicense + dts;
// 4. major hack: update types to allow passing strings
// where enums, longs, or bytes are expected
const enums = getAllEnums(dts);
dts = updateDtsTypes(dts, enums);
return dts;
}
/**
* Returns a combined list of proto files listed in all JSON files given.
*
* @param {string[]} protoJsonFiles List of JSON files to parse
* @return {Promise<string[]>} Resolves to an array of proto files.
*/
async function buildListOfProtos(protoJsonFiles) {
const result = [];
for (const file of protoJsonFiles) {
const directory = path.dirname(file);
const content = await readFile(file);
const list = JSON.parse(content.toString()).map((filePath) => path.join(directory, normalizePath(filePath)));
result.push(...list);
}
return result;
}
/**
* Runs `pbjs` to compile the given proto files, placing the result into
* `./protos/protos.json`. No support for changing output filename for now
* (but it's a TODO!)
*
* @param {string} rootName Name of the root object for pbjs static module (-r option)
* @param {string[]} protos List of proto files to compile.
*/
async function compileProtos(rootName, protos) {
// generate protos.json file from proto list
const jsonOutput = path.join('protos', 'protos.json');
if (protos.length === 0) {
// no input file, just emit an empty object
await writeFile(jsonOutput, '{}');
return;
}
const pbjsArgs4JSON = [
'--target',
'json',
'-p',
'protos',
'-p',
path.join(__dirname, '..', '..', 'protos'),
'-o',
jsonOutput,
];
pbjsArgs4JSON.push(...protos);
await pbjsMain(pbjsArgs4JSON);
// generate protos/protos.js from protos.json
const jsOutput = path.join('protos', 'protos.js');
const pbjsArgs4js = [
'-r',
rootName,
'--target',
'static-module',
'-p',
'protos',
'-p',
path.join(__dirname, '..', '..', 'protos'),
'-o',
jsOutput,
];
pbjsArgs4js.push(...protos);
await pbjsMain(pbjsArgs4js);
let jsResult = (await readFile(jsOutput)).toString();
jsResult = fixJsFile(jsResult);
await writeFile(jsOutput, jsResult);
// generate protos/protos.d.ts
const tsOutput = path.join('protos', 'protos.d.ts');
const pbjsArgs4ts = [jsOutput, '-o', tsOutput];
await pbtsMain(pbjsArgs4ts);
let tsResult = (await readFile(tsOutput)).toString();
tsResult = fixDtsFile(tsResult);
await writeFile(tsOutput, tsResult);
}
/**
*
* @param directories List of directories to process. Normally, just the
* `./src` folder of the given client library.
* @return {Promise<string>} Resolves to a unique name for protobuf root to use in the JS static module, or 'default'.
*/
async function generateRootName(directories) {
// We need to provide `-r root` option to `pbjs -t static-module`, otherwise
// we'll have big problems if two different libraries are used together.
// It's OK to play some guessing game here: if we locate `package.json`
// with a package name, we'll use it; otherwise, we'll fallback to 'default'.
for (const directory of directories) {
const packageJson = path.resolve(directory, '..', 'package.json');
if (fs.existsSync(packageJson)) {
const json = JSON.parse((await readFile(packageJson)).toString());
const name = json.name.replace(/[^\w\d]/g, '_');
const hopefullyUniqueName = `${name}_protos`;
return hopefullyUniqueName;
}
}
return 'default';
}
exports.generateRootName = generateRootName;
/**
* Main function. Takes an array of directories to process.
* Looks for JSON files matching `PROTO_LIST_REGEX`, parses them to get a list of all
* proto files used by the client library, and calls `pbjs` to compile them all into
* JSON (`pbjs -t json`).
*
* Exported to be called from a test.
*
* @param {string[]} directories List of directories to process. Normally, just the
* `./src` folder of the given client library.
*/
async function main(directories) {
const protoJsonFiles = [];
for (const directory of directories) {
protoJsonFiles.push(...(await findProtoJsonFiles(directory)));
}
const rootName = await generateRootName(directories);
const protos = await buildListOfProtos(protoJsonFiles);
await compileProtos(rootName, protos);
}
exports.main = main;
/**
* Shows the usage information.
*/
function usage() {
console.log(`Usage: node ${process.argv[1]} directory ...`);
console.log(`Finds all files matching ${PROTO_LIST_REGEX} in the given directories.`);
console.log('Each of those files should contain a JSON array of proto files used by the');
console.log('client library. Those proto files will be compiled to JSON using pbjs tool');
console.log('from protobufjs.');
}
if (require.main === module) {
if (process.argv.length <= 2) {
usage();
// eslint-disable-next-line no-process-exit
process.exit(1);
}
// argv[0] is node.js binary, argv[1] is script path
main(process.argv.slice(2));
}
//# sourceMappingURL=compileProtos.js.map