홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Thomas Vachuska
Committed by Gerrit Code Review
275d2e82 1 parent cc5f03e8

Refactoring checkstyle daemon into a more general purpose buck daemon. 

Change-Id: I999541e8894f07061141b3a62414e491369f8d08
Inline Side-by-side
Showing 7 changed files with 324 additions and 153 deletions
......@@ -37,10 +37,11 @@ def checkstyle(
sh_test(
name = name + '-checkstyle',
test = '//tools/build/conf:start-checkstyle',
test = '//tools/build/conf:start-buck-daemon',
deps = [ jar_target ],
args = [
'$(location //tools/build/conf:checkstyle-jar)',
'$(location //tools/build/conf:buck-daemon-jar)',
'checkstyle',
'$(location :' + name + '-checkstyle-files)',
'$(location //tools/build/conf:checkstyle-xml)',
'$(location //tools/build/conf:suppressions-xml)',
......
......@@ -14,7 +14,7 @@ export_file (
)
export_file (
name = 'start-checkstyle',
name = 'start-buck-daemon',
visibility = [ 'PUBLIC' ],
)
......@@ -38,9 +38,9 @@ java_library (
)
java_binary (
name = 'checkstyle-jar',
name = 'buck-daemon-jar',
deps = [ ':checkstyle' ] + RUN,
main_class = 'org.onosproject.checkstyle.Main',
main_class = 'org.onosproject.buckdaemon.BuckDaemon',
blacklist = [ 'META-INF/.*' ],
visibility = [ 'PUBLIC' ],
)
......
/*
* Copyright 2016-present Open Networking Laboratory
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -14,54 +14,80 @@
* limitations under the License.
*/
package org.onosproject.checkstyle;
package org.onosproject.buckdaemon;
import com.google.common.io.ByteStreams;
import com.puppycrawl.tools.checkstyle.api.CheckstyleException;
import org.onosproject.checkstyle.CheckstyleRunner;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static java.nio.file.StandardOpenOption.*;
/**
* Main program for executing scenario test warden.
* Buck daemon process.
*/
public final class Main {
public final class BuckDaemon {
private static long POLLING_INTERVAL = 1000; //ms
private final Map<String, BuckTask> tasks = new HashMap<>();
private final String portLock;
private final String buckPid;
// Public construction forbidden
private Main(String[] args) {
private BuckDaemon(String[] args) {
portLock = args[0];
buckPid = args[1];
}
/**
* Main entry point for the cell warden.
* Main entry point for the daemon.
*
* @param args command-line arguments
*/
public static void main(String[] args)
throws CheckstyleException, IOException {
startServer(args);
BuckDaemon daemon = new BuckDaemon(args);
daemon.registerTasks();
daemon.startServer();
}
/**
* Registers re-entrant tasks by their task name.
*/
private void registerTasks() {
tasks.put("checkstyle", new CheckstyleRunner(System.getProperty("checkstyle.config"),
System.getProperty("checkstyle.suppressions")));
// tasks.put("swagger", new SwaggerGenerator());
}
// Monitors another PID and exit when that process exits
private static void watchProcess(String pid) {
/**
* Monitors another PID and exit when that process exits.
*/
private void watchProcess(String pid) {
if (pid == null || pid.equals("0")) {
return;
}
Timer timer = new Timer(true); // start as a daemon, so we don't hang shutdown
timer.scheduleAtFixedRate(new TimerTask() {
private String cmd = "kill -s 0 " + pid;
@Override
public void run() {
try {
......@@ -79,13 +105,10 @@ public final class Main {
}, POLLING_INTERVAL, POLLING_INTERVAL);
}
// Initiates a server.
private static void startServer(String[] args) throws IOException, CheckstyleException {
String portLock = args[0];
String buckPid = args[1];
String checkstyleFile = args[2];
String suppressionsFile = args[3];
/**
* Initiates a server.
*/
private void startServer() throws IOException, CheckstyleException {
// Use a file lock to ensure only one copy of the daemon runs
Path portLockPath = Paths.get(portLock);
FileChannel channel = FileChannel.open(portLockPath, WRITE, CREATE);
......@@ -104,10 +127,8 @@ public final class Main {
// Set up hook to clean up after ourselves
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
if (channel != null) {
channel.truncate(0);
channel.close();
}
channel.truncate(0);
channel.close();
System.err.println("tear down...");
Files.delete(portLockPath);
} catch (IOException e) {
......@@ -122,15 +143,58 @@ public final class Main {
channel.write(ByteBuffer.wrap(Integer.toString(port).getBytes()));
// Instantiate a Checkstyle runner and executor; serve until exit...
CheckstyleRunner runner = new CheckstyleRunner(checkstyleFile, suppressionsFile);
ExecutorService executor = Executors.newCachedThreadPool();
while (true) {
try {
executor.submit(runner.checkClass(server.accept()));
executor.submit(new BuckTaskRunner(server.accept()));
} catch (Exception e) {
e.printStackTrace();
//no-op
}
}
}
/**
* Runnable capable of invoking the appropriate Buck task with input
* consumed form the specified socket and output produced back to that
* socket.
*/
private class BuckTaskRunner implements Runnable {
private final Socket socket;
public BuckTaskRunner(Socket socket) {
this.socket = socket;
}
@Override
public void run() {
try {
BuckTaskContext context = new BuckTaskContext(socket.getInputStream());
String taskName = context.taskName();
if (!taskName.isEmpty()) {
BuckTask task = tasks.get(taskName);
if (task != null) {
System.out.println(String.format("Executing task '%s'", taskName));
task.execute(context);
for (String line : context.output()) {
output(socket, line);
}
} else {
String message = String.format("No task named '%s'", taskName);
System.out.print(message);
output(socket, message);
}
}
socket.getOutputStream().flush();
socket.close();
} catch (IOException e) {
e.printStackTrace();
}
}
private void output(Socket socket, String line) throws IOException {
socket.getOutputStream().write((line + "\n").getBytes());
}
}
}
......
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.buckdaemon;
/**
* Abstraction of a Buck task that can be spawned by the Buck daemon
*/
public interface BuckTask {
/**
* Executes the task, consuming the specified input and producing output.
*
* @param context context for the tast operation
*/
void execute(BuckTaskContext context);
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.buckdaemon;
import com.google.common.collect.ImmutableList;
import com.google.common.io.ByteStreams;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import static com.google.common.base.Preconditions.checkArgument;
/**
* Context for executing a single Buck task.
*/
public class BuckTaskContext {
private final String taskName;
private final ImmutableList<String> input;
private final List<String> output = new ArrayList<>();
BuckTaskContext(InputStream inputString) throws IOException {
String[] split = new String(ByteStreams.toByteArray(inputString)).split("\n");
checkArgument(split.length >= 1, "Request must contain at least task type");
this.taskName = split[0];
ImmutableList.Builder<String> builder = ImmutableList.builder();
for (int i = 1; i < split.length; i++) {
builder.add(split[i]);
}
input = builder.build();
}
/**
* Returns the symbolic task name.
*/
public String taskName() {
return taskName;
}
/**
* Returns the input data a list of strings.
*
* @return input data
*/
public List<String> input() {
return ImmutableList.copyOf(input);
}
/**
* Returns the output data a list of strings.
*
* @return output data
*/
List<String> output() {
return ImmutableList.copyOf(output);
}
/**
* Adds a line to the output data.
*
* @param line line of output data
*/
public void output(String line) {
output.add(line);
}
}
......@@ -15,7 +15,6 @@
*/
package org.onosproject.checkstyle;
import com.google.common.io.ByteStreams;
import com.puppycrawl.tools.checkstyle.Checker;
import com.puppycrawl.tools.checkstyle.ConfigurationLoader;
import com.puppycrawl.tools.checkstyle.DefaultConfiguration;
......@@ -24,64 +23,51 @@ import com.puppycrawl.tools.checkstyle.api.AuditEvent;
import com.puppycrawl.tools.checkstyle.api.AuditListener;
import com.puppycrawl.tools.checkstyle.api.CheckstyleException;
import com.puppycrawl.tools.checkstyle.api.Configuration;
import org.onosproject.buckdaemon.BuckTask;
import org.onosproject.buckdaemon.BuckTaskContext;
import java.io.File;
import java.io.IOException;
import java.net.Socket;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.google.common.base.Strings.isNullOrEmpty;
public class CheckstyleRunner {
/**
* Buck task for executing checkstyle on the specified project files.
*/
public class CheckstyleRunner implements BuckTask {
private final Configuration config;
public CheckstyleRunner(String configLocation, String suppressionLocation)
throws CheckstyleException {
// create a configuration
DefaultConfiguration config = (DefaultConfiguration) ConfigurationLoader.loadConfiguration(
configLocation, new PropertiesExpander(System.getProperties()));
// add the suppression file to the configuration
DefaultConfiguration suppressions = new DefaultConfiguration("SuppressionFilter");
suppressions.addAttribute("file", suppressionLocation);
config.addChild(suppressions);
this.config = config;
}
public Runnable checkClass(Socket socket) {
return () -> {
try {
String input = new String(ByteStreams.toByteArray(socket.getInputStream()));
String output = checkClass(input);
socket.getOutputStream().write(output.getBytes());
socket.getOutputStream().flush();
socket.close();
} catch (IOException e) {
e.printStackTrace();
} catch (CheckstyleException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
};
public CheckstyleRunner(String configLocation, String suppressionLocation) {
try {
// create a configuration
DefaultConfiguration config = (DefaultConfiguration) ConfigurationLoader
.loadConfiguration(configLocation, new PropertiesExpander(System.getProperties()));
// add the suppression file to the configuration
DefaultConfiguration suppressions = new DefaultConfiguration("SuppressionFilter");
suppressions.addAttribute("file", suppressionLocation);
config.addChild(suppressions);
this.config = config;
} catch (CheckstyleException e) {
throw new RuntimeException(e);
}
}
public String checkClass(String input) throws CheckstyleException, InterruptedException {
String[] split = input.split("\n", 3);
if (split.length < 3 || split[2].length() == 0) {
return "";
@Override
public void execute(BuckTaskContext context) {
List<String> input = context.input();
if (input.size() < 3 || input.get(2).length() == 0) {
return;
}
String project = split[0];
String baseDir = split[1];
String files = split[2];
String project = input.get(0);
String baseDir = input.get(1);
// create a listener for output
StringAuditor listener = new StringAuditor();
StringAuditor listener = new StringAuditor(context);
listener.setProjectName(project);
listener.setBaseDir(baseDir);
......@@ -91,102 +77,102 @@ public class CheckstyleRunner {
checker.setModuleClassLoader(moduleClassLoader);
try {
checker.configure(config);
checker.addListener(listener);
// run Checker
List<File> fileList = Stream.of(files.split("\n"))
.map(File::new)
.collect(Collectors.toList());
List<File> fileList = input.subList(2, input.size() - 1).stream()
.map(File::new)
.collect(Collectors.toList());
int errorCounter = checker.process(fileList);
if (errorCounter > 0) {
listener.append("CHECKSTYLE ERROR\n");
context.output("CHECKSTYLE ERROR");
}
listener.await();
} catch (CheckstyleException | InterruptedException e) {
throw new RuntimeException(e);
} finally {
checker.destroy();
}
return listener.getAudit();
}
}
class StringAuditor implements AuditListener {
private CountDownLatch finishedLatch = new CountDownLatch(1);
private StringBuilder output = new StringBuilder();
private String baseDir = "";
private String project = "";
public void setBaseDir(String base) {
this.baseDir = base;
}
static class StringAuditor implements AuditListener {
public void setProjectName(String projectName) {
this.project = projectName;
}
private final BuckTaskContext context;
private CountDownLatch finishedLatch = new CountDownLatch(1);
private String baseDir = "";
private String project = "";
public void append(String s) {
output.append(s);
}
public String getAudit() throws InterruptedException {
finishedLatch.await();
return output.toString();
}
StringAuditor(BuckTaskContext context) {
this.context = context;
}
@Override
public void auditStarted(AuditEvent evt) {
public void setBaseDir(String base) {
this.baseDir = base;
}
}
public void setProjectName(String projectName) {
this.project = projectName;
}
@Override
public void auditFinished(AuditEvent evt) {
finishedLatch.countDown();
}
public void await() throws InterruptedException {
finishedLatch.await();
}
@Override
public void fileStarted(AuditEvent evt) {
@Override
public void auditStarted(AuditEvent evt) {
}
}
@Override
public void auditFinished(AuditEvent evt) {
finishedLatch.countDown();
}
@Override
public void fileFinished(AuditEvent evt) {
@Override
public void fileStarted(AuditEvent evt) {
}
}
@Override
public void fileFinished(AuditEvent evt) {
}
@Override
public void addError(AuditEvent evt) {
switch (evt.getSeverityLevel()) {
case ERROR:
String fileName = evt.getFileName();
if (!isNullOrEmpty(baseDir)) {
int index = fileName.indexOf(baseDir);
if (index >= 0) {
fileName = fileName.substring(index + baseDir.length() + 1);
if (!isNullOrEmpty(project)) {
output.append(project).append(':');
@Override
public void addError(AuditEvent evt) {
switch (evt.getSeverityLevel()) {
case ERROR:
StringBuilder output = new StringBuilder();
String fileName = evt.getFileName();
if (!isNullOrEmpty(baseDir)) {
int index = fileName.indexOf(baseDir);
if (index >= 0) {
fileName = fileName.substring(index + baseDir.length() + 1);
if (!isNullOrEmpty(project)) {
output.append(project).append(':');
}
}
}
}
output.append(fileName).append(':').append(evt.getLine());
if (evt.getColumn() > 0) {
output.append(':').append(evt.getColumn());
}
output.append(": ").append(evt.getMessage());
output.append('\n');
break;
case IGNORE:
case INFO:
case WARNING:
default:
break;
output.append(fileName).append(':').append(evt.getLine());
if (evt.getColumn() > 0) {
output.append(':').append(evt.getColumn());
}
output.append(": ").append(evt.getMessage());
context.output(output.toString());
break;
case IGNORE:
case INFO:
case WARNING:
default:
break;
}
}
}
@Override
public void addException(AuditEvent evt, Throwable throwable) {
addError(evt);
output.append(throwable.getMessage());
@Override
public void addException(AuditEvent evt, Throwable throwable) {
addError(evt);
context.output(throwable.getMessage());
}
}
}
\ No newline at end of file
}
......
#!/bin/bash
CHECKSTYLE=$1
FILES=$2
CONFIG=$3
SUPPRESSIONS=$4
# -----------------------------------------------------------------------------
# Launches Buck daemon if not already running and requests Buck task execution.
# -----------------------------------------------------------------------------
BUCK_DAEMON=$1
TASK=${2:-unspecified}
DATA=${3}
# TODO: Figure out how to parametrize better
BUCK_PROPS="-Dcheckstyle.config=$4 -Dcheckstyle.suppressions=$5"
PORT_FILE="$1.port"
......@@ -39,15 +45,15 @@ function check_socket() {
return $?
}
# check to see if checkstyle daemon is running; if not, start it
# check to see if buck daemon is running; if not, start it
if ! check_socket; then
# Starting checkstyle server...
# Starting buck daemon...
#FIXME change to /dev/null if/when we are confident
nohup java -jar $CHECKSTYLE $PORT_FILE $(buck_pid) $3 $4 >>/tmp/checkstyle.daemon 2>&1 &
nohup java $BUCK_PROPS -jar $BUCK_DAEMON $PORT_FILE $(buck_pid) >>/tmp/buck.daemon 2>&1 &
TRIES=20
i=0
# Wait for checkstyle server to start for 2 seconds
# Wait for buck daemon to start for 2 seconds
while [ $i -lt $TRIES ]; do
if check_socket; then
CONNECTED=true
......@@ -57,15 +63,15 @@ if ! check_socket; then
sleep 0.1
done
if [ -z "$CONNECTED" ]; then
echo "Failed to start checkstyle server"
echo "Failed to start buck daemon"
exit 3
fi
fi
# run the actual checkstyle client
OUT=$(cat $FILES | nc localhost $(port))
# run the actual buck daemon client
OUT=$((printf "%s\n" $TASK; cat $DATA) | nc localhost $(port))
if [ $? -ne 0 ]; then
echo "Error connecting to checkstyle server"
echo "Error connecting to buck daemon server"
exit 2
fi
if [ -n "$OUT" ]; then
......