홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Brian O'Connor
Committed by Gerrit Code Review
be95f689 1 parent f5896bea

Adding Checkstyle daemon 

Lazily instaniate a checkstyle daemon for the first checkstyle job.
Then, each subsequent checkstyle target uses the daemon.

The daemon is terminated when the parent buck or buckd exits.

Change-Id: I4dbea957f20a3f77048dd25d960b7faa1eafef37
Inline Side-by-side
Showing 8 changed files with 513 additions and 41 deletions
......@@ -110,39 +110,29 @@ def osgi_jar(
### Checkstyle
if srcs:
chk_cmd = '#!/bin/bash\n'
base = get_base_path()
chk_cmd += 'OUT=$3\n'
chk_cmd += ' '.join(( 'java -jar $1',
'-c $2',
' '.join(['%s/%s' % (base, s) for s in srcs]) ))
chk_cmd += ' | tee $OUT'
chk_cmd += ' | grep -E "^.*[0-9]+:[0-9]+: error:"'
chk_cmd += ' | sed "s#^.*%s/#%s:#g"\n' % (base, name)
chk_cmd += 'RESULT=(${PIPESTATUS[*]})\n' # RESULT[0] is checkstyle result, RESULT[2] is grep
chk_cmd += 'test ${RESULT[2]} -eq 0 || cat $OUT\n'
chk_cmd += 'rm $OUT\n'
chk_cmd += 'exit ${RESULT[0]}'
files = base + '\n' + '\n'.join(['%s/%s' % (base, s) for s in srcs])
genrule(
name = name + '-checkstyle-sh',
bash = "echo '%s' > $OUT && chmod +x $OUT" % chk_cmd,
name = name + '-checkstyle-files',
bash = "echo '%s' > $OUT" % files,
srcs = srcs,
out = 'checkstyle.sh',
out = 'checkstyle-files.txt',
)
sh_test(
name = name + '-checkstyle',
test = ':' + name + '-checkstyle-sh',
args = [
'$(location //lib:checkstyle)',
'$(location //tools/build/conf:checkstyle-xml)',
'`mktemp /tmp/%s-checkstyle-XXXXXX`' % name,
],
test = '//tools/build/conf:start-checkstyle',
deps = [
':'+ bare_jar_name,
'//tools/build/conf:suppressions-xml',
],
args = [
'$(location //tools/build/conf:checkstyle-jar)',
'$(location :' + name + '-checkstyle-files)',
'$(location //tools/build/conf:checkstyle-xml)',
'$(location //tools/build/conf:suppressions-xml)',
],
test_rule_timeout_ms = 20000,
labels = [ 'checkstyle' ],
)
else:
......@@ -219,4 +209,6 @@ def osgi_jar_with_tests(
resources = test_resources,
resources_root = test_resources_root,
visibility = visibility
)
)
#FIXME need to run checkstyle on test sources
\ No newline at end of file
......
# ***** This file was auto-generated at Wed May 18 14:26:50 PDT 2016. Do not edit this file manually. *****
# ***** This file was auto-generated at Wed May 18 17:55:44 PDT 2016. Do not edit this file manually. *****
osgi_feature_group(
name = 'COMPILE',
visibility = ['PUBLIC'],
......@@ -321,6 +321,15 @@ remote_jar (
)
remote_jar (
name = 'commons-beanutils',
out = 'commons-beanutils-1.9.2.jar',
url = 'mvn:commons-beanutils:commons-beanutils:jar:1.9.2',
sha1 = '7a87d845ad3a155297e8f67d9008f4c1e5656b71',
maven_coords = 'commons-beanutils:commons-beanutils:1.9.2',
visibility = [ 'PUBLIC' ],
)
remote_jar (
name = 'concurrent-trees',
out = 'concurrent-trees-2.4.0.jar',
url = 'mvn:com.googlecode.concurrent-trees:concurrent-trees:jar:2.4.0',
......@@ -420,6 +429,15 @@ remote_jar (
)
remote_jar (
name = 'antlr',
out = 'antlr-2.7.7.jar',
url = 'mvn:antlr:antlr:jar:2.7.7',
sha1 = '83cd2cd674a217ade95a4bb83a8a14f351f48bd0',
maven_coords = 'antlr:antlr:jar:NON-OSGI:2.7.7',
visibility = [ 'PUBLIC' ],
)
remote_jar (
name = 'error_prone_annotations',
out = 'error_prone_annotations-2.0.2.jar',
url = 'mvn:com.google.errorprone:error_prone_annotations:jar:2.0.2',
......@@ -1114,9 +1132,10 @@ remote_jar (
remote_jar (
name = 'checkstyle',
out = 'checkstyle-6.11.2-all.jar',
url = 'http://onlab.vicci.org/onos/third-party/checkstyle-6.11.2-all.jar',
sha1 = 'f504187b1743e73ffe72c2eede0ff57d45536b7d',
out = 'checkstyle-6.11.2.jar',
url = 'mvn:com.puppycrawl.tools:checkstyle:jar:6.11.2',
sha1 = '2705f014697ac0219de0bb2bfc33afb7ec6d22c6',
maven_coords = 'com.puppycrawl.tools:checkstyle:jar:NON-OSGI:6.11.2',
visibility = [ 'PUBLIC' ],
)
......
......@@ -98,6 +98,7 @@
"catalyst-local": "mvn:io.atomix.catalyst:catalyst-local:1.0.4",
"catalyst-serializer": "mvn:io.atomix.catalyst:catalyst-serializer:1.0.4",
"catalyst-transport": "mvn:io.atomix.catalyst:catalyst-transport:1.0.4",
"catalyst-transport": "mvn:io.atomix.catalyst:catalyst-transport:1.0.4",
"commons-codec": "mvn:commons-codec:commons-codec:1.10",
"commons-collections": "mvn:commons-collections:commons-collections:3.2.2",
"commons-configuration": "mvn:commons-configuration:commons-configuration:1.10",
......@@ -107,6 +108,7 @@
"commons-logging": "mvn:commons-logging:commons-logging:1.1.1",
"commons-math3": "mvn:org.apache.commons:commons-math3:3.6.1",
"commons-pool": "mvn:commons-pool:commons-pool:1.6",
"commons-beanutils": "mvn:commons-beanutils:commons-beanutils:1.9.2",
"concurrent-trees": "mvn:com.googlecode.concurrent-trees:concurrent-trees:2.4.0",
"copycat-api": "mvn:org.onosproject:copycat-api:0.5.1.onos",
"copycat-client": "mvn:io.atomix.copycat:copycat-client:1.0.0-rc4",
......@@ -118,6 +120,7 @@
"copycat-state-log": "mvn:org.onosproject:copycat-state-log:0.5.1.onos",
"copycat-state-machine": "mvn:org.onosproject:copycat-state-machine:0.5.1.onos",
"easymock": "mvn:org.easymock:easymock:3.4",
"antlr": "mvn:antlr:antlr:2.7.7",
"error_prone_annotations": "mvn:com.google.errorprone:error_prone_annotations:2.0.2",
"ganymed-ssh2": "mvn:ch.ethz.ganymed:ganymed-ssh2:262",
"jersey-container-jetty-http": "mvn:org.glassfish.jersey.containers:jersey-container-jetty-http:2.22.2",
......@@ -198,7 +201,7 @@
"slf4j-jdk14": "mvn:org.slf4j:slf4j-jdk14:1.7.21",
"typesafe-config": "mvn:com.typesafe:config:1.2.1",
"validation-api": "mvn:javax.validation:validation-api:1.1.0.Final",
"checkstyle": "http://onlab.vicci.org/onos/third-party/checkstyle-6.11.2-all.jar",
"checkstyle": "mvn:com.puppycrawl.tools:checkstyle:6.11.2",
"apache-karaf": "http://onlab.vicci.org/onos/third-party/apache-karaf-3.0.5.tar.gz",
"bndlib": "mvn:biz.aQute.bnd:biz.aQute.bndlib:jar:3.1.0",
"org.apache.felix.scr.bnd": {
......
checkstyle_source = 'src/main/resources/onos/checkstyle.xml'
suppression_source = 'src/main/resources/onos/suppressions.xml'
xml = ('<module name="SuppressionFilter">'
'<property name="file" value="$(location :suppressions-xml)"/>'
'</module>' )
cmd = "sed 's#<module name=\"Checker\">#<module name=\"Checker\">%s#' %s > $OUT" % ( xml, checkstyle_source )
genrule(
export_file (
name = 'checkstyle-xml',
srcs = [ checkstyle_source ],
out = 'checkstyle.xml',
bash = cmd,
visibility = [ 'PUBLIC' ]
src = checkstyle_source,
visibility = [ 'PUBLIC' ],
)
#FIXME location suppression.xml does not trigger this rule
export_file(
export_file (
name = 'suppressions-xml',
src = suppression_source,
visibility = [ 'PUBLIC' ]
visibility = [ 'PUBLIC' ],
)
export_file (
name = 'start-checkstyle',
visibility = [ 'PUBLIC' ],
)
COMPILE = [
'//lib:guava',
'//lib:checkstyle',
]
RUN = [
'//lib:commons-logging',
'//lib:commons-beanutils',
'//lib:commons-lang3',
'//lib:commons-collections',
'//lib:antlr',
]
java_library (
name = 'checkstyle',
srcs = glob([ 'src/main/java/**/*.java' ]),
deps = COMPILE,
)
java_binary (
name = 'checkstyle-jar',
deps = [ ':checkstyle' ] + RUN,
main_class = 'org.onosproject.checkstyle.Main',
blacklist = [ 'META-INF/.*' ],
visibility = [ 'PUBLIC' ],
)
# cmd = '#!/bin/bash\n'
# cmd += '$1 &>/dev/null < /dev/null &'
#
# genrule(
# name = 'checkstyle-sh',
# bash = "echo '%s' > $OUT && chmod +x $OUT" % cmd,
# out = 'checkstyle.sh',
# )
#
# sh_test(
# name = 'checkstyle-runner',
# test = ':checkstyle-sh',
# args = [
# '$(exe :checkstyle-jar)',
# '$(location //lib:checkstyle)',
# '$(location //tools/build/conf:checkstyle-xml)',
# '`mktemp /tmp/%s-checkstyle-XXXXXX`',
# ],
# labels = [ 'checkstyle' ],
# visibility = [ 'PUBLIC' ],
# )
\ No newline at end of file
......
......@@ -34,5 +34,29 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
<version>6.11.2</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<!-- TODO: update once following issue is fixed. -->
<!-- https://jira.codehaus.org/browse/MCOMPILER-205 -->
<version>2.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
......
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.checkstyle;
import com.google.common.io.ByteStreams;
import com.puppycrawl.tools.checkstyle.Checker;
import com.puppycrawl.tools.checkstyle.ConfigurationLoader;
import com.puppycrawl.tools.checkstyle.DefaultConfiguration;
import com.puppycrawl.tools.checkstyle.PropertiesExpander;
import com.puppycrawl.tools.checkstyle.api.AuditEvent;
import com.puppycrawl.tools.checkstyle.api.AuditListener;
import com.puppycrawl.tools.checkstyle.api.CheckstyleException;
import com.puppycrawl.tools.checkstyle.api.Configuration;
import java.io.File;
import java.io.IOException;
import java.net.Socket;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class CheckstyleRunner {
private final Configuration config;
public CheckstyleRunner(String configLocation, String suppressionLocation)
throws CheckstyleException {
// create a configuration
DefaultConfiguration config = (DefaultConfiguration) ConfigurationLoader.loadConfiguration(
configLocation, new PropertiesExpander(System.getProperties()));
// add the suppression file to the configuration
DefaultConfiguration suppressions = new DefaultConfiguration("SuppressionFilter");
suppressions.addAttribute("file", suppressionLocation);
config.addChild(suppressions);
this.config = config;
}
public Runnable checkClass(Socket socket) {
return () -> {
try {
String input = new String(ByteStreams.toByteArray(socket.getInputStream()));
String output = checkClass(input);
socket.getOutputStream().write(output.getBytes());
socket.getOutputStream().flush();
socket.close();
} catch (IOException e) {
e.printStackTrace();
} catch (CheckstyleException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
};
}
public String checkClass(String input) throws CheckstyleException, InterruptedException {
String[] split = input.split("\n", 2);
if (split.length < 2 || split[1].length() == 0) {
return "";
}
String base = split[0];
String files = split[1];
// create a listener for output
StringAuditor listener = new StringAuditor();
listener.setBase(base);
// create Checker object and run it
final Checker checker = new Checker();
final ClassLoader moduleClassLoader = Checker.class.getClassLoader();
checker.setModuleClassLoader(moduleClassLoader);
try {
checker.configure(config);
checker.addListener(listener);
// run Checker
List<File> fileList = Stream.of(files.split("\n"))
.map(File::new)
.collect(Collectors.toList());
int errorCounter = checker.process(fileList);
if (errorCounter > 0) {
listener.append("CHECKSTYLE ERROR\n");
}
} finally {
checker.destroy();
}
return listener.getAudit();
}
}
class StringAuditor implements AuditListener {
private CountDownLatch finishedLatch = new CountDownLatch(1);
private StringBuilder output = new StringBuilder();
private String base = "";
public void setBase(String base) {
this.base = base;
}
public void append(String s) {
output.append(s);
}
public String getAudit() throws InterruptedException {
finishedLatch.await();
return output.toString();
}
@Override
public void auditStarted(AuditEvent evt) {
}
@Override
public void auditFinished(AuditEvent evt) {
finishedLatch.countDown();
}
@Override
public void fileStarted(AuditEvent evt) {
}
@Override
public void fileFinished(AuditEvent evt) {
}
@Override
public void addError(AuditEvent evt) {
switch (evt.getSeverityLevel()) {
case ERROR:
String fileName = evt.getFileName();
int index = fileName.indexOf(base);
if (index >= 0) {
fileName = fileName.substring(index + base.length() + 1);
}
output.append(fileName).append(':').append(evt.getLine());
if (evt.getColumn() > 0) {
output.append(':').append(evt.getColumn());
}
output.append(": ").append(evt.getMessage());
output.append('\n');
break;
case IGNORE:
case INFO:
case WARNING:
default:
break;
}
}
@Override
public void addException(AuditEvent evt, Throwable throwable) {
addError(evt);
output.append(throwable.getMessage());
}
}
\ No newline at end of file
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.checkstyle;
import com.puppycrawl.tools.checkstyle.api.CheckstyleException;
import java.io.IOException;
import java.net.ServerSocket;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static java.nio.file.StandardOpenOption.*;
/**
* Main program for executing scenario test warden.
*/
public final class Main {
private static long POLLING_INTERVAL = 1000; //ms
// Public construction forbidden
private Main(String[] args) {
}
/**
* Main entry point for the cell warden.
*
* @param args command-line arguments
*/
public static void main(String[] args)
throws CheckstyleException, IOException {
startServer(args);
}
// Monitors another PID and exit when that process exits
private static void watchProcess(String pid) {
if (pid == null || pid.equals("0")) {
return;
}
Timer timer = new Timer(true); // start as a daemon, so we don't hang shutdown
timer.scheduleAtFixedRate(new TimerTask() {
private String cmd = "kill -s 0 " + pid;
@Override
public void run() {
try {
Process p = Runtime.getRuntime().exec(cmd);
p.waitFor();
if (p.exitValue() != 0) {
System.err.println("shutting down...");
System.exit(0);
}
} catch (IOException | InterruptedException e) {
//no-op
e.printStackTrace();
}
}
}, POLLING_INTERVAL, POLLING_INTERVAL);
}
// Initiates a server.
private static void startServer(String[] args) throws IOException, CheckstyleException {
String portLock = args[0];
String buckPid = args[1];
String checkstyleFile = args[2];
String suppressionsFile = args[3];
// Use a file lock to ensure only one copy of the daemon runs
Path portLockPath = Paths.get(portLock);
FileChannel channel = FileChannel.open(portLockPath, WRITE, CREATE);
FileLock lock = channel.tryLock();
if (lock == null) {
System.out.println("Server is already running");
System.exit(1);
} //else, hold the lock until the JVM exits
// Start the server and bind it to a random port
ServerSocket server = new ServerSocket(0);
// Monitor the parent buck process
watchProcess(buckPid);
// Set up hook to clean up after ourselves
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
if (channel != null) {
channel.truncate(0);
channel.close();
}
System.err.println("tear down...");
Files.delete(portLockPath);
} catch (IOException e) {
//no-op: shutting down
e.printStackTrace();
}
}));
// Write the bound port to the port file
int port = server.getLocalPort();
channel.truncate(0);
channel.write(ByteBuffer.wrap(Integer.toString(port).getBytes()));
// Instantiate a Checkstyle runner and executor; serve until exit...
CheckstyleRunner runner = new CheckstyleRunner(checkstyleFile, suppressionsFile);
ExecutorService executor = Executors.newCachedThreadPool();
while (true) {
try {
executor.submit(runner.checkClass(server.accept()));
} catch (Exception e) {
e.printStackTrace();
//no-op
}
}
}
}
#!/bin/bash
CHECKSTYLE=$1
FILES=$2
CONFIG=$3
SUPPRESSIONS=$4
PORT_FILE="$1.port"
function ppid() {
ps -p ${1:-$$} -o ppid= -o pid= -o comm=
}
function buck_pid() {
BUCK_PID=($(ppid))
while [ ${BUCK_PID[0]} -ne 0 ]; do
BUCK_PID=($(ppid $BUCK_PID))
if [ "${BUCK_PID[2]}" == "buck" ]; then
# use parent PID of buck
echo ${BUCK_PID[0]}
return
fi
if [ "${BUCK_PID[2]}" == "buckd" ] ||
[[ "${BUCK_PID[2]}" == *"python"* ]]; then
# use PID of buckd or python
echo ${BUCK_PID[1]}
return
fi
done
# fallback last read PID
echo ${BUCK_PID[1]}
}
function port() {
cat $PORT_FILE 2>/dev/null || echo 0
}
function check_socket() {
nc localhost $(port) < /dev/null 2>/dev/null
return $?
}
# check to see if checkstyle daemon is running; if not, start it
if ! check_socket; then
# Starting checkstyle server...
#FIXME change to /dev/null if/when we are confident
nohup java -jar $CHECKSTYLE $PORT_FILE $(buck_pid) $3 $4 >>/tmp/checkstyle.daemon 2>&1 &
TRIES=20
i=0
# Wait for checkstyle server to start for 2 seconds
while [ $i -lt $TRIES ]; do
if check_socket; then
CONNECTED=true
break
fi
let i=i+1
sleep 0.1
done
if [ -z "$CONNECTED" ]; then
echo "Failed to start checkstyle server"
exit 3
fi
fi
# run the actual checkstyle client
OUT=$(cat $FILES | nc localhost $(port))
if [ $? -ne 0 ]; then
echo "Error connecting to checkstyle server"
exit 2
fi
if [ -n "$OUT" ]; then
printf "$OUT"
exit 1
fi
\ No newline at end of file