홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Carmelo Cascone
Committed by Thomas Vachuska
0831efb0 1 parent 17fc9e41

Various changes in BMv2 driver and provider modules (onos1.6 cherry-pick) 

Driver notable changes:
- Implemented new behaviors, removed deprecated ones
- Removed flow rule translator classes (now under protocol module)
- Improved FlowRuleProgrammable: now it uses BMv2TableEntryService
	to lookup/bind flow rules with BMv2 table entries, retrieves flow
	statistics, better exception handling when adding/replacing/removing
	table entries.
- Improved PacketProgrammable: better exception handling and logging

Provider notable changes:
- Bmv2DeviceProvider: detects and notifies device configuration
	changes and reboots to Bmv2DeviceContextService, added support for
	periodic polling of port statistics
- Bmv2PacketProvider: implemented workaround for OutboundPackets with
	flood treatment

Change-Id: I79b756b533d4afb6b70025a137b2e811fd42a4e8
Expand all
Inline Side-by-side
Showing 31 changed files with 421 additions and 599 deletions
......@@ -20,6 +20,6 @@
<feature name="${project.artifactId}" version="${project.version}"
description="${project.description}">
<bundle>mvn:${project.groupId}/${project.artifactId}/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-api/${project.version}</bundle>
</feature>
</features>
......
......@@ -22,11 +22,11 @@
<artifactId>onos-drivers-general</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>onos-drivers-bmv2</artifactId>
<version>1.7.0-SNAPSHOT</version>
<packaging>bundle</packaging>
......@@ -36,7 +36,7 @@
<onos.app.name>org.onosproject.drivers.bmv2</onos.app.name>
<onos.app.origin>ON.Lab</onos.app.origin>
<onos.app.category>Drivers</onos.app.category>
<onos.app.title>BMv2 Device Drivers</onos.app.title>
<onos.app.title>BMv2 Drivers</onos.app.title>
<onos.app.url>http://onosproject.org</onos.app.url>
<onos.app.requires>
org.onosproject.bmv2
......@@ -46,7 +46,7 @@
<dependencies>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-bmv2-protocol</artifactId>
<artifactId>onos-bmv2-protocol-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
......
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.onlab.packet.ChassisId;
import org.onosproject.bmv2.api.runtime.Bmv2DeviceAgent;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.net.AnnotationKeys;
import org.onosproject.net.DefaultAnnotations;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DefaultDeviceDescription;
import org.onosproject.net.device.DefaultPortDescription;
import org.onosproject.net.device.DeviceDescription;
import org.onosproject.net.device.DeviceDescriptionDiscovery;
import org.onosproject.net.device.PortDescription;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigInteger;
import java.util.List;
import static org.onosproject.bmv2.api.runtime.Bmv2Device.*;
import static org.onosproject.net.Device.Type.SWITCH;
/**
* Implementation of the device description discovery behaviour for BMv2.
*/
public class Bmv2DeviceDescriptionDiscovery extends AbstractHandlerBehaviour implements DeviceDescriptionDiscovery {
private static final String JSON_CONFIG_MD5 = "bmv2JsonConfigMd5";
private static final String PROCESS_INSTANCE_ID = "bmv2ProcessInstanceId";
private final Logger log = LoggerFactory.getLogger(this.getClass());
private Bmv2Controller controller;
private boolean init() {
controller = handler().get(Bmv2Controller.class);
if (controller == null) {
log.warn("Failed to get a BMv2 controller");
return false;
}
return true;
}
@Override
public DeviceDescription discoverDeviceDetails() {
if (!init()) {
return null;
}
DeviceId deviceId = handler().data().deviceId();
Bmv2DeviceAgent deviceAgent;
try {
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
log.error("Failed to connect to Bmv2 device", e);
return null;
}
DefaultAnnotations.Builder annotationsBuilder = DefaultAnnotations.builder();
try {
String md5 = deviceAgent.getJsonConfigMd5();
BigInteger i = new BigInteger(1, md5.getBytes());
annotationsBuilder.set(JSON_CONFIG_MD5, String.format("%1$032X", i).toLowerCase());
} catch (Bmv2RuntimeException e) {
log.warn("Unable to dump JSON configuration from {}: {}", deviceId, e.explain());
}
try {
int instanceId = deviceAgent.getProcessInstanceId();
annotationsBuilder.set(PROCESS_INSTANCE_ID, String.valueOf(instanceId));
} catch (Bmv2RuntimeException e) {
log.warn("Unable to get process instance ID from {}: {}", deviceId, e.explain());
}
annotationsBuilder.set(AnnotationKeys.PROTOCOL, PROTOCOL);
return new DefaultDeviceDescription(deviceId.uri(),
SWITCH,
MANUFACTURER,
HW_VERSION,
SW_VERSION,
SERIAL_NUMBER,
new ChassisId(),
annotationsBuilder.build());
}
@Override
public List<PortDescription> discoverPortDetails() {
if (!init()) {
return null;
}
DeviceId deviceId = handler().data().deviceId();
Bmv2DeviceAgent deviceAgent;
try {
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
log.error("Failed to connect to Bmv2 device", e);
return null;
}
List<PortDescription> portDescriptions = Lists.newArrayList();
try {
deviceAgent.getPortsInfo().forEach(p -> {
PortNumber portNumber = PortNumber.portNumber((long) p.number(), p.ifaceName());
portDescriptions.add(new DefaultPortDescription(portNumber, p.isUp(), DefaultAnnotations.EMPTY));
});
} catch (Bmv2RuntimeException e) {
log.error("Unable to get port descriptions of {}: {}", deviceId, e);
}
return ImmutableList.copyOf(portDescriptions);
}
}
......@@ -20,7 +20,7 @@ import org.apache.felix.scr.annotations.Component;
import org.onosproject.net.driver.AbstractDriverLoader;
/**
* Loader for barefoot drivers from specific xml.
* Loader for BMv2 drivers from xml file.
*/
@Component(immediate = true)
public class Bmv2DriversLoader extends AbstractDriverLoader {
......
......@@ -14,7 +14,29 @@
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import org.onosproject.bmv2.api.runtime.Bmv2ExtensionSelector;
import org.onosproject.net.behaviour.ExtensionSelectorResolver;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.flow.criteria.ExtensionSelector;
import org.onosproject.net.flow.criteria.ExtensionSelectorType;
import java.util.Collections;
import static org.onosproject.net.flow.criteria.ExtensionSelectorType.ExtensionSelectorTypes.BMV2_MATCH_PARAMS;
/**
* Translators of ONOS abstractions to BMv2 model-dependent abstractions.
* Implementation of the extension selector resolver behaviour for BMv2.
*/
package org.onosproject.drivers.bmv2.translators;
\ No newline at end of file
public class Bmv2ExtensionSelectorResolver extends AbstractHandlerBehaviour implements ExtensionSelectorResolver {
@Override
public ExtensionSelector getExtensionSelector(ExtensionSelectorType type) {
if (type.equals(BMV2_MATCH_PARAMS.type())) {
return new Bmv2ExtensionSelector(Collections.emptyMap());
}
return null;
}
}
......
......@@ -14,14 +14,26 @@
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
package org.onosproject.drivers.bmv2;
import org.onosproject.bmv2.api.runtime.Bmv2ExtensionTreatment;
import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.BMV2_ACTION;
/**
* BMv2 flow rule translator exception.
* Implementation of the extension treatment resolver behavior for BMv2.
*/
public class Bmv2FlowRuleTranslatorException extends Exception {
public class Bmv2ExtensionTreatmentResolver extends AbstractHandlerBehaviour implements ExtensionTreatmentResolver {
Bmv2FlowRuleTranslatorException(String msg) {
super(msg);
@Override
public ExtensionTreatment getExtensionInstruction(ExtensionTreatmentType type) {
if (type.equals(BMV2_ACTION.type())) {
return new Bmv2ExtensionTreatment(null);
}
return null;
}
}
......
......@@ -17,56 +17,59 @@
package org.onosproject.drivers.bmv2;
import org.onlab.util.ImmutableByteSequence;
import org.onosproject.bmv2.api.runtime.Bmv2Client;
import org.onosproject.bmv2.api.runtime.Bmv2DeviceAgent;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.ctl.Bmv2ThriftClient;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.net.DeviceId;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.instructions.Instructions;
import org.onosproject.net.packet.OutboundPacket;
import org.onosproject.net.packet.PacketProgrammable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import static java.lang.Math.toIntExact;
import static org.onosproject.net.PortNumber.FLOOD;
import static java.util.stream.Collectors.toList;
import static org.onosproject.net.flow.instructions.Instruction.Type.OUTPUT;
import static org.onosproject.net.flow.instructions.Instructions.OutputInstruction;
/**
* Packet programmable device behaviour implementation for BMv2.
* Implementation of the packet programmable behaviour for BMv2.
*/
public class Bmv2PacketProgrammable extends AbstractHandlerBehaviour implements PacketProgrammable {
private static final Logger LOG =
LoggerFactory.getLogger(Bmv2PacketProgrammable.class);
private final Logger log = LoggerFactory.getLogger(this.getClass());
@Override
public void emit(OutboundPacket packet) {
TrafficTreatment treatment = packet.treatment();
treatment.allInstructions().forEach(inst -> {
if (inst.type().equals(OUTPUT)) {
Instructions.OutputInstruction outInst = (Instructions.OutputInstruction) inst;
if (outInst.port().isLogical()) {
if (outInst.port() == FLOOD) {
// TODO: implement flood
LOG.info("Flood not implemented", outInst);
}
LOG.info("Output on logical port not supported: {}", outInst);
} else {
try {
long longPort = outInst.port().toLong();
int portNumber = toIntExact(longPort);
send(portNumber, packet);
} catch (ArithmeticException e) {
LOG.error("Port number overflow! Cannot send packet on port {} (long), as the bmv2" +
" device only accepts int port values.");
}
}
// BMv2 supports only OUTPUT instructions.
List<OutputInstruction> outInstructions = treatment.allInstructions()
.stream()
.filter(i -> i.type().equals(OUTPUT))
.map(i -> (OutputInstruction) i)
.collect(toList());
if (treatment.allInstructions().size() != outInstructions.size()) {
// There are other instructions that are not of type OUTPUT
log.warn("Dropping emit request, treatment nor supported: {}", treatment);
return;
}
outInstructions.forEach(outInst -> {
if (outInst.port().isLogical()) {
log.warn("Dropping emit request, logical port not supported: {}", outInst.port());
} else {
LOG.info("Instruction type not supported: {}", inst.type().name());
try {
int portNumber = toIntExact(outInst.port().toLong());
send(portNumber, packet);
} catch (ArithmeticException e) {
log.error("Dropping emit request, port number too big: {}", outInst.port().toLong());
}
}
});
}
......@@ -75,19 +78,25 @@ public class Bmv2PacketProgrammable extends AbstractHandlerBehaviour implements
DeviceId deviceId = handler().data().deviceId();
Bmv2Client deviceClient;
Bmv2Controller controller = handler().get(Bmv2Controller.class);
if (controller == null) {
log.error("Failed to get BMv2 controller");
return;
}
Bmv2DeviceAgent deviceAgent;
try {
deviceClient = Bmv2ThriftClient.of(deviceId);
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
LOG.error("Failed to connect to Bmv2 device", e);
log.error("Failed to get Bmv2 device agent for {}: {}", deviceId, e.explain());
return;
}
ImmutableByteSequence bs = ImmutableByteSequence.copyFrom(packet.data());
try {
deviceClient.transmitPacket(port, bs);
deviceAgent.transmitPacket(port, bs);
} catch (Bmv2RuntimeException e) {
LOG.info("Unable to push packet to device: deviceId={}, packet={}", deviceId, bs);
log.warn("Unable to emit packet trough {}: {}", deviceId, e.explain());
}
}
}
......
......@@ -37,8 +37,8 @@ public class Bmv2Pipeliner extends AbstractHandlerBehaviour implements Pipeliner
@Override
public void init(DeviceId deviceId, PipelinerContext context) {
// TODO: get multi-table pipeliner dynamically based on BMv2 device running model
// Right now we only support single table pipelines
// TODO: get multi-table pipeliner dynamically based on BMv2 device running model (hard).
// Right now we are able to map flow objectives only in the first table of the pipeline.
pipeliner = new DefaultSingleTablePipeline();
pipeliner.init(deviceId, context);
}
......
/*
* Copyright 2014-2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.onosproject.bmv2.api.runtime.Bmv2Client;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.ctl.Bmv2ThriftClient;
import org.onosproject.net.DefaultAnnotations;
import org.onosproject.net.PortNumber;
import org.onosproject.net.SparseAnnotations;
import org.onosproject.net.behaviour.PortDiscovery;
import org.onosproject.net.device.DefaultPortDescription;
import org.onosproject.net.device.PortDescription;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
public class Bmv2PortDiscovery extends AbstractHandlerBehaviour
implements PortDiscovery {
private final Logger log =
LoggerFactory.getLogger(this.getClass());
@Override
public List<PortDescription> getPorts() {
Bmv2Client deviceClient;
try {
deviceClient = Bmv2ThriftClient.of(handler().data().deviceId());
} catch (Bmv2RuntimeException e) {
log.error("Failed to connect to Bmv2 device", e);
return Collections.emptyList();
}
List<PortDescription> portDescriptions = Lists.newArrayList();
try {
deviceClient.getPortsInfo().forEach(
p -> {
DefaultAnnotations.Builder builder =
DefaultAnnotations.builder();
p.getExtraProperties().forEach(builder::set);
SparseAnnotations annotations = builder.build();
portDescriptions.add(new DefaultPortDescription(
PortNumber.portNumber(
(long) p.portNumber(),
p.ifaceName()),
p.isUp(),
annotations
));
});
} catch (Bmv2RuntimeException e) {
log.error("Unable to get port description from Bmv2 device", e);
}
return ImmutableList.copyOf(portDescriptions);
}
}
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
import com.google.common.annotations.Beta;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.net.flow.criteria.Criterion;
import java.util.Map;
import static org.onosproject.drivers.bmv2.translators.Bmv2FlowRuleTranslator.TranslatorConfig;
/**
* Default implementation of a BMv2 flow rule translator configuration.
*/
@Beta
public abstract class Bmv2DefaultTranslatorConfig implements TranslatorConfig {
private final Bmv2Model model;
private final Map<String, Criterion.Type> fieldMap;
/**
* Creates a new translator configuration.
*
* @param model a BMv2 packet processing model
* @param fieldMap a field-to-criterion type map
*/
protected Bmv2DefaultTranslatorConfig(Bmv2Model model, Map<String, Criterion.Type> fieldMap) {
this.model = model;
this.fieldMap = fieldMap;
}
@Override
public Bmv2Model model() {
return this.model;
}
@Override
public Map<String, Criterion.Type> fieldToCriterionTypeMap() {
return this.fieldMap;
}
}
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
import com.google.common.annotations.Beta;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.bmv2.api.runtime.Bmv2Action;
import org.onosproject.bmv2.api.runtime.Bmv2TableEntry;
import org.onosproject.net.flow.FlowRule;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.criteria.Criterion;
import java.util.Map;
/**
* Translator of ONOS flow rules to BMv2 table entries. Translation depends on a
* {@link TranslatorConfig translator configuration}.
*/
@Beta
public interface Bmv2FlowRuleTranslator {
/**
* Returns a new BMv2 table entry equivalent to the given flow rule.
*
* @param rule a flow rule
* @return a BMv2 table entry
* @throws Bmv2FlowRuleTranslatorException if the flow rule cannot be
* translated
*/
Bmv2TableEntry translate(FlowRule rule) throws Bmv2FlowRuleTranslatorException;
/**
* Returns the configuration of this translator.
*
* @return a translator configuration
*/
TranslatorConfig config();
/**
* BMv2 flow rules translator configuration. Such a configuration is used to
* generate table entries that are compatible with a given {@link Bmv2Model}.
*/
@Beta
interface TranslatorConfig {
/**
* Return the {@link Bmv2Model} associated with this configuration.
*
* @return a BMv2 model
*/
Bmv2Model model();
/**
* Returns a map describing a one-to-one relationship between BMv2
* header field names and ONOS criterion types. Header field names are
* formatted using the notation {@code header_name.field_name}
* representing a specific header field instance extracted by the BMv2
* parser (e.g. {@code ethernet.dstAddr}).
*
* @return a map where the keys represent BMv2 header field names and
* values are criterion types
*/
Map<String, Criterion.Type> fieldToCriterionTypeMap();
/**
* Return a BMv2 action that is equivalent to the given ONOS traffic
* treatment.
*
* @param treatment a traffic treatment
* @return a BMv2 action object
* @throws Bmv2FlowRuleTranslatorException if the treatment cannot be
* translated to a BMv2 action
*/
Bmv2Action buildAction(TrafficTreatment treatment)
throws Bmv2FlowRuleTranslatorException;
}
}
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
import com.google.common.annotations.Beta;
import com.google.common.collect.ImmutableMap;
import org.onlab.util.ImmutableByteSequence;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.bmv2.api.runtime.Bmv2Action;
import org.onosproject.net.PortNumber;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.criteria.Criterion;
import org.onosproject.net.flow.instructions.Instruction;
import org.onosproject.net.flow.instructions.Instructions;
import java.util.Map;
/**
* Implementation of a Bmv2 flow rule translator configuration for the
* simple.p4 model.
*/
@Beta
public class Bmv2SimpleTranslatorConfig extends Bmv2DefaultTranslatorConfig {
// Lazily populate field map.
private static final Map<String, Criterion.Type> FIELD_MAP = ImmutableMap.of(
"standard_metadata.ingress_port", Criterion.Type.IN_PORT,
"ethernet.dstAddr", Criterion.Type.ETH_DST,
"ethernet.srcAddr", Criterion.Type.ETH_SRC,
"ethernet.etherType", Criterion.Type.ETH_TYPE);
/**
* Creates a new simple pipeline translator configuration.
*/
public Bmv2SimpleTranslatorConfig(Bmv2Model model) {
// Populate fieldMap.
super(model, FIELD_MAP);
}
private static Bmv2Action buildDropAction() {
return Bmv2Action.builder()
.withName("_drop")
.build();
}
private static Bmv2Action buildPushToCpAction() {
return Bmv2Action.builder()
.withName("send_to_cpu")
.build();
}
private static Bmv2Action buildFwdAction(Instructions.OutputInstruction inst)
throws Bmv2FlowRuleTranslatorException {
Bmv2Action.Builder actionBuilder = Bmv2Action.builder();
actionBuilder.withName("fwd");
if (inst.port().isLogical()) {
if (inst.port() == PortNumber.CONTROLLER) {
return buildPushToCpAction();
} else {
throw new Bmv2FlowRuleTranslatorException(
"Output logic port number not supported: " + inst);
}
}
actionBuilder.addParameter(
ImmutableByteSequence.copyFrom((short) inst.port().toLong()));
return actionBuilder.build();
}
@Override
public Bmv2Action buildAction(TrafficTreatment treatment)
throws Bmv2FlowRuleTranslatorException {
if (treatment.allInstructions().size() == 0) {
// No instructions means drop.
return buildDropAction();
} else if (treatment.allInstructions().size() > 1) {
// Otherwise, we understand treatments with only 1 instruction.
throw new Bmv2FlowRuleTranslatorException(
"Treatment not supported, more than 1 instructions found: "
+ treatment.toString());
}
Instruction instruction = treatment.allInstructions().get(0);
switch (instruction.type()) {
case OUTPUT:
return buildFwdAction((Instructions.OutputInstruction) instruction);
case NOACTION:
return buildDropAction();
default:
throw new Bmv2FlowRuleTranslatorException(
"Instruction type not supported: "
+ instruction.type().name());
}
}
}
......@@ -16,14 +16,18 @@
~ limitations under the License.
-->
<drivers>
<driver name="bmv2-thrift" manufacturer="p4.org" hwVersion="bmv2" swVersion="unknown">
<behaviour api="org.onosproject.net.behaviour.PortDiscovery"
impl="org.onosproject.drivers.bmv2.Bmv2PortDiscovery"/>
<driver name="bmv2-thrift" manufacturer="p4.org" hwVersion="bmv2" swVersion="n/a">
<behaviour api="org.onosproject.net.device.DeviceDescriptionDiscovery"
impl="org.onosproject.drivers.bmv2.Bmv2DeviceDescriptionDiscovery"/>
<behaviour api="org.onosproject.net.flow.FlowRuleProgrammable"
impl="org.onosproject.drivers.bmv2.Bmv2FlowRuleProgrammable"/>
<behaviour api="org.onosproject.net.behaviour.Pipeliner"
impl="org.onosproject.drivers.bmv2.Bmv2Pipeliner"/>
<behaviour api="org.onosproject.net.packet.PacketProgrammable"
impl="org.onosproject.drivers.bmv2.Bmv2PacketProgrammable"/>
<behaviour api="org.onosproject.net.behaviour.ExtensionSelectorResolver"
impl="org.onosproject.drivers.bmv2.Bmv2ExtensionSelectorResolver"/>
<behaviour api="org.onosproject.net.behaviour.ExtensionTreatmentResolver"
impl="org.onosproject.drivers.bmv2.Bmv2ExtensionTreatmentResolver"/>
</driver>
</drivers>
......
/*
* Copyright 2014-2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import com.eclipsesource.json.Json;
import com.eclipsesource.json.JsonObject;
import com.google.common.testing.EqualsTester;
import org.junit.Before;
import org.junit.Test;
import org.onlab.packet.MacAddress;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.bmv2.api.runtime.Bmv2TableEntry;
import org.onosproject.bmv2.api.runtime.Bmv2TernaryMatchParam;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.DefaultApplicationId;
import org.onosproject.drivers.bmv2.translators.Bmv2DefaultFlowRuleTranslator;
import org.onosproject.drivers.bmv2.translators.Bmv2FlowRuleTranslator;
import org.onosproject.drivers.bmv2.translators.Bmv2SimpleTranslatorConfig;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.flow.DefaultFlowRule;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.FlowRule;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Random;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
/**
* Tests for {@link Bmv2DefaultFlowRuleTranslator}.
*/
public class Bmv2DefaultFlowRuleTranslatorTest {
private static final String JSON_CONFIG_PATH = "/simple.json";
private Random random = new Random();
private Bmv2Model model;
private Bmv2FlowRuleTranslator.TranslatorConfig config;
private Bmv2FlowRuleTranslator translator;
@Before
public void setUp() throws Exception {
InputStream inputStream = Bmv2SimpleTranslatorConfig.class
.getResourceAsStream(JSON_CONFIG_PATH);
InputStreamReader reader = new InputStreamReader(inputStream);
BufferedReader bufReader = new BufferedReader(reader);
JsonObject json = null;
try {
json = Json.parse(bufReader).asObject();
} catch (IOException e) {
throw new RuntimeException("Unable to parse JSON file: " + e.getMessage());
}
this.model = Bmv2Model.parse(json);
this.config = new Bmv2SimpleTranslatorConfig(model);
this.translator = new Bmv2DefaultFlowRuleTranslator(config);
}
@Test
public void testCompiler() throws Exception {
DeviceId deviceId = DeviceId.NONE;
ApplicationId appId = new DefaultApplicationId(1, "test");
int tableId = 0;
MacAddress ethDstMac = MacAddress.valueOf(random.nextLong());
MacAddress ethSrcMac = MacAddress.valueOf(random.nextLong());
short ethType = (short) (0x0000FFFF & random.nextInt());
short outPort = (short) random.nextInt(65);
short inPort = (short) random.nextInt(65);
int timeout = random.nextInt(100);
int priority = random.nextInt(100);
TrafficSelector matchInPort1 = DefaultTrafficSelector
.builder()
.matchInPort(PortNumber.portNumber(inPort))
.matchEthDst(ethDstMac)
.matchEthSrc(ethSrcMac)
.matchEthType(ethType)
.build();
TrafficTreatment outPort2 = DefaultTrafficTreatment
.builder()
.setOutput(PortNumber.portNumber(outPort))
.build();
FlowRule rule1 = DefaultFlowRule.builder()
.forDevice(deviceId)
.forTable(tableId)
.fromApp(appId)
.withSelector(matchInPort1)
.withTreatment(outPort2)
.makeTemporary(timeout)
.withPriority(priority)
.build();
FlowRule rule2 = DefaultFlowRule.builder()
.forDevice(deviceId)
.forTable(tableId)
.fromApp(appId)
.withSelector(matchInPort1)
.withTreatment(outPort2)
.makeTemporary(timeout)
.withPriority(priority)
.build();
Bmv2TableEntry entry1 = translator.translate(rule1);
Bmv2TableEntry entry2 = translator.translate(rule1);
// check equality, i.e. same rules must produce same entries
new EqualsTester()
.addEqualityGroup(rule1, rule2)
.addEqualityGroup(entry1, entry2)
.testEquals();
int numMatchParams = model.table(0).keys().size();
// parse values stored in entry1
Bmv2TernaryMatchParam inPortParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(0);
Bmv2TernaryMatchParam ethDstParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(1);
Bmv2TernaryMatchParam ethSrcParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(2);
Bmv2TernaryMatchParam ethTypeParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(3);
double expectedTimeout = (double) (model.table(0).hasTimeouts() ? rule1.timeout() : -1);
// check that the number of parameters in the entry is the same as the number of table keys
assertThat("Incorrect number of match parameters",
entry1.matchKey().matchParams().size(), is(equalTo(numMatchParams)));
// check that values stored in entry are the same used for the flow rule
assertThat("Incorrect inPort match param value",
inPortParam.value().asReadOnlyBuffer().getShort(), is(equalTo(inPort)));
assertThat("Incorrect ethDestMac match param value",
ethDstParam.value().asArray(), is(equalTo(ethDstMac.toBytes())));
assertThat("Incorrect ethSrcMac match param value",
ethSrcParam.value().asArray(), is(equalTo(ethSrcMac.toBytes())));
assertThat("Incorrect ethType match param value",
ethTypeParam.value().asReadOnlyBuffer().getShort(), is(equalTo(ethType)));
assertThat("Incorrect priority value",
entry1.priority(), is(equalTo(Integer.MAX_VALUE - rule1.priority())));
assertThat("Incorrect timeout value",
entry1.timeout(), is(equalTo(expectedTimeout)));
}
}
\ No newline at end of file
This diff is collapsed. Click to expand it.
......@@ -41,7 +41,7 @@
<module>utilities</module>
<module>lumentum</module>
<module>bti</module>
<!--<module>bmv2</module>-->
<module>bmv2</module>
<module>corsa</module>
<module>optical</module>
</modules>
......
......@@ -21,7 +21,8 @@
<parent>
<artifactId>onos-bmv2-protocol</artifactId>
<groupId>org.onosproject</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -21,7 +21,8 @@
<parent>
<artifactId>onos-bmv2-protocol</artifactId>
<groupId>org.onosproject</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -22,6 +22,7 @@
<artifactId>onos-protocols</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -21,7 +21,8 @@
<parent>
<artifactId>onos-bmv2-protocol</artifactId>
<groupId>org.onosproject</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -22,7 +22,9 @@
<bundle>mvn:${project.groupId}/onos-bmv2-provider-device/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-provider-packet/${project.version}</bundle>
<bundle>mvn:org.apache.thrift/libthrift/0.9.3</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-api/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-ctl/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-thrift-api/${project.version}</bundle>
</feature>
</features>
......
......@@ -22,6 +22,7 @@
<artifactId>onos-bmv2-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......@@ -33,7 +34,7 @@
<properties>
<onos.app.name>org.onosproject.bmv2</onos.app.name>
<onos.app.title>BMv2 Provider</onos.app.title>
<onos.app.title>BMv2 Providers</onos.app.title>
<onos.app.category>Provider</onos.app.category>
</properties>
......
......@@ -22,6 +22,7 @@
<artifactId>onos-bmv2-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......@@ -33,12 +34,12 @@
<dependencies>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-drivers-bmv2</artifactId>
<artifactId>onos-core-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-core-common</artifactId>
<artifactId>onos-bmv2-protocol-api</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
......
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.provider.bmv2.device.impl;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.tuple.Pair;
import org.onosproject.bmv2.api.runtime.Bmv2Action;
import org.onosproject.bmv2.api.runtime.Bmv2DeviceAgent;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.net.Port;
import org.onosproject.net.device.DefaultPortStatistics;
import org.onosproject.net.device.PortStatistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.List;
/**
* Utility class to read port statistics from a BMv2 device.
*/
final class Bmv2PortStatisticsGetter {
// TODO: make counters configuration dependent
private static final String TABLE_NAME = "port_count_table";
private static final String ACTION_NAME = "count_packet";
private static final String EGRESS_COUNTER = "egress_port_counter";
private static final String INGRESS_COUNTER = "ingress_port_counter";
private static final Logger log = LoggerFactory.getLogger(Bmv2PortStatisticsGetter.class);
private Bmv2PortStatisticsGetter() {
// ban constructor.
}
/**
* Returns a collection of port statistics for given ports using the given BMv2 device agent.
*
* @param deviceAgent a device agent
* @param ports a collection of ports
* @return a collection of port statistics
*/
static Collection<PortStatistics> getPortStatistics(Bmv2DeviceAgent deviceAgent, Collection<Port> ports) {
List<PortStatistics> ps = Lists.newArrayList();
for (Port port : ports) {
int portNumber = (int) port.number().toLong();
try {
Pair<Long, Long> egressCounter = deviceAgent.readCounter(EGRESS_COUNTER, portNumber);
Pair<Long, Long> ingressCounter = deviceAgent.readCounter(INGRESS_COUNTER, portNumber);
ps.add(DefaultPortStatistics.builder()
.setPort(portNumber)
.setBytesSent(egressCounter.getLeft())
.setPacketsSent(egressCounter.getRight())
.setBytesReceived(ingressCounter.getLeft())
.setPacketsReceived(ingressCounter.getRight())
.build());
} catch (Bmv2RuntimeException e) {
log.info("Unable to read port statistics from {}: {}", port, e.explain());
}
}
return ps;
}
/**
* Initialize port counters on the given device agent.
*
* @param deviceAgent a device agent.
*/
static void initCounters(Bmv2DeviceAgent deviceAgent) {
try {
deviceAgent.setTableDefaultAction(TABLE_NAME, Bmv2Action.builder().withName(ACTION_NAME).build());
} catch (Bmv2RuntimeException e) {
log.debug("Failed to provision counters on {}: {}", deviceAgent.deviceId(), e.explain());
}
}
}
......@@ -22,6 +22,7 @@
<artifactId>onos-bmv2-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......@@ -34,12 +35,12 @@
<dependencies>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-drivers-bmv2</artifactId>
<artifactId>onos-core-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-core-common</artifactId>
<artifactId>onos-bmv2-protocol-api</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
......
......@@ -23,12 +23,14 @@ import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.onlab.packet.Ethernet;
import org.onlab.util.ImmutableByteSequence;
import org.onosproject.bmv2.api.runtime.Bmv2ControlPlaneServer;
import org.onosproject.bmv2.api.runtime.Bmv2Device;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.bmv2.api.service.Bmv2PacketListener;
import org.onosproject.core.CoreService;
import org.onosproject.net.ConnectPoint;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Port;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.flow.DefaultTrafficTreatment;
......@@ -49,6 +51,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.Optional;
import static org.onosproject.net.PortNumber.FLOOD;
import static org.onosproject.net.flow.DefaultTrafficTreatment.emptyTreatment;
import static org.onosproject.net.flow.instructions.Instruction.Type.OUTPUT;
import static org.onosproject.net.flow.instructions.Instructions.OutputInstruction;
/**
* Implementation of a packet provider for BMv2.
......@@ -56,11 +64,11 @@ import java.nio.ByteBuffer;
@Component(immediate = true)
public class Bmv2PacketProvider extends AbstractProvider implements PacketProvider {
private static final Logger LOG = LoggerFactory.getLogger(Bmv2PacketProvider.class);
private final Logger log = LoggerFactory.getLogger(Bmv2PacketProvider.class);
private static final String APP_NAME = "org.onosproject.bmv2";
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected Bmv2ControlPlaneServer controlPlaneServer;
protected Bmv2Controller controller;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected CoreService coreService;
......@@ -86,28 +94,28 @@ public class Bmv2PacketProvider extends AbstractProvider implements PacketProvid
protected void activate() {
providerService = providerRegistry.register(this);
coreService.registerApplication(APP_NAME);
controlPlaneServer.addPacketListener(packetListener);
LOG.info("Started");
controller.addPacketListener(packetListener);
log.info("Started");
}
@Deactivate
public void deactivate() {
controlPlaneServer.removePacketListener(packetListener);
controller.removePacketListener(packetListener);
providerRegistry.unregister(this);
providerService = null;
LOG.info("Stopped");
log.info("Stopped");
}
@Override
public void emit(OutboundPacket packet) {
if (packet != null) {
DeviceId did = packet.sendThrough();
Device device = deviceService.getDevice(did);
DeviceId deviceId = packet.sendThrough();
Device device = deviceService.getDevice(deviceId);
if (device.is(PacketProgrammable.class)) {
PacketProgrammable packetProgrammable = device.as(PacketProgrammable.class);
packetProgrammable.emit(packet);
} else {
LOG.info("Unable to send packet, no PacketProgrammable behavior for device {}", did);
log.info("No PacketProgrammable behavior for device {}", deviceId);
}
}
}
......@@ -117,47 +125,75 @@ public class Bmv2PacketProvider extends AbstractProvider implements PacketProvid
*/
private class Bmv2PacketContext extends DefaultPacketContext {
public Bmv2PacketContext(long time, InboundPacket inPkt, OutboundPacket outPkt, boolean block) {
Bmv2PacketContext(long time, InboundPacket inPkt, OutboundPacket outPkt, boolean block) {
super(time, inPkt, outPkt, block);
}
@Override
public void send() {
if (!this.block()) {
if (this.outPacket().treatment() == null) {
TrafficTreatment treatment = (this.treatmentBuilder() == null)
? DefaultTrafficTreatment.emptyTreatment()
: this.treatmentBuilder().build();
OutboundPacket newPkt = new DefaultOutboundPacket(this.outPacket().sendThrough(),
treatment,
this.outPacket().data());
emit(newPkt);
} else {
emit(outPacket());
}
if (this.block()) {
log.info("Unable to send, packet context not blocked");
return;
}
DeviceId deviceId = outPacket().sendThrough();
ByteBuffer rawData = outPacket().data();
TrafficTreatment treatment;
if (outPacket().treatment() == null) {
treatment = (treatmentBuilder() == null) ? emptyTreatment() : treatmentBuilder().build();
} else {
treatment = outPacket().treatment();
}
// BMv2 doesn't support FLOOD for packet-outs.
// Workaround here is to perform multiple emits, one for each device port != packet inPort.
Optional<OutputInstruction> floodInst = treatment.allInstructions()
.stream()
.filter(i -> i.type().equals(OUTPUT))
.map(i -> (OutputInstruction) i)
.filter(i -> i.port().equals(FLOOD))
.findAny();
if (floodInst.isPresent() && treatment.allInstructions().size() == 1) {
// Only one instruction and is FLOOD. Do the trick.
PortNumber inPort = inPacket().receivedFrom().port();
deviceService.getPorts(outPacket().sendThrough())
.stream()
.map(Port::number)
.filter(port -> !port.equals(inPort))
.map(outPort -> DefaultTrafficTreatment.builder().setOutput(outPort).build())
.map(outTreatment -> new DefaultOutboundPacket(deviceId, outTreatment, rawData))
.forEach(Bmv2PacketProvider.this::emit);
} else {
LOG.info("Unable to send, packet context not blocked");
// Not FLOOD treatment, what to do is up to driver.
emit(new DefaultOutboundPacket(deviceId, treatment, rawData));
}
}
}
/**
* Internal packet listener to get packet events from the Bmv2ControlPlaneServer.
* Internal packet listener to handle packet-in events received from the BMv2 controller.
*/
private class InternalPacketListener implements Bmv2ControlPlaneServer.PacketListener {
private class InternalPacketListener implements Bmv2PacketListener {
@Override
public void handlePacketIn(Bmv2Device device, int inputPort, long reason, int tableId, int contextId,
ImmutableByteSequence packet) {
Ethernet ethPkt = new Ethernet();
ethPkt.deserialize(packet.asArray(), 0, packet.size());
Ethernet eth = new Ethernet();
eth.deserialize(packet.asArray(), 0, packet.size());
DeviceId deviceId = device.asDeviceId();
ConnectPoint receivedFrom = new ConnectPoint(deviceId, PortNumber.portNumber(inputPort));
ByteBuffer rawData = ByteBuffer.wrap(packet.asArray());
InboundPacket inPkt = new DefaultInboundPacket(receivedFrom, ethPkt, rawData);
OutboundPacket outPkt = new DefaultOutboundPacket(deviceId, null, rawData);
InboundPacket inPkt = new DefaultInboundPacket(new ConnectPoint(device.asDeviceId(),
PortNumber.portNumber(inputPort)),
eth, ByteBuffer.wrap(packet.asArray()));
OutboundPacket outPkt = new DefaultOutboundPacket(device.asDeviceId(), null,
ByteBuffer.wrap(packet.asArray()));
PacketContext pktCtx = new Bmv2PacketContext(System.currentTimeMillis(), inPkt, outPkt, false);
providerService.processPacket(pktCtx);
}
}
......
......@@ -22,6 +22,7 @@
<artifactId>onos-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -46,7 +46,7 @@
<module>lldpcommon</module>
<module>lldp</module>
<module>netcfglinks</module>
<!--<module>bmv2</module>-->
<module>bmv2</module>
<module>isis</module>
</modules>
......