1. XML parser of test suite is in BigTopTestSuiteXML.groovy:
public static List<BigTopIntegrationTestInterface> readTestSuiteFromXMLv2(String fileName) {
List<BigTopIntegrationTestInterface> testCaseList = new ArrayList<BigTopIntegrationTestInterface>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
try {
Document document = dbf.newDocumentBuilder().parse(new File(fileName));
Node previousNode = null;
BigTopIntegrationTestInterface currentTestCase = null;
NodeList nodeList = document.getElementsByTagName("command");
for (int i = 0; i < nodeList.getLength(); i++) {
Node currentnode = nodeList.item(i);
if ( ! (currentnode.getNodeType() == Node.ELEMENT_NODE) ) {
break;
}
if (currentnode.getParentNode()==null ||
currentnode.getParentNode().getParentNode()==null ||
currentnode.getParentNode().getParentNode().getParentNode()==null) {
break;
}
Node grandparentNode = currentnode.getParentNode().getParentNode();
if (grandparentNode.getParentNode()!=previousNode) {
if (currentTestCase!=null) testCaseList.add(currentTestCase);
previousNode = grandparentNode.getParentNode();
currentTestCase = BigTopIntegrationTestFactory.getInstance().createTestCase();
}
if (currentTestCase!=null && grandparentNode!=null) {
BigTopTestCommandInterface command = BigTopIntegrationTestFacade.getInstance().setTestCaseSuiteDetail (currentTestCase,
grandparentNode.getNodeName()+":"+currentnode.getNodeName(), currentnode.getTextContent());
if ( currentnode.getNextSibling() !=null && command!=null) {
Node sib = currentnode.getNextSibling();
while ( sib!=null) {
if ( ( sib.getNodeType() == Node.ELEMENT_NODE ) ) {
BigTopIntegrationTestFacade.getInstance().setTestCommandDetail(command, sib.getNodeName(), sib.getTextContent());
}
sib = sib.getNextSibling();
}
}
}
NodeList childNodeList = grandparentNode.getParentNode().getChildNodes();
for (int j = 0; j < childNodeList.getLength(); j++) {
Node node2 = childNodeList.item(j);
if ( ( node2.getNodeType() == Node.ELEMENT_NODE ) ) {
BigTopIntegrationTestFacade.getInstance().setTestCaseSuiteDetail(currentTestCase, node2.getNodeName(), node2.getTextContent());
}
}
}
if (currentTestCase!=null) testCaseList.add(currentTestCase);
} catch(ParserConfigurationException pce) {
pce.printStackTrace();
} catch(SAXException se) {
se.printStackTrace();
} catch(IOException ioe) {
ioe.printStackTrace();
}
return testCaseList;
}
2. BigTopIntegrationTestFacade.java is to set value for TestCase and TestCommand.
static private BigTopIntegrationTestFacade self=null;
static private Map<String, ComparatorBase> mapCompareClassMap = null;
final static private Map<String, SetTestCommandDetail> commandNameToValueMapping = new HashMap<String, SetTestCommandDetail>() {
private static final long serialVersionUID = -1L;
{
put("command-comparator-type", new SetTestComparatorClass() );
put("command-comparator-compare-to", new SetTestCommandComparator() );
}
};
private static class SetTestComparatorClass implements SetTestCommandDetail {
public void setTestCommandDetail(BigTopTestCommandInterface command, String value) {
command.setComparatorClass(value);
}
}
private static class SetTestCommandComparator implements SetTestCommandDetail {
public void setTestCommandDetail(BigTopTestCommandInterface command, String value) {
command.setCommandComparator(value);
}
}
public static BigTopIntegrationTestFacade getInstance() {
if (self == null) {
synchronized (BigTopIntegrationTestFacade.class) {
if (self == null)
self = new BigTopIntegrationTestFacade();
mapCompareClassMap = new HashMap<String, ComparatorBase>();
}
}
return self;
}
public void setTestCommandDetail(BigTopTestCommandInterface command, String name, String value) {
if (command==null) return;
SetTestCommandDetail setHandler = commandNameToValueMapping.get(name);
if (setHandler==null) return;
setHandler.setTestCommandDetail(command, value);
}
private BigTopIntegrationTestFacade () {}
public static BigTopIntegrationTestFacade getInstance() {
if (self == null) {
synchronized (BigTopIntegrationTestFacade.class) {
if (self == null)
self = new BigTopIntegrationTestFacade();
mapCompareClassMap = new HashMap<String, ComparatorBase>();
}
}
return self;
}
3. RunHadoopTestFromXMLFile.groovy is to run test commands in the ./bigtop-testcases.xml.
static Shell sh = new Shell("/bin/bash -s");
public static void runTestInXMLFile(String fielName) {
List<BigTopIntegrationTestInterface> testCaseList = BigTopTestSuiteXML.readTestSuiteFromXML(fielName);
System.out.println("Run test suite in XML file [" + fielName + "]");
for (BigTopIntegrationTestInterface t: testCaseList) {
//System.out.println( t );
println("Run test case name [" + t.getTestName() + "]");
println("Run test case description [" + t.getTestName() + "]");
for ( BigTopTestCommandInterface command: t.getCommandList() ) {
println "Command line [" + command.getCommand() + "]"
sh.exec(command.getCommand());
String stdout = sh.getOut();
if ( command.getComparatorClass() !=null && command.getCommandComparator()!=null ) {
System.out.println( "ComparatorClass - " + command.getComparatorClass() );
System.out.println( "CommandComparator - " + command.getCommandComparator() );
println "CommandComparator line [" + command.getCommandComparator() + "]"
sh.exec(command.getCommandComparator());
String stdout2 = sh.getOut();
System.out.println( "CommandComparator return code is " + sh.getRet() + " Output is " + stdout2 );
ComparatorBase compare = BigTopIntegrationTestFacade.getInstance().getComparatorClass(command.getComparatorClass());
if (compare==null) {
System.out.println( "No such ComparatorClass - " + command.getComparatorClass() );
} else {
if (stdout.length()>=2 && stdout2.length()>=2 ) {
boolean ret = compare.compare( stripOutLeadingBracket (stdout) , stripOutLeadingBracket(stdout2) );
if (ret) {
System.out.println( " SUCCESS! \n actual output - " + stdout + " \n expected -" + stdout2 + " \n compare class - "+ command.getComparatorClass());
} else {
System.out.println( " FAIL! \n actual output - " + stdout + " \n expected -" + stdout2 + " \n compare class - "+ command.getComparatorClass());
}
}
}
} else {
if (sh.getRet()==0)
System.out.println( "SUCCESS! return code is " + sh.getRet() + " Output is " + stdout);
else
System.out.println( "FAIL! return code is " + sh.getRet() + " Output is " + stdout);
}
}
}
}
public static void main(String[] args) {
if (args==null || args.length==0 ) {
runTestInXMLFile("./bigtop-testcases.xml");
} else {
for (String filename : args) {
runTestInXMLFile(filename);
}
}
}
4. ExtactComparatorIgnoreWhiteSpace.java is to compare strings ignoring all space and ,
package com.lei.bigtop.hadoop.integration.test;
public class ExtactComparatorIgnoreWhiteSpace extends org.apache.hadoop.cli.util.ExactComparator {
public boolean compare(String actual, String expected) {
if (actual==null || expected==null) return false;
String actual2 = actual.replaceAll("\\s", "").replaceAll(",", "");
String expected2 = expected.replaceAll("\\s", "").replaceAll(",", "");
return super.compare(actual2, expected2);
}
}
5. The last is the project pom file:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.lei.bigtop</groupId>
<artifactId>LeiBigTop</artifactId>
<version>1.1</version>
<packaging>jar</packaging>
<name>LeiBigTop</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>net.sf.json-lib</groupId>
<artifactId>json-lib</artifactId>
<version>2.4</version>
<classifier>jdk15</classifier>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.bigtop.itest</groupId>
<artifactId>itest-common</artifactId>
<version>0.3.0-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>0.20.205.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
<version>0.20.205.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.groovy.maven.runtime</groupId>
<artifactId>gmaven-runtime-1.6</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.4.3</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src</sourceDirectory>
<directory>target</directory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.groovy.maven</groupId>
<artifactId>gmaven-plugin</artifactId>
<version>1.0</version>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>generateStubs</goal>
<goal>compile</goal>
<goal>generateTestStubs</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<sources>
<fileset>
<directory>${pom.basedir}/src/com/lei/bigtop/hadoop/test</directory>
<includes>
<include>**/*.groovy</include>
</includes>
</fileset>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<executions>
<execution>
<phase>integration-test</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions>
<configuration>
<mainClass>com.lei.bigtop.hadoop.test.RunHadoopTestFromXMLFile</mainClass>
<!--
<mainClass>com.lei.bigtop.hadoop.test.RunHadoopTestFromPropFile</mainClass>
-->
</configuration>
</plugin>
</plugins>
</build>
</project>
Well, that is it. Enjoy the journey! If you want the source, send me a request, I will let you know where to get it.
how to process a xml file via mapreduce and load them in hbase and hive . Please provide the sample code . thanks in advance.
ReplyDelete