atom feed1 message in org.apache.incubator.accumulo-commitssvn commit: r1242503 - in /incubator/...
FromSent OnAttachments
ktur...@apache.orgFeb 9, 2012 12:20 pm 
Subject:svn commit: r1242503 - in /incubator/accumulo/trunk: ./ src/core/ src/core/src/main/java/org/apache/accumulo/core/util/shell/ src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ src/examples/wikisearch/ingest/ src/server/ src/server/sr...
From:ktur...@apache.org (ktur@apache.org)
Date:Feb 9, 2012 12:20:44 pm
List:org.apache.incubator.accumulo-commits

Author: kturner Date: Thu Feb 9 20:20:44 2012 New Revision: 1242503

URL: http://svn.apache.org/viewvc?rev=1242503&view=rev Log: ACCUMULO-300 ACCUMULO-303 ACCUMULO-381 merged from 1.4

Modified: incubator/accumulo/trunk/ (props changed) incubator/accumulo/trunk/src/core/ (props changed) incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FormatterCommand.java incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java incubator/accumulo/trunk/src/examples/wikisearch/ingest/pom.xml incubator/accumulo/trunk/src/server/ (props changed) incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java

Propchange: incubator/accumulo/trunk/

------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Feb 9 20:20:44 2012 @@ -1,3 +1,3 @@ /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873 /incubator/accumulo/branches/1.3.5rc:1209938 -/incubator/accumulo/branches/1.4:1201902-1242131 +/incubator/accumulo/branches/1.4:1201902-1242502

Propchange: incubator/accumulo/trunk/src/core/

------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Feb 9 20:20:44 2012 @@ -1,3 +1,3 @@ /incubator/accumulo/branches/1.3.5rc/src/core:1209938 /incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215 -/incubator/accumulo/branches/1.4/src/core:1201902-1242131 +/incubator/accumulo/branches/1.4/src/core:1201902-1242502

Modified:
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java URL:
http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java?rev=1242503&r1=1242502&r2=1242503&view=diff ============================================================================== ---
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
(original) +++
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
Thu Feb 9 20:20:44 2012 @@ -917,8 +917,13 @@ public class Shell { }

public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean
printTimestamps, boolean paginate) throws IOException { - Class<? extends Formatter> formatterClass =
getFormatterClass(this.tableName); + Class<? extends Formatter> formatterClass = getFormatter();

+ printRecords(scanner, printTimestamps, paginate, formatterClass); + } + + public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean
printTimestamps, boolean paginate, Class<? extends Formatter> formatterClass) + throws IOException { printLines(FormatterFactory.getFormatter(formatterClass, scanner,
printTimestamps), paginate); }

@@ -1006,25 +1011,25 @@ public class Shell { public void setFormatterClass(String tableName, Class<? extends Formatter>
formatter) { this.tableFormatters.put(tableName, formatter); } - + /** - * Pull the current formatter for the given table and cache it. + * Return the formatter for this table, . * * @param tableName * @return The formatter class for the given table */ - public Class<? extends Formatter> getFormatterClass(String tableName) { - if (this.tableFormatters.containsKey(tableName) && null !=
this.tableFormatters.get(tableName)) { - return this.tableFormatters.get(tableName); + public Class<? extends Formatter> getFormatter() { + return getFormatter(this.tableName); + } + + public Class<? extends Formatter> getFormatter(String tableName) { + Class<? extends Formatter> formatter =
FormatterCommand.getCurrentFormatter(tableName, this); + + if (null == formatter) { + log.error("Could not load the specified formatter. Using the
DefaultFormatter"); + return this.defaultFormatterClass; } else { - Class<? extends Formatter> formatter =
FormatterCommand.getCurrentFormatter(tableName, this); - - if (null == formatter) { - return this.defaultFormatterClass; - } else { - return formatter; - } + return formatter; } } - }

Modified:
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java URL:
http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java?rev=1242503&r1=1242502&r2=1242503&view=diff ============================================================================== ---
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java
(original) +++
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ConfigCommand.java
Thu Feb 9 20:20:44 2012 @@ -32,11 +32,9 @@ import org.apache.accumulo.core.conf.Acc import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.util.BadArgumentException; -import org.apache.accumulo.core.util.format.Formatter; import org.apache.accumulo.core.util.shell.Shell; import org.apache.accumulo.core.util.shell.Shell.Command; import org.apache.accumulo.core.util.shell.Token; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionGroup; @@ -76,12 +74,7 @@ public class ConfigCommand extends Comma if (tableName != null) { if (!Property.isValidTablePropertyKey(property)) Shell.log.warn("Invalid per-table property : " + property + ", still
removing from zookeeper if its there."); - - // Fall back to the default formatter - if (property.equals(Property.TABLE_FORMATTER_CLASS.getKey())) { - shellState.setFormatterClass(tableName,
AccumuloClassLoader.loadClass(Property.TABLE_FORMATTER_CLASS.getDefaultValue(),
Formatter.class)); - } - + shellState.getConnector().tableOperations().removeProperty(tableName,
property); Shell.log.debug("Successfully deleted table configuration option."); } else { @@ -105,14 +98,8 @@ public class ConfigCommand extends Comma throw new BadArgumentException("Invalid per-table property.",
fullCommand, fullCommand.indexOf(property));

if
(property.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) - new ColumnVisibility(value); // validate that it is a valid - // expression - - // Load the formatter before setting the parameter - if (property.equals(Property.TABLE_FORMATTER_CLASS.getKey())) { - shellState.setFormatterClass(tableName,
AccumuloClassLoader.loadClass(value, Formatter.class)); - } - + new ColumnVisibility(value); // validate that it is a valid
expression + shellState.getConnector().tableOperations().setProperty(tableName,
property, value); Shell.log.debug("Successfully set table configuration option."); } else {

Modified:
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java URL:
http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java?rev=1242503&r1=1242502&r2=1242503&view=diff ============================================================================== ---
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
(original) +++
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
Thu Feb 9 20:20:44 2012 @@ -36,10 +36,8 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig; import org.apache.accumulo.core.security.VisibilityConstraint; import org.apache.accumulo.core.util.BadArgumentException; -import org.apache.accumulo.core.util.format.Formatter; import org.apache.accumulo.core.util.shell.Shell; import org.apache.accumulo.core.util.shell.Shell.Command; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionGroup; @@ -182,9 +180,7 @@ public class CreateTableCommand extends // Load custom formatter if set if (cl.hasOption(createTableOptFormatter.getOpt())) { String formatterClass =
cl.getOptionValue(createTableOptFormatter.getOpt()); - - shellState.setFormatterClass(tableName,
AccumuloClassLoader.loadClass(formatterClass, Formatter.class)); - + shellState.getConnector().tableOperations().setProperty(tableName,
Property.TABLE_FORMATTER_CLASS.toString(), formatterClass); }

@@ -214,12 +210,13 @@ public class CreateTableCommand extends createTableNoDefaultIters = new Option("ndi", "no-default-iterators",
false, "prevents creation of the normal default iterator set"); createTableOptEVC = new Option("evc", "enable-visibility-constraint",
false, "prevents users from writing data they can not read. When enabling
this may want to consider disabling bulk import and alter table"); - createTableOptFormatter = new Option("f", "formatter", false, "default
formatter to set"); + createTableOptFormatter = new Option("f", "formatter", true, "default
formatter to set");

createTableOptCopyConfig.setArgName("table"); createTableOptCopySplits.setArgName("table"); createTableOptSplit.setArgName("filename"); createTableOptAgg.setArgName("{<columnfamily>[:<columnqualifier>]=<aggregation_class>}"); + createTableOptFormatter.setArgName("className");

// Splits and CopySplits are put in an optionsgroup to make them // mutually exclusive

Modified:
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FormatterCommand.java URL:
http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FormatterCommand.java?rev=1242503&r1=1242502&r2=1242503&view=diff ============================================================================== ---
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FormatterCommand.java
(original) +++
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FormatterCommand.java
Thu Feb 9 20:20:44 2012 @@ -51,9 +51,6 @@ public class FormatterCommand extends Co // Remove the property shellState.getConnector().tableOperations().removeProperty(tableName,
Property.TABLE_FORMATTER_CLASS.toString());

- // Reset the shell formatter on this table - shellState.setFormatterClass(tableName,
AccumuloClassLoader.loadClass(Property.TABLE_FORMATTER_CLASS.getDefaultValue(),
Formatter.class)); - shellState.getReader().printString("Removed formatter on " + tableName +
"\n"); } else if (cl.hasOption(listClassOption.getOpt())) { // Get the options for this table @@ -71,9 +68,7 @@ public class FormatterCommand extends Co // Set the formatter with the provided options String className = cl.getOptionValue(formatterClassOption.getOpt());

- // Update the shell formatter for this table - shellState.setFormatterClass(tableName,
AccumuloClassLoader.loadClass(className, Formatter.class)); - + // Set the formatter property on the table shellState.getConnector().tableOperations().setProperty(tableName,
Property.TABLE_FORMATTER_CLASS.toString(), className); }

@@ -96,7 +91,6 @@ public class FormatterCommand extends Co Class<? extends Formatter> formatter; try { formatter = AccumuloClassLoader.loadClass(ent.getValue(),
Formatter.class); - shellState.setFormatterClass(tableName, formatter); } catch (ClassNotFoundException e) { return null; }

Modified:
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java URL:
http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java?rev=1242503&r1=1242502&r2=1242503&view=diff ============================================================================== ---
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
(original) +++
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
Thu Feb 9 20:20:44 2012 @@ -31,8 +31,10 @@ import org.apache.accumulo.core.data.Ran import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.format.BinaryFormatter; +import org.apache.accumulo.core.util.format.Formatter; import org.apache.accumulo.core.util.shell.Shell; import org.apache.accumulo.core.util.shell.Shell.Command; +import org.apache.accumulo.start.classloader.AccumuloClassLoader; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -41,22 +43,29 @@ import org.apache.hadoop.io.Text;

public class ScanCommand extends Command {

- private Option scanOptAuths, scanOptStartRow, scanOptEndRow, scanOptRow,
scanOptColumns, disablePaginationOpt, tableOpt, showFewOpt; + private Option scanOptAuths, scanOptStartRow, scanOptEndRow, scanOptRow,
scanOptColumns, disablePaginationOpt, tableOpt, showFewOpt, formatterOpt; protected Option timestampOpt;

public int execute(String fullCommand, CommandLine cl, Shell shellState)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException, ParseException {

String tableName; + Class<? extends Formatter> formatter = null;

if (cl.hasOption(tableOpt.getOpt())) { tableName = cl.getOptionValue(tableOpt.getOpt()); if (!shellState.getConnector().tableOperations().exists(tableName)) throw new TableNotFoundException(null, tableName, null); + + // Use the configured formatter unless one was provided + if (!cl.hasOption(formatterOpt.getOpt())) { + formatter = FormatterCommand.getCurrentFormatter(tableName,
shellState); + } } else { shellState.checkTableState(); tableName = shellState.getTableName(); } + // handle first argument, if present, the authorizations list to // scan with Authorizations auths = getAuths(cl, shellState); @@ -88,7 +97,11 @@ public class ScanCommand extends Command }

} else { - printRecords(cl, shellState, scanner); + if (null == formatter) { + printRecords(cl, shellState, scanner); + } else { + printRecords(cl, shellState, scanner, formatter); + } }

return 0; @@ -113,7 +126,22 @@ public class ScanCommand extends Command }

protected void printRecords(CommandLine cl, Shell shellState,
Iterable<Entry<Key,Value>> scanner) throws IOException { - shellState.printRecords(scanner, cl.hasOption(timestampOpt.getOpt()),
!cl.hasOption(disablePaginationOpt.getOpt())); + if (cl.hasOption(formatterOpt.getOpt())) { + try { + String className = cl.getOptionValue(formatterOpt.getOpt()); + Class<? extends Formatter> formatterClass =
AccumuloClassLoader.loadClass(className, Formatter.class); + + printRecords(cl, shellState, scanner, formatterClass); + } catch (ClassNotFoundException e) { + shellState.getReader().printString("Formatter class could not be
loaded.\n" + e.getMessage() + "\n"); + } + } else { + shellState.printRecords(scanner, cl.hasOption(timestampOpt.getOpt()),
!cl.hasOption(disablePaginationOpt.getOpt())); + } + } + + protected void printRecords(CommandLine cl, Shell shellState,
Iterable<Entry<Key,Value>> scanner, Class<? extends Formatter> formatter) throws
IOException { + shellState.printRecords(scanner, cl.hasOption(timestampOpt.getOpt()),
!cl.hasOption(disablePaginationOpt.getOpt()), formatter); }

protected void printBinaryRecords(CommandLine cl, Shell shellState,
Iterable<Entry<Key,Value>> scanner) throws IOException { @@ -175,6 +203,7 @@ public class ScanCommand extends Command disablePaginationOpt = new Option("np", "no-pagination", false, "disables
pagination of output"); tableOpt = new Option(Shell.tableOption, "tableName", true, "table to be
scanned"); showFewOpt = new Option("f", "show few", true, "Only shows certain amount
of characters"); + formatterOpt = new Option("fm", "formatter", true, "fully qualified name of
the formatter class to use");

scanOptAuths.setArgName("comma-separated-authorizations"); scanOptRow.setArgName("row"); @@ -185,6 +214,7 @@ public class ScanCommand extends Command tableOpt.setRequired(false); showFewOpt.setRequired(false); showFewOpt.setArgName("int"); + formatterOpt.setArgName("className");

o.addOption(scanOptAuths); o.addOption(scanOptRow); @@ -195,6 +225,7 @@ public class ScanCommand extends Command o.addOption(disablePaginationOpt); o.addOption(tableOpt); o.addOption(showFewOpt); + o.addOption(formatterOpt);

return o; }

Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/pom.xml URL:
http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/pom.xml?rev=1242503&r1=1242502&r2=1242503&view=diff ============================================================================== --- incubator/accumulo/trunk/src/examples/wikisearch/ingest/pom.xml (original) +++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/pom.xml Thu Feb 9
20:20:44 2012 @@ -107,7 +107,7 @@ <configuration> <outputDirectory>lib</outputDirectory> <!-- just grab the non-provided runtime dependencies --> -
<includeArtifactIds>commons-lang,google-collections,lucene-core,lucene-analyzers,lucene-wikipedia,protobuf-java,accumulo-core,hadoop-core,libthrift,cloudtrace,zookeeper</includeArtifactIds> +
<includeArtifactIds>commons-lang,google-collections,lucene-core,lucene-analyzers,lucene-wikipedia,protobuf-java,accumulo-core,hadoop-core,libthrift,cloudtrace,zookeeper,commons-codec</includeArtifactIds> <excludeTransitive>false</excludeTransitive> </configuration> </execution>

Propchange: incubator/accumulo/trunk/src/server/

------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Feb 9 20:20:44 2012 @@ -1,3 +1,3 @@ /incubator/accumulo/branches/1.3.5rc/src/server:1209938 /incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611 -/incubator/accumulo/branches/1.4/src/server:1201902-1242131 +/incubator/accumulo/branches/1.4/src/server:1201902-1242502

Modified:
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java URL:
http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java?rev=1242503&r1=1242502&r2=1242503&view=diff ============================================================================== ---
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java
(original) +++
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java
Thu Feb 9 20:20:44 2012 @@ -136,7 +136,7 @@ public class MasterServlet extends Basic masterStatus.addSortableColumn("Loggers", new NumberType<Integer>((int)
(slaves.size() * .8), Integer.MAX_VALUE, 1, Integer.MAX_VALUE), "The number of write-ahead loggers. This should be approximately the
same as the number of tablet servers (and greater than zero)."); masterStatus.addSortableColumn("Last&nbsp;GC", null, "The last time files
were cleaned-up from HDFS."); - masterStatus.addSortableColumn("#&nbsp;Tablets", new
NumberType<Integer>(0, 300000, 2, 400000), null); + masterStatus.addSortableColumn("#&nbsp;Tablets", new
NumberType<Integer>(0, Integer.MAX_VALUE, 2, Integer.MAX_VALUE), null); masterStatus.addSortableColumn("#&nbsp;Unassigned<br />Tablets", new
NumberType<Integer>(0, 0), null); masterStatus.addSortableColumn("Entries", new NumberType<Long>(), "The
total number of key/value pairs in Accumulo"); masterStatus.addSortableColumn("Ingest", new NumberType<Long>(), "The
number of Key/Value pairs inserted, per second. "