Skip to content

Commit

Permalink
s/prepareImport/importFileset/ and add importPaths
Browse files Browse the repository at this point in the history
Simplified method for clients which do not have access
to Bio-Formats and the ImportLibrary for filling out
the Fileset and ImportSettings objects. This prevents
a good deal of cut-n-paste but also reduces functionality.
  • Loading branch information
joshmoore committed Jan 3, 2013
1 parent 795d04f commit a71b04d
Show file tree
Hide file tree
Showing 8 changed files with 83 additions and 40 deletions.
16 changes: 13 additions & 3 deletions components/blitz/resources/omero/Repositories.ice
Original file line number Diff line number Diff line change
Expand Up @@ -296,13 +296,13 @@ module omero {

/**
* [ImportSettings] which are provided by the
* client on the call to [ManagedRepository::prepareImport].
* client on the call to [ManagedRepository::importFileset].
**/
ImportSettings settings;

/**
* [ImportLocation] which is calculated during
* the call to [ManagedRepository::prepareImport].
* the call to [ManagedRepository::importFileset].
**/
ImportLocation location;

Expand Down Expand Up @@ -342,7 +342,17 @@ module omero {
* overwritten or interfering with one another, a new directory
* may be created for the current session.
**/
ImportProcess* prepareImport(omero::model::Fileset fs, ImportSettings settings) throws ServerError;
ImportProcess* importFileset(omero::model::Fileset fs, ImportSettings settings) throws ServerError;

/**
* For clients without access to Bio-Formats, the simplified
* []importPaths] method allows passing solely the absolute
* path of the files to be uploaded (no directories) and all
* configuration happens server-side. Much of the functionality
* provided via [omero::model::Fileset] and [omero::grid::ImportSettings]
* is of course lost.
**/
ImportProcess* importPaths(omero::api::StringSet filePaths) throws ServerError;

/**
* List imports that are currently running in this importer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ public class ImportContainer
private IObject target;

public ImportContainer(File file,
IObject target,
Double[] userPixels, String reader, String[] usedFiles, Boolean isSPW)
IObject target,
Double[] userPixels, String reader, String[] usedFiles, Boolean isSPW)
{
this.file = file;
this.target = target;
Expand Down
8 changes: 4 additions & 4 deletions components/blitz/src/ome/formats/importer/ImportLibrary.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,10 @@ public class ImportLibrary implements IObservable

/** The class used to identify the dataset target.*/
private static final String DATASET_CLASS = "omero.model.Dataset";

/** The class used to identify the screen target.*/
private static final String SCREEN_CLASS = "omero.model.Screen";

private final ArrayList<IObserver> observers = new ArrayList<IObserver>();

private final OMEROMetadataStoreClient store;
Expand Down Expand Up @@ -140,7 +140,7 @@ public void notifyObservers(ImportEvent event)

/**
* Primary user method for importing a number
*
*
* @param config The configuration information.
* @param candidates Hosts information about the files to import.
*/
Expand Down Expand Up @@ -213,7 +213,7 @@ public ImportProcessPrx createImport(final ImportContainer container)
final ImportSettings settings = new ImportSettings();
final Fileset fs = new FilesetI();
container.fillData(new ImportConfig(), settings, fs);
return repo.prepareImport(fs, settings);
return repo.importFileset(fs, settings);

}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public void ice_exception(Exception ex) {

/**
* The import location as defined by the managed repository during
* prepareImport. Never null.
* importFileset. Never null.
*/
private final ImportLocation location;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import Ice.Current;

import ome.formats.importer.ImportConfig;
import ome.formats.importer.ImportContainer;
import ome.services.blitz.impl.ServiceFactoryI;

import omero.ServerError;
Expand All @@ -51,9 +52,11 @@
import omero.grid._ManagedRepositoryTie;
import omero.model.Fileset;
import omero.model.FilesetEntry;
import omero.model.FilesetI;
import omero.model.FilesetJobLink;
import omero.model.FilesetVersionInfo;
import omero.model.FilesetVersionInfoI;
import omero.model.IObject;
import omero.model.IndexingJobI;
import omero.model.Job;
import omero.model.MetadataImportJob;
Expand Down Expand Up @@ -126,7 +129,7 @@ public Ice.Object tie() {
*
* @FIXME For the moment only the top-level directory is being incremented.
*/
public ImportProcessPrx prepareImport(Fileset fs, ImportSettings settings,
public ImportProcessPrx importFileset(Fileset fs, ImportSettings settings,
Ice.Current __current) throws omero.ServerError {

if (fs == null || fs.sizeOfUsedFiles() < 1) {
Expand Down Expand Up @@ -161,6 +164,25 @@ public ImportProcessPrx prepareImport(Fileset fs, ImportSettings settings,
return createImportProcess(fs, location, settings, __current);
}

public ImportProcessPrx importPaths(List<String> paths,
Ice.Current __current) throws ServerError {

if (paths == null || paths.size() < 1) {
throw new omero.ApiUsageException(null, null, "No paths provided");
}

final ImportContainer container = new ImportContainer(
null /*file*/, null /*target*/, null /*userPixels*/,
"Unknown" /*reader*/, paths.toArray(new String[0]),
false /*spw*/);

final ImportSettings settings = new ImportSettings();
final Fileset fs = new FilesetI();
container.fillData(new ImportConfig(), settings, fs);

return importFileset(fs, settings, __current);
}

public List<ImportProcessPrx> listImports(Ice.Current __current) throws omero.ServerError {

final List<Long> filesetIds = new ArrayList<Long>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ public void testBasicImportExample() throws Exception {
Fileset fs = new FilesetI();
ic.fillData(new ImportConfig(), settings, fs);

ImportProcessPrx i = repo.prepareImport(fs, settings, curr());
ImportProcessPrx i = repo.importFileset(fs, settings, curr());
assertNotNull(i);

upload(i.getUploader(0));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,14 @@
import omero.model.OriginalFile;
import omero.sys.EventContext;

/**
/**
* Collections of tests for the <code>ManagedRepository</code> service.
*
* @author Colin Blackburn &nbsp;&nbsp;&nbsp;&nbsp;
* <a href="mailto:[email protected]">[email protected]</a>
*/
@Test(groups = {"integration", "fs"})
public class ManagedRepositoryTest
public class ManagedRepositoryTest
extends AbstractServerTest
{

Expand Down Expand Up @@ -67,7 +67,7 @@ public void setRepo() throws Exception {
*/
void assertFileExists(String message, String path)
throws ServerError
{
{
assertTrue(message + path, repo.fileExists(path));
}

Expand All @@ -77,9 +77,9 @@ void assertFileExists(String message, String path)
*/
void assertFileDoesNotExist(String message, String path)
throws ServerError
{
{
assertFalse(message + path, repo.fileExists(path));
}
}

/**
* Construct a path from its elements
Expand All @@ -96,7 +96,7 @@ String buildPath(String[] pathElements)
}

// Primarily to get code compiling during the major refactoring
ImportLocation prepareImport(List<String> srcPaths) {
ImportLocation importFileset(List<String> srcPaths) {
fail("NYI");
return null;
}
Expand Down Expand Up @@ -128,30 +128,30 @@ public void testGetCurrentRepoDirSimple()
srcPaths.add(buildPath(src));
String[] dest = {uniquePath, file1};
destPath = buildPath(dest);
ImportLocation data = prepareImport(srcPaths);
ImportLocation data = importFileset(srcPaths);
assertContains(data.usedFiles.get(0), destPath);
touch(uploadUsedFile(data, data.usedFiles.get(0)));

// Different file that should go in existing directory
src[1] = file2;
srcPaths.set(0, buildPath(src));
dest[1] = file2;
destPath = buildPath(dest);
data = prepareImport(srcPaths);
data = importFileset(srcPaths);
assertContains(data.usedFiles.get(0), destPath);
touch(uploadUsedFile(data, data.usedFiles.get(0)));

// Same file that should go in new directory
dest[0] = uniquePath + "-1";
destPath = buildPath(dest);
data = prepareImport(srcPaths);
data = importFileset(srcPaths);
assertContains(data.usedFiles.get(0), destPath);
touch(uploadUsedFile(data, data.usedFiles.get(0)));

// Same file again that should go in new directory
dest[0] = uniquePath + "-2";
destPath = buildPath(dest);
data = prepareImport(srcPaths);
data = importFileset(srcPaths);
assertContains(data.usedFiles.get(0), destPath);
}

Expand All @@ -162,9 +162,9 @@ public void testGetCurrentRepoDirSimple()
* @throws Exception Thrown if an error occurred.
*/
@Test
public void testGetCurrentRepoDirMultipleFiles()
public void testGetCurrentRepoDirMultipleFiles()
throws Exception
{
{
List<String> srcPaths = new ArrayList<String>();
List<String> destPaths = new ArrayList<String>();

Expand All @@ -183,7 +183,7 @@ public void testGetCurrentRepoDirMultipleFiles()
destPaths.add(buildPath(dest));
dest[1] = file2;
destPaths.add(buildPath(dest));
ImportLocation data = prepareImport(srcPaths);
ImportLocation data = importFileset(srcPaths);
assertTrue(data.usedFiles.size()==destPaths.size());
for (int i=0; i<data.usedFiles.size(); i++) {
assertContains(data.usedFiles.get(i), destPaths.get(i));
Expand All @@ -200,7 +200,7 @@ public void testGetCurrentRepoDirMultipleFiles()
destPaths.set(0, buildPath(dest));
dest[1] = file4;
destPaths.set(1, buildPath(dest));
data = prepareImport(srcPaths);
data = importFileset(srcPaths);
assertTrue(data.usedFiles.size()==destPaths.size());
for (int i=0; i<data.usedFiles.size(); i++) {
assertContains(data.usedFiles.get(i), destPaths.get(i));
Expand All @@ -217,7 +217,7 @@ public void testGetCurrentRepoDirMultipleFiles()
destPaths.set(0, buildPath(dest));
dest[1] = file4;
destPaths.set(1, buildPath(dest));
data = prepareImport(srcPaths);
data = importFileset(srcPaths);
assertTrue(data.usedFiles.size()==destPaths.size());
for (int i=0; i<data.usedFiles.size(); i++) {
assertContains(data.usedFiles.get(i), destPaths.get(i));
Expand All @@ -230,7 +230,7 @@ public void testGetCurrentRepoDirMultipleFiles()
destPaths.set(0, buildPath(dest));
dest[1] = file4;
destPaths.set(1, buildPath(dest));
data = prepareImport(srcPaths);
data = importFileset(srcPaths);
assertTrue(data.usedFiles.size()==destPaths.size());
for (int i=0; i<data.usedFiles.size(); i++) {
assertContains(data.usedFiles.get(i), destPaths.get(i));
Expand Down Expand Up @@ -275,7 +275,7 @@ public void testGetCurrentRepoDirNested()
dest[2] = subSubDir;
dest = (String[]) ArrayUtils.add(dest,file3);
destPaths.add(buildPath(dest));
ImportLocation data = prepareImport(srcPaths);
ImportLocation data = importFileset(srcPaths);
assertTrue(data.usedFiles.size()==destPaths.size());
for (int i=0; i<data.usedFiles.size(); i++) {
assertContains(data.usedFiles.get(i), destPaths.get(i));
Expand All @@ -292,7 +292,7 @@ public void testGetCurrentRepoDirNested()
dest2[2] = subSubDir;
dest2 = (String[]) ArrayUtils.add(dest2,file3);
destPaths.add(buildPath(dest2));
data = prepareImport(srcPaths);
data = importFileset(srcPaths);
assertTrue(data.usedFiles.size()==destPaths.size());
for (int i=0; i<data.usedFiles.size(); i++) {
assertContains(data.usedFiles.get(i), destPaths.get(i));
Expand All @@ -315,7 +315,7 @@ public void testDeleteUploadedFileSimple()

String[] src = {uniquePath, file1};
srcPaths.add(buildPath(src));
ImportLocation data = prepareImport(srcPaths);
ImportLocation data = importFileset(srcPaths);
touch(uploadUsedFile(data, data.usedFiles.get(0)));
for (String path : data.usedFiles) {
assertFileExists("Upload failed. File does not exist: ", path);
Expand Down Expand Up @@ -344,7 +344,7 @@ public void testDeleteUploadedMultipleFilesSimple()
srcPaths.add(buildPath(src));
src[1] = file2;
srcPaths.add(buildPath(src));
ImportLocation data = prepareImport(srcPaths);
ImportLocation data = importFileset(srcPaths);
for (String path : data.usedFiles) {
touch(uploadUsedFile(data, path));
assertFileExists("Upload failed. File does not exist: ", path);
Expand Down Expand Up @@ -373,7 +373,7 @@ public void testDeleteUploadedPartialFiles()
srcPaths.add(buildPath(src));
src[1] = file2;
srcPaths.add(buildPath(src));
ImportLocation data = prepareImport(srcPaths);
ImportLocation data = importFileset(srcPaths);
touch(uploadUsedFile(data, data.usedFiles.get(0)));
assertFileExists("Upload failed. File does not exist: ", data.usedFiles.get(0));
assertFileDoesNotExist("Something wrong. File does exist!: ", data.usedFiles.get(1));
Expand Down Expand Up @@ -411,7 +411,7 @@ public void testDeleteUploadedMultipleFilesNested()
src = (String[]) ArrayUtils.add(src,file3);
srcPaths.add(buildPath(src));

ImportLocation data = prepareImport(srcPaths);
ImportLocation data = importFileset(srcPaths);
for (String path : data.usedFiles) {
touch(uploadUsedFile(data, path));
assertFileExists("Upload failed. File does not exist: ", path);
Expand Down Expand Up @@ -440,7 +440,7 @@ public void testDeleteUploadedMultipleSetsDeleteOneSet()
srcPaths.add(buildPath(src));
src[1] = file2;
srcPaths.add(buildPath(src));
ImportLocation data1 = prepareImport(srcPaths);
ImportLocation data1 = importFileset(srcPaths);

srcPaths.clear();
file1 = UUID.randomUUID().toString() + ".dv";
Expand All @@ -449,7 +449,7 @@ public void testDeleteUploadedMultipleSetsDeleteOneSet()
srcPaths.add(buildPath(src));
src[1] = file2;
srcPaths.add(buildPath(src));
ImportLocation data2 = prepareImport(srcPaths);
ImportLocation data2 = importFileset(srcPaths);

for (String path : data1.usedFiles) {
touch(uploadUsedFile(data1, path));
Expand Down
17 changes: 14 additions & 3 deletions components/tools/OmeroPy/test/integration/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -429,14 +429,26 @@ def upload_folder(self, proc, folder):
rfs.close()
return ret_val

def testSimpleImport(self):
def testImportFileset(self):
client = self.new_client()
mrepo = self.getManagedRepo(client)
folder = self.create_test_dir()
fileset = self.create_fileset(folder)
settings = self.create_settings()

proc = mrepo.prepareImport(fileset, settings)
proc = mrepo.importFileset(fileset, settings)
self.assertImport(client, proc, folder)

def testImportPaths(self):
client = self.new_client()
mrepo = self.getManagedRepo(client)
folder = self.create_test_dir()
paths = folder.files()

proc = mrepo.importPaths(paths)
self.assertImport(client, proc, folder)

def assertImport(self, client, proc, folder):
hashes = self.upload_folder(proc, folder)
handle = proc.verifyUpload(hashes)
cb = CmdCallbackI(client, handle)
Expand All @@ -447,6 +459,5 @@ def testSimpleImport(self):
else:
self.assertEquals(1, len(rsp.pixels))


if __name__ == '__main__':
unittest.main()

0 comments on commit a71b04d

Please sign in to comment.