Source code for QueryLayer.TileQuery
import logging as log
from ImageLayer import HDF5
from ImageLayer import Mojo
from ImageLayer import ImageStack
from ImageLayer import BossGrid
from ImageLayer import Sparse
from urllib2 import URLError
from Query import Query
import numpy as np
import sys
import os
[docs]class TileQuery(Query):
""" Describe requsts for tiles from a :class:`Datasource`
Arguments
----------
args: list
* (:class:`DataQuery`) gives the full volume request
* (numpy.ndarray) 3x1 tile offset of the given tile
* (numpy.ndarray) 2x3 subregion within given tile
keywords: dict
unused
Attributes
-----------
source_list: list
from :class:`RUNTIME` ``.IMAGE.SOURCE.LIST``
SOURCES: dict
* source_list[0]: :class:`HDF5`
* source_list[1]: :class:`BossGrid`
"""
def __init__(self, *args, **keywords):
Query.__init__(self, **keywords)
query, zyx_index, kji_pixels = args
self.source_list = self.RUNTIME.IMAGE.SOURCE.LIST
self.SOURCES = {
self.source_list[0]: HDF5,
self.source_list[1]: BossGrid,
self.source_list[2]: Mojo,
self.source_list[3]: ImageStack,
}
self.RUNTIME.TILE.ZYX.VALUE = zyx_index
self.RUNTIME.TILE.KJI.VALUE = kji_pixels
self.RUNTIME.TILE.SCALES.VALUE = query.scales
# Set blocksize from the query
self.blocksize = query.blocksize
# Get the right blocksize, datatype, and path
q_type = query.OUTPUT.INFO.TYPE.VALUE
q_path = query.OUTPUT.INFO.PATH.VALUE
# Set the right blocksize, datatype, and path
self.OUTPUT.INFO.TYPE.VALUE = q_type
self.OUTPUT.INFO.PATH.VALUE = q_path
# Very important to get the right datasource
query_source = query.RUNTIME.IMAGE.SOURCE
self_source = self.RUNTIME.IMAGE.SOURCE
self_source.VALUE = query_source.VALUE
# Only applies to HDF5 datasource
query_h5 = query.RUNTIME.IMAGE.SOURCE.HDF5
self_h5 = self.RUNTIME.IMAGE.SOURCE.HDF5
self_h5.VALUE = query_h5.VALUE
# Only applies to Mojo datasource
query_format = query.RUNTIME.IMAGE.SOURCE.MOJO.FORMAT
self_format = self.RUNTIME.IMAGE.SOURCE.MOJO.FORMAT
self_format.VALUE = query_format.VALUE
# Only applies to Boss datasource
self_source.BOSS = query_source.BOSS
# Get the XY resolution for Mojo
query_xy = query.INPUT.RESOLUTION.XY
self_xy = self.INPUT.RESOLUTION.XY
self_xy.VALUE = query_xy.VALUE
@property
def key(self):
""" return the key for the database
Returns
-------
str
the path value from :meth:`path` \
joined with the tile_values from \
:meth:`index_zyx` and :meth:`all_scales`
"""
origin = self.index_zyx
scales = self.all_scales
tile_values = np.r_[scales,origin]
tile_key = np.array2string(tile_values)
return self.path + tile_key
@property
def tile(self):
""" Load the requested tile from the source
Returns
--------
numpy.ndarray
The full image loaded by :meth:`my_source`
"""
return self.my_source.load_tile(self)
@property
def my_source(self):
""" return the source that loads this tile
Returns
-------
:class:`Datasource`
the source value from ``RUNTIME.IMAGE`` \
as an actual :class:`Datasource` \
that can load a ``TileQuery``.
"""
my_source = self.RUNTIME.IMAGE.SOURCE.VALUE
return self.get_source(my_source)
@property
def all_scales(self):
""" A scale array from ``RUNTIME.TILE``
Returns
-------
numpy.ndarray
A 3x1 array of the target/source voxel ratios
"""
return np.uint32(self.RUNTIME.TILE.SCALES.VALUE)
@property
def pixels_kji(self):
"""The pixels bounds needed from the tile
Returns
--------
numpy.ndarray
The 2x3 subregion within the given tile
"""
return np.uint32(self.RUNTIME.TILE.KJI.VALUE)
@property
def index_zyx(self):
"""The tile offsets needed to get the tile
Returns
--------
numpy.ndarray
3x1 tile offset of the given tile
"""
return np.uint32(self.RUNTIME.TILE.ZYX.VALUE)
@property
def target_tile_bounds(self):
"""The scaled target bounds of the tile
Returns
--------
numpy.ndarray
2x3 scaled target pixel bounds
"""
# Get the target offset from the target origin
tile_origin = self.index_zyx * self.blocksize
offset = np.outer([0,1], self.blocksize)
return tile_origin + offset
@property
def source_tile_bounds(self):
"""The full image bounds of the tile
Returns
--------
numpy.ndarray
2x3 full image pixel bounds
"""
return self.all_scales * self.target_tile_bounds
@property
def target_origin(self):
"""The origin of query in target coordinates
Returns
--------
numpy.ndarray
3x1 target pixel lower bound
"""
return self.target_bounds[0]
@property
def target_bounds(self):
"""The bounds of query in target coordinates
Returns
--------
numpy.ndarray
2x3 target pixel boundaries
"""
return self.pixels_kji + self.target_tile_bounds[0]
@property
def source_bounds(self):
"""The bounds of query in full coordinates
Returns
--------
numpy.ndarray
2x3 full source pixel bounds in tile
"""
return self.all_scales * self.target_bounds
@property
def preload_source(self):
""" Get a :meth:`valid_source` with a measured size
Returns
--------
dict
Empty if no :meth:`valid_source`
* :class:`RUNTIME` ``.CACHE.META.NAME``
(int) -- The byte size of this dictionary
* :class:`RUNTIME` ``.IMAGE.SOURCE.NAME``
(str) -- The :class:`Datasource` subclass
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Get all the metadata needed for the cache
cache_meta = self.RUNTIME.CACHE.META
# Preload the metadata from the source
keywords = self.valid_source
# Get the size of this dictionary for the cache
dict_size = np.uint32(sys.getsizeof({}))
keywords[cache_meta.NAME] = dict_size
# Size calculation function
def get_bytes(v):
# Estimate memory usage for sparse matrix
if type(v) in Sparse.TYPES:
return Sparse.count_bytes(v)
# Works for dictionaries of similarly size elements
if type(v) is dict:
# number of items X size of item
any_k = next(iter(v), None)
any_v = v.get(any_k, None)
key_size = sys.getsizeof(any_k)
val_size = len(v)*get_bytes(any_v)
return key_size + val_size + dict_size
# Works for numpy arrays, strings, numbers
return sys.getsizeof(v)
# calculate the size of everything
for key in keywords.keys():
v = keywords[key]
n_bytes = get_bytes(v)
keywords[cache_meta.NAME] += n_bytes
# Return keywords for cache and dataQuery
return keywords
@property
def valid_source(self):
""" Call ``preload_source`` for all :data:`source_list`
Returns
--------
dict
Empty if no :meth:`Datasource.preload_source` \
can find this filname to give a valid volume.
* :class:`RUNTIME` ``.IMAGE.SOURCE.NAME``
(str) -- The :class:`Datasource` subclass
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Get the path key and value
k_path = self.OUTPUT.INFO.PATH.NAME
v_path = self.OUTPUT.INFO.PATH.VALUE
# Make sure we have a valid pathname
if not os.path.exists(v_path):
msg = 'The {} {} does not exist on the server.'
msg = msg.format(k_path, v_path)
raise URLError([msg, 503])
# Get the default source
my_source = self.RUNTIME.IMAGE.SOURCE
# Validate the source of self.path
for name in self.source_list:
source = self.get_source(name)
# Ask if source can load self path
keywords = source.preload_source(self)
# If sucessfully loaded
if len(keywords):
# Set valid source
keywords[my_source.NAME] = name
# Set channel path
keywords[k_path] = v_path
return keywords
# return empty
return {}
[docs] def get_source(self, name):
""" Gets a ``Datasource`` from :data:`SOURCES`
Arguments
----------
name: str
The key to a ``Datasource`` subclass \
as listed in :data:`SOURCES`
Returns
--------
:class:`Datasource`
A ``Datasource`` subclass given by ``name``
"""
return self.SOURCES.get(name, HDF5)
