eoxserver.resources.coverages.models

1 #------------------------------------------------------------------------------- 2 # $Id: models.py 2445 2013-04-26 18:14:11Z meissls $ 3 # 4 # Project: EOxServer <http://eoxserver.org> 5 # Authors: Stephan Krause <stephan.krause@eox.at> 6 # Stephan Meissl <stephan.meissl@eox.at> 7 # Martin Paces <martin.paces@eox.at> 8 # 9 #------------------------------------------------------------------------------- 10 # Copyright (C) 2011 EOX IT Services GmbH 11 # 12 # Permission is hereby granted, free of charge, to any person obtaining a copy 13 # of this software and associated documentation files (the "Software"), to deal 14 # in the Software without restriction, including without limitation the rights 15 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 # copies of the Software, and to permit persons to whom the Software is 17 # furnished to do so, subject to the following conditions: 18 # 19 # The above copyright notice and this permission notice shall be included in all 20 # copies of this Software or works derived from this Software. 21 # 22 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 28 # THE SOFTWARE. 29 #------------------------------------------------------------------------------- 30 31 import re 32 import logging 33 34 from django.core.validators import RegexValidator 35 from django.core.exceptions import ValidationError 36 37 from django.contrib.gis.db import models 38 from django.contrib.gis.geos import Point, LinearRing, Polygon, MultiPolygon 39 from django.contrib.gis.geos.error import GEOSException 40 41 from eoxserver.contrib import gdal 42 from eoxserver.core.models import Resource 43 from eoxserver.backends.models import ( 44 Location, LocalPath, RemotePath, 45 RasdamanLocation, CacheFile 46 ) 47 from eoxserver.resources.coverages.validators import ( 48 validateEOOM, validateCoverageIDnotInEOOM 49 ) 50 51 from eoxserver.resources.coverages.formats import _gerexValMime as regexMIMEType 52 53 54 logger = logging.getLogger(__name__) 55 56 MIMETypeValidator = RegexValidator( regexMIMEType , message="The field must contain a valid MIME Type!" ) 57 NCNameValidator = RegexValidator(re.compile(r'^[a-zA-z_][a-zA-Z0-9_.-]*$'), message="This field must contain a valid NCName.") 58

59 -class NilValueRecord(models.Model):

60 reason = models.CharField(max_length=128, default="http://www.opengis.net/def/nil/OGC/0/unknown", 61 choices=( 62 ("http://www.opengis.net/def/nil/OGC/0/inapplicable", "Inapplicable (There is no value)"), 63 ("http://www.opengis.net/def/nil/OGC/0/missing", "Missing"), 64 ("http://www.opengis.net/def/nil/OGC/0/template", "Template (The value will be available later)"), 65 ("http://www.opengis.net/def/nil/OGC/0/unknown", "Unknown"), 66 ("http://www.opengis.net/def/nil/OGC/0/withheld", "Withheld (The value is not divulged)"), 67 ("http://www.opengis.net/def/nil/OGC/0/AboveDetectionRange", "Above detection range"), 68 ("http://www.opengis.net/def/nil/OGC/0/BelowDetectionRange", "Below detection range") 69 ) 70 ) 71 value = models.IntegerField() 72

73 - def __unicode__(self):

74 return self.reason+" "+str(self.value)

75

76 - class Meta:

77 verbose_name = "Nil Value"

78

79 -class BandRecord(models.Model):

80 name = models.CharField(max_length=256) 81 identifier = models.CharField(max_length=256) 82 description = models.TextField() 83 definition = models.CharField(max_length=256) 84 uom = models.CharField("UOM", max_length=16) 85 nil_values = models.ManyToManyField(NilValueRecord, null=True, blank=True, verbose_name="Nil Value") 86 gdal_interpretation = models.IntegerField("GDAL Interpretation", default=gdal.GCI_Undefined, 87 choices=gdal.GCI_TO_NAME.items() 88 ) 89

90 - def __unicode__(self):

91 return self.name

92

93 - class Meta:

94 verbose_name = "Band"

95 96

97 -class RangeTypeRecord(models.Model):

98 name = models.CharField(max_length=256) 99 data_type = models.IntegerField(choices=gdal.GDT_TO_NAME.items()) 100 bands = models.ManyToManyField(BandRecord, through="RangeType2Band") 101

102 - def __unicode__(self):

103 return self.name

104

105 - class Meta:

106 verbose_name = "Range Type"

107

108 -class RangeType2Band(models.Model):

109 band = models.ForeignKey(BandRecord) 110 range_type = models.ForeignKey(RangeTypeRecord) 111 no = models.PositiveIntegerField() 112

113 - class Meta:

114 verbose_name = "Band in Range type"

115

116 -class ExtentRecord(models.Model):

117 srid = models.IntegerField("SRID") 118 size_x = models.IntegerField() 119 size_y = models.IntegerField() 120 minx = models.FloatField() 121 miny = models.FloatField() 122 maxx = models.FloatField() 123 maxy = models.FloatField() 124

125 - def __unicode__(self):

126 return "Extent (SRID=%d; %f, %f, %f, %f; Size: %d x %d)" % ( 127 self.srid, self.minx, self.miny, self.maxx, self.maxy, self.size_x, self.size_y 128 )

129

130 - class Meta:

131 verbose_name = "Extent"

132

133 -class LayerMetadataRecord(models.Model):

134 key = models.CharField(max_length=256) 135 value = models.TextField() 136

137 - def __unicode__(self):

138 return self.key

139

140 - class Meta:

141 verbose_name = "Layer Metadata" 142 verbose_name_plural = "Layer Metadata"

143

144 -class LineageRecord(models.Model):

145

146 - class Meta:

147 verbose_name = "Lineage Entry" 148 verbose_name_plural = "Lineage Entries"

149

150 -class EOMetadataRecord(models.Model):

151 timestamp_begin = models.DateTimeField("Begin of acquisition") 152 timestamp_end = models.DateTimeField("End of acquisition") 153 footprint = models.MultiPolygonField(srid=4326) 154 eo_gml = models.TextField("EO O&M", blank=True, validators=[validateEOOM]) # validate against schema 155 objects = models.GeoManager() 156

157 - class Meta:

158 verbose_name = "EO Metadata Entry" 159 verbose_name_plural = "EO Metadata Entries"

160

161 - def __unicode__(self):

162 return ("BeginTime: %s" % self.timestamp_begin)

163 164

165 - def save(self, *args, **kwargs):

166 from eoxserver.core.util.timetools import UTCOffsetTimeZoneInfo 167 if self.timestamp_begin.tzinfo is None: 168 dt = self.timestamp_begin.replace(tzinfo=UTCOffsetTimeZoneInfo()) 169 self.timestamp_begin = dt.astimezone(UTCOffsetTimeZoneInfo()) 170 171 if self.timestamp_end.tzinfo is None: 172 dt = self.timestamp_end.replace(tzinfo=UTCOffsetTimeZoneInfo()) 173 self.timestamp_end = dt.astimezone(UTCOffsetTimeZoneInfo()) 174 models.Model.save(self, *args, **kwargs)

175

176 - def clean(self):

177 pass

178

179 -class DataSource(models.Model): # Maybe make two sub models for local and remote storages.

180 """ 181 182 """ 183 location = models.ForeignKey(Location, related_name="data_sources") 184 search_pattern = models.CharField(max_length=1024, null=True) 185

186 - class Meta:

187 unique_together = ('location', 'search_pattern')

188

189 - def __unicode__(self):

190 return "%s: %s" % (str(self.location), self.search_pattern)

191

192 -class DataPackage(models.Model):

193 data_package_type = models.CharField(max_length=32, editable=False) 194 metadata_format_name = models.CharField(max_length=128, null=True, blank=True) 195

196 - def __unicode__(self):

197 if self.data_package_type == "local": 198 return "Local Data Package: %s / %s" % ( 199 self.localdatapackage.data_location, 200 self.localdatapackage.metadata_location, 201 ) 202 203 elif self.data_package_type == "remote": 204 return "Remote Data Package: %s / %s" % ( 205 self.remotedatapackage.data_location, 206 self.remotedatapackage.metadata_location, 207 ) 208 elif self.data_package_type == "rasdaman": 209 return "Rasdaman Data Package: %s / %s" % ( 210 self.rasdamandatapackage.data_location, 211 self.rasdamandatapackage.metadata_location, 212 ) 213 else: 214 return "Unknown location type"

215 216

217 -class LocalDataPackage(DataPackage):

218 DATA_PACKAGE_TYPE = "local" 219 220 data_location = models.ForeignKey(LocalPath, related_name="data_file_packages") 221 metadata_location = models.ForeignKey(LocalPath, related_name="metadata_file_packages", null=True) 222 source_format = models.CharField( max_length=64, null=False, blank=False, validators = [ MIMETypeValidator ] )

223

224 -class RemoteDataPackage(DataPackage):

225 DATA_PACKAGE_TYPE = "remote" 226 227 data_location = models.ForeignKey(RemotePath, related_name="data_file_packages") 228 metadata_location = models.ForeignKey(RemotePath, related_name="metadata_file_packages", null=True) 229 source_format = models.CharField( max_length=64, null=False, blank=False, validators = [ MIMETypeValidator ] ) 230 231 cache_file = models.ForeignKey(CacheFile, related_name="remote_data_packages", null=True) 232

233 - def delete(self):

234 cache_file = self.cache_file 235 super(RemoteDataPackage, self).delete() 236 cache_file.delete()

237

238 -class RasdamanDataPackage(DataPackage):

239 DATA_PACKAGE_TYPE = "rasdaman" 240 241 data_location = models.ForeignKey(RasdamanLocation, related_name="data_packages") 242 metadata_location = models.ForeignKey(LocalPath, related_name="rasdaman_metadata_file_packages", null=True)

243

244 -class TileIndex(models.Model):

245 storage_dir = models.CharField(max_length=1024) 246

247 - class Meta:

248 verbose_name = "Tile Index" 249 verbose_name_plural = "Tile Indices"

250

251 -class ReservedCoverageIdRecord(models.Model):

252 until = models.DateTimeField() 253 request_id = models.CharField(max_length=256, null=True) 254 coverage_id = models.CharField("Coverage ID", max_length=256, unique=True, validators=[NCNameValidator])

255

256 -class CoverageRecord(Resource):

257 coverage_id = models.CharField("Coverage ID", max_length=256, unique=True, validators=[NCNameValidator]) 258 259 range_type = models.ForeignKey(RangeTypeRecord, on_delete=models.PROTECT) 260 layer_metadata = models.ManyToManyField(LayerMetadataRecord, null=True, blank=True) 261 automatic = models.BooleanField(default=False) # True means that the dataset was automatically generated from a dataset series's data dir 262 data_source = models.ForeignKey(DataSource, related_name="%(class)s_set", null=True, blank=True, on_delete=models.SET_NULL) # Has to be set if automatic is true. 263

264 - def clean(self):

265 super(CoverageRecord, self).clean() 266 if self.automatic and self.data_source is None: 267 raise ValidationError('DataSource has to be set if automatic is true.')

268

269 - def __unicode__(self):

270 return self.coverage_id

271

272 -class PlainCoverageRecord(CoverageRecord):

273 extent = models.ForeignKey(ExtentRecord, related_name = "single_file_coverages") 274 data_package = models.ForeignKey(DataPackage, related_name="plain_coverages") 275

276 - class Meta:

277 verbose_name = "Single File Coverage" 278 verbose_name_plural = "Single File Coverages"

279

280 - def delete(self):

281 extent = self.extent 282 super(PlainCoverageRecord, self).delete() 283 extent.delete()

284

285 -class EOCoverageMixIn(models.Model):

286 eo_id = models.CharField("EO ID", max_length=256, unique=True, validators=[NCNameValidator]) 287 eo_metadata = models.OneToOneField(EOMetadataRecord, 288 related_name="%(class)s_set", 289 verbose_name="EO Metadata Entry") 290 lineage = models.OneToOneField(LineageRecord, related_name="%(class)s_set") 291

292 - class Meta:

293 abstract = True

294

295 - def delete(self):

296 eo_metadata = self.eo_metadata 297 lineage = self.lineage 298 super(EOCoverageMixIn, self).delete() 299 eo_metadata.delete() 300 lineage.delete()

301

302 -class EODatasetMixIn(EOCoverageMixIn):

303 data_package = models.ForeignKey(DataPackage, related_name="%(class)s_set") 304 visible = models.BooleanField(default=False) # True means that the dataset is visible in the GetCapabilities response 305

306 - def __unicode__(self):

307 return self.eo_id

308

309 - class Meta:

310 abstract=True

311

312 - def delete(self):

313 data_package = self.data_package 314 super(EODatasetMixIn, self).delete() 315 data_package.delete()

316

317 -def _checkFootprint(footprint, extent):

318 """ Check footprint to match the extent. """ 319 320 #TODO: Make the rtol value configurable. 321 # allow footprint to exceed extent by given % of smaller extent size 322 rtol = 0.005 # .5% 323 difx = abs(extent.maxx - extent.minx) 324 dify = abs(extent.maxy - extent.miny) 325 atol = rtol * min(difx, dify) 326 327 try: 328 bbox = Polygon.from_bbox((extent.minx, extent.miny, extent.maxx, extent.maxy)) 329 bbox.srid = int(extent.srid) 330 331 bbox_ll = bbox.transform(footprint.srs, clone=True) 332 333 normalized_space = Polygon.from_bbox((-180, -90, 180, 90)) 334 non_normalized_space = Polygon.from_bbox((180, -90, 360, 90)) 335 336 normalized_space.srid = int(extent.srid) 337 non_normalized_space.srid = int(extent.srid) 338 339 if not normalized_space.contains(bbox_ll): 340 # create 2 bboxes for each side of the date line 341 bbox_ll1 = bbox_ll.intersection(normalized_space) 342 bbox_ll2 = bbox_ll.intersection(non_normalized_space) 343 344 bbox_ll2 = Polygon(LinearRing([Point(x - 360, y) for x, y in bbox_ll2.exterior_ring]), srid=bbox_ll2.srid) 345 346 bbox_ll1.transform(extent.srid) 347 bbox_ll2.transform(extent.srid) 348 349 e1 = bbox_ll1.extent 350 e2 = bbox_ll2.extent 351 352 bbox = MultiPolygon( 353 Polygon.from_bbox((e1[0] - atol, e2[1] - atol, e1[2] + atol, e1[3] + atol)), 354 Polygon.from_bbox((e2[0] - atol, e2[1] - atol, e2[2] + atol, e2[3] + atol)), 355 srid=extent.srid 356 ) 357 else: 358 # just use the tolerance for a slightly larger bbox 359 bbox = Polygon.from_bbox((extent.minx - atol, extent.miny - atol, 360 extent.maxx + atol, extent.maxy + atol)) 361 bbox.srid = int(extent.srid) 362 363 if footprint.srid != bbox.srid: 364 footprint_bboxsrs = footprint.transform(bbox.srs, clone=True) 365 else: 366 footprint_bboxsrs = footprint 367 368 logger.debug("Extent: %s" % bbox.wkt) 369 logger.debug("Footprint: %s" % footprint_bboxsrs.wkt) 370 371 if not bbox.contains(footprint_bboxsrs): 372 raise ValidationError("The datasets's extent does not surround its" 373 " footprint. Extent: '%s' Footprint: '%s'." 374 % (bbox.wkt, footprint_bboxsrs.wkt) 375 ) 376 377 except GEOSException: 378 pass

379 380

381 -class RectifiedDatasetRecord(CoverageRecord, EODatasetMixIn):

382 extent = models.ForeignKey(ExtentRecord, related_name="rect_datasets") 383

384 - class Meta:

385 verbose_name = "Rectified Dataset" 386 verbose_name_plural = "Rectified Datasets"

387

388 - def clean(self):

389 super(RectifiedDatasetRecord, self).clean() 390 391 # TODO: this does not work in the admins changelist.save method 392 # A wrong WKB is inside the eo_metadata.footprint entry 393 _checkFootprint( self.eo_metadata.footprint , self.extent ) 394 395 validateCoverageIDnotInEOOM(self.coverage_id, self.eo_metadata.eo_gml)

396

397 - def delete(self):

398 extent = self.extent 399 super(RectifiedDatasetRecord, self).delete() 400 extent.delete()

401

402 -class ReferenceableDatasetRecord(CoverageRecord, EODatasetMixIn):

403 extent = models.ForeignKey(ExtentRecord, related_name="refa_datasets") 404

405 - class Meta:

406 verbose_name = "Referenceable Dataset" 407 verbose_name_plural = "Referenceable Datasets"

408

409 - def clean(self):

410 super(ReferenceableDatasetRecord, self).clean() 411 412 # TODO: taken from Rectified DS, check if applicable to Referenceable DS too 413 # TODO: this does not work in the admins changelist.save method 414 # A wrong WKB is inside the eo_metadata.footprint entry 415 _checkFootprint( self.eo_metadata.footprint , self.extent ) 416 417 validateCoverageIDnotInEOOM(self.coverage_id, self.eo_metadata.eo_gml)

418

419 - def delete(self):

420 extent = self.extent 421 super(EOCoverageMixIn, self).delete() 422 extent.delete()

423

424 -class RectifiedStitchedMosaicRecord(CoverageRecord, EOCoverageMixIn):

425 extent = models.ForeignKey(ExtentRecord, related_name="rect_stitched_mosaics") 426 data_sources = models.ManyToManyField(DataSource, 427 null=True, blank=True, 428 related_name="rect_stitched_mosaics") 429 tile_index = models.ForeignKey(TileIndex, related_name="rect_stitched_mosaics") 430 rect_datasets = models.ManyToManyField(RectifiedDatasetRecord, 431 null=True, blank=True, 432 related_name="rect_stitched_mosaics", 433 verbose_name="Rectified Dataset(s)") 434

435 - def __unicode__(self):

436 return self.eo_id

437

438 - class Meta:

439 verbose_name = "Stitched Mosaic" 440 verbose_name_plural = "Stitched Mosaics"

441

442 - def clean(self):

443 super(RectifiedStitchedMosaicRecord, self).clean() 444 445 footprint = self.eo_metadata.footprint 446 bbox = Polygon.from_bbox((self.extent.minx, self.extent.miny, 447 self.extent.maxx, self.extent.maxy)) 448 bbox.set_srid(int(self.extent.srid)) 449 450 if footprint.srid != bbox.srid: 451 footprint.transform(bbox.srs) 452 453 if not bbox.contains(footprint): 454 raise ValidationError( 455 "Extent does not surround footprint. Extent: '%s' Footprint: " 456 "'%s'" % (str(bbox), str(footprint)) 457 ) 458 459 validateCoverageIDnotInEOOM(self.coverage_id, self.eo_metadata.eo_gml)

460

461 - def delete(self):

462 tile_index = self.tile_index 463 # TODO maybe delete only automatic datasets? 464 for dataset in self.rect_datasets.all(): 465 dataset.delete() 466 super(RectifiedStitchedMosaicRecord, self).delete() 467 tile_index.delete()

468

469 -class DatasetSeriesRecord(Resource):

470 eo_id = models.CharField("EO ID", max_length=256, unique=True, validators=[NCNameValidator]) 471 eo_metadata = models.OneToOneField(EOMetadataRecord, 472 related_name="dataset_series_set", 473 verbose_name="EO Metadata Entry") 474 data_sources = models.ManyToManyField(DataSource, 475 null=True, blank=True, 476 related_name="dataset_series_set") 477 rect_stitched_mosaics = models.ManyToManyField(RectifiedStitchedMosaicRecord, 478 blank=True, null=True, 479 related_name="dataset_series_set", 480 verbose_name="Stitched Mosaic(s)") 481 rect_datasets = models.ManyToManyField(RectifiedDatasetRecord, 482 blank=True, null=True, 483 related_name="dataset_series_set", 484 verbose_name="Rectified Dataset(s)") 485 ref_datasets = models.ManyToManyField(ReferenceableDatasetRecord, 486 blank=True, null=True, 487 related_name="dataset_series_set", 488 verbose_name="Referenceable Dataset(s)") 489 490 layer_metadata = models.ManyToManyField(LayerMetadataRecord, null=True, blank=True) 491

492 - def __unicode__(self):

493 return self.eo_id

494

495 - class Meta:

496 verbose_name = "Dataset Series" 497 verbose_name_plural = "Dataset Series"

498

499 - def delete(self):

500 eo_metadata = self.eo_metadata 501 for dataset in self.rect_datasets.filter(automatic=True): 502 if dataset.dataset_series_set.count() == 1 and \ 503 dataset.rect_stitched_mosaics.count() == 0: 504 dataset.delete() 505 for dataset in self.ref_datasets.filter(automatic=True): 506 if dataset.dataset_series_set.count() == 1: 507 dataset.delete() 508 super(DatasetSeriesRecord, self).delete() 509 eo_metadata.delete()

510

Source Code for Module eoxserver.resources.coverages.models