From 393e2226f672a962c02b83be2cf850ca09f9b63c Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Dec 2023 16:26:41 +0100 Subject: [PATCH 1/8] wip: major refactoring again new stam-rust API --- src/annotation.rs | 638 +++++++++++++++++++++++++++++---------- src/annotationdata.rs | 337 +++++++++++++++------ src/annotationdataset.rs | 74 ++++- src/annotationstore.rs | 80 +++-- src/iterparams.rs | 388 ++++++++++-------------- src/lib.rs | 11 - src/resources.rs | 175 +++++++---- src/textselection.rs | 461 ++++++++++++++++++++-------- stam.pyi | 166 ++++++---- 9 files changed, 1552 insertions(+), 778 deletions(-) diff --git a/src/annotation.rs b/src/annotation.rs index d2836ff..6050aba 100644 --- a/src/annotation.rs +++ b/src/annotation.rs @@ -10,7 +10,7 @@ use crate::annotationdata::{PyAnnotationData, PyData}; use crate::annotationdataset::PyAnnotationDataSet; use crate::annotationstore::MapStore; use crate::error::PyStamError; -use crate::iterparams::IterParams; +use crate::iterparams::*; use crate::resources::{PyOffset, PyTextResource}; use crate::selector::{PySelector, PySelectorKind}; use crate::textselection::{PyTextSelectionOperator, PyTextSelections}; @@ -128,9 +128,9 @@ impl PyAnnotation { /// Returns the text of the annotation. /// If the annotation references multiple text slices, they will be concatenated with a space as a delimiter, - /// but note that in reality the different parts may be non-contingent! + /// but note that in reality the different parts may be non-contingent or non-delimited! /// - /// Use `text()` instead to retrieve a list of texts + /// Use `text()` instead to retrieve a list of texts, which you can subsequently concatenate as you please. fn __str__(&self) -> PyResult { self.map(|annotation| { let elements: Vec<&str> = annotation.text().collect(); @@ -142,52 +142,144 @@ impl PyAnnotation { /// Returns a list of all textselections of the annotation. /// Note that this will always return a list (even it if only contains a single element), /// as an annotation may reference multiple text selections. - #[pyo3(signature = (**kwargs))] - fn textselections(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotation| { - let iter = annotation.textselections(); - iterparams.evaluate_to_pytextselections(iter, annotation.store(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn textselections(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotation| { + Ok(PyTextSelections::from_iter( + annotation.textselections().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::TextSelection, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |annotation, query| { + Ok(PyTextSelections::from_query( + query, + annotation.store(), + &self.store, + limit, + )) + }, + ) + } } /// Returns annotations this annotation refers to (i.e. using an AnnotationSelector) - #[pyo3(signature = (**kwargs))] - fn annotations_in_targets(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - let mut recursive: bool = false; - if let Some(kwargs) = kwargs { - if let Ok(Some(v)) = kwargs.get_item("recursive") { - if let Ok(v) = v.extract() { - recursive = v; - } - } + #[pyo3(signature = (*args, **kwargs))] + fn annotations_in_targets( + &self, + args: &PyList, + kwargs: Option<&PyDict>, + ) -> PyResult { + let limit = get_limit(kwargs); + let recursive = get_recursive(kwargs, false); + if !has_filters(args, kwargs) { + self.map(|annotation| { + Ok(PyAnnotations::from_iter( + annotation.annotations_in_targets(recursive).limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable( + "main", + if recursive { + AnnotationQualifier::RecursiveTarget + } else { + AnnotationQualifier::Target + }, + ), + args, + kwargs, + |annotation, query| { + Ok(PyAnnotations::from_query( + query, + annotation.store(), + &self.store, + limit, + )) + }, + ) } - self.map(|annotation| { - let iter = annotation.annotations_in_targets(recursive); - iterparams.evaluate_to_pyannotations(iter, annotation.store(), &self.store) - }) } /// Returns annotations that are referring to this annotation (i.e. others using an AnnotationSelector) - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotation| { - let iter = annotation.annotations(); - iterparams.evaluate_to_pyannotations(iter, annotation.store(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotation| { + Ok(PyAnnotations::from_iter( + annotation.annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |annotation, query| { + Ok(PyAnnotations::from_query( + query, + annotation.store(), + &self.store, + limit, + )) + }, + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotation| { - let iter = annotation.annotations(); - Ok(iterparams - .evaluate_annotations(iter, annotation.store())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|annotation| Ok(annotation.annotations().test())) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |annotation, query| Ok(annotation.store().query(query).test()), + ) + } + } + + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations_in_targets( + &self, + args: &PyList, + kwargs: Option<&PyDict>, + ) -> PyResult { + let recursive = get_recursive(kwargs, false); + if !has_filters(args, kwargs) { + self.map(|annotation| Ok(annotation.annotations_in_targets(recursive).test())) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable( + "main", + if recursive { + AnnotationQualifier::RecursiveTarget + } else { + AnnotationQualifier::Target + }, + ), + args, + kwargs, + |annotation, query| Ok(annotation.store().query(query).test()), + ) + } } /// Returns a list of resources this annotation refers to @@ -261,22 +353,47 @@ impl PyAnnotation { } /// Returns annotation data instances that pertain to this annotation. - #[pyo3(signature = (**kwargs))] - fn data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotation| { - let iter = annotation.data(); - iterparams.evaluate_to_pydata(iter, annotation.store(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotation| { + Ok(PyData::from_iter( + annotation.data().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::AnnotationData, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |annotation, query| { + Ok(PyData::from_query( + query, + annotation.store(), + &self.store, + limit, + )) + }, + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotation| { - let iter = annotation.data(); - Ok(iterparams.evaluate_data(iter, annotation.store())?.test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|annotation| Ok(annotation.data().test())) + } else { + self.map_with_query( + Type::AnnotationData, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |annotation, query| Ok(annotation.store().query(query).test()), + ) + } } /// Returns the number of data items under this annotation @@ -285,17 +402,40 @@ impl PyAnnotation { .unwrap() } - #[pyo3(signature = (operator, **kwargs))] + #[pyo3(signature = (operator, *args, **kwargs))] fn related_text( &self, operator: PyTextSelectionOperator, + args: &PyList, kwargs: Option<&PyDict>, ) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotation| { - let iter = annotation.related_text(operator.operator); - iterparams.evaluate_to_pytextselections(iter, annotation.rootstore(), &self.store) - }) + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotation| { + Ok(PyTextSelections::from_iter( + annotation.related_text(operator.operator).limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::TextSelection, + Constraint::TextRelation { + var: "main", + operator: operator.operator, //MAYBE TODO: check if we need to invert an operator here? + }, + args, + kwargs, + |annotation, query| { + Ok(PyTextSelections::from_query( + query, + annotation.store(), + &self.store, + limit, + )) + }, + ) + } } } @@ -304,8 +444,6 @@ pub struct PyAnnotations { pub(crate) annotations: Vec, pub(crate) store: Arc>, pub(crate) cursor: usize, - /// Indicates whether the annotations are sorted chronologically (by handle) - pub(crate) sorted: bool, } #[pymethods] @@ -344,87 +482,209 @@ impl PyAnnotations { !pyself.annotations.is_empty() } - fn is_sorted(pyself: PyRef<'_, Self>) -> bool { - pyself.sorted + /// Returns annotation data instances used by the annotations in this collection. + #[pyo3(signature = (*args, **kwargs))] + fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotations, store| { + Ok(PyData::from_iter( + annotations.items().data().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::AnnotationData, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |query, store| Ok(PyData::from_query(query, store, &self.store, limit)), + ) + } } - /// Returns annotation data instances used by the annotations in this collection. - #[pyo3(signature = (**kwargs))] - fn data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotations, store| { - let iter = Annotations::from_handles(Cow::Borrowed(annotations), self.sorted, store) - .iter() - .data(); - iterparams.evaluate_to_pydata(iter, store, &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|annotations, _| Ok(annotations.items().data().test())) + } else { + self.map_with_query( + Type::AnnotationData, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |query, store| Ok(store.query(query).test()), + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotations, store| { - let iter = Annotations::from_handles(Cow::Borrowed(annotations), self.sorted, store) - .iter() - .data(); - Ok(iterparams.evaluate_data(iter, store)?.test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotations, store| { + Ok(PyAnnotations::from_iter( + annotations.items().annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |query, store| Ok(PyAnnotations::from_query(query, store, &self.store, limit)), + ) + } } - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotations, store| { - let iter = Annotations::from_handles(Cow::Borrowed(annotations), self.sorted, store) - .iter() - .annotations(); - iterparams.evaluate_to_pyannotations(iter, store, &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|annotations, _| Ok(annotations.items().annotations().test())) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |query, store| Ok(store.query(query).test()), + ) + } } - #[pyo3(signature = (**kwargs))] - fn annotations_in_targets(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - let mut recursive: bool = false; - if let Some(kwargs) = kwargs { - if let Ok(Some(v)) = kwargs.get_item("recursive") { - if let Ok(v) = v.extract() { - recursive = v; - } - } + #[pyo3(signature = (*args, **kwargs))] + fn annotations_in_targets( + &self, + args: &PyList, + kwargs: Option<&PyDict>, + ) -> PyResult { + let limit = get_limit(kwargs); + let recursive = get_recursive(kwargs, false); + if !has_filters(args, kwargs) { + self.map(|annotations, store| { + Ok(PyAnnotations::from_iter( + annotations + .items() + .annotations_in_targets(recursive) + .limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable( + "main", + if recursive { + AnnotationQualifier::RecursiveTarget + } else { + AnnotationQualifier::Target + }, + ), + args, + kwargs, + |query, store| Ok(PyAnnotations::from_query(query, store, &self.store, limit)), + ) } - self.map(|annotations, store| { - let iter = Annotations::from_handles(Cow::Borrowed(annotations), self.sorted, store) - .iter() - .annotations_in_targets(recursive); - iterparams.evaluate_to_pyannotations(iter, store, &self.store) - }) } - #[pyo3(signature = (**kwargs))] - fn textselections(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotations, store| { - let iter = Annotations::from_handles(Cow::Borrowed(annotations), self.sorted, store) - .iter() - .textselections(); - iterparams.evaluate_to_pytextselections(iter, store, &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations_in_targets( + &self, + args: &PyList, + kwargs: Option<&PyDict>, + ) -> PyResult { + let recursive = get_recursive(kwargs, false); + if !has_filters(args, kwargs) { + self.map(|annotations, _| { + Ok(annotations.items().annotations_in_targets(recursive).test()) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::AnnotationVariable( + "main", + if recursive { + AnnotationQualifier::RecursiveTarget + } else { + AnnotationQualifier::Target + }, + ), + args, + kwargs, + |query, store| Ok(store.query(query).test()), + ) + } } - #[pyo3(signature = (operator, **kwargs))] + #[pyo3(signature = (*args,**kwargs))] + fn textselections(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotations, store| { + Ok(PyTextSelections::from_iter( + annotations.items().textselections().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::TextSelection, + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + args, + kwargs, + |query, store| { + Ok(PyTextSelections::from_query( + query, + store, + &self.store, + limit, + )) + }, + ) + } + } + + #[pyo3(signature = (operator, *args, **kwargs))] fn related_text( &self, operator: PyTextSelectionOperator, + args: &PyList, kwargs: Option<&PyDict>, ) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|annotations, store| { - let iter = Annotations::from_handles(Cow::Borrowed(annotations), self.sorted, store) - .iter() - .related_text(operator.operator); - iterparams.evaluate_to_pytextselections(iter, store, &self.store) - }) + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|annotations, store| { + Ok(PyTextSelections::from_iter( + annotations + .items() + .related_text(operator.operator) + .limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::TextSelection, + Constraint::TextRelation { + var: "main", + operator: operator.operator, //MAYBE TODO: check if we need to invert an operator here? + }, + args, + kwargs, + |query, store| { + Ok(PyTextSelections::from_query( + query, + store, + &self.store, + limit, + )) + }, + ) + } } fn textual_order(mut pyself: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { @@ -447,12 +707,49 @@ impl PyAnnotations { } impl PyAnnotations { - fn map(&self, f: F) -> Result + pub(crate) fn from_iter<'store>( + iter: impl Iterator>, + wrappedstore: &Arc>, + ) -> Self { + Self { + annotations: iter.map(|x| x.handle()).collect(), + store: wrappedstore.clone(), + cursor: 0, + } + } + + pub(crate) fn from_query<'store>( + query: Query<'store>, + store: &'store AnnotationStore, + wrappedstore: &Arc>, + limit: Option, + ) -> Self { + assert!(query.resulttype() == Some(Type::Annotation)); + Self { + annotations: store + .query(query) + .limit(limit) + .map(|resultitems| { + //we use the deepest item if there are multiple + if let Some(QueryResultItem::Annotation(annotation)) = resultitems.pop_last() { + annotation.handle() + } else { + unreachable!("Unexpected QueryResultItem"); + } + }) + .collect(), + store: wrappedstore.clone(), + cursor: 0, + } + } + + fn map<'store, T, F>(&self, f: F) -> Result where - F: FnOnce(&Vec, &AnnotationStore) -> Result, + F: FnOnce(Handles<'store, Annotation>, &'store AnnotationStore) -> Result, { if let Ok(store) = self.store.read() { - f(&self.annotations, &store).map_err(|err| PyStamError::new_err(format!("{}", err))) + let handles = Annotations::new(Cow::Borrowed(&self.annotations), false, &store); + f(handles, &store).map_err(|err| PyStamError::new_err(format!("{}", err))) } else { Err(PyRuntimeError::new_err( "Unable to obtain store (should never happen)", @@ -472,6 +769,39 @@ impl PyAnnotations { )) } } + + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + constraint: Constraint<'a>, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result + where + F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, + { + self.map(|annotations, store| { + let query = Query::new(QueryType::Select, Some(Type::Annotation), Some("main")) + .with_constraint(Constraint::Handle(Filter::Annotations( + annotations, + FilterMode::Any, + ))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(constraint), + args, + kwargs, + store, + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(query, store) + }) + } } #[pyclass(name = "DataIter")] @@ -548,34 +878,34 @@ impl PyAnnotation { )) } } -} -impl<'py> IterParams<'py> { - pub(crate) fn evaluate_to_pyannotations<'store>( - self, - iter: AnnotationsIter<'store>, - store: &'store AnnotationStore, - wrappedstore: &Arc>, - ) -> Result + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + constraint: Constraint<'a>, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result where - 'py: 'store, + F: FnOnce(ResultItem<'a, Annotation>, Query<'a>) -> Result, { - let limit = self.limit(); - match self.evaluate_annotations(iter, store) { - Ok(iter) => { - let sorted = iter.returns_sorted(); - Ok(PyAnnotations { - annotations: if let Some(limit) = limit { - iter.to_collection_limit(limit).take() - } else { - iter.to_collection().take() - }, - store: wrappedstore.clone(), - cursor: 0, - sorted, - }) - } - Err(e) => Err(e.into()), - } + self.map(|annotation| { + let query = Query::new(QueryType::Select, Some(Type::Annotation), Some("main")) + .with_constraint(Constraint::Handle(Filter::Annotation(annotation.handle()))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(constraint), + args, + kwargs, + annotation.store(), + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(annotation, query) + }) } } diff --git a/src/annotationdata.rs b/src/annotationdata.rs index 3da5173..c8c9906 100644 --- a/src/annotationdata.rs +++ b/src/annotationdata.rs @@ -11,7 +11,7 @@ use crate::annotation::PyAnnotations; use crate::annotationdataset::PyAnnotationDataSet; use crate::annotationstore::MapStore; use crate::error::PyStamError; -use crate::iterparams::IterParams; +use crate::iterparams::*; use stam::*; #[pyclass(dict, module = "stam", name = "DataKey")] @@ -81,42 +81,65 @@ impl PyDataKey { }) } - #[pyo3(signature = (**kwargs))] - fn data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|key| { - let iter = key.data(); - iterparams.evaluate_to_pydata(iter, key.rootstore(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|key| Ok(PyData::from_iter(key.data().limit(limit), &self.store))) + } else { + self.map_with_query(Type::AnnotationData, args, kwargs, |key, query| { + Ok(PyData::from_query( + query, + key.rootstore(), + &self.store, + limit, + )) + }) + } } - #[pyo3(signature = (**kwargs))] - fn test_data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|key| { - let iter = key.data(); - Ok(iterparams.evaluate_data(iter, key.rootstore())?.test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|key| Ok(key.data().test())) + } else { + self.map_with_query(Type::AnnotationData, args, kwargs, |key, query| { + Ok(key.rootstore().query(query).test()) + }) + } } - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|key| { - let iter = key.annotations(); - iterparams.evaluate_to_pyannotations(iter, key.rootstore(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|key| { + Ok(PyAnnotations::from_iter( + key.annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query(Type::Annotation, args, kwargs, |key, query| { + Ok(PyAnnotations::from_query( + query, + key.rootstore(), + &self.store, + limit, + )) + }) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|key| { - let iter = key.annotations(); - Ok(iterparams - .evaluate_annotations(iter, key.rootstore())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|key| Ok(key.annotations().test())) + } else { + self.map_with_query(Type::Annotation, args, kwargs, |key, query| { + Ok(key.rootstore().query(query).test()) + }) + } } fn annotations_count(&self) -> usize { @@ -152,6 +175,38 @@ impl PyDataKey { )) } } + + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result + where + F: FnOnce(ResultItem<'a, DataKey>, Query<'a>) -> Result, + { + self.map(|data| { + let query = Query::new(QueryType::Select, Some(Type::DataKey), Some("main")) + .with_constraint(Constraint::Handle(Filter::DataKey( + data.set().handle(), + data.handle(), + ))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(Constraint::DataKeyVariable("main")), + args, + kwargs, + data.rootstore(), + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(data, query) + }) + } } #[pyclass(dict, module = "stam", name = "AnnotationData")] @@ -396,24 +451,37 @@ impl PyAnnotationData { Ok(PyAnnotationDataSet::new(self.set, &self.store)) } - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|data| { - let iter = data.annotations(); - iterparams.evaluate_to_pyannotations(iter, data.rootstore(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|data| { + Ok(PyAnnotations::from_iter( + data.annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query(Type::Annotation, args, kwargs, |data, query| { + Ok(PyAnnotations::from_query( + query, + data.rootstore(), + &self.store, + limit, + )) + }) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|data| { - let iter = data.annotations(); - Ok(iterparams - .evaluate_annotations(iter, data.rootstore())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|key| Ok(key.annotations().test())) + } else { + self.map_with_query(Type::Annotation, args, kwargs, |key, query| { + Ok(key.rootstore().query(query).test()) + }) + } } fn annotations_len(&self) -> usize { @@ -449,6 +517,38 @@ impl PyAnnotationData { )) } } + + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result + where + F: FnOnce(ResultItem<'a, AnnotationData>, Query<'a>) -> Result, + { + self.map(|data| { + let query = Query::new(QueryType::Select, Some(Type::AnnotationData), Some("main")) + .with_constraint(Constraint::Handle(Filter::AnnotationData( + data.set().handle(), + data.handle(), + ))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(Constraint::DataVariable("main")), + args, + kwargs, + data.rootstore(), + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(data, query) + }) + } } /// Build an AnnotationDataBuilder from a python dictionary (or string referring to an existing public ID) @@ -745,8 +845,6 @@ pub struct PyData { pub(crate) data: Vec<(AnnotationDataSetHandle, AnnotationDataHandle)>, pub(crate) store: Arc>, pub(crate) cursor: usize, - /// Indicates whether the annotations are sorted chronologically (by handle) - pub(crate) sorted: bool, } #[pymethods] @@ -785,73 +883,114 @@ impl PyData { !pyself.data.is_empty() } - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|data, store| { - let iter = Data::from_handles(Cow::Borrowed(data), self.sorted, store) - .iter() - .annotations(); - iterparams.evaluate_to_pyannotations(iter, store, &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|data, store| { + Ok(PyAnnotations::from_iter( + data.items().annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query(Type::Annotation, args, kwargs, |query, store| { + Ok(PyAnnotations::from_query(query, store, &self.store, limit)) + }) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|data, store| { - let iter = Data::from_handles(Cow::Borrowed(data), self.sorted, store) - .iter() - .annotations(); - Ok(iterparams.evaluate_annotations(iter, store)?.test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|data, _| Ok(data.items().annotations().test())) + } else { + self.map_with_query(Type::Annotation, args, kwargs, |query, store| { + Ok(store.query(query).test()) + }) + } } } impl PyData { - fn map(&self, f: F) -> Result + pub(crate) fn from_iter<'store>( + iter: impl Iterator>, + wrappedstore: &Arc>, + ) -> Self { + Self { + data: iter + .map(|item| (item.set().handle(), item.handle())) + .collect(), + store: wrappedstore.clone(), + cursor: 0, + } + } + + pub(crate) fn from_query<'store>( + query: Query<'store>, + store: &'store AnnotationStore, + wrappedstore: &Arc>, + limit: Option, + ) -> Self { + assert!(query.resulttype() == Some(Type::Annotation)); + Self { + data: store + .query(query) + .limit(limit) + .map(|resultitems| { + //we use the deepest item if there are multiple + if let Some(QueryResultItem::AnnotationData(data)) = resultitems.pop_last() { + (data.set().handle(), data.handle()) + } else { + unreachable!("Unexpected QueryResultItem"); + } + }) + .collect(), + store: wrappedstore.clone(), + cursor: 0, + } + } + + fn map<'store, T, F>(&self, f: F) -> Result where - F: FnOnce( - &Vec<(AnnotationDataSetHandle, AnnotationDataHandle)>, - &AnnotationStore, - ) -> Result, + F: FnOnce(Handles<'store, AnnotationData>, &'store AnnotationStore) -> Result, { if let Ok(store) = self.store.read() { - f(&self.data, &store).map_err(|err| PyStamError::new_err(format!("{}", err))) + let handles = Data::new(Cow::Borrowed(&self.data), false, &store); + f(handles, &store).map_err(|err| PyStamError::new_err(format!("{}", err))) } else { Err(PyRuntimeError::new_err( "Unable to obtain store (should never happen)", )) } } -} -impl<'py> IterParams<'py> { - pub(crate) fn evaluate_to_pydata<'store>( - self, - iter: DataIter<'store>, - store: &'store AnnotationStore, - wrappedstore: &Arc>, - ) -> Result + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result where - 'py: 'store, + F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, { - let limit = self.limit(); - match self.evaluate_data(iter, store) { - Ok(iter) => { - let sorted = iter.returns_sorted(); - Ok(PyData { - data: if let Some(limit) = limit { - iter.to_collection_limit(limit).take() - } else { - iter.to_collection().take() - }, - store: wrappedstore.clone(), - cursor: 0, - sorted, - }) - } - Err(e) => Err(e.into()), - } + self.map(|data, store| { + let query = Query::new(QueryType::Select, Some(Type::AnnotationData), Some("main")) + .with_constraint(Constraint::Handle(Filter::Data(data, FilterMode::Any))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(Constraint::DataVariable("main")), + args, + kwargs, + store, + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(query, store) + }) } } diff --git a/src/annotationdataset.rs b/src/annotationdataset.rs index c075c79..8c637ee 100644 --- a/src/annotationdataset.rs +++ b/src/annotationdataset.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, RwLock}; use crate::annotationdata::{datavalue_from_py, PyAnnotationData, PyData, PyDataKey}; use crate::error::PyStamError; -use crate::iterparams::IterParams; +use crate::iterparams::*; use crate::selector::{PySelector, PySelectorKind}; use stam::*; @@ -172,22 +172,32 @@ impl PyAnnotationDataSet { }) } - #[pyo3(signature = (**kwargs))] - fn data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|dataset| { - let iter = dataset.data(); - iterparams.evaluate_to_pydata(iter, dataset.store(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|dataset| Ok(PyData::from_iter(dataset.data().limit(limit), &self.store))) + } else { + self.map_with_query(Type::AnnotationData, args, kwargs, |dataset, query| { + Ok(PyData::from_query( + query, + dataset.store(), + &self.store, + limit, + )) + }) + } } - #[pyo3(signature = (**kwargs))] - fn test_data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|dataset| { - let iter = dataset.data(); - Ok(iterparams.evaluate_data(iter, dataset.store())?.test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|dataset| Ok(dataset.data().test())) + } else { + self.map_with_query(Type::AnnotationData, args, kwargs, |dataset, query| { + Ok(dataset.store().query(query).test()) + }) + } } /// Returns a Selector (DataSetSelector) pointing to this AnnotationDataSet @@ -241,6 +251,40 @@ impl PyAnnotationDataSet { )) } } + + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result + where + F: FnOnce(ResultItem<'a, AnnotationDataSet>, Query<'a>) -> Result, + { + self.map(|dataset| { + let query = Query::new( + QueryType::Select, + Some(Type::AnnotationDataSet), + Some("main"), + ) + .with_constraint(Constraint::Handle(Filter::AnnotationDataSet( + dataset.handle(), + ))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")).with_constraint( + Constraint::AnnotationVariable("main", AnnotationQualifier::None), + ), + args, + kwargs, + dataset.store(), + ) + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))?, + ); + f(dataset, query) + }) + } } #[pyclass(name = "DataKeyIter")] diff --git a/src/annotationstore.rs b/src/annotationstore.rs index f431305..8294957 100644 --- a/src/annotationstore.rs +++ b/src/annotationstore.rs @@ -9,8 +9,7 @@ use crate::annotationdata::{annotationdata_builder, data_request_parser, PyData} use crate::annotationdataset::PyAnnotationDataSet; use crate::config::get_config; use crate::error::PyStamError; -use crate::get_limit; -use crate::iterparams::IterParams; +use crate::iterparams::*; use crate::resources::PyTextResource; use crate::selector::PySelector; use stam::*; @@ -247,13 +246,21 @@ impl PyAnnotationStore { }) } - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|store| { - let iter = store.annotations(); - iterparams.evaluate_to_pyannotations(iter, store, &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|store| { + Ok(PyAnnotations::from_iter( + store.annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query(Type::Annotation, args, kwargs, |query, store| { + Ok(PyAnnotations::from_query(query, store, &self.store, limit)) + }) + } } /// Returns a generator over all annotations in this store @@ -266,6 +273,7 @@ impl PyAnnotationStore { /// Returns a generator over all resources in this store fn resources(&self) -> PyResult { + //TODO: transform to PyResources Ok(PyResourceIter { store: self.store.clone(), index: 0, @@ -291,30 +299,16 @@ impl PyAnnotationStore { self.map_mut(|store| Ok(store.shrink_to_fit(true))) } - /// Find annotation data for the specified set, key and value - /// Returns all found AnnotationData instances - #[pyo3(signature = (**kwargs))] - fn find_data(&self, kwargs: Option<&PyDict>) -> PyResult { + #[pyo3(signature = (*args, **kwargs))] + fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); - self.map( - |store| match data_request_parser(kwargs, store, None, None) { - Ok((sethandle, keyhandle, op)) => { - let iter = store.find_data(sethandle, keyhandle, op); - let sorted = iter.returns_sorted(); - Ok(PyData { - data: if let Some(limit) = limit { - iter.to_collection_limit(limit).take() - } else { - iter.to_collection().take() - }, - store: self.store.clone(), - cursor: 0, - sorted, - }) - } - Err(e) => Err(e), - }, - ) + if !has_filters(args, kwargs) { + self.map(|store| Ok(PyData::from_iter(store.data().limit(limit), &self.store))) + } else { + self.map_with_query(Type::AnnotationData, args, kwargs, |query, store| { + Ok(PyData::from_query(query, store, &self.store, limit)) + }) + } } } @@ -374,6 +368,28 @@ impl PyAnnotationStore { { self.map_store_mut(f) } + + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result + where + F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, + { + self.map_store(|store| { + let query = build_query( + Query::new(QueryType::Select, Some(resulttype), None), + args, + kwargs, + store, + ) + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))?; + f(query, store) + }) + } } #[pyclass(name = "AnnotationIter")] diff --git a/src/iterparams.rs b/src/iterparams.rs index e10c3a3..f2f7a53 100644 --- a/src/iterparams.rs +++ b/src/iterparams.rs @@ -9,52 +9,59 @@ use crate::error::PyStamError; use crate::textselection::PyTextSelectionOperator; use stam::*; -pub enum Filter<'a> { - Annotation(AnnotationHandle), - Annotations((Vec, bool)), //the boolean expresses whether the data is sorted or not - AnnotationData(AnnotationDataSetHandle, AnnotationDataHandle), - Data((Vec<(AnnotationDataSetHandle, AnnotationDataHandle)>, bool)), //the boolean expresses whether the data is sorted or not - DataKey(AnnotationDataSetHandle, DataKeyHandle), - Value(DataOperator<'a>), - TextRelation(TextSelectionOperator), - //FindData(AnnotationDataSetHandle, DataKeyHandle, DataOperator<'a>), -} - -pub struct IterParams<'a> { +pub struct QueryBuilder<'a> { filters: Vec>, limit: Option, } -fn add_filter<'py>(filters: &mut Vec>, filter: &'py PyAny) -> PyResult<()> { +fn add_filter<'a>( + query: &mut Query<'a>, + store: &'a AnnotationStore, + filter: &'a PyAny, + operator: &mut Option, +) -> PyResult<()> { if filter.is_instance_of::() { let vec: Vec<&PyAny> = filter.extract()?; - add_multi_filter(filters, vec)?; + add_multi_filter(query, store, vec)?; } else if filter.is_instance_of::() { let vec: Vec<&PyAny> = filter.extract()?; - add_multi_filter(filters, vec)?; + add_multi_filter(query, store, vec)?; } else if filter.is_instance_of::() { let adata: PyRef<'_, PyAnnotationData> = filter.extract()?; - filters.push(Filter::AnnotationData(adata.set, adata.handle)); + query.constrain(Constraint::Filter(Filter::AnnotationData( + adata.set, + adata.handle, + ))); } else if filter.is_instance_of::() { let key: PyRef<'_, PyDataKey> = filter.extract()?; - filters.push(Filter::DataKey(key.set, key.handle)); + if operator.is_some() { + query.constrain(Constraint::Filter(Filter::DataKeyAndOperator( + key.set, + key.handle, + operator.take().unwrap(), + ))); + } else { + query.constrain(Constraint::Filter(Filter::DataKey(key.set, key.handle))); + } } else if filter.is_instance_of::() { let annotation: PyRef<'_, PyAnnotation> = filter.extract()?; - filters.push(Filter::Annotation(annotation.handle)); + query.constrain(Constraint::Filter(Filter::Annotation(annotation.handle))); } else if filter.is_instance_of::() { let operator: PyRef<'_, PyTextSelectionOperator> = filter.extract()?; - filters.push(Filter::TextRelation(operator.operator)); - } else if filter.is_instance_of::() { - let annotations: PyRef<'py, PyAnnotations> = filter.extract()?; - filters.push(Filter::Annotations(( - annotations.annotations.iter().copied().collect(), - annotations.sorted, + query.constrain(Constraint::Filter(Filter::TextSelectionOperator( + operator.operator, ))); + } else if filter.is_instance_of::() { + let annotations: PyRef<'a, PyAnnotations> = filter.extract()?; + query.constrain(Constraint::Filter(Filter::Annotations(Handles::from_iter( + annotations.annotations.iter().copied(), + store, + )))); } else if filter.is_instance_of::() { let data: PyRef<'_, PyData> = filter.extract()?; - filters.push(Filter::Data(( - data.data.iter().copied().collect(), - data.sorted, + query.constrain(Constraint::Filter(Filter::Data( + Handles::from_iter(data.data.iter().copied(), store), + FilterMode::Any, ))); } else { return Err(PyValueError::new_err( @@ -64,248 +71,175 @@ fn add_filter<'py>(filters: &mut Vec>, filter: &'py PyAny) -> PyResu Ok(()) } -fn add_multi_filter<'a>(filters: &mut Vec>, filter: Vec<&PyAny>) -> PyResult<()> { +fn add_multi_filter<'a>( + query: &mut Query<'a>, + store: &AnnotationStore, + filter: Vec<&PyAny>, +) -> PyResult<()> { if filter.iter().all(|x| x.is_instance_of::()) { - filters.push(Filter::Annotations(( - filter - .iter() - .map(|x| { - let annotation: PyRef<'_, PyAnnotation> = x.extract().unwrap(); - annotation.handle - }) - .collect(), - false, //we don't know if the data is sorted - ))); + query.constrain(Constraint::Filter(Filter::Annotations(Handles::from_iter( + filter.iter().map(|x| { + let annotation: PyRef<'_, PyAnnotation> = x.extract().unwrap(); + annotation.handle + }), + store, + )))); } else if filter .iter() .all(|x| x.is_instance_of::()) { - filters.push(Filter::Data(( - filter - .iter() - .map(|x| { + query.constrain(Constraint::Filter(Filter::Data( + Handles::from_iter( + filter.iter().map(|x| { let adata: PyRef<'_, PyAnnotationData> = x.extract().unwrap(); (adata.set, adata.handle) - }) - .collect(), - false, //we don't know if the data is sorted + }), + store, + ), + FilterMode::Any, ))); } Ok(()) } -impl<'py> IterParams<'py> { - pub fn new(kwargs: Option<&'py PyDict>) -> PyResult { - let mut filters = Vec::new(); - let mut limit: Option = None; - if let Some(kwargs) = kwargs { - if let Ok(Some(v)) = kwargs.get_item("limit") { - match v.extract() { - Ok(v) => limit = v, - Err(e) => { - return Err(PyValueError::new_err(format!( - "Limit should be an integer or None: {}", - e - ))) - } +pub(crate) fn build_query<'py>( + mut query: Query<'py>, + args: &'py PyList, //TODO: implement! + kwargs: Option<&'py PyDict>, + store: &'py AnnotationStore, +) -> PyResult> { + if let Some(kwargs) = kwargs { + let mut operator = + dataoperator_from_kwargs(kwargs).map_err(|e| PyStamError::new_err(format!("{}", e)))?; + if let Ok(Some(filter)) = kwargs.get_item("filter") { + add_filter(&mut query, store, filter, &mut operator)?; + } else if let Ok(Some(filter)) = kwargs.get_item("filters") { + if filter.is_instance_of::() { + let vec = filter.downcast::()?; + for filter in vec { + add_filter(&mut query, store, filter, &mut operator)?; + } + } else if filter.is_instance_of::() { + let vec = filter.downcast::()?; + for filter in vec { + add_filter(&mut query, store, filter, &mut operator)?; } } + } + //if the operator has not been consumed yet in an earlier add_filter step, add a constraint for it now: + if let Some(operator) = operator { + query.constrain(Constraint::Filter(Filter::DataOperator(operator))); + } + } + Ok(query) +} + +impl<'py> QueryBuilder<'py> { + pub fn new( + resulttype: Type, + store: &'py AnnotationStore, + kwargs: Option<&'py PyDict>, + name: Option<&'py str>, + ) -> PyResult> { + let mut query = Query::new(QueryType::Select, Some(resulttype), name); + + if let Some(kwargs) = kwargs { + let mut operator = dataoperator_from_kwargs(kwargs) + .map_err(|e| PyStamError::new_err(format!("{}", e)))?; if let Ok(Some(filter)) = kwargs.get_item("filter") { - add_filter(&mut filters, filter)?; + add_filter(&mut query, store, filter, &mut operator)?; } else if let Ok(Some(filter)) = kwargs.get_item("filters") { if filter.is_instance_of::() { let vec = filter.downcast::()?; for filter in vec { - add_filter(&mut filters, filter)?; + add_filter(&mut query, store, filter, &mut operator)?; } } else if filter.is_instance_of::() { let vec = filter.downcast::()?; for filter in vec { - add_filter(&mut filters, filter)?; + add_filter(&mut query, store, filter, &mut operator)?; } } } - if let Some(operator) = dataoperator_from_kwargs(kwargs) - .map_err(|e| PyStamError::new_err(format!("{}", e)))? - { - filters.push(Filter::Value(operator)); + //if the operator has not been consumed yet in an earlier add_filter step, add a constraint for it now: + if let Some(operator) = operator { + query.constrain(Constraint::Filter(Filter::DataOperator(operator))); } } - Ok(Self { filters, limit }) + Ok(query) } +} - pub fn limit(&self) -> Option { - self.limit +pub(crate) fn has_filters(args: &PyList, kwargs: Option<&PyDict>) -> bool { + if !args.is_empty() { + return true; } - - pub fn evaluate_annotations<'store>( - self, - mut iter: stam::AnnotationsIter<'store>, - store: &'store AnnotationStore, - ) -> Result, StamError> - where - 'py: 'store, - { - let mut has_value_filter = false; - let mut datakey_filter: Option> = None; - for filter in self.filters.iter() { - if let Filter::Value(op) = filter { - if let DataOperator::Any = op { - //ignore - } else { - has_value_filter = true; - } - } - } - for filter in self.filters.into_iter() { - match filter { - Filter::Annotation(handle) => { - iter = iter.filter_handle(handle); - } - Filter::AnnotationData(set_handle, data_handle) => { - iter = iter.filter_annotationdata_handle(set_handle, data_handle); - } - Filter::DataKey(set_handle, key_handle) => { - let dataset = store - .dataset(set_handle) - .ok_or_else(|| StamError::HandleError("Unable to find dataset"))?; - let key = dataset - .key(key_handle) - .ok_or_else(|| StamError::HandleError("Unable to find key"))?; - //check if we also have a value filter - if !has_value_filter { - iter = iter.filter_data(key.data().to_collection()); - } else { - datakey_filter = Some(key); //will be handled further in Filter::Value arm - } - } - Filter::Annotations((annotations, sorted)) => { - let annotations = - Annotations::from_handles(Cow::Owned(annotations), sorted, store); - iter = iter.filter_annotations(annotations.iter()); - } - Filter::Data((data, sorted)) => { - let data = Data::from_handles(Cow::Owned(data), sorted, store); - iter = iter.filter_data(data); - } - Filter::Value(op) => { - if let Some(key) = &datakey_filter { - iter = iter.filter_data(key.data().filter_value(op).to_collection()); - } else { - return Err(StamError::OtherError( - "Python: You can specify a value filter only if you pass filter=DataKey (Annotations)", - )); - } - } - Filter::TextRelation(op) => { - iter = iter.filter_related_text(op); - } + if let Some(kwargs) = kwargs { + for key in kwargs.keys() { + if let Ok(Some("limit")) | Ok(Some("recursive")) = key.extract() { + continue; //this doesn't count as a filter + } else { + return true; } } - Ok(iter) } + false +} - pub fn evaluate_data<'store>( - self, - mut iter: stam::DataIter<'store>, - store: &'store AnnotationStore, - ) -> Result, StamError> - where - 'py: 'store, - { - for filter in self.filters.into_iter() { - match filter { - Filter::Annotation(handle) => { - if let Some(annotation) = store.annotation(handle) { - iter = iter.filter_annotation(&annotation); - } - } - Filter::Annotations((annotations, sorted)) => { - let annotations = - Annotations::from_handles(Cow::Owned(annotations), sorted, store); - iter = iter.filter_annotations(annotations.iter()); - } - Filter::Data((data, sorted)) => { - let data = Data::from_handles(Cow::Owned(data), sorted, store); - iter = iter.filter_data(data.iter()); - } - Filter::DataKey(set_handle, key_handle) => { - iter = iter.filter_key_handle(set_handle, key_handle); - } - Filter::Value(operator) => { - iter = iter.filter_value(operator.clone()); - } - _ => { - return Err(StamError::OtherError( - "Python: not a valid filter in this context (Data)", - )); - } +pub(crate) fn get_recursive(kwargs: Option<&PyDict>, default: bool) -> bool { + if let Some(kwargs) = kwargs { + if let Ok(Some(v)) = kwargs.get_item("recursive") { + if let Ok(v) = v.extract::() { + return v; } } - Ok(iter) } + default +} - pub fn evaluate_textselections<'store>( - self, - mut iter: stam::TextSelectionsIter<'store>, - store: &'store AnnotationStore, - ) -> Result, StamError> - where - 'py: 'store, - { - let mut has_value_filter = false; - let mut datakey_filter: Option> = None; - for filter in self.filters.iter() { - if let Filter::Value(op) = filter { - if let DataOperator::Any = op { - //ignore - } else { - has_value_filter = true; - } +pub(crate) fn get_limit(kwargs: Option<&PyDict>) -> Option { + if let Some(kwargs) = kwargs { + if let Ok(Some(limit)) = kwargs.get_item("limit") { + if let Ok(limit) = limit.extract::() { + return Some(limit); } } - for filter in self.filters.into_iter() { - match filter { - Filter::TextRelation(op) => { - iter = iter.related_text(op); - } - Filter::Annotation(handle) => { - iter = iter.filter_annotation_handle(handle); - } - Filter::Annotations((annotations, sorted)) => { - let annotations = - Annotations::from_handles(Cow::Owned(annotations), sorted, store); - iter = iter.filter_annotations(annotations); - } - Filter::AnnotationData(set_handle, data_handle) => { - iter = iter.filter_annotationdata_handle(set_handle, data_handle); - } - Filter::Data((data, sorted)) => { - let data = Data::from_handles(Cow::Owned(data), sorted, store); - iter = iter.filter_data(data); - } - Filter::DataKey(set_handle, key_handle) => { - if let Some(dataset) = store.dataset(set_handle) { - if let Some(key) = dataset.key(key_handle) { - if !has_value_filter { - iter = iter.filter_data(key.data().to_collection()); - } else { - datakey_filter = Some(key); //will be handled further in Filter::Value arm - } - } - } - } - Filter::Value(operator) => { - if let Some(key) = &datakey_filter { - iter = iter.filter_data(key.data().filter_value(operator).to_collection()); - } else { - return Err(StamError::OtherError( - "Python: You can specify a value filter only if you pass filter=DataKey (TextSelections)", - )); - } - } + } + None +} + +pub(crate) struct LimitIter { + inner: I, + limit: Option, +} + +impl Iterator for LimitIter +where + I: Iterator, +{ + type Item = I::Item; + fn next(&mut self) -> Option { + if let Some(remainder) = self.limit.as_mut() { + if *remainder > 0 { + *remainder -= 1; + self.inner.next() + } else { + None } + } else { + self.inner.next() } - Ok(iter) } } + +pub(crate) trait LimitIterator +where + Self: Iterator, + Self: Sized, +{ + fn limit(self, limit: Option) -> LimitIter { + LimitIter { inner: self, limit } + } +} + +impl LimitIterator for I where I: Iterator {} diff --git a/src/lib.rs b/src/lib.rs index d85af2d..3731b40 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,14 +47,3 @@ fn stam(py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; Ok(()) } - -pub(crate) fn get_limit(kwargs: Option<&PyDict>) -> Option { - if let Some(kwargs) = kwargs { - if let Ok(Some(limit)) = kwargs.get_item("limit") { - if let Ok(limit) = limit.extract::() { - return Some(limit); - } - } - } - None -} diff --git a/src/resources.rs b/src/resources.rs index 48c57c4..6f0a428 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, RwLock}; use crate::annotation::PyAnnotations; use crate::error::PyStamError; -use crate::iterparams::IterParams; +use crate::iterparams::*; use crate::selector::{PySelector, PySelectorKind}; use crate::textselection::{ PyTextSelection, PyTextSelectionIter, PyTextSelectionOperator, PyTextSelections, @@ -379,67 +379,102 @@ impl PyTextResource { self.map(|res| res.beginaligned_cursor(&Cursor::EndAligned(endalignedcursor))) } - /// Returns all annotations (:obj:`Annotation`) that reference this resource via either a TextSelector or a ResourceSelector (if any). - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|resource| { - let iter = resource.annotations(); - iterparams.evaluate_to_pyannotations(iter, resource.store(), &self.store) - }) - } - - #[pyo3(signature = (**kwargs))] - fn annotations_as_metadata(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|resource| { - let iter = resource.annotations_as_metadata(); - iterparams.evaluate_to_pyannotations(iter, resource.store(), &self.store) - }) - } - - #[pyo3(signature = (**kwargs))] - fn annotations_on_text(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|resource| { - let iter = resource.annotations_on_text(); - iterparams.evaluate_to_pyannotations(iter, resource.store(), &self.store) - }) + /// Returns annotations that are referring to this resource via a TextSelector + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|resource| { + Ok(PyAnnotations::from_iter( + resource.annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::ResourceVariable("main", SelectionQualifier::Normal), + args, + kwargs, + |annotation, query| { + Ok(PyAnnotations::from_query( + query, + annotation.store(), + &self.store, + limit, + )) + }, + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|resource| { - let iter = resource.annotations(); - Ok(iterparams - .evaluate_annotations(iter, resource.store())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations_as_metadata( + &self, + args: &PyList, + kwargs: Option<&PyDict>, + ) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|resource| { + Ok(PyAnnotations::from_iter( + resource.annotations_as_metadata().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::ResourceVariable("main", SelectionQualifier::Metadata), + args, + kwargs, + |annotation, query| { + Ok(PyAnnotations::from_query( + query, + annotation.store(), + &self.store, + limit, + )) + }, + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations_as_metadata(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|resource| { - let iter = resource.annotations_as_metadata(); - Ok(iterparams - .evaluate_annotations(iter, resource.store())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|resource| Ok(resource.annotations().test())) + } else { + self.map_with_query( + Type::Annotation, + Constraint::ResourceVariable("main", SelectionQualifier::Normal), + args, + kwargs, + |resource, query| Ok(resource.store().query(query).test()), + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations_on_text(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|resource| { - let iter = resource.annotations_on_text(); - Ok(iterparams - .evaluate_annotations(iter, resource.store())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations_as_metadata( + &self, + args: &PyList, + kwargs: Option<&PyDict>, + ) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|resource| Ok(resource.annotations_as_metadata().test())) + } else { + self.map_with_query( + Type::Annotation, + Constraint::ResourceVariable("main", SelectionQualifier::Metadata), + args, + kwargs, + |resource, query| Ok(resource.store().query(query).test()), + ) + } } + /* /// Applies a `TextSelectionOperator` to find all other text selections that are in a specific /// relation with the reference selections (a list of :obj:`TextSelection` instances). Returns /// all matching TextSelections in a list @@ -451,7 +486,6 @@ impl PyTextResource { referenceselections: Vec, kwargs: Option<&PyDict>, ) -> PyResult { - let iterparams = IterParams::new(kwargs)?; self.map(|textselection| { let mut refset = TextSelectionSet::new(self.handle); refset.extend(referenceselections.into_iter().map(|x| x.textselection)); @@ -459,6 +493,7 @@ impl PyTextResource { iterparams.evaluate_to_pytextselections(iter, textselection.rootstore(), &self.store) }) } + */ } impl PyTextResource { @@ -478,6 +513,36 @@ impl PyTextResource { )) } } + + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + constraint: Constraint<'a>, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result + where + F: FnOnce(ResultItem<'a, TextResource>, Query<'a>) -> Result, + { + self.map(|resource| { + let query = Query::new(QueryType::Select, Some(Type::TextResource), Some("main")) + .with_constraint(Constraint::Handle(Filter::TextResource(resource.handle()))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(constraint), + args, + kwargs, + resource.store(), + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(resource, query) + }) + } } #[pyclass(dict, module = "stam", name = "Cursor")] diff --git a/src/textselection.rs b/src/textselection.rs index be0172a..31bc19f 100644 --- a/src/textselection.rs +++ b/src/textselection.rs @@ -2,13 +2,15 @@ use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::*; use pyo3::pyclass::CompareOp; use pyo3::types::*; +use std::borrow::Cow; use std::hash::{Hash, Hasher}; use std::ops::FnOnce; use std::sync::{Arc, RwLock}; use crate::annotation::PyAnnotations; +use crate::annotationdata::PyData; use crate::error::PyStamError; -use crate::iterparams::IterParams; +use crate::iterparams::*; use crate::resources::{PyOffset, PyTextResource}; use crate::textselection::TextSelectionHandle; use stam::*; @@ -294,48 +296,98 @@ impl PyTextSelection { }) } - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselection| { - let iter = textselection.annotations(); - iterparams.evaluate_to_pyannotations(iter, textselection.rootstore(), &self.store) - }) + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|textselection| { + Ok(PyAnnotations::from_iter( + textselection.annotations().limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::TextVariable("main"), + args, + kwargs, + |textselection, query| { + Ok(PyAnnotations::from_query( + query, + textselection.rootstore(), + &self.store, + limit, + )) + }, + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselection| { - let iter = textselection.annotations(); - Ok(iterparams - .evaluate_annotations(iter, textselection.rootstore())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|textselection| Ok(textselection.annotations().test())) + } else { + self.map_with_query( + Type::Annotation, + Constraint::TextVariable("main"), + args, + kwargs, + |textselection, query| Ok(textselection.rootstore().query(query).test()), + ) + } } - #[pyo3(signature = (**kwargs))] - fn test_data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselection| { - let iter = textselection.annotations().data_unchecked(); - Ok(iterparams - .evaluate_data(iter, textselection.rootstore())? - .test()) - }) + #[pyo3(signature = (*args, **kwargs))] + fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|textselection| Ok(textselection.annotations().data().test())) + } else { + self.map_with_query( + Type::AnnotationData, + Constraint::TextVariable("main"), + args, + kwargs, + |textselection, query| Ok(textselection.rootstore().query(query).test()), + ) + } } - #[pyo3(signature = (operator, **kwargs))] + #[pyo3(signature = (operator, *args, **kwargs))] fn related_text( &self, operator: PyTextSelectionOperator, + args: &PyList, kwargs: Option<&PyDict>, ) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselection| { - let iter = textselection.related_text(operator.operator); - iterparams.evaluate_to_pytextselections(iter, textselection.rootstore(), &self.store) - }) + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|textselection| { + Ok(PyTextSelections::from_iter( + textselection.related_text(operator.operator).limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::TextSelection, + Constraint::TextRelation { + var: "main", + operator: operator.operator, //MAYBE TODO: check if we need to invert an operator here? + }, + args, + kwargs, + |textselection, query| { + Ok(PyTextSelections::from_query( + query, + textselection.rootstore(), + &self.store, + limit, + )) + }, + ) + } } fn annotations_len(&self) -> usize { @@ -428,6 +480,41 @@ impl PyTextSelection { )) } } + + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + constraint: Constraint<'a>, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result + where + F: FnOnce(ResultTextSelection<'a>, Query<'a>) -> Result, + { + self.map(|textselection| { + let query = Query::new(QueryType::Select, Some(Type::TextSelection), Some("main")) + .with_constraint(Constraint::Handle(Filter::TextSelection( + textselection.resource().handle(), + textselection + .handle() + .expect("textselection must have handle"), + ))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(constraint), + args, + kwargs, + textselection.rootstore(), + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(textselection, query) + }) + } } impl From for TextSelection { @@ -453,21 +540,19 @@ impl PyTextSelections { fn __next__(mut pyself: PyRefMut<'_, Self>) -> Option { pyself.cursor += 1; //increment first (prevent exclusive mutability issues) pyself - .map( - |textselections: &Vec<(TextResourceHandle, TextSelectionHandle)>, store| { - //index is one ahead, prevents exclusive lock issues - if let Some((res_handle, handle)) = textselections.get(pyself.cursor - 1) { - let resource = store.get(*res_handle)?; - let textselection = resource.get(*handle)?; - return Ok(PyTextSelection::new( - textselection.clone(), - *res_handle, - &pyself.store, - )); - } - Err(StamError::HandleError("a handle did not resolve")) - }, - ) + .map(|textselections, store| { + //index is one ahead, prevents exclusive lock issues + if let Some((res_handle, handle)) = textselections.get(pyself.cursor - 1) { + let resource = store.get(res_handle)?; + let textselection = resource.get(handle)?; + return Ok(PyTextSelection::new( + textselection.clone(), + res_handle, + &pyself.store, + )); + } + Err(StamError::HandleError("a handle did not resolve")) + }) .ok() } @@ -503,81 +588,153 @@ impl PyTextSelections { } fn text_join(pyself: PyRef<'_, Self>, delimiter: &str) -> PyResult { - pyself.map(|textselections, store| { - let iter = stam::TextSelectionsIter::from_handles( - textselections.iter().copied().collect(), //MAYBE TODO: work away the extra copy - store, - ); - Ok(iter.text_join(delimiter)) + pyself.map(|textselections, _store| { + Ok(ResultTextSelections::new(textselections.items()).text_join(delimiter)) }) } fn text(pyself: PyRef<'_, Self>) -> PyResult> { - pyself.map(|textselections, store| { - let iter = stam::TextSelectionsIter::from_handles( - textselections.iter().copied().collect(), //MAYBE TODO: work away the extra copy - store, - ); - let v: Vec = iter.text().map(|s| s.to_string()).collect(); + pyself.map(|textselections, _store| { + let v: Vec = ResultTextSelections::new(textselections.items()) + .text() + .map(|s| s.to_string()) + .collect(); Ok(v) }) } - #[pyo3(signature = (**kwargs))] - fn annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselections, store| { - let iter = stam::TextSelectionsIter::from_handles( - textselections.iter().copied().collect(), //MAYBE TODO: work away the extra copy - store, + #[pyo3(signature = (*args, **kwargs))] + fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|textselections, store| { + Ok(PyAnnotations::from_iter( + textselections + .items() + .map(|x| x.as_resulttextselection()) + .annotations() + .limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::TextVariable("main"), + args, + kwargs, + |query, store| Ok(PyAnnotations::from_query(query, store, &self.store, limit)), ) - .annotations(); - iterparams.evaluate_to_pyannotations(iter, store, &self.store) - }) + } } - #[pyo3(signature = (**kwargs))] - fn test_annotations(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselections, store| { - let iter = stam::TextSelectionsIter::from_handles( - textselections.iter().copied().collect(), //MAYBE TODO: work away the extra copy - store, + #[pyo3(signature = (*args, **kwargs))] + fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|annotations, _| { + Ok(annotations + .items() + .map(|x| x.as_resulttextselection()) + .annotations() + .test()) + }) + } else { + self.map_with_query( + Type::Annotation, + Constraint::TextVariable("main"), + args, + kwargs, + |query, store| Ok(store.query(query).test()), ) - .annotations(); - Ok(iterparams.evaluate_annotations(iter, store)?.test()) - }) + } } - #[pyo3(signature = (**kwargs))] - fn test_data(&self, kwargs: Option<&PyDict>) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselections, store| { - let iter = stam::TextSelectionsIter::from_handles( - textselections.iter().copied().collect(), //MAYBE TODO: work away the extra copy - store, + #[pyo3(signature = (*args, **kwargs))] + fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|textselections, store| { + Ok(PyData::from_iter( + textselections + .items() + .map(|x| x.as_resulttextselection()) + .annotations() + .data() + .limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::AnnotationData, + Constraint::TextVariable("main"), + args, + kwargs, + |query, store| Ok(PyData::from_query(query, store, &self.store, limit)), ) - .annotations_unchecked() - .data_unchecked(); - Ok(iterparams.evaluate_data(iter, store)?.test()) - }) + } + } + + #[pyo3(signature = (*args, **kwargs))] + fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + if !has_filters(args, kwargs) { + self.map(|textselections, _| { + Ok(textselections + .items() + .map(|x| x.as_resulttextselection()) + .annotations() + .data() + .test()) + }) + } else { + self.map_with_query( + Type::AnnotationData, + Constraint::TextVariable("main"), + args, + kwargs, + |query, store| Ok(store.query(query).test()), + ) + } } - #[pyo3(signature = (operator, **kwargs))] + #[pyo3(signature = (operator, *args, **kwargs))] fn related_text( &self, operator: PyTextSelectionOperator, + args: &PyList, kwargs: Option<&PyDict>, ) -> PyResult { - let iterparams = IterParams::new(kwargs)?; - self.map(|textselections, store| { - let iter = stam::TextSelectionsIter::from_handles( - textselections.iter().copied().collect(), //MAYBE TODO: work away the extra copy - store, + let limit = get_limit(kwargs); + if !has_filters(args, kwargs) { + self.map(|textselections, store| { + Ok(PyTextSelections::from_iter( + textselections + .items() + .map(|x| x.as_resulttextselection()) + .related_text(operator.operator) + .limit(limit), + &self.store, + )) + }) + } else { + self.map_with_query( + Type::TextSelection, + Constraint::TextRelation { + var: "main", + operator: operator.operator, //MAYBE TODO: check if we need to invert an operator here? + }, + args, + kwargs, + |query, store| { + Ok(PyTextSelections::from_query( + query, + store, + &self.store, + limit, + )) + }, ) - .related_text(operator.operator); - iterparams.evaluate_to_pytextselections(iter, store, &self.store) - }) + } } fn textual_order(mut pyself: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { @@ -608,15 +765,61 @@ impl PyTextSelections { } impl PyTextSelections { - fn map(&self, f: F) -> Result + pub(crate) fn from_iter<'store>( + iter: impl Iterator>, + wrappedstore: &Arc>, + ) -> Self { + Self { + textselections: iter + .map(|textselection| { + ( + textselection.resource().handle(), + textselection.handle().expect("textselection must be bound"), + ) + }) + .collect(), + store: wrappedstore.clone(), + cursor: 0, + } + } + + pub(crate) fn from_query<'store>( + query: Query<'store>, + store: &'store AnnotationStore, + wrappedstore: &Arc>, + limit: Option, + ) -> Self { + assert!(query.resulttype() == Some(Type::Annotation)); + Self { + textselections: store + .query(query) + .limit(limit) + .map(|resultitems| { + //we use the deepest item if there are multiple + if let Some(QueryResultItem::TextSelection(textselection)) = + resultitems.pop_last() + { + ( + textselection.resource().handle(), + textselection.handle().expect("textselection must be bound"), + ) + } else { + unreachable!("Unexpected QueryResultItem"); + } + }) + .collect(), + store: wrappedstore.clone(), + cursor: 0, + } + } + + fn map<'store, T, F>(&self, f: F) -> Result where - F: FnOnce( - &Vec<(TextResourceHandle, TextSelectionHandle)>, - &AnnotationStore, - ) -> Result, + F: FnOnce(TextSelections<'store>, &'store AnnotationStore) -> Result, { if let Ok(store) = self.store.read() { - f(&self.textselections, &store).map_err(|err| PyStamError::new_err(format!("{}", err))) + let handles = TextSelections::new(Cow::Borrowed(&self.textselections), false, &store); + f(handles, &store).map_err(|err| PyStamError::new_err(format!("{}", err))) } else { Err(PyRuntimeError::new_err( "Unable to obtain store (should never happen)", @@ -640,31 +843,35 @@ impl PyTextSelections { )) } } -} -impl<'py> IterParams<'py> { - pub(crate) fn evaluate_to_pytextselections<'store>( - self, - iter: TextSelectionsIter<'store>, - store: &'store AnnotationStore, - wrappedstore: &Arc>, - ) -> Result + fn map_with_query<'a, T, F>( + &'a self, + resulttype: Type, + constraint: Constraint<'a>, + args: &PyList, + kwargs: Option<&PyDict>, + f: F, + ) -> Result where - 'py: 'store, + F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, { - let limit = self.limit(); - match self.evaluate_textselections(iter, store) { - Ok(iter) => Ok(PyTextSelections { - textselections: if let Some(limit) = limit { - iter.to_handles_limit(limit) - } else { - iter.to_handles() - }, - store: wrappedstore.clone(), - cursor: 0, - }), - Err(e) => Err(e.into()), - } + self.map(|textselections, store| { + let query = Query::new(QueryType::Select, Some(Type::Annotation), Some("main")) + .with_constraint(Constraint::Handle(Filter::TextSelections(textselections))) + .with_subquery( + build_query( + Query::new(QueryType::Select, Some(resulttype), Some("sub")) + .with_constraint(constraint), + args, + kwargs, + store, + ) + .map_err(|e| { + StamError::QuerySyntaxError(format!("{}", e), "(python to query)") + })?, + ); + f(query, store) + }) } } diff --git a/stam.pyi b/stam.pyi index 3a36794..79a9c5b 100644 --- a/stam.pyi +++ b/stam.pyi @@ -152,7 +152,7 @@ class AnnotationStore: or use the equally named method on other objects for more constrained and filterable annotations (e.g. :meth:`DataKey.annotations`, :meth:`AnnotationDataSet.annotations`, :meth:`TextResource.annotations`) """ - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """Returns an iterator over all annotations (:class:`Annotation`) in this store. Filtering can be applied using keyword arguments. It is recommended to only use this method if you apply further filtering, otherwise the memory overhead may be very large if you have many annotations. @@ -200,22 +200,32 @@ class AnnotationStore: def shrink_to_fit(self): """Reallocates internal data structures to tight fits to conserve memory space (if necessary). You can use this after having added lots of annotations to possibly reduce the memory consumption.""" - def find_data(self, **kwargs) -> Data: - """ - Find annotation data by set, key and value. - Returns :class:`Data`, which holds a collection of :class:`AnnotationData` instances. + def data(self, *args, **kwargs) -> Data: + """Returns an iterator over all annotations (:class:`Annotation`) in this store. - Keyword arguments + Filtering can be applied using keyword arguments. It is recommended to only use this method if you apply further filtering, otherwise the memory overhead may be very large if you have a lot of data. + + Keyword Arguments ------------------- - set: Optional[Union[str,AnnotationDataSet]] - The set to search for; it will search all sets if not specified - key: Optional[Union[str,DataKey]] - The key to search for; it will search all keys if not specified. If you specify a key, you must also specify a set! - value: Optional[Union[str,int,float,bool]] - The exact value to search for, if this or any of its variants mentioned below is omitted, it will search all values. + limit: Optional[int] = None + The maximum number of results to return (default: unlimited) + filter: Union[AnnotationData,Tuple[AnnotationData],List[AnnotationData],DataKey,Annotations,Data] + If you want to add multiple different filters, use `filters` instead. + Filter annotations based on: + + * :class:`AnnotationData` - Returns only this exact data (you can only pass this once). Use :meth:`test_data` instead. + * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. + * :class:`DataKey` - Returns data matching this key (you can only pass this once). + * a tuple/list of :class:`DataKey` + * :class:`Annotations` - Returns data that is used by by annotations in the provided :obj:`Annotations` collection. + * :class:`Data` - Returns only data that is in the provided :obj:`Data` collection. + filters: List[Union[AnnotationData,DataKey,Annotations,Data]] + value: Optional[Union[str,int,float,bool,List[Union[str,int,float,bool]]]] + Search for data matching a specific value. + This holds exact value to search for. Further variants of this keyword are listed below: value_not: Optional[Union[str,int,float,bool]] - Value + Value must not match value_greater: Optional[Union[int,float]] Value must be greater than specified (int or float) value_less: Optional[Union[int,float]] @@ -230,29 +240,61 @@ class AnnotationStore: Value must not match any in the tuple value_in_range: Optional[Tuple[Union[int,float]]] Must be a numeric 2-tuple with min and max (inclusive) values + """ +# def find_data(self, **kwargs) -> Data: +# """ +# Find annotation data by set, key and value. +# Returns :class:`Data`, which holds a collection of :class:`AnnotationData` instances. - Examples - ------------- +# Keyword arguments +# ------------------- - Query for specific annotation data:: +# set: Optional[Union[str,AnnotationDataSet]] +# The set to search for; it will search all sets if not specified +# key: Optional[Union[str,DataKey]] +# The key to search for; it will search all keys if not specified. If you specify a key, you must also specify a set! +# value: Optional[Union[str,int,float,bool]] +# The exact value to search for, if this or any of its variants mentioned below is omitted, it will search all values. +# value_not: Optional[Union[str,int,float,bool]] +# Value +# value_greater: Optional[Union[int,float]] +# Value must be greater than specified (int or float) +# value_less: Optional[Union[int,float]] +# Value must be less than specified (int or float) +# value_greatereq: Optional[Union[int,float]] +# Value must be greater than specified or equal (int or float) +# value_lesseq: Optional[Union[int,float]] +# Value must be less than specified or equal (int or float) +# value_in: Optional[Tuple[Union[str,int,float,bool]]] +# Value must match any in the tuple (this is a logical OR statement) +# value_not_in: Optional[Tuple[Union[str,int,float,bool]]] +# Value must not match any in the tuple +# value_in_range: Optional[Tuple[Union[int,float]]] +# Must be a numeric 2-tuple with min and max (inclusive) values - for annotationdata in store.find_data(set="some-set", key="structuretype", value="word"): - # only returns one - ... - Query for all data for a key:: +# Examples +# ------------- - for annotationdata in store.find_data(set="some-set", key="structuretype"): - ... +# Query for specific annotation data:: - Note, the latter can be accomplished more efficiently as:: +# for annotationdata in store.find_data(set="some-set", key="structuretype", value="word"): +# # only returns one +# ... - for annotationdata in store.dataset("some-set").key("structuretype").data(): - ... +# Query for all data for a key:: - `find_data` should be considered as a convenience/shortcut method. - """ +# for annotationdata in store.find_data(set="some-set", key="structuretype"): +# ... + +# Note, the latter can be accomplished more efficiently as:: + +# for annotationdata in store.dataset("some-set").key("structuretype").data(): +# ... + +# `find_data` should be considered as a convenience/shortcut method. +# """ class Annotation: @@ -619,6 +661,20 @@ class Annotations: Follow AnnotationSelectors recursively (default False) """ + def test_annotation(self, **kwargs) -> bool: + """ + Tests whether certain annotations reference any annotation in this collection. + The annotation can be filtered using keyword arguments. See :meth:`annotations`. + Unlike :meth:`annotations`, this method merely tests without returning the data, and as such is more performant. + """ + + def test_annotations_in_targets(self, **kwargs) -> Annotations: + """ + Tests whether annotations in this collection targets the specified annotation. + The annotation can be filtered using keyword arguments. See :meth:`annotations`. + Unlike :meth:`annotations_in_targets`, this method merely tests without returning the data, and as such is more performant. + """ + def textselections(self, limit: Optional[int] = None) -> TextSelections: """ Returns a collection of all textselections associated with the annotations in this collection. @@ -965,6 +1021,14 @@ class TextSelections: The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. """ + def data(self, **kwargs) -> Data: + """ + Returns annotation data (:class:`Data` containing :class:`AnnotationData`) used by annotations referring to the text selections in this collection. + + The data can be filtered using keyword arguments; see :meth:`Annotation.data`. + If no filters are set (default), all data from all annotations on all text selections are returned (without duplicates). + """ + def test_data(self, **kwargs) -> bool: """ Tests whether there are any annotations that reference any of the text selections in the iterator, with data that passes the provided filters. @@ -1346,12 +1410,6 @@ class TextResource: """ def annotations(self, **kwargs) -> Annotations: - """Returns a collection of annotations (:class:`Annotations` contiaining :class:`Annotation`) that reference this resource via any selector. - - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. - """ - - def annotations_on_text(self, **kwargs) -> Annotations: """Returns a collection of annotations (:class:`Annotation`) that reference this resource via a *TextSelector* (if any). Does *NOT* include those that use a ResourceSelector, use :meth:`annotations_metadata` instead for those instead. @@ -1368,7 +1426,7 @@ class TextResource: def test_annotations(self, **kwargs) -> bool: """ - Tests whether there are any annotations that reference this resource (via any selector). + Tests whether there are any annotations that reference the text of this resource (via a TextSelector). This method is like :meth:`annotations`, but only tests and does not return the annotations, as such it is more performant. @@ -1384,34 +1442,26 @@ class TextResource: The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. """ - def test_annotations_on_text(self, **kwargs) -> bool: - """ - Tests whether there are any annotations that reference the text of this resource (via a TextSelector). - - This method is like :meth:`annotations_on_text`, but only tests and does not return the annotations, as such it is more performant. - - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. - """ - def related_text(self, operator: TextSelectionOperator, referenceselections: List[TextSelection], **kwargs) -> TextSelections: - """ - Applies a :class:`TextSelectionOperator` to find all other - text selections who are in a specific relation with the ones from `referenceselections`. - Returns all matching :class:`TextSelection` instances in a collection :class:`TextSelections`. +# def related_text(self, operator: TextSelectionOperator, referenceselections: List[TextSelection], **kwargs) -> TextSelections: +# """ +# Applies a :class:`TextSelectionOperator` to find all other +# text selections who are in a specific relation with the ones from `referenceselections`. +# Returns all matching :class:`TextSelection` instances in a collection :class:`TextSelections`. - Text selections will be returned in textual order. They may be filtered via keyword arguments. See :meth:`Annotation.textselections`. - - Parameters - ------------ +# Text selections will be returned in textual order. They may be filtered via keyword arguments. See :meth:`Annotation.textselections`. +# +# Parameters +# ------------ - operator: TextSelectionOperator - The operator to apply when comparing text selections - referenceselections: List[TextSelection] - Text selections to use as reference - +# operator: TextSelectionOperator +# The operator to apply when comparing text selections +# referenceselections: List[TextSelection] +# Text selections to use as reference +# - See :meth:`Annotation.related_text` for allowed keyword arguments. - """ +# See :meth:`Annotation.related_text` for allowed keyword arguments. +# """ class TextSelection: """ From 17af37440ba14828a3330f170c8001786adf2b02 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Dec 2023 20:59:15 +0100 Subject: [PATCH 2/8] renamed query.rs --- src/annotation.rs | 2 +- src/annotationdata.rs | 2 +- src/annotationdataset.rs | 2 +- src/annotationstore.rs | 2 +- src/lib.rs | 2 +- src/{iterparams.rs => query.rs} | 0 src/resources.rs | 2 +- src/textselection.rs | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename src/{iterparams.rs => query.rs} (100%) diff --git a/src/annotation.rs b/src/annotation.rs index 6050aba..850aa9c 100644 --- a/src/annotation.rs +++ b/src/annotation.rs @@ -10,7 +10,7 @@ use crate::annotationdata::{PyAnnotationData, PyData}; use crate::annotationdataset::PyAnnotationDataSet; use crate::annotationstore::MapStore; use crate::error::PyStamError; -use crate::iterparams::*; +use crate::query::*; use crate::resources::{PyOffset, PyTextResource}; use crate::selector::{PySelector, PySelectorKind}; use crate::textselection::{PyTextSelectionOperator, PyTextSelections}; diff --git a/src/annotationdata.rs b/src/annotationdata.rs index c8c9906..96068eb 100644 --- a/src/annotationdata.rs +++ b/src/annotationdata.rs @@ -11,7 +11,7 @@ use crate::annotation::PyAnnotations; use crate::annotationdataset::PyAnnotationDataSet; use crate::annotationstore::MapStore; use crate::error::PyStamError; -use crate::iterparams::*; +use crate::query::*; use stam::*; #[pyclass(dict, module = "stam", name = "DataKey")] diff --git a/src/annotationdataset.rs b/src/annotationdataset.rs index 8c637ee..1228767 100644 --- a/src/annotationdataset.rs +++ b/src/annotationdataset.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, RwLock}; use crate::annotationdata::{datavalue_from_py, PyAnnotationData, PyData, PyDataKey}; use crate::error::PyStamError; -use crate::iterparams::*; +use crate::query::*; use crate::selector::{PySelector, PySelectorKind}; use stam::*; diff --git a/src/annotationstore.rs b/src/annotationstore.rs index 8294957..7ededbb 100644 --- a/src/annotationstore.rs +++ b/src/annotationstore.rs @@ -9,7 +9,7 @@ use crate::annotationdata::{annotationdata_builder, data_request_parser, PyData} use crate::annotationdataset::PyAnnotationDataSet; use crate::config::get_config; use crate::error::PyStamError; -use crate::iterparams::*; +use crate::query::*; use crate::resources::PyTextResource; use crate::selector::PySelector; use stam::*; diff --git a/src/lib.rs b/src/lib.rs index 3731b40..b4322c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,7 @@ mod annotationstore; mod config; mod error; //mod query; -mod iterparams; +mod query; mod resources; mod selector; mod textselection; diff --git a/src/iterparams.rs b/src/query.rs similarity index 100% rename from src/iterparams.rs rename to src/query.rs diff --git a/src/resources.rs b/src/resources.rs index 6f0a428..58afcdf 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, RwLock}; use crate::annotation::PyAnnotations; use crate::error::PyStamError; -use crate::iterparams::*; +use crate::query::*; use crate::selector::{PySelector, PySelectorKind}; use crate::textselection::{ PyTextSelection, PyTextSelectionIter, PyTextSelectionOperator, PyTextSelections, diff --git a/src/textselection.rs b/src/textselection.rs index 31bc19f..734fb96 100644 --- a/src/textselection.rs +++ b/src/textselection.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, RwLock}; use crate::annotation::PyAnnotations; use crate::annotationdata::PyData; use crate::error::PyStamError; -use crate::iterparams::*; +use crate::query::*; use crate::resources::{PyOffset, PyTextResource}; use crate::textselection::TextSelectionHandle; use stam::*; From 68b5073f4ef1edca38ae64ab0f866638b376cf92 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 9 Jan 2024 19:27:25 +0100 Subject: [PATCH 3/8] wip: refactoring against new API --- src/annotation.rs | 162 ++++++++++++---------- src/annotationdata.rs | 103 ++++++-------- src/annotationdataset.rs | 66 ++++----- src/annotationstore.rs | 9 +- src/query.rs | 292 +++++++++++++++++++++++++-------------- src/resources.rs | 42 +++--- src/textselection.rs | 59 ++++---- 7 files changed, 402 insertions(+), 331 deletions(-) diff --git a/src/annotation.rs b/src/annotation.rs index 850aa9c..da4cf14 100644 --- a/src/annotation.rs +++ b/src/annotation.rs @@ -155,7 +155,11 @@ impl PyAnnotation { } else { self.map_with_query( Type::TextSelection, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |annotation, query| { @@ -178,7 +182,7 @@ impl PyAnnotation { kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); - let recursive = get_recursive(kwargs, false); + let recursive = get_recursive(kwargs, AnnotationDepth::One); if !has_filters(args, kwargs) { self.map(|annotation| { Ok(PyAnnotations::from_iter( @@ -189,14 +193,7 @@ impl PyAnnotation { } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable( - "main", - if recursive { - AnnotationQualifier::RecursiveTarget - } else { - AnnotationQualifier::Target - }, - ), + Constraint::AnnotationVariable("main", SelectionQualifier::Metadata, recursive), args, kwargs, |annotation, query| { @@ -225,7 +222,11 @@ impl PyAnnotation { } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |annotation, query| { @@ -247,7 +248,11 @@ impl PyAnnotation { } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |annotation, query| Ok(annotation.store().query(query).test()), @@ -261,20 +266,13 @@ impl PyAnnotation { args: &PyList, kwargs: Option<&PyDict>, ) -> PyResult { - let recursive = get_recursive(kwargs, false); + let recursive = get_recursive(kwargs, AnnotationDepth::One); if !has_filters(args, kwargs) { self.map(|annotation| Ok(annotation.annotations_in_targets(recursive).test())) } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable( - "main", - if recursive { - AnnotationQualifier::RecursiveTarget - } else { - AnnotationQualifier::Target - }, - ), + Constraint::AnnotationVariable("main", SelectionQualifier::Metadata, recursive), args, kwargs, |annotation, query| Ok(annotation.store().query(query).test()), @@ -366,7 +364,11 @@ impl PyAnnotation { } else { self.map_with_query( Type::AnnotationData, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |annotation, query| { @@ -388,7 +390,11 @@ impl PyAnnotation { } else { self.map_with_query( Type::AnnotationData, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |annotation, query| Ok(annotation.store().query(query).test()), @@ -496,7 +502,11 @@ impl PyAnnotations { } else { self.map_with_query( Type::AnnotationData, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |query, store| Ok(PyData::from_query(query, store, &self.store, limit)), @@ -511,7 +521,11 @@ impl PyAnnotations { } else { self.map_with_query( Type::AnnotationData, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |query, store| Ok(store.query(query).test()), @@ -532,7 +546,11 @@ impl PyAnnotations { } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |query, store| Ok(PyAnnotations::from_query(query, store, &self.store, limit)), @@ -547,7 +565,11 @@ impl PyAnnotations { } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |query, store| Ok(store.query(query).test()), @@ -562,7 +584,7 @@ impl PyAnnotations { kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); - let recursive = get_recursive(kwargs, false); + let recursive = get_recursive(kwargs, AnnotationDepth::One); if !has_filters(args, kwargs) { self.map(|annotations, store| { Ok(PyAnnotations::from_iter( @@ -576,14 +598,7 @@ impl PyAnnotations { } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable( - "main", - if recursive { - AnnotationQualifier::RecursiveTarget - } else { - AnnotationQualifier::Target - }, - ), + Constraint::AnnotationVariable("main", SelectionQualifier::Normal, recursive), args, kwargs, |query, store| Ok(PyAnnotations::from_query(query, store, &self.store, limit)), @@ -597,7 +612,7 @@ impl PyAnnotations { args: &PyList, kwargs: Option<&PyDict>, ) -> PyResult { - let recursive = get_recursive(kwargs, false); + let recursive = get_recursive(kwargs, AnnotationDepth::One); if !has_filters(args, kwargs) { self.map(|annotations, _| { Ok(annotations.items().annotations_in_targets(recursive).test()) @@ -605,14 +620,7 @@ impl PyAnnotations { } else { self.map_with_query( Type::Annotation, - Constraint::AnnotationVariable( - "main", - if recursive { - AnnotationQualifier::RecursiveTarget - } else { - AnnotationQualifier::Target - }, - ), + Constraint::AnnotationVariable("main", SelectionQualifier::Normal, recursive), args, kwargs, |query, store| Ok(store.query(query).test()), @@ -633,7 +641,11 @@ impl PyAnnotations { } else { self.map_with_query( Type::TextSelection, - Constraint::AnnotationVariable("main", AnnotationQualifier::None), + Constraint::AnnotationVariable( + "main", + SelectionQualifier::Normal, + AnnotationDepth::One, + ), args, kwargs, |query, store| { @@ -729,7 +741,7 @@ impl PyAnnotations { annotations: store .query(query) .limit(limit) - .map(|resultitems| { + .map(|mut resultitems| { //we use the deepest item if there are multiple if let Some(QueryResultItem::Annotation(annotation)) = resultitems.pop_last() { annotation.handle() @@ -743,9 +755,9 @@ impl PyAnnotations { } } - fn map<'store, T, F>(&self, f: F) -> Result + fn map(&self, f: F) -> Result where - F: FnOnce(Handles<'store, Annotation>, &'store AnnotationStore) -> Result, + F: FnOnce(Handles, &AnnotationStore) -> Result, { if let Ok(store) = self.store.read() { let handles = Annotations::new(Cow::Borrowed(&self.annotations), false, &store); @@ -770,23 +782,24 @@ impl PyAnnotations { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, - constraint: Constraint<'a>, + constraint: Constraint, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, + F: FnOnce(Query, &AnnotationStore) -> Result, { self.map(|annotations, store| { let query = Query::new(QueryType::Select, Some(Type::Annotation), Some("main")) - .with_constraint(Constraint::Handle(Filter::Annotations( + .with_constraint(Constraint::Annotations( annotations, - FilterMode::Any, - ))) + SelectionQualifier::Normal, + AnnotationDepth::One, + )) .with_subquery( build_query( Query::new(QueryType::Select, Some(resulttype), Some("sub")) @@ -797,7 +810,8 @@ impl PyAnnotations { ) .map_err(|e| { StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, + })? + .0, ); f(query, store) }) @@ -863,7 +877,7 @@ impl MapStore for PyAnnotation { impl PyAnnotation { /// Map function to act on the actual underlying store, helps reduce boilerplate - fn map(&self, f: F) -> Result + pub(crate) fn map(&self, f: F) -> Result where F: FnOnce(ResultItem<'_, Annotation>) -> Result, { @@ -879,32 +893,28 @@ impl PyAnnotation { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, - constraint: Constraint<'a>, + constraint: Constraint, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(ResultItem<'a, Annotation>, Query<'a>) -> Result, + F: FnOnce(ResultItem, Query) -> Result, { self.map(|annotation| { - let query = Query::new(QueryType::Select, Some(Type::Annotation), Some("main")) - .with_constraint(Constraint::Handle(Filter::Annotation(annotation.handle()))) - .with_subquery( - build_query( - Query::new(QueryType::Select, Some(resulttype), Some("sub")) - .with_constraint(constraint), - args, - kwargs, - annotation.store(), - ) - .map_err(|e| { - StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, - ); + let query = build_query( + Query::new(QueryType::Select, Some(resulttype), Some("result")) + .with_constraint(constraint), + args, + kwargs, + annotation.store(), + ) + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? + .0 + .with_annotationvar("main", annotation.clone()); f(annotation, query) }) } diff --git a/src/annotationdata.rs b/src/annotationdata.rs index 96068eb..630b2f2 100644 --- a/src/annotationdata.rs +++ b/src/annotationdata.rs @@ -157,7 +157,7 @@ impl MapStore for PyDataKey { } impl PyDataKey { - fn map(&self, f: F) -> Result + pub(crate) fn map(&self, f: F) -> Result where F: FnOnce(ResultItem) -> Result, { @@ -176,35 +176,28 @@ impl PyDataKey { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(ResultItem<'a, DataKey>, Query<'a>) -> Result, + F: FnOnce(ResultItem, Query) -> Result, { - self.map(|data| { - let query = Query::new(QueryType::Select, Some(Type::DataKey), Some("main")) - .with_constraint(Constraint::Handle(Filter::DataKey( - data.set().handle(), - data.handle(), - ))) - .with_subquery( - build_query( - Query::new(QueryType::Select, Some(resulttype), Some("sub")) - .with_constraint(Constraint::DataKeyVariable("main")), - args, - kwargs, - data.rootstore(), - ) - .map_err(|e| { - StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, - ); - f(data, query) + self.map(|key| { + let query = build_query( + Query::new(QueryType::Select, Some(resulttype), Some("result")) + .with_constraint(Constraint::KeyVariable("main", SelectionQualifier::Normal)), + args, + kwargs, + key.rootstore(), + ) + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? + .0 + .with_keyvar("main", key.clone()); + f(key, query) }) } } @@ -499,7 +492,7 @@ impl MapStore for PyAnnotationData { } impl PyAnnotationData { - fn map(&self, f: F) -> Result + pub(crate) fn map(&self, f: F) -> Result where F: FnOnce(ResultItem) -> Result, { @@ -518,34 +511,27 @@ impl PyAnnotationData { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(ResultItem<'a, AnnotationData>, Query<'a>) -> Result, + F: FnOnce(ResultItem, Query) -> Result, { self.map(|data| { - let query = Query::new(QueryType::Select, Some(Type::AnnotationData), Some("main")) - .with_constraint(Constraint::Handle(Filter::AnnotationData( - data.set().handle(), - data.handle(), - ))) - .with_subquery( - build_query( - Query::new(QueryType::Select, Some(resulttype), Some("sub")) - .with_constraint(Constraint::DataVariable("main")), - args, - kwargs, - data.rootstore(), - ) - .map_err(|e| { - StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, - ); + let query = build_query( + Query::new(QueryType::Select, Some(resulttype), Some("result")) + .with_constraint(Constraint::DataVariable("main", SelectionQualifier::Normal)), + args, + kwargs, + data.rootstore(), + ) + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? + .0 + .with_datavar("main", data.clone()); f(data, query) }) } @@ -608,12 +594,7 @@ pub(crate) fn annotationdata_builder<'a>(data: &'a PyAny) -> PyResult( - kwargs: &'py PyDict, -) -> Result>, StamError> -where - 'py: 'a, -{ +pub(crate) fn dataoperator_from_kwargs(kwargs: &PyDict) -> Result, StamError> { if let Ok(Some(value)) = kwargs.get_item("value") { Ok(Some(dataoperator_from_py(value)?)) } else if let Ok(Some(value)) = kwargs.get_item("value_not") { @@ -937,7 +918,7 @@ impl PyData { data: store .query(query) .limit(limit) - .map(|resultitems| { + .map(|mut resultitems| { //we use the deepest item if there are multiple if let Some(QueryResultItem::AnnotationData(data)) = resultitems.pop_last() { (data.set().handle(), data.handle()) @@ -951,9 +932,9 @@ impl PyData { } } - fn map<'store, T, F>(&self, f: F) -> Result + fn map(&self, f: F) -> Result where - F: FnOnce(Handles<'store, AnnotationData>, &'store AnnotationStore) -> Result, + F: FnOnce(Handles, &AnnotationStore) -> Result, { if let Ok(store) = self.store.read() { let handles = Data::new(Cow::Borrowed(&self.data), false, &store); @@ -965,30 +946,34 @@ impl PyData { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, + F: FnOnce(Query, &AnnotationStore) -> Result, { self.map(|data, store| { let query = Query::new(QueryType::Select, Some(Type::AnnotationData), Some("main")) - .with_constraint(Constraint::Handle(Filter::Data(data, FilterMode::Any))) + .with_constraint(Constraint::Data(data, SelectionQualifier::Normal)) .with_subquery( build_query( Query::new(QueryType::Select, Some(resulttype), Some("sub")) - .with_constraint(Constraint::DataVariable("main")), + .with_constraint(Constraint::DataVariable( + "main", + SelectionQualifier::Normal, + )), args, kwargs, store, ) .map_err(|e| { StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, + })? + .0, ); f(query, store) }) diff --git a/src/annotationdataset.rs b/src/annotationdataset.rs index 1228767..61e2af1 100644 --- a/src/annotationdataset.rs +++ b/src/annotationdataset.rs @@ -178,14 +178,20 @@ impl PyAnnotationDataSet { if !has_filters(args, kwargs) { self.map(|dataset| Ok(PyData::from_iter(dataset.data().limit(limit), &self.store))) } else { - self.map_with_query(Type::AnnotationData, args, kwargs, |dataset, query| { - Ok(PyData::from_query( - query, - dataset.store(), - &self.store, - limit, - )) - }) + self.map_with_query( + Type::AnnotationData, + Constraint::DataSetVariable("main", SelectionQualifier::Normal), + args, + kwargs, + |dataset, query| { + Ok(PyData::from_query( + query, + dataset.store(), + &self.store, + limit, + )) + }, + ) } } @@ -194,9 +200,13 @@ impl PyAnnotationDataSet { if !has_filters(args, kwargs) { self.map(|dataset| Ok(dataset.data().test())) } else { - self.map_with_query(Type::AnnotationData, args, kwargs, |dataset, query| { - Ok(dataset.store().query(query).test()) - }) + self.map_with_query( + Type::AnnotationData, + Constraint::DataSetVariable("main", SelectionQualifier::Normal), + args, + kwargs, + |dataset, query| Ok(dataset.store().query(query).test()), + ) } } @@ -252,36 +262,28 @@ impl PyAnnotationDataSet { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, + constraint: Constraint, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(ResultItem<'a, AnnotationDataSet>, Query<'a>) -> Result, + F: FnOnce(ResultItem, Query) -> Result, { self.map(|dataset| { - let query = Query::new( - QueryType::Select, - Some(Type::AnnotationDataSet), - Some("main"), + let query = build_query( + Query::new(QueryType::Select, Some(resulttype), Some("result")) + .with_constraint(constraint), + args, + kwargs, + dataset.store(), ) - .with_constraint(Constraint::Handle(Filter::AnnotationDataSet( - dataset.handle(), - ))) - .with_subquery( - build_query( - Query::new(QueryType::Select, Some(resulttype), Some("sub")).with_constraint( - Constraint::AnnotationVariable("main", AnnotationQualifier::None), - ), - args, - kwargs, - dataset.store(), - ) - .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))?, - ); + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? + .0 + .with_datasetvar("main", dataset.clone()); f(dataset, query) }) } diff --git a/src/annotationstore.rs b/src/annotationstore.rs index 7ededbb..564fb4d 100644 --- a/src/annotationstore.rs +++ b/src/annotationstore.rs @@ -369,15 +369,15 @@ impl PyAnnotationStore { self.map_store_mut(f) } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, + F: FnOnce(Query, &AnnotationStore) -> Result, { self.map_store(|store| { let query = build_query( @@ -386,7 +386,8 @@ impl PyAnnotationStore { kwargs, store, ) - .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))?; + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? + .0; f(query, store) }) } diff --git a/src/query.rs b/src/query.rs index f2f7a53..e5c79dc 100644 --- a/src/query.rs +++ b/src/query.rs @@ -1,71 +1,180 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::*; -use std::borrow::Cow; +use std::collections::HashSet; use crate::annotation::{PyAnnotation, PyAnnotations}; use crate::annotationdata::{dataoperator_from_kwargs, PyAnnotationData, PyData, PyDataKey}; +use crate::annotationdataset::PyAnnotationDataSet; use crate::error::PyStamError; use crate::textselection::PyTextSelectionOperator; use stam::*; -pub struct QueryBuilder<'a> { - filters: Vec>, - limit: Option, +pub struct ContextVariables(Vec); + +impl ContextVariables { + pub fn add(&mut self) -> &str { + let var = format!("v{}", self.0.len() + 1); + self.0.push(var); + self.last().expect("was just added") + } + + pub fn last(&self) -> Option<&str> { + self.0.iter().last().map(|x| x.as_str()) + } } -fn add_filter<'a>( - query: &mut Query<'a>, - store: &'a AnnotationStore, - filter: &'a PyAny, - operator: &mut Option, -) -> PyResult<()> { - if filter.is_instance_of::() { +impl Default for ContextVariables { + fn default() -> Self { + Self(Vec::new()) + } +} + +fn add_filter<'store, 'py>( + query: &mut Query<'store>, + store: &'store AnnotationStore, + filter: &'py PyAny, + operator: Option>, + contextvariables: &mut ContextVariables, +) -> PyResult<()> +where + 'py: 'store, +{ + if filter.is_instance_of::() { + let filter: &PyDict = filter.extract()?; + let operator = dataoperator_from_kwargs(filter) + .map_err(|err| PyValueError::new_err(format!("{}", err)))? + .or(operator); + let key = filter.get_item("key")?.expect("already checked"); + if key.is_instance_of::() { + let key: PyRef<'py, PyDataKey> = filter.extract()?; + if let Some(key) = store.key(key.set, key.handle) { + let varname = contextvariables.add(); + query.bind_keyvar(varname, key); + if let Some(operator) = operator { + query.constrain(Constraint::KeyValueVariable( + varname, + operator, + SelectionQualifier::Normal, + )); + } else { + query.constrain(Constraint::KeyVariable(varname, SelectionQualifier::Normal)); + } + } else { + return Err(PyValueError::new_err( + "Passed key instance is invalid (should not happen)", + )); + } + } else if filter.contains("set")? { + if key.is_instance_of::() { + let key = key.downcast::()?; + let set = filter.get_item("set")?.expect("already checked"); + let key = if set.is_instance_of::() { + let set: PyRef<'py, PyAnnotationDataSet> = filter.extract()?; + let key: &str = key.to_str()?; + set.map(|set| Ok(set.key(key))).unwrap() + } else if set.is_instance_of::() { + let set = set.downcast::()?; + if let Some(set) = store.dataset(set.to_str()?) { + set.key(key.to_str()?) + } else { + None + } + } else { + None + }; + if let Some(key) = key { + let varname = key.as_ref().temp_id().expect("temp id must work"); + query.bind_keyvar(varname.as_str(), key); + if operator.is_some() { + query.constrain(Constraint::KeyValueVariable( + varname.as_str(), + operator.take().unwrap(), + SelectionQualifier::Normal, + )); + } else { + query.constrain(Constraint::KeyVariable( + varname.as_str(), + SelectionQualifier::Normal, + )); + } + } + } else { + return Err(PyValueError::new_err( + "'key' parameter in filter dictionary should be of type `str`", + )); + } + } else { + return Err(PyValueError::new_err( + "'key' parameter in filter dictionary should be of type DataKey, it can also be `str` if you also provide `set` as well", + )); + } + } else if filter.is_instance_of::() { let vec: Vec<&PyAny> = filter.extract()?; add_multi_filter(query, store, vec)?; } else if filter.is_instance_of::() { let vec: Vec<&PyAny> = filter.extract()?; add_multi_filter(query, store, vec)?; } else if filter.is_instance_of::() { - let adata: PyRef<'_, PyAnnotationData> = filter.extract()?; - query.constrain(Constraint::Filter(Filter::AnnotationData( - adata.set, - adata.handle, - ))); - } else if filter.is_instance_of::() { - let key: PyRef<'_, PyDataKey> = filter.extract()?; + let data: PyRef<'_, PyAnnotationData> = filter.extract()?; if operator.is_some() { - query.constrain(Constraint::Filter(Filter::DataKeyAndOperator( - key.set, - key.handle, - operator.take().unwrap(), - ))); - } else { - query.constrain(Constraint::Filter(Filter::DataKey(key.set, key.handle))); + return Err(PyValueError::new_err( + "'value' parameter can not be used in combination with an AnnotationData instance (it already restrains to a single value)", + )); } + data.map(|data| { + let varname = contextvariables.add(); + query.bind_datavar(varname, data); + query.constrain(Constraint::DataVariable( + varname, + SelectionQualifier::Normal, + )); + Ok(()) + }); + } else if filter.is_instance_of::() { + let key: PyRef<'py, PyDataKey> = filter.extract()?; + key.map(|key| { + let varname = contextvariables.add(); + query.bind_keyvar(varname, key); + if operator.is_some() { + query.constrain(Constraint::KeyValueVariable( + varname, + operator.take().unwrap(), + SelectionQualifier::Normal, + )); + } else { + query.constrain(Constraint::KeyVariable(varname, SelectionQualifier::Normal)); + } + Ok(()) + }); } else if filter.is_instance_of::() { - let annotation: PyRef<'_, PyAnnotation> = filter.extract()?; - query.constrain(Constraint::Filter(Filter::Annotation(annotation.handle))); - } else if filter.is_instance_of::() { - let operator: PyRef<'_, PyTextSelectionOperator> = filter.extract()?; - query.constrain(Constraint::Filter(Filter::TextSelectionOperator( - operator.operator, - ))); + let annotation: PyRef<'py, PyAnnotation> = filter.extract()?; + annotation.map(|annotation| { + let varname = annotation.as_ref().temp_id()?; + query.bind_annotationvar(varname.as_str(), annotation); + query.constrain(Constraint::AnnotationVariable( + varname.as_str(), + SelectionQualifier::Normal, + AnnotationDepth::One, + )); + Ok(()) + }); } else if filter.is_instance_of::() { - let annotations: PyRef<'a, PyAnnotations> = filter.extract()?; - query.constrain(Constraint::Filter(Filter::Annotations(Handles::from_iter( - annotations.annotations.iter().copied(), - store, - )))); + let annotations: PyRef<'py, PyAnnotations> = filter.extract()?; + query.constrain(Constraint::Annotations( + Handles::from_iter(annotations.annotations.iter().copied(), store), + SelectionQualifier::Normal, + AnnotationDepth::One, + )); } else if filter.is_instance_of::() { - let data: PyRef<'_, PyData> = filter.extract()?; - query.constrain(Constraint::Filter(Filter::Data( + let data: PyRef<'py, PyData> = filter.extract()?; + query.constrain(Constraint::Data( Handles::from_iter(data.data.iter().copied(), store), - FilterMode::Any, - ))); + SelectionQualifier::Normal, + )); } else { return Err(PyValueError::new_err( - "Got argument of unexpected type for filter=/filters=", + "Got filter argument of unexpected type", )); } Ok(()) @@ -77,18 +186,22 @@ fn add_multi_filter<'a>( filter: Vec<&PyAny>, ) -> PyResult<()> { if filter.iter().all(|x| x.is_instance_of::()) { - query.constrain(Constraint::Filter(Filter::Annotations(Handles::from_iter( - filter.iter().map(|x| { - let annotation: PyRef<'_, PyAnnotation> = x.extract().unwrap(); - annotation.handle - }), - store, - )))); + query.constrain(Constraint::Annotations( + Handles::from_iter( + filter.iter().map(|x| { + let annotation: PyRef<'_, PyAnnotation> = x.extract().unwrap(); + annotation.handle + }), + store, + ), + SelectionQualifier::Normal, + AnnotationDepth::One, + )); } else if filter .iter() .all(|x| x.is_instance_of::()) { - query.constrain(Constraint::Filter(Filter::Data( + query.constrain(Constraint::Data( Handles::from_iter( filter.iter().map(|x| { let adata: PyRef<'_, PyAnnotationData> = x.extract().unwrap(); @@ -96,78 +209,45 @@ fn add_multi_filter<'a>( }), store, ), - FilterMode::Any, - ))); + SelectionQualifier::Normal, + )); } Ok(()) } -pub(crate) fn build_query<'py>( - mut query: Query<'py>, +pub(crate) fn build_query<'store, 'py>( + mut query: Query<'store>, args: &'py PyList, //TODO: implement! kwargs: Option<&'py PyDict>, - store: &'py AnnotationStore, -) -> PyResult> { + store: &'store AnnotationStore, +) -> PyResult<(Query<'store>, ContextVariables)> { + let mut contextvariables = ContextVariables::default(); + let operator = if let Some(kwargs) = kwargs { + dataoperator_from_kwargs(kwargs).map_err(|e| PyStamError::new_err(format!("{}", e)))? + } else { + None + }; + for filter in args { + add_filter(&mut query, store, filter, operator, &mut contextvariables)?; + } if let Some(kwargs) = kwargs { - let mut operator = - dataoperator_from_kwargs(kwargs).map_err(|e| PyStamError::new_err(format!("{}", e)))?; if let Ok(Some(filter)) = kwargs.get_item("filter") { - add_filter(&mut query, store, filter, &mut operator)?; + add_filter(&mut query, store, filter, operator, &mut contextvariables)?; } else if let Ok(Some(filter)) = kwargs.get_item("filters") { if filter.is_instance_of::() { let vec = filter.downcast::()?; for filter in vec { - add_filter(&mut query, store, filter, &mut operator)?; + add_filter(&mut query, store, filter, operator, &mut contextvariables)?; } } else if filter.is_instance_of::() { let vec = filter.downcast::()?; for filter in vec { - add_filter(&mut query, store, filter, &mut operator)?; + add_filter(&mut query, store, filter, operator, &mut contextvariables)?; } } } - //if the operator has not been consumed yet in an earlier add_filter step, add a constraint for it now: - if let Some(operator) = operator { - query.constrain(Constraint::Filter(Filter::DataOperator(operator))); - } - } - Ok(query) -} - -impl<'py> QueryBuilder<'py> { - pub fn new( - resulttype: Type, - store: &'py AnnotationStore, - kwargs: Option<&'py PyDict>, - name: Option<&'py str>, - ) -> PyResult> { - let mut query = Query::new(QueryType::Select, Some(resulttype), name); - - if let Some(kwargs) = kwargs { - let mut operator = dataoperator_from_kwargs(kwargs) - .map_err(|e| PyStamError::new_err(format!("{}", e)))?; - if let Ok(Some(filter)) = kwargs.get_item("filter") { - add_filter(&mut query, store, filter, &mut operator)?; - } else if let Ok(Some(filter)) = kwargs.get_item("filters") { - if filter.is_instance_of::() { - let vec = filter.downcast::()?; - for filter in vec { - add_filter(&mut query, store, filter, &mut operator)?; - } - } else if filter.is_instance_of::() { - let vec = filter.downcast::()?; - for filter in vec { - add_filter(&mut query, store, filter, &mut operator)?; - } - } - } - //if the operator has not been consumed yet in an earlier add_filter step, add a constraint for it now: - if let Some(operator) = operator { - query.constrain(Constraint::Filter(Filter::DataOperator(operator))); - } - } - Ok(query) } + Ok((query, contextvariables)) } pub(crate) fn has_filters(args: &PyList, kwargs: Option<&PyDict>) -> bool { @@ -186,11 +266,15 @@ pub(crate) fn has_filters(args: &PyList, kwargs: Option<&PyDict>) -> bool { false } -pub(crate) fn get_recursive(kwargs: Option<&PyDict>, default: bool) -> bool { +pub(crate) fn get_recursive(kwargs: Option<&PyDict>, default: AnnotationDepth) -> AnnotationDepth { if let Some(kwargs) = kwargs { if let Ok(Some(v)) = kwargs.get_item("recursive") { if let Ok(v) = v.extract::() { - return v; + if v { + return AnnotationDepth::Max; + } else { + return AnnotationDepth::One; + } } } } diff --git a/src/resources.rs b/src/resources.rs index 58afcdf..3d3cbb9 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -310,12 +310,10 @@ impl PyTextResource { fn textselections(&self) -> PyResult { self.map(|resource| { - let iter = resource.textselections(); - Ok(PyTextSelections { - textselections: iter.to_handles(), - store: self.store.clone(), - cursor: 0, - }) + Ok(PyTextSelections::from_iter( + resource.textselections(), + &self.store.clone(), + )) }) } @@ -514,32 +512,28 @@ impl PyTextResource { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, - constraint: Constraint<'a>, + constraint: Constraint, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(ResultItem<'a, TextResource>, Query<'a>) -> Result, + F: FnOnce(ResultItem, Query) -> Result, { self.map(|resource| { - let query = Query::new(QueryType::Select, Some(Type::TextResource), Some("main")) - .with_constraint(Constraint::Handle(Filter::TextResource(resource.handle()))) - .with_subquery( - build_query( - Query::new(QueryType::Select, Some(resulttype), Some("sub")) - .with_constraint(constraint), - args, - kwargs, - resource.store(), - ) - .map_err(|e| { - StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, - ); + let query = build_query( + Query::new(QueryType::Select, Some(resulttype), Some("result")) + .with_constraint(constraint), + args, + kwargs, + resource.store(), + ) + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? + .0 + .with_resourcevar("main", resource.clone()); f(resource, query) }) } diff --git a/src/textselection.rs b/src/textselection.rs index 734fb96..ffacd09 100644 --- a/src/textselection.rs +++ b/src/textselection.rs @@ -481,37 +481,28 @@ impl PyTextSelection { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, - constraint: Constraint<'a>, + constraint: Constraint, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(ResultTextSelection<'a>, Query<'a>) -> Result, + F: FnOnce(ResultTextSelection, Query) -> Result, { self.map(|textselection| { - let query = Query::new(QueryType::Select, Some(Type::TextSelection), Some("main")) - .with_constraint(Constraint::Handle(Filter::TextSelection( - textselection.resource().handle(), - textselection - .handle() - .expect("textselection must have handle"), - ))) - .with_subquery( - build_query( - Query::new(QueryType::Select, Some(resulttype), Some("sub")) - .with_constraint(constraint), - args, - kwargs, - textselection.rootstore(), - ) - .map_err(|e| { - StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, - ); + let query = build_query( + Query::new(QueryType::Select, Some(resulttype), Some("result")) + .with_constraint(constraint), + args, + kwargs, + textselection.rootstore(), + ) + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? + .0 + .with_textvar("main", textselection.clone()); f(textselection, query) }) } @@ -794,7 +785,7 @@ impl PyTextSelections { textselections: store .query(query) .limit(limit) - .map(|resultitems| { + .map(|mut resultitems| { //we use the deepest item if there are multiple if let Some(QueryResultItem::TextSelection(textselection)) = resultitems.pop_last() @@ -813,9 +804,9 @@ impl PyTextSelections { } } - fn map<'store, T, F>(&self, f: F) -> Result + fn map(&self, f: F) -> Result where - F: FnOnce(TextSelections<'store>, &'store AnnotationStore) -> Result, + F: FnOnce(TextSelections, &AnnotationStore) -> Result, { if let Ok(store) = self.store.read() { let handles = TextSelections::new(Cow::Borrowed(&self.textselections), false, &store); @@ -844,20 +835,23 @@ impl PyTextSelections { } } - fn map_with_query<'a, T, F>( - &'a self, + fn map_with_query( + &self, resulttype: Type, - constraint: Constraint<'a>, + constraint: Constraint, args: &PyList, kwargs: Option<&PyDict>, f: F, ) -> Result where - F: FnOnce(Query<'a>, &'a AnnotationStore) -> Result, + F: FnOnce(Query, &AnnotationStore) -> Result, { self.map(|textselections, store| { let query = Query::new(QueryType::Select, Some(Type::Annotation), Some("main")) - .with_constraint(Constraint::Handle(Filter::TextSelections(textselections))) + .with_constraint(Constraint::TextSelections( + textselections, + SelectionQualifier::Normal, + )) .with_subquery( build_query( Query::new(QueryType::Select, Some(resulttype), Some("sub")) @@ -868,7 +862,8 @@ impl PyTextSelections { ) .map_err(|e| { StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })?, + })? + .0, ); f(query, store) }) From 4fb46663d00508093fb632069cb146c209c6d1f1 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 10 Jan 2024 11:35:12 +0100 Subject: [PATCH 4/8] wip --- src/query.rs | 78 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/src/query.rs b/src/query.rs index e5c79dc..7dc4a9b 100644 --- a/src/query.rs +++ b/src/query.rs @@ -30,15 +30,16 @@ impl Default for ContextVariables { } } -fn add_filter<'store, 'py>( +fn add_filter<'store, 'py, 'context>( query: &mut Query<'store>, store: &'store AnnotationStore, filter: &'py PyAny, operator: Option>, - contextvariables: &mut ContextVariables, + contextvariables: &'context mut ContextVariables, ) -> PyResult<()> where 'py: 'store, + 'context: 'store, { if filter.is_instance_of::() { let filter: &PyDict = filter.extract()?; @@ -62,7 +63,7 @@ where } } else { return Err(PyValueError::new_err( - "Passed key instance is invalid (should not happen)", + "Passed DataKey instance is invalid (should never happen)", )); } } else if filter.contains("set")? { @@ -71,30 +72,43 @@ where let set = filter.get_item("set")?.expect("already checked"); let key = if set.is_instance_of::() { let set: PyRef<'py, PyAnnotationDataSet> = filter.extract()?; - let key: &str = key.to_str()?; - set.map(|set| Ok(set.key(key))).unwrap() + if let Some(dataset) = store.dataset(set.handle) { + if let Some(key) = dataset.key(key.to_str()?) { + Some(key) + } else { + return Err(PyValueError::new_err("specified key not found in set")); + } + } else { + return Err(PyValueError::new_err( + "Passed AnnotationDataSet instance is invalid (should never happen)", + )); + } } else if set.is_instance_of::() { let set = set.downcast::()?; - if let Some(set) = store.dataset(set.to_str()?) { - set.key(key.to_str()?) + if let Some(dataset) = store.dataset(set.to_str()?) { + if let Some(key) = dataset.key(key.to_str()?) { + Some(key) + } else { + return Err(PyValueError::new_err("specified key not found in set")); + } } else { - None + return Err(PyValueError::new_err("specified dataset not found")); } } else { None }; if let Some(key) = key { - let varname = key.as_ref().temp_id().expect("temp id must work"); - query.bind_keyvar(varname.as_str(), key); + let varname = contextvariables.add(); + query.bind_keyvar(varname, key); if operator.is_some() { query.constrain(Constraint::KeyValueVariable( - varname.as_str(), + varname, operator.take().unwrap(), SelectionQualifier::Normal, )); } else { query.constrain(Constraint::KeyVariable( - varname.as_str(), + varname, SelectionQualifier::Normal, )); } @@ -122,43 +136,48 @@ where "'value' parameter can not be used in combination with an AnnotationData instance (it already restrains to a single value)", )); } - data.map(|data| { + if let Some(data) = store.annotationdata(data.set, data.handle) { let varname = contextvariables.add(); query.bind_datavar(varname, data); query.constrain(Constraint::DataVariable( varname, SelectionQualifier::Normal, )); - Ok(()) - }); + } } else if filter.is_instance_of::() { let key: PyRef<'py, PyDataKey> = filter.extract()?; - key.map(|key| { + if let Some(key) = store.key(key.set, key.handle) { let varname = contextvariables.add(); query.bind_keyvar(varname, key); - if operator.is_some() { + if let Some(operator) = operator { query.constrain(Constraint::KeyValueVariable( varname, - operator.take().unwrap(), + operator, SelectionQualifier::Normal, )); } else { query.constrain(Constraint::KeyVariable(varname, SelectionQualifier::Normal)); } - Ok(()) - }); + } else { + return Err(PyValueError::new_err( + "Passed DataKey instance is invalid (should never happen)", + )); + } } else if filter.is_instance_of::() { let annotation: PyRef<'py, PyAnnotation> = filter.extract()?; - annotation.map(|annotation| { - let varname = annotation.as_ref().temp_id()?; - query.bind_annotationvar(varname.as_str(), annotation); + if let Some(annotation) = store.annotation(annotation.handle) { + let varname = contextvariables.add(); + query.bind_annotationvar(varname, annotation); query.constrain(Constraint::AnnotationVariable( - varname.as_str(), + varname, SelectionQualifier::Normal, AnnotationDepth::One, )); - Ok(()) - }); + } else { + return Err(PyValueError::new_err( + "Passed DataKey instance is invalid (should never happen)", + )); + } } else if filter.is_instance_of::() { let annotations: PyRef<'py, PyAnnotations> = filter.extract()?; query.constrain(Constraint::Annotations( @@ -182,7 +201,7 @@ where fn add_multi_filter<'a>( query: &mut Query<'a>, - store: &AnnotationStore, + store: &'a AnnotationStore, filter: Vec<&PyAny>, ) -> PyResult<()> { if filter.iter().all(|x| x.is_instance_of::()) { @@ -220,7 +239,10 @@ pub(crate) fn build_query<'store, 'py>( args: &'py PyList, //TODO: implement! kwargs: Option<&'py PyDict>, store: &'store AnnotationStore, -) -> PyResult<(Query<'store>, ContextVariables)> { +) -> PyResult<(Query<'store>, ContextVariables)> +where + 'py: 'store, +{ let mut contextvariables = ContextVariables::default(); let operator = if let Some(kwargs) = kwargs { dataoperator_from_kwargs(kwargs).map_err(|e| PyStamError::new_err(format!("{}", e)))? From 08b11a475a481d466915a0f28325582a257837ef Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 10 Jan 2024 11:40:42 +0100 Subject: [PATCH 5/8] wip --- stam.pyi | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/stam.pyi b/stam.pyi index 79a9c5b..b014043 100644 --- a/stam.pyi +++ b/stam.pyi @@ -155,9 +155,24 @@ class AnnotationStore: def annotations(self, *args, **kwargs) -> Annotations: """Returns an iterator over all annotations (:class:`Annotation`) in this store. - Filtering can be applied using keyword arguments. It is recommended to only use this method if you apply further filtering, otherwise the memory overhead may be very large if you have many annotations. + Filtering can be applied using positional arguments and/or keyword arguments. It is recommended to only use this method if you apply further filtering, otherwise the memory overhead may be very large if you have many annotations. Otherwise you can fall back to a more low-level iterator, :meth:`__iter__` instead + Positional Arguments + ------------------- + + Positional arguments can be of the following types: + + * :class:`AnnotationData` - Returns only annotations that have this exact data (you can only pass this once). + * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. + * :class:`DataKey` - Returns annotations with data matching this key (you can only pass this once). + * a tuple/list of :class:`DataKey` + * :class:`Annotations` - Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) + * :class:`Data` - Returns only annotations with data that is in the provided :obj:`Data` collection. + * :class:`TextSelectionOperator` - Returns only annotations that are in a particular textual relationship with the current one (e.g. overlap,embedding,adjacency,etc). + * a dictionary: + + Keyword Arguments ------------------- From 6e8c3151922c796b592d1ff3573c8e902d294bc5 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 10 Jan 2024 12:19:04 +0100 Subject: [PATCH 6/8] wip --- src/annotation.rs | 4 +- src/annotationdata.rs | 5 +-- src/annotationdataset.rs | 1 - src/annotationstore.rs | 3 +- src/query.rs | 90 ++++++++++++++++++++++++---------------- src/resources.rs | 1 - src/textselection.rs | 4 +- 7 files changed, 59 insertions(+), 49 deletions(-) diff --git a/src/annotation.rs b/src/annotation.rs index da4cf14..f7d14c5 100644 --- a/src/annotation.rs +++ b/src/annotation.rs @@ -810,8 +810,7 @@ impl PyAnnotations { ) .map_err(|e| { StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })? - .0, + })?, ); f(query, store) }) @@ -913,7 +912,6 @@ impl PyAnnotation { annotation.store(), ) .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? - .0 .with_annotationvar("main", annotation.clone()); f(annotation, query) }) diff --git a/src/annotationdata.rs b/src/annotationdata.rs index 630b2f2..6b42836 100644 --- a/src/annotationdata.rs +++ b/src/annotationdata.rs @@ -195,7 +195,6 @@ impl PyDataKey { key.rootstore(), ) .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? - .0 .with_keyvar("main", key.clone()); f(key, query) }) @@ -530,7 +529,6 @@ impl PyAnnotationData { data.rootstore(), ) .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? - .0 .with_datavar("main", data.clone()); f(data, query) }) @@ -972,8 +970,7 @@ impl PyData { ) .map_err(|e| { StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })? - .0, + })?, ); f(query, store) }) diff --git a/src/annotationdataset.rs b/src/annotationdataset.rs index 61e2af1..5fbab17 100644 --- a/src/annotationdataset.rs +++ b/src/annotationdataset.rs @@ -282,7 +282,6 @@ impl PyAnnotationDataSet { dataset.store(), ) .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? - .0 .with_datasetvar("main", dataset.clone()); f(dataset, query) }) diff --git a/src/annotationstore.rs b/src/annotationstore.rs index 564fb4d..b6a2a00 100644 --- a/src/annotationstore.rs +++ b/src/annotationstore.rs @@ -386,8 +386,7 @@ impl PyAnnotationStore { kwargs, store, ) - .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? - .0; + .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))?; f(query, store) }) } diff --git a/src/query.rs b/src/query.rs index 7dc4a9b..b98c7a8 100644 --- a/src/query.rs +++ b/src/query.rs @@ -10,24 +10,18 @@ use crate::error::PyStamError; use crate::textselection::PyTextSelectionOperator; use stam::*; -pub struct ContextVariables(Vec); +const CONTEXTVARNAMES: [&str; 25] = [ + "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", +]; -impl ContextVariables { - pub fn add(&mut self) -> &str { - let var = format!("v{}", self.0.len() + 1); - self.0.push(var); - self.last().expect("was just added") - } - - pub fn last(&self) -> Option<&str> { - self.0.iter().last().map(|x| x.as_str()) - } -} - -impl Default for ContextVariables { - fn default() -> Self { - Self(Vec::new()) - } +fn new_contextvar(used_contextvarnames: &mut usize) -> &'static str { + let varname = CONTEXTVARNAMES + .get(*used_contextvarnames) + .map(|x| *x) + .expect("no free context variables present"); + *used_contextvarnames += 1; + varname } fn add_filter<'store, 'py, 'context>( @@ -35,8 +29,8 @@ fn add_filter<'store, 'py, 'context>( store: &'store AnnotationStore, filter: &'py PyAny, operator: Option>, - contextvariables: &'context mut ContextVariables, -) -> PyResult<()> + mut used_contextvarnames: usize, +) -> PyResult where 'py: 'store, 'context: 'store, @@ -50,7 +44,7 @@ where if key.is_instance_of::() { let key: PyRef<'py, PyDataKey> = filter.extract()?; if let Some(key) = store.key(key.set, key.handle) { - let varname = contextvariables.add(); + let varname = new_contextvar(&mut used_contextvarnames); query.bind_keyvar(varname, key); if let Some(operator) = operator { query.constrain(Constraint::KeyValueVariable( @@ -98,12 +92,12 @@ where None }; if let Some(key) = key { - let varname = contextvariables.add(); + let varname = new_contextvar(&mut used_contextvarnames); query.bind_keyvar(varname, key); - if operator.is_some() { + if let Some(operator) = operator { query.constrain(Constraint::KeyValueVariable( varname, - operator.take().unwrap(), + operator, SelectionQualifier::Normal, )); } else { @@ -137,7 +131,7 @@ where )); } if let Some(data) = store.annotationdata(data.set, data.handle) { - let varname = contextvariables.add(); + let varname = new_contextvar(&mut used_contextvarnames); query.bind_datavar(varname, data); query.constrain(Constraint::DataVariable( varname, @@ -147,7 +141,7 @@ where } else if filter.is_instance_of::() { let key: PyRef<'py, PyDataKey> = filter.extract()?; if let Some(key) = store.key(key.set, key.handle) { - let varname = contextvariables.add(); + let varname = new_contextvar(&mut used_contextvarnames); query.bind_keyvar(varname, key); if let Some(operator) = operator { query.constrain(Constraint::KeyValueVariable( @@ -166,7 +160,7 @@ where } else if filter.is_instance_of::() { let annotation: PyRef<'py, PyAnnotation> = filter.extract()?; if let Some(annotation) = store.annotation(annotation.handle) { - let varname = contextvariables.add(); + let varname = new_contextvar(&mut used_contextvarnames); query.bind_annotationvar(varname, annotation); query.constrain(Constraint::AnnotationVariable( varname, @@ -196,7 +190,7 @@ where "Got filter argument of unexpected type", )); } - Ok(()) + Ok(used_contextvarnames) } fn add_multi_filter<'a>( @@ -236,40 +230,66 @@ fn add_multi_filter<'a>( pub(crate) fn build_query<'store, 'py>( mut query: Query<'store>, - args: &'py PyList, //TODO: implement! + args: &'py PyList, kwargs: Option<&'py PyDict>, store: &'store AnnotationStore, -) -> PyResult<(Query<'store>, ContextVariables)> +) -> PyResult> where 'py: 'store, { - let mut contextvariables = ContextVariables::default(); + let mut used_contextvarnames: usize = 0; let operator = if let Some(kwargs) = kwargs { dataoperator_from_kwargs(kwargs).map_err(|e| PyStamError::new_err(format!("{}", e)))? } else { None }; for filter in args { - add_filter(&mut query, store, filter, operator, &mut contextvariables)?; + used_contextvarnames = add_filter( + &mut query, + store, + filter, + operator.clone(), + used_contextvarnames, + )?; } if let Some(kwargs) = kwargs { if let Ok(Some(filter)) = kwargs.get_item("filter") { - add_filter(&mut query, store, filter, operator, &mut contextvariables)?; + add_filter(&mut query, store, filter, operator, used_contextvarnames)?; } else if let Ok(Some(filter)) = kwargs.get_item("filters") { if filter.is_instance_of::() { let vec = filter.downcast::()?; for filter in vec { - add_filter(&mut query, store, filter, operator, &mut contextvariables)?; + used_contextvarnames = add_filter( + &mut query, + store, + filter, + operator.clone(), + used_contextvarnames, + )?; } } else if filter.is_instance_of::() { let vec = filter.downcast::()?; for filter in vec { - add_filter(&mut query, store, filter, operator, &mut contextvariables)?; + used_contextvarnames = add_filter( + &mut query, + store, + filter, + operator.clone(), + used_contextvarnames, + )?; } } + } else { + add_filter( + &mut query, + store, + kwargs.as_ref(), + operator, + used_contextvarnames, + )?; } } - Ok((query, contextvariables)) + Ok(query) } pub(crate) fn has_filters(args: &PyList, kwargs: Option<&PyDict>) -> bool { diff --git a/src/resources.rs b/src/resources.rs index 3d3cbb9..dae6179 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -532,7 +532,6 @@ impl PyTextResource { resource.store(), ) .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? - .0 .with_resourcevar("main", resource.clone()); f(resource, query) }) diff --git a/src/textselection.rs b/src/textselection.rs index ffacd09..ef71b2c 100644 --- a/src/textselection.rs +++ b/src/textselection.rs @@ -501,7 +501,6 @@ impl PyTextSelection { textselection.rootstore(), ) .map_err(|e| StamError::QuerySyntaxError(format!("{}", e), "(python to query)"))? - .0 .with_textvar("main", textselection.clone()); f(textselection, query) }) @@ -862,8 +861,7 @@ impl PyTextSelections { ) .map_err(|e| { StamError::QuerySyntaxError(format!("{}", e), "(python to query)") - })? - .0, + })?, ); f(query, store) }) From 6c9d0e810258179542037cf72da3c0832e4e4d57 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 10 Jan 2024 12:41:40 +0100 Subject: [PATCH 7/8] implemented DataKeySelector and AnnotationDataSelector #10 --- src/annotation.rs | 2 + src/annotationdata.rs | 37 +++++++++++++ src/annotationdataset.rs | 2 + src/resources.rs | 2 + src/selector.rs | 115 ++++++++++++++++++++++++++++++++++++++- stam.pyi | 6 ++ 6 files changed, 163 insertions(+), 1 deletion(-) diff --git a/src/annotation.rs b/src/annotation.rs index f7d14c5..25206d3 100644 --- a/src/annotation.rs +++ b/src/annotation.rs @@ -92,6 +92,8 @@ impl PyAnnotation { annotation: Some(annotation.handle()), resource: None, dataset: None, + key: None, + data: None, offset: if annotation .as_ref() .target() diff --git a/src/annotationdata.rs b/src/annotationdata.rs index 6b42836..f8d218e 100644 --- a/src/annotationdata.rs +++ b/src/annotationdata.rs @@ -12,6 +12,7 @@ use crate::annotationdataset::PyAnnotationDataSet; use crate::annotationstore::MapStore; use crate::error::PyStamError; use crate::query::*; +use crate::selector::{PySelector, PySelectorKind}; use stam::*; #[pyclass(dict, module = "stam", name = "DataKey")] @@ -145,6 +146,24 @@ impl PyDataKey { fn annotations_count(&self) -> usize { self.map(|key| Ok(key.annotations_count())).unwrap() } + + /// Returns a Selector (DataKeySelector) pointing to this DataKey + fn select(&self) -> PyResult { + self.map(|key| { + Ok(PySelector { + kind: PySelectorKind { + kind: SelectorKind::DataKeySelector, + }, + dataset: None, + annotation: None, + resource: None, + key: Some((key.set().handle(), key.handle())), + data: None, + offset: None, + subselectors: Vec::new(), + }) + }) + } } impl MapStore for PyDataKey { @@ -479,6 +498,24 @@ impl PyAnnotationData { fn annotations_len(&self) -> usize { self.map(|data| Ok(data.annotations_len())).unwrap() } + + /// Returns a Selector (AnnotationDataSelector) pointing to this AnnotationData + fn select(&self) -> PyResult { + self.map(|data| { + Ok(PySelector { + kind: PySelectorKind { + kind: SelectorKind::AnnotationDataSelector, + }, + dataset: None, + annotation: None, + resource: None, + data: Some((data.set().handle(), data.handle())), + key: None, + offset: None, + subselectors: Vec::new(), + }) + }) + } } impl MapStore for PyAnnotationData { diff --git a/src/annotationdataset.rs b/src/annotationdataset.rs index 5fbab17..86d4ce5 100644 --- a/src/annotationdataset.rs +++ b/src/annotationdataset.rs @@ -220,6 +220,8 @@ impl PyAnnotationDataSet { dataset: Some(dataset.handle()), annotation: None, resource: None, + key: None, + data: None, offset: None, subselectors: Vec::new(), }) diff --git a/src/resources.rs b/src/resources.rs index dae6179..9315188 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -302,6 +302,8 @@ impl PyTextResource { resource: Some(resource.handle()), annotation: None, dataset: None, + key: None, + data: None, offset: None, subselectors: Vec::new(), }) diff --git a/src/selector.rs b/src/selector.rs index 19e8508..4bbd430 100644 --- a/src/selector.rs +++ b/src/selector.rs @@ -3,6 +3,7 @@ use pyo3::prelude::*; use pyo3::pyclass::CompareOp; use crate::annotation::PyAnnotation; +use crate::annotationdata::{PyAnnotationData, PyDataKey}; use crate::annotationdataset::PyAnnotationDataSet; use crate::annotationstore::{MapStore, PyAnnotationStore}; use crate::resources::{PyOffset, PyTextResource}; @@ -63,6 +64,8 @@ pub(crate) struct PySelector { pub(crate) resource: Option, pub(crate) annotation: Option, pub(crate) dataset: Option, + pub(crate) key: Option<(AnnotationDataSetHandle, DataKeyHandle)>, + pub(crate) data: Option<(AnnotationDataSetHandle, AnnotationDataHandle)>, pub(crate) offset: Option, pub(crate) subselectors: Vec, } @@ -93,6 +96,26 @@ impl PySelector { .expect("pyselector of type datasetselector must have dataset, was checked on instantiation") .into(), ), + SelectorKind::DataKeySelector => SelectorBuilder::DataKeySelector( + self.key + .expect("pyselector of type datakeyselector must have key, was checked on instantiation") + .0 + .into(), + self.key + .expect("pyselector of type datakeyselector must have key, was checked on instantiation") + .1 + .into(), + ), + SelectorKind::AnnotationDataSelector => SelectorBuilder::AnnotationDataSelector( + self.data + .expect("pyselector of type annotationdataselector must have key, was checked on instantiation") + .0 + .into(), + self.data + .expect("pyselector of type annotationdataselector must have key, was checked on instantiation") + .1 + .into(), + ), SelectorKind::MultiSelector => { SelectorBuilder::MultiSelector(self.subselectors.iter().map(|subselector| subselector.build()).collect()) } @@ -110,12 +133,14 @@ impl PySelector { #[pymethods] impl PySelector { #[new] - #[pyo3(signature = (kind, resource=None, annotation=None, dataset=None, offset=None, subselectors=Vec::new()))] + #[pyo3(signature = (kind, resource=None, annotation=None, dataset=None, key=None, data=None, offset=None, subselectors=Vec::new()))] fn new( kind: &PySelectorKind, resource: Option>, annotation: Option>, dataset: Option>, + key: Option>, + data: Option>, offset: Option>, subselectors: Vec>, ) -> PyResult { @@ -127,6 +152,8 @@ impl PySelector { resource: Some(resource.handle), annotation: None, dataset: None, + key: None, + data: None, offset: None, subselectors: Vec::new(), }) @@ -142,6 +169,8 @@ impl PySelector { annotation: Some(annotation.handle), resource: None, dataset: None, + key: None, + data: None, offset: Some(offset.clone()), subselectors: Vec::new(), }) @@ -151,6 +180,8 @@ impl PySelector { annotation: Some(annotation.handle), resource: None, dataset: None, + key: None, + data: None, offset: None, subselectors: Vec::new(), }) @@ -167,6 +198,8 @@ impl PySelector { resource: Some(resource.handle), annotation: None, dataset: None, + key: None, + data: None, offset: Some(offset.clone()), subselectors: Vec::new(), }) @@ -184,6 +217,8 @@ impl PySelector { resource: None, annotation: None, dataset: Some(dataset.handle), + key: None, + data: None, offset: None, subselectors: Vec::new(), }) @@ -191,6 +226,38 @@ impl PySelector { Err(PyValueError::new_err("'dataset' keyword argument must be specified for DataSetSelector and point to an AnnotationDataSet instance")) } } + SelectorKind::DataKeySelector => { + if let Some(key) = key { + Ok(PySelector { + kind: kind.clone(), + resource: None, + annotation: None, + dataset: None, + key: Some((key.set, key.handle)), + data: None, + offset: None, + subselectors: Vec::new(), + }) + } else { + Err(PyValueError::new_err("'key' keyword argument must be specified for DataKeySelector and point to a DataKey instance")) + } + } + SelectorKind::AnnotationDataSelector => { + if let Some(data) = data { + Ok(PySelector { + kind: kind.clone(), + resource: None, + annotation: None, + dataset: None, + data: Some((data.set, data.handle)), + key: None, + offset: None, + subselectors: Vec::new(), + }) + } else { + Err(PyValueError::new_err("'key' keyword argument must be specified for DataKeySelector and point to a DataKey instance")) + } + } SelectorKind::MultiSelector | SelectorKind::CompositeSelector | SelectorKind::DirectionalSelector => { @@ -202,6 +269,8 @@ impl PySelector { resource: None, annotation: None, dataset: None, + key: None, + data: None, offset: None, subselectors: subselectors.into_iter().map(|sel| sel.clone()).collect(), }) @@ -221,6 +290,8 @@ impl PySelector { Some(resource), None, None, + None, + None, Some(offset), Vec::new(), ) @@ -237,6 +308,8 @@ impl PySelector { None, Some(annotation), None, + None, + None, offset, Vec::new(), ) @@ -251,6 +324,8 @@ impl PySelector { None, None, None, + None, + None, Vec::new(), ) } @@ -264,6 +339,8 @@ impl PySelector { None, Some(annotationset), None, + None, + None, Vec::new(), ) } @@ -278,6 +355,8 @@ impl PySelector { None, None, None, + None, + None, subselectors, ) } @@ -292,6 +371,8 @@ impl PySelector { None, None, None, + None, + None, subselectors, ) } @@ -306,6 +387,8 @@ impl PySelector { None, None, None, + None, + None, subselectors, ) } @@ -370,6 +453,28 @@ impl PySelector { }) } + /// Returns the key this selector points at, if any. + /// Works only for DataKeySelector, returns None otherwise. + /// Requires to explicitly pass the store so the resource can be found. + fn key(&self, store: PyRef) -> Option { + self.key.map(|(set_handle, key_handle)| PyDataKey { + set: set_handle, + handle: key_handle, + store: store.get_store().clone(), + }) + } + + /// Returns the annotationdata this selector points at, if any. + /// Works only for AnnotationDataSelector, returns None otherwise. + /// Requires to explicitly pass the store so the resource can be found. + fn annotationdata(&self, store: PyRef) -> Option { + self.data.map(|(set_handle, data_handle)| PyAnnotationData { + set: set_handle, + handle: data_handle, + store: store.get_store().clone(), + }) + } + /// Returns the annotation this selector points at, if any. /// Works only for AnnotationSelector, returns None otherwise. /// Requires to explicitly pass the store so the resource can be found. @@ -420,6 +525,14 @@ impl PySelector { Selector::AnnotationSelector(a_id, ..) => Some(*a_id), _ => None, }, + key: match selector { + Selector::DataKeySelector(set_id, key_id) => Some((*set_id, *key_id)), + _ => None, + }, + data: match selector { + Selector::AnnotationDataSelector(set_id, data_id) => Some((*set_id, *data_id)), + _ => None, + }, offset: selector.offset(store).map(|offset| PyOffset { offset }), subselectors: if selector.is_complex() { if let Some(subselectors) = selector.subselectors() { diff --git a/stam.pyi b/stam.pyi index b014043..2de9275 100644 --- a/stam.pyi +++ b/stam.pyi @@ -868,6 +868,9 @@ class DataKey: The maximum number of results to return (default: unlimited) """ + def select(self) -> Selector: + """Returns a selector pointing to this key (DataKeySelector)""" + class DataValue: """Encapsulates a value and its type. Held by :class:`AnnotationData`. This type is not a reference but holds the actual value.""" @@ -963,6 +966,9 @@ class AnnotationData: The maximum number of results to return (default: unlimited) """ + def select(self) -> Selector: + """Returns a selector pointing to this data (AnnotationDataSelector)""" + class Data: """ A `Data` object holds an arbitrary collection of annotation data. From 1c811b022ffe3a4607378e994d884c984cd8c54f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 10 Jan 2024 13:01:27 +0100 Subject: [PATCH 8/8] improved filter parsing #8 --- src/annotation.rs | 44 +++--- src/annotationdata.rs | 24 ++-- src/annotationdataset.rs | 6 +- src/annotationstore.rs | 6 +- src/query.rs | 159 ++++++++++++---------- src/resources.rs | 10 +- src/textselection.rs | 24 ++-- stam.pyi | 285 ++++++++++++++++++++------------------- test.py | 4 +- 9 files changed, 297 insertions(+), 265 deletions(-) diff --git a/src/annotation.rs b/src/annotation.rs index 25206d3..33052ed 100644 --- a/src/annotation.rs +++ b/src/annotation.rs @@ -145,7 +145,11 @@ impl PyAnnotation { /// Note that this will always return a list (even it if only contains a single element), /// as an annotation may reference multiple text selections. #[pyo3(signature = (*args, **kwargs))] - fn textselections(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn textselections( + &self, + args: &PyTuple, + kwargs: Option<&PyDict>, + ) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|annotation| { @@ -180,7 +184,7 @@ impl PyAnnotation { #[pyo3(signature = (*args, **kwargs))] fn annotations_in_targets( &self, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); @@ -212,7 +216,7 @@ impl PyAnnotation { /// Returns annotations that are referring to this annotation (i.e. others using an AnnotationSelector) #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|annotation| { @@ -244,7 +248,7 @@ impl PyAnnotation { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|annotation| Ok(annotation.annotations().test())) } else { @@ -265,7 +269,7 @@ impl PyAnnotation { #[pyo3(signature = (*args, **kwargs))] fn test_annotations_in_targets( &self, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let recursive = get_recursive(kwargs, AnnotationDepth::One); @@ -354,7 +358,7 @@ impl PyAnnotation { /// Returns annotation data instances that pertain to this annotation. #[pyo3(signature = (*args, **kwargs))] - fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|annotation| { @@ -386,7 +390,7 @@ impl PyAnnotation { } #[pyo3(signature = (*args, **kwargs))] - fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|annotation| Ok(annotation.data().test())) } else { @@ -414,7 +418,7 @@ impl PyAnnotation { fn related_text( &self, operator: PyTextSelectionOperator, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); @@ -492,7 +496,7 @@ impl PyAnnotations { /// Returns annotation data instances used by the annotations in this collection. #[pyo3(signature = (*args, **kwargs))] - fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|annotations, store| { @@ -517,7 +521,7 @@ impl PyAnnotations { } #[pyo3(signature = (*args, **kwargs))] - fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|annotations, _| Ok(annotations.items().data().test())) } else { @@ -536,7 +540,7 @@ impl PyAnnotations { } #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|annotations, store| { @@ -561,7 +565,7 @@ impl PyAnnotations { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|annotations, _| Ok(annotations.items().annotations().test())) } else { @@ -582,7 +586,7 @@ impl PyAnnotations { #[pyo3(signature = (*args, **kwargs))] fn annotations_in_targets( &self, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); @@ -611,7 +615,7 @@ impl PyAnnotations { #[pyo3(signature = (*args, **kwargs))] fn test_annotations_in_targets( &self, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let recursive = get_recursive(kwargs, AnnotationDepth::One); @@ -631,7 +635,11 @@ impl PyAnnotations { } #[pyo3(signature = (*args,**kwargs))] - fn textselections(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn textselections( + &self, + args: &PyTuple, + kwargs: Option<&PyDict>, + ) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|annotations, store| { @@ -666,7 +674,7 @@ impl PyAnnotations { fn related_text( &self, operator: PyTextSelectionOperator, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); @@ -788,7 +796,7 @@ impl PyAnnotations { &self, resulttype: Type, constraint: Constraint, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result @@ -898,7 +906,7 @@ impl PyAnnotation { &self, resulttype: Type, constraint: Constraint, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result diff --git a/src/annotationdata.rs b/src/annotationdata.rs index f8d218e..b1c253c 100644 --- a/src/annotationdata.rs +++ b/src/annotationdata.rs @@ -83,7 +83,7 @@ impl PyDataKey { } #[pyo3(signature = (*args, **kwargs))] - fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|key| Ok(PyData::from_iter(key.data().limit(limit), &self.store))) @@ -100,7 +100,7 @@ impl PyDataKey { } #[pyo3(signature = (*args, **kwargs))] - fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|key| Ok(key.data().test())) } else { @@ -111,7 +111,7 @@ impl PyDataKey { } #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|key| { @@ -133,7 +133,7 @@ impl PyDataKey { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|key| Ok(key.annotations().test())) } else { @@ -198,7 +198,7 @@ impl PyDataKey { fn map_with_query( &self, resulttype: Type, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result @@ -463,7 +463,7 @@ impl PyAnnotationData { } #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|data| { @@ -485,7 +485,7 @@ impl PyAnnotationData { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|key| Ok(key.annotations().test())) } else { @@ -550,7 +550,7 @@ impl PyAnnotationData { fn map_with_query( &self, resulttype: Type, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result @@ -900,7 +900,7 @@ impl PyData { } #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|data, store| { @@ -917,7 +917,7 @@ impl PyData { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|data, _| Ok(data.items().annotations().test())) } else { @@ -948,7 +948,7 @@ impl PyData { wrappedstore: &Arc>, limit: Option, ) -> Self { - assert!(query.resulttype() == Some(Type::Annotation)); + assert!(query.resulttype() == Some(Type::AnnotationData)); Self { data: store .query(query) @@ -984,7 +984,7 @@ impl PyData { fn map_with_query( &self, resulttype: Type, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result diff --git a/src/annotationdataset.rs b/src/annotationdataset.rs index 86d4ce5..02d9636 100644 --- a/src/annotationdataset.rs +++ b/src/annotationdataset.rs @@ -173,7 +173,7 @@ impl PyAnnotationDataSet { } #[pyo3(signature = (*args, **kwargs))] - fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|dataset| Ok(PyData::from_iter(dataset.data().limit(limit), &self.store))) @@ -196,7 +196,7 @@ impl PyAnnotationDataSet { } #[pyo3(signature = (*args, **kwargs))] - fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|dataset| Ok(dataset.data().test())) } else { @@ -268,7 +268,7 @@ impl PyAnnotationDataSet { &self, resulttype: Type, constraint: Constraint, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result diff --git a/src/annotationstore.rs b/src/annotationstore.rs index b6a2a00..097133f 100644 --- a/src/annotationstore.rs +++ b/src/annotationstore.rs @@ -247,7 +247,7 @@ impl PyAnnotationStore { } #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|store| { @@ -300,7 +300,7 @@ impl PyAnnotationStore { } #[pyo3(signature = (*args, **kwargs))] - fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|store| Ok(PyData::from_iter(store.data().limit(limit), &self.store))) @@ -372,7 +372,7 @@ impl PyAnnotationStore { fn map_with_query( &self, resulttype: Type, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result diff --git a/src/query.rs b/src/query.rs index b98c7a8..27dae53 100644 --- a/src/query.rs +++ b/src/query.rs @@ -7,7 +7,6 @@ use crate::annotation::{PyAnnotation, PyAnnotations}; use crate::annotationdata::{dataoperator_from_kwargs, PyAnnotationData, PyData, PyDataKey}; use crate::annotationdataset::PyAnnotationDataSet; use crate::error::PyStamError; -use crate::textselection::PyTextSelectionOperator; use stam::*; const CONTEXTVARNAMES: [&str; 25] = [ @@ -40,58 +39,13 @@ where let operator = dataoperator_from_kwargs(filter) .map_err(|err| PyValueError::new_err(format!("{}", err)))? .or(operator); - let key = filter.get_item("key")?.expect("already checked"); - if key.is_instance_of::() { - let key: PyRef<'py, PyDataKey> = filter.extract()?; - if let Some(key) = store.key(key.set, key.handle) { - let varname = new_contextvar(&mut used_contextvarnames); - query.bind_keyvar(varname, key); - if let Some(operator) = operator { - query.constrain(Constraint::KeyValueVariable( - varname, - operator, - SelectionQualifier::Normal, - )); - } else { - query.constrain(Constraint::KeyVariable(varname, SelectionQualifier::Normal)); - } - } else { - return Err(PyValueError::new_err( - "Passed DataKey instance is invalid (should never happen)", - )); - } - } else if filter.contains("set")? { - if key.is_instance_of::() { - let key = key.downcast::()?; - let set = filter.get_item("set")?.expect("already checked"); - let key = if set.is_instance_of::() { - let set: PyRef<'py, PyAnnotationDataSet> = filter.extract()?; - if let Some(dataset) = store.dataset(set.handle) { - if let Some(key) = dataset.key(key.to_str()?) { - Some(key) - } else { - return Err(PyValueError::new_err("specified key not found in set")); - } - } else { - return Err(PyValueError::new_err( - "Passed AnnotationDataSet instance is invalid (should never happen)", - )); - } - } else if set.is_instance_of::() { - let set = set.downcast::()?; - if let Some(dataset) = store.dataset(set.to_str()?) { - if let Some(key) = dataset.key(key.to_str()?) { - Some(key) - } else { - return Err(PyValueError::new_err("specified key not found in set")); - } - } else { - return Err(PyValueError::new_err("specified dataset not found")); - } - } else { - None - }; - if let Some(key) = key { + if filter.contains("key")? { + let key = filter + .get_item("key")? + .expect("key field was checked to exist in filter"); + if key.is_instance_of::() { + let key: PyRef<'py, PyDataKey> = filter.extract()?; + if let Some(key) = store.key(key.set, key.handle) { let varname = new_contextvar(&mut used_contextvarnames); query.bind_keyvar(varname, key); if let Some(operator) = operator { @@ -106,23 +60,78 @@ where SelectionQualifier::Normal, )); } + } else { + return Err(PyValueError::new_err( + "Passed DataKey instance is invalid (should never happen)", + )); + } + } else if filter.contains("set")? { + if key.is_instance_of::() { + let key = key.downcast::()?; + let set = filter.get_item("set")?.expect("already checked"); + let key = if set.is_instance_of::() { + let set: PyRef<'py, PyAnnotationDataSet> = filter.extract()?; + if let Some(dataset) = store.dataset(set.handle) { + if let Some(key) = dataset.key(key.to_str()?) { + Some(key) + } else { + return Err(PyValueError::new_err( + "specified key not found in set", + )); + } + } else { + return Err(PyValueError::new_err( + "Passed AnnotationDataSet instance is invalid (should never happen)", + )); + } + } else if set.is_instance_of::() { + let set = set.downcast::()?; + if let Some(dataset) = store.dataset(set.to_str()?) { + if let Some(key) = dataset.key(key.to_str()?) { + Some(key) + } else { + return Err(PyValueError::new_err( + "specified key not found in set", + )); + } + } else { + return Err(PyValueError::new_err("specified dataset not found")); + } + } else { + None + }; + if let Some(key) = key { + let varname = new_contextvar(&mut used_contextvarnames); + query.bind_keyvar(varname, key); + if let Some(operator) = operator { + query.constrain(Constraint::KeyValueVariable( + varname, + operator, + SelectionQualifier::Normal, + )); + } else { + query.constrain(Constraint::KeyVariable( + varname, + SelectionQualifier::Normal, + )); + } + } + } else { + return Err(PyValueError::new_err( + "'key' parameter in filter dictionary should be of type `str`", + )); } - } else { - return Err(PyValueError::new_err( - "'key' parameter in filter dictionary should be of type `str`", - )); } - } else { - return Err(PyValueError::new_err( - "'key' parameter in filter dictionary should be of type DataKey, it can also be `str` if you also provide `set` as well", - )); + } else if let Some(operator) = operator { + //no key specified but we do have an operator + query.constrain(Constraint::Value(operator, SelectionQualifier::Normal)); } } else if filter.is_instance_of::() { let vec: Vec<&PyAny> = filter.extract()?; - add_multi_filter(query, store, vec)?; + used_contextvarnames = add_multi_filter(query, store, vec, used_contextvarnames)?; } else if filter.is_instance_of::() { let vec: Vec<&PyAny> = filter.extract()?; - add_multi_filter(query, store, vec)?; + used_contextvarnames = add_multi_filter(query, store, vec, used_contextvarnames)?; } else if filter.is_instance_of::() { let data: PyRef<'_, PyAnnotationData> = filter.extract()?; if operator.is_some() { @@ -196,8 +205,9 @@ where fn add_multi_filter<'a>( query: &mut Query<'a>, store: &'a AnnotationStore, - filter: Vec<&PyAny>, -) -> PyResult<()> { + filter: Vec<&'a PyAny>, + mut used_contextvarnames: usize, +) -> PyResult { if filter.iter().all(|x| x.is_instance_of::()) { query.constrain(Constraint::Annotations( Handles::from_iter( @@ -224,13 +234,17 @@ fn add_multi_filter<'a>( ), SelectionQualifier::Normal, )); + } else { + for item in filter.iter() { + used_contextvarnames = add_filter(query, store, item, None, used_contextvarnames)?; + } } - Ok(()) + Ok(used_contextvarnames) } pub(crate) fn build_query<'store, 'py>( mut query: Query<'store>, - args: &'py PyList, + args: &'py PyTuple, kwargs: Option<&'py PyDict>, store: &'store AnnotationStore, ) -> PyResult> @@ -243,7 +257,9 @@ where } else { None }; + let mut has_args = false; for filter in args { + has_args = true; used_contextvarnames = add_filter( &mut query, store, @@ -254,8 +270,10 @@ where } if let Some(kwargs) = kwargs { if let Ok(Some(filter)) = kwargs.get_item("filter") { + //backwards compatibility add_filter(&mut query, store, filter, operator, used_contextvarnames)?; } else if let Ok(Some(filter)) = kwargs.get_item("filters") { + //backwards compatibility if filter.is_instance_of::() { let vec = filter.downcast::()?; for filter in vec { @@ -279,12 +297,13 @@ where )?; } } - } else { + } else if !has_args { + //we have no args, handle kwargs standalone add_filter( &mut query, store, kwargs.as_ref(), - operator, + None, used_contextvarnames, )?; } @@ -292,7 +311,7 @@ where Ok(query) } -pub(crate) fn has_filters(args: &PyList, kwargs: Option<&PyDict>) -> bool { +pub(crate) fn has_filters(args: &PyTuple, kwargs: Option<&PyDict>) -> bool { if !args.is_empty() { return true; } diff --git a/src/resources.rs b/src/resources.rs index 9315188..f3a69fd 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -381,7 +381,7 @@ impl PyTextResource { /// Returns annotations that are referring to this resource via a TextSelector #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|resource| { @@ -411,7 +411,7 @@ impl PyTextResource { #[pyo3(signature = (*args, **kwargs))] fn annotations_as_metadata( &self, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); @@ -441,7 +441,7 @@ impl PyTextResource { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|resource| Ok(resource.annotations().test())) } else { @@ -458,7 +458,7 @@ impl PyTextResource { #[pyo3(signature = (*args, **kwargs))] fn test_annotations_as_metadata( &self, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { if !has_filters(args, kwargs) { @@ -518,7 +518,7 @@ impl PyTextResource { &self, resulttype: Type, constraint: Constraint, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result diff --git a/src/textselection.rs b/src/textselection.rs index ef71b2c..2063bf3 100644 --- a/src/textselection.rs +++ b/src/textselection.rs @@ -297,7 +297,7 @@ impl PyTextSelection { } #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|textselection| { @@ -325,7 +325,7 @@ impl PyTextSelection { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|textselection| Ok(textselection.annotations().test())) } else { @@ -340,7 +340,7 @@ impl PyTextSelection { } #[pyo3(signature = (*args, **kwargs))] - fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|textselection| Ok(textselection.annotations().data().test())) } else { @@ -358,7 +358,7 @@ impl PyTextSelection { fn related_text( &self, operator: PyTextSelectionOperator, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); @@ -485,7 +485,7 @@ impl PyTextSelection { &self, resulttype: Type, constraint: Constraint, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result @@ -594,7 +594,7 @@ impl PyTextSelections { } #[pyo3(signature = (*args, **kwargs))] - fn annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|textselections, store| { @@ -619,7 +619,7 @@ impl PyTextSelections { } #[pyo3(signature = (*args, **kwargs))] - fn test_annotations(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_annotations(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|annotations, _| { Ok(annotations @@ -640,7 +640,7 @@ impl PyTextSelections { } #[pyo3(signature = (*args, **kwargs))] - fn data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { let limit = get_limit(kwargs); if !has_filters(args, kwargs) { self.map(|textselections, store| { @@ -666,7 +666,7 @@ impl PyTextSelections { } #[pyo3(signature = (*args, **kwargs))] - fn test_data(&self, args: &PyList, kwargs: Option<&PyDict>) -> PyResult { + fn test_data(&self, args: &PyTuple, kwargs: Option<&PyDict>) -> PyResult { if !has_filters(args, kwargs) { self.map(|textselections, _| { Ok(textselections @@ -691,7 +691,7 @@ impl PyTextSelections { fn related_text( &self, operator: PyTextSelectionOperator, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, ) -> PyResult { let limit = get_limit(kwargs); @@ -779,7 +779,7 @@ impl PyTextSelections { wrappedstore: &Arc>, limit: Option, ) -> Self { - assert!(query.resulttype() == Some(Type::Annotation)); + assert!(query.resulttype() == Some(Type::TextSelection)); Self { textselections: store .query(query) @@ -838,7 +838,7 @@ impl PyTextSelections { &self, resulttype: Type, constraint: Constraint, - args: &PyList, + args: &PyTuple, kwargs: Option<&PyDict>, f: F, ) -> Result diff --git a/stam.pyi b/stam.pyi index 2de9275..11ef41b 100644 --- a/stam.pyi +++ b/stam.pyi @@ -163,35 +163,22 @@ class AnnotationStore: Positional arguments can be of the following types: - * :class:`AnnotationData` - Returns only annotations that have this exact data (you can only pass this once). - * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. - * :class:`DataKey` - Returns annotations with data matching this key (you can only pass this once). - * a tuple/list of :class:`DataKey` - * :class:`Annotations` - Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) - * :class:`Data` - Returns only annotations with data that is in the provided :obj:`Data` collection. - * :class:`TextSelectionOperator` - Returns only annotations that are in a particular textual relationship with the current one (e.g. overlap,embedding,adjacency,etc). - * a dictionary: - + * :class:`DataKey` - Returns annotations with data matching this key + * :class:`AnnotationData` - Returns only annotations that have this exact data + * :class:`Annotations` or a tuple/list of :class:`Annotation`- Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) + * :class:`Data` a tuple/list of :class:`AnnotationData `- Returns only annotations with data that is in the provided collection. + * a dictionary: + * `set` - An ID of an dataset (or a :class:`DataAnnotationSet` instance), only needed when specifying `key` as a string (see below) + * `key` - An key, either an instance of :class:`DataKey` or a string, in the latter case you need to specify `set` as well. + * `value` (see keyword arguments below) Keyword Arguments ------------------- limit: Optional[int] = None The maximum number of results to return (default: unlimited) - filter: Union[AnnotationData,Tuple[AnnotationData],List[AnnotationData],DataKey,Annotations,Data,TextSelectionOperator] - If you want to add multiple different filters, use `filters` instead. - Filter annotations based on: - - * :class:`AnnotationData` - Returns only annotations that have this exact data (you can only pass this once). - * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. - * :class:`DataKey` - Returns annotations with data matching this key (you can only pass this once). - * a tuple/list of :class:`DataKey` - * :class:`Annotations` - Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) - * :class:`Data` - Returns only annotations with data that is in the provided :obj:`Data` collection. - * :class:`TextSelectionOperator` - Returns only annotations that are in a particular textual relationship with the current one (e.g. overlap,embedding,adjacency,etc). - filters: List[Union[AnnotationData,DataKey,Annotations,Data]] value: Optional[Union[str,int,float,bool]] - Constrain the search to annotations with data of a certain value. This can only be used with `filter=DataKey`. + Constrain the search to annotations with data of a certain value. This can only be used when you also pass a :class:`DataKey` as filter. This holds the exact value to search for, there are other variants of this keyword available, see :meth:`data` for a full list. """ @@ -216,26 +203,30 @@ class AnnotationStore: """Reallocates internal data structures to tight fits to conserve memory space (if necessary). You can use this after having added lots of annotations to possibly reduce the memory consumption.""" def data(self, *args, **kwargs) -> Data: - """Returns an iterator over all annotations (:class:`Annotation`) in this store. + """Returns an iterator over all data (:class:`AnnotationData`) in this store. + + Filtering can be applied using positional arguments and/or keyword arguments. It is recommended to only use this method if you apply further filtering, otherwise the memory overhead may be very large if you have a lot of data. + + Positional Arguments + ------------------- + + Positional arguments can be of the following types: - Filtering can be applied using keyword arguments. It is recommended to only use this method if you apply further filtering, otherwise the memory overhead may be very large if you have a lot of data. + * :class:`DataKey` - Returns data matching this key + * :class:`Annotation` - Returns data referenced by the mentioned annotation + * :class:`AnnotationData` - Returns only this exact data. Not very useful, use :meth:`test_data` instead. + * :class:`Annotations` or a tuple/list of :class:`Annotation` - Returns data references by annotations in the provided collection. + * :class:`Data` a tuple/list of :class:`AnnotationData `- Returns only data that is in the provided :obj:`Data` collection (intersection) + * a dictionary: + * `set` - An ID of an dataset (or a :class:`DataAnnotationSet` instance), only needed when specifying `key` as a string (see below) + * `key` - An key, either an instance of :class:`DataKey` or a string, in the latter case you need to specify `set` as well. + * `value` or variants (see keyword arguments below) Keyword Arguments ------------------- limit: Optional[int] = None The maximum number of results to return (default: unlimited) - filter: Union[AnnotationData,Tuple[AnnotationData],List[AnnotationData],DataKey,Annotations,Data] - If you want to add multiple different filters, use `filters` instead. - Filter annotations based on: - - * :class:`AnnotationData` - Returns only this exact data (you can only pass this once). Use :meth:`test_data` instead. - * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. - * :class:`DataKey` - Returns data matching this key (you can only pass this once). - * a tuple/list of :class:`DataKey` - * :class:`Annotations` - Returns data that is used by by annotations in the provided :obj:`Annotations` collection. - * :class:`Data` - Returns only data that is in the provided :obj:`Data` collection. - filters: List[Union[AnnotationData,DataKey,Annotations,Data]] value: Optional[Union[str,int,float,bool,List[Union[str,int,float,bool]]]] Search for data matching a specific value. This holds exact value to search for. Further variants of this keyword are listed below: @@ -315,7 +306,7 @@ class AnnotationStore: class Annotation: """ `Annotation` represents a particular *instance of annotation* and is the central - concept of the model. They can be considered the primary nodes of the graph model. The + concept of the model. Annotations can be considered the primary nodes of the graph model. The instance of annotation is strictly decoupled from the *data* or key/value of the annotation (:class:`AnnotationData`). After all, multiple instances can be annotated with the same label (multiple annotations may share the same annotation data). @@ -366,33 +357,36 @@ class Annotation: Text selections will be returned in textual order, except if a DirectionalSelector was used. - Text selections may be filtered using the following keyword arguments: + Text selections may be filtered using the following positionl and/or keyword arguments: + + Positional Arguments + ------------------- + + Positional arguments can be of the following types: + + * :class:`DataKey` - Returns text selections referenced by annotations with data matching this key + * :class:`AnnotationData` - Returns text selections referenced by annotations that have this exact data + * :class:`Annotations` or a tuple/list of :class:`Annotation`- Returns text selections referenced by any annotations that are already in the provided :obj:`Annotations` collection (intersection) + * :class:`Data` a tuple/list of :class:`AnnotationData `- Returns only textselections referenced by annotations with data that is in the provided collection. + * a dictionary: + * `set` - An ID of an dataset (or a :class:`DataAnnotationSet` instance), only needed when specifying `key` as a string (see below) + * `key` - An key, either an instance of :class:`DataKey` or a string, in the latter case you need to specify `set` as well. + * `value` (see keyword arguments below) Keyword Arguments ------------------- limit: Optional[int] = None The maximum number of results to return (default: unlimited) - filter: Union[AnnotationData,Tuple[AnnotationData],List[AnnotationData],DataKey,Annotations,Data,TextSelectionOperator] - If you want to add multiple different filters, use `filters` instead. - Filter annotations based on: - - * :class:`AnnotationData` - Returns only text selections referenced by annotations that have this exact data (you can only pass this once). - * a tuple/list of :class:`AnnotationData` - Returns only text selections referenced by annotations with data that matches one of the items in the tuple/list. - * :class:`DataKey` - Returns text selections referenced by annotations with data matching this key (you can only pass this once). - * a tuple/list of :class:`DataKey` - * :class:`Annotations` - Returns only text selections referenced by annotations that are already in the provided :obj:`Annotations` collection (intersection) - * :class:`Data` - Returns only text selections referenced by annotations with data that is in the provided :obj:`Data` collection. - filters: List[Union[AnnotationData,DataKey,Annotations,Data]] value: Optional[Union[str,int,float,bool]] - Constrain the search to text selections referenced by annotations with data of a certain value. This can only be used with `filter=DataKey`. + Constrain the search to text selections referenced by annotations with data of a certain value. This is usually used together with passing a :obj:`DataKey` as filter in the positional arguments. This holds the exact value to search for, there are other variants of this keyword available, see :meth:`data` for a full list. """ - def annotations_in_targets(self, recursive= False, limit: Optional[int] = None) -> Annotations: + def annotations_in_targets(self, *args, **kwargs) -> Annotations: """Returns annotations (:class:`Annotations` containing :class:`Annotation`) this annotation refers to (i.e. using an *AnnotationSelector*) - The annotations can be filtered using keyword arguments; see :meth:`annotations`. One extra keyword argument is available for this method (see below). + The annotations can be filtered using positional and/or keyword arguments; see :meth:`annotations`. One extra keyword argument is available for this method (see below). Annotations will returned be in textual order unless recursive is set or a DirectionalSelector is involved. @@ -403,50 +397,51 @@ class Annotation: Follow AnnotationSelectors recursively (default False) """ - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """ Returns annotations (:class:`Annotations` containing :class:`Annotation`) that are referring to this annotation (i.e. others using an AnnotationSelector). - The annotations can be filtered using keyword arguments. + The annotations can be filtered using positional and/or keyword arguments. + + Positional Arguments + ------------------- + + Positional arguments can be of the following types: + + * :class:`DataKey` - Returns annotations with data matching this key + * :class:`AnnotationData` - Returns only annotations that have this exact data + * :class:`Annotations` or a tuple/list of :class:`Annotation`- Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) + * :class:`Data` a tuple/list of :class:`AnnotationData `- Returns only annotations with data that is in the provided collection. + * a dictionary: + * `set` - An ID of an dataset (or a :class:`DataAnnotationSet` instance), only needed when specifying `key` as a string (see below) + * `key` - An key, either an instance of :class:`DataKey` or a string, in the latter case you need to specify `set` as well. + * `value` or variants (see keyword arguments below) Keyword Arguments ------------------- limit: Optional[int] = None The maximum number of results to return (default: unlimited) - filter: Union[AnnotationData,Tuple[AnnotationData],List[AnnotationData],DataKey,Annotations,Data] - If you want to add multiple different filters, use `filters` instead. - Filter annotations based on: - - * :class:`AnnotationData` - Returns only annotations that have this exact data (you can only pass this once). - * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. - * :class:`DataKey` - Returns annotations with data matching this key (you can only pass this once). - * a tuple/list of :class:`DataKey` - * :class:`Annotations` - Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) - * :class:`Data` - Returns only annotations with data that is in the provided :obj:`Data` collection. - filters: List[Union[AnnotationData,DataKey,Annotations,Data]] value: Optional[Union[str,int,float,bool]] - Constrain the search to annotations with data of a certain value. This can only be used with `filter=DataKey`. + Constrain the search to annotations with data of a certain value. This is usually used together with passing a :obj:`DataKey` as filter in the positional arguments. This holds the exact value to search for, there are other variants of this keyword available, see :meth:`data` for a full list. - Example --------- Filter by data key and value:: key = store.dataset("linguistic-set").key("part-of-speech") - for annotation in store.annotations(filter=key, value="noun"): + for annotation in store.annotations(key, value="noun"): ... But if you already have the key, like in the example above, you may just as well do (more efficient):: for annotation in key.annotations(value="noun"): ... - """ - def test_annotations(self, **kwargs) -> bool: + def test_annotations(self, *args, **kwargs) -> bool: """ Tests whwther there are annotations (:class:`Annotations` containing :class:`Annotation`) that are referring to this annotation (i.e. others using an AnnotationSelector). This method is like :meth:`annotations`, but only tests and does not return the annotations, as such it is more performant. @@ -493,7 +488,7 @@ class Annotation: def selector_kind(self) -> SelectorKind: """Returns the type of the selector of this annotation""" - def data(self, **kwargs) -> Data: + def data(self, *args, **kwargs) -> Data: """ Returns annotation data (:class:`Data` containing :class:`AnnotationData`) used by this annotation. @@ -501,22 +496,26 @@ class Annotation: the data, then just iterating over the annotation directly (:meth:`__iter__`) will be more efficient. Do note that implementing any filtering yourself in Python is much less performant than letting this data method do it for you. + Positional Arguments + ------------------------- + + Positional arguments can be of the following types: + + * :class:`DataKey` - Returns data matching this key + * :class:`Annotation` - Returns data referenced by the mentioned annotation + * :class:`AnnotationData` - Returns only this exact data. Not very useful, use :meth:`test_data` instead. + * :class:`Annotations` or a tuple/list of :class:`Annotation` - Returns data references by annotations in the provided collection. + * :class:`Data` a tuple/list of :class:`AnnotationData `- Returns only data that is in the provided :obj:`Data` collection (intersection) + * a dictionary: + * `set` - An ID of an dataset (or a :class:`DataAnnotationSet` instance), only needed when specifying `key` as a string (see below) + * `key` - An key, either an instance of :class:`DataKey` or a string, in the latter case you need to specify `set` as well. + * `value` or variants (see keyword arguments below) + Keyword Arguments ------------------- limit: Optional[int] = None The maximum number of results to return (default: unlimited) - filter: Union[AnnotationData,Tuple[AnnotationData],List[AnnotationData],DataKey,Annotations,Data] - If you want to add multiple different filters, use `filters` instead. - Filter annotations based on: - - * :class:`AnnotationData` - Returns only this exact data (you can only pass this once). Use :meth:`test_data` instead. - * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. - * :class:`DataKey` - Returns data matching this key (you can only pass this once). - * a tuple/list of :class:`DataKey` - * :class:`Annotations` - Returns data that is used by by annotations in the provided :obj:`Annotations` collection. - * :class:`Data` - Returns only data that is in the provided :obj:`Data` collection. - filters: List[Union[AnnotationData,DataKey,Annotations,Data]] value: Optional[Union[str,int,float,bool,List[Union[str,int,float,bool]]]] Search for data matching a specific value. This holds exact value to search for. Further variants of this keyword are listed below: @@ -549,20 +548,20 @@ class Annotation: """ - def test_data(self, **kwargs) -> bool: + def test_data(self, *args, **kwargs) -> bool: """ Tests whether certain annotation data is used by this annotation. - The data can be filtered using keyword arguments. See :meth:`data`. + The data can be filtered using positional and/or keyword arguments. See :meth:`data`. Unlike :meth:`data`, this method merely tests without returning the data, and as such is more performant. """ - def related_text(self, operator: TextSelectionOperator, **kwargs) -> TextSelections: + def related_text(self, operator: TextSelectionOperator, *args, **kwargs) -> TextSelections: """ Applies a :class:`TextSelectionOperator` to find all other text selections who are in a specific relation with the ones from the current annotation. Returns a collection :class:`TextSelections` containing all matching :class:`TextSelection` instances. - Text selections will be returned in textual order. They may be filtered via keyword arguments. See :meth:`Annotation.textselections`. + Text selections will be returned in textual order. They may be filtered via positional and/or keyword arguments. See :meth:`Annotation.textselections`. If you are interested in the annotations associated with the found text selections, then add `.annotations()` to the result. @@ -628,26 +627,26 @@ class Annotations: def is_sorted(self) -> bool: """Returns a boolean indicating whether the annotations in this collection are sorted chronologically (earlier annotations before later once). Note that this is distinct from any textual ordering.""" - def data(self, **kwargs) -> Data: + def data(self, *args, **kwargs) -> Data: """ Returns annotation data (:class:`Data` containing :class:`AnnotationData`) used by annotations in this collection. - The data can be filtered using keyword arguments; see :meth:`Annotation.data`. + The data can be filtered using positional and/or keyword arguments; see :meth:`Annotation.data`. If no filters are set (default), all data from all annotations are returned (without duplicates). """ - def test_data(self, **kwargs) -> bool: + def test_data(self, *args, **kwargs) -> bool: """ Tests whether certain annotation data is used by any annotation in this collection. The data can be filtered using keyword arguments. See :meth:`data`. Unlike :meth:`data`, this method merely tests without returning the data, and as such is more performant. """ - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """ Returns annotations (:class:`Annotations` containing :class:`Annotation`) that reference annotations in the current collection (e.g. annotations that target of the current any annotations using an AnnotationSelector). - The annotations can be filtered using keyword arguments; see :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments; see :meth:`Annotation.annotations`. If no filters are set (default), all annotations are returned (without duplicates) in chronological order. Example @@ -661,11 +660,11 @@ class Annotations: ... """ - def annotations_in_targets(self, **kwargs) -> Annotations: + def annotations_in_targets(self, *args, **kwargs) -> Annotations: """ Returns annotations (:class:`Annotations` containing :class:`Annotation`) that are being referenced by annotations in the current collection (e.g. annotations we target using an AnnotationSelector). - The annotations can be filtered using keyword arguments; see :meth:`Annotation.annotations`. One extra keyword argument is available and explained below. + The annotations can be filtered using positional and/or keyword arguments; see :meth:`Annotation.annotations`. One extra keyword argument is available and explained below. If no filters are set (default), all annotations are returned (without duplicates). Annotations are returned in chronological order. @@ -676,17 +675,17 @@ class Annotations: Follow AnnotationSelectors recursively (default False) """ - def test_annotation(self, **kwargs) -> bool: + def test_annotation(self, *args, **kwargs) -> bool: """ Tests whether certain annotations reference any annotation in this collection. - The annotation can be filtered using keyword arguments. See :meth:`annotations`. + The annotation can be filtered using positional and/or keyword arguments. See :meth:`annotations`. Unlike :meth:`annotations`, this method merely tests without returning the data, and as such is more performant. """ - def test_annotations_in_targets(self, **kwargs) -> Annotations: + def test_annotations_in_targets(self, *args, **kwargs) -> Annotations: """ Tests whether annotations in this collection targets the specified annotation. - The annotation can be filtered using keyword arguments. See :meth:`annotations`. + The annotation can be filtered using positional and/or keyword arguments. See :meth:`annotations`. Unlike :meth:`annotations_in_targets`, this method merely tests without returning the data, and as such is more performant. """ @@ -757,18 +756,18 @@ class AnnotationDataSet: def __iter__(self) -> Iterator[AnnotationData]: """Returns an iterator over all :class:`AnnotationData` in the dataset. If you want to do any filtering, use :meth:`data` instead.""" - def data(self, **kwargs) -> Data: + def data(self, *args, **kwargs) -> Data: """ Returns annotation data (:class:`Data` containing :class:`AnnotationData`) used by this key. - The data can be filtered using keyword arguments. See :meth:`Annotation.data`. + The data can be filtered using positional and/or keyword arguments. See :meth:`Annotation.data`. If you don't intend to do any filtering at all, then just using :meth:`__iter__` may be faster. """ - def test_data(self, **kwargs) -> bool: + def test_data(self, *args, **kwargs) -> bool: """ Tests whether certain annotation data exists in this set. - The data can be filtered using keyword arguments. See :meth:`Annotation.data`. + The data can be filtered using positional and/or keyword arguments. See :meth:`Annotation.data`. This method is like :meth:`data`, but merely tests without returning the data, and as such is more performant. """ @@ -793,11 +792,11 @@ class DataKey: def dataset(self) -> AnnotationDataSet: """Returns the :class:`AnnotationDataSet` this key is part of""" - def data(self, **kwargs) -> Data: + def data(self, *args, **kwargs) -> Data: """ Returns annotation data (:class:`Data` containing :class:`AnnotationData`) used by this key. - The data can be filtered using keyword arguments. See :meth:`Annotation.data`. Note that only a subset makes sense in this context, set and key are already fixed. + The data can be filtered using positional and/or keyword arguments. See :meth:`Annotation.data`. Note that only a subset makes sense in this context, set and key are already fixed. Example -------- @@ -808,7 +807,7 @@ class DataKey: # returns only one """ - def test_data(self, **kwargs) -> bool: + def test_data(self, *args, **kwargs) -> bool: """ Tests whether certain annotation data exists for this key The data can be filtered using keyword arguments. See :meth:`Annotation.data`. Note that only a subset makes sense in this context, set and key are already fixed. @@ -825,7 +824,7 @@ class DataKey: ... """ - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """ Returns annotations (:class:`Annotations` containing :class:`Annotation`) that make use of this key. @@ -840,7 +839,7 @@ class DataKey: ... """ - def test_annotations(self, **kwargs) -> bool: + def test_annotations(self, *args, **kwargs) -> bool: """ Tests whether there are any annotations that make use of this key. This method is like :meth:`annotations`, but only tests and does not return the annotations, as such it is more performant. @@ -921,33 +920,37 @@ class AnnotationData: def dataset(self) -> AnnotationDataSet: """Returns the :class:`AnnotationDataSet` this data is part of""" - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """ Returns annotations (:class:`Annotations` containing :class:`Annotation`) that make use of this data. - The annotations can be filtered using keyword arguments. + The annotations can be filtered using positional and/or keyword arguments. + + Positional Arguments + ------------------- + + Positional arguments can be of the following types: + + * :class:`DataKey` - Returns annotations with data matching this key + * :class:`AnnotationData` - Returns only annotations that have this exact data + * :class:`Annotations` or a tuple/list of :class:`Annotation`- Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) + * :class:`Data` a tuple/list of :class:`AnnotationData `- Returns only annotations with data that is in the provided collection. + * a dictionary: + * `set` - An ID of an dataset (or a :class:`DataAnnotationSet` instance), only needed when specifying `key` as a string (see below) + * `key` - An key, either an instance of :class:`DataKey` or a string, in the latter case you need to specify `set` as well. + * `value` (see keyword arguments below) Keyword Arguments ------------------- limit: Optional[int] = None The maximum number of results to return (default: unlimited) - filter: Union[AnnotationData,Tuple[AnnotationData],List[AnnotationData],DataKey,Annotations,Data] - If you want to add multiple different filters, use `filters` instead. - Filter annotations based on: - - * :class:`AnnotationData` - Returns only annotations that have this exact data (you can only pass this once). - * a tuple/list of :class:`AnnotationData` - Returns only annotations with data that matches one of the items in the tuple/list. - * :class:`DataKey` - Returns annotations with data matching this key (you can only pass this once). - * :class:`Annotations` - Returns only annotations that are already in the provided :obj:`Annotations` collection (intersection) - * :class:`Data` - Returns only annotations with data that is in the provided :obj:`Data` collection. - filters: List[Union[AnnotationData,DataKey,Annotations,Data]] value: Optional[Union[str,int,float,bool]] - Constrain the search to annotations with data of a certain value. This can only be used with `filter=DataKey`. + Constrain the search to annotations with data of a certain value. This holds the exact value to search for, there are other variants of this keyword available, see :meth:`data` for a full list. """ - def test_annotations(self, **kwargs) -> bool: + def test_annotations(self, *args, **kwargs) -> bool: """ Tests whether there are any annotations that make use of this data. This method is like :meth:`annotations`, but only tests and does not return the annotations, as such it is more performant. @@ -984,19 +987,19 @@ class Data: def __getitem__(self, int) -> AnnotationData: """Returns data in the collection by index""" - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """ Returns annotations (:class:`Annotations` containing :class:`Annotation`) that are make use of any of the data in this collection - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ - def test_annotations(self, **kwargs) -> bool: + def test_annotations(self, *args, **kwargs) -> bool: """ Tests whether there are any annotations that make use of any of the data in this collection This method is like :meth:`annotations`, but does only tests and does not return the annotations, as such it is more performant. - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ class TextSelections: @@ -1026,11 +1029,11 @@ class TextSelections: def text(self, delimiter: str) -> List[str]: """Returns the text of all textselections in a list""" - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """ Returns annotations (:class:`Annotations` containing :class:`Annotation`) that refer to any of the text selections in this collection - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ def test_annotations(self, **kwargs) -> bool: @@ -1039,32 +1042,32 @@ class TextSelections: This method is like :meth:`annotations`, but only tests and does not return the annotations, as such it is more performant. - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ - def data(self, **kwargs) -> Data: + def data(self, *args, **kwargs) -> Data: """ Returns annotation data (:class:`Data` containing :class:`AnnotationData`) used by annotations referring to the text selections in this collection. - The data can be filtered using keyword arguments; see :meth:`Annotation.data`. + The data can be filtered using positional and/or keyword arguments; see :meth:`Annotation.data`. If no filters are set (default), all data from all annotations on all text selections are returned (without duplicates). """ - def test_data(self, **kwargs) -> bool: + def test_data(self, *args, **kwargs) -> bool: """ Tests whether there are any annotations that reference any of the text selections in the iterator, with data that passes the provided filters. The result is functionally equivalent to doing `.annotations().test_data()`, but this shortcut method is implemented much more efficiently and therefore recommended. - The data can be filtered using keyword arguments. See :meth:`Annotations.data`. + The data can be filtered using positional and/or keyword arguments. See :meth:`Annotations.data`. """ - def related_text(self, operator: TextSelectionOperator, **kwargs) -> TextSelections: + def related_text(self, operator: TextSelectionOperator, *args, **kwargs) -> TextSelections: """ Applies a :class:`TextSelectionOperator` to find all other text selections who are in a specific relation with the ones from the current collections. Returns a collection of all matching :class:`TextSelection` instances. - Text selections will be returned in textual order. They may be filtered via keyword arguments. See :meth:`Annotation.textselections`. + Text selections will be returned in textual order. They may be filtered via positional and/or keyword arguments. See :meth:`Annotation.textselections`. If you are interested in the annotations associated with the found text selections, then add `.annotations()` to the result. @@ -1254,6 +1257,8 @@ class SelectorKind: ANNOTATIONSELECTOR: SelectorKind TEXTSELECTOR: SelectorKind DATASETSELECTOR: SelectorKind + DATAKEYSELECTOR: SelectorKind + ANNOTATIONDATASELECTOR: SelectorKind MULTISELECTOR: SelectorKind COMPOSITESELECTOR: SelectorKind DIRECTIONALSELECTOR: SelectorKind @@ -1430,37 +1435,37 @@ class TextResource: The parameter value must be 0 or negative. """ - def annotations(self, **kwargs) -> Annotations: + def annotations(self, *args, **kwargs) -> Annotations: """Returns a collection of annotations (:class:`Annotation`) that reference this resource via a *TextSelector* (if any). Does *NOT* include those that use a ResourceSelector, use :meth:`annotations_metadata` instead for those instead. - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ - def annotations_as_metadata(self, **kwargs) -> Annotations: + def annotations_as_metadata(self, *args, **kwargs) -> Annotations: """Returns a collection of annotations (:class:`Annotation`) that reference this resource via a *ResourceSelector* (if any). Does *NOT* include those that use a TextSelector, use :meth:`annotations` instead for those instead. - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ - def test_annotations(self, **kwargs) -> bool: + def test_annotations(self,*args, **kwargs) -> bool: """ Tests whether there are any annotations that reference the text of this resource (via a TextSelector). This method is like :meth:`annotations`, but only tests and does not return the annotations, as such it is more performant. - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ - def test_annotations_as_metadata(self, **kwargs) -> bool: + def test_annotations_as_metadata(self, *args, **kwargs) -> bool: """ Tests whether there are any annotations that reference this resource as metadata (via a ResourceSelector). This method is like :meth:`annotations_as_metadata`, but only tests and does not return the annotations, as such it is more performant. - The annotations can be filtered using keyword arguments. See :meth:`Annotation.annotations`. + The annotations can be filtered using positional and/or keyword arguments. See :meth:`Annotation.annotations`. """ diff --git a/test.py b/test.py index c924aab..a52c252 100644 --- a/test.py +++ b/test.py @@ -235,7 +235,7 @@ def test_annotations_by_data(self): def test_find_data(self): """Find annotationdata by value""" dataset = self.store.dataset("testdataset") - results = dataset.data(filter=dataset.key("pos"), value="noun") + results = dataset.data(dataset.key("pos"), value="noun") self.assertIsInstance(results, Data) self.assertEqual(len(results), 1) self.assertIsInstance(results[0], AnnotationData) @@ -254,7 +254,7 @@ def test_find_data_from_key(self): def test_find_data_missing(self): """Find annotationdata by value, test mismatches""" dataset = self.store.dataset("testdataset") - results = dataset.data(filter=dataset.key("pos"),value="non-existent") + results = dataset.data(dataset.key("pos"),value="non-existent") self.assertFalse(results) #empty evaluates to False