Created starter files for the project.

This commit is contained in:
Batuhan Berk Başoğlu 2020-10-02 21:26:03 -04:00
commit 73f0c0db42
1992 changed files with 769897 additions and 0 deletions

View file

@ -0,0 +1,26 @@
__all__ = [
"__version__",
"AbstractProvider",
"AbstractResolver",
"BaseReporter",
"InconsistentCandidate",
"Resolver",
"RequirementsConflicted",
"ResolutionError",
"ResolutionImpossible",
"ResolutionTooDeep",
]
__version__ = "0.4.0"
from .providers import AbstractProvider, AbstractResolver
from .reporters import BaseReporter
from .resolvers import (
InconsistentCandidate,
RequirementsConflicted,
Resolver,
ResolutionError,
ResolutionImpossible,
ResolutionTooDeep,
)

View file

@ -0,0 +1,6 @@
__all__ = ["Sequence"]
try:
from collections.abc import Sequence
except ImportError:
from collections import Sequence

View file

@ -0,0 +1,109 @@
class AbstractProvider(object):
"""Delegate class to provide requirement interface for the resolver.
"""
def identify(self, dependency):
"""Given a dependency, return an identifier for it.
This is used in many places to identify the dependency, e.g. whether
two requirements should have their specifier parts merged, whether
two specifications would conflict with each other (because they the
same name but different versions).
"""
raise NotImplementedError
def get_preference(self, resolution, candidates, information):
"""Produce a sort key for given specification based on preference.
The preference is defined as "I think this requirement should be
resolved first". The lower the return value is, the more preferred
this group of arguments is.
:param resolution: Currently pinned candidate, or `None`.
:param candidates: A list of possible candidates.
:param information: A list of requirement information.
Each information instance is a named tuple with two entries:
* `requirement` specifies a requirement contributing to the current
candidate list
* `parent` specifies the candidate that provides (dependend on) the
requirement, or `None` to indicate a root requirement.
The preference could depend on a various of issues, including (not
necessarily in this order):
* Is this package pinned in the current resolution result?
* How relaxed is the requirement? Stricter ones should probably be
worked on first? (I don't know, actually.)
* How many possibilities are there to satisfy this requirement? Those
with few left should likely be worked on first, I guess?
* Are there any known conflicts for this requirement? We should
probably work on those with the most known conflicts.
A sortable value should be returned (this will be used as the `key`
parameter of the built-in sorting function). The smaller the value is,
the more preferred this specification is (i.e. the sorting function
is called with `reverse=False`).
"""
raise NotImplementedError
def find_matches(self, requirements):
"""Find all possible candidates that satisfy the given requirements.
This should try to get candidates based on the requirements' types.
For VCS, local, and archive requirements, the one-and-only match is
returned, and for a "named" requirement, the index(es) should be
consulted to find concrete candidates for this requirement.
:param requirements: A collection of requirements which all of the the
returned candidates must match. All requirements are guaranteed to
have the same identifier. The collection is never empty.
:returns: An iterable that orders candidates by preference, e.g. the
most preferred candidate should come first.
"""
raise NotImplementedError
def is_satisfied_by(self, requirement, candidate):
"""Whether the given requirement can be satisfied by a candidate.
The candidate is guarenteed to have been generated from the
requirement.
A boolean should be returned to indicate whether `candidate` is a
viable solution to the requirement.
"""
raise NotImplementedError
def get_dependencies(self, candidate):
"""Get dependencies of a candidate.
This should return a collection of requirements that `candidate`
specifies as its dependencies.
"""
raise NotImplementedError
class AbstractResolver(object):
"""The thing that performs the actual resolution work.
"""
base_exception = Exception
def __init__(self, provider, reporter):
self.provider = provider
self.reporter = reporter
def resolve(self, requirements, **kwargs):
"""Take a collection of constraints, spit out the resolution result.
This returns a representation of the final resolution state, with one
guarenteed attribute ``mapping`` that contains resolved candidates as
values. The keys are their respective identifiers.
:param requirements: A collection of constraints.
:param kwargs: Additional keyword arguments that subclasses may accept.
:raises: ``self.base_exception`` or its subclass.
"""
raise NotImplementedError

View file

@ -0,0 +1,42 @@
class BaseReporter(object):
"""Delegate class to provider progress reporting for the resolver.
"""
def starting(self):
"""Called before the resolution actually starts.
"""
def starting_round(self, index):
"""Called before each round of resolution starts.
The index is zero-based.
"""
def ending_round(self, index, state):
"""Called before each round of resolution ends.
This is NOT called if the resolution ends at this round. Use `ending`
if you want to report finalization. The index is zero-based.
"""
def ending(self, state):
"""Called before the resolution ends successfully.
"""
def adding_requirement(self, requirement, parent):
"""Called when adding a new requirement into the resolve criteria.
:param requirement: The additional requirement to be applied to filter
the available candidaites.
:param parent: The candidate that requires ``requirement`` as a
dependency, or None if ``requirement`` is one of the root
requirements passed in from ``Resolver.resolve()``.
"""
def backtracking(self, candidate):
"""Called when rejecting a candidate during backtracking.
"""
def pinning(self, candidate):
"""Called when adding a candidate to the potential solution.
"""

View file

@ -0,0 +1,428 @@
import collections
from .compat import collections_abc
from .providers import AbstractResolver
from .structs import DirectedGraph
RequirementInformation = collections.namedtuple(
"RequirementInformation", ["requirement", "parent"]
)
class ResolverException(Exception):
"""A base class for all exceptions raised by this module.
Exceptions derived by this class should all be handled in this module. Any
bubbling pass the resolver should be treated as a bug.
"""
class RequirementsConflicted(ResolverException):
def __init__(self, criterion):
super(RequirementsConflicted, self).__init__(criterion)
self.criterion = criterion
def __str__(self):
return "Requirements conflict: {}".format(
", ".join(repr(r) for r in self.criterion.iter_requirement()),
)
class InconsistentCandidate(ResolverException):
def __init__(self, candidate, criterion):
super(InconsistentCandidate, self).__init__(candidate, criterion)
self.candidate = candidate
self.criterion = criterion
def __str__(self):
return "Provided candidate {!r} does not satisfy {}".format(
self.candidate,
", ".join(repr(r) for r in self.criterion.iter_requirement()),
)
class Criterion(object):
"""Representation of possible resolution results of a package.
This holds three attributes:
* `information` is a collection of `RequirementInformation` pairs.
Each pair is a requirement contributing to this criterion, and the
candidate that provides the requirement.
* `incompatibilities` is a collection of all known not-to-work candidates
to exclude from consideration.
* `candidates` is a collection containing all possible candidates deducted
from the union of contributing requirements and known incompatibilities.
It should never be empty, except when the criterion is an attribute of a
raised `RequirementsConflicted` (in which case it is always empty).
.. note::
This class is intended to be externally immutable. **Do not** mutate
any of its attribute containers.
"""
def __init__(self, candidates, information, incompatibilities):
self.candidates = candidates
self.information = information
self.incompatibilities = incompatibilities
def __repr__(self):
requirements = ", ".join(
"({!r}, via={!r})".format(req, parent)
for req, parent in self.information
)
return "Criterion({})".format(requirements)
@classmethod
def from_requirement(cls, provider, requirement, parent):
"""Build an instance from a requirement.
"""
candidates = provider.find_matches([requirement])
if not isinstance(candidates, collections_abc.Sequence):
candidates = list(candidates)
criterion = cls(
candidates=candidates,
information=[RequirementInformation(requirement, parent)],
incompatibilities=[],
)
if not candidates:
raise RequirementsConflicted(criterion)
return criterion
def iter_requirement(self):
return (i.requirement for i in self.information)
def iter_parent(self):
return (i.parent for i in self.information)
def merged_with(self, provider, requirement, parent):
"""Build a new instance from this and a new requirement.
"""
infos = list(self.information)
infos.append(RequirementInformation(requirement, parent))
candidates = provider.find_matches([r for r, _ in infos])
if not isinstance(candidates, collections_abc.Sequence):
candidates = list(candidates)
criterion = type(self)(candidates, infos, list(self.incompatibilities))
if not candidates:
raise RequirementsConflicted(criterion)
return criterion
def excluded_of(self, candidate):
"""Build a new instance from this, but excluding specified candidate.
Returns the new instance, or None if we still have no valid candidates.
"""
incompats = list(self.incompatibilities)
incompats.append(candidate)
candidates = [c for c in self.candidates if c != candidate]
if not candidates:
return None
criterion = type(self)(candidates, list(self.information), incompats)
return criterion
class ResolutionError(ResolverException):
pass
class ResolutionImpossible(ResolutionError):
def __init__(self, causes):
super(ResolutionImpossible, self).__init__(causes)
# causes is a list of RequirementInformation objects
self.causes = causes
class ResolutionTooDeep(ResolutionError):
def __init__(self, round_count):
super(ResolutionTooDeep, self).__init__(round_count)
self.round_count = round_count
# Resolution state in a round.
State = collections.namedtuple("State", "mapping criteria")
class Resolution(object):
"""Stateful resolution object.
This is designed as a one-off object that holds information to kick start
the resolution process, and holds the results afterwards.
"""
def __init__(self, provider, reporter):
self._p = provider
self._r = reporter
self._states = []
@property
def state(self):
try:
return self._states[-1]
except IndexError:
raise AttributeError("state")
def _push_new_state(self):
"""Push a new state into history.
This new state will be used to hold resolution results of the next
coming round.
"""
try:
base = self._states[-1]
except IndexError:
state = State(mapping=collections.OrderedDict(), criteria={})
else:
state = State(
mapping=base.mapping.copy(), criteria=base.criteria.copy(),
)
self._states.append(state)
def _merge_into_criterion(self, requirement, parent):
self._r.adding_requirement(requirement, parent)
name = self._p.identify(requirement)
try:
crit = self.state.criteria[name]
except KeyError:
crit = Criterion.from_requirement(self._p, requirement, parent)
else:
crit = crit.merged_with(self._p, requirement, parent)
return name, crit
def _get_criterion_item_preference(self, item):
name, criterion = item
try:
pinned = self.state.mapping[name]
except KeyError:
pinned = None
return self._p.get_preference(
pinned, criterion.candidates, criterion.information,
)
def _is_current_pin_satisfying(self, name, criterion):
try:
current_pin = self.state.mapping[name]
except KeyError:
return False
return all(
self._p.is_satisfied_by(r, current_pin)
for r in criterion.iter_requirement()
)
def _get_criteria_to_update(self, candidate):
criteria = {}
for r in self._p.get_dependencies(candidate):
name, crit = self._merge_into_criterion(r, parent=candidate)
criteria[name] = crit
return criteria
def _attempt_to_pin_criterion(self, name, criterion):
causes = []
for candidate in criterion.candidates:
try:
criteria = self._get_criteria_to_update(candidate)
except RequirementsConflicted as e:
causes.append(e.criterion)
continue
# Check the newly-pinned candidate actually works. This should
# always pass under normal circumstances, but in the case of a
# faulty provider, we will raise an error to notify the implementer
# to fix find_matches() and/or is_satisfied_by().
satisfied = all(
self._p.is_satisfied_by(r, candidate)
for r in criterion.iter_requirement()
)
if not satisfied:
raise InconsistentCandidate(candidate, criterion)
# Put newly-pinned candidate at the end. This is essential because
# backtracking looks at this mapping to get the last pin.
self._r.pinning(candidate)
self.state.mapping.pop(name, None)
self.state.mapping[name] = candidate
self.state.criteria.update(criteria)
return []
# All candidates tried, nothing works. This criterion is a dead
# end, signal for backtracking.
return causes
def _backtrack(self):
# Drop the current state, it's known not to work.
del self._states[-1]
# We need at least 2 states here:
# (a) One to backtrack to.
# (b) One to restore state (a) to its state prior to candidate-pinning,
# so we can pin another one instead.
while len(self._states) >= 2:
# Retract the last candidate pin.
prev_state = self._states.pop()
try:
name, candidate = prev_state.mapping.popitem()
except KeyError:
continue
self._r.backtracking(candidate)
# Create a new state to work on, with the newly known not-working
# candidate excluded.
self._push_new_state()
# Mark the retracted candidate as incompatible.
criterion = self.state.criteria[name].excluded_of(candidate)
if criterion is None:
# This state still does not work. Try the still previous state.
del self._states[-1]
continue
self.state.criteria[name] = criterion
return True
return False
def resolve(self, requirements, max_rounds):
if self._states:
raise RuntimeError("already resolved")
self._push_new_state()
for r in requirements:
try:
name, crit = self._merge_into_criterion(r, parent=None)
except RequirementsConflicted as e:
raise ResolutionImpossible(e.criterion.information)
self.state.criteria[name] = crit
self._r.starting()
for round_index in range(max_rounds):
self._r.starting_round(round_index)
self._push_new_state()
curr = self.state
unsatisfied_criterion_items = [
item
for item in self.state.criteria.items()
if not self._is_current_pin_satisfying(*item)
]
# All criteria are accounted for. Nothing more to pin, we are done!
if not unsatisfied_criterion_items:
del self._states[-1]
self._r.ending(curr)
return self.state
# Choose the most preferred unpinned criterion to try.
name, criterion = min(
unsatisfied_criterion_items,
key=self._get_criterion_item_preference,
)
failure_causes = self._attempt_to_pin_criterion(name, criterion)
# Backtrack if pinning fails.
if failure_causes:
result = self._backtrack()
if not result:
causes = [
i for crit in failure_causes for i in crit.information
]
raise ResolutionImpossible(causes)
self._r.ending_round(round_index, curr)
raise ResolutionTooDeep(max_rounds)
def _has_route_to_root(criteria, key, all_keys, connected):
if key in connected:
return True
if key not in criteria:
return False
for p in criteria[key].iter_parent():
try:
pkey = all_keys[id(p)]
except KeyError:
continue
if pkey in connected:
connected.add(key)
return True
if _has_route_to_root(criteria, pkey, all_keys, connected):
connected.add(key)
return True
return False
Result = collections.namedtuple("Result", "mapping graph criteria")
def _build_result(state):
mapping = state.mapping
all_keys = {id(v): k for k, v in mapping.items()}
all_keys[id(None)] = None
graph = DirectedGraph()
graph.add(None) # Sentinel as root dependencies' parent.
connected = {None}
for key, criterion in state.criteria.items():
if not _has_route_to_root(state.criteria, key, all_keys, connected):
continue
if key not in graph:
graph.add(key)
for p in criterion.iter_parent():
try:
pkey = all_keys[id(p)]
except KeyError:
continue
if pkey not in graph:
graph.add(pkey)
graph.connect(pkey, key)
return Result(
mapping={k: v for k, v in mapping.items() if k in connected},
graph=graph,
criteria=state.criteria,
)
class Resolver(AbstractResolver):
"""The thing that performs the actual resolution work.
"""
base_exception = ResolverException
def resolve(self, requirements, max_rounds=100):
"""Take a collection of constraints, spit out the resolution result.
The return value is a representation to the final resolution result. It
is a tuple subclass with three public members:
* `mapping`: A dict of resolved candidates. Each key is an identifier
of a requirement (as returned by the provider's `identify` method),
and the value is the resolved candidate.
* `graph`: A `DirectedGraph` instance representing the dependency tree.
The vertices are keys of `mapping`, and each edge represents *why*
a particular package is included. A special vertex `None` is
included to represent parents of user-supplied requirements.
* `criteria`: A dict of "criteria" that hold detailed information on
how edges in the graph are derived. Each key is an identifier of a
requirement, and the value is a `Criterion` instance.
The following exceptions may be raised if a resolution cannot be found:
* `ResolutionImpossible`: A resolution cannot be found for the given
combination of requirements. The `causes` attribute of the
exception is a list of (requirement, parent), giving the
requirements that could not be satisfied.
* `ResolutionTooDeep`: The dependency tree is too deeply nested and
the resolver gave up. This is usually caused by a circular
dependency, but you can try to resolve this by increasing the
`max_rounds` argument.
"""
resolution = Resolution(self.provider, self.reporter)
state = resolution.resolve(requirements, max_rounds=max_rounds)
return _build_result(state)

View file

@ -0,0 +1,68 @@
class DirectedGraph(object):
"""A graph structure with directed edges.
"""
def __init__(self):
self._vertices = set()
self._forwards = {} # <key> -> Set[<key>]
self._backwards = {} # <key> -> Set[<key>]
def __iter__(self):
return iter(self._vertices)
def __len__(self):
return len(self._vertices)
def __contains__(self, key):
return key in self._vertices
def copy(self):
"""Return a shallow copy of this graph.
"""
other = DirectedGraph()
other._vertices = set(self._vertices)
other._forwards = {k: set(v) for k, v in self._forwards.items()}
other._backwards = {k: set(v) for k, v in self._backwards.items()}
return other
def add(self, key):
"""Add a new vertex to the graph.
"""
if key in self._vertices:
raise ValueError("vertex exists")
self._vertices.add(key)
self._forwards[key] = set()
self._backwards[key] = set()
def remove(self, key):
"""Remove a vertex from the graph, disconnecting all edges from/to it.
"""
self._vertices.remove(key)
for f in self._forwards.pop(key):
self._backwards[f].remove(key)
for t in self._backwards.pop(key):
self._forwards[t].remove(key)
def connected(self, f, t):
return f in self._backwards[t] and t in self._forwards[f]
def connect(self, f, t):
"""Connect two existing vertices.
Nothing happens if the vertices are already connected.
"""
if t not in self._vertices:
raise KeyError(t)
self._forwards[f].add(t)
self._backwards[t].add(f)
def iter_edges(self):
for f, children in self._forwards.items():
for t in children:
yield f, t
def iter_children(self, key):
return iter(self._forwards[key])
def iter_parents(self, key):
return iter(self._backwards[key])