From abbd75307ccae61ade582043800756e65a18cede Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 14 Feb 2015 19:42:27 -0500 Subject: [PATCH 01/10] dep-resolv: Allow the use of short module names in dependency-data. This change also prefers the use of short module names. Additionally the 'inherent module dependencies' feature would likely be removed, although I'm commiting the ported version here in case we'd ever need to bring it back. --- modules/ksb/DependencyResolver.pm | 96 ++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 2cfce15..1b77e5f 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -30,8 +30,12 @@ sub new my $class = shift; my $self = { - # hash table mapping full module names (m) to a hashref key by branch - # name, the value of which is yet another hashref (see readDependencyData) + # hash table mapping short module names (m) to a hashref key by branch + # name, the value of which is yet another hashref (see + # readDependencyData). Note that this assumes KDE git infrastructure + # ensures that all full module names (e.g. + # kde/workspace/plasma-workspace) map to a *unique* short name (e.g. + # plasma-workspace) by stripping leading path components dependenciesOf => { }, # hash table mapping a wildcarded module name with no branch to a @@ -42,6 +46,29 @@ sub new return bless $self, $class; } +# Function: shortenModuleName +# +# Internal: +# +# This method returns the 'short' module name of kde-project full project paths. +# E.g. 'kde/kdelibs/foo' would be shortened to 'foo'. +# +# This is a static function, not an object method. +# +# Parameters: +# +# path - A string holding the full module virtual path +# +# Returns: +# +# The module name. +sub _shortenModuleName +{ + my $name = shift; + $name =~ s{^.*/}{}; # Uses greedy capture by default + return $name; +} + # Method: readDependencyData # # Reads in dependency data in a pseudo-Makefile format. @@ -109,6 +136,11 @@ sub readDependencyData next; } + # No catch-alls here, these are direct dependencies so remove optional + # path components + $sourceItem = _shortenModuleName($sourceItem); + $dependentItem = _shortenModuleName($dependentItem); + # Initialize with hashref if not already defined. The hashref will hold # - => [ ] (list of explicit *NON* dependencies of item:$branch), # + => [ ] (list of dependencies of item:$branch) @@ -166,25 +198,38 @@ sub _addInherentDependencies # successor. Consider kde/foo, kde/foobar, kde/foo/a. The dependency # here would be missed that way. Instead we strip off the last path # component and see if that matches an existing module name. - for my $testModule (keys %{$modulesFromNameRef}) { - my $candidateBaseModule = $testModule; + for my $testModule (values %{$modulesFromNameRef}) { + my $candidateBaseModule = $testModule->fullProjectPath(); # Remove trailing component, bail if unable to do so. next unless $candidateBaseModule =~ s(/[^/]+$)(); - if ($candidateBaseModule && - exists $modulesFromNameRef->{$candidateBaseModule}) + # See if new base path matches up to an existing module + my $candidateModuleName = _shortenModuleName($candidateBaseModule); + if ($candidateModuleName ne $testModule->name() && + exists $modulesFromNameRef->{$candidateModuleName}) { + # Verify this possible base's full path is actually fully-contained + # within the dependent item, to account for things like + # kdevelop/kdevelop and phonon/phonon + my $candidateModule = $modulesFromNameRef->{$candidateModuleName}; + my $candidateModulePath = $candidateModule->fullProjectPath(); + + next if $candidateBaseModule !~ /^$candidateModulePath/; + + my $shortDependencyName = $candidateModuleName; + my $shortDependentName = _shortenModuleName($testModule); + # Add candidateBaseModule as dependency of testModule. - $dependenciesOfRef->{"$testModule:*"} //= { + $dependenciesOfRef->{"$shortDependentName:*"} //= { '-' => [ ], '+' => [ ], }; - my $moduleDepsRef = $dependenciesOfRef->{"$testModule:*"}->{'+'}; - if (!first { $_ eq $candidateBaseModule } @{$moduleDepsRef}) { - debug ("dep-resolv: Adding $testModule as dependency of $candidateBaseModule"); - push @{$moduleDepsRef}, "$candidateBaseModule:*"; + my $moduleDepsRef = $dependenciesOfRef->{"$shortDependentName:*"}->{'+'}; + if (!first { $_ eq $shortDependencyName } @{$moduleDepsRef}) { + debug ("dep-resolv: Adding $shortDependentName as dependency of $shortDependencyName"); + push @{$moduleDepsRef}, "$shortDependencyName:*"; } } } @@ -203,15 +248,15 @@ sub _addInherentDependencies # Parameters: # dependenciesOfRef - hashref to the table of dependencies as read by # . -# module - The full name (just the name) of the kde-project module to list +# module - The short name (just the name) of the kde-project module to list # dependencies of. # branch - The branch to assume for module. This must be specified, but use # '*' if you have no specific branch in mind. # # Returns: # A list of dependencies. Every item of the list will be of the form -# "$moduleName:$branch", where $moduleName will be the full kde-project module -# name (e.g. kde/kdelibs) and $branch will be a specific git branch or '*'. +# "$moduleName:$branch", where $moduleName will be the short kde-project module +# name (e.g. kdelibs) and $branch will be a specific git branch or '*'. # The order of the entries within the list is not important. sub _directDependenciesOf { @@ -258,7 +303,7 @@ sub _directDependenciesOf # # Parameters: # optionsRef - The hashref as provided to <_visitModuleAndDependencies> -# item - The kde-project module to generate dependencies of. +# item - The kde-project short module name to generate dependencies for. sub _makeCatchAllRules { my ($optionsRef, $item) = @_; @@ -302,9 +347,10 @@ sub _getBranchOf # Internal: # # This method is used to topographically sort dependency data. It accepts a -# , ensures that any KDE Projects it depends on present on the -# build list are re-ordered before the module, and then adds the -# to the build list (whether it is a KDE Project or not, to preserve ordering). +# , ensures that any KDE Projects it depends on (which are present +# on the build list) are re-ordered before the module, and then adds the +# to the build list (whether it is a KDE Project or not, to +# preserve ordering). # # See also _visitDependencyItemAndDependencies, which actually does most of # the work of handling dependencies, and calls back to this function when it @@ -324,9 +370,8 @@ sub _visitModuleAndDependencies my ($optionsRef, $module, $level) = @_; assert_isa($module, 'ksb::Module'); - my $item = $module->fullProjectPath() if $module->scmType() eq 'proj'; - - if ($item) { + if ($module->scmType() eq 'proj') { + my $item = _shortenModuleName($module->fullProjectPath()); my $branch = _getBranchOf($module) // '*'; _visitDependencyItemAndDependencies($optionsRef, "$item:$branch", $level); } @@ -358,11 +403,10 @@ sub _visitModuleAndDependencies # optionsRef - hashref to the module dependencies, catch-all dependencies, # module build list, module name to mapping, and auxiliary data # to see if a module has already been visited. -# dependencyItem - a string containing the kde-projects full path for the module, +# dependencyItem - a string containing the kde-projects short name for the module, # ':', and the specific branch name for the dependency if needed. The branch # name is '*' if the branch doesn't matter (or can be determined only by the -# branch-group in use). E.g. 'kde/kdelibs/baloo:*' or -# 'kdesupport/akonadi:master'. +# branch-group in use). E.g. 'baloo:*' or 'akonadi:master'. # # Returns: # Nothing. The proper build order can be read out from the optionsRef passed @@ -396,7 +440,7 @@ sub _visitDependencyItemAndDependencies _makeCatchAllRules($optionsRef, $item); for my $subItem (_directDependenciesOf($dependenciesOfRef, $item, $branch)) { - my ($subItemName, $subItemBranch) = ($subItem =~ m/^([^:]+):(.*)$/); + my ($subItemName, $subItemBranch) = split(':', $subItem, 2); croak_internal("Invalid dependency item: $subItem") if !$subItemName; next if $subItemName eq $item; # Catch-all deps might make this happen @@ -466,7 +510,7 @@ sub resolveDependencies # will map names back to their Modules modulesFromName => { - map { $_->fullProjectPath() => $_ } + map { $_->name() => $_ } grep { $_->scmType() eq 'proj' } @modules }, From f6e2022e1b83de58f8d08f14527a9c66d0e35268 Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 14 Feb 2015 20:11:23 -0500 Subject: [PATCH 02/10] dep-resolv: Do not implicitly add base-path modules as dependencies. This feature had been intended for cases where a module kde/foo was a git repository while also being a base for other git repositories kde/foo/bar, kde/foo/baz, etc. For instance, kde/kdelibs is a git repository that also held sub-children like kde/kdelibs/kactivities. However this is fairly rare in KDE repositories now, and this feature in its easiest-to-implement form has always had to navigate around several corner cases, so it's better just to explicitly specify dependencies. --- modules/ksb/DependencyResolver.pm | 68 ------------------------------- 1 file changed, 68 deletions(-) diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 1b77e5f..457c3b9 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -171,70 +171,6 @@ sub readDependencyData } } -# Function: addInherentDependencies -# -# Internal: -# -# This method adds any full module names as dependencies of any module that -# begins with that full module name. E.g. kde/kdelibs/foo automatically depends -# on kde/kdelibs if both are present in the build. -# -# This is a static function, not an object method. -# -# Parameters: -# -# options - Hashref to the internal options as given to -# -# Returns: -# -# Nothing. -sub _addInherentDependencies -{ - my $optionsRef = shift; - my $dependenciesOfRef = $optionsRef->{dependenciesOf}; - my $modulesFromNameRef = $optionsRef->{modulesFromName}; - - # It's not good enough to just sort modules and compare one to its - # successor. Consider kde/foo, kde/foobar, kde/foo/a. The dependency - # here would be missed that way. Instead we strip off the last path - # component and see if that matches an existing module name. - for my $testModule (values %{$modulesFromNameRef}) { - my $candidateBaseModule = $testModule->fullProjectPath(); - - # Remove trailing component, bail if unable to do so. - next unless $candidateBaseModule =~ s(/[^/]+$)(); - - # See if new base path matches up to an existing module - my $candidateModuleName = _shortenModuleName($candidateBaseModule); - if ($candidateModuleName ne $testModule->name() && - exists $modulesFromNameRef->{$candidateModuleName}) - { - # Verify this possible base's full path is actually fully-contained - # within the dependent item, to account for things like - # kdevelop/kdevelop and phonon/phonon - my $candidateModule = $modulesFromNameRef->{$candidateModuleName}; - my $candidateModulePath = $candidateModule->fullProjectPath(); - - next if $candidateBaseModule !~ /^$candidateModulePath/; - - my $shortDependencyName = $candidateModuleName; - my $shortDependentName = _shortenModuleName($testModule); - - # Add candidateBaseModule as dependency of testModule. - $dependenciesOfRef->{"$shortDependentName:*"} //= { - '-' => [ ], - '+' => [ ], - }; - - my $moduleDepsRef = $dependenciesOfRef->{"$shortDependentName:*"}->{'+'}; - if (!first { $_ eq $shortDependencyName } @{$moduleDepsRef}) { - debug ("dep-resolv: Adding $shortDependentName as dependency of $shortDependencyName"); - push @{$moduleDepsRef}, "$shortDependencyName:*"; - } - } - } -} - # Function: directDependenciesOf # # Internal: @@ -519,10 +455,6 @@ sub resolveDependencies modulesNeeded => scalar @modules, }; - # Adds things like kde/kdelibs/foo to automatically depend on - # kde/kdelibs if both are present in the build. - _addInherentDependencies($optionsRef); - for my $module (@modules) { _visitModuleAndDependencies($optionsRef, $module); } From 22e77dea40a01fa76109d76de479c0df0feabde3 Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 21 Feb 2015 17:43:32 -0500 Subject: [PATCH 03/10] Indent --print-modules output based on dependency resolution. This is somewhat confusing; since modules are listed in build order, indented modules show up *before* the modules that require them. Essentially, the module that caused an indented module to show up in its position in the build is the next module in the build that is more outdented. Additionally, this only indents in situations where a module's build order had to be rearranged. Modules that were already in the right order are not specially indented even if a valid dependency exists. With all that said, this can be a useful way of determining why kdesrc-build thinks a given order is required without having to go the route of a complicated ASCII DAG renderer. --- modules/ksb/Application.pm | 4 +++- modules/ksb/DependencyResolver.pm | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index 072e3f2..d80ab5f 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -771,7 +771,9 @@ sub runAllModulePhases if ($ctx->getOption('print-modules')) { info (" * Module list", $metadataModule ? " in dependency order" : ''); - say "$_" foreach @modules; + for my $m (@modules) { + say ((" " x ($m->getOption('#dependency-level', 'module') // 0)), "$m"); + } return 0; # Abort execution early! } diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 457c3b9..0eb1e07 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -315,6 +315,7 @@ sub _visitModuleAndDependencies # It's possible for _visitDependencyItemAndDependencies to add *this* # module without it being a cycle, so make sure we don't duplicate. if (! grep { $_->name() eq $module->name() } @{$optionsRef->{properBuildOrder}}) { + $module->setOption('#dependency-level', $level); push @{$optionsRef->{properBuildOrder}}, $module; --($optionsRef->{modulesNeeded}); } From 6e05bb60f15d62064b61ed5fe7562bd71fecbffb Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 21 Feb 2015 17:55:57 -0500 Subject: [PATCH 04/10] Code cleanups; kde-build-metadata is always downloaded now. --- modules/ksb/Application.pm | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index d80ab5f..fa47a7e 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -320,7 +320,8 @@ DONE # # All pending options are set into each module. Global options are set by # removing any existing rc-file option value, so you must setup the build context -# separately to have the needed option for this to work. +# separately to have the needed option for this to work. Additionally, the +# KDE project metadata must be available. # # Returns a list of in build order. # @@ -331,9 +332,7 @@ DONE # initialization - Do not call from this function. # # Parameters: -# ctx - in use. This function might call -# setKDEProjectMetadataModule on the build context. If so, the project -# metadata module should be updated before the build phase. +# ctx - in use. # # selectors - listref to hold the list of module or module-set selectors to # build, in the order desired by the user. The value of this parameter @@ -2161,8 +2160,7 @@ EOF # processed. # # Parameters: -# $ctx - in use for this script execution. Additionally this -# method might call setKDEProjectMetadataModuleNeeded on the $ctx. +# $ctx - in use for this script execution. # $modNew - Reference to a subroutine to be run for every new # created. See _resolveSelectorsIntoModules for full details. # @modules - list of , to be expanded. @@ -2183,10 +2181,6 @@ sub _expandModuleSets my @moduleResults = map { &$filter } (@buildModuleList); - if (first { $_->scmType() eq 'proj' } @moduleResults) { - $ctx->setKDEProjectMetadataModuleNeeded(); - } - return @moduleResults; } From 3821990bac88765cd0a38a97e883f5c16d699937 Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 21 Feb 2015 21:35:47 -0500 Subject: [PATCH 05/10] kde-projects: Don't return duplicate phonon modules from KDE XML. The way kdesrc-build handles letting users type "use-modules frameworks" and getting everything in frameworks/* back is by implementing the wildcard search in the KDE XML reader that parses kde_projects.xml. This uses a two-pass search for simple names, looking for 'name/*' and 'name'. This results in duplicates, which are normally OK due to other features in kdesrc-build that prevent duplicate modules from making it into the build list. However a feature I'm implementing was made much more difficult by this; easier instead just to prevent finding dups in the first place. --- modules/ksb/KDEXMLReader.pm | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/modules/ksb/KDEXMLReader.pm b/modules/ksb/KDEXMLReader.pm index 72d04ab..ec25896 100644 --- a/modules/ksb/KDEXMLReader.pm +++ b/modules/ksb/KDEXMLReader.pm @@ -90,7 +90,19 @@ sub getModulesForProject $parser->parse($self->inputHandle()); } - my @results; + # A hash is used to hold results since the keys inherently form a set, + # since we don't want dups. + my %results; + my $findResults = sub { + for my $result ( + grep { + _projectPathMatchesWildcardSearch( + $repositories{$_}->{'fullName'}, $proj) + } keys %repositories) + { + $results{$result} = 1; + }; + }; # Wildcard matches happen as specified if asked for. Non-wildcard matches # have an implicit "$proj/*" search as well for compatibility with previous @@ -98,20 +110,16 @@ sub getModulesForProject if ($proj !~ /\*/) { # We have to do a search to account for over-specified module names # like phonon/phonon - push @results, grep { - _projectPathMatchesWildcardSearch($repositories{$_}->{'fullName'}, $proj) - } keys %repositories; + $findResults->(); # Now setup for a wildcard search to find things like kde/kdelibs/baloo # if just 'kdelibs' is asked for. $proj .= '/*'; } - push @results, grep { - _projectPathMatchesWildcardSearch($repositories{$_}->{'fullName'}, $proj) - } keys %repositories; + $findResults->(); - return @repositories{@results}; + return @repositories{keys %results}; } sub xmlTagStart From 6f5e4f8542c78d957542521bd2ebf5c97b4fee0f Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 21 Feb 2015 21:43:52 -0500 Subject: [PATCH 06/10] dep-resolv: Shorten catch-all source names too. Source items can never be catch-all/wildcard items, so we must canonicalize these too to prevent e.g. extra-cmake-modules from depending on kdesupport/extra-cmake-modules due to a '*: kdesupport/extra-cmake-modules' rule. --- modules/ksb/DependencyResolver.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 0eb1e07..77c1e6e 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -129,6 +129,10 @@ sub readDependencyData $dependentBranch ||= '*'; # If no branch, apply catch-all flag $sourceBranch ||= '*'; + # Source can never be a catch-all so we can shorten early. Also, + # we *must* shorten early to avoid a dependency on a long path. + $sourceItem = _shortenModuleName($sourceItem); + # Handle catch-all dependent groupings if ($dependentItem =~ /\*$/) { $self->{catchAllDependencies}->{$dependentItem} //= [ ]; @@ -136,9 +140,6 @@ sub readDependencyData next; } - # No catch-alls here, these are direct dependencies so remove optional - # path components - $sourceItem = _shortenModuleName($sourceItem); $dependentItem = _shortenModuleName($dependentItem); # Initialize with hashref if not already defined. The hashref will hold From a2e48acc7396bdae9c6acbf76dcf1278bdbd9e70 Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 21 Feb 2015 21:48:46 -0500 Subject: [PATCH 07/10] GIT_SILENT: Make anchor conform to naming convention. --- doc/index.docbook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.docbook b/doc/index.docbook index 14a9677..03ac845 100644 --- a/doc/index.docbook +++ b/doc/index.docbook @@ -2025,7 +2025,7 @@ on also use that proxy server, if possible, by setting the - + ignore-kde-structure Module setting overrides global This option is used to store the source and the build files directly From b452401f50d3c740e8529986bf19e074920be0c6 Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sat, 21 Feb 2015 22:07:47 -0500 Subject: [PATCH 08/10] Add new option 'include-dependencies'. Currently it (probably) doesn't support pulling in options for these modules from their host module-sets (although using 'options' sections might actually work). I think it ignores the 'ignore-modules' option as well. Other than that, it should do pretty much what it says on the tin, although I had to implement a few other bug fixes as well to get it to that point. I want to say that there was an open bug for this but I can't seem to find it now. --- doc/index.docbook | 20 ++++++++ kdesrc-buildrc.xml | 3 +- kf5-workspace-build-include | 4 ++ modules/ksb/Application.pm | 44 +++++++++++++++-- modules/ksb/DependencyResolver.pm | 78 +++++++++++++++++++++++-------- vim/syntax/kdesrc-buildrc.vim | 6 +-- 6 files changed, 127 insertions(+), 28 deletions(-) diff --git a/doc/index.docbook b/doc/index.docbook index 03ac845..0103c5d 100644 --- a/doc/index.docbook +++ b/doc/index.docbook @@ -2067,6 +2067,26 @@ is compared). + +include-dependencies +Module setting overrides global + +This option, when set to true requests that +&kdesrc-build; also include known dependencies of this module in its build, +without requiring you to mention those dependencies (even indirectly). + +This option only works for kde-project-based +modules, and requires that the metadata maintained by the &kde; +developers is accurate for your selected branch-group. + +This option is disabled by default, to avoid inadvertently selecting a +bunch of modules that were not desired. It was added with &kdesrc-build; +1.16 in 2015. + + + install-after-build Module setting overrides global diff --git a/kdesrc-buildrc.xml b/kdesrc-buildrc.xml index 2abf2c9..327d7c6 100644 --- a/kdesrc-buildrc.xml +++ b/kdesrc-buildrc.xml @@ -4,7 +4,7 @@ it can go to $KDEHOME/share/apps/katepart/syntax --> use-modules ignore-modules + include-dependencies diff --git a/kf5-workspace-build-include b/kf5-workspace-build-include index 5f9903d..4e71d6b 100644 --- a/kf5-workspace-build-include +++ b/kf5-workspace-build-include @@ -21,6 +21,10 @@ module-set kf5-workspace-modules use-modules workspace plasmate kde-gtk-config libkscreen + # kdesrc-build will build dependencies (that it knows about) even if you forget + # to list them all. + include-dependencies true + # Remove if you're somehow using Windows ignore-modules kwindowsaddons end module-set diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index fa47a7e..90cf8e9 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -58,6 +58,8 @@ sub new metadata_module => undef, run_mode => 'build', modules => undef, + module_factory => undef, # ref to sub that makes a new Module. + # See generateModuleList _base_pid => $$, # See finish() }, $class; @@ -512,7 +514,7 @@ sub _resolveSelectorsIntoModules } # Generates the build context and module list based on the command line options -# and module selectors provided. +# and module selectors provided, and sets up the module factory. # # After this function is called all module set selectors will have been # expanded, and we will know if we need to download kde-projects metadata or @@ -614,6 +616,9 @@ sub generateModuleList delete @{$module->{options}}{@globalCmdlineArgs}; }; + # Called here since it depends on the closure above + $self->_defineNewModuleFactory($newModuleSub); + if ($commandLineModules) { # select our modules and module-sets, and expand them out @modules = $self->_resolveSelectorsIntoModules( @@ -698,8 +703,9 @@ sub _downloadKDEProjectMetadata # Returns a list of Modules in the proper build order according to the # kde-build-metadata dependency information. # -# The kde-build-metadata repository must have already been updated. The Modules -# to reorder must be passed as arguments. +# The kde-build-metadata repository must have already been updated, and the +# module factory must be setup. The Modules to reorder must be passed as +# arguments. sub _resolveModuleDependencies { my $self = shift; @@ -708,7 +714,7 @@ sub _resolveModuleDependencies my @modules = @_; @modules = eval { - my $dependencyResolver = ksb::DependencyResolver->new(); + my $dependencyResolver = ksb::DependencyResolver->new($self->{module_factory}); my $branchGroup = $ctx->effectiveBranchGroup(); for my $file ('dependency-data-common', "dependency-data-$branchGroup") @@ -2184,6 +2190,36 @@ sub _expandModuleSets return @moduleResults; } +# This defines the factory function needed for lower-level code to properly be +# able to create ksb::Module objects from just the module name, while still +# having the options be properly set and having the module properly tied into a +# context. +sub _defineNewModuleFactory +{ + my ($self, $newModuleSub) = @_; + my $ctx = $self->context(); + my $projSet = ksb::ModuleSet::KDEProjects->new($ctx, ''); + + $self->{module_factory} = sub { + my $name = shift; + $projSet->setModulesToFind($name); + my @results = $projSet->convertToModules($ctx); + + # Thought experiment: a module depends on phonon/phonon, which gets duly + # shortened to 'phonon'. Our kde-project code expands that by default to + # 'phonon/*', which returns {phonon,phonon-vlc,phonon-gstreamer}, etc. + # We need to make sure to return only matching modules. + my @mods = grep { $_->name() eq $name } (@results); + if (@mods > 1) { + croak_runtime ("Too many modules match $name; results were " . + join(', ', @mods)."\nCandidates @results"); + } + + $newModuleSub->($mods[0]); + return $mods[0]; + }; +} + # This function converts any 'l10n' references on the command line to return a l10n # module with the proper build system, scm type, etc. # diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 77c1e6e..8d39bda 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -18,16 +18,23 @@ use List::Util qw(first); # Constructor: new # -# Constructs a new . No parameters are taken. +# Constructs a new . +# +# Parameters: +# +# moduleFactoryRef - Reference to a sub that creates ksb::Modules from +# kde-project module names. Used for ksb::Modules for which the user +# requested recursive dependency inclusion. # # Synposis: # -# > my $resolver = new DependencyResolver; +# > my $resolver = new DependencyResolver($modNewRef); # > $resolver->readDependencyData(open my $fh, '<', 'file.txt'); # > $resolver->resolveDependencies(@modules); sub new { my $class = shift; + my $moduleFactoryRef = shift; my $self = { # hash table mapping short module names (m) to a hashref key by branch @@ -41,6 +48,11 @@ sub new # hash table mapping a wildcarded module name with no branch to a # listref of module:branch dependencies. catchAllDependencies => { }, + + # reference to a sub that will properly create a ksb::Module from a + # given kde-project module name. Used to support automatically adding + # dependencies to a build. + moduleFactoryRef => $moduleFactoryRef, }; return bless $self, $class; @@ -298,29 +310,31 @@ sub _getBranchOf # module build list, module name to mapping, and auxiliary data # to see if a module has already been visited. # module - The to properly order in the build list. +# level - The level of recursion of this call. +# dependent - Identical to the same param as _visitDependencyItemAndDependencies # # Returns: # Nothing. The proper build order can be read out from the optionsRef passed # in. sub _visitModuleAndDependencies { - my ($optionsRef, $module, $level) = @_; + my ($optionsRef, $module, $level, $dependentName) = @_; assert_isa($module, 'ksb::Module'); if ($module->scmType() eq 'proj') { my $item = _shortenModuleName($module->fullProjectPath()); my $branch = _getBranchOf($module) // '*'; - _visitDependencyItemAndDependencies($optionsRef, "$item:$branch", $level); - } - # It's possible for _visitDependencyItemAndDependencies to add *this* - # module without it being a cycle, so make sure we don't duplicate. - if (! grep { $_->name() eq $module->name() } @{$optionsRef->{properBuildOrder}}) { - $module->setOption('#dependency-level', $level); - push @{$optionsRef->{properBuildOrder}}, $module; - --($optionsRef->{modulesNeeded}); + $dependentName //= $item if $module->getOption('include-dependencies'); + _visitDependencyItemAndDependencies($optionsRef, "$item:$branch", $level, $dependentName); + + $optionsRef->{visitedItems}->{$item} = 3; # Mark as also in build list } + $module->setOption('#dependency-level', $level); + push @{$optionsRef->{properBuildOrder}}, $module; + --($optionsRef->{modulesNeeded}); + return; } @@ -345,18 +359,26 @@ sub _visitModuleAndDependencies # ':', and the specific branch name for the dependency if needed. The branch # name is '*' if the branch doesn't matter (or can be determined only by the # branch-group in use). E.g. 'baloo:*' or 'akonadi:master'. +# level - Level of recursion of the current call. +# dependent - *if set*, is the name of the module that requires that all of its +# dependencies be added to the build list (properly ordered) even if not +# specifically selected in the configuration file or command line. If not set, +# recursive dependencies are not pulled into the build even if they are not +# in the build list. # # Returns: # Nothing. The proper build order can be read out from the optionsRef passed -# in. +# in. Note that the generated build list might be longer than the build list that +# was input, in the case that recursive dependency inclusion was requested. sub _visitDependencyItemAndDependencies { - my ($optionsRef, $dependencyItem, $level) = @_; + my ($optionsRef, $dependencyItem, $level, $dependentName) = @_; my $visitedItemsRef = $optionsRef->{visitedItems}; my $properBuildOrderRef = $optionsRef->{properBuildOrder}; my $dependenciesOfRef = $optionsRef->{dependenciesOf}; my $modulesFromNameRef = $optionsRef->{modulesFromName}; + my $moduleFactoryRef = $optionsRef->{moduleFactoryRef}; $level //= 0; my ($item, $branch) = split(':', $dependencyItem, 2); @@ -366,14 +388,20 @@ sub _visitDependencyItemAndDependencies $visitedItemsRef->{$item} //= 0; # This module may have already been added to build. - return if $visitedItemsRef->{$item} == 1; + # 0 == Not visited + # 1 == Currently visiting. Running into a module in visit state 1 indicates a cycle. + # 2 == Visited, but not in build (this may happen for common dependencies with siblings, or for + # modules that are not in our build list but are part of dependency chain for other modules + # that *are* in build list). + # 3 == Visited, placed in build queue. + return if $visitedItemsRef->{$item} >= 2; # But if the value is 2 that means we've detected a cycle. - if ($visitedItemsRef->{$item} > 1) { + if ($visitedItemsRef->{$item} == 1) { croak_internal("Somehow there is a dependency cycle involving $item! :("); } - $visitedItemsRef->{$item} = 2; # Mark as currently-visiting for cycle detection. + $visitedItemsRef->{$item} = 1; # Mark as currently-visiting for cycle detection. _makeCatchAllRules($optionsRef, $item); @@ -385,28 +413,36 @@ sub _visitDependencyItemAndDependencies # This keeps us from doing a deep recursive search for dependencies # on an item we've already asked about. - next if (($visitedItemsRef->{$subItemName} // 0) == 1); + next if (($visitedItemsRef->{$subItemName} // 0) >= 2); debug ("\tdep-resolv: $item:$branch depends on $subItem"); my $subModule = $modulesFromNameRef->{$subItemName}; - if (!$subModule) { + if (!$subModule && !$dependentName) { whisper (" y[b[*] $dependencyItem depends on $subItem, but no module builds $subItem for this run."); _visitDependencyItemAndDependencies($optionsRef, $subItem, $level + 1); } else { + # Add in the dependent module if requested. + if (!$subModule) { + $subModule = $moduleFactoryRef->($subItemName); + $modulesFromNameRef->{$subModule->name()} = $subModule; + ++($optionsRef->{modulesNeeded}); + } + if ($subItemBranch ne '*' && (_getBranchOf($subModule) // '') ne $subItemBranch) { my $wrongBranch = _getBranchOf($subModule) // '?'; error (" r[b[*] $item needs $subItem, not $subItemName:$wrongBranch"); } - _visitModuleAndDependencies($optionsRef, $subModule, $level + 1); + _visitModuleAndDependencies($optionsRef, $subModule, $level + 1, $dependentName); } last if $optionsRef->{modulesNeeded} == 0; } - $visitedItemsRef->{$item} = 1; # Mark as done visiting. + # Mark as done visiting. + $visitedItemsRef->{$item} = 2; return; } @@ -453,6 +489,8 @@ sub resolveDependencies @modules }, + moduleFactoryRef => $self->{moduleFactoryRef}, + # Help _visitModuleAndDependencies to optimize modulesNeeded => scalar @modules, }; diff --git a/vim/syntax/kdesrc-buildrc.vim b/vim/syntax/kdesrc-buildrc.vim index e5d03fd..8b60c5c 100644 --- a/vim/syntax/kdesrc-buildrc.vim +++ b/vim/syntax/kdesrc-buildrc.vim @@ -1,7 +1,7 @@ " Vim syntax file " Language: kdesrc-build configuration file " Maintainer: Michael Pyne -" Latest Revision: 8 February 2015 +" Latest Revision: 21 February 2015 " Copyright (c) 2014,2015 Michael Pyne " Redistribution and use in source and binary forms, with or without @@ -54,12 +54,12 @@ syn keyword ksbrcErrorGlobalOption contained skipwhite nextgroup=ksbrcStringValu \ kde-languages niceness debug-level persistent-data-file set-env syn keyword ksbrcModuleSetOption contained skipwhite nextgroup=ksbrcStringValue - \ use-modules ignore-modules + \ use-modules ignore-modules include-dependencies " MUST BE CONSISTENT WITH ABOVE. Used when a module-set option is used in the " wrong spot to highlight the error. syn keyword ksbrcErrorModuleSetOption contained skipwhite nextgroup=ksbrcStringValue - \ use-modules ignore-modules + \ use-modules ignore-modules include-dependencies syn keyword ksbrcBoolOption contained skipwhite nextgroup=ksbrcBoolValue \ build-system-only build-when-unchanged ignore-kde-structure From 96fe204e59dc6cbe4700afbca17463ea3af06feb Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Sun, 22 Feb 2015 18:59:07 -0500 Subject: [PATCH 09/10] dep-resolv: There's a reason I included a visit-guard before. Even without include-dependencies, without this visit guard the tree traversal code will add a ksb::Module dependency (when they are encountered) to the build list every time it is encountered, not just the first time it's encountered. --- modules/ksb/DependencyResolver.pm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 8d39bda..789399b 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -325,6 +325,12 @@ sub _visitModuleAndDependencies my $item = _shortenModuleName($module->fullProjectPath()); my $branch = _getBranchOf($module) // '*'; + # Since the initial build list is visited start to finish it is + # possible for this module to already be in the ordered list if + # reordering has already happened or if dependencies are included (i.e. + # this was a dependency of some other module). + return if ($optionsRef->{visitedItems}->{$item} // 0) == 3; + $dependentName //= $item if $module->getOption('include-dependencies'); _visitDependencyItemAndDependencies($optionsRef, "$item:$branch", $level, $dependentName); From 633159b6ac7e211e9158758b6e0530f7a9603f4e Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Fri, 27 Feb 2015 21:25:48 -0500 Subject: [PATCH 10/10] dep-resolv: Revert to more restrictive syntax for dependency item. --- modules/ksb/DependencyResolver.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 789399b..6bb7bad 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -412,7 +412,7 @@ sub _visitDependencyItemAndDependencies _makeCatchAllRules($optionsRef, $item); for my $subItem (_directDependenciesOf($dependenciesOfRef, $item, $branch)) { - my ($subItemName, $subItemBranch) = split(':', $subItem, 2); + my ($subItemName, $subItemBranch) = ($subItem =~ m/^([^:]+):(.*)$/); croak_internal("Invalid dependency item: $subItem") if !$subItemName; next if $subItemName eq $item; # Catch-all deps might make this happen