diff --git a/doc/index.docbook b/doc/index.docbook index 14a9677..0103c5d 100644 --- a/doc/index.docbook +++ b/doc/index.docbook @@ -2025,7 +2025,7 @@ on also use that proxy server, if possible, by setting the - + ignore-kde-structure Module setting overrides global This option is used to store the source and the build files directly @@ -2067,6 +2067,26 @@ is compared). + +include-dependencies +Module setting overrides global + +This option, when set to true requests that +&kdesrc-build; also include known dependencies of this module in its build, +without requiring you to mention those dependencies (even indirectly). + +This option only works for kde-project-based +modules, and requires that the metadata maintained by the &kde; +developers is accurate for your selected branch-group. + +This option is disabled by default, to avoid inadvertently selecting a +bunch of modules that were not desired. It was added with &kdesrc-build; +1.16 in 2015. + + + install-after-build Module setting overrides global diff --git a/kdesrc-buildrc.xml b/kdesrc-buildrc.xml index 2abf2c9..327d7c6 100644 --- a/kdesrc-buildrc.xml +++ b/kdesrc-buildrc.xml @@ -4,7 +4,7 @@ it can go to $KDEHOME/share/apps/katepart/syntax --> use-modules ignore-modules + include-dependencies diff --git a/kf5-workspace-build-include b/kf5-workspace-build-include index 5f9903d..4e71d6b 100644 --- a/kf5-workspace-build-include +++ b/kf5-workspace-build-include @@ -21,6 +21,10 @@ module-set kf5-workspace-modules use-modules workspace plasmate kde-gtk-config libkscreen + # kdesrc-build will build dependencies (that it knows about) even if you forget + # to list them all. + include-dependencies true + # Remove if you're somehow using Windows ignore-modules kwindowsaddons end module-set diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index 5ad19ce..fabfb1f 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -58,6 +58,8 @@ sub new metadata_module => undef, run_mode => 'build', modules => undef, + module_factory => undef, # ref to sub that makes a new Module. + # See generateModuleList _base_pid => $$, # See finish() }, $class; @@ -320,7 +322,8 @@ DONE # # All pending options are set into each module. Global options are set by # removing any existing rc-file option value, so you must setup the build context -# separately to have the needed option for this to work. +# separately to have the needed option for this to work. Additionally, the +# KDE project metadata must be available. # # Returns a list of in build order. # @@ -331,9 +334,7 @@ DONE # initialization - Do not call from this function. # # Parameters: -# ctx - in use. This function might call -# setKDEProjectMetadataModule on the build context. If so, the project -# metadata module should be updated before the build phase. +# ctx - in use. # # selectors - listref to hold the list of module or module-set selectors to # build, in the order desired by the user. The value of this parameter @@ -521,7 +522,7 @@ sub _resolveSelectorsIntoModules } # Generates the build context and module list based on the command line options -# and module selectors provided. +# and module selectors provided, and sets up the module factory. # # After this function is called all module set selectors will have been # expanded, and we will know if we need to download kde-projects metadata or @@ -624,6 +625,9 @@ sub generateModuleList delete @{$module->{options}}{@globalCmdlineArgs}; }; + # Called here since it depends on the closure above + $self->_defineNewModuleFactory($newModuleSub); + if ($commandLineModules) { # select our modules and module-sets, and expand them out @modules = $self->_resolveSelectorsIntoModules( @@ -711,8 +715,9 @@ sub _downloadKDEProjectMetadata # Returns a list of Modules in the proper build order according to the # kde-build-metadata dependency information. # -# The kde-build-metadata repository must have already been updated. The Modules -# to reorder must be passed as arguments. +# The kde-build-metadata repository must have already been updated, and the +# module factory must be setup. The Modules to reorder must be passed as +# arguments. sub _resolveModuleDependencies { my $self = shift; @@ -721,7 +726,7 @@ sub _resolveModuleDependencies my @modules = @_; @modules = eval { - my $dependencyResolver = ksb::DependencyResolver->new(); + my $dependencyResolver = ksb::DependencyResolver->new($self->{module_factory}); my $branchGroup = $ctx->effectiveBranchGroup(); for my $file ('dependency-data-common', "dependency-data-$branchGroup") @@ -783,7 +788,9 @@ sub runAllModulePhases if ($ctx->getOption('print-modules')) { info (" * Module list", $metadataModule ? " in dependency order" : ''); - say "$_" foreach @modules; + for my $m (@modules) { + say ((" " x ($m->getOption('#dependency-level', 'module') // 0)), "$m"); + } return 0; # Abort execution early! } @@ -2167,8 +2174,7 @@ EOF # processed. # # Parameters: -# $ctx - in use for this script execution. Additionally this -# method might call setKDEProjectMetadataModuleNeeded on the $ctx. +# $ctx - in use for this script execution. # $modNew - Reference to a subroutine to be run for every new # created. See _resolveSelectorsIntoModules for full details. # @modules - list of , to be expanded. @@ -2189,13 +2195,39 @@ sub _expandModuleSets my @moduleResults = map { &$filter } (@buildModuleList); - if (first { $_->scmType() eq 'proj' } @moduleResults) { - $ctx->setKDEProjectMetadataModuleNeeded(); - } - return @moduleResults; } +# This defines the factory function needed for lower-level code to properly be +# able to create ksb::Module objects from just the module name, while still +# having the options be properly set and having the module properly tied into a +# context. +sub _defineNewModuleFactory +{ + my ($self, $newModuleSub) = @_; + my $ctx = $self->context(); + my $projSet = ksb::ModuleSet::KDEProjects->new($ctx, ''); + + $self->{module_factory} = sub { + my $name = shift; + $projSet->setModulesToFind($name); + my @results = $projSet->convertToModules($ctx); + + # Thought experiment: a module depends on phonon/phonon, which gets duly + # shortened to 'phonon'. Our kde-project code expands that by default to + # 'phonon/*', which returns {phonon,phonon-vlc,phonon-gstreamer}, etc. + # We need to make sure to return only matching modules. + my @mods = grep { $_->name() eq $name } (@results); + if (@mods > 1) { + croak_runtime ("Too many modules match $name; results were " . + join(', ', @mods)."\nCandidates @results"); + } + + $newModuleSub->($mods[0]); + return $mods[0]; + }; +} + # This function converts any 'l10n' references on the command line to return a l10n # module with the proper build system, scm type, etc. # diff --git a/modules/ksb/DependencyResolver.pm b/modules/ksb/DependencyResolver.pm index 2cfce15..6bb7bad 100644 --- a/modules/ksb/DependencyResolver.pm +++ b/modules/ksb/DependencyResolver.pm @@ -18,30 +18,69 @@ use List::Util qw(first); # Constructor: new # -# Constructs a new . No parameters are taken. +# Constructs a new . +# +# Parameters: +# +# moduleFactoryRef - Reference to a sub that creates ksb::Modules from +# kde-project module names. Used for ksb::Modules for which the user +# requested recursive dependency inclusion. # # Synposis: # -# > my $resolver = new DependencyResolver; +# > my $resolver = new DependencyResolver($modNewRef); # > $resolver->readDependencyData(open my $fh, '<', 'file.txt'); # > $resolver->resolveDependencies(@modules); sub new { my $class = shift; + my $moduleFactoryRef = shift; my $self = { - # hash table mapping full module names (m) to a hashref key by branch - # name, the value of which is yet another hashref (see readDependencyData) + # hash table mapping short module names (m) to a hashref key by branch + # name, the value of which is yet another hashref (see + # readDependencyData). Note that this assumes KDE git infrastructure + # ensures that all full module names (e.g. + # kde/workspace/plasma-workspace) map to a *unique* short name (e.g. + # plasma-workspace) by stripping leading path components dependenciesOf => { }, # hash table mapping a wildcarded module name with no branch to a # listref of module:branch dependencies. catchAllDependencies => { }, + + # reference to a sub that will properly create a ksb::Module from a + # given kde-project module name. Used to support automatically adding + # dependencies to a build. + moduleFactoryRef => $moduleFactoryRef, }; return bless $self, $class; } +# Function: shortenModuleName +# +# Internal: +# +# This method returns the 'short' module name of kde-project full project paths. +# E.g. 'kde/kdelibs/foo' would be shortened to 'foo'. +# +# This is a static function, not an object method. +# +# Parameters: +# +# path - A string holding the full module virtual path +# +# Returns: +# +# The module name. +sub _shortenModuleName +{ + my $name = shift; + $name =~ s{^.*/}{}; # Uses greedy capture by default + return $name; +} + # Method: readDependencyData # # Reads in dependency data in a pseudo-Makefile format. @@ -102,6 +141,10 @@ sub readDependencyData $dependentBranch ||= '*'; # If no branch, apply catch-all flag $sourceBranch ||= '*'; + # Source can never be a catch-all so we can shorten early. Also, + # we *must* shorten early to avoid a dependency on a long path. + $sourceItem = _shortenModuleName($sourceItem); + # Handle catch-all dependent groupings if ($dependentItem =~ /\*$/) { $self->{catchAllDependencies}->{$dependentItem} //= [ ]; @@ -109,6 +152,8 @@ sub readDependencyData next; } + $dependentItem = _shortenModuleName($dependentItem); + # Initialize with hashref if not already defined. The hashref will hold # - => [ ] (list of explicit *NON* dependencies of item:$branch), # + => [ ] (list of dependencies of item:$branch) @@ -139,57 +184,6 @@ sub readDependencyData } } -# Function: addInherentDependencies -# -# Internal: -# -# This method adds any full module names as dependencies of any module that -# begins with that full module name. E.g. kde/kdelibs/foo automatically depends -# on kde/kdelibs if both are present in the build. -# -# This is a static function, not an object method. -# -# Parameters: -# -# options - Hashref to the internal options as given to -# -# Returns: -# -# Nothing. -sub _addInherentDependencies -{ - my $optionsRef = shift; - my $dependenciesOfRef = $optionsRef->{dependenciesOf}; - my $modulesFromNameRef = $optionsRef->{modulesFromName}; - - # It's not good enough to just sort modules and compare one to its - # successor. Consider kde/foo, kde/foobar, kde/foo/a. The dependency - # here would be missed that way. Instead we strip off the last path - # component and see if that matches an existing module name. - for my $testModule (keys %{$modulesFromNameRef}) { - my $candidateBaseModule = $testModule; - - # Remove trailing component, bail if unable to do so. - next unless $candidateBaseModule =~ s(/[^/]+$)(); - - if ($candidateBaseModule && - exists $modulesFromNameRef->{$candidateBaseModule}) - { - # Add candidateBaseModule as dependency of testModule. - $dependenciesOfRef->{"$testModule:*"} //= { - '-' => [ ], - '+' => [ ], - }; - - my $moduleDepsRef = $dependenciesOfRef->{"$testModule:*"}->{'+'}; - if (!first { $_ eq $candidateBaseModule } @{$moduleDepsRef}) { - debug ("dep-resolv: Adding $testModule as dependency of $candidateBaseModule"); - push @{$moduleDepsRef}, "$candidateBaseModule:*"; - } - } - } -} - # Function: directDependenciesOf # # Internal: @@ -203,15 +197,15 @@ sub _addInherentDependencies # Parameters: # dependenciesOfRef - hashref to the table of dependencies as read by # . -# module - The full name (just the name) of the kde-project module to list +# module - The short name (just the name) of the kde-project module to list # dependencies of. # branch - The branch to assume for module. This must be specified, but use # '*' if you have no specific branch in mind. # # Returns: # A list of dependencies. Every item of the list will be of the form -# "$moduleName:$branch", where $moduleName will be the full kde-project module -# name (e.g. kde/kdelibs) and $branch will be a specific git branch or '*'. +# "$moduleName:$branch", where $moduleName will be the short kde-project module +# name (e.g. kdelibs) and $branch will be a specific git branch or '*'. # The order of the entries within the list is not important. sub _directDependenciesOf { @@ -258,7 +252,7 @@ sub _directDependenciesOf # # Parameters: # optionsRef - The hashref as provided to <_visitModuleAndDependencies> -# item - The kde-project module to generate dependencies of. +# item - The kde-project short module name to generate dependencies for. sub _makeCatchAllRules { my ($optionsRef, $item) = @_; @@ -302,9 +296,10 @@ sub _getBranchOf # Internal: # # This method is used to topographically sort dependency data. It accepts a -# , ensures that any KDE Projects it depends on present on the -# build list are re-ordered before the module, and then adds the -# to the build list (whether it is a KDE Project or not, to preserve ordering). +# , ensures that any KDE Projects it depends on (which are present +# on the build list) are re-ordered before the module, and then adds the +# to the build list (whether it is a KDE Project or not, to +# preserve ordering). # # See also _visitDependencyItemAndDependencies, which actually does most of # the work of handling dependencies, and calls back to this function when it @@ -315,29 +310,37 @@ sub _getBranchOf # module build list, module name to mapping, and auxiliary data # to see if a module has already been visited. # module - The to properly order in the build list. +# level - The level of recursion of this call. +# dependent - Identical to the same param as _visitDependencyItemAndDependencies # # Returns: # Nothing. The proper build order can be read out from the optionsRef passed # in. sub _visitModuleAndDependencies { - my ($optionsRef, $module, $level) = @_; + my ($optionsRef, $module, $level, $dependentName) = @_; assert_isa($module, 'ksb::Module'); - my $item = $module->fullProjectPath() if $module->scmType() eq 'proj'; - - if ($item) { + if ($module->scmType() eq 'proj') { + my $item = _shortenModuleName($module->fullProjectPath()); my $branch = _getBranchOf($module) // '*'; - _visitDependencyItemAndDependencies($optionsRef, "$item:$branch", $level); - } - # It's possible for _visitDependencyItemAndDependencies to add *this* - # module without it being a cycle, so make sure we don't duplicate. - if (! grep { $_->name() eq $module->name() } @{$optionsRef->{properBuildOrder}}) { - push @{$optionsRef->{properBuildOrder}}, $module; - --($optionsRef->{modulesNeeded}); + # Since the initial build list is visited start to finish it is + # possible for this module to already be in the ordered list if + # reordering has already happened or if dependencies are included (i.e. + # this was a dependency of some other module). + return if ($optionsRef->{visitedItems}->{$item} // 0) == 3; + + $dependentName //= $item if $module->getOption('include-dependencies'); + _visitDependencyItemAndDependencies($optionsRef, "$item:$branch", $level, $dependentName); + + $optionsRef->{visitedItems}->{$item} = 3; # Mark as also in build list } + $module->setOption('#dependency-level', $level); + push @{$optionsRef->{properBuildOrder}}, $module; + --($optionsRef->{modulesNeeded}); + return; } @@ -358,23 +361,30 @@ sub _visitModuleAndDependencies # optionsRef - hashref to the module dependencies, catch-all dependencies, # module build list, module name to mapping, and auxiliary data # to see if a module has already been visited. -# dependencyItem - a string containing the kde-projects full path for the module, +# dependencyItem - a string containing the kde-projects short name for the module, # ':', and the specific branch name for the dependency if needed. The branch # name is '*' if the branch doesn't matter (or can be determined only by the -# branch-group in use). E.g. 'kde/kdelibs/baloo:*' or -# 'kdesupport/akonadi:master'. +# branch-group in use). E.g. 'baloo:*' or 'akonadi:master'. +# level - Level of recursion of the current call. +# dependent - *if set*, is the name of the module that requires that all of its +# dependencies be added to the build list (properly ordered) even if not +# specifically selected in the configuration file or command line. If not set, +# recursive dependencies are not pulled into the build even if they are not +# in the build list. # # Returns: # Nothing. The proper build order can be read out from the optionsRef passed -# in. +# in. Note that the generated build list might be longer than the build list that +# was input, in the case that recursive dependency inclusion was requested. sub _visitDependencyItemAndDependencies { - my ($optionsRef, $dependencyItem, $level) = @_; + my ($optionsRef, $dependencyItem, $level, $dependentName) = @_; my $visitedItemsRef = $optionsRef->{visitedItems}; my $properBuildOrderRef = $optionsRef->{properBuildOrder}; my $dependenciesOfRef = $optionsRef->{dependenciesOf}; my $modulesFromNameRef = $optionsRef->{modulesFromName}; + my $moduleFactoryRef = $optionsRef->{moduleFactoryRef}; $level //= 0; my ($item, $branch) = split(':', $dependencyItem, 2); @@ -384,14 +394,20 @@ sub _visitDependencyItemAndDependencies $visitedItemsRef->{$item} //= 0; # This module may have already been added to build. - return if $visitedItemsRef->{$item} == 1; + # 0 == Not visited + # 1 == Currently visiting. Running into a module in visit state 1 indicates a cycle. + # 2 == Visited, but not in build (this may happen for common dependencies with siblings, or for + # modules that are not in our build list but are part of dependency chain for other modules + # that *are* in build list). + # 3 == Visited, placed in build queue. + return if $visitedItemsRef->{$item} >= 2; # But if the value is 2 that means we've detected a cycle. - if ($visitedItemsRef->{$item} > 1) { + if ($visitedItemsRef->{$item} == 1) { croak_internal("Somehow there is a dependency cycle involving $item! :("); } - $visitedItemsRef->{$item} = 2; # Mark as currently-visiting for cycle detection. + $visitedItemsRef->{$item} = 1; # Mark as currently-visiting for cycle detection. _makeCatchAllRules($optionsRef, $item); @@ -403,28 +419,36 @@ sub _visitDependencyItemAndDependencies # This keeps us from doing a deep recursive search for dependencies # on an item we've already asked about. - next if (($visitedItemsRef->{$subItemName} // 0) == 1); + next if (($visitedItemsRef->{$subItemName} // 0) >= 2); debug ("\tdep-resolv: $item:$branch depends on $subItem"); my $subModule = $modulesFromNameRef->{$subItemName}; - if (!$subModule) { + if (!$subModule && !$dependentName) { whisper (" y[b[*] $dependencyItem depends on $subItem, but no module builds $subItem for this run."); _visitDependencyItemAndDependencies($optionsRef, $subItem, $level + 1); } else { + # Add in the dependent module if requested. + if (!$subModule) { + $subModule = $moduleFactoryRef->($subItemName); + $modulesFromNameRef->{$subModule->name()} = $subModule; + ++($optionsRef->{modulesNeeded}); + } + if ($subItemBranch ne '*' && (_getBranchOf($subModule) // '') ne $subItemBranch) { my $wrongBranch = _getBranchOf($subModule) // '?'; error (" r[b[*] $item needs $subItem, not $subItemName:$wrongBranch"); } - _visitModuleAndDependencies($optionsRef, $subModule, $level + 1); + _visitModuleAndDependencies($optionsRef, $subModule, $level + 1, $dependentName); } last if $optionsRef->{modulesNeeded} == 0; } - $visitedItemsRef->{$item} = 1; # Mark as done visiting. + # Mark as done visiting. + $visitedItemsRef->{$item} = 2; return; } @@ -466,19 +490,17 @@ sub resolveDependencies # will map names back to their Modules modulesFromName => { - map { $_->fullProjectPath() => $_ } + map { $_->name() => $_ } grep { $_->scmType() eq 'proj' } @modules }, + moduleFactoryRef => $self->{moduleFactoryRef}, + # Help _visitModuleAndDependencies to optimize modulesNeeded => scalar @modules, }; - # Adds things like kde/kdelibs/foo to automatically depend on - # kde/kdelibs if both are present in the build. - _addInherentDependencies($optionsRef); - for my $module (@modules) { _visitModuleAndDependencies($optionsRef, $module); } diff --git a/modules/ksb/KDEXMLReader.pm b/modules/ksb/KDEXMLReader.pm index 72d04ab..ec25896 100644 --- a/modules/ksb/KDEXMLReader.pm +++ b/modules/ksb/KDEXMLReader.pm @@ -90,7 +90,19 @@ sub getModulesForProject $parser->parse($self->inputHandle()); } - my @results; + # A hash is used to hold results since the keys inherently form a set, + # since we don't want dups. + my %results; + my $findResults = sub { + for my $result ( + grep { + _projectPathMatchesWildcardSearch( + $repositories{$_}->{'fullName'}, $proj) + } keys %repositories) + { + $results{$result} = 1; + }; + }; # Wildcard matches happen as specified if asked for. Non-wildcard matches # have an implicit "$proj/*" search as well for compatibility with previous @@ -98,20 +110,16 @@ sub getModulesForProject if ($proj !~ /\*/) { # We have to do a search to account for over-specified module names # like phonon/phonon - push @results, grep { - _projectPathMatchesWildcardSearch($repositories{$_}->{'fullName'}, $proj) - } keys %repositories; + $findResults->(); # Now setup for a wildcard search to find things like kde/kdelibs/baloo # if just 'kdelibs' is asked for. $proj .= '/*'; } - push @results, grep { - _projectPathMatchesWildcardSearch($repositories{$_}->{'fullName'}, $proj) - } keys %repositories; + $findResults->(); - return @repositories{@results}; + return @repositories{keys %results}; } sub xmlTagStart diff --git a/vim/syntax/kdesrc-buildrc.vim b/vim/syntax/kdesrc-buildrc.vim index e5d03fd..8b60c5c 100644 --- a/vim/syntax/kdesrc-buildrc.vim +++ b/vim/syntax/kdesrc-buildrc.vim @@ -1,7 +1,7 @@ " Vim syntax file " Language: kdesrc-build configuration file " Maintainer: Michael Pyne -" Latest Revision: 8 February 2015 +" Latest Revision: 21 February 2015 " Copyright (c) 2014,2015 Michael Pyne " Redistribution and use in source and binary forms, with or without @@ -54,12 +54,12 @@ syn keyword ksbrcErrorGlobalOption contained skipwhite nextgroup=ksbrcStringValu \ kde-languages niceness debug-level persistent-data-file set-env syn keyword ksbrcModuleSetOption contained skipwhite nextgroup=ksbrcStringValue - \ use-modules ignore-modules + \ use-modules ignore-modules include-dependencies " MUST BE CONSISTENT WITH ABOVE. Used when a module-set option is used in the " wrong spot to highlight the error. syn keyword ksbrcErrorModuleSetOption contained skipwhite nextgroup=ksbrcStringValue - \ use-modules ignore-modules + \ use-modules ignore-modules include-dependencies syn keyword ksbrcBoolOption contained skipwhite nextgroup=ksbrcBoolValue \ build-system-only build-when-unchanged ignore-kde-structure