From 319eb52b9b4604b4f4a9a43aa8202b85eb644d37 Mon Sep 17 00:00:00 2001 From: Michael Pyne Date: Fri, 29 Nov 2013 16:51:53 -0500 Subject: [PATCH] Fix bugs in option application to module-set modules. Handling options for the modules generated from a module-set has always been especially clunky, and at some point even the hacks failed and it became impossible to do things like setting an option for a specific module picked out of a larger module-set. E.g. the idea was always that the following would work: module-set foo repository kde-projects use-modules kdelibs kde-workspace kde-runtime cmake-options -DCMAKE_BUILD_TYPE=Release end module-set # Override the above, but only for kde-runtime module kde-runtime cmake-options -DCMAKE_BUILD_TYPE=Debug end module Probably I should have named "module" as "options" or even "override" (and that's still feasible), but in any event this has been broken for awhile: whenever kde-runtime is built it would end up with a Release build type instead of a Debug one. Worse yet, the second mention of kde-runtime was handled as a separate module. Usually this duplicate module was weeded out by accident during the dependency resolution phase, but that leaves open the question of which internal Module object was the "winner"... now we can't even rely on the option handling being predictably broken. This was partially helped by c565d4c which at least prevents the code from spitting out duplicate modules from within a given module-set (which is quite easy to do by accident with kdelibs). This commit reorganizes the command line and option reading code to do the following: - Add a "pending option" tracker, to hold option values that should be applied to a named module, if one happens to be created, either via explicit module-set expansion (e.g. if you ask for nepomuk-core or nepomuk-widgets) or via implicit expansion (e.g. if you ask for kdemultimedia, you get juk and a lot of others). - Add a "selector" method which is responsible for translating module entries on the command line into appropriate module-set or module selections from the Modules and ModuleSets read in from the rc-file. - Pass a subroutine to the module-set expansion method and the selector method to ensure that any new Modules created as a part of either process are checked for pending options (either from the rc-file due to the "override" module method, or from the command line. - While I was digging it out I made the long-overdue switch to Getopt::Long away from my custom option parser. There are a couple of minor features lost in this process but if they are needed I can add them back. Now it should be possible to override options for individual modules within a module-set. However as before, the module you wish to override *must* have been mentioned somewhere before in a use-modules entry so that kdesrc-build can recognize that it must hold onto those option values for later application. It is still safe to mention a kde-projects module and its superset, they will be re-ordered if necessary. BUG:321883 FIXED-IN:1.16 --- modules/ksb/Application.pm | 1145 ++++++++++++++++++------------------ modules/ksb/ModuleSet.pm | 9 + 2 files changed, 595 insertions(+), 559 deletions(-) diff --git a/modules/ksb/Application.pm b/modules/ksb/Application.pm index c03d5cb..cbc446b 100644 --- a/modules/ksb/Application.pm +++ b/modules/ksb/Application.pm @@ -30,6 +30,7 @@ use List::Util qw(first min); use File::Basename; # basename use File::Glob ':glob'; use POSIX qw(:sys_wait_h _exit); +use Getopt::Long qw(GetOptionsFromArray :config gnu_getopt nobundling pass_through); use IO::Handle; use IO::Select; @@ -86,6 +87,383 @@ sub new return $self; } +# Method: _readCommandLineOptionsAndSelectors +# +# Returns a list of module/module-set selectors, selected module/module-set +# options, and global options, based on the command-line arguments passed to +# this function. +# +# This is a package method, should be called as +# $app->_readCommandLineOptionsAndSelectors +# +# Phase: +# initialization - Do not call from this function. +# +# Parameters: +# pendingOptions - hashref to hold parsed modules options to be applied later. +# *Note* this must be done separately, it is not handled by this subroutine. +# Global options will be stored in a hashref at $pendingOptions->{global}. +# Module or module-set options will be stored in a hashref at +# $pendingOptions->{$moduleName} (it will be necessary to disambiguate +# later in the run whether it is a module set or a single module). +# +# If the global option 'start-program' is set, then the selectors parameter +# (see below) will not be selector at all, but will be a list of command +# line options to pass to the chosen program to start (the program will be +# the value of the 'start-program' option mentioned above). +# +# selectors - listref to hold the list of module or module-set selectors to +# build, in the order desired by the user. These will just be strings, the +# caller will have to figure out whether the selector is a module or +# module-set, and create any needed objects, and then set the recommended +# options as listed in pendingOptions. +# +# See pendingOptions for the special note about the value of this listref +# when start-options is in effect. +# +# phases - to hold the list of phases to run. See . +# This will be the PhaseList that should be used for the global context +# object. +# +# @options - The remainder of the arguments are treated as command line +# arguments to process. +# +# Returns: +# Nothing. An exception will be raised on failure, or this function may quit +# the program directly (e.g. to handle --help, --usage). +sub _readCommandLineOptionsAndSelectors +{ + my $self = shift; + my ($pendingOptionsRef, $selectorsRef, $phases, @options) = @_; + my @savedOptions = @options; # Copied for use in debugging. + my $version = "kdesrc-build $SCRIPT_VERSION"; + my $author = < + +Many people have contributed code, bugfixes, and documentation. + +Please report bugs using the KDE Bugzilla, at http://bugs.kde.org/ +DONE + + my %foundOptions; + %foundOptions = ( + version => sub { say $version; exit }, + author => sub { say $author; exit }, + help => sub { _showHelpMessage(); exit 0 }, + install => sub { + $self->{run_mode} = 'install'; + $phases->phases('install'); + }, + uninstall => sub { + $self->{run_mode} = 'uninstall'; + $phases->phases('uninstall'); + }, + 'no-src' => sub { + $phases->filterOutPhase('update'); + }, + 'no-install' => sub { + $phases->filterOutPhase('install'); + }, + 'no-tests' => sub { + # The "right thing" to do + $phases->filterOutPhase('test'); + + # What actually works at this point. + $foundOptions{'run-tests'} = 0; + }, + 'no-build' => sub { + $phases->filterOutPhase('build'); + }, + # Mostly equivalent to the above + 'src-only' => sub { + $phases->phases('update'); + + # We have an auto-switching function that we only want to run + # if --src-only was passed to the command line, so we still + # need to set a flag for it. + $foundOptions{'allow-auto-repo-move'} = 1; + }, + 'build-only' => sub { + $phases->phases('build'); + }, + prefix => sub { + my ($optName, $arg) = @_; + $foundOptions{prefix} = $arg; + $foundOptions{kdedir} = $arg; #TODO: Still needed for compat? + $foundOptions{reconfigure} = 1; + }, + pretend => sub { + # Set pretend mode but also force the build process to run. + $foundOptions{pretend} = 1; + $foundOptions{'build-when-unchanged'} = 1; + }, + verbose => sub { $foundOptions{'debug-level'} = ksb::Debug::WHISPER }, + quiet => sub { $foundOptions{'debug-level'} = ksb::Debug::NOTE }, + 'really-quiet' => sub { $foundOptions{'debug-level'} = ksb::Debug::WARNING }, + debug => sub { + $foundOptions{'debug-level'} = ksb::Debug::DEBUG; + debug ("Commandline was: ", join(', ', @savedOptions)); + }, + + # Hack to set module options + 'set-module-option-value' => sub { + my ($optName, $arg) = @_; + my ($module, $option, $value) = split (',', $arg, 3); + if ($module && $option) { + $pendingOptionsRef->{$module} //= { }; + $pendingOptionsRef->{$module}->{$option} = $value; + } + }, + + # Getopt::Long doesn't set these up for us even though we specify an + # array. Set them up ourselves. + 'start-program' => [ ], + 'ignore-modules' => [ ], + + # Module selectors, the <> is Getopt::Long shortcut for an + # unrecognized non-option value (i.e. an actual argument) + '<>' => sub { + my $arg = shift; + push @{$selectorsRef}, $arg; + }, + ); + + GetOptionsFromArray(\@options, \%foundOptions, + 'version', 'author', 'help', 'disable-snapshots|no-snapshots', + 'install', 'uninstall', 'no-src|no-svn', 'no-install', 'no-build', + 'no-tests', 'build-when-unchanged|force-build', + 'verbose|v', 'quiet|quite|q', 'really-quiet', 'debug', + 'reconfigure', 'colorful-output|color!', 'async!', + 'src-only|svn-only', 'build-only', 'build-system-only', + 'rc-file=s', 'prefix=s', 'niceness|nice:10', 'ignore-modules=s{,}', + 'pretend|dry-run|p', 'refresh-build', 'delete-my-patches', + 'start-program|run=s{,}', + 'delete-my-settings', 'revision=i', 'resume-from=s', 'resume-after=s', + 'stop-after=s', 'stop-before=s', 'set-module-option-value=s', + '<>', # Required to read non-option args + ); + + # TODO: Handle unrecognized options by searching within our list of global + # preset options as before? + say "Warning: Unrecognized option $_" foreach @options; + + $pendingOptionsRef->{'global'} //= { }; + + # To store the values we found, need to strip out the values that are + # subroutines, as those are the ones we created. Alternately, place the + # subs inline as an argument to the appropriate option in the + # GetOptionsFromArray call above, but that's ugly too. + my @readOptionNames = grep { + ref($foundOptions{$_}) ne 'CODE' + } (keys %foundOptions); + + # Slice assignment: $left{$key} = $right{$key} foreach $key (@keys), but + # with hashref syntax everywhere. + @{ $pendingOptionsRef->{'global'} }{@readOptionNames} + = @foundOptions{@readOptionNames}; +} + +# Method: _resolveSelectorsIntoModules +# +# Takes the provided list of module/module-set selectors, pending options to set, +# and module/module-sets to choose from (as read from rc-file) and selects the +# appropriate rc-file module/module-sets. +# +# Additionally the module-sets are expanded into modules. +# +# All pending options are set into each module. Global options are set by +# removing any existing rc-file option value, so you must setup the build context +# separately to have the needed option for this to work. +# +# Returns a list of in build order. +# +# This is a package method, should be called as +# $app->_resolveSelectorsIntoModules +# +# Phase: +# initialization - Do not call from this function. +# +# Parameters: +# ctx - in use. +# +# selectors - listref to hold the list of module or module-set selectors to +# build, in the order desired by the user. The value of this parameter +# WILL BE CHANGED by this function (each string name is replaced by matching +# or . +# +# modNew - reference to a subroutine to run that will accept a newly-made +# and perform any needed setup. The idea is that this subroutine +# will also setup any pending options (either from the cmdline or as rc-file +# overlay). This could easily be an object except it doesn't seem to warrant +# a full separate class. +# +# rcFileModulesAndModuleSets - listref to a list of or (and +# no other types) that can be selected from. +# +# Returns: +# A list: ($metadataModule, @modules), where @modules is a list of to +# build (all module-sets expanded), with options correctly setup from the +# rc-file and cmdline, in the same relative order as determined in selectors. +# $metadataModule is 'undef' if kde-projects metadata is not required, or a +# to use to perform the download otherwise. +# +# If any passed-in selector does not match a module or module set from +# rcFileModulesAndModuleSets then a will be created and assumed to +# come from the kde-projects repository. A special option flag +# '#guessed-kde-project' will be set for such modules. +sub _resolveSelectorsIntoModules +{ + my ($self, $ctx, $selectorsRef, $modNewRef, $rcFileModulesAndModuleSetsRef) + = @_; + my @modules = @{$selectorsRef}; + my $metadataModule; + + # Lookup modules/module-sets by name + my %lookupTable = + map { $_->name() => $_ } (@$rcFileModulesAndModuleSetsRef); + + # Separate lookup table for the use-modules entries for module-sets to allow + # for partial module-set expansion. This and lookupTable *should* end up + # with disjoint sets of module-names, at least as long as the user declared + # their option-overriding module *after* the appropriate use-modules line. + my %setEntryLookupTable; + for my $moduleSet (grep { $_->isa('ksb::ModuleSet') } (@{$rcFileModulesAndModuleSetsRef})) + { + my @results = $moduleSet->moduleNamesToFind(); + + # Have each use-module item point to the source module-set. The + # parens in front of 'x' are semantically required for repetition! + @setEntryLookupTable{@results} = ($moduleSet) x scalar @results; + } + + my %expandedModuleSets; # Holds module-sets expanded on-the-fly + + # This is the meat of our procedure, so we wrap in a sub. + # If not for the dependency on almost all of our params and the lifetime + # requirements this could be a separate function. + my $lookupSelector = sub { + my $selector = shift; + my $selectorName = $selector; + + # Module selectors beginning with '+' force treatment as a kde-projects + # module, which means they won't be matched here (we're only looking for + # sets). + my $forcedToKDEProject = substr($selectorName, 0, 1) eq '+'; + substr($selectorName, 0, 1, '') if $forcedToKDEProject; + + # This test applies for both modules and (entire) module-sets. Note + # that a partially-expanded module-set has its module results + # re-inserted into lookup table though. + if (exists $lookupTable{$selectorName}) { + $selector = $lookupTable{$selectorName}; + $selector->{options}->{'#selected-by'} = 'name'; + } + # This applies to module-sets when only a partial expansion is needed. + elsif (exists $setEntryLookupTable{$selectorName}) { + my $neededModuleSet = $setEntryLookupTable{$selectorName}; + + # _expandModuleSets applies pending/cmdline options. + if (!exists $expandedModuleSets{$neededModuleSet}) { + my (undef, @moduleResults) = _expandModuleSets( + $ctx, $modNewRef, $neededModuleSet); + $expandedModuleSets{$neededModuleSet} = \@moduleResults; + } + + $selector = first { + $_->name() eq $selectorName + } @{$expandedModuleSets{$neededModuleSet}}; + + if (!$selector) { + croak_internal("Didn't select a module that was here earlier in the code path. Please report a bug."); + } + + $lookupTable{$selectorName} = $selector; + $selector->setOption('#selected-by', + 'partial-expansion-' . $neededModuleSet->name()); + } + elsif (ref $selector && $selector->isa('ksb::Module')) { + # We couldn't find anything better than what we were provided, + # just give it back. + $selector = $selector; + $selector->setOption('#selected-by', 'best-guess-after-full-search'); + } + elsif ($forcedToKDEProject) { + # Just assume it's a kde-projects module and expand away... + my $tempSet = ksb::ModuleSet::KDEProjects->new($ctx, '_cmdline'); + $tempSet->setModulesToFind($selectorName); + (undef, $selector) = _expandModuleSets($ctx, $modNewRef, $tempSet); + } + else { + # Neither a named Module, ModuleSet, or use-modules entry within a + # known ModuleSet. It's possible it might be a to-be-expanded + # ModuleSet entry though, so create a shell Module for now and mark + # it as a guess so we can see if it can be sorted out later. + $selector = ksb::Module->new($ctx, $selectorName); + $selector->phases()->phases($ctx->phases()->phases()); + + if ($selectorName eq 'l10n') { + $_->setScmType('l10n') + } + + $selector->setScmType('proj'); + $selector->setOption('#guessed-kde-project', 1); + $selector->setOption('#selected-by', 'initial-guess'); + } + + return $selector; + }; + + # We have to be careful to maintain order of selectors throughout. + for my $selector (@modules) { + $selector = $lookupSelector->($selector); + $modNewRef->($selector); # Perform module option setup + } + + # Filter --resume-foo first so entire module-sets can be skipped. + # Wrap in eval to catch runtime errors + eval { @modules = _applyModuleFilters($ctx, @modules); }; + + ($metadataModule, @modules) = _expandModuleSets($ctx, $modNewRef, @modules); + + # If we have any 'guessed' modules then they had no obvious source in the + # rc-file. But they might still be implicitly from one of our module-sets. + # We want them to use ksb::Modules from the rc-file modules/module-sets + # instead of our shell Modules, if possible. + # But we didn't expand module-sets in rcFileModulesAndModuleSets + # unconditionally, only ones that had been selected via the selectors. + # Because of this we may need to go a step further and expand out all + # remaining module-sets in rcFileModulesAndModuleSets if we have 'guess' + # modules still left over, and see if we can then successfully match. + if (first { $_->getOption('#guessed-kde-project', 'module') } @modules) { + my (undef, @expandedOptionModules) = + _expandModuleSets($ctx, $modNewRef, @$rcFileModulesAndModuleSetsRef); + %lookupTable = map { $_->name() => $_ } @expandedOptionModules; + + for my $guessedModule (grep { + $_->getOption('#guessed-kde-project', 'module') } @modules) + { + # If the module we want could be found from within our rc-file + # module-sets (even implicitly), use it. Otherwise assume + # kde-projects and evaluate now. + if (exists $lookupTable{$guessedModule->name()}) { + $guessedModule = $lookupTable{$guessedModule->name()}; + } + else { + my $set = ksb::ModuleSet::KDEProjects->new($ctx, "guessed_from_cmdline"); + $set->setModulesToFind($guessedModule->name()); + + my (undef, @results) = _expandModuleSets($ctx, $modNewRef, $set); + $guessedModule = shift @results; + } + } + } + + ### TODO: Metadata Module should go away + + return ($metadataModule, @modules); +} + # Runs the pre-initialization phase and takes the kdesrc-build lock. Once this # function successfully completes it is required for the main process to call # finish() to remove the lock. @@ -98,168 +476,109 @@ sub generateModuleList # doing. my $ctx = $self->context(); - my $pendingOptions = { }; - - debug ("+++ Reached pre-init phase"); + my $pendingOptions = { global => { }, }; + my $pendingGlobalOptions = $pendingOptions->{global}; # Process --help, --install, etc. first. - my @modules = $self->_processCmdlineArguments($ctx, $pendingOptions, @argv); + my @selectors; + $self->_readCommandLineOptionsAndSelectors($pendingOptions, \@selectors, + $ctx->phases(), @argv); + + my %ignoredSelectors; + @ignoredSelectors{@{$pendingGlobalOptions->{'ignore-modules'}}} = undef; + + my @startProgramArgs = @{$pendingGlobalOptions->{'start-program'}}; + delete @{$pendingGlobalOptions}{qw/ignore-modules start-program/}; - debug ("--- Arguments read: ", scalar @modules, " command line modules"); + # Everything else in pendingOptions should be OK to apply directly as a module + # or context option. + + # rc-file needs special handling. + if (exists $pendingGlobalOptions->{'rc-file'} && $pendingGlobalOptions->{'rc-file'}) { + $ctx->setRcFile($pendingGlobalOptions->{'rc-file'}); + } + + # disable async if only running a single phase. + $pendingGlobalOptions->{async} = 0 if (scalar $ctx->phases()->phases() == 1); # Output time once we know if pretending or not. my $time = localtime; info ("Script started processing at g[$time]") unless pretending(); - # Change name and type of command line entries beginning with + to force - # them to be XML project modules. - foreach (@modules) { - if (substr($_->{name}, 0, 1) eq '+') { - debug ("Forcing ", $_->name(), " to be an XML module"); - $_->setScmType('proj'); - substr($_->{name}, 0, 1) = ''; # Remove first char - } - } - my $fh = $ctx->loadRcFile(); - # If we're still here, read the options - my @optionModulesAndSets = _readConfigurationOptions($ctx, $fh); + # _readConfigurationOptions will add pending global opts to ctx while ensuring + # returned modules/sets have any such options stripped out. It will also add + # module-specific options to any returned modules/sets. + my @optionModulesAndSets = _readConfigurationOptions($ctx, $fh, $pendingOptions); close $fh; - debug ("--- Config file read: ", scalar @optionModulesAndSets, " modules/module sets."); - debug (" --- " , (scalar grep { $_->isa('ksb::ModuleSet') } @optionModulesAndSets) , " are module sets."); - - # Modify the options read from the rc-file to have the pending changes from - # the command line. - foreach my $pendingModule (keys %{$pendingOptions}) { - my $options = ${$pendingOptions}{$pendingModule}; - my ($module) = grep { - $_->isa('ksb::Module') && $pendingModule eq $_->name() - } (@optionModulesAndSets); - - if (!$module) { - warning ("Tried to set options for unknown module b[y[$pendingModule]"); - next; - } - - while (my ($key, $value) = each %{$options}) { - $module->setOption($key, $value); - } - } # Check if we're supposed to drop into an interactive shell instead. If so, # here's the stop off point. - if (my $prog = $ctx->getOption('#start-program')) + if (@startProgramArgs) { # @modules is the command line arguments to pass in this case. - _executeCommandLineProgram($prog, @modules); + # TODO: Have context setup environment variables here first. + _executeCommandLineProgram(@startProgramArgs); # noreturn } - my $commandLineModules = scalar @modules; + # At this point we have our list of candidate modules / module-sets (as read in + # from rc-file). The module sets have not been expanded into modules. + # We also might have cmdline "selectors" to determine which modules or + # module-sets to choose. First let's select module sets, and expand them. + + my @modules; + my @globalCmdlineArgs = keys %{$pendingGlobalOptions}; + my $commandLineModules = scalar @selectors; my $metadataModule; - # Allow named module-sets to be given on the command line. - if ($commandLineModules) { - # Copy ksb::Module and ksb::ModuleSet objects from the ones created by - # _readConfigurationOptions since their module-type will actually be set. - _spliceOptionModules(\@modules, \@optionModulesAndSets); - debug ("--- spliced rc-file modules/sets into command line modules"); - debug (" --- now ", scalar @modules, " modules present"); - - # After this splicing, the only bare ksb::Modules with a 'proj' - # scm type should be the ones from the command line that we got from - # _processCmdlineArguments. - - # Modify l10n module inline, if present. - for (@modules) { - if ($_->name() eq 'l10n' && $_->isa('ksb::Module')) { - $_->setScmType('l10n') - } + my $newModuleSub = sub { + my $module = shift; + while (my ($k, $v) = each %{$pendingOptions->{$module->name()}}) { + $module->setOption($k, $v); } - # Filter --resume-foo first so entire module-sets can be skipped. - # Wrap in eval to catch runtime errors - eval { @modules = _applyModuleFilters($ctx, @modules); }; - - debug ("--- Applied command-line --resume-foo pass, ", scalar @modules, " remain."); - debug (" --- " , (scalar grep { $_->isa("ksb::ModuleSet") } @modules) , " are module sets."); - - ($metadataModule, @modules) = _expandModuleSets($ctx, @modules); - - debug ("--- Expanded command-line module sets into ", scalar @modules, " modules."); - debug (" --- Metadata module needed? ", $metadataModule ? "Yes" : "No"); - - # If we have any 'guessed' modules they came from the command line. We - # want them to use ksb::Modules that came from - # _readConfigurationOptions() instead of ones from _processCmdlineArguments(), - # if any are available. But we don't expand module-sets from - # _readConfigurationOptions unconditionally, only ones where the name - # matched between the command line and rc-file, so the previous - # _spliceOptionModules might not have found them. (e.g. if you named a - # module on the command line that isn't named directly in the rc file - # but would eventually be found implicitly). To make this work we have - # to expand out our rc-file Modules and try splicing again. - if (first { $_->getOption('#guessed-kde-project', 'module') } @modules) { - my (undef, @expandedOptionModules) = _expandModuleSets($ctx, @optionModulesAndSets); - _spliceOptionModules(\@modules, \@expandedOptionModules); - - debug ("--- re-spliced rc-file modules/sets into command line modules"); - debug (" --- now ", scalar @modules, " modules present"); - } + # Just in case + delete @{$module->{options}}{@globalCmdlineArgs}; + }; + + if ($commandLineModules) { + # select our modules and module-sets, and expand them out + ($metadataModule, @modules) = $self->_resolveSelectorsIntoModules( + $ctx, \@selectors, $newModuleSub, \@optionModulesAndSets); - # At this point @modules has no more module sets, kde-projects or - # otherwise. ksb::Module->setModuleSource('cmdline'); } else { # Build everything in the rc-file, in the order specified. - ($metadataModule, @modules) = _expandModuleSets($ctx, @optionModulesAndSets); - debug ("--- Expanded rc-file module sets into ", scalar @modules, " modules."); - debug (" --- Metadata module needed? ", $metadataModule ? "Yes" : "No"); + # Check for ignored module-sets and modules (pre-expansion) + @optionModulesAndSets = grep { ! exists $ignoredSelectors{$_->name()} } @optionModulesAndSets; + ($metadataModule, @modules) = _expandModuleSets( + $ctx, $newModuleSub, @optionModulesAndSets); if ($ctx->getOption('kde-languages')) { - my $l10nModule = ksb::Module->new($ctx, 'l10n'); - $l10nModule->setScmType('l10n'); - $l10nModule->setBuildSystem($l10nModule->scm()); - - debug ("--- Added l10n module to end of build"); - push @modules, $l10nModule; + @modules = _expandl10nModules($ctx, @modules); } ksb::Module->setModuleSource('config'); } + # Check for ignored modules (post-expansion) + @modules = grep { ! exists $ignoredSelectors{$_->name()} } @modules; + # Filter --resume-foo options. This might be a second pass, but that should # be OK since there's nothing different going on from the first pass in that # event. @modules = _applyModuleFilters($ctx, @modules); - debug ("--- Applied all-modules --resume-foo pass, ", scalar @modules, " remain."); - debug (" --- ", (scalar grep { $_->isa("ksb::ModuleSet") } @modules), " are module sets (should be 0!)."); - - # Apply kde-languages, by appending needed l10n modules to the end of the - # build. - @modules = _expandl10nModules($ctx, @modules); - - debug ("--- Expanded l10n modules. ", scalar @modules, " remain."); - - # Remove ignored modules and/or module-sets - @modules = grep { - not exists $ignore_list{$_->name()} && not exists $ignore_list{$_->moduleSet()->name()} - } (@modules); - - debug ("--- Handled ignore lists. ", scalar @modules, " remain."); - # If modules were on the command line then they are effectively forced to # process unless overridden by command line options as well. If phases # *were* overridden on the command line, then no update pass is required # (all modules already have correct phases) @modules = _updateModulePhases(@modules) unless $commandLineModules; - debug ("--- Updated module phases. ", scalar @modules, " remain."); - # Save our metadata module, if used. $self->{metadata_module} = $metadataModule; $ctx->setKDEProjectMetadataModule($metadataModule); @@ -278,8 +597,6 @@ sub runAllModulePhases $ctx->loadPersistentOptions(); - debug ("+++ Reached execution phase"); - # If we have kde-build-metadata we must process it first, ASAP. if ($metadataModule) { eval { @@ -730,10 +1047,23 @@ EOF # initialization - Do not call from this function. # # Parameters: -# ctx - The to update based on the configuration read. +# ctx - The to update based on the configuration read and +# any pending options (see pendingGlobalOptions). +# # filehandle - The I/O object to read from. Must handle _eof_ and _readline_ # methods (e.g. subclass). # +# pendingOptions - hashref holding key/value pairs of pending +# options. Any read-in global options matching a key in this hash will be +# ignored in the result list (i.e. the global options will mask the read-in +# ones), except for ctx, which will have its options modified to match the +# pending ones. +# +# Conversely, any module/set options to be applied in pendingOptions are +# applied before this function returns. Options for a module are removed from +# pendingOptions when they are applied, so that you can tell which options have +# not yet been applied. +# # Returns: # @module - Heterogenous list of and defined in the # configuration file. No module sets will have been expanded out (either @@ -745,6 +1075,7 @@ sub _readConfigurationOptions { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $fh = shift; + my $pendingOptionsRef = shift; my @module_list; my $rcfile = $ctx->rcFile(); my ($option, $modulename, %readModules); @@ -770,10 +1101,17 @@ sub _readConfigurationOptions # Now read in each global option. _parseModuleOptions($ctx, $fileReader, 'global'); + while (my ($k, $v) = each %{$pendingOptionsRef->{global}}) { + $ctx->setOption($k, $v); + } + last; } my $using_default = 1; + my @pendingOptsKeys = keys %{$pendingOptionsRef->{global}}; + my %seenModules; # NOTE! *not* module-sets, *just* modules. + my %seenModuleSetItems; # To track option override modules. # Now read in module settings while ($_ = $fileReader->readLine()) @@ -784,9 +1122,10 @@ sub _readConfigurationOptions # Get modulename (has dash, dots, slashes, or letters/numbers) ($modulename) = /^module\s+([-\/\.\w]+)\s*$/; + my $newModule; - if (not $modulename) - { + # Module-set? + if (not $modulename) { my $moduleSetRE = qr/^module-set\s*([-\/\.\w]+)?\s*$/; ($modulename) = m/$moduleSetRE/; @@ -798,20 +1137,53 @@ sub _readConfigurationOptions } # A moduleset can give us more than one module to add. - push @module_list, _parseModuleSetOptions($ctx, $fileReader, $modulename); + $newModule = _parseModuleSetOptions($ctx, $fileReader, $modulename); + + # Save 'use-modules' entries so we can see if later module decls + # are overriding/overlaying their options. + my @moduleSetItems = $newModule->moduleNamesToFind(); + @seenModuleSetItems{@moduleSetItems} = ($newModule) x scalar @moduleSetItems; } - else { - # Overwrite options set for existing modules. - if (my @modules = grep { $_->isa('ksb::Module') && ($_->name() eq $modulename) } @module_list) { - # We check for definedness as a module-set can exist but be - # unnamed. - warning ("Multiple module declarations for $modules[0]"); - _parseModuleOptions($ctx, $fileReader, $modules[0]); # Don't re-add - } - else { - push @module_list, _parseModuleOptions($ctx, $fileReader, $modulename); + # Module override? + elsif (exists $seenModuleSetItems{$modulename}) { + # Parse the modules... + $newModule = _parseModuleOptions($ctx, $fileReader, "#overlay_$modulename"); + + # but only keep the options. Any existing pending options came from + # cmdline so do not overwrite existing keys. + $pendingOptionsRef->{$modulename} //= { }; + my $moduleOptsRef = $pendingOptionsRef->{$modulename}; + while (my ($k, $v) = each %{$newModule->{options}}) { + $moduleOptsRef->{$k} = $v unless exists $moduleOptsRef->{$k}; } + + # Don't mask global cmdline options. + delete @{$moduleOptsRef}{@pendingOptsKeys}; + + next; # Don't add to module list } + # Duplicate module entry? + elsif (exists $seenModules{$modulename}) { + # Overwrite options set for existing modules. + warning ("Multiple module declarations for $modulename"); + + $newModule = $seenModules{$modulename}; + + # _parseModuleOptions will re-use newModule, but we still need to + # be careful not to mask cmdline options in pendingOptsKeys. + _parseModuleOptions($ctx, $fileReader, $newModule); + + delete @{$newModule->{options}}{@pendingOptsKeys}; + + next; # Skip all the stuff below + } + else { + $newModule = _parseModuleOptions($ctx, $fileReader, $modulename); + $seenModules{$modulename} = $newModule; + } + + delete @{$newModule->{options}}{@pendingOptsKeys}; + push @module_list, $newModule; # Don't build default modules if user has their own wishes. $using_default = 0; @@ -1630,6 +2002,8 @@ EOF # # Parameters: # $ctx - in use for this script execution. +# $modNew - Reference to a subroutine to be run for every new +# created. See _resolveSelectorsIntoModules for full details. # @modules - list of , to be expanded. # # Returns: @@ -1638,33 +2012,20 @@ EOF # @modules - List of with any module-sets expanded into . sub _expandModuleSets { - my ($ctx, @buildModuleList) = @_; + my ($ctx, $modNewSub, @buildModuleList) = @_; my $filter = sub { - my $moduleOrSetName = $_->name(); - - # 'proj' module types can only come from command line -- we assume the - # user is trying to build a module from the kde-projects repo without - # first putting into rc-file. - if ($_->isa('ksb::Module') && $_->scmType() ne 'proj') { - return $_; - } - if ($_->isa('ksb::ModuleSet')) { - return $_->convertToModules($ctx); + return map { $modNewSub->($_); $_ } ($_->convertToModules($ctx)); } - my $moduleSet = ksb::ModuleSet::KDEProjects->new($ctx, ''); - $moduleSet->setModulesToFind($_->name()); - $moduleSet->{options}->{'#guessed-kde-project'} = 1; - - debug ("--- Trying to find a home for $_"); - return $moduleSet->convertToModules($ctx); + return $_; }; my @moduleResults = map { &$filter } (@buildModuleList); my $metadataModule; + # TODO Extract this to a higher-level function. if (first { $_->scmType() eq 'proj' } @moduleResults) { debug ("Introducing metadata module into the build"); $metadataModule = ksb::ModuleSet::KDEProjects::getMetadataModule($ctx); @@ -1791,417 +2152,6 @@ sub _extractOptionValueRequired return $returnValue; } -# Method: _processCmdlineArguments -# -# Processes the command line arguments, which are used to modify the given -# and possibly return a list of . -# -# This is a package method, should be called as $app->_processCmdlineArguments -# -# Phase: -# initialization - Do not call from this function. -# -# Parameters: -# ctx - BuildContext in use. -# pendingOptions - hashref to hold parsed modules options to be applied later. -# *Note* this must be done separately, it is not handled by this subroutine. -# @options - The remainder of the arguments are treated as command line -# arguments to process. -# -# Returns: -# - List of that represent modules specifically entered on the -# command-line, _or_ -# - List of options to pass to a command named by the --run command line -# option. (This is true if and only if the _ctx_ ends up with the -# _#start-program_ option set). -sub _processCmdlineArguments -{ - my $self = shift; - my $ctx = assert_isa(shift, 'ksb::BuildContext'); - my $pendingOptions = shift; - my $phases = $ctx->phases(); - my @savedOptions = @_; # Used for --debug - my @options = @_; - my $arg; - my $version = "kdesrc-build $SCRIPT_VERSION"; - my $author = < - -Many people have contributed code, bugfixes, and documentation. - -Please report bugs using the KDE Bugzilla, at http://bugs.kde.org/ -DONE - - my @enteredModules; - - while ($_ = shift @options) - { - SWITCH: { - /^(--version)$/ && do { print "$version\n"; exit; }; - /^--author$/ && do { print $author; exit; }; - /^(-h)|(--?help)$/ && do { - print < Read configuration from filename instead of default. - - --resume-from= Skips modules until just before the given package, - then operates as normal. - --resume-after= Skips modules up to and including the given package, - then operates as normal. - - --stop-before= Skips the given package and all later packages. - --stop-after= Skips all packages after the given package. - - --reconfigure Run CMake/configure again, but don't clean the build - directory. - --build-system-only Create the build infrastructure, but don't actually - perform the build. - - --