# This chunk of stuff was generated by App::FatPacker. To find the original # file's code, look for the end of this BEGIN block or the string 'FATPACK' BEGIN { my %fatpacked; $fatpacked{"Dist/Surveyor.pm"} = <<'DIST_SURVEYOR'; package Dist::Surveyor; { $Dist::Surveyor::VERSION = '0.013'; } =head1 NAME Dist::Surveyor - Survey installed modules and determine the specific distribution versions they came from =head1 VERSION version 0.013 =head1 SYNOPSIS my $options = { opt_match => $opt_match, opt_perlver => $opt_perlver, opt_remnants => $opt_remnants, distro_key_mod_names => $distro_key_mod_names, }; my @installed_releases = determine_installed_releases($options, \@libdirs); =head1 DESCRIPTION Surveys your huge ball of Perl modules, jammed together inside a directory, and tells you exactly which module is installed there. For quick start, and a fine example of this module usage, see L. This module have one exported function - determine_installed_releases =cut use strict; use warnings; use version; use Carp; # core use Data::Dumper; # core use File::Find; # core use File::Spec; # core use List::Util qw(max sum); # core use Dist::Surveyor::Inquiry; # internal use Module::CoreList; use Module::Metadata; use constant ON_WIN32 => $^O eq 'MSWin32'; use constant ON_VMS => $^O eq 'VMS'; if (ON_VMS) { require File::Spec::Unix; } our ($DEBUG, $VERBOSE); *DEBUG = \$::DEBUG; *VERBOSE = \$::VERBOSE; require Exporter; our @ISA = qw{Exporter}; our @EXPORT = qw{determine_installed_releases}; =head1 determine_installed_releases($options, $search_dirs) $options includes: =over =item opt_match A regex qr//. If exists, will ignore modules that doesn't match this regex =item opt_perlver Skip modules that are included as core in this Perl version =item opt_remnants If true, output will include old distribution versions that have left old modules behind =item distro_key_mod_names A hash-ref, with a list of irregular named releases. i.e. 'libwww-perl' => 'LWP'. =back $search_dirs is an array-ref containing the list of directories to survey. Returns a list, where each element is a hashref representing one installed distibution. This hashref is what MetaCPAN returns for http://api.metacpan.org/v0/release/$author/$release, with two additional keys: =over =item * 'url' - that same as 'download_url', but without the hostname. can be used to download the file for your favorite mirror =item * 'dist_data' - Hashref containing info about the release, i.e. percent_installed. (fully installed releases will have '100.00') =back =cut sub determine_installed_releases { my ($options, $search_dirs) = @_; $options->{opt_perlver} ||= version->parse( $] )->numify; my %installed_mod_info; warn "Finding modules in @$search_dirs\n"; my ($installed_mod_files, $installed_meta) = find_installed_modules(@$search_dirs); # get the installed version of each installed module and related info warn "Finding candidate releases for the ".keys(%$installed_mod_files)." installed modules\n"; foreach my $module ( sort keys %$installed_mod_files ) { my $mod_file = $installed_mod_files->{$module}; if (my $opt_match = $options->{opt_match}) { if ($module !~ m/$opt_match/o) { delete $installed_mod_files->{$module}; next; } } module_progress_indicator($module) unless $VERBOSE; my $mi = get_installed_mod_info($options, $module, $mod_file); $installed_mod_info{$module} = $mi if $mi; } # Map modules to dists using the accumulated %installed_mod_info info warn "*** Mapping modules to releases\n"; my %best_dist; foreach my $mod ( sort keys %installed_mod_info ) { my $mi = $installed_mod_info{$mod}; module_progress_indicator($mod) unless $VERBOSE; # find best match among the cpan releases that included this module my $ccdr = $installed_mod_info{$mod}{candidate_cpan_dist_releases} or next; # no candidates, warned about above (for mods with a version) my $best_dist_cache_key = join " ", sort keys %$ccdr; our %best_dist_cache; my $best = $best_dist_cache{$best_dist_cache_key} ||= pick_best_cpan_dist_release($ccdr, \%installed_mod_info); my $note = ""; if ((@$best > 1) and $installed_meta->{perllocalpod}) { # try using perllocal.pod to narrow the options, if there is one # XXX TODO move this logic into the per-candidate-distro loop below # it doesn't make much sense to be here at the per-module level my @in_perllocal = grep { my $distname = $_->{distribution}; my ($v, $dist_mod_name) = perllocal_distro_mod_version( $options->{distro_key_mod_names}, $distname, $installed_meta->{perllocalpod}); warn "$dist_mod_name in perllocal.pod: ".($v ? "YES" : "NO")."\n" if $DEBUG; $v; } @$best; if (@in_perllocal && @in_perllocal < @$best) { $note = sprintf "narrowed from %d via perllocal", scalar @$best; $best = \@in_perllocal; } } if (@$best > 1 or $note) { # note the poor match for this module # but not if there's no version (as that's common) my $best_desc = join " or ", map { $_->{release} } @$best; my $pct = sprintf "%.2f%%", $best->[0]{fraction_installed} * 100; warn "$mod $mi->{version} odd best match: $best_desc $note ($best->[0]{fraction_installed})\n" if $note or $VERBOSE or ($mi->{version} and $best->[0]{fraction_installed} < 0.3); # if the module has no version and multiple best matches # then it's unlikely make a useful contribution, so ignore it # XXX there's a risk that we'd ignore all the modules of a release # where none of the modules has a version, but that seems unlikely. next if not $mi->{version}; } for my $dist (@$best) { # two level hash to make it easier to handle versions my $di = $best_dist{ $dist->{distribution} }{ $dist->{release} } ||= { dist => $dist }; push @{ $di->{modules} }, $mi; $di->{or}{$_->{release}}++ for grep { $_ != $dist } @$best; } } warn "*** Refining releases\n"; # $best_dist{ Foo }{ Foo-1.23 }{ dist=>$dist_struct, modules=>, or=>{ Foo-1.22 => $dist_struct } } my @installed_releases; # Dist-Name => { ... } for my $distname ( sort keys %best_dist ) { my $releases = $best_dist{$distname}; push @installed_releases, refine_releases($options, $distname, $releases); } # sorting into dependency order could be added later, maybe return @installed_releases; } sub refine_releases { my ($options, $distname, $releases) = @_; my @dist_by_version = sort { $a->{dist}{version_obj} <=> $b->{dist}{version_obj} or $a->{dist}{fraction_installed} <=> $b->{dist}{fraction_installed} } values %$releases; my @dist_by_fraction = sort { $a->{dist}{fraction_installed} <=> $b->{dist}{fraction_installed} or $a->{dist}{version_obj} <=> $b->{dist}{version_obj} } values %$releases; my @remnant_dists = @dist_by_version; my $installed_dist = pop @remnant_dists; # is the most recent candidate dist version also the one with the # highest fraction_installed? if ($dist_by_version[-1] == $dist_by_fraction[-1]) { # this is the common case: we'll assume that's installed and the # rest are remnants of earlier versions } elsif ($dist_by_fraction[-1]{dist}{fraction_installed} == 100) { warn "Unsure which $distname is installed from among @{[ keys %$releases ]}\n"; @remnant_dists = @dist_by_fraction; $installed_dist = pop @remnant_dists; warn "Selecting the one that apprears to be 100% installed\n"; } else { # else grumble so the user knows to ponder the possibilities warn "Can't determine which $distname is installed from among @{[ keys %$releases ]}\n"; warn Dumper([\@dist_by_version, \@dist_by_fraction]); warn "\tSelecting based on latest version\n"; } if (@remnant_dists or $DEBUG) { warn "Distributions with remnants (chosen release is first):\n" unless our $dist_with_remnants_warning++; warn "@{[ map { $_->{dist}{release} } reverse @dist_by_fraction ]}\n"; for ($installed_dist, @remnant_dists) { my $fi = $_->{dist}{fraction_installed}; my $modules = $_->{modules}; my $mv_desc = join(", ", map { "$_->{module} $_->{version}" } @$modules); warn sprintf "\t%s\t%s%% installed: %s\n", $_->{dist}{release}, $_->{dist}{percent_installed}, (@$modules > 4 ? "(".@$modules." modules)" : $mv_desc), } } my @installed_releases; # note ordering: remnants first for (($options->{opt_remnants} ? @remnant_dists : ()), $installed_dist) { my ($author, $release) = @{$_->{dist}}{qw(author release)}; my $release_data = get_release_info($author, $release); next unless $release_data; # shortcuts (my $url = $release_data->{download_url}) =~ s{ .*? \b authors/ }{authors/}x; push @installed_releases, { # %$release_data, # extra items mushed inhandy shortcuts url => $url, # raw data structures dist_data => $_->{dist}, }; } #die Dumper(\@installed_releases); return @installed_releases; } # for each installed module, get the list of releases that it exists in # Parameters: # $options - uses only opt_perlver # $module - module name (i.e. 'Dist::Surveyor') # $mod_file - the location of this module on the filesystem # Return: # undef if this module should be skipped # otherwise, a hashref containing: # file => $mod_file, # module => $module, # version => $mod_version, # version_obj => same as version, but as an object, # size => $mod_file_size, # # optional flags: # file_size_mismatch => 1, # cpan_dist_fallback => 1, # could not find this module/version on cpan, # # but found a release with that version, containing such module # version_not_on_cpan> 1, # can not find this file on CPAN. # # releases info # candidate_cpan_dist_releases => hashref, # # candidate_cpan_dist_releases hashref contain a map of all the releases # that this module exists in. see get_candidate_cpan_dist_releases for more # info. sub get_installed_mod_info { my ($options, $module, $mod_file) = @_; my $mod_version = do { # silence warnings about duplicate VERSION declarations # eg Catalyst::Controller::DBIC::API* 2.002001 local $SIG{__WARN__} = sub { warn @_ if $_[0] !~ /already declared with version/ }; my $mm = Module::Metadata->new_from_file($mod_file); $mm->version; # only one version for one package in file }; $mod_version ||= 0; # XXX my $mod_file_size = -s $mod_file; # Eliminate modules that will be supplied by the target perl version if ( my $cv = $Module::CoreList::version{ $options->{opt_perlver} }->{$module} ) { $cv =~ s/ //g; if (version->parse($cv) >= version->parse($mod_version)) { warn "$module is core in perl $options->{opt_perlver} (lib: $mod_version, core: $cv) - skipped\n"; return; } } my $mi = { file => $mod_file, module => $module, version => $mod_version, version_obj => version->parse($mod_version), size => $mod_file_size, }; # ignore modules we know aren't indexed return $mi if $module =~ /^Moose::Meta::Method::Accessor::Native::/; # XXX could also consider file mtime: releases newer than the mtime # of the module file can't be the origin of that module file. # (assuming clocks and file times haven't been messed with) eval { my $ccdr = get_candidate_cpan_dist_releases($module, $mod_version, $mod_file_size); if (not %$ccdr) { $ccdr = get_candidate_cpan_dist_releases($module, $mod_version, 0); if (%$ccdr) { # probably either a local change/patch or installed direct from repo # but with a version number that matches a release warn "$module $mod_version on CPAN but with different file size (not $mod_file_size)\n" if $mod_version or $VERBOSE; $mi->{file_size_mismatch}++; } elsif ($ccdr = get_candidate_cpan_dist_releases_fallback($module, $mod_version) and %$ccdr) { warn "$module $mod_version not on CPAN but assumed to be from @{[ sort keys %$ccdr ]}\n" if $mod_version or $VERBOSE; $mi->{cpan_dist_fallback}++; } else { $mi->{version_not_on_cpan}++; # Possibly: # - a local change/patch or installed direct from repo # with a version number that was never released. # - a private module never released on cpan. # - a build-time create module eg common/sense.pm.PL warn "$module $mod_version not found on CPAN\n" if $mi->{version} # no version implies uninteresting or $VERBOSE; # XXX could try finding the module with *any* version on cpan # to help with later advice. ie could select as candidates # the version above and the version below the number we have, # and set a flag to inform later logic. } } $mi->{candidate_cpan_dist_releases} = $ccdr if %$ccdr; }; if ($@) { warn "Failed get_candidate_cpan_dist_releases($module, $mod_version, $mod_file_size): $@"; } return $mi; } # pick_best_cpan_dist_release - memoized # for each %$ccdr adds a fraction_installed based on %$installed_mod_info # returns ref to array of %$ccdr values that have the max fraction_installed sub pick_best_cpan_dist_release { my ($ccdr, $installed_mod_info) = @_; for my $release (sort keys %$ccdr) { my $release_info = $ccdr->{$release}; $release_info->{fraction_installed} = dist_fraction_installed($release_info->{author}, $release, $installed_mod_info); $release_info->{percent_installed} # for informal use = sprintf "%.2f", $release_info->{fraction_installed} * 100; } my $max_fraction_installed = max( map { $_->{fraction_installed} } values %$ccdr ); my @best = grep { $_->{fraction_installed} == $max_fraction_installed } values %$ccdr; return \@best; } # returns a number from 0 to 1 representing the fraction of the modules # in a particular release match the coresponding modules in %$installed_mod_info sub dist_fraction_installed { my ($author, $release, $installed_mod_info) = @_; my $tag = "$author/$release"; my $mods_in_rel = get_module_versions_in_release($author, $release); my $mods_in_rel_count = keys %$mods_in_rel; my $mods_inst_count = sum( map { my $mi = $installed_mod_info->{ $_->{name} }; # XXX we stash the version_obj into the mods_in_rel hash # (though with little/no caching effect with current setup) $_->{version_obj} ||= eval { version->parse($_->{version}) }; my $hit = ($mi && $mi->{version_obj} == $_->{version_obj}) ? 1 : 0; # demote to a low-scoring partial match if the file size differs # XXX this isn't good as the effect varies with the number of modules $hit = 0.1 if $mi && $mi->{size} != $_->{size}; warn sprintf "%s %s %s %s: %s\n", $tag, $_->{name}, $_->{version_obj}, $_->{size}, ($hit == 1) ? "matches" : ($mi) ? "differs ($mi->{version_obj}, $mi->{size})" : "not installed", if $DEBUG; $hit; } values %$mods_in_rel) || 0; my $fraction_installed = ($mods_in_rel_count) ? $mods_inst_count/$mods_in_rel_count : 0; warn "$author/$release:\tfraction_installed $fraction_installed ($mods_inst_count/$mods_in_rel_count)\n" if $VERBOSE or !$mods_in_rel_count; return $fraction_installed; } sub get_file_mtime { my ($file) = @_; # try to find the time the file was 'installed' # by looking for the commit date in svn or git # else fallback to the file modification time return (stat($file))[9]; } sub find_installed_modules { my (@dirs) = @_; ### File::Find uses follow_skip => 1 by default, which doesn't die ### on duplicates, unless they are directories or symlinks. ### Ticket #29796 shows this code dying on Alien::WxWidgets, ### which uses symlinks. ### File::Find doc says to use follow_skip => 2 to ignore duplicates ### so this will stop it from dying. my %find_args = ( follow_skip => 2 ); ### File::Find uses lstat, which quietly becomes stat on win32 ### it then uses -l _ which is not allowed by the statbuffer because ### you did a stat, not an lstat (duh!). so don't tell win32 to ### follow symlinks, as that will break badly # XXX disabled because we want the postprocess hook to work #$find_args{'follow_fast'} = 1 unless ON_WIN32; ### never use the @INC hooks to find installed versions of ### modules -- they're just there in case they're not on the ### perl install, but the user shouldn't trust them for *other* ### modules! ### XXX CPANPLUS::inc is now obsolete, remove the calls #local @INC = CPANPLUS::inc->original_inc; # sort @dirs to put longest first to make it easy to handle # elements that are within other elements (e.g., an archdir) my @dirs_ordered = sort { length $b <=> length $a } @dirs; my %seen_mod; my %dir_done; my %meta; # return metadata about the search for my $dir (@dirs_ordered) { next if $dir eq '.'; ### not a directory after all ### may be coderef or some such next unless -d $dir; ### make sure to clean up the directories just in case, ### as we're making assumptions about the length ### This solves rt.cpan issue #19738 ### John M. notes: On VMS cannonpath can not currently handle ### the $dir values that are in UNIX format. $dir = File::Spec->canonpath($dir) unless ON_VMS; ### have to use F::S::Unix on VMS, or things will break my $file_spec = ON_VMS ? 'File::Spec::Unix' : 'File::Spec'; ### XXX in some cases File::Find can actually die! ### so be safe and wrap it in an eval. eval { File::Find::find( { %find_args, postprocess => sub { $dir_done{$File::Find::dir}++; }, wanted => sub { unless (/\.pm$/i) { # skip all dot-dirs (eg .git .svn) $File::Find::prune = 1 if -d $File::Find::name and /^\.\w/; # don't reenter a dir we've already done $File::Find::prune = 1 if $dir_done{$File::Find::name}; # remember perllocal.pod if we see it push @{$meta{perllocalpod}}, $File::Find::name if $_ eq 'perllocal.pod'; return; } my $mod = $File::Find::name; ### make sure it's in Unix format, as it ### may be in VMS format on VMS; $mod = VMS::Filespec::unixify($mod) if ON_VMS; $mod = substr( $mod, length($dir) + 1, -3 ); $mod = join '::', $file_spec->splitdir($mod); return if $seen_mod{$mod}; $seen_mod{$mod} = $File::Find::name; ### ignore files that don't contain a matching package declaration ### warn about those that do contain some kind of package declaration #use File::Slurp; #my $content = read_file($File::Find::name); #unless ( $content =~ m/^ \s* package \s+ (\#.*\n\s*)? $mod \b/xm ) { #warn "No 'package $mod' seen in $File::Find::name\n" #if $VERBOSE && $content =~ /\b package \b/x; #return; #} }, }, $dir ); 1; } or die "File::Find died: $@"; } return (\%seen_mod, \%meta); } sub perllocal_distro_mod_version { my ($distro_key_mod_names, $distname, $perllocalpod) = @_; ( my $dist_mod_name = $distname ) =~ s/-/::/g; my $key_mod_name = $distro_key_mod_names->{$distname} || $dist_mod_name; our $perllocal_distro_mod_version; if (not $perllocal_distro_mod_version) { # initial setup warn "Only first perllocal.pod file will be processed: @$perllocalpod\n" if ref $perllocalpod eq 'ARRAY' and @$perllocalpod > 1; $perllocal_distro_mod_version = {}; # extract data from perllocal.pod if (my $plp = shift @$perllocalpod) { # The VERSION isn't always the same as that in the distro file if (eval { require ExtUtils::Perllocal::Parser }) { my $p = ExtUtils::Perllocal::Parser->new; $perllocal_distro_mod_version = { map { $_->name => $_->{data}{VERSION} } $p->parse_from_file($plp) }; warn "Details of ".keys(%$perllocal_distro_mod_version)." distributions found in $plp\n"; } else { warn "Wanted to use perllocal.pod but can't because ExtUtils::Perllocal::Parser isn't available\n"; } } else { warn "No perllocal.pod found to aid disambiguation\n"; } } return $perllocal_distro_mod_version->{$key_mod_name}; } sub module_progress_indicator { my ($module) = @_; my $crnt = (split /::/, $module)[0]; our $last ||= ''; if ($last ne $crnt) { warn "\t$crnt...\n"; $last = $crnt; } } =head1 OTHERS This module checks $::DEBUG and $::VERBOSE for obvious proposes. This module uses L to communicate with MetaCPAN. Check that module's documentation for options and caching. You can use L to take the list of releases and create a mini-cpan containing them. =head1 AUTHOR Written by Tim Bunce ETim.Bunce@pobox.comE Maintained by Fomberg Shmuel, Eshmuelfomberg@gmail.comE =head1 COPYRIGHT AND LICENSE Copyright 2011-2013 by Tim Bunce. This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut 1; DIST_SURVEYOR $fatpacked{"Dist/Surveyor/DB_File.pm"} = <<'DIST_SURVEYOR_DB_FILE'; package Dist::Surveyor::DB_File; { $Dist::Surveyor::DB_File::VERSION = '0.013'; } use strict; use warnings; use Storable qw(freeze thaw); our @ISA; if (eval { require DB_File; 1; }) { @ISA = ('DB_File'); } elsif (eval { require SDBM_File; 1; }) { @ISA = ('SDBM_File'); } else { die "Need either DB_file or SDBM_File installed to run"; } # DB_File can store only strings as values, and not Perl structures # this small wrapper fixes the problem sub STORE { my ($self, $key, $val) = @_; $self->SUPER::STORE($key, freeze($val)); } sub FETCH { my ($self, $key) = @_; my $val = $self->SUPER::FETCH($key); return thaw($val); } return 1; DIST_SURVEYOR_DB_FILE $fatpacked{"Dist/Surveyor/Inquiry.pm"} = <<'DIST_SURVEYOR_INQUIRY'; package Dist::Surveyor::Inquiry; { $Dist::Surveyor::Inquiry::VERSION = '0.013'; } use strict; use warnings; use Memoize; # core use FindBin; use Fcntl qw(:DEFAULT :flock); # core use Dist::Surveyor::DB_File; # internal use LWP::UserAgent; use JSON; use Scalar::Util qw(looks_like_number); # core use Data::Dumper; use version; =head1 NAME Dist::Surveyor::Inquiry - Handling the meta-cpan API access for Dist::Surveyor =head1 VERSION version 0.013 =head1 DESCRIPTION There are a few things that needed to be known in this module: =over =item * $metacpan_size - internally defined global to limit the maximum size of every API call =item * $metacpan_calls - internally defined global counting how many API call happen. =item * This module checks $::DEBUG and $::VERBOSE for obvious proposes. =item * For initating cache-on-disk, call Dist::Surveyor::Inquiry->perma_cache() (this should be usually done, except in testing environment) =back =cut # We have to limit the number of results when using MetaCPAN::API. # We can'r make it too large as it hurts the server (it preallocates) # but need to make it large enough for worst case distros (eg eBay-API). # TODO: switching to the ElasticSearch module, with cursor support, will # probably avoid the need for this. Else we could dynamically adjust. our $metacpan_size = 2500; our $metacpan_calls = 0; our ($DEBUG, $VERBOSE); *DEBUG = \$::DEBUG; *VERBOSE = \$::VERBOSE; my $ua = LWP::UserAgent->new( agent => $0, timeout => 10, keep_alive => 3, ); require Exporter; our @ISA = qw{Exporter}; our @EXPORT = qw{ get_candidate_cpan_dist_releases get_candidate_cpan_dist_releases_fallback get_module_versions_in_release get_release_info }; # caching via persistent memoize my %memoize_cache; my $locking_file; =head1 CLASS METHODS =head2 Dist::Surveyor::Inquiry->perma_cache() Enable caching to disk of all the MetaCPAN API requests. This cache can grew to be quite big - 40MB is one case, but it worth it, as if you will need to run this program again, it will run much faster. =cut sub perma_cache { my $class = shift; my $db_generation = 3; # XXX increment on incompatible change my $pname = $FindBin::Script; $pname =~ s/\..*$//; my $memoize_file = "$pname-$db_generation.db"; open $locking_file, ">", "$memoize_file.lock" or die "Unable to open lock file: $!"; flock ($locking_file, LOCK_EX) || die "flock: $!"; tie %memoize_cache => 'Dist::Surveyor::DB_File', $memoize_file, O_CREAT|O_RDWR, 0640 or die "Unable to use persistent cache: $!"; } my @memoize_subs = qw( get_candidate_cpan_dist_releases get_candidate_cpan_dist_releases_fallback get_module_versions_in_release get_release_info ); for my $subname (@memoize_subs) { my %memoize_args = ( SCALAR_CACHE => [ HASH => \%memoize_cache ], LIST_CACHE => 'FAULT', NORMALIZER => sub { return join("\034", $subname, @_) } ); memoize($subname, %memoize_args); } =head1 FUNCTIONS =head2 get_release_info($author, $release) Receive release info, such as: get_release_info('SEMUELF', 'Dist-Surveyor-0.009') Returns a hashref containing all that release meta information, returned by http://api.metacpan.org/v0/release/$author/$release (but not information on the files inside the module) Dies on HTTP error, and warns on empty response. =cut sub get_release_info { my ($author, $release) = @_; $metacpan_calls++; my $response = $ua->get("http://api.metacpan.org/v0/release/$author/$release"); die $response->status_line unless $response->is_success; my $release_data = decode_json $response->decoded_content; if (!$release_data) { warn "Can't find release details for $author/$release - SKIPPED!\n"; return; # XXX could fake some of $release_data instead } return $release_data; } =head2 get_candidate_cpan_dist_releases($module, $version, $file_size) Return a hashref containing all the releases that contain this module (with the specific version and file size combination) The keys are the release name (i.e. 'Dist-Surveyor-0.009') and the value is a hashref containing release information and file information: 'Dist-Surveyor-0.009' => { # release information 'date' => '2013-02-20T06:48:35.000Z', 'version' => '0.009', 'author' => 'SEMUELF', 'version_numified' => '0.009', 'release' => 'Dist-Surveyor-0.009', 'distribution' => 'Dist-Surveyor', 'version_obj' => , # File information 'path' => 'lib/Dist/Surveyor/DB_File.pm', 'stat.mtime' => 1361342736, 'module.version' => '0.009' 'module.version_numified' => '0.009', } =cut sub get_candidate_cpan_dist_releases { my ($module, $version, $file_size) = @_; my $funcstr = "get_candidate_cpan_dist_releases($module, $version, $file_size)"; my $version_qual = _prepare_version_query(0, $version); my @and_quals = ( {"term" => {"file.module.name" => $module }}, (@$version_qual > 1 ? { "or" => $version_qual } : $version_qual->[0]), ); push @and_quals, {"term" => {"file.stat.size" => $file_size }} if $file_size; # XXX doesn't cope with odd cases like # http://explorer.metacpan.org/?url=/module/MLEHMANN/common-sense-3.4/sense.pm.PL $metacpan_calls++; my $query = { "size" => $metacpan_size, "query" => { "filtered" => { "filter" => {"and" => \@and_quals }, "query" => {"match_all" => {}}, }}, "fields" => [qw( release _parent author version version_numified file.module.version file.module.version_numified date stat.mtime distribution file.path )] }; my $response = $ua->post( 'http://api.metacpan.org/v0/file', Content_Type => 'application/json', Content => to_json( $query, { canonical => 1 } ), ); die $response->status_line unless $response->is_success; return _process_response($funcstr, $response); } =head2 get_candidate_cpan_dist_releases_fallback($module, $version) Similar to get_candidate_cpan_dist_releases, but getting called when get_candidate_cpan_dist_releases fails for find matching file and release. Maybe the file was tempared somehow, so the file size does not match anymore. =cut sub get_candidate_cpan_dist_releases_fallback { my ($module, $version) = @_; # fallback to look for distro of the same name as the module # for odd cases like # http://explorer.metacpan.org/?url=/module/MLEHMANN/common-sense-3.4/sense.pm.PL (my $distname = $module) =~ s/::/-/g; my $version_qual = _prepare_version_query(1, $version); my @and_quals = ( {"term" => {"distribution" => $distname }}, (@$version_qual > 1 ? { "or" => $version_qual } : $version_qual->[0]), ); # XXX doesn't cope with odd cases like $metacpan_calls++; my $query = { "size" => $metacpan_size, "query" => { "filtered" => { "filter" => {"and" => \@and_quals }, "query" => {"match_all" => {}}, }}, "fields" => [qw( release _parent author version version_numified file.module.version file.module.version_numified date stat.mtime distribution file.path)] }; my $response = $ua->post( 'http://api.metacpan.org/v0/file', Content_Type => 'application/json', Content => to_json( $query, { canonical => 1 } ), ); die $response->status_line unless $response->is_success; return _process_response("get_candidate_cpan_dist_releases_fallback($module, $version)", $response); } sub _prepare_version_query { my ($is_fallback, $version) = @_; $version = 0 if not defined $version; # XXX my ($v_key, $num_key) = $is_fallback ? qw{ version version_numified } : qw{ file.module.version file.module.version_numified }; # timbunce: So, the current situation is that: version_numified is a float # holding version->parse($raw_version)->numify, and version is a string # also holding version->parse($raw_version)->numify at the moment, and # that'll change to ->stringify at some point. Is that right now? # mo: yes, I already patched the indexer, so new releases are already # indexed ok, but for older ones I need to reindex cpan my $v = (ref $version && $version->isa('version')) ? $version : version->parse($version); my %v = map { $_ => 1 } "$version", $v->stringify, $v->numify; my @version_qual; push @version_qual, { term => { $v_key => $_ } } for keys %v; push @version_qual, { term => { $num_key => $_ }} for grep { looks_like_number($_) } keys %v; return \@version_qual; } sub _process_response { my ($funcname, $response) = @_; my $results = decode_json $response->decoded_content; my $hits = $results->{hits}{hits}; die "$funcname: too many results (>$metacpan_size)" if @$hits >= $metacpan_size; warn "$funcname: ".Dumper($results) if grep { not $_->{fields}{release} } @$hits; # XXX temp, seen once but not since # filter out perl-like releases @$hits = grep { $_->{fields}{path} !~ m!^(?:t|xt|tests?|inc|samples?|ex|examples?|bak|local-lib)\b! } grep { $_->{fields}{release} !~ /^(perl|ponie|parrot|kurila|SiePerl-)/ } @$hits; for my $hit (@$hits) { $hit->{release_id} = delete $hit->{_parent}; # add version_obj for convenience (will fail and be undef for releases like "0.08124-TRIAL") $hit->{fields}{version_obj} = eval { version->parse($hit->{fields}{version}) }; } # we'll return { "Dist-Name-Version" => { details }, ... } my %dists = map { $_->{fields}{release} => $_->{fields} } @$hits; warn "$funcname: @{[ sort keys %dists ]}\n" if $VERBOSE; return \%dists; } =head2 get_module_versions_in_release($author, $release) Receive release info, such as: get_module_versions_in_release('SEMUELF', 'Dist-Surveyor-0.009') And returns a hashref, that contains one entry for each module that exists in the release. module information is the format: 'Dist::Surveyor' => { 'version' => '0.009', 'name' => 'Dist::Surveyor', 'path' => 'lib/Dist/Surveyor.pm', 'size' => 43879 }, this function can be called for all sorts of releases that are only vague possibilities and aren't actually installed, so generally it's quiet =cut sub get_module_versions_in_release { my ($author, $release) = @_; $metacpan_calls++; my $results = eval { my $query = { "size" => $metacpan_size, "query" => { "filtered" => { "filter" => {"and" => [ {"term" => {"release" => $release }}, {"term" => {"author" => $author }}, {"term" => {"mime" => "text/x-script.perl-module"}}, ]}, "query" => {"match_all" => {}}, }}, "fields" => ["path","name","_source.module", "_source.stat.size"], }; my $response = $ua->post( 'http://api.metacpan.org/v0/file', Content_Type => 'application/json', Content => to_json( $query, { canonical => 1 } ), ); die $response->status_line unless $response->is_success; decode_json $response->decoded_content; }; if (not $results) { warn "Failed get_module_versions_in_release for $author/$release: $@"; return {}; } my $hits = $results->{hits}{hits}; die "get_module_versions_in_release($author, $release): too many results" if @$hits >= $metacpan_size; my %modules_in_release; for my $hit (@$hits) { my $path = $hit->{fields}{path}; # XXX try to ignore files that won't get installed # XXX should use META noindex! if ($path =~ m!^(?:t|xt|tests?|inc|samples?|ex|examples?|bak|local-lib)\b!) { warn "$author/$release: ignored non-installed module $path\n" if $DEBUG; next; } my $size = $hit->{fields}{"_source.stat.size"}; # files can contain more than one package ('module') my $rel_mods = $hit->{fields}{"_source.module"} || []; for my $mod (@$rel_mods) { # actually packages in the file # Some files may contain multiple packages. We want to ignore # all except the one that matches the name of the file. # We use a fairly loose (but still very effective) test because we # can't rely on $path including the full package name. (my $filebasename = $hit->{fields}{name}) =~ s/\.pm$//; if ($mod->{name} !~ m/\b$filebasename$/) { warn "$author/$release: ignored $mod->{name} in $path\n" if $DEBUG; next; } # warn if package previously seen in this release # with a different version or file size if (my $prev = $modules_in_release{$mod->{name}}) { my $version_obj = eval { version->parse($mod->{version}) }; die "$author/$release: $mod $mod->{version}: $@" if $@; if ($VERBOSE) { # XXX could add a show-only-once cache here my $msg = "$mod->{name} $mod->{version} ($size) seen in $path after $prev->{path} $prev->{version} ($prev->{size})"; warn "$release: $msg\n" if ($version_obj != version->parse($prev->{version}) or $size != $prev->{size}); } } # keep result small as Storable thawing this is major runtime cost # (specifically we avoid storing a version_obj here) $modules_in_release{$mod->{name}} = { name => $mod->{name}, path => $path, version => $mod->{version}, size => $size, }; } } warn "\n$author/$release contains: @{[ map { qq($_->{name} $_->{version}) } values %modules_in_release ]}\n" if $DEBUG; return \%modules_in_release; } =head1 License, Copyright Please see L for details =cut 1; DIST_SURVEYOR_INQUIRY $fatpacked{"Dist/Surveyor/MakeCpan.pm"} = <<'DIST_SURVEYOR_MAKECPAN'; package Dist::Surveyor::MakeCpan; { $Dist::Surveyor::MakeCpan::VERSION = '0.013'; } use strict; use warnings; use Carp; # core use Data::Dumper; # core use File::Path; # core use CPAN::DistnameInfo; use File::Basename qw{dirname}; # core use LWP::Simple qw{is_error mirror}; use LWP::UserAgent; use Dist::Surveyor::Inquiry; use List::Util qw(max); # core our $verbose; *verbose = \$::VERBOSE; sub new { my ($class, $cpan_dir, $progname, $irregularities) = @_; require Compress::Zlib; mkpath("$cpan_dir/modules"); # --- write extra data files that may be useful XXX may change # XXX these don't all (yet?) merge with existing data my $survey_datadump_dir = "$cpan_dir/$progname"; mkpath($survey_datadump_dir); # Write list of releases, like default stdout open my $rel_fh, ">", "$survey_datadump_dir/releases.txt"; # dump the primary result data for additional info and debugging my $gzwrite = Compress::Zlib::gzopen("$survey_datadump_dir/_data_dump.perl.gz", 'wb') or croak "Cannot open $survey_datadump_dir/_data_dump.perl.gz for writing: " . $Compress::Zlib::gzerrno; $gzwrite->gzwrite("[\n"); my $self = { errors => 0, cpan_dir => $cpan_dir, irregularities => $irregularities, pkg_ver_rel => {}, # for 02packages progname => $progname, rel_fh => $rel_fh, gzwrite => $gzwrite, }; return bless $self, $class; } sub close { my $self = shift; # --- write 02packages file my $pkg_lines = _readpkgs($self->{cpan_dir}); my %packages; for my $line (@$pkg_lines, map { $_->{line} } values %{ $self->{pkg_ver_rel} }) { my ($pkg) = split(/\s+/, $line, 2); if ($packages{$pkg} and $packages{$pkg} ne $line) { warn "Old $packages{$pkg}\nNew $line\n" if $verbose; } $packages{$pkg} = $line; }; _writepkgs($self->{cpan_dir}, [ sort values %packages ] ); # Write list of token packages - each should match only one release. # This makes it _much_ faster to do installs via cpanm because it # can skip the modules it knows are installed (whereas using a list of # distros it has to reinstall _all_ of them every time). # XXX maybe add as a separate option: "--mainpkgs mainpkgs.lst" my %dist_packages; while ( my ($pkg, $line) = each %packages) { my $distpath = (split /\s+/, $line)[2]; $dist_packages{$distpath}{$pkg}++; } my %token_package; my %token_package_pri = ( # alter install order for some modules 'Module::Build' => 100, # should be near first Moose => 50, # install distros that use Module::Install late so their dependencies # have already been resolved (else they try to fetch them directly, # bypassing our cpanm --mirror-only goal) 'Olson::Abbreviations' => -90, # distros with special needs 'Term::ReadKey' => -100, # tests hang if run in background ); for my $distpath (sort keys %dist_packages) { my $dp = $dist_packages{$distpath}; my $di = CPAN::DistnameInfo->new($distpath); #warn Dumper([ $distpath, $di->dist, $di]); (my $token_pkg = $di->dist) =~ s/-/::/g; if (!$dp->{$token_pkg}) { if (my $keypkg = $self->{irregularities}->{$di->dist}) { $token_pkg = $keypkg; } else { # XXX not good - may pick a dummy test package $token_pkg = (grep { $_ } keys %$dp)[0] || $token_pkg; warn "Picked $token_pkg as token package for ".$di->distvname."\n"; } } $token_package{$token_pkg} = $token_package_pri{$token_pkg} || 0; } my @main_pkgs = sort { $token_package{$b} <=> $token_package{$a} or $a cmp $b } keys %token_package; open my $key_pkg_fh, ">", join('/', $self->{cpan_dir}, $self->{progname}, "token_packages.txt"); print $key_pkg_fh "$_\n" for @main_pkgs; close $key_pkg_fh; close $self->{rel_fh}; $self->{gzwrite}->gzwrite("]\n"); $self->{gzwrite}->gzclose; warn $self->{cpan_dir}." updated.\n"; return $self->{errors}; } sub add_release { my ($self, $ri) = @_; # --- get the file my $main_url = $ri->{download_url}; my $di = distname_info_from_url($main_url); my $pathfile = "authors/id/".$di->pathname; my $destfile = $self->{cpan_dir}."/$pathfile"; mkpath(dirname($destfile)); my @urls = ($main_url); for my $mirror ('http://backpan.perl.org') { push @urls, "$mirror/$pathfile"; } my $mirror_status; for my $url (@urls) { $mirror_status = eval { mirror($url, $destfile) }; last if not is_error($mirror_status||500); } if ($@ || is_error($mirror_status)) { my $err = ($@ and chomp $@) ? $@ : $mirror_status; my $msg = "Error $err mirroring $main_url"; if (-f $destfile) { warn "$msg - using existing file\n"; } else { # better to keep going and add the packages to the index # than abort at this stage due to network/mirror problems # the user can drop the files in later warn "$msg - continuing, ADD FILE MANUALLY!\n"; $self->{errors}++; } } else { warn "$mirror_status $main_url\n" if $verbose; } my $mods_in_rel = get_module_versions_in_release($ri->{author}, $ri->{name}); if (!keys %$mods_in_rel) { # XXX hack for common::sense (my $dist_as_pkg = $ri->{distribution}) =~ s/-/::/g; warn "$ri->{author}/$ri->{name} has no modules! Adding fake module $dist_as_pkg ".$di->version."\n"; $mods_in_rel->{$dist_as_pkg} = { name => $dist_as_pkg, version => $di->version, version_obj => version->parse($di->version), }; } # --- accumulate package info for 02packages file for my $pkg (sort keys %$mods_in_rel ) { # pi => { name=>, version=>, version_obj=> } my $pi = $mods_in_rel->{$pkg}; # for selecting which dist a package belongs to # XXX should factor in authorization status my $p_r_match_score = p_r_match_score($pkg, $ri); if (my $pvr = $self->{pkg_ver_rel}->{$pkg}) { # already seen same package name in different distribution if ($p_r_match_score < $pvr->{p_r_match_score}) { warn "$pkg seen in $pvr->{ri}{name} so ignoring one in $ri->{name}\n"; next; } warn "$pkg seen in $pvr->{ri}{name} - now overridden by $ri->{name}\n"; } my $line = _fmtmodule($pkg, $di->pathname, $pi->{version}); $self->{pkg_ver_rel}->{$pkg} = { line => $line, pi => $pi, ri => $ri, p_r_match_score => $p_r_match_score }; } printf { $self->{rel_fh} } "%s\n", ( exists $ri->{url} ? $ri->{url} : "?url" ); $self->{gzwrite}->gzwrite(Dumper($ri)); $self->{gzwrite}->gzwrite(","); } sub p_r_match_score { my ($pkg_name, $ri) = @_; my @p = split /\W/, $pkg_name; my @r = split /\W/, $ri->{name}; for my $i (0..max(scalar @p, scalar @r)) { return $i if not defined $p[$i] or not defined $r[$i] or $p[$i] ne $r[$i] } die; # unreached } # copied from CPAN::Mini::Inject and hacked sub _readpkgs { my ($cpandir) = @_; my $packages_file = $cpandir.'/modules/02packages.details.txt.gz'; return [] if not -f $packages_file; my $gzread = Compress::Zlib::gzopen($packages_file, 'rb') or croak "Cannot open $packages_file: " . $Compress::Zlib::gzerrno . "\n"; my $inheader = 1; my @packages; my $package; while ( $gzread->gzreadline( $package ) ) { if ( $inheader ) { $inheader = 0 unless $package =~ /\S/; next; } chomp $package; push @packages, $package; } $gzread->gzclose; return \@packages; } sub _writepkgs { my ($cpandir, $pkgs) = @_; my $packages_file = $cpandir.'/modules/02packages.details.txt.gz'; my $gzwrite = Compress::Zlib::gzopen($packages_file, 'wb') or croak "Cannot open $packages_file for writing: " . $Compress::Zlib::gzerrno; $gzwrite->gzwrite( "File: 02packages.details.txt\n" ); $gzwrite->gzwrite( "URL: http://www.perl.com/CPAN/modules/02packages.details.txt\n" ); $gzwrite->gzwrite( 'Description: Package names found in directory $CPAN/authors/id/' . "\n" ); $gzwrite->gzwrite( "Columns: package name, version, path\n" ); $gzwrite->gzwrite( "Intended-For: Automated fetch routines, namespace documentation.\n" ); $gzwrite->gzwrite( "Written-By: $0 0.001\n" ); # XXX TODO $gzwrite->gzwrite( "Line-Count: " . scalar( @$pkgs ) . "\n" ); # Last-Updated: Sat, 19 Mar 2005 19:49:10 GMT my @date = split( /\s+/, scalar( gmtime ) ); $gzwrite->gzwrite( "Last-Updated: $date[0], $date[2] $date[1] $date[4] $date[3] GMT\n\n" ); $gzwrite->gzwrite( "$_\n" ) for ( @$pkgs ); $gzwrite->gzclose; } sub distname_info_from_url { my ($url) = @_; $url =~ s{.* \b authors/id/ }{}x or warn "No authors/ in '$url'\n"; my $di = CPAN::DistnameInfo->new($url); return $di; } sub _fmtmodule { my ( $module, $file, $version ) = @_; $version = "undef" if not defined $version; my $fw = 38 - length $version; $fw = length $module if $fw < length $module; return sprintf "%-${fw}s %s %s", $module, $version, $file; } sub errors { my $self = shift; return $self->{errors}; } 1; =head1 NAME Dist::Surveyor::MakeCpan - Create a Mini-CPAN for the surveyed modules =head1 VERSION version 0.013 =head1 SYNOPSIS use Dist::Surveyor::MakeCpan; my $cpan = Dist::Surveyor::MakeCpan->new( $cpan_dir, $progname, $irregularities); foreach my $rel (@releases) { $cpan->add_release($rel); } $cpan->close(); say "There where ", $cpan->errors(), " errors"; =head1 DESCRIPTION Create a mini-CPAN for the surveyed modules, so you will be able to re-install the same setup in a new computer. =head1 CONSTRUCTOR my $cpan = Dist::Surveyor::MakeCpan->new( $cpan_dir, $progname, $irregularities, $verbose); =over =item $cpan_dir The directory where the mini-cpan will be created =item $progname The name of the running program - will be used to create a subdirectory inside $cpan_dir, that will contain debug information. =item $irregularities A hashref with a list of irregular named releases. i.e. 'libwww-perl' => 'LWP'. =back =head1 METHODS =head2 $cpan->add_release($rel) Add one release to the mini-cpan. the $rel should be a hashref, and contain the following fields: $rel = { download_url => 'http://cpan.metacpan.org/authors/id/S/SE/SEMUELF/Dist-Surveyor-0.009.tar.gz', url => 'authors/id/S/SE/SEMUELF/Dist-Surveyor-0.009.tar.gz', author => 'SEMUELF', name => 'Dist-Surveyor-0.009', distribution => 'Dist-Surveyor', } =head2 $cpan->close() Close the mini-CPAN, and close all the debug data dump files. =head1 License, Copyright Please see L for details =cut DIST_SURVEYOR_MAKECPAN $fatpacked{"JSON.pm"} = <<'JSON'; package JSON; use strict; use Carp (); use base qw(Exporter); @JSON::EXPORT = qw(from_json to_json jsonToObj objToJson encode_json decode_json); BEGIN { $JSON::VERSION = '2.53'; $JSON::DEBUG = 0 unless (defined $JSON::DEBUG); $JSON::DEBUG = $ENV{ PERL_JSON_DEBUG } if exists $ENV{ PERL_JSON_DEBUG }; } my $Module_XS = 'JSON::XS'; my $Module_PP = 'JSON::PP'; my $Module_bp = 'JSON::backportPP'; # included in JSON distribution my $PP_Version = '2.27200'; my $XS_Version = '2.27'; # XS and PP common methods my @PublicMethods = qw/ ascii latin1 utf8 pretty indent space_before space_after relaxed canonical allow_nonref allow_blessed convert_blessed filter_json_object filter_json_single_key_object shrink max_depth max_size encode decode decode_prefix allow_unknown /; my @Properties = qw/ ascii latin1 utf8 indent space_before space_after relaxed canonical allow_nonref allow_blessed convert_blessed shrink max_depth max_size allow_unknown /; my @XSOnlyMethods = qw//; # Currently nothing my @PPOnlyMethods = qw/ indent_length sort_by allow_singlequote allow_bignum loose allow_barekey escape_slash as_nonblessed /; # JSON::PP specific # used in _load_xs and _load_pp ($INSTALL_ONLY is not used currently) my $_INSTALL_DONT_DIE = 1; # When _load_xs fails to load XS, don't die. my $_INSTALL_ONLY = 2; # Don't call _set_methods() my $_ALLOW_UNSUPPORTED = 0; my $_UNIV_CONV_BLESSED = 0; my $_USSING_bpPP = 0; # Check the environment variable to decide worker module. unless ($JSON::Backend) { $JSON::DEBUG and Carp::carp("Check used worker module..."); my $backend = exists $ENV{PERL_JSON_BACKEND} ? $ENV{PERL_JSON_BACKEND} : 1; if ($backend eq '1' or $backend =~ /JSON::XS\s*,\s*JSON::PP/) { _load_xs($_INSTALL_DONT_DIE) or _load_pp(); } elsif ($backend eq '0' or $backend eq 'JSON::PP') { _load_pp(); } elsif ($backend eq '2' or $backend eq 'JSON::XS') { _load_xs(); } elsif ($backend eq 'JSON::backportPP') { $_USSING_bpPP = 1; _load_pp(); } else { Carp::croak "The value of environmental variable 'PERL_JSON_BACKEND' is invalid."; } } sub import { my $pkg = shift; my @what_to_export; my $no_export; for my $tag (@_) { if ($tag eq '-support_by_pp') { if (!$_ALLOW_UNSUPPORTED++) { JSON::Backend::XS ->support_by_pp(@PPOnlyMethods) if ($JSON::Backend eq $Module_XS); } next; } elsif ($tag eq '-no_export') { $no_export++, next; } elsif ( $tag eq '-convert_blessed_universally' ) { eval q| require B; *UNIVERSAL::TO_JSON = sub { my $b_obj = B::svref_2object( $_[0] ); return $b_obj->isa('B::HV') ? { %{ $_[0] } } : $b_obj->isa('B::AV') ? [ @{ $_[0] } ] : undef ; } | if ( !$_UNIV_CONV_BLESSED++ ); next; } push @what_to_export, $tag; } return if ($no_export); __PACKAGE__->export_to_level(1, $pkg, @what_to_export); } # OBSOLETED sub jsonToObj { my $alternative = 'from_json'; if (defined $_[0] and UNIVERSAL::isa($_[0], 'JSON')) { shift @_; $alternative = 'decode'; } Carp::carp "'jsonToObj' will be obsoleted. Please use '$alternative' instead."; return JSON::from_json(@_); }; sub objToJson { my $alternative = 'to_json'; if (defined $_[0] and UNIVERSAL::isa($_[0], 'JSON')) { shift @_; $alternative = 'encode'; } Carp::carp "'objToJson' will be obsoleted. Please use '$alternative' instead."; JSON::to_json(@_); }; # INTERFACES sub to_json ($@) { if ( ref($_[0]) eq 'JSON' or (@_ > 2 and $_[0] eq 'JSON') ) { Carp::croak "to_json should not be called as a method."; } my $json = new JSON; if (@_ == 2 and ref $_[1] eq 'HASH') { my $opt = $_[1]; for my $method (keys %$opt) { $json->$method( $opt->{$method} ); } } $json->encode($_[0]); } sub from_json ($@) { if ( ref($_[0]) eq 'JSON' or $_[0] eq 'JSON' ) { Carp::croak "from_json should not be called as a method."; } my $json = new JSON; if (@_ == 2 and ref $_[1] eq 'HASH') { my $opt = $_[1]; for my $method (keys %$opt) { $json->$method( $opt->{$method} ); } } return $json->decode( $_[0] ); } sub true { $JSON::true } sub false { $JSON::false } sub null { undef; } sub require_xs_version { $XS_Version; } sub backend { my $proto = shift; $JSON::Backend; } #*module = *backend; sub is_xs { return $_[0]->module eq $Module_XS; } sub is_pp { return not $_[0]->xs; } sub pureperl_only_methods { @PPOnlyMethods; } sub property { my ($self, $name, $value) = @_; if (@_ == 1) { my %props; for $name (@Properties) { my $method = 'get_' . $name; if ($name eq 'max_size') { my $value = $self->$method(); $props{$name} = $value == 1 ? 0 : $value; next; } $props{$name} = $self->$method(); } return \%props; } elsif (@_ > 3) { Carp::croak('property() can take only the option within 2 arguments.'); } elsif (@_ == 2) { if ( my $method = $self->can('get_' . $name) ) { if ($name eq 'max_size') { my $value = $self->$method(); return $value == 1 ? 0 : $value; } $self->$method(); } } else { $self->$name($value); } } # INTERNAL sub _load_xs { my $opt = shift; $JSON::DEBUG and Carp::carp "Load $Module_XS."; # if called after install module, overload is disable.... why? JSON::Boolean::_overrride_overload($Module_XS); JSON::Boolean::_overrride_overload($Module_PP); eval qq| use $Module_XS $XS_Version (); |; if ($@) { if (defined $opt and $opt & $_INSTALL_DONT_DIE) { $JSON::DEBUG and Carp::carp "Can't load $Module_XS...($@)"; return 0; } Carp::croak $@; } unless (defined $opt and $opt & $_INSTALL_ONLY) { _set_module( $JSON::Backend = $Module_XS ); my $data = join("", ); # this code is from Jcode 2.xx. close(DATA); eval $data; JSON::Backend::XS->init; } return 1; }; sub _load_pp { my $opt = shift; my $backend = $_USSING_bpPP ? $Module_bp : $Module_PP; $JSON::DEBUG and Carp::carp "Load $backend."; # if called after install module, overload is disable.... why? JSON::Boolean::_overrride_overload($Module_XS); JSON::Boolean::_overrride_overload($backend); if ( $_USSING_bpPP ) { eval qq| require $backend |; } else { eval qq| use $backend $PP_Version () |; } if ($@) { if ( $backend eq $Module_PP ) { $JSON::DEBUG and Carp::carp "Can't load $Module_PP ($@), so try to load $Module_bp"; $_USSING_bpPP++; $backend = $Module_bp; JSON::Boolean::_overrride_overload($backend); local $^W; # if PP installed but invalid version, backportPP redifines methods. eval qq| require $Module_bp |; } Carp::croak $@ if $@; } unless (defined $opt and $opt & $_INSTALL_ONLY) { _set_module( $JSON::Backend = $Module_PP ); # even if backportPP, set $Backend with 'JSON::PP' JSON::Backend::PP->init; } }; sub _set_module { return if defined $JSON::true; my $module = shift; local $^W; no strict qw(refs); $JSON::true = ${"$module\::true"}; $JSON::false = ${"$module\::false"}; push @JSON::ISA, $module; push @{"$module\::Boolean::ISA"}, qw(JSON::Boolean); *{"JSON::is_bool"} = \&{"$module\::is_bool"}; for my $method ($module eq $Module_XS ? @PPOnlyMethods : @XSOnlyMethods) { *{"JSON::$method"} = sub { Carp::carp("$method is not supported in $module."); $_[0]; }; } return 1; } # # JSON Boolean # package JSON::Boolean; my %Installed; sub _overrride_overload { return if ($Installed{ $_[0] }++); my $boolean = $_[0] . '::Boolean'; eval sprintf(q| package %s; use overload ( '""' => sub { ${$_[0]} == 1 ? 'true' : 'false' }, 'eq' => sub { my ($obj, $op) = ref ($_[0]) ? ($_[0], $_[1]) : ($_[1], $_[0]); if ($op eq 'true' or $op eq 'false') { return "$obj" eq 'true' ? 'true' eq $op : 'false' eq $op; } else { return $obj ? 1 == $op : 0 == $op; } }, ); |, $boolean); if ($@) { Carp::croak $@; } return 1; } # # Helper classes for Backend Module (PP) # package JSON::Backend::PP; sub init { local $^W; no strict qw(refs); # this routine may be called after JSON::Backend::XS init was called. *{"JSON::decode_json"} = \&{"JSON::PP::decode_json"}; *{"JSON::encode_json"} = \&{"JSON::PP::encode_json"}; *{"JSON::PP::is_xs"} = sub { 0 }; *{"JSON::PP::is_pp"} = sub { 1 }; return 1; } # # To save memory, the below lines are read only when XS backend is used. # package JSON; 1; __DATA__ # # Helper classes for Backend Module (XS) # package JSON::Backend::XS; use constant INDENT_LENGTH_FLAG => 15 << 12; use constant UNSUPPORTED_ENCODE_FLAG => { ESCAPE_SLASH => 0x00000010, ALLOW_BIGNUM => 0x00000020, AS_NONBLESSED => 0x00000040, EXPANDED => 0x10000000, # for developer's }; use constant UNSUPPORTED_DECODE_FLAG => { LOOSE => 0x00000001, ALLOW_BIGNUM => 0x00000002, ALLOW_BAREKEY => 0x00000004, ALLOW_SINGLEQUOTE => 0x00000008, EXPANDED => 0x20000000, # for developer's }; sub init { local $^W; no strict qw(refs); *{"JSON::decode_json"} = \&{"JSON::XS::decode_json"}; *{"JSON::encode_json"} = \&{"JSON::XS::encode_json"}; *{"JSON::XS::is_xs"} = sub { 1 }; *{"JSON::XS::is_pp"} = sub { 0 }; return 1; } sub support_by_pp { my ($class, @methods) = @_; local $^W; no strict qw(refs); my $JSON_XS_encode_orignal = \&JSON::XS::encode; my $JSON_XS_decode_orignal = \&JSON::XS::decode; my $JSON_XS_incr_parse_orignal = \&JSON::XS::incr_parse; *JSON::XS::decode = \&JSON::Backend::XS::Supportable::_decode; *JSON::XS::encode = \&JSON::Backend::XS::Supportable::_encode; *JSON::XS::incr_parse = \&JSON::Backend::XS::Supportable::_incr_parse; *{JSON::XS::_original_decode} = $JSON_XS_decode_orignal; *{JSON::XS::_original_encode} = $JSON_XS_encode_orignal; *{JSON::XS::_original_incr_parse} = $JSON_XS_incr_parse_orignal; push @JSON::Backend::XS::Supportable::ISA, 'JSON'; my $pkg = 'JSON::Backend::XS::Supportable'; *{JSON::new} = sub { my $proto = new JSON::XS; $$proto = 0; bless $proto, $pkg; }; for my $method (@methods) { my $flag = uc($method); my $type |= (UNSUPPORTED_ENCODE_FLAG->{$flag} || 0); $type |= (UNSUPPORTED_DECODE_FLAG->{$flag} || 0); next unless($type); $pkg->_make_unsupported_method($method => $type); } push @{"JSON::XS::Boolean::ISA"}, qw(JSON::PP::Boolean); push @{"JSON::PP::Boolean::ISA"}, qw(JSON::Boolean); $JSON::DEBUG and Carp::carp("set -support_by_pp mode."); return 1; } # # Helper classes for XS # package JSON::Backend::XS::Supportable; $Carp::Internal{'JSON::Backend::XS::Supportable'} = 1; sub _make_unsupported_method { my ($pkg, $method, $type) = @_; local $^W; no strict qw(refs); *{"$pkg\::$method"} = sub { local $^W; if (defined $_[1] ? $_[1] : 1) { ${$_[0]} |= $type; } else { ${$_[0]} &= ~$type; } $_[0]; }; *{"$pkg\::get_$method"} = sub { ${$_[0]} & $type ? 1 : ''; }; } sub _set_for_pp { JSON::_load_pp( $_INSTALL_ONLY ); my $type = shift; my $pp = new JSON::PP; my $prop = $_[0]->property; for my $name (keys %$prop) { $pp->$name( $prop->{$name} ? $prop->{$name} : 0 ); } my $unsupported = $type eq 'encode' ? JSON::Backend::XS::UNSUPPORTED_ENCODE_FLAG : JSON::Backend::XS::UNSUPPORTED_DECODE_FLAG; my $flags = ${$_[0]} || 0; for my $name (keys %$unsupported) { next if ($name eq 'EXPANDED'); # for developer's my $enable = ($flags & $unsupported->{$name}) ? 1 : 0; my $method = lc $name; $pp->$method($enable); } $pp->indent_length( $_[0]->get_indent_length ); return $pp; } sub _encode { # using with PP encod if (${$_[0]}) { _set_for_pp('encode' => @_)->encode($_[1]); } else { $_[0]->_original_encode( $_[1] ); } } sub _decode { # if unsupported-flag is set, use PP if (${$_[0]}) { _set_for_pp('decode' => @_)->decode($_[1]); } else { $_[0]->_original_decode( $_[1] ); } } sub decode_prefix { # if unsupported-flag is set, use PP _set_for_pp('decode' => @_)->decode_prefix($_[1]); } sub _incr_parse { if (${$_[0]}) { _set_for_pp('decode' => @_)->incr_parse($_[1]); } else { $_[0]->_original_incr_parse( $_[1] ); } } sub get_indent_length { ${$_[0]} << 4 >> 16; } sub indent_length { my $length = $_[1]; if (!defined $length or $length > 15 or $length < 0) { Carp::carp "The acceptable range of indent_length() is 0 to 15."; } else { local $^W; $length <<= 12; ${$_[0]} &= ~ JSON::Backend::XS::INDENT_LENGTH_FLAG; ${$_[0]} |= $length; *JSON::XS::encode = \&JSON::Backend::XS::Supportable::_encode; } $_[0]; } 1; __END__ =head1 NAME JSON - JSON (JavaScript Object Notation) encoder/decoder =head1 SYNOPSIS use JSON; # imports encode_json, decode_json, to_json and from_json. # simple and fast interfaces (expect/generate UTF-8) $utf8_encoded_json_text = encode_json $perl_hash_or_arrayref; $perl_hash_or_arrayref = decode_json $utf8_encoded_json_text; # OO-interface $json = JSON->new->allow_nonref; $json_text = $json->encode( $perl_scalar ); $perl_scalar = $json->decode( $json_text ); $pretty_printed = $json->pretty->encode( $perl_scalar ); # pretty-printing # If you want to use PP only support features, call with '-support_by_pp' # When XS unsupported feature is enable, using PP (de|en)code instead of XS ones. use JSON -support_by_pp; # option-acceptable interfaces (expect/generate UNICODE by default) $json_text = to_json( $perl_scalar, { ascii => 1, pretty => 1 } ); $perl_scalar = from_json( $json_text, { utf8 => 1 } ); # Between (en|de)code_json and (to|from)_json, if you want to write # a code which communicates to an outer world (encoded in UTF-8), # recommend to use (en|de)code_json. =head1 VERSION 2.53 This version is compatible with JSON::XS B<2.27> and later. =head1 NOTE JSON::PP was inculded in C distribution. It comes to be a perl core module in Perl 5.14. And L will be split away it. C distribution will inculde yet another JSON::PP modules. They are JSNO::backportPP and so on. JSON.pm should work as it did at all. =head1 DESCRIPTION ************************** CAUTION ******************************** * This is 'JSON module version 2' and there are many differences * * to version 1.xx * * Please check your applications useing old version. * * See to 'INCOMPATIBLE CHANGES TO OLD VERSION' * ******************************************************************* JSON (JavaScript Object Notation) is a simple data format. See to L and C(L). This module converts Perl data structures to JSON and vice versa using either L or L. JSON::XS is the fastest and most proper JSON module on CPAN which must be compiled and installed in your environment. JSON::PP is a pure-Perl module which is bundled in this distribution and has a strong compatibility to JSON::XS. This module try to use JSON::XS by default and fail to it, use JSON::PP instead. So its features completely depend on JSON::XS or JSON::PP. See to L. To distinguish the module name 'JSON' and the format type JSON, the former is quoted by CEE (its results vary with your using media), and the latter is left just as it is. Module name : C Format type : JSON =head2 FEATURES =over =item * correct unicode handling This module (i.e. backend modules) knows how to handle Unicode, documents how and when it does so, and even documents what "correct" means. Even though there are limitations, this feature is available since Perl version 5.6. JSON::XS requires Perl 5.8.2 (but works correctly in 5.8.8 or later), so in older versions C sholud call JSON::PP as the backend which can be used since Perl 5.005. With Perl 5.8.x JSON::PP works, but from 5.8.0 to 5.8.2, because of a Perl side problem, JSON::PP works slower in the versions. And in 5.005, the Unicode handling is not available. See to L for more information. See also to L and L. =item * round-trip integrity When you serialise a perl data structure using only data types supported by JSON and Perl, the deserialised data structure is identical on the Perl level. (e.g. the string "2.0" doesn't suddenly become "2" just because it looks like a number). There I minor exceptions to this, read the L section below to learn about those. =item * strict checking of JSON correctness There is no guessing, no generating of illegal JSON texts by default, and only JSON is accepted as input by default (the latter is a security feature). See to L and L. =item * fast This module returns a JSON::XS object itself if available. Compared to other JSON modules and other serialisers such as Storable, JSON::XS usually compares favourably in terms of speed, too. If not available, C returns a JSON::PP object instead of JSON::XS and it is very slow as pure-Perl. =item * simple to use This module has both a simple functional interface as well as an object oriented interface interface. =item * reasonably versatile output formats You can choose between the most compact guaranteed-single-line format possible (nice for simple line-based protocols), a pure-ASCII format (for when your transport is not 8-bit clean, still supports the whole Unicode range), or a pretty-printed format (for when you want to read that stuff). Or you can combine those features in whatever way you like. =back =head1 FUNCTIONAL INTERFACE Some documents are copied and modified from L. C and C are additional functions. =head2 encode_json $json_text = encode_json $perl_scalar Converts the given Perl data structure to a UTF-8 encoded, binary string. This function call is functionally identical to: $json_text = JSON->new->utf8->encode($perl_scalar) =head2 decode_json $perl_scalar = decode_json $json_text The opposite of C: expects an UTF-8 (binary) string and tries to parse that as an UTF-8 encoded JSON text, returning the resulting reference. This function call is functionally identical to: $perl_scalar = JSON->new->utf8->decode($json_text) =head2 to_json $json_text = to_json($perl_scalar) Converts the given Perl data structure to a json string. This function call is functionally identical to: $json_text = JSON->new->encode($perl_scalar) Takes a hash reference as the second. $json_text = to_json($perl_scalar, $flag_hashref) So, $json_text = to_json($perl_scalar, {utf8 => 1, pretty => 1}) equivalent to: $json_text = JSON->new->utf8(1)->pretty(1)->encode($perl_scalar) If you want to write a modern perl code which communicates to outer world, you should use C (supposed that JSON data are encoded in UTF-8). =head2 from_json $perl_scalar = from_json($json_text) The opposite of C: expects a json string and tries to parse it, returning the resulting reference. This function call is functionally identical to: $perl_scalar = JSON->decode($json_text) Takes a hash reference as the second. $perl_scalar = from_json($json_text, $flag_hashref) So, $perl_scalar = from_json($json_text, {utf8 => 1}) equivalent to: $perl_scalar = JSON->new->utf8(1)->decode($json_text) If you want to write a modern perl code which communicates to outer world, you should use C (supposed that JSON data are encoded in UTF-8). =head2 JSON::is_bool $is_boolean = JSON::is_bool($scalar) Returns true if the passed scalar represents either JSON::true or JSON::false, two constants that act like C<1> and C<0> respectively and are also used to represent JSON C and C in Perl strings. =head2 JSON::true Returns JSON true value which is blessed object. It C JSON::Boolean object. =head2 JSON::false Returns JSON false value which is blessed object. It C JSON::Boolean object. =head2 JSON::null Returns C. See L, below, for more information on how JSON values are mapped to Perl. =head1 HOW DO I DECODE A DATA FROM OUTER AND ENCODE TO OUTER This section supposes that your perl vresion is 5.8 or later. If you know a JSON text from an outer world - a network, a file content, and so on, is encoded in UTF-8, you should use C or C module object with C enable. And the decoded result will contain UNICODE characters. # from network my $json = JSON->new->utf8; my $json_text = CGI->new->param( 'json_data' ); my $perl_scalar = $json->decode( $json_text ); # from file content local $/; open( my $fh, '<', 'json.data' ); $json_text = <$fh>; $perl_scalar = decode_json( $json_text ); If an outer data is not encoded in UTF-8, firstly you should C it. use Encode; local $/; open( my $fh, '<', 'json.data' ); my $encoding = 'cp932'; my $unicode_json_text = decode( $encoding, <$fh> ); # UNICODE # or you can write the below code. # # open( my $fh, "<:encoding($encoding)", 'json.data' ); # $unicode_json_text = <$fh>; In this case, C<$unicode_json_text> is of course UNICODE string. So you B use C nor C module object with C enable. Instead of them, you use C module object with C disable or C. $perl_scalar = $json->utf8(0)->decode( $unicode_json_text ); # or $perl_scalar = from_json( $unicode_json_text ); Or C and C: $perl_scalar = decode_json( encode( 'utf8', $unicode_json_text ) ); # this way is not efficient. And now, you want to convert your C<$perl_scalar> into JSON data and send it to an outer world - a network or a file content, and so on. Your data usually contains UNICODE strings and you want the converted data to be encoded in UTF-8, you should use C or C module object with C enable. print encode_json( $perl_scalar ); # to a network? file? or display? # or print $json->utf8->encode( $perl_scalar ); If C<$perl_scalar> does not contain UNICODE but C<$encoding>-encoded strings for some reason, then its characters are regarded as B for perl (because it does not concern with your $encoding). You B use C nor C module object with C enable. Instead of them, you use C module object with C disable or C. Note that the resulted text is a UNICODE string but no problem to print it. # $perl_scalar contains $encoding encoded string values $unicode_json_text = $json->utf8(0)->encode( $perl_scalar ); # or $unicode_json_text = to_json( $perl_scalar ); # $unicode_json_text consists of characters less than 0x100 print $unicode_json_text; Or C all string values and C: $perl_scalar->{ foo } = decode( $encoding, $perl_scalar->{ foo } ); # ... do it to each string values, then encode_json $json_text = encode_json( $perl_scalar ); This method is a proper way but probably not efficient. See to L, L. =head1 COMMON OBJECT-ORIENTED INTERFACE =head2 new $json = new JSON Returns a new C object inherited from either JSON::XS or JSON::PP that can be used to de/encode JSON strings. All boolean flags described below are by default I. The mutators for flags all return the JSON object again and thus calls can be chained: my $json = JSON->new->utf8->space_after->encode({a => [1,2]}) => {"a": [1, 2]} =head2 ascii $json = $json->ascii([$enable]) $enabled = $json->get_ascii If $enable is true (or missing), then the encode method will not generate characters outside the code range 0..127. Any Unicode characters outside that range will be escaped using either a single \uXXXX or a double \uHHHH\uLLLLL escape sequence, as per RFC4627. If $enable is false, then the encode method will not escape Unicode characters unless required by the JSON syntax or other flags. This results in a faster and more compact format. This feature depends on the used Perl version and environment. See to L if the backend is PP. JSON->new->ascii(1)->encode([chr 0x10401]) => ["\ud801\udc01"] =head2 latin1 $json = $json->latin1([$enable]) $enabled = $json->get_latin1 If $enable is true (or missing), then the encode method will encode the resulting JSON text as latin1 (or iso-8859-1), escaping any characters outside the code range 0..255. If $enable is false, then the encode method will not escape Unicode characters unless required by the JSON syntax or other flags. JSON->new->latin1->encode (["\x{89}\x{abc}"] => ["\x{89}\\u0abc"] # (perl syntax, U+abc escaped, U+89 not) =head2 utf8 $json = $json->utf8([$enable]) $enabled = $json->get_utf8 If $enable is true (or missing), then the encode method will encode the JSON result into UTF-8, as required by many protocols, while the decode method expects to be handled an UTF-8-encoded string. Please note that UTF-8-encoded strings do not contain any characters outside the range 0..255, they are thus useful for bytewise/binary I/O. In future versions, enabling this option might enable autodetection of the UTF-16 and UTF-32 encoding families, as described in RFC4627. If $enable is false, then the encode method will return the JSON string as a (non-encoded) Unicode string, while decode expects thus a Unicode string. Any decoding or encoding (e.g. to UTF-8 or UTF-16) needs to be done yourself, e.g. using the Encode module. Example, output UTF-16BE-encoded JSON: use Encode; $jsontext = encode "UTF-16BE", JSON::XS->new->encode ($object); Example, decode UTF-32LE-encoded JSON: use Encode; $object = JSON::XS->new->decode (decode "UTF-32LE", $jsontext); See to L if the backend is PP. =head2 pretty $json = $json->pretty([$enable]) This enables (or disables) all of the C, C and C (and in the future possibly more) flags in one call to generate the most readable (or most compact) form possible. Equivalent to: $json->indent->space_before->space_after The indent space length is three and JSON::XS cannot change the indent space length. =head2 indent $json = $json->indent([$enable]) $enabled = $json->get_indent If C<$enable> is true (or missing), then the C method will use a multiline format as output, putting every array member or object/hash key-value pair into its own line, identing them properly. If C<$enable> is false, no newlines or indenting will be produced, and the resulting JSON text is guarenteed not to contain any C. This setting has no effect when decoding JSON texts. The indent space length is three. With JSON::PP, you can also access C to change indent space length. =head2 space_before $json = $json->space_before([$enable]) $enabled = $json->get_space_before If C<$enable> is true (or missing), then the C method will add an extra optional space before the C<:> separating keys from values in JSON objects. If C<$enable> is false, then the C method will not add any extra space at those places. This setting has no effect when decoding JSON texts. Example, space_before enabled, space_after and indent disabled: {"key" :"value"} =head2 space_after $json = $json->space_after([$enable]) $enabled = $json->get_space_after If C<$enable> is true (or missing), then the C method will add an extra optional space after the C<:> separating keys from values in JSON objects and extra whitespace after the C<,> separating key-value pairs and array members. If C<$enable> is false, then the C method will not add any extra space at those places. This setting has no effect when decoding JSON texts. Example, space_before and indent disabled, space_after enabled: {"key": "value"} =head2 relaxed $json = $json->relaxed([$enable]) $enabled = $json->get_relaxed If C<$enable> is true (or missing), then C will accept some extensions to normal JSON syntax (see below). C will not be affected in anyway. I. I suggest only to use this option to parse application-specific files written by humans (configuration files, resource files etc.) If C<$enable> is false (the default), then C will only accept valid JSON texts. Currently accepted extensions are: =over 4 =item * list items can have an end-comma JSON I array elements and key-value pairs with commas. This can be annoying if you write JSON texts manually and want to be able to quickly append elements, so this extension accepts comma at the end of such items not just between them: [ 1, 2, <- this comma not normally allowed ] { "k1": "v1", "k2": "v2", <- this comma not normally allowed } =item * shell-style '#'-comments Whenever JSON allows whitespace, shell-style comments are additionally allowed. They are terminated by the first carriage-return or line-feed character, after which more white-space and comments are allowed. [ 1, # this comment not allowed in JSON # neither this one... ] =back =head2 canonical $json = $json->canonical([$enable]) $enabled = $json->get_canonical If C<$enable> is true (or missing), then the C method will output JSON objects by sorting their keys. This is adding a comparatively high overhead. If C<$enable> is false, then the C method will output key-value pairs in the order Perl stores them (which will likely change between runs of the same script). This option is useful if you want the same data structure to be encoded as the same JSON text (given the same overall settings). If it is disabled, the same hash might be encoded differently even if contains the same data, as key-value pairs have no inherent ordering in Perl. This setting has no effect when decoding JSON texts. =head2 allow_nonref $json = $json->allow_nonref([$enable]) $enabled = $json->get_allow_nonref If C<$enable> is true (or missing), then the C method can convert a non-reference into its corresponding string, number or null JSON value, which is an extension to RFC4627. Likewise, C will accept those JSON values instead of croaking. If C<$enable> is false, then the C method will croak if it isn't passed an arrayref or hashref, as JSON texts must either be an object or array. Likewise, C will croak if given something that is not a JSON object or array. JSON->new->allow_nonref->encode ("Hello, World!") => "Hello, World!" =head2 allow_unknown $json = $json->allow_unknown ([$enable]) $enabled = $json->get_allow_unknown If $enable is true (or missing), then "encode" will *not* throw an exception when it encounters values it cannot represent in JSON (for example, filehandles) but instead will encode a JSON "null" value. Note that blessed objects are not included here and are handled separately by c. If $enable is false (the default), then "encode" will throw an exception when it encounters anything it cannot encode as JSON. This option does not affect "decode" in any way, and it is recommended to leave it off unless you know your communications partner. =head2 allow_blessed $json = $json->allow_blessed([$enable]) $enabled = $json->get_allow_blessed If C<$enable> is true (or missing), then the C method will not barf when it encounters a blessed reference. Instead, the value of the B option will decide whether C (C disabled or no C method found) or a representation of the object (C enabled and C method found) is being encoded. Has no effect on C. If C<$enable> is false (the default), then C will throw an exception when it encounters a blessed object. =head2 convert_blessed $json = $json->convert_blessed([$enable]) $enabled = $json->get_convert_blessed If C<$enable> is true (or missing), then C, upon encountering a blessed object, will check for the availability of the C method on the object's class. If found, it will be called in scalar context and the resulting scalar will be encoded instead of the object. If no C method is found, the value of C will decide what to do. The C method may safely call die if it wants. If C returns other blessed objects, those will be handled in the same way. C must take care of not causing an endless recursion cycle (== crash) in this case. The name of C was chosen because other methods called by the Perl core (== not by the user of the object) are usually in upper case letters and to avoid collisions with the C function or method. This setting does not yet influence C in any way. If C<$enable> is false, then the C setting will decide what to do when a blessed object is found. =over =item convert_blessed_universally mode If use C with C<-convert_blessed_universally>, the C subroutine is defined as the below code: *UNIVERSAL::TO_JSON = sub { my $b_obj = B::svref_2object( $_[0] ); return $b_obj->isa('B::HV') ? { %{ $_[0] } } : $b_obj->isa('B::AV') ? [ @{ $_[0] } ] : undef ; } This will cause that C method converts simple blessed objects into JSON objects as non-blessed object. JSON -convert_blessed_universally; $json->allow_blessed->convert_blessed->encode( $blessed_object ) This feature is experimental and may be removed in the future. =back =head2 filter_json_object $json = $json->filter_json_object([$coderef]) When C<$coderef> is specified, it will be called from C each time it decodes a JSON object. The only argument passed to the coderef is a reference to the newly-created hash. If the code references returns a single scalar (which need not be a reference), this value (i.e. a copy of that scalar to avoid aliasing) is inserted into the deserialised data structure. If it returns an empty list (NOTE: I C, which is a valid scalar), the original deserialised hash will be inserted. This setting can slow down decoding considerably. When C<$coderef> is omitted or undefined, any existing callback will be removed and C will not change the deserialised hash in any way. Example, convert all JSON objects into the integer 5: my $js = JSON->new->filter_json_object (sub { 5 }); # returns [5] $js->decode ('[{}]'); # the given subroutine takes a hash reference. # throw an exception because allow_nonref is not enabled # so a lone 5 is not allowed. $js->decode ('{"a":1, "b":2}'); =head2 filter_json_single_key_object $json = $json->filter_json_single_key_object($key [=> $coderef]) Works remotely similar to C, but is only called for JSON objects having a single key named C<$key>. This C<$coderef> is called before the one specified via C, if any. It gets passed the single value in the JSON object. If it returns a single value, it will be inserted into the data structure. If it returns nothing (not even C but the empty list), the callback from C will be called next, as if no single-key callback were specified. If C<$coderef> is omitted or undefined, the corresponding callback will be disabled. There can only ever be one callback for a given key. As this callback gets called less often then the C one, decoding speed will not usually suffer as much. Therefore, single-key objects make excellent targets to serialise Perl objects into, especially as single-key JSON objects are as close to the type-tagged value concept as JSON gets (it's basically an ID/VALUE tuple). Of course, JSON does not support this in any way, so you need to make sure your data never looks like a serialised Perl hash. Typical names for the single object key are C<__class_whatever__>, or C<$__dollars_are_rarely_used__$> or C<}ugly_brace_placement>, or even things like C<__class_md5sum(classname)__>, to reduce the risk of clashing with real hashes. Example, decode JSON objects of the form C<< { "__widget__" => } >> into the corresponding C<< $WIDGET{} >> object: # return whatever is in $WIDGET{5}: JSON ->new ->filter_json_single_key_object (__widget__ => sub { $WIDGET{ $_[0] } }) ->decode ('{"__widget__": 5') # this can be used with a TO_JSON method in some "widget" class # for serialisation to json: sub WidgetBase::TO_JSON { my ($self) = @_; unless ($self->{id}) { $self->{id} = ..get..some..id..; $WIDGET{$self->{id}} = $self; } { __widget__ => $self->{id} } } =head2 shrink $json = $json->shrink([$enable]) $enabled = $json->get_shrink With JSON::XS, this flag resizes strings generated by either C or C to their minimum size possible. This can save memory when your JSON texts are either very very long or you have many short strings. It will also try to downgrade any strings to octet-form if possible: perl stores strings internally either in an encoding called UTF-X or in octet-form. The latter cannot store everything but uses less space in general (and some buggy Perl or C code might even rely on that internal representation being used). With JSON::PP, it is noop about resizing strings but tries C to the returned string by C. See to L. See to L and L. =head2 max_depth $json = $json->max_depth([$maximum_nesting_depth]) $max_depth = $json->get_max_depth Sets the maximum nesting level (default C<512>) accepted while encoding or decoding. If a higher nesting level is detected in JSON text or a Perl data structure, then the encoder and decoder will stop and croak at that point. Nesting level is defined by number of hash- or arrayrefs that the encoder needs to traverse to reach a given point or the number of C<{> or C<[> characters without their matching closing parenthesis crossed to reach a given character in a string. If no argument is given, the highest possible setting will be used, which is rarely useful. Note that nesting is implemented by recursion in C. The default value has been chosen to be as large as typical operating systems allow without crashing. (JSON::XS) With JSON::PP as the backend, when a large value (100 or more) was set and it de/encodes a deep nested object/text, it may raise a warning 'Deep recursion on subroutin' at the perl runtime phase. See L for more info on why this is useful. =head2 max_size $json = $json->max_size([$maximum_string_size]) $max_size = $json->get_max_size Set the maximum length a JSON text may have (in bytes) where decoding is being attempted. The default is C<0>, meaning no limit. When C is called on a string that is longer then this many bytes, it will not attempt to decode the string but throw an exception. This setting has no effect on C (yet). If no argument is given, the limit check will be deactivated (same as when C<0> is specified). See L, below, for more info on why this is useful. =head2 encode $json_text = $json->encode($perl_scalar) Converts the given Perl data structure (a simple scalar or a reference to a hash or array) to its JSON representation. Simple scalars will be converted into JSON string or number sequences, while references to arrays become JSON arrays and references to hashes become JSON objects. Undefined Perl values (e.g. C) become JSON C values. References to the integers C<0> and C<1> are converted into C and C. =head2 decode $perl_scalar = $json->decode($json_text) The opposite of C: expects a JSON text and tries to parse it, returning the resulting simple scalar or reference. Croaks on error. JSON numbers and strings become simple Perl scalars. JSON arrays become Perl arrayrefs and JSON objects become Perl hashrefs. C becomes C<1> (C), C becomes C<0> (C) and C becomes C. =head2 decode_prefix ($perl_scalar, $characters) = $json->decode_prefix($json_text) This works like the C method, but instead of raising an exception when there is trailing garbage after the first JSON object, it will silently stop parsing there and return the number of characters consumed so far. JSON->new->decode_prefix ("[1] the tail") => ([], 3) See to L =head2 property $boolean = $json->property($property_name) Returns a boolean value about above some properties. The available properties are C, C, C, C,C, C, C, C, C, C, C, C, C, C and C. $boolean = $json->property('utf8'); => 0 $json->utf8; $boolean = $json->property('utf8'); => 1 Sets the property with a given boolean value. $json = $json->property($property_name => $boolean); With no argumnt, it returns all the above properties as a hash reference. $flag_hashref = $json->property(); =head1 INCREMENTAL PARSING Most of this section are copied and modified from L. In some cases, there is the need for incremental parsing of JSON texts. This module does allow you to parse a JSON stream incrementally. It does so by accumulating text until it has a full JSON object, which it then can decode. This process is similar to using C to see if a full JSON object is available, but is much more efficient (and can be implemented with a minimum of method calls). The backend module will only attempt to parse the JSON text once it is sure it has enough text to get a decisive result, using a very simple but truly incremental parser. This means that it sometimes won't stop as early as the full parser, for example, it doesn't detect parenthese mismatches. The only thing it guarantees is that it starts decoding as soon as a syntactically valid JSON text has been seen. This means you need to set resource limits (e.g. C) to ensure the parser will stop parsing in the presence if syntax errors. The following methods implement this incremental parser. =head2 incr_parse $json->incr_parse( [$string] ) # void context $obj_or_undef = $json->incr_parse( [$string] ) # scalar context @obj_or_empty = $json->incr_parse( [$string] ) # list context This is the central parsing function. It can both append new text and extract objects from the stream accumulated so far (both of these functions are optional). If C<$string> is given, then this string is appended to the already existing JSON fragment stored in the C<$json> object. After that, if the function is called in void context, it will simply return without doing anything further. This can be used to add more text in as many chunks as you want. If the method is called in scalar context, then it will try to extract exactly I JSON object. If that is successful, it will return this object, otherwise it will return C. If there is a parse error, this method will croak just as C would do (one can then use C to skip the errornous part). This is the most common way of using the method. And finally, in list context, it will try to extract as many objects from the stream as it can find and return them, or the empty list otherwise. For this to work, there must be no separators between the JSON objects or arrays, instead they must be concatenated back-to-back. If an error occurs, an exception will be raised as in the scalar context case. Note that in this case, any previously-parsed JSON texts will be lost. Example: Parse some JSON arrays/objects in a given string and return them. my @objs = JSON->new->incr_parse ("[5][7][1,2]"); =head2 incr_text $lvalue_string = $json->incr_text This method returns the currently stored JSON fragment as an lvalue, that is, you can manipulate it. This I works when a preceding call to C in I successfully returned an object. Under all other circumstances you must not call this function (I mean it. although in simple tests it might actually work, it I fail under real world conditions). As a special exception, you can also call this method before having parsed anything. This function is useful in two cases: a) finding the trailing text after a JSON object or b) parsing multiple JSON objects separated by non-JSON text (such as commas). $json->incr_text =~ s/\s*,\s*//; In Perl 5.005, C attribute is not available. You must write codes like the below: $string = $json->incr_text; $string =~ s/\s*,\s*//; $json->incr_text( $string ); =head2 incr_skip $json->incr_skip This will reset the state of the incremental parser and will remove the parsed text from the input buffer. This is useful after C died, in which case the input buffer and incremental parser state is left unchanged, to skip the text parsed so far and to reset the parse state. =head2 incr_reset $json->incr_reset This completely resets the incremental parser, that is, after this call, it will be as if the parser had never parsed anything. This is useful if you want ot repeatedly parse JSON objects and want to ignore any trailing data, which means you have to reset the parser after each successful decode. See to L for examples. =head1 JSON::PP SUPPORT METHODS The below methods are JSON::PP own methods, so when C works with JSON::PP (i.e. the created object is a JSON::PP object), available. See to L in detail. If you use C with additonal C<-support_by_pp>, some methods are available even with JSON::XS. See to L. BEING { $ENV{PERL_JSON_BACKEND} = 'JSON::XS' } use JSON -support_by_pp; my $json = new JSON; $json->allow_nonref->escape_slash->encode("/"); # functional interfaces too. print to_json(["/"], {escape_slash => 1}); print from_json('["foo"]', {utf8 => 1}); If you do not want to all functions but C<-support_by_pp>, use C<-no_export>. use JSON -support_by_pp, -no_export; # functional interfaces are not exported. =head2 allow_singlequote $json = $json->allow_singlequote([$enable]) If C<$enable> is true (or missing), then C will accept any JSON strings quoted by single quotations that are invalid JSON format. $json->allow_singlequote->decode({"foo":'bar'}); $json->allow_singlequote->decode({'foo':"bar"}); $json->allow_singlequote->decode({'foo':'bar'}); As same as the C option, this option may be used to parse application-specific files written by humans. =head2 allow_barekey $json = $json->allow_barekey([$enable]) If C<$enable> is true (or missing), then C will accept bare keys of JSON object that are invalid JSON format. As same as the C option, this option may be used to parse application-specific files written by humans. $json->allow_barekey->decode('{foo:"bar"}'); =head2 allow_bignum $json = $json->allow_bignum([$enable]) If C<$enable> is true (or missing), then C will convert the big integer Perl cannot handle as integer into a L object and convert a floating number (any) into a L. On the contary, C converts C objects and C objects into JSON numbers with C enable. $json->allow_nonref->allow_blessed->allow_bignum; $bigfloat = $json->decode('2.000000000000000000000000001'); print $json->encode($bigfloat); # => 2.000000000000000000000000001 See to L aboout the conversion of JSON number. =head2 loose $json = $json->loose([$enable]) The unescaped [\x00-\x1f\x22\x2f\x5c] strings are invalid in JSON strings and the module doesn't allow to C to these (except for \x2f). If C<$enable> is true (or missing), then C will accept these unescaped strings. $json->loose->decode(qq|["abc def"]|); See to L. =head2 escape_slash $json = $json->escape_slash([$enable]) According to JSON Grammar, I (U+002F) is escaped. But by default JSON backend modules encode strings without escaping slash. If C<$enable> is true (or missing), then C will escape slashes. =head2 indent_length $json = $json->indent_length($length) With JSON::XS, The indent space length is 3 and cannot be changed. With JSON::PP, it sets the indent space length with the given $length. The default is 3. The acceptable range is 0 to 15. =head2 sort_by $json = $json->sort_by($function_name) $json = $json->sort_by($subroutine_ref) If $function_name or $subroutine_ref are set, its sort routine are used. $js = $pc->sort_by(sub { $JSON::PP::a cmp $JSON::PP::b })->encode($obj); # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|); $js = $pc->sort_by('own_sort')->encode($obj); # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|); sub JSON::PP::own_sort { $JSON::PP::a cmp $JSON::PP::b } As the sorting routine runs in the JSON::PP scope, the given subroutine name and the special variables C<$a>, C<$b> will begin with 'JSON::PP::'. If $integer is set, then the effect is same as C on. See to L. =head1 MAPPING This section is copied from JSON::XS and modified to C. JSON::XS and JSON::PP mapping mechanisms are almost equivalent. See to L. =head2 JSON -> PERL =over 4 =item object A JSON object becomes a reference to a hash in Perl. No ordering of object keys is preserved (JSON does not preserver object key ordering itself). =item array A JSON array becomes a reference to an array in Perl. =item string A JSON string becomes a string scalar in Perl - Unicode codepoints in JSON are represented by the same codepoints in the Perl string, so no manual decoding is necessary. =item number A JSON number becomes either an integer, numeric (floating point) or string scalar in perl, depending on its range and any fractional parts. On the Perl level, there is no difference between those as Perl handles all the conversion details, but an integer may take slightly less memory and might represent more values exactly than floating point numbers. If the number consists of digits only, C will try to represent it as an integer value. If that fails, it will try to represent it as a numeric (floating point) value if that is possible without loss of precision. Otherwise it will preserve the number as a string value (in which case you lose roundtripping ability, as the JSON number will be re-encoded toa JSON string). Numbers containing a fractional or exponential part will always be represented as numeric (floating point) values, possibly at a loss of precision (in which case you might lose perfect roundtripping ability, but the JSON number will still be re-encoded as a JSON number). Note that precision is not accuracy - binary floating point values cannot represent most decimal fractions exactly, and when converting from and to floating point, C only guarantees precision up to but not including the leats significant bit. If the backend is JSON::PP and C is enable, the big integers and the numeric can be optionally converted into L and L objects. =item true, false These JSON atoms become C and C, respectively. They are overloaded to act almost exactly like the numbers C<1> and C<0>. You can check wether a scalar is a JSON boolean by using the C function. If C and C are used as strings or compared as strings, they represent as C and C respectively. print JSON::true . "\n"; => true print JSON::true + 1; => 1 ok(JSON::true eq 'true'); ok(JSON::true eq '1'); ok(JSON::true == 1); C will install these missing overloading features to the backend modules. =item null A JSON null atom becomes C in Perl. C returns C. =back =head2 PERL -> JSON The mapping from Perl to JSON is slightly more difficult, as Perl is a truly typeless language, so we can only guess which JSON type is meant by a Perl value. =over 4 =item hash references Perl hash references become JSON objects. As there is no inherent ordering in hash keys (or JSON objects), they will usually be encoded in a pseudo-random order that can change between runs of the same program but stays generally the same within a single run of a program. C optionally sort the hash keys (determined by the I flag), so the same datastructure will serialise to the same JSON text (given same settings and version of JSON::XS), but this incurs a runtime overhead and is only rarely useful, e.g. when you want to compare some JSON text against another for equality. In future, the ordered object feature will be added to JSON::PP using C mechanism. =item array references Perl array references become JSON arrays. =item other references Other unblessed references are generally not allowed and will cause an exception to be thrown, except for references to the integers C<0> and C<1>, which get turned into C and C atoms in JSON. You can also use C and C to improve readability. to_json [\0,JSON::true] # yields [false,true] =item JSON::true, JSON::false, JSON::null These special values become JSON true and JSON false values, respectively. You can also use C<\1> and C<\0> directly if you want. JSON::null returns C. =item blessed objects Blessed objects are not directly representable in JSON. See the C and C methods on various options on how to deal with this: basically, you can choose between throwing an exception, encoding the reference as if it weren't blessed, or provide your own serialiser method. With C mode, C converts blessed hash references or blessed array references (contains other blessed references) into JSON members and arrays. use JSON -convert_blessed_universally; JSON->new->allow_blessed->convert_blessed->encode( $blessed_object ); See to L. =item simple scalars Simple Perl scalars (any scalar that is not a reference) are the most difficult objects to encode: JSON::XS and JSON::PP will encode undefined scalars as JSON C values, scalars that have last been used in a string context before encoding as JSON strings, and anything else as number value: # dump as number encode_json [2] # yields [2] encode_json [-3.0e17] # yields [-3e+17] my $value = 5; encode_json [$value] # yields [5] # used as string, so dump as string print $value; encode_json [$value] # yields ["5"] # undef becomes null encode_json [undef] # yields [null] You can force the type to be a string by stringifying it: my $x = 3.1; # some variable containing a number "$x"; # stringified $x .= ""; # another, more awkward way to stringify print $x; # perl does it for you, too, quite often You can force the type to be a number by numifying it: my $x = "3"; # some variable containing a string $x += 0; # numify it, ensuring it will be dumped as a number $x *= 1; # same thing, the choise is yours. You can not currently force the type in other, less obscure, ways. Note that numerical precision has the same meaning as under Perl (so binary to decimal conversion follows the same rules as in Perl, which can differ to other languages). Also, your perl interpreter might expose extensions to the floating point numbers of your platform, such as infinities or NaN's - these cannot be represented in JSON, and it is an error to pass those in. =item Big Number If the backend is JSON::PP and C is enable, C converts C objects and C objects into JSON numbers. =back =head1 JSON and ECMAscript See to L. =head1 JSON and YAML JSON is not a subset of YAML. See to L. =head1 BACKEND MODULE DECISION When you use C, C tries to C JSON::XS. If this call failed, it will C JSON::PP. The required JSON::XS version is I<2.2> or later. The C constructor method returns an object inherited from the backend module, and JSON::XS object is a blessed scaler reference while JSON::PP is a blessed hash reference. So, your program should not depend on the backend module, especially returned objects should not be modified. my $json = JSON->new; # XS or PP? $json->{stash} = 'this is xs object'; # this code may raise an error! To check the backend module, there are some methods - C, C and C. JSON->backend; # 'JSON::XS' or 'JSON::PP' JSON->backend->is_pp: # 0 or 1 JSON->backend->is_xs: # 1 or 0 $json->is_xs; # 1 or 0 $json->is_pp; # 0 or 1 If you set an enviornment variable C, The calling action will be changed. =over =item PERL_JSON_BACKEND = 0 or PERL_JSON_BACKEND = 'JSON::PP' Always use JSON::PP =item PERL_JSON_BACKEND == 1 or PERL_JSON_BACKEND = 'JSON::XS,JSON::PP' (The default) Use compiled JSON::XS if it is properly compiled & installed, otherwise use JSON::PP. =item PERL_JSON_BACKEND == 2 or PERL_JSON_BACKEND = 'JSON::XS' Always use compiled JSON::XS, die if it isn't properly compiled & installed. =item PERL_JSON_BACKEND = 'JSON::backportPP' Always use JSON::backportPP. JSON::backportPP is JSON::PP back port module. C includs JSON::backportPP instead of JSON::PP. =back These ideas come from L mechanism. example: BEGIN { $ENV{PERL_JSON_BACKEND} = 'JSON::PP' } use JSON; # always uses JSON::PP In future, it may be able to specify another module. =head1 USE PP FEATURES EVEN THOUGH XS BACKEND Many methods are available with either JSON::XS or JSON::PP and when the backend module is JSON::XS, if any JSON::PP specific (i.e. JSON::XS unspported) method is called, it will C and be noop. But If you C C passing the optional string C<-support_by_pp>, it makes a part of those unupported methods available. This feature is achieved by using JSON::PP in C. BEGIN { $ENV{PERL_JSON_BACKEND} = 2 } # with JSON::XS use JSON -support_by_pp; my $json = new JSON; $json->allow_nonref->escape_slash->encode("/"); At this time, the returned object is a C object (re-blessed XS object), and by checking JSON::XS unsupported flags in de/encoding, can support some unsupported methods - C, C, C, C, C and C. When any unsupported methods are not enable, C will be used as is. The switch is achieved by changing the symbolic tables. C<-support_by_pp> is effective only when the backend module is JSON::XS and it makes the de/encoding speed down a bit. See to L. =head1 INCOMPATIBLE CHANGES TO OLD VERSION There are big incompatibility between new version (2.00) and old (1.xx). If you use old C 1.xx in your code, please check it. See to L =over =item jsonToObj and objToJson are obsoleted. Non Perl-style name C and C are obsoleted (but not yet deleted from the source). If you use these functions in your code, please replace them with C and C. =item Global variables are no longer available. C class variables - C<$JSON::AUTOCONVERT>, C<$JSON::BareKey>, etc... - are not available any longer. Instead, various features can be used through object methods. =item Package JSON::Converter and JSON::Parser are deleted. Now C bundles with JSON::PP which can handle JSON more properly than them. =item Package JSON::NotString is deleted. There was C class which represents JSON value C, C, C and numbers. It was deleted and replaced by C. C represents C and C. C does not represent C. C returns C. C makes L and L is-a relation to L. =item function JSON::Number is obsoleted. C is now needless because JSON::XS and JSON::PP have round-trip integrity. =item JSONRPC modules are deleted. Perl implementation of JSON-RPC protocol - C, C and C are deleted in this distribution. Instead of them, there is L which supports JSON-RPC protocol version 1.1. =back =head2 Transition ways from 1.xx to 2.xx. You should set C mode firstly, because it is always successful for the below codes even with JSON::XS. use JSON -support_by_pp; =over =item Exported jsonToObj (simple) from_json($json_text); =item Exported objToJson (simple) to_json($perl_scalar); =item Exported jsonToObj (advanced) $flags = {allow_barekey => 1, allow_singlequote => 1}; from_json($json_text, $flags); equivalent to: $JSON::BareKey = 1; $JSON::QuotApos = 1; jsonToObj($json_text); =item Exported objToJson (advanced) $flags = {allow_blessed => 1, allow_barekey => 1}; to_json($perl_scalar, $flags); equivalent to: $JSON::BareKey = 1; objToJson($perl_scalar); =item jsonToObj as object method $json->decode($json_text); =item objToJson as object method $json->encode($perl_scalar); =item new method with parameters The C method in 2.x takes any parameters no longer. You can set parameters instead; $json = JSON->new->pretty; =item $JSON::Pretty, $JSON::Indent, $JSON::Delimiter If C is enable, that means C<$JSON::Pretty> flag set. And C<$JSON::Delimiter> was substituted by C and C. In conclusion: $json->indent->space_before->space_after; Equivalent to: $json->pretty; To change indent length, use C. (Only with JSON::PP, if C<-support_by_pp> is not used.) $json->pretty->indent_length(2)->encode($perl_scalar); =item $JSON::BareKey (Only with JSON::PP, if C<-support_by_pp> is not used.) $json->allow_barekey->decode($json_text) =item $JSON::ConvBlessed use C<-convert_blessed_universally>. See to L. =item $JSON::QuotApos (Only with JSON::PP, if C<-support_by_pp> is not used.) $json->allow_singlequote->decode($json_text) =item $JSON::SingleQuote Disable. C does not make such a invalid JSON string any longer. =item $JSON::KeySort $json->canonical->encode($perl_scalar) This is the ascii sort. If you want to use with your own sort routine, check the C method. (Only with JSON::PP, even if C<-support_by_pp> is used currently.) $json->sort_by($sort_routine_ref)->encode($perl_scalar) $json->sort_by(sub { $JSON::PP::a <=> $JSON::PP::b })->encode($perl_scalar) Can't access C<$a> and C<$b> but C<$JSON::PP::a> and C<$JSON::PP::b>. =item $JSON::SkipInvalid $json->allow_unknown =item $JSON::AUTOCONVERT Needless. C backend modules have the round-trip integrity. =item $JSON::UTF8 Needless because C (JSON::XS/JSON::PP) sets the UTF8 flag on properly. # With UTF8-flagged strings $json->allow_nonref; $str = chr(1000); # UTF8-flagged $json_text = $json->utf8(0)->encode($str); utf8::is_utf8($json_text); # true $json_text = $json->utf8(1)->encode($str); utf8::is_utf8($json_text); # false $str = '"' . chr(1000) . '"'; # UTF8-flagged $perl_scalar = $json->utf8(0)->decode($str); utf8::is_utf8($perl_scalar); # true $perl_scalar = $json->utf8(1)->decode($str); # died because of 'Wide character in subroutine' See to L. =item $JSON::UnMapping Disable. See to L. =item $JSON::SelfConvert This option was deleted. Instead of it, if a givien blessed object has the C method, C will be executed with C. $json->convert_blessed->encode($bleesed_hashref_or_arrayref) # if need, call allow_blessed Note that it was C in old version, but now not C but C. =back =head1 TODO =over =item example programs =back =head1 THREADS No test with JSON::PP. If with JSON::XS, See to L. =head1 BUGS Please report bugs relevant to C to Emakamaka[at]cpan.orgE. =head1 SEE ALSO Most of the document is copied and modified from JSON::XS doc. L, L C(L) =head1 AUTHOR Makamaka Hannyaharamitu, Emakamaka[at]cpan.orgE JSON::XS was written by Marc Lehmann The relese of this new version owes to the courtesy of Marc Lehmann. =head1 COPYRIGHT AND LICENSE Copyright 2005-2011 by Makamaka Hannyaharamitu This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut JSON $fatpacked{"JSON/backportPP.pm"} = <<'JSON_BACKPORTPP'; package # This is JSON::backportPP JSON::PP; # JSON-2.0 use 5.005; use strict; use base qw(Exporter); use overload (); use Carp (); use B (); #use Devel::Peek; $JSON::PP::VERSION = '2.27200'; @JSON::PP::EXPORT = qw(encode_json decode_json from_json to_json); # instead of hash-access, i tried index-access for speed. # but this method is not faster than what i expected. so it will be changed. use constant P_ASCII => 0; use constant P_LATIN1 => 1; use constant P_UTF8 => 2; use constant P_INDENT => 3; use constant P_CANONICAL => 4; use constant P_SPACE_BEFORE => 5; use constant P_SPACE_AFTER => 6; use constant P_ALLOW_NONREF => 7; use constant P_SHRINK => 8; use constant P_ALLOW_BLESSED => 9; use constant P_CONVERT_BLESSED => 10; use constant P_RELAXED => 11; use constant P_LOOSE => 12; use constant P_ALLOW_BIGNUM => 13; use constant P_ALLOW_BAREKEY => 14; use constant P_ALLOW_SINGLEQUOTE => 15; use constant P_ESCAPE_SLASH => 16; use constant P_AS_NONBLESSED => 17; use constant P_ALLOW_UNKNOWN => 18; use constant OLD_PERL => $] < 5.008 ? 1 : 0; BEGIN { my @xs_compati_bit_properties = qw( latin1 ascii utf8 indent canonical space_before space_after allow_nonref shrink allow_blessed convert_blessed relaxed allow_unknown ); my @pp_bit_properties = qw( allow_singlequote allow_bignum loose allow_barekey escape_slash as_nonblessed ); # Perl version check, Unicode handling is enable? # Helper module sets @JSON::PP::_properties. if ($] < 5.008 ) { my $helper = $] >= 5.006 ? 'JSON::backportPP::Compat5006' : 'JSON::backportPP::Compat5005'; eval qq| require $helper |; if ($@) { Carp::croak $@; } } for my $name (@xs_compati_bit_properties, @pp_bit_properties) { my $flag_name = 'P_' . uc($name); eval qq/ sub $name { my \$enable = defined \$_[1] ? \$_[1] : 1; if (\$enable) { \$_[0]->{PROPS}->[$flag_name] = 1; } else { \$_[0]->{PROPS}->[$flag_name] = 0; } \$_[0]; } sub get_$name { \$_[0]->{PROPS}->[$flag_name] ? 1 : ''; } /; } } # Functions my %encode_allow_method = map {($_ => 1)} qw/utf8 pretty allow_nonref latin1 self_encode escape_slash allow_blessed convert_blessed indent indent_length allow_bignum as_nonblessed /; my %decode_allow_method = map {($_ => 1)} qw/utf8 allow_nonref loose allow_singlequote allow_bignum allow_barekey max_size relaxed/; my $JSON; # cache sub encode_json ($) { # encode ($JSON ||= __PACKAGE__->new->utf8)->encode(@_); } sub decode_json { # decode ($JSON ||= __PACKAGE__->new->utf8)->decode(@_); } # Obsoleted sub to_json($) { Carp::croak ("JSON::PP::to_json has been renamed to encode_json."); } sub from_json($) { Carp::croak ("JSON::PP::from_json has been renamed to decode_json."); } # Methods sub new { my $class = shift; my $self = { max_depth => 512, max_size => 0, indent => 0, FLAGS => 0, fallback => sub { encode_error('Invalid value. JSON can only reference.') }, indent_length => 3, }; bless $self, $class; } sub encode { return $_[0]->PP_encode_json($_[1]); } sub decode { return $_[0]->PP_decode_json($_[1], 0x00000000); } sub decode_prefix { return $_[0]->PP_decode_json($_[1], 0x00000001); } # accessor # pretty printing sub pretty { my ($self, $v) = @_; my $enable = defined $v ? $v : 1; if ($enable) { # indent_length(3) for JSON::XS compatibility $self->indent(1)->indent_length(3)->space_before(1)->space_after(1); } else { $self->indent(0)->space_before(0)->space_after(0); } $self; } # etc sub max_depth { my $max = defined $_[1] ? $_[1] : 0x80000000; $_[0]->{max_depth} = $max; $_[0]; } sub get_max_depth { $_[0]->{max_depth}; } sub max_size { my $max = defined $_[1] ? $_[1] : 0; $_[0]->{max_size} = $max; $_[0]; } sub get_max_size { $_[0]->{max_size}; } sub filter_json_object { $_[0]->{cb_object} = defined $_[1] ? $_[1] : 0; $_[0]->{F_HOOK} = ($_[0]->{cb_object} or $_[0]->{cb_sk_object}) ? 1 : 0; $_[0]; } sub filter_json_single_key_object { if (@_ > 1) { $_[0]->{cb_sk_object}->{$_[1]} = $_[2]; } $_[0]->{F_HOOK} = ($_[0]->{cb_object} or $_[0]->{cb_sk_object}) ? 1 : 0; $_[0]; } sub indent_length { if (!defined $_[1] or $_[1] > 15 or $_[1] < 0) { Carp::carp "The acceptable range of indent_length() is 0 to 15."; } else { $_[0]->{indent_length} = $_[1]; } $_[0]; } sub get_indent_length { $_[0]->{indent_length}; } sub sort_by { $_[0]->{sort_by} = defined $_[1] ? $_[1] : 1; $_[0]; } sub allow_bigint { Carp::carp("allow_bigint() is obsoleted. use allow_bignum() insted."); } ############################### ### ### Perl => JSON ### { # Convert my $max_depth; my $indent; my $ascii; my $latin1; my $utf8; my $space_before; my $space_after; my $canonical; my $allow_blessed; my $convert_blessed; my $indent_length; my $escape_slash; my $bignum; my $as_nonblessed; my $depth; my $indent_count; my $keysort; sub PP_encode_json { my $self = shift; my $obj = shift; $indent_count = 0; $depth = 0; my $idx = $self->{PROPS}; ($ascii, $latin1, $utf8, $indent, $canonical, $space_before, $space_after, $allow_blessed, $convert_blessed, $escape_slash, $bignum, $as_nonblessed) = @{$idx}[P_ASCII .. P_SPACE_AFTER, P_ALLOW_BLESSED, P_CONVERT_BLESSED, P_ESCAPE_SLASH, P_ALLOW_BIGNUM, P_AS_NONBLESSED]; ($max_depth, $indent_length) = @{$self}{qw/max_depth indent_length/}; $keysort = $canonical ? sub { $a cmp $b } : undef; if ($self->{sort_by}) { $keysort = ref($self->{sort_by}) eq 'CODE' ? $self->{sort_by} : $self->{sort_by} =~ /\D+/ ? $self->{sort_by} : sub { $a cmp $b }; } encode_error("hash- or arrayref expected (not a simple scalar, use allow_nonref to allow this)") if(!ref $obj and !$idx->[ P_ALLOW_NONREF ]); my $str = $self->object_to_json($obj); $str .= "\n" if ( $indent ); # JSON::XS 2.26 compatible unless ($ascii or $latin1 or $utf8) { utf8::upgrade($str); } if ($idx->[ P_SHRINK ]) { utf8::downgrade($str, 1); } return $str; } sub object_to_json { my ($self, $obj) = @_; my $type = ref($obj); if($type eq 'HASH'){ return $self->hash_to_json($obj); } elsif($type eq 'ARRAY'){ return $self->array_to_json($obj); } elsif ($type) { # blessed object? if (blessed($obj)) { return $self->value_to_json($obj) if ( $obj->isa('JSON::PP::Boolean') ); if ( $convert_blessed and $obj->can('TO_JSON') ) { my $result = $obj->TO_JSON(); if ( defined $result and ref( $result ) ) { if ( refaddr( $obj ) eq refaddr( $result ) ) { encode_error( sprintf( "%s::TO_JSON method returned same object as was passed instead of a new one", ref $obj ) ); } } return $self->object_to_json( $result ); } return "$obj" if ( $bignum and _is_bignum($obj) ); return $self->blessed_to_json($obj) if ($allow_blessed and $as_nonblessed); # will be removed. encode_error( sprintf("encountered object '%s', but neither allow_blessed " . "nor convert_blessed settings are enabled", $obj) ) unless ($allow_blessed); return 'null'; } else { return $self->value_to_json($obj); } } else{ return $self->value_to_json($obj); } } sub hash_to_json { my ($self, $obj) = @_; my @res; encode_error("json text or perl structure exceeds maximum nesting level (max_depth set too low?)") if (++$depth > $max_depth); my ($pre, $post) = $indent ? $self->_up_indent() : ('', ''); my $del = ($space_before ? ' ' : '') . ':' . ($space_after ? ' ' : ''); for my $k ( _sort( $obj ) ) { if ( OLD_PERL ) { utf8::decode($k) } # key for Perl 5.6 / be optimized push @res, string_to_json( $self, $k ) . $del . ( $self->object_to_json( $obj->{$k} ) || $self->value_to_json( $obj->{$k} ) ); } --$depth; $self->_down_indent() if ($indent); return '{' . ( @res ? $pre : '' ) . ( @res ? join( ",$pre", @res ) . $post : '' ) . '}'; } sub array_to_json { my ($self, $obj) = @_; my @res; encode_error("json text or perl structure exceeds maximum nesting level (max_depth set too low?)") if (++$depth > $max_depth); my ($pre, $post) = $indent ? $self->_up_indent() : ('', ''); for my $v (@$obj){ push @res, $self->object_to_json($v) || $self->value_to_json($v); } --$depth; $self->_down_indent() if ($indent); return '[' . ( @res ? $pre : '' ) . ( @res ? join( ",$pre", @res ) . $post : '' ) . ']'; } sub value_to_json { my ($self, $value) = @_; return 'null' if(!defined $value); my $b_obj = B::svref_2object(\$value); # for round trip problem my $flags = $b_obj->FLAGS; return $value # as is if $flags & ( B::SVp_IOK | B::SVp_NOK ) and !( $flags & B::SVp_POK ); # SvTYPE is IV or NV? my $type = ref($value); if(!$type){ return string_to_json($self, $value); } elsif( blessed($value) and $value->isa('JSON::PP::Boolean') ){ return $$value == 1 ? 'true' : 'false'; } elsif ($type) { if ((overload::StrVal($value) =~ /=(\w+)/)[0]) { return $self->value_to_json("$value"); } if ($type eq 'SCALAR' and defined $$value) { return $$value eq '1' ? 'true' : $$value eq '0' ? 'false' : $self->{PROPS}->[ P_ALLOW_UNKNOWN ] ? 'null' : encode_error("cannot encode reference to scalar"); } if ( $self->{PROPS}->[ P_ALLOW_UNKNOWN ] ) { return 'null'; } else { if ( $type eq 'SCALAR' or $type eq 'REF' ) { encode_error("cannot encode reference to scalar"); } else { encode_error("encountered $value, but JSON can only represent references to arrays or hashes"); } } } else { return $self->{fallback}->($value) if ($self->{fallback} and ref($self->{fallback}) eq 'CODE'); return 'null'; } } my %esc = ( "\n" => '\n', "\r" => '\r', "\t" => '\t', "\f" => '\f', "\b" => '\b', "\"" => '\"', "\\" => '\\\\', "\'" => '\\\'', ); sub string_to_json { my ($self, $arg) = @_; $arg =~ s/([\x22\x5c\n\r\t\f\b])/$esc{$1}/g; $arg =~ s/\//\\\//g if ($escape_slash); $arg =~ s/([\x00-\x08\x0b\x0e-\x1f])/'\\u00' . unpack('H2', $1)/eg; if ($ascii) { $arg = JSON_PP_encode_ascii($arg); } if ($latin1) { $arg = JSON_PP_encode_latin1($arg); } if ($utf8) { utf8::encode($arg); } return '"' . $arg . '"'; } sub blessed_to_json { my $reftype = reftype($_[1]) || ''; if ($reftype eq 'HASH') { return $_[0]->hash_to_json($_[1]); } elsif ($reftype eq 'ARRAY') { return $_[0]->array_to_json($_[1]); } else { return 'null'; } } sub encode_error { my $error = shift; Carp::croak "$error"; } sub _sort { defined $keysort ? (sort $keysort (keys %{$_[0]})) : keys %{$_[0]}; } sub _up_indent { my $self = shift; my $space = ' ' x $indent_length; my ($pre,$post) = ('',''); $post = "\n" . $space x $indent_count; $indent_count++; $pre = "\n" . $space x $indent_count; return ($pre,$post); } sub _down_indent { $indent_count--; } sub PP_encode_box { { depth => $depth, indent_count => $indent_count, }; } } # Convert sub _encode_ascii { join('', map { $_ <= 127 ? chr($_) : $_ <= 65535 ? sprintf('\u%04x', $_) : sprintf('\u%x\u%x', _encode_surrogates($_)); } unpack('U*', $_[0]) ); } sub _encode_latin1 { join('', map { $_ <= 255 ? chr($_) : $_ <= 65535 ? sprintf('\u%04x', $_) : sprintf('\u%x\u%x', _encode_surrogates($_)); } unpack('U*', $_[0]) ); } sub _encode_surrogates { # from perlunicode my $uni = $_[0] - 0x10000; return ($uni / 0x400 + 0xD800, $uni % 0x400 + 0xDC00); } sub _is_bignum { $_[0]->isa('Math::BigInt') or $_[0]->isa('Math::BigFloat'); } # # JSON => Perl # my $max_intsize; BEGIN { my $checkint = 1111; for my $d (5..64) { $checkint .= 1; my $int = eval qq| $checkint |; if ($int =~ /[eE]/) { $max_intsize = $d - 1; last; } } } { # PARSE my %escapes = ( # by Jeremy Muhlich b => "\x8", t => "\x9", n => "\xA", f => "\xC", r => "\xD", '\\' => '\\', '"' => '"', '/' => '/', ); my $text; # json data my $at; # offset my $ch; # 1chracter my $len; # text length (changed according to UTF8 or NON UTF8) # INTERNAL my $depth; # nest counter my $encoding; # json text encoding my $is_valid_utf8; # temp variable my $utf8_len; # utf8 byte length # FLAGS my $utf8; # must be utf8 my $max_depth; # max nest nubmer of objects and arrays my $max_size; my $relaxed; my $cb_object; my $cb_sk_object; my $F_HOOK; my $allow_bigint; # using Math::BigInt my $singlequote; # loosely quoting my $loose; # my $allow_barekey; # bareKey # $opt flag # 0x00000001 .... decode_prefix # 0x10000000 .... incr_parse sub PP_decode_json { my ($self, $opt); # $opt is an effective flag during this decode_json. ($self, $text, $opt) = @_; ($at, $ch, $depth) = (0, '', 0); if ( !defined $text or ref $text ) { decode_error("malformed JSON string, neither array, object, number, string or atom"); } my $idx = $self->{PROPS}; ($utf8, $relaxed, $loose, $allow_bigint, $allow_barekey, $singlequote) = @{$idx}[P_UTF8, P_RELAXED, P_LOOSE .. P_ALLOW_SINGLEQUOTE]; if ( $utf8 ) { utf8::downgrade( $text, 1 ) or Carp::croak("Wide character in subroutine entry"); } else { utf8::upgrade( $text ); } $len = length $text; ($max_depth, $max_size, $cb_object, $cb_sk_object, $F_HOOK) = @{$self}{qw/max_depth max_size cb_object cb_sk_object F_HOOK/}; if ($max_size > 1) { use bytes; my $bytes = length $text; decode_error( sprintf("attempted decode of JSON text of %s bytes size, but max_size is set to %s" , $bytes, $max_size), 1 ) if ($bytes > $max_size); } # Currently no effect # should use regexp my @octets = unpack('C4', $text); $encoding = ( $octets[0] and $octets[1]) ? 'UTF-8' : (!$octets[0] and $octets[1]) ? 'UTF-16BE' : (!$octets[0] and !$octets[1]) ? 'UTF-32BE' : ( $octets[2] ) ? 'UTF-16LE' : (!$octets[2] ) ? 'UTF-32LE' : 'unknown'; white(); # remove head white space my $valid_start = defined $ch; # Is there a first character for JSON structure? my $result = value(); return undef if ( !$result && ( $opt & 0x10000000 ) ); # for incr_parse decode_error("malformed JSON string, neither array, object, number, string or atom") unless $valid_start; if ( !$idx->[ P_ALLOW_NONREF ] and !ref $result ) { decode_error( 'JSON text must be an object or array (but found number, string, true, false or null,' . ' use allow_nonref to allow this)', 1); } Carp::croak('something wrong.') if $len < $at; # we won't arrive here. my $consumed = defined $ch ? $at - 1 : $at; # consumed JSON text length white(); # remove tail white space if ( $ch ) { return ( $result, $consumed ) if ($opt & 0x00000001); # all right if decode_prefix decode_error("garbage after JSON object"); } ( $opt & 0x00000001 ) ? ( $result, $consumed ) : $result; } sub next_chr { return $ch = undef if($at >= $len); $ch = substr($text, $at++, 1); } sub value { white(); return if(!defined $ch); return object() if($ch eq '{'); return array() if($ch eq '['); return string() if($ch eq '"' or ($singlequote and $ch eq "'")); return number() if($ch =~ /[0-9]/ or $ch eq '-'); return word(); } sub string { my ($i, $s, $t, $u); my $utf16; my $is_utf8; ($is_valid_utf8, $utf8_len) = ('', 0); $s = ''; # basically UTF8 flag on if($ch eq '"' or ($singlequote and $ch eq "'")){ my $boundChar = $ch; OUTER: while( defined(next_chr()) ){ if($ch eq $boundChar){ next_chr(); if ($utf16) { decode_error("missing low surrogate character in surrogate pair"); } utf8::decode($s) if($is_utf8); return $s; } elsif($ch eq '\\'){ next_chr(); if(exists $escapes{$ch}){ $s .= $escapes{$ch}; } elsif($ch eq 'u'){ # UNICODE handling my $u = ''; for(1..4){ $ch = next_chr(); last OUTER if($ch !~ /[0-9a-fA-F]/); $u .= $ch; } # U+D800 - U+DBFF if ($u =~ /^[dD][89abAB][0-9a-fA-F]{2}/) { # UTF-16 high surrogate? $utf16 = $u; } # U+DC00 - U+DFFF elsif ($u =~ /^[dD][c-fC-F][0-9a-fA-F]{2}/) { # UTF-16 low surrogate? unless (defined $utf16) { decode_error("missing high surrogate character in surrogate pair"); } $is_utf8 = 1; $s .= JSON_PP_decode_surrogates($utf16, $u) || next; $utf16 = undef; } else { if (defined $utf16) { decode_error("surrogate pair expected"); } if ( ( my $hex = hex( $u ) ) > 127 ) { $is_utf8 = 1; $s .= JSON_PP_decode_unicode($u) || next; } else { $s .= chr $hex; } } } else{ unless ($loose) { $at -= 2; decode_error('illegal backslash escape sequence in string'); } $s .= $ch; } } else{ if ( ord $ch > 127 ) { if ( $utf8 ) { unless( $ch = is_valid_utf8($ch) ) { $at -= 1; decode_error("malformed UTF-8 character in JSON string"); } else { $at += $utf8_len - 1; } } else { utf8::encode( $ch ); } $is_utf8 = 1; } if (!$loose) { if ($ch =~ /[\x00-\x1f\x22\x5c]/) { # '/' ok $at--; decode_error('invalid character encountered while parsing JSON string'); } } $s .= $ch; } } } decode_error("unexpected end of string while parsing JSON string"); } sub white { while( defined $ch ){ if($ch le ' '){ next_chr(); } elsif($ch eq '/'){ next_chr(); if(defined $ch and $ch eq '/'){ 1 while(defined(next_chr()) and $ch ne "\n" and $ch ne "\r"); } elsif(defined $ch and $ch eq '*'){ next_chr(); while(1){ if(defined $ch){ if($ch eq '*'){ if(defined(next_chr()) and $ch eq '/'){ next_chr(); last; } } else{ next_chr(); } } else{ decode_error("Unterminated comment"); } } next; } else{ $at--; decode_error("malformed JSON string, neither array, object, number, string or atom"); } } else{ if ($relaxed and $ch eq '#') { # correctly? pos($text) = $at; $text =~ /\G([^\n]*(?:\r\n|\r|\n|$))/g; $at = pos($text); next_chr; next; } last; } } } sub array { my $a = $_[0] || []; # you can use this code to use another array ref object. decode_error('json text or perl structure exceeds maximum nesting level (max_depth set too low?)') if (++$depth > $max_depth); next_chr(); white(); if(defined $ch and $ch eq ']'){ --$depth; next_chr(); return $a; } else { while(defined($ch)){ push @$a, value(); white(); if (!defined $ch) { last; } if($ch eq ']'){ --$depth; next_chr(); return $a; } if($ch ne ','){ last; } next_chr(); white(); if ($relaxed and $ch eq ']') { --$depth; next_chr(); return $a; } } } decode_error(", or ] expected while parsing array"); } sub object { my $o = $_[0] || {}; # you can use this code to use another hash ref object. my $k; decode_error('json text or perl structure exceeds maximum nesting level (max_depth set too low?)') if (++$depth > $max_depth); next_chr(); white(); if(defined $ch and $ch eq '}'){ --$depth; next_chr(); if ($F_HOOK) { return _json_object_hook($o); } return $o; } else { while (defined $ch) { $k = ($allow_barekey and $ch ne '"' and $ch ne "'") ? bareKey() : string(); white(); if(!defined $ch or $ch ne ':'){ $at--; decode_error("':' expected"); } next_chr(); $o->{$k} = value(); white(); last if (!defined $ch); if($ch eq '}'){ --$depth; next_chr(); if ($F_HOOK) { return _json_object_hook($o); } return $o; } if($ch ne ','){ last; } next_chr(); white(); if ($relaxed and $ch eq '}') { --$depth; next_chr(); if ($F_HOOK) { return _json_object_hook($o); } return $o; } } } $at--; decode_error(", or } expected while parsing object/hash"); } sub bareKey { # doesn't strictly follow Standard ECMA-262 3rd Edition my $key; while($ch =~ /[^\x00-\x23\x25-\x2F\x3A-\x40\x5B-\x5E\x60\x7B-\x7F]/){ $key .= $ch; next_chr(); } return $key; } sub word { my $word = substr($text,$at-1,4); if($word eq 'true'){ $at += 3; next_chr; return $JSON::PP::true; } elsif($word eq 'null'){ $at += 3; next_chr; return undef; } elsif($word eq 'fals'){ $at += 3; if(substr($text,$at,1) eq 'e'){ $at++; next_chr; return $JSON::PP::false; } } $at--; # for decode_error report decode_error("'null' expected") if ($word =~ /^n/); decode_error("'true' expected") if ($word =~ /^t/); decode_error("'false' expected") if ($word =~ /^f/); decode_error("malformed JSON string, neither array, object, number, string or atom"); } sub number { my $n = ''; my $v; # According to RFC4627, hex or oct digts are invalid. if($ch eq '0'){ my $peek = substr($text,$at,1); my $hex = $peek =~ /[xX]/; # 0 or 1 if($hex){ decode_error("malformed number (leading zero must not be followed by another digit)"); ($n) = ( substr($text, $at+1) =~ /^([0-9a-fA-F]+)/); } else{ # oct ($n) = ( substr($text, $at) =~ /^([0-7]+)/); if (defined $n and length $n > 1) { decode_error("malformed number (leading zero must not be followed by another digit)"); } } if(defined $n and length($n)){ if (!$hex and length($n) == 1) { decode_error("malformed number (leading zero must not be followed by another digit)"); } $at += length($n) + $hex; next_chr; return $hex ? hex($n) : oct($n); } } if($ch eq '-'){ $n = '-'; next_chr; if (!defined $ch or $ch !~ /\d/) { decode_error("malformed number (no digits after initial minus)"); } } while(defined $ch and $ch =~ /\d/){ $n .= $ch; next_chr; } if(defined $ch and $ch eq '.'){ $n .= '.'; next_chr; if (!defined $ch or $ch !~ /\d/) { decode_error("malformed number (no digits after decimal point)"); } else { $n .= $ch; } while(defined(next_chr) and $ch =~ /\d/){ $n .= $ch; } } if(defined $ch and ($ch eq 'e' or $ch eq 'E')){ $n .= $ch; next_chr; if(defined($ch) and ($ch eq '+' or $ch eq '-')){ $n .= $ch; next_chr; if (!defined $ch or $ch =~ /\D/) { decode_error("malformed number (no digits after exp sign)"); } $n .= $ch; } elsif(defined($ch) and $ch =~ /\d/){ $n .= $ch; } else { decode_error("malformed number (no digits after exp sign)"); } while(defined(next_chr) and $ch =~ /\d/){ $n .= $ch; } } $v .= $n; if ($v !~ /[.eE]/ and length $v > $max_intsize) { if ($allow_bigint) { # from Adam Sussman require Math::BigInt; return Math::BigInt->new($v); } else { return "$v"; } } elsif ($allow_bigint) { require Math::BigFloat; return Math::BigFloat->new($v); } return 0+$v; } sub is_valid_utf8 { $utf8_len = $_[0] =~ /[\x00-\x7F]/ ? 1 : $_[0] =~ /[\xC2-\xDF]/ ? 2 : $_[0] =~ /[\xE0-\xEF]/ ? 3 : $_[0] =~ /[\xF0-\xF4]/ ? 4 : 0 ; return unless $utf8_len; my $is_valid_utf8 = substr($text, $at - 1, $utf8_len); return ( $is_valid_utf8 =~ /^(?: [\x00-\x7F] |[\xC2-\xDF][\x80-\xBF] |[\xE0][\xA0-\xBF][\x80-\xBF] |[\xE1-\xEC][\x80-\xBF][\x80-\xBF] |[\xED][\x80-\x9F][\x80-\xBF] |[\xEE-\xEF][\x80-\xBF][\x80-\xBF] |[\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF] |[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF] |[\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF] )$/x ) ? $is_valid_utf8 : ''; } sub decode_error { my $error = shift; my $no_rep = shift; my $str = defined $text ? substr($text, $at) : ''; my $mess = ''; my $type = $] >= 5.008 ? 'U*' : $] < 5.006 ? 'C*' : utf8::is_utf8( $str ) ? 'U*' # 5.6 : 'C*' ; for my $c ( unpack( $type, $str ) ) { # emulate pv_uni_display() ? $mess .= $c == 0x07 ? '\a' : $c == 0x09 ? '\t' : $c == 0x0a ? '\n' : $c == 0x0d ? '\r' : $c == 0x0c ? '\f' : $c < 0x20 ? sprintf('\x{%x}', $c) : $c == 0x5c ? '\\\\' : $c < 0x80 ? chr($c) : sprintf('\x{%x}', $c) ; if ( length $mess >= 20 ) { $mess .= '...'; last; } } unless ( length $mess ) { $mess = '(end of string)'; } Carp::croak ( $no_rep ? "$error" : "$error, at character offset $at (before \"$mess\")" ); } sub _json_object_hook { my $o = $_[0]; my @ks = keys %{$o}; if ( $cb_sk_object and @ks == 1 and exists $cb_sk_object->{ $ks[0] } and ref $cb_sk_object->{ $ks[0] } ) { my @val = $cb_sk_object->{ $ks[0] }->( $o->{$ks[0]} ); if (@val == 1) { return $val[0]; } } my @val = $cb_object->($o) if ($cb_object); if (@val == 0 or @val > 1) { return $o; } else { return $val[0]; } } sub PP_decode_box { { text => $text, at => $at, ch => $ch, len => $len, depth => $depth, encoding => $encoding, is_valid_utf8 => $is_valid_utf8, }; } } # PARSE sub _decode_surrogates { # from perlunicode my $uni = 0x10000 + (hex($_[0]) - 0xD800) * 0x400 + (hex($_[1]) - 0xDC00); my $un = pack('U*', $uni); utf8::encode( $un ); return $un; } sub _decode_unicode { my $un = pack('U', hex shift); utf8::encode( $un ); return $un; } # # Setup for various Perl versions (the code from JSON::PP58) # BEGIN { unless ( defined &utf8::is_utf8 ) { require Encode; *utf8::is_utf8 = *Encode::is_utf8; } if ( $] >= 5.008 ) { *JSON::PP::JSON_PP_encode_ascii = \&_encode_ascii; *JSON::PP::JSON_PP_encode_latin1 = \&_encode_latin1; *JSON::PP::JSON_PP_decode_surrogates = \&_decode_surrogates; *JSON::PP::JSON_PP_decode_unicode = \&_decode_unicode; } if ($] >= 5.008 and $] < 5.008003) { # join() in 5.8.0 - 5.8.2 is broken. package JSON::PP; require subs; subs->import('join'); eval q| sub join { return '' if (@_ < 2); my $j = shift; my $str = shift; for (@_) { $str .= $j . $_; } return $str; } |; } sub JSON::PP::incr_parse { local $Carp::CarpLevel = 1; ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_parse( @_ ); } sub JSON::PP::incr_skip { ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_skip; } sub JSON::PP::incr_reset { ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_reset; } eval q{ sub JSON::PP::incr_text : lvalue { $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new; if ( $_[0]->{_incr_parser}->{incr_parsing} ) { Carp::croak("incr_text can not be called when the incremental parser already started parsing"); } $_[0]->{_incr_parser}->{incr_text}; } } if ( $] >= 5.006 ); } # Setup for various Perl versions (the code from JSON::PP58) ############################### # Utilities # BEGIN { eval 'require Scalar::Util'; unless($@){ *JSON::PP::blessed = \&Scalar::Util::blessed; *JSON::PP::reftype = \&Scalar::Util::reftype; *JSON::PP::refaddr = \&Scalar::Util::refaddr; } else{ # This code is from Sclar::Util. # warn $@; eval 'sub UNIVERSAL::a_sub_not_likely_to_be_here { ref($_[0]) }'; *JSON::PP::blessed = sub { local($@, $SIG{__DIE__}, $SIG{__WARN__}); ref($_[0]) ? eval { $_[0]->a_sub_not_likely_to_be_here } : undef; }; my %tmap = qw( B::NULL SCALAR B::HV HASH B::AV ARRAY B::CV CODE B::IO IO B::GV GLOB B::REGEXP REGEXP ); *JSON::PP::reftype = sub { my $r = shift; return undef unless length(ref($r)); my $t = ref(B::svref_2object($r)); return exists $tmap{$t} ? $tmap{$t} : length(ref($$r)) ? 'REF' : 'SCALAR'; }; *JSON::PP::refaddr = sub { return undef unless length(ref($_[0])); my $addr; if(defined(my $pkg = blessed($_[0]))) { $addr .= bless $_[0], 'Scalar::Util::Fake'; bless $_[0], $pkg; } else { $addr .= $_[0] } $addr =~ /0x(\w+)/; local $^W; #no warnings 'portable'; hex($1); } } } # shamely copied and modified from JSON::XS code. $JSON::PP::true = do { bless \(my $dummy = 1), "JSON::backportPP::Boolean" }; $JSON::PP::false = do { bless \(my $dummy = 0), "JSON::backportPP::Boolean" }; sub is_bool { defined $_[0] and UNIVERSAL::isa($_[0], "JSON::PP::Boolean"); } sub true { $JSON::PP::true } sub false { $JSON::PP::false } sub null { undef; } ############################### package JSON::backportPP::Boolean; @JSON::backportPP::Boolean::ISA = ('JSON::PP::Boolean'); use overload ( "0+" => sub { ${$_[0]} }, "++" => sub { $_[0] = ${$_[0]} + 1 }, "--" => sub { $_[0] = ${$_[0]} - 1 }, fallback => 1, ); ############################### package JSON::PP::IncrParser; use strict; use constant INCR_M_WS => 0; # initial whitespace skipping use constant INCR_M_STR => 1; # inside string use constant INCR_M_BS => 2; # inside backslash use constant INCR_M_JSON => 3; # outside anything, count nesting use constant INCR_M_C0 => 4; use constant INCR_M_C1 => 5; $JSON::PP::IncrParser::VERSION = '1.01'; my $unpack_format = $] < 5.006 ? 'C*' : 'U*'; sub new { my ( $class ) = @_; bless { incr_nest => 0, incr_text => undef, incr_parsing => 0, incr_p => 0, }, $class; } sub incr_parse { my ( $self, $coder, $text ) = @_; $self->{incr_text} = '' unless ( defined $self->{incr_text} ); if ( defined $text ) { if ( utf8::is_utf8( $text ) and !utf8::is_utf8( $self->{incr_text} ) ) { utf8::upgrade( $self->{incr_text} ) ; utf8::decode( $self->{incr_text} ) ; } $self->{incr_text} .= $text; } my $max_size = $coder->get_max_size; if ( defined wantarray ) { $self->{incr_mode} = INCR_M_WS unless defined $self->{incr_mode}; if ( wantarray ) { my @ret; $self->{incr_parsing} = 1; do { push @ret, $self->_incr_parse( $coder, $self->{incr_text} ); unless ( !$self->{incr_nest} and $self->{incr_mode} == INCR_M_JSON ) { $self->{incr_mode} = INCR_M_WS if $self->{incr_mode} != INCR_M_STR; } } until ( length $self->{incr_text} >= $self->{incr_p} ); $self->{incr_parsing} = 0; return @ret; } else { # in scalar context $self->{incr_parsing} = 1; my $obj = $self->_incr_parse( $coder, $self->{incr_text} ); $self->{incr_parsing} = 0 if defined $obj; # pointed by Martin J. Evans return $obj ? $obj : undef; # $obj is an empty string, parsing was completed. } } } sub _incr_parse { my ( $self, $coder, $text, $skip ) = @_; my $p = $self->{incr_p}; my $restore = $p; my @obj; my $len = length $text; if ( $self->{incr_mode} == INCR_M_WS ) { while ( $len > $p ) { my $s = substr( $text, $p, 1 ); $p++ and next if ( 0x20 >= unpack($unpack_format, $s) ); $self->{incr_mode} = INCR_M_JSON; last; } } while ( $len > $p ) { my $s = substr( $text, $p++, 1 ); if ( $s eq '"' ) { if (substr( $text, $p - 2, 1 ) eq '\\' ) { next; } if ( $self->{incr_mode} != INCR_M_STR ) { $self->{incr_mode} = INCR_M_STR; } else { $self->{incr_mode} = INCR_M_JSON; unless ( $self->{incr_nest} ) { last; } } } if ( $self->{incr_mode} == INCR_M_JSON ) { if ( $s eq '[' or $s eq '{' ) { if ( ++$self->{incr_nest} > $coder->get_max_depth ) { Carp::croak('json text or perl structure exceeds maximum nesting level (max_depth set too low?)'); } } elsif ( $s eq ']' or $s eq '}' ) { last if ( --$self->{incr_nest} <= 0 ); } elsif ( $s eq '#' ) { while ( $len > $p ) { last if substr( $text, $p++, 1 ) eq "\n"; } } } } $self->{incr_p} = $p; return if ( $self->{incr_mode} == INCR_M_STR and not $self->{incr_nest} ); return if ( $self->{incr_mode} == INCR_M_JSON and $self->{incr_nest} > 0 ); return '' unless ( length substr( $self->{incr_text}, 0, $p ) ); local $Carp::CarpLevel = 2; $self->{incr_p} = $restore; $self->{incr_c} = $p; my ( $obj, $tail ) = $coder->PP_decode_json( substr( $self->{incr_text}, 0, $p ), 0x10000001 ); $self->{incr_text} = substr( $self->{incr_text}, $p ); $self->{incr_p} = 0; return $obj or ''; } sub incr_text { if ( $_[0]->{incr_parsing} ) { Carp::croak("incr_text can not be called when the incremental parser already started parsing"); } $_[0]->{incr_text}; } sub incr_skip { my $self = shift; $self->{incr_text} = substr( $self->{incr_text}, $self->{incr_c} ); $self->{incr_p} = 0; } sub incr_reset { my $self = shift; $self->{incr_text} = undef; $self->{incr_p} = 0; $self->{incr_mode} = 0; $self->{incr_nest} = 0; $self->{incr_parsing} = 0; } ############################### 1; __END__ =pod =head1 NAME JSON::PP - JSON::XS compatible pure-Perl module. =head1 SYNOPSIS use JSON::PP; # exported functions, they croak on error # and expect/generate UTF-8 $utf8_encoded_json_text = encode_json $perl_hash_or_arrayref; $perl_hash_or_arrayref = decode_json $utf8_encoded_json_text; # OO-interface $coder = JSON::PP->new->ascii->pretty->allow_nonref; $json_text = $json->encode( $perl_scalar ); $perl_scalar = $json->decode( $json_text ); $pretty_printed = $json->pretty->encode( $perl_scalar ); # pretty-printing # Note that JSON version 2.0 and above will automatically use # JSON::XS or JSON::PP, so you should be able to just: use JSON; =head1 VERSION 2.27200 L 2.27 (~2.30) compatible. =head1 DESCRIPTION This module is L compatible pure Perl module. (Perl 5.8 or later is recommended) JSON::XS is the fastest and most proper JSON module on CPAN. It is written by Marc Lehmann in C, so must be compiled and installed in the used environment. JSON::PP is a pure-Perl module and has compatibility to JSON::XS. =head2 FEATURES =over =item * correct unicode handling This module knows how to handle Unicode (depending on Perl version). See to L and L. =item * round-trip integrity When you serialise a perl data structure using only data types supported by JSON and Perl, the deserialised data structure is identical on the Perl level. (e.g. the string "2.0" doesn't suddenly become "2" just because it looks like a number). There I minor exceptions to this, read the MAPPING section below to learn about those. =item * strict checking of JSON correctness There is no guessing, no generating of illegal JSON texts by default, and only JSON is accepted as input by default (the latter is a security feature). But when some options are set, loose chcking features are available. =back =head1 FUNCTIONAL INTERFACE Some documents are copied and modified from L. =head2 encode_json $json_text = encode_json $perl_scalar Converts the given Perl data structure to a UTF-8 encoded, binary string. This function call is functionally identical to: $json_text = JSON::PP->new->utf8->encode($perl_scalar) =head2 decode_json $perl_scalar = decode_json $json_text The opposite of C: expects an UTF-8 (binary) string and tries to parse that as an UTF-8 encoded JSON text, returning the resulting reference. This function call is functionally identical to: $perl_scalar = JSON::PP->new->utf8->decode($json_text) =head2 JSON::PP::is_bool $is_boolean = JSON::PP::is_bool($scalar) Returns true if the passed scalar represents either JSON::PP::true or JSON::PP::false, two constants that act like C<1> and C<0> respectively and are also used to represent JSON C and C in Perl strings. =head2 JSON::PP::true Returns JSON true value which is blessed object. It C JSON::PP::Boolean object. =head2 JSON::PP::false Returns JSON false value which is blessed object. It C JSON::PP::Boolean object. =head2 JSON::PP::null Returns C. See L, below, for more information on how JSON values are mapped to Perl. =head1 HOW DO I DECODE A DATA FROM OUTER AND ENCODE TO OUTER This section supposes that your perl vresion is 5.8 or later. If you know a JSON text from an outer world - a network, a file content, and so on, is encoded in UTF-8, you should use C or C module object with C enable. And the decoded result will contain UNICODE characters. # from network my $json = JSON::PP->new->utf8; my $json_text = CGI->new->param( 'json_data' ); my $perl_scalar = $json->decode( $json_text ); # from file content local $/; open( my $fh, '<', 'json.data' ); $json_text = <$fh>; $perl_scalar = decode_json( $json_text ); If an outer data is not encoded in UTF-8, firstly you should C it. use Encode; local $/; open( my $fh, '<', 'json.data' ); my $encoding = 'cp932'; my $unicode_json_text = decode( $encoding, <$fh> ); # UNICODE # or you can write the below code. # # open( my $fh, "<:encoding($encoding)", 'json.data' ); # $unicode_json_text = <$fh>; In this case, C<$unicode_json_text> is of course UNICODE string. So you B use C nor C module object with C enable. Instead of them, you use C module object with C disable. $perl_scalar = $json->utf8(0)->decode( $unicode_json_text ); Or C and C: $perl_scalar = decode_json( encode( 'utf8', $unicode_json_text ) ); # this way is not efficient. And now, you want to convert your C<$perl_scalar> into JSON data and send it to an outer world - a network or a file content, and so on. Your data usually contains UNICODE strings and you want the converted data to be encoded in UTF-8, you should use C or C module object with C enable. print encode_json( $perl_scalar ); # to a network? file? or display? # or print $json->utf8->encode( $perl_scalar ); If C<$perl_scalar> does not contain UNICODE but C<$encoding>-encoded strings for some reason, then its characters are regarded as B for perl (because it does not concern with your $encoding). You B use C nor C module object with C enable. Instead of them, you use C module object with C disable. Note that the resulted text is a UNICODE string but no problem to print it. # $perl_scalar contains $encoding encoded string values $unicode_json_text = $json->utf8(0)->encode( $perl_scalar ); # $unicode_json_text consists of characters less than 0x100 print $unicode_json_text; Or C all string values and C: $perl_scalar->{ foo } = decode( $encoding, $perl_scalar->{ foo } ); # ... do it to each string values, then encode_json $json_text = encode_json( $perl_scalar ); This method is a proper way but probably not efficient. See to L, L. =head1 METHODS Basically, check to L or L. =head2 new $json = JSON::PP->new Rturns a new JSON::PP object that can be used to de/encode JSON strings. All boolean flags described below are by default I. The mutators for flags all return the JSON object again and thus calls can be chained: my $json = JSON::PP->new->utf8->space_after->encode({a => [1,2]}) => {"a": [1, 2]} =head2 ascii $json = $json->ascii([$enable]) $enabled = $json->get_ascii If $enable is true (or missing), then the encode method will not generate characters outside the code range 0..127. Any Unicode characters outside that range will be escaped using either a single \uXXXX or a double \uHHHH\uLLLLL escape sequence, as per RFC4627. (See to L). In Perl 5.005, there is no character having high value (more than 255). See to L. If $enable is false, then the encode method will not escape Unicode characters unless required by the JSON syntax or other flags. This results in a faster and more compact format. JSON::PP->new->ascii(1)->encode([chr 0x10401]) => ["\ud801\udc01"] =head2 latin1 $json = $json->latin1([$enable]) $enabled = $json->get_latin1 If $enable is true (or missing), then the encode method will encode the resulting JSON text as latin1 (or iso-8859-1), escaping any characters outside the code range 0..255. If $enable is false, then the encode method will not escape Unicode characters unless required by the JSON syntax or other flags. JSON::XS->new->latin1->encode (["\x{89}\x{abc}"] => ["\x{89}\\u0abc"] # (perl syntax, U+abc escaped, U+89 not) See to L. =head2 utf8 $json = $json->utf8([$enable]) $enabled = $json->get_utf8 If $enable is true (or missing), then the encode method will encode the JSON result into UTF-8, as required by many protocols, while the decode method expects to be handled an UTF-8-encoded string. Please note that UTF-8-encoded strings do not contain any characters outside the range 0..255, they are thus useful for bytewise/binary I/O. (In Perl 5.005, any character outside the range 0..255 does not exist. See to L.) In future versions, enabling this option might enable autodetection of the UTF-16 and UTF-32 encoding families, as described in RFC4627. If $enable is false, then the encode method will return the JSON string as a (non-encoded) Unicode string, while decode expects thus a Unicode string. Any decoding or encoding (e.g. to UTF-8 or UTF-16) needs to be done yourself, e.g. using the Encode module. Example, output UTF-16BE-encoded JSON: use Encode; $jsontext = encode "UTF-16BE", JSON::PP->new->encode ($object); Example, decode UTF-32LE-encoded JSON: use Encode; $object = JSON::PP->new->decode (decode "UTF-32LE", $jsontext); =head2 pretty $json = $json->pretty([$enable]) This enables (or disables) all of the C, C and C flags in one call to generate the most readable (or most compact) form possible. Equivalent to: $json->indent->space_before->space_after =head2 indent $json = $json->indent([$enable]) $enabled = $json->get_indent The default indent space length is three. You can use C to change the length. =head2 space_before $json = $json->space_before([$enable]) $enabled = $json->get_space_before If C<$enable> is true (or missing), then the C method will add an extra optional space before the C<:> separating keys from values in JSON objects. If C<$enable> is false, then the C method will not add any extra space at those places. This setting has no effect when decoding JSON texts. Example, space_before enabled, space_after and indent disabled: {"key" :"value"} =head2 space_after $json = $json->space_after([$enable]) $enabled = $json->get_space_after If C<$enable> is true (or missing), then the C method will add an extra optional space after the C<:> separating keys from values in JSON objects and extra whitespace after the C<,> separating key-value pairs and array members. If C<$enable> is false, then the C method will not add any extra space at those places. This setting has no effect when decoding JSON texts. Example, space_before and indent disabled, space_after enabled: {"key": "value"} =head2 relaxed $json = $json->relaxed([$enable]) $enabled = $json->get_relaxed If C<$enable> is true (or missing), then C will accept some extensions to normal JSON syntax (see below). C will not be affected in anyway. I. I suggest only to use this option to parse application-specific files written by humans (configuration files, resource files etc.) If C<$enable> is false (the default), then C will only accept valid JSON texts. Currently accepted extensions are: =over 4 =item * list items can have an end-comma JSON I array elements and key-value pairs with commas. This can be annoying if you write JSON texts manually and want to be able to quickly append elements, so this extension accepts comma at the end of such items not just between them: [ 1, 2, <- this comma not normally allowed ] { "k1": "v1", "k2": "v2", <- this comma not normally allowed } =item * shell-style '#'-comments Whenever JSON allows whitespace, shell-style comments are additionally allowed. They are terminated by the first carriage-return or line-feed character, after which more white-space and comments are allowed. [ 1, # this comment not allowed in JSON # neither this one... ] =back =head2 canonical $json = $json->canonical([$enable]) $enabled = $json->get_canonical If C<$enable> is true (or missing), then the C method will output JSON objects by sorting their keys. This is adding a comparatively high overhead. If C<$enable> is false, then the C method will output key-value pairs in the order Perl stores them (which will likely change between runs of the same script). This option is useful if you want the same data structure to be encoded as the same JSON text (given the same overall settings). If it is disabled, the same hash might be encoded differently even if contains the same data, as key-value pairs have no inherent ordering in Perl. This setting has no effect when decoding JSON texts. If you want your own sorting routine, you can give a code referece or a subroutine name to C. See to C. =head2 allow_nonref $json = $json->allow_nonref([$enable]) $enabled = $json->get_allow_nonref If C<$enable> is true (or missing), then the C method can convert a non-reference into its corresponding string, number or null JSON value, which is an extension to RFC4627. Likewise, C will accept those JSON values instead of croaking. If C<$enable> is false, then the C method will croak if it isn't passed an arrayref or hashref, as JSON texts must either be an object or array. Likewise, C will croak if given something that is not a JSON object or array. JSON::PP->new->allow_nonref->encode ("Hello, World!") => "Hello, World!" =head2 allow_unknown $json = $json->allow_unknown ([$enable]) $enabled = $json->get_allow_unknown If $enable is true (or missing), then "encode" will *not* throw an exception when it encounters values it cannot represent in JSON (for example, filehandles) but instead will encode a JSON "null" value. Note that blessed objects are not included here and are handled separately by c. If $enable is false (the default), then "encode" will throw an exception when it encounters anything it cannot encode as JSON. This option does not affect "decode" in any way, and it is recommended to leave it off unless you know your communications partner. =head2 allow_blessed $json = $json->allow_blessed([$enable]) $enabled = $json->get_allow_blessed If C<$enable> is true (or missing), then the C method will not barf when it encounters a blessed reference. Instead, the value of the B option will decide whether C (C disabled or no C method found) or a representation of the object (C enabled and C method found) is being encoded. Has no effect on C. If C<$enable> is false (the default), then C will throw an exception when it encounters a blessed object. =head2 convert_blessed $json = $json->convert_blessed([$enable]) $enabled = $json->get_convert_blessed If C<$enable> is true (or missing), then C, upon encountering a blessed object, will check for the availability of the C method on the object's class. If found, it will be called in scalar context and the resulting scalar will be encoded instead of the object. If no C method is found, the value of C will decide what to do. The C method may safely call die if it wants. If C returns other blessed objects, those will be handled in the same way. C must take care of not causing an endless recursion cycle (== crash) in this case. The name of C was chosen because other methods called by the Perl core (== not by the user of the object) are usually in upper case letters and to avoid collisions with the C function or method. This setting does not yet influence C in any way. If C<$enable> is false, then the C setting will decide what to do when a blessed object is found. =head2 filter_json_object $json = $json->filter_json_object([$coderef]) When C<$coderef> is specified, it will be called from C each time it decodes a JSON object. The only argument passed to the coderef is a reference to the newly-created hash. If the code references returns a single scalar (which need not be a reference), this value (i.e. a copy of that scalar to avoid aliasing) is inserted into the deserialised data structure. If it returns an empty list (NOTE: I C, which is a valid scalar), the original deserialised hash will be inserted. This setting can slow down decoding considerably. When C<$coderef> is omitted or undefined, any existing callback will be removed and C will not change the deserialised hash in any way. Example, convert all JSON objects into the integer 5: my $js = JSON::PP->new->filter_json_object (sub { 5 }); # returns [5] $js->decode ('[{}]'); # the given subroutine takes a hash reference. # throw an exception because allow_nonref is not enabled # so a lone 5 is not allowed. $js->decode ('{"a":1, "b":2}'); =head2 filter_json_single_key_object $json = $json->filter_json_single_key_object($key [=> $coderef]) Works remotely similar to C, but is only called for JSON objects having a single key named C<$key>. This C<$coderef> is called before the one specified via C, if any. It gets passed the single value in the JSON object. If it returns a single value, it will be inserted into the data structure. If it returns nothing (not even C but the empty list), the callback from C will be called next, as if no single-key callback were specified. If C<$coderef> is omitted or undefined, the corresponding callback will be disabled. There can only ever be one callback for a given key. As this callback gets called less often then the C one, decoding speed will not usually suffer as much. Therefore, single-key objects make excellent targets to serialise Perl objects into, especially as single-key JSON objects are as close to the type-tagged value concept as JSON gets (it's basically an ID/VALUE tuple). Of course, JSON does not support this in any way, so you need to make sure your data never looks like a serialised Perl hash. Typical names for the single object key are C<__class_whatever__>, or C<$__dollars_are_rarely_used__$> or C<}ugly_brace_placement>, or even things like C<__class_md5sum(classname)__>, to reduce the risk of clashing with real hashes. Example, decode JSON objects of the form C<< { "__widget__" => } >> into the corresponding C<< $WIDGET{} >> object: # return whatever is in $WIDGET{5}: JSON::PP ->new ->filter_json_single_key_object (__widget__ => sub { $WIDGET{ $_[0] } }) ->decode ('{"__widget__": 5') # this can be used with a TO_JSON method in some "widget" class # for serialisation to json: sub WidgetBase::TO_JSON { my ($self) = @_; unless ($self->{id}) { $self->{id} = ..get..some..id..; $WIDGET{$self->{id}} = $self; } { __widget__ => $self->{id} } } =head2 shrink $json = $json->shrink([$enable]) $enabled = $json->get_shrink In JSON::XS, this flag resizes strings generated by either C or C to their minimum size possible. It will also try to downgrade any strings to octet-form if possible. In JSON::PP, it is noop about resizing strings but tries C to the returned string by C. See to L. See to L =head2 max_depth $json = $json->max_depth([$maximum_nesting_depth]) $max_depth = $json->get_max_depth Sets the maximum nesting level (default C<512>) accepted while encoding or decoding. If a higher nesting level is detected in JSON text or a Perl data structure, then the encoder and decoder will stop and croak at that point. Nesting level is defined by number of hash- or arrayrefs that the encoder needs to traverse to reach a given point or the number of C<{> or C<[> characters without their matching closing parenthesis crossed to reach a given character in a string. If no argument is given, the highest possible setting will be used, which is rarely useful. See L for more info on why this is useful. When a large value (100 or more) was set and it de/encodes a deep nested object/text, it may raise a warning 'Deep recursion on subroutin' at the perl runtime phase. =head2 max_size $json = $json->max_size([$maximum_string_size]) $max_size = $json->get_max_size Set the maximum length a JSON text may have (in bytes) where decoding is being attempted. The default is C<0>, meaning no limit. When C is called on a string that is longer then this many bytes, it will not attempt to decode the string but throw an exception. This setting has no effect on C (yet). If no argument is given, the limit check will be deactivated (same as when C<0> is specified). See L for more info on why this is useful. =head2 encode $json_text = $json->encode($perl_scalar) Converts the given Perl data structure (a simple scalar or a reference to a hash or array) to its JSON representation. Simple scalars will be converted into JSON string or number sequences, while references to arrays become JSON arrays and references to hashes become JSON objects. Undefined Perl values (e.g. C) become JSON C values. References to the integers C<0> and C<1> are converted into C and C. =head2 decode $perl_scalar = $json->decode($json_text) The opposite of C: expects a JSON text and tries to parse it, returning the resulting simple scalar or reference. Croaks on error. JSON numbers and strings become simple Perl scalars. JSON arrays become Perl arrayrefs and JSON objects become Perl hashrefs. C becomes C<1> (C), C becomes C<0> (C) and C becomes C. =head2 decode_prefix ($perl_scalar, $characters) = $json->decode_prefix($json_text) This works like the C method, but instead of raising an exception when there is trailing garbage after the first JSON object, it will silently stop parsing there and return the number of characters consumed so far. JSON->new->decode_prefix ("[1] the tail") => ([], 3) =head1 INCREMENTAL PARSING Most of this section are copied and modified from L. In some cases, there is the need for incremental parsing of JSON texts. This module does allow you to parse a JSON stream incrementally. It does so by accumulating text until it has a full JSON object, which it then can decode. This process is similar to using C to see if a full JSON object is available, but is much more efficient (and can be implemented with a minimum of method calls). This module will only attempt to parse the JSON text once it is sure it has enough text to get a decisive result, using a very simple but truly incremental parser. This means that it sometimes won't stop as early as the full parser, for example, it doesn't detect parenthese mismatches. The only thing it guarantees is that it starts decoding as soon as a syntactically valid JSON text has been seen. This means you need to set resource limits (e.g. C) to ensure the parser will stop parsing in the presence if syntax errors. The following methods implement this incremental parser. =head2 incr_parse $json->incr_parse( [$string] ) # void context $obj_or_undef = $json->incr_parse( [$string] ) # scalar context @obj_or_empty = $json->incr_parse( [$string] ) # list context This is the central parsing function. It can both append new text and extract objects from the stream accumulated so far (both of these functions are optional). If C<$string> is given, then this string is appended to the already existing JSON fragment stored in the C<$json> object. After that, if the function is called in void context, it will simply return without doing anything further. This can be used to add more text in as many chunks as you want. If the method is called in scalar context, then it will try to extract exactly I JSON object. If that is successful, it will return this object, otherwise it will return C. If there is a parse error, this method will croak just as C would do (one can then use C to skip the errornous part). This is the most common way of using the method. And finally, in list context, it will try to extract as many objects from the stream as it can find and return them, or the empty list otherwise. For this to work, there must be no separators between the JSON objects or arrays, instead they must be concatenated back-to-back. If an error occurs, an exception will be raised as in the scalar context case. Note that in this case, any previously-parsed JSON texts will be lost. Example: Parse some JSON arrays/objects in a given string and return them. my @objs = JSON->new->incr_parse ("[5][7][1,2]"); =head2 incr_text $lvalue_string = $json->incr_text This method returns the currently stored JSON fragment as an lvalue, that is, you can manipulate it. This I works when a preceding call to C in I successfully returned an object. Under all other circumstances you must not call this function (I mean it. although in simple tests it might actually work, it I fail under real world conditions). As a special exception, you can also call this method before having parsed anything. This function is useful in two cases: a) finding the trailing text after a JSON object or b) parsing multiple JSON objects separated by non-JSON text (such as commas). $json->incr_text =~ s/\s*,\s*//; In Perl 5.005, C attribute is not available. You must write codes like the below: $string = $json->incr_text; $string =~ s/\s*,\s*//; $json->incr_text( $string ); =head2 incr_skip $json->incr_skip This will reset the state of the incremental parser and will remove the parsed text from the input buffer. This is useful after C died, in which case the input buffer and incremental parser state is left unchanged, to skip the text parsed so far and to reset the parse state. =head2 incr_reset $json->incr_reset This completely resets the incremental parser, that is, after this call, it will be as if the parser had never parsed anything. This is useful if you want ot repeatedly parse JSON objects and want to ignore any trailing data, which means you have to reset the parser after each successful decode. See to L for examples. =head1 JSON::PP OWN METHODS =head2 allow_singlequote $json = $json->allow_singlequote([$enable]) If C<$enable> is true (or missing), then C will accept JSON strings quoted by single quotations that are invalid JSON format. $json->allow_singlequote->decode({"foo":'bar'}); $json->allow_singlequote->decode({'foo':"bar"}); $json->allow_singlequote->decode({'foo':'bar'}); As same as the C option, this option may be used to parse application-specific files written by humans. =head2 allow_barekey $json = $json->allow_barekey([$enable]) If C<$enable> is true (or missing), then C will accept bare keys of JSON object that are invalid JSON format. As same as the C option, this option may be used to parse application-specific files written by humans. $json->allow_barekey->decode('{foo:"bar"}'); =head2 allow_bignum $json = $json->allow_bignum([$enable]) If C<$enable> is true (or missing), then C will convert the big integer Perl cannot handle as integer into a L object and convert a floating number (any) into a L. On the contary, C converts C objects and C objects into JSON numbers with C enable. $json->allow_nonref->allow_blessed->allow_bignum; $bigfloat = $json->decode('2.000000000000000000000000001'); print $json->encode($bigfloat); # => 2.000000000000000000000000001 See to L aboout the normal conversion of JSON number. =head2 loose $json = $json->loose([$enable]) The unescaped [\x00-\x1f\x22\x2f\x5c] strings are invalid in JSON strings and the module doesn't allow to C to these (except for \x2f). If C<$enable> is true (or missing), then C will accept these unescaped strings. $json->loose->decode(qq|["abc def"]|); See L. =head2 escape_slash $json = $json->escape_slash([$enable]) According to JSON Grammar, I (U+002F) is escaped. But default JSON::PP (as same as JSON::XS) encodes strings without escaping slash. If C<$enable> is true (or missing), then C will escape slashes. =head2 indent_length $json = $json->indent_length($length) JSON::XS indent space length is 3 and cannot be changed. JSON::PP set the indent space length with the given $length. The default is 3. The acceptable range is 0 to 15. =head2 sort_by $json = $json->sort_by($function_name) $json = $json->sort_by($subroutine_ref) If $function_name or $subroutine_ref are set, its sort routine are used in encoding JSON objects. $js = $pc->sort_by(sub { $JSON::PP::a cmp $JSON::PP::b })->encode($obj); # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|); $js = $pc->sort_by('own_sort')->encode($obj); # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|); sub JSON::PP::own_sort { $JSON::PP::a cmp $JSON::PP::b } As the sorting routine runs in the JSON::PP scope, the given subroutine name and the special variables C<$a>, C<$b> will begin 'JSON::PP::'. If $integer is set, then the effect is same as C on. =head1 INTERNAL For developers. =over =item PP_encode_box Returns { depth => $depth, indent_count => $indent_count, } =item PP_decode_box Returns { text => $text, at => $at, ch => $ch, len => $len, depth => $depth, encoding => $encoding, is_valid_utf8 => $is_valid_utf8, }; =back =head1 MAPPING This section is copied from JSON::XS and modified to C. JSON::XS and JSON::PP mapping mechanisms are almost equivalent. See to L. =head2 JSON -> PERL =over 4 =item object A JSON object becomes a reference to a hash in Perl. No ordering of object keys is preserved (JSON does not preserver object key ordering itself). =item array A JSON array becomes a reference to an array in Perl. =item string A JSON string becomes a string scalar in Perl - Unicode codepoints in JSON are represented by the same codepoints in the Perl string, so no manual decoding is necessary. =item number A JSON number becomes either an integer, numeric (floating point) or string scalar in perl, depending on its range and any fractional parts. On the Perl level, there is no difference between those as Perl handles all the conversion details, but an integer may take slightly less memory and might represent more values exactly than floating point numbers. If the number consists of digits only, C will try to represent it as an integer value. If that fails, it will try to represent it as a numeric (floating point) value if that is possible without loss of precision. Otherwise it will preserve the number as a string value (in which case you lose roundtripping ability, as the JSON number will be re-encoded toa JSON string). Numbers containing a fractional or exponential part will always be represented as numeric (floating point) values, possibly at a loss of precision (in which case you might lose perfect roundtripping ability, but the JSON number will still be re-encoded as a JSON number). Note that precision is not accuracy - binary floating point values cannot represent most decimal fractions exactly, and when converting from and to floating point, C only guarantees precision up to but not including the leats significant bit. When C is enable, the big integers and the numeric can be optionally converted into L and L objects. =item true, false These JSON atoms become C and C, respectively. They are overloaded to act almost exactly like the numbers C<1> and C<0>. You can check wether a scalar is a JSON boolean by using the C function. print JSON::PP::true . "\n"; => true print JSON::PP::true + 1; => 1 ok(JSON::true eq '1'); ok(JSON::true == 1); C will install these missing overloading features to the backend modules. =item null A JSON null atom becomes C in Perl. C returns C. =back =head2 PERL -> JSON The mapping from Perl to JSON is slightly more difficult, as Perl is a truly typeless language, so we can only guess which JSON type is meant by a Perl value. =over 4 =item hash references Perl hash references become JSON objects. As there is no inherent ordering in hash keys (or JSON objects), they will usually be encoded in a pseudo-random order that can change between runs of the same program but stays generally the same within a single run of a program. C optionally sort the hash keys (determined by the I flag), so the same datastructure will serialise to the same JSON text (given same settings and version of JSON::XS), but this incurs a runtime overhead and is only rarely useful, e.g. when you want to compare some JSON text against another for equality. =item array references Perl array references become JSON arrays. =item other references Other unblessed references are generally not allowed and will cause an exception to be thrown, except for references to the integers C<0> and C<1>, which get turned into C and C atoms in JSON. You can also use C and C to improve readability. to_json [\0,JSON::PP::true] # yields [false,true] =item JSON::PP::true, JSON::PP::false, JSON::PP::null These special values become JSON true and JSON false values, respectively. You can also use C<\1> and C<\0> directly if you want. JSON::PP::null returns C. =item blessed objects Blessed objects are not directly representable in JSON. See the C and C methods on various options on how to deal with this: basically, you can choose between throwing an exception, encoding the reference as if it weren't blessed, or provide your own serialiser method. See to L. =item simple scalars Simple Perl scalars (any scalar that is not a reference) are the most difficult objects to encode: JSON::XS and JSON::PP will encode undefined scalars as JSON C values, scalars that have last been used in a string context before encoding as JSON strings, and anything else as number value: # dump as number encode_json [2] # yields [2] encode_json [-3.0e17] # yields [-3e+17] my $value = 5; encode_json [$value] # yields [5] # used as string, so dump as string print $value; encode_json [$value] # yields ["5"] # undef becomes null encode_json [undef] # yields [null] You can force the type to be a string by stringifying it: my $x = 3.1; # some variable containing a number "$x"; # stringified $x .= ""; # another, more awkward way to stringify print $x; # perl does it for you, too, quite often You can force the type to be a number by numifying it: my $x = "3"; # some variable containing a string $x += 0; # numify it, ensuring it will be dumped as a number $x *= 1; # same thing, the choise is yours. You can not currently force the type in other, less obscure, ways. Note that numerical precision has the same meaning as under Perl (so binary to decimal conversion follows the same rules as in Perl, which can differ to other languages). Also, your perl interpreter might expose extensions to the floating point numbers of your platform, such as infinities or NaN's - these cannot be represented in JSON, and it is an error to pass those in. =item Big Number When C is enable, C converts C objects and C objects into JSON numbers. =back =head1 UNICODE HANDLING ON PERLS If you do not know about Unicode on Perl well, please check L. =head2 Perl 5.8 and later Perl can handle Unicode and the JSON::PP de/encode methods also work properly. $json->allow_nonref->encode(chr hex 3042); $json->allow_nonref->encode(chr hex 12345); Reuturns C<"\u3042"> and C<"\ud808\udf45"> respectively. $json->allow_nonref->decode('"\u3042"'); $json->allow_nonref->decode('"\ud808\udf45"'); Returns UTF-8 encoded strings with UTF8 flag, regarded as C and C. Note that the versions from Perl 5.8.0 to 5.8.2, Perl built-in C was broken, so JSON::PP wraps the C with a subroutine. Thus JSON::PP works slow in the versions. =head2 Perl 5.6 Perl can handle Unicode and the JSON::PP de/encode methods also work. =head2 Perl 5.005 Perl 5.005 is a byte sementics world -- all strings are sequences of bytes. That means the unicode handling is not available. In encoding, $json->allow_nonref->encode(chr hex 3042); # hex 3042 is 12354. $json->allow_nonref->encode(chr hex 12345); # hex 12345 is 74565. Returns C and C, as C takes a value more than 255, it treats as C<$value % 256>, so the above codes are equivalent to : $json->allow_nonref->encode(chr 66); $json->allow_nonref->encode(chr 69); In decoding, $json->decode('"\u00e3\u0081\u0082"'); The returned is a byte sequence C<0xE3 0x81 0x82> for UTF-8 encoded japanese character (C). And if it is represented in Unicode code point, C. Next, $json->decode('"\u3042"'); We ordinary expect the returned value is a Unicode character C. But here is 5.005 world. This is C<0xE3 0x81 0x82>. $json->decode('"\ud808\udf45"'); This is not a character C but bytes - C<0xf0 0x92 0x8d 0x85>. =head1 TODO =over =item speed =item memory saving =back =head1 SEE ALSO Most of the document are copied and modified from JSON::XS doc. L RFC4627 (L) =head1 AUTHOR Makamaka Hannyaharamitu, Emakamaka[at]cpan.orgE =head1 COPYRIGHT AND LICENSE Copyright 2007-2011 by Makamaka Hannyaharamitu This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut JSON_BACKPORTPP $fatpacked{"JSON/backportPP/Boolean.pm"} = <<'JSON_BACKPORTPP_BOOLEAN'; =head1 NAME JSON::PP::Boolean - dummy module providing JSON::PP::Boolean =head1 SYNOPSIS # do not "use" yourself =head1 DESCRIPTION This module exists only to provide overload resolution for Storable and similar modules. See L for more info about this class. =cut use JSON::backportPP (); use strict; 1; =head1 AUTHOR This idea is from L written by Marc Lehmann =cut JSON_BACKPORTPP_BOOLEAN $fatpacked{"JSON/backportPP/Compat5005.pm"} = <<'JSON_BACKPORTPP_COMPAT5005'; package # This is JSON::backportPP JSON::backportPP5005; use 5.005; use strict; my @properties; $JSON::PP5005::VERSION = '1.10'; BEGIN { sub utf8::is_utf8 { 0; # It is considered that UTF8 flag off for Perl 5.005. } sub utf8::upgrade { } sub utf8::downgrade { 1; # must always return true. } sub utf8::encode { } sub utf8::decode { } *JSON::PP::JSON_PP_encode_ascii = \&_encode_ascii; *JSON::PP::JSON_PP_encode_latin1 = \&_encode_latin1; *JSON::PP::JSON_PP_decode_surrogates = \&_decode_surrogates; *JSON::PP::JSON_PP_decode_unicode = \&_decode_unicode; # missing in B module. sub B::SVp_IOK () { 0x01000000; } sub B::SVp_NOK () { 0x02000000; } sub B::SVp_POK () { 0x04000000; } $INC{'bytes.pm'} = 1; # dummy } sub _encode_ascii { join('', map { $_ <= 127 ? chr($_) : sprintf('\u%04x', $_) } unpack('C*', $_[0]) ); } sub _encode_latin1 { join('', map { chr($_) } unpack('C*', $_[0]) ); } sub _decode_surrogates { # from http://homepage1.nifty.com/nomenclator/unicode/ucs_utf.htm my $uni = 0x10000 + (hex($_[0]) - 0xD800) * 0x400 + (hex($_[1]) - 0xDC00); # from perlunicode my $bit = unpack('B32', pack('N', $uni)); if ( $bit =~ /^00000000000(...)(......)(......)(......)$/ ) { my ($w, $x, $y, $z) = ($1, $2, $3, $4); return pack('B*', sprintf('11110%s10%s10%s10%s', $w, $x, $y, $z)); } else { Carp::croak("Invalid surrogate pair"); } } sub _decode_unicode { my ($u) = @_; my ($utf8bit); if ( $u =~ /^00([89a-f][0-9a-f])$/i ) { # 0x80-0xff return pack( 'H2', $1 ); } my $bit = unpack("B*", pack("H*", $u)); if ( $bit =~ /^00000(.....)(......)$/ ) { $utf8bit = sprintf('110%s10%s', $1, $2); } elsif ( $bit =~ /^(....)(......)(......)$/ ) { $utf8bit = sprintf('1110%s10%s10%s', $1, $2, $3); } else { Carp::croak("Invalid escaped unicode"); } return pack('B*', $utf8bit); } sub JSON::PP::incr_text { $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new; if ( $_[0]->{_incr_parser}->{incr_parsing} ) { Carp::croak("incr_text can not be called when the incremental parser already started parsing"); } $_[0]->{_incr_parser}->{incr_text} = $_[1] if ( @_ > 1 ); $_[0]->{_incr_parser}->{incr_text}; } 1; __END__ =pod =head1 NAME JSON::PP5005 - Helper module in using JSON::PP in Perl 5.005 =head1 DESCRIPTION JSON::PP calls internally. =head1 AUTHOR Makamaka Hannyaharamitu, Emakamaka[at]cpan.orgE =head1 COPYRIGHT AND LICENSE Copyright 2007-2010 by Makamaka Hannyaharamitu This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut JSON_BACKPORTPP_COMPAT5005 $fatpacked{"JSON/backportPP/Compat5006.pm"} = <<'JSON_BACKPORTPP_COMPAT5006'; package # This is JSON::backportPP JSON::backportPP56; use 5.006; use strict; my @properties; $JSON::PP56::VERSION = '1.08'; BEGIN { sub utf8::is_utf8 { my $len = length $_[0]; # char length { use bytes; # byte length; return $len != length $_[0]; # if !=, UTF8-flagged on. } } sub utf8::upgrade { ; # noop; } sub utf8::downgrade ($;$) { return 1 unless ( utf8::is_utf8( $_[0] ) ); if ( _is_valid_utf8( $_[0] ) ) { my $downgrade; for my $c ( unpack( "U*", $_[0] ) ) { if ( $c < 256 ) { $downgrade .= pack("C", $c); } else { $downgrade .= pack("U", $c); } } $_[0] = $downgrade; return 1; } else { Carp::croak("Wide character in subroutine entry") unless ( $_[1] ); 0; } } sub utf8::encode ($) { # UTF8 flag off if ( utf8::is_utf8( $_[0] ) ) { $_[0] = pack( "C*", unpack( "C*", $_[0] ) ); } else { $_[0] = pack( "U*", unpack( "C*", $_[0] ) ); $_[0] = pack( "C*", unpack( "C*", $_[0] ) ); } } sub utf8::decode ($) { # UTF8 flag on if ( _is_valid_utf8( $_[0] ) ) { utf8::downgrade( $_[0] ); $_[0] = pack( "U*", unpack( "U*", $_[0] ) ); } } *JSON::PP::JSON_PP_encode_ascii = \&_encode_ascii; *JSON::PP::JSON_PP_encode_latin1 = \&_encode_latin1; *JSON::PP::JSON_PP_decode_surrogates = \&JSON::PP::_decode_surrogates; *JSON::PP::JSON_PP_decode_unicode = \&JSON::PP::_decode_unicode; unless ( defined &B::SVp_NOK ) { # missing in B module. eval q{ sub B::SVp_NOK () { 0x02000000; } }; } } sub _encode_ascii { join('', map { $_ <= 127 ? chr($_) : $_ <= 65535 ? sprintf('\u%04x', $_) : sprintf('\u%x\u%x', JSON::PP::_encode_surrogates($_)); } _unpack_emu($_[0]) ); } sub _encode_latin1 { join('', map { $_ <= 255 ? chr($_) : $_ <= 65535 ? sprintf('\u%04x', $_) : sprintf('\u%x\u%x', JSON::PP::_encode_surrogates($_)); } _unpack_emu($_[0]) ); } sub _unpack_emu { # for Perl 5.6 unpack warnings return !utf8::is_utf8($_[0]) ? unpack('C*', $_[0]) : _is_valid_utf8($_[0]) ? unpack('U*', $_[0]) : unpack('C*', $_[0]); } sub _is_valid_utf8 { my $str = $_[0]; my $is_utf8; while ($str =~ /(?: ( [\x00-\x7F] |[\xC2-\xDF][\x80-\xBF] |[\xE0][\xA0-\xBF][\x80-\xBF] |[\xE1-\xEC][\x80-\xBF][\x80-\xBF] |[\xED][\x80-\x9F][\x80-\xBF] |[\xEE-\xEF][\x80-\xBF][\x80-\xBF] |[\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF] |[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF] |[\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF] ) | (.) )/xg) { if (defined $1) { $is_utf8 = 1 if (!defined $is_utf8); } else { $is_utf8 = 0 if (!defined $is_utf8); if ($is_utf8) { # eventually, not utf8 return; } } } return $is_utf8; } 1; __END__ =pod =head1 NAME JSON::PP56 - Helper module in using JSON::PP in Perl 5.6 =head1 DESCRIPTION JSON::PP calls internally. =head1 AUTHOR Makamaka Hannyaharamitu, Emakamaka[at]cpan.orgE =head1 COPYRIGHT AND LICENSE Copyright 2007-2009 by Makamaka Hannyaharamitu This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut JSON_BACKPORTPP_COMPAT5006 $fatpacked{"Module/Metadata.pm"} = <<'MODULE_METADATA'; # -*- mode: cperl; tab-width: 8; indent-tabs-mode: nil; basic-offset: 2 -*- # vim:ts=8:sw=2:et:sta:sts=2 package Module::Metadata; # Adapted from Perl-licensed code originally distributed with # Module-Build by Ken Williams # This module provides routines to gather information about # perl modules (assuming this may be expanded in the distant # parrot future to look at other types of modules). use strict; use vars qw($VERSION); $VERSION = '1.000011'; $VERSION = eval $VERSION; use Carp qw/croak/; use File::Spec; use IO::File; use version 0.87; BEGIN { if ($INC{'Log/Contextual.pm'}) { Log::Contextual->import('log_info'); } else { *log_info = sub (&) { warn $_[0]->() }; } } use File::Find qw(find); my $V_NUM_REGEXP = qr{v?[0-9._]+}; # crudely, a v-string or decimal my $PKG_REGEXP = qr{ # match a package declaration ^[\s\{;]* # intro chars on a line package # the word 'package' \s+ # whitespace ([\w:]+) # a package name \s* # optional whitespace ($V_NUM_REGEXP)? # optional version number \s* # optional whitesapce [;\{] # semicolon line terminator or block start (since 5.16) }x; my $VARNAME_REGEXP = qr{ # match fully-qualified VERSION name ([\$*]) # sigil - $ or * ( ( # optional leading package name (?:::|\')? # possibly starting like just :: (Ì la $::VERSION) (?:\w+(?:::|\'))* # Foo::Bar:: ... )? VERSION )\b }x; my $VERS_REGEXP = qr{ # match a VERSION definition (?: \(\s*$VARNAME_REGEXP\s*\) # with parens | $VARNAME_REGEXP # without parens ) \s* =[^=~] # = but not ==, nor =~ }x; sub new_from_file { my $class = shift; my $filename = File::Spec->rel2abs( shift ); return undef unless defined( $filename ) && -f $filename; return $class->_init(undef, $filename, @_); } sub new_from_handle { my $class = shift; my $handle = shift; my $filename = shift; return undef unless defined($handle) && defined($filename); $filename = File::Spec->rel2abs( $filename ); return $class->_init(undef, $filename, @_, handle => $handle); } sub new_from_module { my $class = shift; my $module = shift; my %props = @_; $props{inc} ||= \@INC; my $filename = $class->find_module_by_name( $module, $props{inc} ); return undef unless defined( $filename ) && -f $filename; return $class->_init($module, $filename, %props); } { my $compare_versions = sub { my ($v1, $op, $v2) = @_; $v1 = version->new($v1) unless UNIVERSAL::isa($v1,'version'); my $eval_str = "\$v1 $op \$v2"; my $result = eval $eval_str; log_info { "error comparing versions: '$eval_str' $@" } if $@; return $result; }; my $normalize_version = sub { my ($version) = @_; if ( $version =~ /[=<>!,]/ ) { # logic, not just version # take as is without modification } elsif ( ref $version eq 'version' ) { # version objects $version = $version->is_qv ? $version->normal : $version->stringify; } elsif ( $version =~ /^[^v][^.]*\.[^.]+\./ ) { # no leading v, multiple dots # normalize string tuples without "v": "1.2.3" -> "v1.2.3" $version = "v$version"; } else { # leave alone } return $version; }; # separate out some of the conflict resolution logic my $resolve_module_versions = sub { my $packages = shift; my( $file, $version ); my $err = ''; foreach my $p ( @$packages ) { if ( defined( $p->{version} ) ) { if ( defined( $version ) ) { if ( $compare_versions->( $version, '!=', $p->{version} ) ) { $err .= " $p->{file} ($p->{version})\n"; } else { # same version declared multiple times, ignore } } else { $file = $p->{file}; $version = $p->{version}; } } $file ||= $p->{file} if defined( $p->{file} ); } if ( $err ) { $err = " $file ($version)\n" . $err; } my %result = ( file => $file, version => $version, err => $err ); return \%result; }; sub provides { my $class = shift; croak "provides() requires key/value pairs \n" if @_ % 2; my %args = @_; croak "provides() takes only one of 'dir' or 'files'\n" if $args{dir} && $args{files}; croak "provides() requires a 'version' argument" unless defined $args{version}; croak "provides() does not support version '$args{version}' metadata" unless grep { $args{version} eq $_ } qw/1.4 2/; $args{prefix} = 'lib' unless defined $args{prefix}; my $p; if ( $args{dir} ) { $p = $class->package_versions_from_directory($args{dir}); } else { croak "provides() requires 'files' to be an array reference\n" unless ref $args{files} eq 'ARRAY'; $p = $class->package_versions_from_directory($args{files}); } # Now, fix up files with prefix if ( length $args{prefix} ) { # check in case disabled with q{} $args{prefix} =~ s{/$}{}; for my $v ( values %$p ) { $v->{file} = "$args{prefix}/$v->{file}"; } } return $p } sub package_versions_from_directory { my ( $class, $dir, $files ) = @_; my @files; if ( $files ) { @files = @$files; } else { find( { wanted => sub { push @files, $_ if -f $_ && /\.pm$/; }, no_chdir => 1, }, $dir ); } # First, we enumerate all packages & versions, # separating into primary & alternative candidates my( %prime, %alt ); foreach my $file (@files) { my $mapped_filename = File::Spec::Unix->abs2rel( $file, $dir ); my @path = split( /\//, $mapped_filename ); (my $prime_package = join( '::', @path )) =~ s/\.pm$//; my $pm_info = $class->new_from_file( $file ); foreach my $package ( $pm_info->packages_inside ) { next if $package eq 'main'; # main can appear numerous times, ignore next if $package eq 'DB'; # special debugging package, ignore next if grep /^_/, split( /::/, $package ); # private package, ignore my $version = $pm_info->version( $package ); $prime_package = $package if lc($prime_package) eq lc($package); if ( $package eq $prime_package ) { if ( exists( $prime{$package} ) ) { croak "Unexpected conflict in '$package'; multiple versions found.\n"; } else { $mapped_filename = "$package.pm" if lc("$package.pm") eq lc($mapped_filename); $prime{$package}{file} = $mapped_filename; $prime{$package}{version} = $version if defined( $version ); } } else { push( @{$alt{$package}}, { file => $mapped_filename, version => $version, } ); } } } # Then we iterate over all the packages found above, identifying conflicts # and selecting the "best" candidate for recording the file & version # for each package. foreach my $package ( keys( %alt ) ) { my $result = $resolve_module_versions->( $alt{$package} ); if ( exists( $prime{$package} ) ) { # primary package selected if ( $result->{err} ) { # Use the selected primary package, but there are conflicting # errors among multiple alternative packages that need to be # reported log_info { "Found conflicting versions for package '$package'\n" . " $prime{$package}{file} ($prime{$package}{version})\n" . $result->{err} }; } elsif ( defined( $result->{version} ) ) { # There is a primary package selected, and exactly one # alternative package if ( exists( $prime{$package}{version} ) && defined( $prime{$package}{version} ) ) { # Unless the version of the primary package agrees with the # version of the alternative package, report a conflict if ( $compare_versions->( $prime{$package}{version}, '!=', $result->{version} ) ) { log_info { "Found conflicting versions for package '$package'\n" . " $prime{$package}{file} ($prime{$package}{version})\n" . " $result->{file} ($result->{version})\n" }; } } else { # The prime package selected has no version so, we choose to # use any alternative package that does have a version $prime{$package}{file} = $result->{file}; $prime{$package}{version} = $result->{version}; } } else { # no alt package found with a version, but we have a prime # package so we use it whether it has a version or not } } else { # No primary package was selected, use the best alternative if ( $result->{err} ) { log_info { "Found conflicting versions for package '$package'\n" . $result->{err} }; } # Despite possible conflicting versions, we choose to record # something rather than nothing $prime{$package}{file} = $result->{file}; $prime{$package}{version} = $result->{version} if defined( $result->{version} ); } } # Normalize versions. Can't use exists() here because of bug in YAML::Node. # XXX "bug in YAML::Node" comment seems irrelvant -- dagolden, 2009-05-18 for (grep defined $_->{version}, values %prime) { $_->{version} = $normalize_version->( $_->{version} ); } return \%prime; } } sub _init { my $class = shift; my $module = shift; my $filename = shift; my %props = @_; my $handle = delete $props{handle}; my( %valid_props, @valid_props ); @valid_props = qw( collect_pod inc ); @valid_props{@valid_props} = delete( @props{@valid_props} ); warn "Unknown properties: @{[keys %props]}\n" if scalar( %props ); my %data = ( module => $module, filename => $filename, version => undef, packages => [], versions => {}, pod => {}, pod_headings => [], collect_pod => 0, %valid_props, ); my $self = bless(\%data, $class); if ( $handle ) { $self->_parse_fh($handle); } else { $self->_parse_file(); } unless($self->{module} and length($self->{module})) { my ($v, $d, $f) = File::Spec->splitpath($self->{filename}); if($f =~ /\.pm$/) { $f =~ s/\..+$//; my @candidates = grep /$f$/, @{$self->{packages}}; $self->{module} = shift(@candidates); # punt } else { if(grep /main/, @{$self->{packages}}) { $self->{module} = 'main'; } else { $self->{module} = $self->{packages}[0] || ''; } } } $self->{version} = $self->{versions}{$self->{module}} if defined( $self->{module} ); return $self; } # class method sub _do_find_module { my $class = shift; my $module = shift || croak 'find_module_by_name() requires a package name'; my $dirs = shift || \@INC; my $file = File::Spec->catfile(split( /::/, $module)); foreach my $dir ( @$dirs ) { my $testfile = File::Spec->catfile($dir, $file); return [ File::Spec->rel2abs( $testfile ), $dir ] if -e $testfile and !-d _; # For stuff like ExtUtils::xsubpp return [ File::Spec->rel2abs( "$testfile.pm" ), $dir ] if -e "$testfile.pm"; } return; } # class method sub find_module_by_name { my $found = shift()->_do_find_module(@_) or return; return $found->[0]; } # class method sub find_module_dir_by_name { my $found = shift()->_do_find_module(@_) or return; return $found->[1]; } # given a line of perl code, attempt to parse it if it looks like a # $VERSION assignment, returning sigil, full name, & package name sub _parse_version_expression { my $self = shift; my $line = shift; my( $sig, $var, $pkg ); if ( $line =~ /$VERS_REGEXP/o ) { ( $sig, $var, $pkg ) = $2 ? ( $1, $2, $3 ) : ( $4, $5, $6 ); if ( $pkg ) { $pkg = ($pkg eq '::') ? 'main' : $pkg; $pkg =~ s/::$//; } } return ( $sig, $var, $pkg ); } sub _parse_file { my $self = shift; my $filename = $self->{filename}; my $fh = IO::File->new( $filename ) or croak( "Can't open '$filename': $!" ); $self->_handle_bom($fh, $filename); $self->_parse_fh($fh); } # Look for a UTF-8/UTF-16BE/UTF-16LE BOM at the beginning of the stream. # If there's one, then skip it and set the :encoding layer appropriately. sub _handle_bom { my ($self, $fh, $filename) = @_; my $pos = $fh->getpos; return unless defined $pos; my $buf = ' ' x 2; my $count = $fh->read( $buf, length $buf ); return unless defined $count and $count >= 2; my $encoding; if ( $buf eq "\x{FE}\x{FF}" ) { $encoding = 'UTF-16BE'; } elsif ( $buf eq "\x{FF}\x{FE}" ) { $encoding = 'UTF-16LE'; } elsif ( $buf eq "\x{EF}\x{BB}" ) { $buf = ' '; $count = $fh->read( $buf, length $buf ); if ( defined $count and $count >= 1 and $buf eq "\x{BF}" ) { $encoding = 'UTF-8'; } } if ( defined $encoding ) { if ( "$]" >= 5.008 ) { # $fh->binmode requires perl 5.10 binmode( $fh, ":encoding($encoding)" ); } } else { $fh->setpos($pos) or croak( sprintf "Can't reset position to the top of '$filename'" ); } return $encoding; } sub _parse_fh { my ($self, $fh) = @_; my( $in_pod, $seen_end, $need_vers ) = ( 0, 0, 0 ); my( @pkgs, %vers, %pod, @pod ); my $pkg = 'main'; my $pod_sect = ''; my $pod_data = ''; while (defined( my $line = <$fh> )) { my $line_num = $.; chomp( $line ); # From toke.c : any line that begins by "=X", where X is an alphabetic # character, introduces a POD segment. my $is_cut; if ( $line =~ /^=([a-zA-Z].*)/ ) { my $cmd = $1; # Then it goes back to Perl code for "=cutX" where X is a non-alphabetic # character (which includes the newline, but here we chomped it away). $is_cut = $cmd =~ /^cut(?:[^a-zA-Z]|$)/; $in_pod = !$is_cut; } if ( $in_pod ) { if ( $line =~ /^=head[1-4]\s+(.+)\s*$/ ) { push( @pod, $1 ); if ( $self->{collect_pod} && length( $pod_data ) ) { $pod{$pod_sect} = $pod_data; $pod_data = ''; } $pod_sect = $1; } elsif ( $self->{collect_pod} ) { $pod_data .= "$line\n"; } } elsif ( $is_cut ) { if ( $self->{collect_pod} && length( $pod_data ) ) { $pod{$pod_sect} = $pod_data; $pod_data = ''; } $pod_sect = ''; } else { # Skip comments in code next if $line =~ /^\s*#/; # Would be nice if we could also check $in_string or something too last if $line =~ /^__(?:DATA|END)__$/; # parse $line to see if it's a $VERSION declaration my( $vers_sig, $vers_fullname, $vers_pkg ) = ($line =~ /VERSION/) ? $self->_parse_version_expression( $line ) : (); if ( $line =~ /$PKG_REGEXP/o ) { $pkg = $1; push( @pkgs, $pkg ) unless grep( $pkg eq $_, @pkgs ); $vers{$pkg} = $2 unless exists( $vers{$pkg} ); $need_vers = defined $2 ? 0 : 1; # VERSION defined with full package spec, i.e. $Module::VERSION } elsif ( $vers_fullname && $vers_pkg ) { push( @pkgs, $vers_pkg ) unless grep( $vers_pkg eq $_, @pkgs ); $need_vers = 0 if $vers_pkg eq $pkg; unless ( defined $vers{$vers_pkg} && length $vers{$vers_pkg} ) { $vers{$vers_pkg} = $self->_evaluate_version_line( $vers_sig, $vers_fullname, $line ); } # first non-comment line in undeclared package main is VERSION } elsif ( !exists($vers{main}) && $pkg eq 'main' && $vers_fullname ) { $need_vers = 0; my $v = $self->_evaluate_version_line( $vers_sig, $vers_fullname, $line ); $vers{$pkg} = $v; push( @pkgs, 'main' ); # first non-comment line in undeclared package defines package main } elsif ( !exists($vers{main}) && $pkg eq 'main' && $line =~ /\w+/ ) { $need_vers = 1; $vers{main} = ''; push( @pkgs, 'main' ); # only keep if this is the first $VERSION seen } elsif ( $vers_fullname && $need_vers ) { $need_vers = 0; my $v = $self->_evaluate_version_line( $vers_sig, $vers_fullname, $line ); unless ( defined $vers{$pkg} && length $vers{$pkg} ) { $vers{$pkg} = $v; } } } } if ( $self->{collect_pod} && length($pod_data) ) { $pod{$pod_sect} = $pod_data; } $self->{versions} = \%vers; $self->{packages} = \@pkgs; $self->{pod} = \%pod; $self->{pod_headings} = \@pod; } { my $pn = 0; sub _evaluate_version_line { my $self = shift; my( $sigil, $var, $line ) = @_; # Some of this code came from the ExtUtils:: hierarchy. # We compile into $vsub because 'use version' would cause # compiletime/runtime issues with local() my $vsub; $pn++; # everybody gets their own package my $eval = qq{BEGIN { q# Hide from _packages_inside() #; package Module::Metadata::_version::p$pn; use version; no strict; \$vsub = sub { local $sigil$var; \$$var=undef; $line; \$$var }; }}; local $^W; # Try to get the $VERSION eval $eval; # some modules say $VERSION = $Foo::Bar::VERSION, but Foo::Bar isn't # installed, so we need to hunt in ./lib for it if ( $@ =~ /Can't locate/ && -d 'lib' ) { local @INC = ('lib',@INC); eval $eval; } warn "Error evaling version line '$eval' in $self->{filename}: $@\n" if $@; (ref($vsub) eq 'CODE') or croak "failed to build version sub for $self->{filename}"; my $result = eval { $vsub->() }; croak "Could not get version from $self->{filename} by executing:\n$eval\n\nThe fatal error was: $@\n" if $@; # Upgrade it into a version object my $version = eval { _dwim_version($result) }; croak "Version '$result' from $self->{filename} does not appear to be valid:\n$eval\n\nThe fatal error was: $@\n" unless defined $version; # "0" is OK! return $version; } } # Try to DWIM when things fail the lax version test in obvious ways { my @version_prep = ( # Best case, it just works sub { return shift }, # If we still don't have a version, try stripping any # trailing junk that is prohibited by lax rules sub { my $v = shift; $v =~ s{([0-9])[a-z-].*$}{$1}i; # 1.23-alpha or 1.23b return $v; }, # Activestate apparently creates custom versions like '1.23_45_01', which # cause version.pm to think it's an invalid alpha. So check for that # and strip them sub { my $v = shift; my $num_dots = () = $v =~ m{(\.)}g; my $num_unders = () = $v =~ m{(_)}g; my $leading_v = substr($v,0,1) eq 'v'; if ( ! $leading_v && $num_dots < 2 && $num_unders > 1 ) { $v =~ s{_}{}g; $num_unders = () = $v =~ m{(_)}g; } return $v; }, # Worst case, try numifying it like we would have before version objects sub { my $v = shift; no warnings 'numeric'; return 0 + $v; }, ); sub _dwim_version { my ($result) = shift; return $result if ref($result) eq 'version'; my ($version, $error); for my $f (@version_prep) { $result = $f->($result); $version = eval { version->new($result) }; $error ||= $@ if $@; # capture first failure last if defined $version; } croak $error unless defined $version; return $version; } } ############################################################ # accessors sub name { $_[0]->{module} } sub filename { $_[0]->{filename} } sub packages_inside { @{$_[0]->{packages}} } sub pod_inside { @{$_[0]->{pod_headings}} } sub contains_pod { $#{$_[0]->{pod_headings}} } sub version { my $self = shift; my $mod = shift || $self->{module}; my $vers; if ( defined( $mod ) && length( $mod ) && exists( $self->{versions}{$mod} ) ) { return $self->{versions}{$mod}; } else { return undef; } } sub pod { my $self = shift; my $sect = shift; if ( defined( $sect ) && length( $sect ) && exists( $self->{pod}{$sect} ) ) { return $self->{pod}{$sect}; } else { return undef; } } 1; =head1 NAME Module::Metadata - Gather package and POD information from perl module files =head1 SYNOPSIS use Module::Metadata; # information about a .pm file my $info = Module::Metadata->new_from_file( $file ); my $version = $info->version; # CPAN META 'provides' field for .pm files in a directory my $provides = Module::Metadata->provides( dir => 'lib', version => 2 ); =head1 DESCRIPTION This module provides a standard way to gather metadata about a .pm file without executing unsafe code. =head1 USAGE =head2 Class methods =over 4 =item C<< new_from_file($filename, collect_pod => 1) >> Constructs a C object given the path to a file. Returns undef if the filename does not exist. C is a optional boolean argument that determines whether POD data is collected and stored for reference. POD data is not collected by default. POD headings are always collected. If the file begins by an UTF-8, UTF-16BE or UTF-16LE byte-order mark, then it is skipped before processing, and the content of the file is also decoded appropriately starting from perl 5.8. =item C<< new_from_handle($handle, $filename, collect_pod => 1) >> This works just like C, except that a handle can be provided as the first argument. Note that there is no validation to confirm that the handle is a handle or something that can act like one. Passing something that isn't a handle will cause a exception when trying to read from it. The C argument is mandatory or undef will be returned. You are responsible for setting the decoding layers on C<$handle> if required. =item C<< new_from_module($module, collect_pod => 1, inc => \@dirs) >> Constructs a C object given a module or package name. Returns undef if the module cannot be found. In addition to accepting the C argument as described above, this method accepts a C argument which is a reference to an array of directories to search for the module. If none are given, the default is @INC. If the file that contains the module begins by an UTF-8, UTF-16BE or UTF-16LE byte-order mark, then it is skipped before processing, and the content of the file is also decoded appropriately starting from perl 5.8. =item C<< find_module_by_name($module, \@dirs) >> Returns the path to a module given the module or package name. A list of directories can be passed in as an optional parameter, otherwise @INC is searched. Can be called as either an object or a class method. =item C<< find_module_dir_by_name($module, \@dirs) >> Returns the entry in C<@dirs> (or C<@INC> by default) that contains the module C<$module>. A list of directories can be passed in as an optional parameter, otherwise @INC is searched. Can be called as either an object or a class method. =item C<< provides( %options ) >> This is a convenience wrapper around C to generate a CPAN META C data structure. It takes key/value pairs. Valid option keys include: =over =item version B<(required)> Specifies which version of the L should be used as the format of the C output. Currently only '1.4' and '2' are supported (and their format is identical). This may change in the future as the definition of C changes. The C option is required. If it is omitted or if an unsupported version is given, then C will throw an error. =item dir Directory to search recursively for F<.pm> files. May not be specified with C. =item files Array reference of files to examine. May not be specified with C. =item prefix String to prepend to the C field of the resulting output. This defaults to F, which is the common case for most CPAN distributions with their F<.pm> files in F. This option ensures the META information has the correct relative path even when the C or C arguments are absolute or have relative paths from a location other than the distribution root. =back For example, given C of 'lib' and C of 'lib', the return value is a hashref of the form: { 'Package::Name' => { version => '0.123', file => 'lib/Package/Name.pm' }, 'OtherPackage::Name' => ... } =item C<< package_versions_from_directory($dir, \@files?) >> Scans C<$dir> for .pm files (unless C<@files> is given, in which case looks for those files in C<$dir> - and reads each file for packages and versions, returning a hashref of the form: { 'Package::Name' => { version => '0.123', file => 'Package/Name.pm' }, 'OtherPackage::Name' => ... } The C and C
packages are always omitted, as are any "private" packages that have leading underscores in the namespace (e.g. C) Note that the file path is relative to C<$dir> if that is specified. This B be used directly for CPAN META C. See the C method instead. =item C<< log_info (internal) >> Used internally to perform logging; imported from Log::Contextual if Log::Contextual has already been loaded, otherwise simply calls warn. =back =head2 Object methods =over 4 =item C<< name() >> Returns the name of the package represented by this module. If there are more than one packages, it makes a best guess based on the filename. If it's a script (i.e. not a *.pm) the package name is 'main'. =item C<< version($package) >> Returns the version as defined by the $VERSION variable for the package as returned by the C method if no arguments are given. If given the name of a package it will attempt to return the version of that package if it is specified in the file. =item C<< filename() >> Returns the absolute path to the file. =item C<< packages_inside() >> Returns a list of packages. Note: this is a raw list of packages discovered (or assumed, in the case of C
). It is not filtered for C, C
or private packages the way the C method does. =item C<< pod_inside() >> Returns a list of POD sections. =item C<< contains_pod() >> Returns true if there is any POD in the file. =item C<< pod($section) >> Returns the POD data in the given section. =back =head1 AUTHOR Original code from Module::Build::ModuleInfo by Ken Williams , Randy W. Sims Released as Module::Metadata by Matt S Trout (mst) with assistance from David Golden (xdg) . =head1 COPYRIGHT & LICENSE Original code Copyright (c) 2001-2011 Ken Williams. Additional code Copyright (c) 2010-2011 Matt Trout and David Golden. All rights reserved. This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut MODULE_METADATA $fatpacked{"version.pm"} = <<'VERSION'; #!perl -w package version; use 5.005_04; use strict; use vars qw(@ISA $VERSION $CLASS $STRICT $LAX *declare *qv); $VERSION = 0.9901; $CLASS = 'version'; #--------------------------------------------------------------------------# # Version regexp components #--------------------------------------------------------------------------# # Fraction part of a decimal version number. This is a common part of # both strict and lax decimal versions my $FRACTION_PART = qr/\.[0-9]+/; # First part of either decimal or dotted-decimal strict version number. # Unsigned integer with no leading zeroes (except for zero itself) to # avoid confusion with octal. my $STRICT_INTEGER_PART = qr/0|[1-9][0-9]*/; # First part of either decimal or dotted-decimal lax version number. # Unsigned integer, but allowing leading zeros. Always interpreted # as decimal. However, some forms of the resulting syntax give odd # results if used as ordinary Perl expressions, due to how perl treats # octals. E.g. # version->new("010" ) == 10 # version->new( 010 ) == 8 # version->new( 010.2) == 82 # "8" . "2" my $LAX_INTEGER_PART = qr/[0-9]+/; # Second and subsequent part of a strict dotted-decimal version number. # Leading zeroes are permitted, and the number is always decimal. # Limited to three digits to avoid overflow when converting to decimal # form and also avoid problematic style with excessive leading zeroes. my $STRICT_DOTTED_DECIMAL_PART = qr/\.[0-9]{1,3}/; # Second and subsequent part of a lax dotted-decimal version number. # Leading zeroes are permitted, and the number is always decimal. No # limit on the numerical value or number of digits, so there is the # possibility of overflow when converting to decimal form. my $LAX_DOTTED_DECIMAL_PART = qr/\.[0-9]+/; # Alpha suffix part of lax version number syntax. Acts like a # dotted-decimal part. my $LAX_ALPHA_PART = qr/_[0-9]+/; #--------------------------------------------------------------------------# # Strict version regexp definitions #--------------------------------------------------------------------------# # Strict decimal version number. my $STRICT_DECIMAL_VERSION = qr/ $STRICT_INTEGER_PART $FRACTION_PART? /x; # Strict dotted-decimal version number. Must have both leading "v" and # at least three parts, to avoid confusion with decimal syntax. my $STRICT_DOTTED_DECIMAL_VERSION = qr/ v $STRICT_INTEGER_PART $STRICT_DOTTED_DECIMAL_PART{2,} /x; # Complete strict version number syntax -- should generally be used # anchored: qr/ \A $STRICT \z /x $STRICT = qr/ $STRICT_DECIMAL_VERSION | $STRICT_DOTTED_DECIMAL_VERSION /x; #--------------------------------------------------------------------------# # Lax version regexp definitions #--------------------------------------------------------------------------# # Lax decimal version number. Just like the strict one except for # allowing an alpha suffix or allowing a leading or trailing # decimal-point my $LAX_DECIMAL_VERSION = qr/ $LAX_INTEGER_PART (?: \. | $FRACTION_PART $LAX_ALPHA_PART? )? | $FRACTION_PART $LAX_ALPHA_PART? /x; # Lax dotted-decimal version number. Distinguished by having either # leading "v" or at least three non-alpha parts. Alpha part is only # permitted if there are at least two non-alpha parts. Strangely # enough, without the leading "v", Perl takes .1.2 to mean v0.1.2, # so when there is no "v", the leading part is optional my $LAX_DOTTED_DECIMAL_VERSION = qr/ v $LAX_INTEGER_PART (?: $LAX_DOTTED_DECIMAL_PART+ $LAX_ALPHA_PART? )? | $LAX_INTEGER_PART? $LAX_DOTTED_DECIMAL_PART{2,} $LAX_ALPHA_PART? /x; # Complete lax version number syntax -- should generally be used # anchored: qr/ \A $LAX \z /x # # The string 'undef' is a special case to make for easier handling # of return values from ExtUtils::MM->parse_version $LAX = qr/ undef | $LAX_DECIMAL_VERSION | $LAX_DOTTED_DECIMAL_VERSION /x; #--------------------------------------------------------------------------# { local $SIG{'__DIE__'}; eval "use version::vxs $VERSION"; if ( $@ ) { # don't have the XS version installed eval "use version::vpp $VERSION"; # don't tempt fate die "$@" if ( $@ ); push @ISA, "version::vpp"; local $^W; *version::qv = \&version::vpp::qv; *version::declare = \&version::vpp::declare; *version::_VERSION = \&version::vpp::_VERSION; *version::vcmp = \&version::vpp::vcmp; if ($] >= 5.009000) { no strict 'refs'; *version::stringify = \&version::vpp::stringify; *{'version::(""'} = \&version::vpp::stringify; *{'version::(<=>'} = \&version::vpp::vcmp; *version::new = \&version::vpp::new; *version::parse = \&version::vpp::parse; } } else { # use XS module push @ISA, "version::vxs"; local $^W; *version::declare = \&version::vxs::declare; *version::qv = \&version::vxs::qv; *version::_VERSION = \&version::vxs::_VERSION; *version::vcmp = \&version::vxs::VCMP; if ($] >= 5.009000) { no strict 'refs'; *version::stringify = \&version::vxs::stringify; *{'version::(""'} = \&version::vxs::stringify; *{'version::(<=>'} = \&version::vxs::VCMP; *version::new = \&version::vxs::new; *version::parse = \&version::vxs::parse; } } } # Preloaded methods go here. sub import { no strict 'refs'; my ($class) = shift; # Set up any derived class unless ($class eq 'version') { local $^W; *{$class.'::declare'} = \&version::declare; *{$class.'::qv'} = \&version::qv; } my %args; if (@_) { # any remaining terms are arguments map { $args{$_} = 1 } @_ } else { # no parameters at all on use line %args = ( qv => 1, 'UNIVERSAL::VERSION' => 1, ); } my $callpkg = caller(); if (exists($args{declare})) { *{$callpkg.'::declare'} = sub {return $class->declare(shift) } unless defined(&{$callpkg.'::declare'}); } if (exists($args{qv})) { *{$callpkg.'::qv'} = sub {return $class->qv(shift) } unless defined(&{$callpkg.'::qv'}); } if (exists($args{'UNIVERSAL::VERSION'})) { local $^W; *UNIVERSAL::VERSION = \&version::_VERSION; } if (exists($args{'VERSION'})) { *{$callpkg.'::VERSION'} = \&version::_VERSION; } if (exists($args{'is_strict'})) { *{$callpkg.'::is_strict'} = \&version::is_strict unless defined(&{$callpkg.'::is_strict'}); } if (exists($args{'is_lax'})) { *{$callpkg.'::is_lax'} = \&version::is_lax unless defined(&{$callpkg.'::is_lax'}); } } sub is_strict { defined $_[0] && $_[0] =~ qr/ \A $STRICT \z /x } sub is_lax { defined $_[0] && $_[0] =~ qr/ \A $LAX \z /x } 1; VERSION $fatpacked{"version/vpp.pm"} = <<'VERSION_VPP'; package charstar; # a little helper class to emulate C char* semantics in Perl # so that prescan_version can use the same code as in C use overload ( '""' => \&thischar, '0+' => \&thischar, '++' => \&increment, '--' => \&decrement, '+' => \&plus, '-' => \&minus, '*' => \&multiply, 'cmp' => \&cmp, '<=>' => \&spaceship, 'bool' => \&thischar, '=' => \&clone, ); sub new { my ($self, $string) = @_; my $class = ref($self) || $self; my $obj = { string => [split(//,$string)], current => 0, }; return bless $obj, $class; } sub thischar { my ($self) = @_; my $last = $#{$self->{string}}; my $curr = $self->{current}; if ($curr >= 0 && $curr <= $last) { return $self->{string}->[$curr]; } else { return ''; } } sub increment { my ($self) = @_; $self->{current}++; } sub decrement { my ($self) = @_; $self->{current}--; } sub plus { my ($self, $offset) = @_; my $rself = $self->clone; $rself->{current} += $offset; return $rself; } sub minus { my ($self, $offset) = @_; my $rself = $self->clone; $rself->{current} -= $offset; return $rself; } sub multiply { my ($left, $right, $swapped) = @_; my $char = $left->thischar(); return $char * $right; } sub spaceship { my ($left, $right, $swapped) = @_; unless (ref($right)) { # not an object already $right = $left->new($right); } return $left->{current} <=> $right->{current}; } sub cmp { my ($left, $right, $swapped) = @_; unless (ref($right)) { # not an object already if (length($right) == 1) { # comparing single character only return $left->thischar cmp $right; } $right = $left->new($right); } return $left->currstr cmp $right->currstr; } sub bool { my ($self) = @_; my $char = $self->thischar; return ($char ne ''); } sub clone { my ($left, $right, $swapped) = @_; $right = { string => [@{$left->{string}}], current => $left->{current}, }; return bless $right, ref($left); } sub currstr { my ($self, $s) = @_; my $curr = $self->{current}; my $last = $#{$self->{string}}; if (defined($s) && $s->{current} < $last) { $last = $s->{current}; } my $string = join('', @{$self->{string}}[$curr..$last]); return $string; } package version::vpp; use strict; use POSIX qw/locale_h/; use locale; use vars qw ($VERSION @ISA @REGEXS); $VERSION = 0.9901; use overload ( '""' => \&stringify, '0+' => \&numify, 'cmp' => \&vcmp, '<=>' => \&vcmp, 'bool' => \&vbool, '+' => \&vnoop, '-' => \&vnoop, '*' => \&vnoop, '/' => \&vnoop, '+=' => \&vnoop, '-=' => \&vnoop, '*=' => \&vnoop, '/=' => \&vnoop, 'abs' => \&vnoop, ); eval "use warnings"; if ($@) { eval ' package warnings; sub enabled {return $^W;} 1; '; } my $VERSION_MAX = 0x7FFFFFFF; # implement prescan_version as closely to the C version as possible use constant TRUE => 1; use constant FALSE => 0; sub isDIGIT { my ($char) = shift->thischar(); return ($char =~ /\d/); } sub isALPHA { my ($char) = shift->thischar(); return ($char =~ /[a-zA-Z]/); } sub isSPACE { my ($char) = shift->thischar(); return ($char =~ /\s/); } sub BADVERSION { my ($s, $errstr, $error) = @_; if ($errstr) { $$errstr = $error; } return $s; } sub prescan_version { my ($s, $strict, $errstr, $sqv, $ssaw_decimal, $swidth, $salpha) = @_; my $qv = defined $sqv ? $$sqv : FALSE; my $saw_decimal = defined $ssaw_decimal ? $$ssaw_decimal : 0; my $width = defined $swidth ? $$swidth : 3; my $alpha = defined $salpha ? $$salpha : FALSE; my $d = $s; if ($qv && isDIGIT($d)) { goto dotted_decimal_version; } if ($d eq 'v') { # explicit v-string $d++; if (isDIGIT($d)) { $qv = TRUE; } else { # degenerate v-string # requires v1.2.3 return BADVERSION($s,$errstr,"Invalid version format (dotted-decimal versions require at least three parts)"); } dotted_decimal_version: if ($strict && $d eq '0' && isDIGIT($d+1)) { # no leading zeros allowed return BADVERSION($s,$errstr,"Invalid version format (no leading zeros)"); } while (isDIGIT($d)) { # integer part $d++; } if ($d eq '.') { $saw_decimal++; $d++; # decimal point } else { if ($strict) { # require v1.2.3 return BADVERSION($s,$errstr,"Invalid version format (dotted-decimal versions require at least three parts)"); } else { goto version_prescan_finish; } } { my $i = 0; my $j = 0; while (isDIGIT($d)) { # just keep reading $i++; while (isDIGIT($d)) { $d++; $j++; # maximum 3 digits between decimal if ($strict && $j > 3) { return BADVERSION($s,$errstr,"Invalid version format (maximum 3 digits between decimals)"); } } if ($d eq '_') { if ($strict) { return BADVERSION($s,$errstr,"Invalid version format (no underscores)"); } if ( $alpha ) { return BADVERSION($s,$errstr,"Invalid version format (multiple underscores)"); } $d++; $alpha = TRUE; } elsif ($d eq '.') { if ($alpha) { return BADVERSION($s,$errstr,"Invalid version format (underscores before decimal)"); } $saw_decimal++; $d++; } elsif (!isDIGIT($d)) { last; } $j = 0; } if ($strict && $i < 2) { # requires v1.2.3 return BADVERSION($s,$errstr,"Invalid version format (dotted-decimal versions require at least three parts)"); } } } # end if dotted-decimal else { # decimal versions my $j = 0; # special $strict case for leading '.' or '0' if ($strict) { if ($d eq '.') { return BADVERSION($s,$errstr,"Invalid version format (0 before decimal required)"); } if ($d eq '0' && isDIGIT($d+1)) { return BADVERSION($s,$errstr,"Invalid version format (no leading zeros)"); } } # and we never support negative version numbers if ($d eq '-') { return BADVERSION($s,$errstr,"Invalid version format (negative version number)"); } # consume all of the integer part while (isDIGIT($d)) { $d++; } # look for a fractional part if ($d eq '.') { # we found it, so consume it $saw_decimal++; $d++; } elsif (!$d || $d eq ';' || isSPACE($d) || $d eq '}') { if ( $d == $s ) { # found nothing return BADVERSION($s,$errstr,"Invalid version format (version required)"); } # found just an integer goto version_prescan_finish; } elsif ( $d == $s ) { # didn't find either integer or period return BADVERSION($s,$errstr,"Invalid version format (non-numeric data)"); } elsif ($d eq '_') { # underscore can't come after integer part if ($strict) { return BADVERSION($s,$errstr,"Invalid version format (no underscores)"); } elsif (isDIGIT($d+1)) { return BADVERSION($s,$errstr,"Invalid version format (alpha without decimal)"); } else { return BADVERSION($s,$errstr,"Invalid version format (misplaced underscore)"); } } elsif ($d) { # anything else after integer part is just invalid data return BADVERSION($s,$errstr,"Invalid version format (non-numeric data)"); } # scan the fractional part after the decimal point if ($d && !isDIGIT($d) && ($strict || ! ($d eq ';' || isSPACE($d) || $d eq '}') )) { # $strict or lax-but-not-the-end return BADVERSION($s,$errstr,"Invalid version format (fractional part required)"); } while (isDIGIT($d)) { $d++; $j++; if ($d eq '.' && isDIGIT($d-1)) { if ($alpha) { return BADVERSION($s,$errstr,"Invalid version format (underscores before decimal)"); } if ($strict) { return BADVERSION($s,$errstr,"Invalid version format (dotted-decimal versions must begin with 'v')"); } $d = $s; # start all over again $qv = TRUE; goto dotted_decimal_version; } if ($d eq '_') { if ($strict) { return BADVERSION($s,$errstr,"Invalid version format (no underscores)"); } if ( $alpha ) { return BADVERSION($s,$errstr,"Invalid version format (multiple underscores)"); } if ( ! isDIGIT($d+1) ) { return BADVERSION($s,$errstr,"Invalid version format (misplaced underscore)"); } $width = $j; $d++; $alpha = TRUE; } } } version_prescan_finish: while (isSPACE($d)) { $d++; } if ($d && !isDIGIT($d) && (! ($d eq ';' || $d eq '}') )) { # trailing non-numeric data return BADVERSION($s,$errstr,"Invalid version format (non-numeric data)"); } if (defined $sqv) { $$sqv = $qv; } if (defined $swidth) { $$swidth = $width; } if (defined $ssaw_decimal) { $$ssaw_decimal = $saw_decimal; } if (defined $salpha) { $$salpha = $alpha; } return $d; } sub scan_version { my ($s, $rv, $qv) = @_; my $start; my $pos; my $last; my $errstr; my $saw_decimal = 0; my $width = 3; my $alpha = FALSE; my $vinf = FALSE; my @av; $s = new charstar $s; while (isSPACE($s)) { # leading whitespace is OK $s++; } $last = prescan_version($s, FALSE, \$errstr, \$qv, \$saw_decimal, \$width, \$alpha); if ($errstr) { # 'undef' is a special case and not an error if ( $s ne 'undef') { use Carp; Carp::croak($errstr); } } $start = $s; if ($s eq 'v') { $s++; } $pos = $s; if ( $qv ) { $$rv->{qv} = $qv; } if ( $alpha ) { $$rv->{alpha} = $alpha; } if ( !$qv && $width < 3 ) { $$rv->{width} = $width; } while (isDIGIT($pos)) { $pos++; } if (!isALPHA($pos)) { my $rev; for (;;) { $rev = 0; { # this is atoi() that delimits on underscores my $end = $pos; my $mult = 1; my $orev; # the following if() will only be true after the decimal # point of a version originally created with a bare # floating point number, i.e. not quoted in any way # if ( !$qv && $s > $start && $saw_decimal == 1 ) { $mult *= 100; while ( $s < $end ) { $orev = $rev; $rev += $s * $mult; $mult /= 10; if ( (abs($orev) > abs($rev)) || (abs($rev) > $VERSION_MAX )) { warn("Integer overflow in version %d", $VERSION_MAX); $s = $end - 1; $rev = $VERSION_MAX; $vinf = 1; } $s++; if ( $s eq '_' ) { $s++; } } } else { while (--$end >= $s) { $orev = $rev; $rev += $end * $mult; $mult *= 10; if ( (abs($orev) > abs($rev)) || (abs($rev) > $VERSION_MAX )) { warn("Integer overflow in version"); $end = $s - 1; $rev = $VERSION_MAX; $vinf = 1; } } } } # Append revision push @av, $rev; if ( $vinf ) { $s = $last; last; } elsif ( $pos eq '.' ) { $s = ++$pos; } elsif ( $pos eq '_' && isDIGIT($pos+1) ) { $s = ++$pos; } elsif ( $pos eq ',' && isDIGIT($pos+1) ) { $s = ++$pos; } elsif ( isDIGIT($pos) ) { $s = $pos; } else { $s = $pos; last; } if ( $qv ) { while ( isDIGIT($pos) ) { $pos++; } } else { my $digits = 0; while ( ( isDIGIT($pos) || $pos eq '_' ) && $digits < 3 ) { if ( $pos ne '_' ) { $digits++; } $pos++; } } } } if ( $qv ) { # quoted versions always get at least three terms my $len = $#av; # This for loop appears to trigger a compiler bug on OS X, as it # loops infinitely. Yes, len is negative. No, it makes no sense. # Compiler in question is: # gcc version 3.3 20030304 (Apple Computer, Inc. build 1640) # for ( len = 2 - len; len > 0; len-- ) # av_push(MUTABLE_AV(sv), newSViv(0)); # $len = 2 - $len; while ($len-- > 0) { push @av, 0; } } # need to save off the current version string for later if ( $vinf ) { $$rv->{original} = "v.Inf"; $$rv->{vinf} = 1; } elsif ( $s > $start ) { $$rv->{original} = $start->currstr($s); if ( $qv && $saw_decimal == 1 && $start ne 'v' ) { # need to insert a v to be consistent $$rv->{original} = 'v' . $$rv->{original}; } } else { $$rv->{original} = '0'; push(@av, 0); } # And finally, store the AV in the hash $$rv->{version} = \@av; # fix RT#19517 - special case 'undef' as string if ($s eq 'undef') { $s += 5; } return $s; } sub new { my ($class, $value) = @_; my $self = bless ({}, ref ($class) || $class); my $qv = FALSE; if ( ref($value) && eval('$value->isa("version")') ) { # Can copy the elements directly $self->{version} = [ @{$value->{version} } ]; $self->{qv} = 1 if $value->{qv}; $self->{alpha} = 1 if $value->{alpha}; $self->{original} = ''.$value->{original}; return $self; } my $currlocale = setlocale(LC_ALL); # if the current locale uses commas for decimal points, we # just replace commas with decimal places, rather than changing # locales if ( localeconv()->{decimal_point} eq ',' ) { $value =~ tr/,/./; } if ( not defined $value or $value =~ /^undef$/ ) { # RT #19517 - special case for undef comparison # or someone forgot to pass a value push @{$self->{version}}, 0; $self->{original} = "0"; return ($self); } if ( $#_ == 2 ) { # must be CVS-style $value = $_[2]; $qv = TRUE; } $value = _un_vstring($value); # exponential notation if ( $value =~ /\d+.?\d*e[-+]?\d+/ ) { $value = sprintf("%.9f",$value); $value =~ s/(0+)$//; # trim trailing zeros } my $s = scan_version($value, \$self, $qv); if ($s) { # must be something left over warn("Version string '%s' contains invalid data; " ."ignoring: '%s'", $value, $s); } return ($self); } *parse = \&new; sub numify { my ($self) = @_; unless (_verify($self)) { require Carp; Carp::croak("Invalid version object"); } my $width = $self->{width} || 3; my $alpha = $self->{alpha} || ""; my $len = $#{$self->{version}}; my $digit = $self->{version}[0]; my $string = sprintf("%d.", $digit ); for ( my $i = 1 ; $i < $len ; $i++ ) { $digit = $self->{version}[$i]; if ( $width < 3 ) { my $denom = 10**(3-$width); my $quot = int($digit/$denom); my $rem = $digit - ($quot * $denom); $string .= sprintf("%0".$width."d_%d", $quot, $rem); } else { $string .= sprintf("%03d", $digit); } } if ( $len > 0 ) { $digit = $self->{version}[$len]; if ( $alpha && $width == 3 ) { $string .= "_"; } $string .= sprintf("%0".$width."d", $digit); } else # $len = 0 { $string .= sprintf("000"); } return $string; } sub normal { my ($self) = @_; unless (_verify($self)) { require Carp; Carp::croak("Invalid version object"); } my $alpha = $self->{alpha} || ""; my $len = $#{$self->{version}}; my $digit = $self->{version}[0]; my $string = sprintf("v%d", $digit ); for ( my $i = 1 ; $i < $len ; $i++ ) { $digit = $self->{version}[$i]; $string .= sprintf(".%d", $digit); } if ( $len > 0 ) { $digit = $self->{version}[$len]; if ( $alpha ) { $string .= sprintf("_%0d", $digit); } else { $string .= sprintf(".%0d", $digit); } } if ( $len <= 2 ) { for ( $len = 2 - $len; $len != 0; $len-- ) { $string .= sprintf(".%0d", 0); } } return $string; } sub stringify { my ($self) = @_; unless (_verify($self)) { require Carp; Carp::croak("Invalid version object"); } return exists $self->{original} ? $self->{original} : exists $self->{qv} ? $self->normal : $self->numify; } sub vcmp { require UNIVERSAL; my ($left,$right,$swap) = @_; my $class = ref($left); unless ( UNIVERSAL::isa($right, $class) ) { $right = $class->new($right); } if ( $swap ) { ($left, $right) = ($right, $left); } unless (_verify($left)) { require Carp; Carp::croak("Invalid version object"); } unless (_verify($right)) { require Carp; Carp::croak("Invalid version format"); } my $l = $#{$left->{version}}; my $r = $#{$right->{version}}; my $m = $l < $r ? $l : $r; my $lalpha = $left->is_alpha; my $ralpha = $right->is_alpha; my $retval = 0; my $i = 0; while ( $i <= $m && $retval == 0 ) { $retval = $left->{version}[$i] <=> $right->{version}[$i]; $i++; } # tiebreaker for alpha with identical terms if ( $retval == 0 && $l == $r && $left->{version}[$m] == $right->{version}[$m] && ( $lalpha || $ralpha ) ) { if ( $lalpha && !$ralpha ) { $retval = -1; } elsif ( $ralpha && !$lalpha) { $retval = +1; } } # possible match except for trailing 0's if ( $retval == 0 && $l != $r ) { if ( $l < $r ) { while ( $i <= $r && $retval == 0 ) { if ( $right->{version}[$i] != 0 ) { $retval = -1; # not a match after all } $i++; } } else { while ( $i <= $l && $retval == 0 ) { if ( $left->{version}[$i] != 0 ) { $retval = +1; # not a match after all } $i++; } } } return $retval; } sub vbool { my ($self) = @_; return vcmp($self,$self->new("0"),1); } sub vnoop { require Carp; Carp::croak("operation not supported with version object"); } sub is_alpha { my ($self) = @_; return (exists $self->{alpha}); } sub qv { my $value = shift; my $class = 'version'; if (@_) { $class = ref($value) || $value; $value = shift; } $value = _un_vstring($value); $value = 'v'.$value unless $value =~ /(^v|\d+\.\d+\.\d)/; my $obj = version->new($value); return bless $obj, $class; } *declare = \&qv; sub is_qv { my ($self) = @_; return (exists $self->{qv}); } sub _verify { my ($self) = @_; if ( ref($self) && eval { exists $self->{version} } && ref($self->{version}) eq 'ARRAY' ) { return 1; } else { return 0; } } sub _is_non_alphanumeric { my $s = shift; $s = new charstar $s; while ($s) { return 0 if isSPACE($s); # early out return 1 unless (isALPHA($s) || isDIGIT($s) || $s =~ /[.-]/); $s++; } return 0; } sub _un_vstring { my $value = shift; # may be a v-string if ( length($value) >= 3 && $value !~ /[._]/ && _is_non_alphanumeric($value)) { my $tvalue; if ( $] ge 5.008_001 ) { $tvalue = _find_magic_vstring($value); $value = $tvalue if length $tvalue; } elsif ( $] ge 5.006_000 ) { $tvalue = sprintf("v%vd",$value); if ( $tvalue =~ /^v\d+(\.\d+){2,}$/ ) { # must be a v-string $value = $tvalue; } } } return $value; } sub _find_magic_vstring { my $value = shift; my $tvalue = ''; require B; my $sv = B::svref_2object(\$value); my $magic = ref($sv) eq 'B::PVMG' ? $sv->MAGIC : undef; while ( $magic ) { if ( $magic->TYPE eq 'V' ) { $tvalue = $magic->PTR; $tvalue =~ s/^v?(.+)$/v$1/; last; } else { $magic = $magic->MOREMAGIC; } } return $tvalue; } sub _VERSION { my ($obj, $req) = @_; my $class = ref($obj) || $obj; no strict 'refs'; if ( exists $INC{"$class.pm"} and not %{"$class\::"} and $] >= 5.008) { # file but no package require Carp; Carp::croak( "$class defines neither package nor VERSION" ."--version check failed"); } my $version = eval "\$$class\::VERSION"; if ( defined $version ) { local $^W if $] <= 5.008; $version = version::vpp->new($version); } if ( defined $req ) { unless ( defined $version ) { require Carp; my $msg = $] < 5.006 ? "$class version $req required--this is only version " : "$class does not define \$$class\::VERSION" ."--version check failed"; if ( $ENV{VERSION_DEBUG} ) { Carp::confess($msg); } else { Carp::croak($msg); } } $req = version::vpp->new($req); if ( $req > $version ) { require Carp; if ( $req->is_qv ) { Carp::croak( sprintf ("%s version %s required--". "this is only version %s", $class, $req->normal, $version->normal) ); } else { Carp::croak( sprintf ("%s version %s required--". "this is only version %s", $class, $req->stringify, $version->stringify) ); } } } return defined $version ? $version->stringify : undef; } 1; #this line is important and will help the module return a true value VERSION_VPP s/^ //mg for values %fatpacked; unshift @INC, sub { if (my $fat = $fatpacked{$_[1]}) { open my $fh, '<', \$fat or die "FatPacker error loading $_[1] (could be a perl installation issue?)"; return $fh; } return }; } # END OF FATPACK CODE #!/usr/bin/env perl =head1 NAME dist_surveyor - determine exactly what dist versions are installed =head1 SYNOPSIS dist_surveyor [options] /some/perl/lib/dir Typically a perl library directory will have an architecture specific library as a subdirectory. The dist_surveyor script will detect and add it automatically if the perl being used has the same 'archname' the same as the one in the library. If not, then specify the "archlib" directory explicitly I: dist_surveyor [options] /some/perl/lib/dir/archname /some/perl/lib/dir =head1 DESCRIPTION This utility examines all the modules installed within the specified perl library directory and uses the metacpan API to work out what versions of what distributions could have provided those modules. It then works out which of those candidate distributions is the most likely one. It is fairly robust and copes well with edge cases like installation of non-released versions from git repos and local modifications. Distributions are written to stdout. Progress and issues are reported to stderr. It can take a long time to run for the first time on a directory with a large number of modules and candidate distributions. The data fetched from metacpan is cached so future runs are much faster. (The system this code was tested on took about 60 minutes to process around 500 distributions with no cached data, and under 10 minutes for later runs that could reuse the cached data. The cache file ended up about 40MB in size.) =head1 Fatpacked script A fatpacked version of this script exists in: L Please note that the packed version expect that the following modules already installed on the local system: L, L and L. Also, if you are planing to --makecpan, you also need L =head1 OPTIONS --verbose Show more detailed progress --debug Show much more information --match R Ignore modules that don't match regex R (unanchored) --perlver V Ignore modules that are shipped with perl version V --remnants Include old distribution versions that have left old modules behind --uncached Don't use or update the persistent cache --makecpan D Create a CPAN repository in directory D --output S List of field names to output, separate by spaces. --format S Printf format string with a %s for each field in --output =head2 --makecpan Creates a CPAN repository in the specified directory by fetching the selected distributions into authors/id/... and writing the index files into modules/... If the directory already exists then selected distributions that already exist are not refetched, any distributions that already exist but aren't selected by this run are left in place. New package distribution information is merged into the modules/02packages index file. Some additional files are written into a dist_surveyor subdirectory: =head3 dist_surveyor/token_packages.txt This file lists one unique 'token package' per distribution. It's very useful to speed up re-running a full install after some distributions have failed. =head1 SURVEY USAGE Run a survey and create a mini-CPAN repository containing the distributions: dist_surveyor --makecpan my_cpan /some/perl/lib/dir > installed_dists.txt It's important to give the correct perl lib directory path. It's important to check the results related to any modules that generated warnings during the run. =head1 INSTALLATION USAGE Then, to install those distributions into a new library: cpanm --mirror file:$PWD/my_cpan --mirror-only [-l new_lib] < installed_dists.txt It's very likely that some distributions will fail tests and not install, which will, in turn, cause others to fail. Once the initial run is complete study the cpam build log file carefully and resolve the test failures. Running cpanm with a list of distributions, as above, will always reinstall I the listed distributions. Even those already sucessfully installed. It's much (I) faster to give cpanm a list of package names as that allows it to skip those that it knows are already installed. The L option writes a list of 'token packages', one per distribution, so you can use that with cpanm: cpanm --mirror file:$PWD/my_cpan --mirror-only [-l new_lib] < my_cpan/dist_surveyor/token_packages.txt When a distro fails tests I use the cpanm C<--look> option to investigate: cpanm --mirror file:$PWD/my_cpan --mirror-only --look Some::Package I'll often end up building, testing and installing the distro from within that cpanm look shell. Once installed I'll rerun cpanm using the full C file again. If there are more failures I'll repeat that sequence till they're all resolved. =head1 BUGS Probably. =head1 TODO * Auto-detect when directory given isn't the root of a perl library dir tree. E.g. by matching file names to module names * Add support for matching Foo.pm.PL files (e.g. FCGI and common::sense) * For installed modules get the file modification time (last commit time) and use it to eliminate candidate dists that were released after that time. * Consider factoring in release status ('authorized') so rogue releases that ship copies of many other modules (like Net-Braintree-0.1.1) are given a lower priority. * Sort out ExtUtils::Perllocal::Parser situation Avoid hard-coded %distro_key_mod_names related to perllocal.pod where the dist name doesn't match the key module name. Or maybe just remove use of distro_key_mod_names and perllocal entirely? * Optimise use of metacpan. Check caching. Use ElasticSearch.pm. * Fully handle merging of pre-existing --makecpan directory data files. * Consider factoring install date in the output ordering. May help with edge cases where a package P is installed via distros A then B. If A is reinstalled after B then the reinstalled P will be from A but should be from B. (I don't know of any cases, but it's certainly a possibility. The LWP breakup and Class::MOP spring to mind as possible candidates.) =cut use strict; use warnings; use Getopt::Long; # core use Config; # core $| = 1; use Dist::Surveyor; use Dist::Surveyor::Inquiry; # internal use constant PROGNAME => 'dist_surveyor'; GetOptions( 'match=s' => \my $opt_match, 'v|verbose!' => \my $opt_verbose, 'd|debug!' => \my $opt_debug, # target perl version, re core modules 'perlver=s' => \my $opt_perlver, # include old dists that have remnant/orphaned modules installed 'remnants!' => \my $opt_remnants, # don't use a persistent cache 'uncached!' => \my $opt_uncached, 'makecpan=s' => \my $opt_makecpan, # e.g., 'download_url author' 'output=s' => \(my $opt_output ||= 'url'), # e.g., 'some-command --foo --file %s --authorid %s' 'format=s' => \my $opt_format, ) or exit 1; $opt_verbose++ if $opt_debug; $opt_perlver = version->parse($opt_perlver || $])->numify; our $VERBOSE = $opt_verbose; our $DEBUG = $opt_debug; my $major_error_count = 0; # exit status my $distro_key_mod_names = { 'PathTools' => 'File::Spec', 'Template-Toolkit' => 'Template', 'TermReadKey' => 'Term::ReadKey', 'libwww-perl' => 'LWP', 'ack' => 'App::Ack', }; sub main { die "Usage: $0 perl-lib-directory [...]\n" unless @ARGV; my @libdirs = @ARGV; # check dirs and add archlib's if appropriate for my $libdir (@libdirs) { die "$libdir isn't a directory\n" unless -d $libdir; my $archdir = "$libdir/$Config{archname}"; if (-d $archdir) { unshift @libdirs, $archdir unless grep { $_ eq $archdir } @libdirs; } } $::DEBUG = $opt_debug; $::VERBOSE = $opt_verbose; Dist::Surveyor::Inquiry->perma_cache() unless $opt_uncached; my $options = { opt_match => $opt_match, opt_perlver => $opt_perlver, opt_remnants => $opt_remnants, distro_key_mod_names => $distro_key_mod_names, }; my @installed_releases = determine_installed_releases($options, \@libdirs); write_fields(\@installed_releases, $opt_format, [split ' ', $opt_output], \*STDOUT); warn sprintf "Completed survey in %.1f minutes using %d metacpan calls.\n", (time-$^T)/60, $Dist::Surveyor::Inquiry::metacpan_calls; if ($opt_makecpan) { require Dist::Surveyor::MakeCpan; my $cpan = Dist::Surveyor::MakeCpan->new( $opt_makecpan, PROGNAME, $distro_key_mod_names); warn "Updating $opt_makecpan for ".@installed_releases." releases...\n"; for my $ri (@installed_releases) { $cpan->add_release($ri); } $cpan->close(); $major_error_count += $cpan->errors(); } exit $major_error_count; } sub write_fields { my ($releases, $format, $fields, $fh) = @_; $format ||= join("\t", ('%s') x @$fields); $format .= "\n"; for my $release_data (@$releases) { printf $fh $format, map { exists $release_data->{$_} ? $release_data->{$_} : "?$_" } @$fields; } } main(@ARGV); exit 0;