[freeside-commits] branch master updated. 6e638a436bee0c28fbb702374659d75b928a9a3a
Jonathan Prykop
jonathan at 420.am
Thu Sep 15 07:34:49 PDT 2016
The branch, master has been updated
via 6e638a436bee0c28fbb702374659d75b928a9a3a (commit)
from bc330b42139caf8fadbf4c0c5b4e866f53271a95 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 6e638a436bee0c28fbb702374659d75b928a9a3a
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Thu Sep 15 09:32:33 2016 -0500
40361: Vocus CDR Format
diff --git a/FS/FS/cdr.pm b/FS/FS/cdr.pm
index c61a7b3..c4e9c47 100644
--- a/FS/FS/cdr.pm
+++ b/FS/FS/cdr.pm
@@ -32,7 +32,7 @@ use IO::Socket::SSL;
use Cpanel::JSON::XS qw(decode_json);
@ISA = qw(FS::Record);
- at EXPORT_OK = qw( _cdr_date_parser_maker _cdr_min_parser_maker );
+ at EXPORT_OK = qw( _cdr_date_parser_maker _cdr_min_parser_maker _cdr_date_parse );
$DEBUG = 0;
$me = '[FS::cdr]';
diff --git a/FS/FS/cdr/vocus.pm b/FS/FS/cdr/vocus.pm
new file mode 100644
index 0000000..0de9942
--- /dev/null
+++ b/FS/FS/cdr/vocus.pm
@@ -0,0 +1,73 @@
+package FS::cdr::vocus;
+
+use strict;
+use vars qw( @ISA %info $CDR_TYPES );
+use FS::cdr qw( _cdr_date_parse );
+use FS::Record qw( qsearch );
+
+
+ at ISA = qw(FS::cdr);
+
+%info = (
+ 'name' => 'Vocus',
+ 'weight' => 120,
+ 'import_fields' => [
+
+ #The first column is reserved for future use.
+ skip(1),
+ #The second column is the call identifier generated on our system.
+ 'uniqueid',
+ #The third column is the date of the call in UTC.
+ 'startdate',
+ #The fourth column is the time of the call in UTC.
+ sub {
+ # combine cols 3 & 4 and parse
+ my($cdr, $time, $conf, $param) = @_;
+ $cdr->startdate(_cdr_date_parse($cdr->startdate.' '.$time, gmt => 1));
+ },
+ #The fifth column is for Vocus use.
+ skip(1),
+ #The sixth column is the call duration in seconds.
+ 'billsec',
+ #The seventh column is the calling number presented to our soft switch in E164 format.
+ 'src',
+ #The eight column is the called number presented to our soft switch in E164 format.
+ 'dst',
+ #The ninth column is the time and date at which the call was rated and the
+ # CDR generated in our system. It's just there for your information.
+ skip(1),
+ #The tenth column is the SZU of the calling party.
+ 'upstream_src_regionname',
+ #The eleventh column is the SZU of the called party, if applicable.
+ 'upstream_dst_regionname',
+ #The twelfth column is the tariff type - Mobile, Regional, International,
+ #etc. This matches up with the tariff types under the Voice Access Point on your bill.
+ sub {
+ my($cdr, $cdrtypename, $conf, $param) = @_;
+ return unless length($cdrtypename);
+ _init_cdr_types();
+ die "no matching cdrtypenum for $cdrtypename"
+ unless defined $CDR_TYPES->{$cdrtypename};
+ $cdr->cdrtypenum($CDR_TYPES->{$cdrtypename});
+ },
+ #The thirteenth column is the cost of the call, ex GST.
+ 'upstream_price',
+
+ ],
+);
+
+sub skip { map {''} (1..$_[0]) }
+
+sub _init_cdr_types {
+ unless ($CDR_TYPES) {
+ $CDR_TYPES = {};
+ foreach my $cdr_type ( qsearch('cdr_type') ) {
+ die "multiple cdr_types with same cdrtypename".$cdr_type->cdrtypename
+ if defined $CDR_TYPES->{$cdr_type->cdrtypename};
+ $CDR_TYPES->{$cdr_type->cdrtypename} = $cdr_type->cdrtypenum;
+ }
+ }
+}
+
+1;
+
diff --git a/FS/bin/freeside-cdr-vocus b/FS/bin/freeside-cdr-vocus
new file mode 100755
index 0000000..cc6190e
--- /dev/null
+++ b/FS/bin/freeside-cdr-vocus
@@ -0,0 +1,232 @@
+#!/usr/bin/perl
+
+use strict;
+use Getopt::Std;
+use Date::Format;
+use Net::SFTP::Foreign::Compat;
+use FS::UID qw(adminsuidsetup datasrc);
+use FS::cdr;
+
+### So much false laziness with freeside-cdr-sftp_and_import
+### but vocus needs special handling to choose which files to load
+
+###
+# parse command line
+###
+
+use vars qw( $opt_r $opt_d $opt_v $opt_s );
+getopts('r:d:vs');
+
+my $user = shift or die &usage;
+adminsuidsetup $user;
+
+# %%%FREESIDE_CACHE%%%
+my $cachedir = '%%%FREESIDE_CACHE%%%/cache.'. datasrc. '/cdrs';
+mkdir $cachedir unless -d $cachedir;
+
+use vars qw( $servername );
+$servername = shift or die &usage;
+
+###
+# get the file list
+###
+
+warn "Retrieving directory listing\n" if $opt_v;
+
+my $ls;
+
+my $ls_sftp = sftp();
+
+$ls_sftp->setcwd($opt_r) or die "can't chdir to $opt_r\n"
+ if $opt_r;
+
+$ls = $ls_sftp->ls('.', wanted => qr/^VAP.*\.csv\.gz$/i,
+ names_only => 1 );
+
+
+###
+# vocus-specific part -- only use highest-numbered file for day
+###
+my %dates;
+foreach my $filename ( @$ls ) {
+ my ($filepre,$filedate,$iter) = $filename =~ /^(VAP\d+\-)(\d{4}\-\d{2}\-\d{2})(\.\d+)?.csv.gz$/;
+ unless ($filepre && $filedate) {
+ die "unparsable filename $filename"; #no warn and skip, might process wrong file for date
+ }
+ $iter =~ s/\.// if length($iter);
+ #not clear if same day can have different initial digits after VAP,
+ # stated rule is "use the highest-numbered file for a given day",
+ # so using date as key, but die if iter can't resolve conflict
+ if (!$dates{$filedate}) {
+ $dates{$filedate} = {};
+ } elsif ($dates{$filedate}{'iter'} eq $iter) {
+ die "duplicate iterators found for $filedate\n"
+ }
+ $dates{$filedate}{'files'} ||= [];
+ push @{$dates{$filedate}{'files'}}, $filename;
+ # don't actually expect iter of 0, but just in case, 0 trumps undef
+ if (!defined($dates{$filedate}{'iter'}) or (($iter || 0) > $dates{$filedate}{'iter'})) {
+ $dates{$filedate}{'iter'} = $iter;
+ $dates{$filedate}{'pre'} = $filepre;
+ }
+}
+
+###
+# import each file
+###
+
+foreach my $filedate ( keys %dates ) {
+
+ my $filename = $dates{$filedate}{'pre'} . $filedate;
+ $filename .= '.'.$dates{$filedate}{'iter'}
+ if defined($dates{$filedate}{'iter'});
+ $filename .= '.csv.gz';
+
+ warn "Downloading $filename\n" if $opt_v;
+
+ #get the file
+ my $sftp = sftp();
+ $sftp->get($filename, "$cachedir/$filename")
+ or do {
+ unlink "$cachedir/$filename";
+ my $error = "Can't get $filename: ". $sftp->error . "\n";
+ if ( $opt_s ) {
+ warn $error;
+ next;
+ } else {
+ die $error;
+ }
+ };
+
+ warn "Processing $filename\n" if $opt_v;
+
+ my $ungziped = $filename;
+ $ungziped =~ s/\.gz$//;
+ if(system('gunzip', "$cachedir/$filename") != 0) {
+ unlink "$cachedir/$filename";
+ my $error = "gunzip of '$cachedir/$filename' failed\n";
+ if ( $opt_s ) {
+ warn $error;
+ next;
+ } else {
+ die $error;
+ }
+ }
+
+ my $import_options = {
+ 'file' => "$cachedir/$ungziped",
+ 'format' => 'vocus',
+ 'batch_namevalue' => 'vocus-'.$filedate, #should further ensure only one file per date
+# 'empty_ok' => 1,
+ };
+
+ my $error = FS::cdr::batch_import($import_options);
+
+ if ( $error ) {
+
+ unlink "$cachedir/$filename";
+ unlink "$cachedir/$ungziped";
+ $error = "Error importing $ungziped: $error\n";
+ if ( $opt_s ) {
+ warn $error;
+ next;
+ } else {
+ die $error;
+ }
+
+ } else {
+
+ if ( $opt_d ) {
+ my $timestamp = time2str('%Y-%m-%d', time);
+ my $sftp = sftp();
+ foreach my $mfilename (@{$dates{$filedate}{'files'}}) {
+ warn "Moving $mfilename\n" if $opt_v;
+ $sftp->rename($mfilename, "$opt_d/$mfilename-$timestamp")
+ or do {
+ unlink "$cachedir/$filename";
+ unlink "$cachedir/$ungziped";
+ $error = "$mfilename imported, but can't move to $opt_d: ". $sftp->error . "\n";
+ if ( $opt_s ) {
+ warn $error;
+ next;
+ } else {
+ die $error;
+ }
+ };
+ }
+ }
+
+ }
+
+ unlink "$cachedir/$filename";
+ unlink "$cachedir/$ungziped";
+
+}
+
+###
+# subs
+###
+
+sub usage {
+ "Usage:
+ freeside-cdr-vocus
+ [ -r remotefolder ] [ -d donefolder ] [ -v level ]
+ [ -s ] user [sftpuser@]servername
+ ";
+}
+
+use vars qw( $sftp );
+
+sub sftp {
+
+ #reuse connections
+ return $sftp if $sftp && $sftp->cwd;
+
+ my %sftp = ( host => $servername );
+
+ $sftp = Net::SFTP::Foreign->new(%sftp);
+ $sftp->error and die "SFTP connection failed: ". $sftp->error;
+
+ $sftp;
+}
+
+=head1 NAME
+
+freeside-cdr-sftp_and_import - Download Vocus CDR files
+
+=head1 SYNOPSIS
+
+ freeside-cdr-sftp_and_import
+ [ -r remotefolder ] [ -d donefolder ] [ -v level ]
+ [ -s ] user [sftpuser@]servername
+
+=head1 DESCRIPTION
+
+Command line tool to download Vocus CDR files from a remote server via SFTP
+and then import them into the database. CDR tarrif types need to be
+configured as CDR types in freeside. If multiple files for a given day are
+found, the one with the highest iterator will be used (though upon successful
+import, all existing files for the day will be moved by the -d option.)
+
+-r: if specified, changes into this remote folder before starting
+
+-d: if specified, moves files to the specified folder when done
+
+-v: verbose
+
+-s: Warn and skip files which could not be imported rather than abort
+
+user: freeside username
+
+[sftpuser@]servername: remote server
+
+=head1 BUGS
+
+=head1 SEE ALSO
+
+L<FS::cdr>
+
+=cut
+
+1;
+
diff --git a/httemplate/edit/cdr_type.cgi b/httemplate/edit/cdr_type.cgi
index c696106..651de71 100644
--- a/httemplate/edit/cdr_type.cgi
+++ b/httemplate/edit/cdr_type.cgi
@@ -17,7 +17,7 @@ configured in the rate tables.
<INPUT NAME="cdrtypenum" SIZE=16 MAXLENGTH=16 ALIGN="right">
</TD>
<TD>
- <INPUT NAME="cdrtypename" SIZE=16 MAXLENGTH=16>
+ <INPUT NAME="cdrtypename" SIZE=16 MAXLENGTH=80>
</TD>
</TR>
<& /elements/auto-table.html,
-----------------------------------------------------------------------
Summary of changes:
FS/FS/cdr.pm | 2 +-
FS/FS/cdr/vocus.pm | 73 +++++++++++++
FS/bin/freeside-cdr-vocus | 232 ++++++++++++++++++++++++++++++++++++++++++
httemplate/edit/cdr_type.cgi | 2 +-
4 files changed, 307 insertions(+), 2 deletions(-)
create mode 100644 FS/FS/cdr/vocus.pm
create mode 100755 FS/bin/freeside-cdr-vocus
More information about the freeside-commits
mailing list