[freeside-commits] branch FREESIDE_4_BRANCH updated. 222df23fcacb65a7a92f1031f7e871f408601060
Jonathan Prykop
jonathan at 420.am
Thu Dec 3 20:29:37 PST 2015
The branch, FREESIDE_4_BRANCH has been updated
via 222df23fcacb65a7a92f1031f7e871f408601060 (commit)
via 1e42c6c5fcfe661861b3d45d1b17d16ca36ab7f6 (commit)
via cfdfc8bc874bff592679a6e3df067338bbd51067 (commit)
from 2bd324173312ea6356c68b32f176c46317b28ce0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 222df23fcacb65a7a92f1031f7e871f408601060
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Thu Dec 3 22:13:31 2015 -0600
RT#37901: Galactic Telecom CDRs [skip duplicates, midnight start]
diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import
index d9a606a..9a02a72 100644
--- a/FS/bin/freeside-cdr-portaone-import
+++ b/FS/bin/freeside-cdr-portaone-import
@@ -7,9 +7,12 @@ use Date::Parse 'str2time';
use Getopt::Long;
use JSON;
use Net::HTTPS::Any qw(https_post);
+use Time::Local;
use FS::Record qw(qsearchs dbh);
use FS::UID qw(adminsuidsetup);
+use FS::cdr;
+use FS::cdr_batch;
sub usage {
"Usage:
@@ -54,7 +57,8 @@ unless ($startdate) {
}
$startdate ||= '2010-01-01 00:00:00'; #seems decently in the past
-my $now = time;
+my @now = localtime();
+my $now = timelocal(0,0,0,$now[3],$now[4],$now[5]); #most recent midnight
if ($enddate) {
$enddate = str2time($enddate) or die "Can't parse enddate $enddate";
$now = $enddate;
@@ -65,8 +69,11 @@ $enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now);
$FS::UID::AutoCommit = 0;
+my $cdrbatchname = 'portaone-import-'. time2str('%Y/%m/%d-%T',$now);
+die "Batch $cdrbatchname already exists, please specify a different end date. " . usage()
+ if FS::cdr_batch->row_exists('cdrbatch = ?', $cdrbatchname);
my $cdr_batch = new FS::cdr_batch({
- 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
+ 'cdrbatch' => $cdrbatchname,
'_date' => $now,
});
my $error = $cdr_batch->insert;
@@ -102,8 +109,13 @@ foreach my $custnum (@custnum) {
'offset' => $offset,
});
my @xdrs = @{$xdrs->{'xdr_list'}};
- print "Inserting ". at xdrs." records\n" if $verbose && @xdrs;
+ print "Retrieved ". at xdrs." records\n" if $verbose && @xdrs;
+ my $skipped = 0; # for verbose display only
foreach my $xdr (@xdrs) {
+ if ( FS::cdr->row_exists('uniqueid = ?', $xdr->{'i_xdr'}) ) {
+ $skipped += 1;
+ next;
+ }
my $desc = $xdr->{'country'};
if ($xdr->{'subdivision'}) {
$desc = ', ' . $desc if $desc;
@@ -131,11 +143,12 @@ foreach my $custnum (@custnum) {
die "Error inserting cdr: $error";
}
} #foreach $xdr
- $totalcount += @xdrs;
+ print "Skipped $skipped duplicate records\n" if $verbose && $skipped;
+ $totalcount += @xdrs - $skipped;
$lastcount = @xdrs;
$offset += $step;
} #while $lastcount == $step
- print scalar($totalcount) . " xdrs retrieved\n" if $verbose;
+ print scalar($totalcount) . " records inserted\n" if $verbose;
} #foreach $custnum
call_api('Session','logout',$auth_info);
commit 1e42c6c5fcfe661861b3d45d1b17d16ca36ab7f6
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Thu Dec 3 16:45:15 2015 -0600
RT#37901: Galactic Telecom CDRs [field mappings]
diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import
index be2555b..d9a606a 100644
--- a/FS/bin/freeside-cdr-portaone-import
+++ b/FS/bin/freeside-cdr-portaone-import
@@ -62,6 +62,19 @@ if ($enddate) {
}
$enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now);
+
+$FS::UID::AutoCommit = 0;
+
+my $cdr_batch = new FS::cdr_batch({
+ 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
+ '_date' => $now,
+});
+my $error = $cdr_batch->insert;
+if ($error) {
+ dbh->rollback;
+ die "Error creating batch: $error";
+}
+
print "Downloading records from $startdate to $enddate\n" if $verbose;
my $auth_info; # needs to be declared undef for call_api
@@ -74,58 +87,61 @@ my $results = {};
my $custlist = call_api('Customer','get_customer_list');
my @custnum = map { $_->{'i_customer'} } @{$custlist->{'customer_list'}};
foreach my $custnum (@custnum) {
- print "Processing customer $custnum\n" if $verbose;
- my $xdrs = call_api('Customer','get_customer_xdrs',{
- 'i_customer' => $custnum,
- 'from_date' => $startdate,
- 'to_date' => $enddate,
- 'cdr_entity' => 'C', # can also be 'A', or blank for both
- });
- my @xdrs = @{$xdrs->{'xdr_list'}};
- print scalar(@xdrs) . " xdrs retrieved\n" if $verbose;
- $results->{$custnum} = \@xdrs;
-}
+ print "Retrieving for customer $custnum\n" if $verbose;
+ my $step = 500; # too many records was crashing server, so we request in chunks
+ my $lastcount = $step; # to get the while loop rolling
+ my $totalcount = 0; # for verbose display only
+ my $offset = 0;
+ while ($lastcount == $step) {
+ my $xdrs = call_api('Customer','get_customer_xdrs',{
+ 'i_customer' => $custnum,
+ 'from_date' => $startdate,
+ 'to_date' => $enddate,
+ 'cdr_entity' => 'A',
+ 'limit' => $step,
+ 'offset' => $offset,
+ });
+ my @xdrs = @{$xdrs->{'xdr_list'}};
+ print "Inserting ". at xdrs." records\n" if $verbose && @xdrs;
+ foreach my $xdr (@xdrs) {
+ my $desc = $xdr->{'country'};
+ if ($xdr->{'subdivision'}) {
+ $desc = ', ' . $desc if $desc;
+ $desc = $xdr->{'subdivision'} . $desc;
+ }
+ if ($xdr->{'description'}) {
+ $desc = ' (' . $desc . ')' if $desc;
+ $desc = $xdr->{'description'} . $desc;
+ }
+ my $cdr = FS::cdr->new ({
+ 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
+ 'uniqueid' => $xdr->{'i_xdr'},
+ 'src' => $xdr->{'CLI'},
+ 'dst' => $xdr->{'CLD'},
+ 'upstream_price' => $xdr->{'charged_amount'},
+ 'startdate' => $xdr->{'unix_connect_time'},
+ 'enddate' => $xdr->{'unix_disconnect_time'},
+ 'accountcode' => $xdr->{'account_id'},
+ 'billsec' => $xdr->{'charged_quantity'},
+ 'upstream_dst_regionname' => $desc,
+ });
+ $error = $cdr->insert;
+ if ($error) {
+ dbh->rollback;
+ die "Error inserting cdr: $error";
+ }
+ } #foreach $xdr
+ $totalcount += @xdrs;
+ $lastcount = @xdrs;
+ $offset += $step;
+ } #while $lastcount == $step
+ print scalar($totalcount) . " xdrs retrieved\n" if $verbose;
+} #foreach $custnum
call_api('Session','logout',$auth_info);
-$FS::UID::AutoCommit = 0;
-
-my $cdr_batch = new FS::cdr_batch({
- 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
- '_date' => $now,
-});
-my $error = $cdr_batch->insert;
-if ($error) {
- dbh->rollback;
- die "Error inserting batch: $error";
-}
-
-foreach my $custnum (keys %$results) {
- foreach my $xdr (@{$results->{$custnum}}) {
- my $cdr = FS::cdr->new ({
- 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
- 'acctid' => $xdr->{'i_xdr'}, #???
- 'accountcode' => $xdr->{'account_id'}, #???
- 'startdate' => $xdr->{'unix_connect_time'},
- 'enddate' => $xdr->{'unix_disconnect_time'},
-
-### XXX NEED TO MAP FIELDS:
+### Full list of fields returned by API:
#i_xdr int The unique ID of the xdr record
-#account_id int The unique ID of the account database record
-#CLI string Calling Line Identification
-#CLD string Called Line Identification
-#charged_amount float Amount charged
-#charged_quantity int Units charged
-#country string Country
-#subdivision string Country subdivision
-#description string Destination description
-#disconnect_cause string The code of disconnect cause
-#bill_status string Call bill status
-#disconnect_reason string Call disconnect reason
-#connect_time dateTime Call connect time
-#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch)
-#disconnect_time dateTime Call disconnect time
-#unix_disconnect_time int Call disconnect time i_xdr int The unique ID of the xdr record
#account_id int The unique ID of the account database record
#CLI string Calling Line Identification
#CLD string Called Line Identification
@@ -144,21 +160,8 @@ foreach my $custnum (keys %$results) {
#bill_time dateTime Call bill time
#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
#call_recording_url string Path to recorded .wav files
-#call_recording_server_url string URL to the recording server (expressed in Unix time format - seconds since epoch)
-#bill_time dateTime Call bill time
-#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
-#call_recording_url string Path to recorded .wav files
#call_recording_server_url string URL to the recording server
- });
- $error = $cdr->insert;
- if ($error) {
- dbh->rollback;
- die "Error inserting cdr: $error";
- }
- } #foreach $xdr
-} #foreach $custnum
-
dbh->commit;
exit;
@@ -167,7 +170,7 @@ sub call_api {
my ($service,$method,$params) = @_;
my %auth_info = $auth_info ? ('auth_info' => encode_json($auth_info)) : ();
$params ||= {};
- print "Calling $service/$method...\n" if $verbose;
+ print "Calling $service/$method\n" if $verbose;
my ( $page, $response, %reply_headers ) = https_post(
'host' => $host,
'port' => $port,
@@ -175,6 +178,7 @@ sub call_api {
'args' => [ %auth_info, 'params' => encode_json($params) ],
);
return decode_json($page) if $response eq '200 OK';
+ dbh->rollback;
if ($response =~ /^500/) {
my $error = decode_json($page);
die "Server returned error during $service/$method: ".$error->{'faultstring'}
commit cfdfc8bc874bff592679a6e3df067338bbd51067
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Tue Dec 1 02:27:19 2015 -0600
RT37901: Galactic Telecom CDRs [need to finish field mapping
diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import
new file mode 100644
index 0000000..be2555b
--- /dev/null
+++ b/FS/bin/freeside-cdr-portaone-import
@@ -0,0 +1,186 @@
+#!/usr/bin/perl
+
+use strict;
+
+use Date::Format 'time2str';
+use Date::Parse 'str2time';
+use Getopt::Long;
+use JSON;
+use Net::HTTPS::Any qw(https_post);
+
+use FS::Record qw(qsearchs dbh);
+use FS::UID qw(adminsuidsetup);
+
+sub usage {
+ "Usage:
+freeside-cdr-portaone-import -h 'your.domain.com:443' -u switchusername -p switchpass
+ [-s startdate] [-e enddate] [-v] freesideuser
+";
+}
+
+my ($host,$username,$password,$startdate,$enddate,$verbose);
+GetOptions(
+ "enddate=s" => \$enddate,
+ "host=s" => \$host,
+ "password=s" => \$password,
+ "startdate=s" => \$startdate,
+ "username=s" => \$username,
+ "verbose" => \$verbose,
+);
+
+my $fsuser = $ARGV[-1];
+
+die usage() unless $host && $password && $username && $fsuser;
+
+adminsuidsetup($fsuser);
+
+my $port = 443;
+if ($host =~ /^(.*)\:(.*)$/) {
+ $host = $1;
+ $port = $2;
+}
+
+if ($startdate) {
+ $startdate = str2time($startdate) or die "Can't parse startdate $startdate";
+ $startdate = time2str("%Y-%m-%d %H:%M:%S",$startdate);
+}
+unless ($startdate) {
+ my $lastbatch = qsearchs({
+ 'table' => 'cdr_batch',
+ 'hashref' => { 'cdrbatch' => {op=>'like', value=>'portaone-import%'}},
+ 'order_by' => 'ORDER BY _date DESC LIMIT 1',
+ });
+ $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date + 1) if $lastbatch;
+}
+$startdate ||= '2010-01-01 00:00:00'; #seems decently in the past
+
+my $now = time;
+if ($enddate) {
+ $enddate = str2time($enddate) or die "Can't parse enddate $enddate";
+ $now = $enddate;
+ $enddate = time2str("%Y-%m-%d %H:%M:%S",$enddate);
+}
+$enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now);
+
+print "Downloading records from $startdate to $enddate\n" if $verbose;
+
+my $auth_info; # needs to be declared undef for call_api
+$auth_info = call_api('Session','login',{
+ 'login' => $username,
+ 'password' => $password,
+});
+
+my $results = {};
+my $custlist = call_api('Customer','get_customer_list');
+my @custnum = map { $_->{'i_customer'} } @{$custlist->{'customer_list'}};
+foreach my $custnum (@custnum) {
+ print "Processing customer $custnum\n" if $verbose;
+ my $xdrs = call_api('Customer','get_customer_xdrs',{
+ 'i_customer' => $custnum,
+ 'from_date' => $startdate,
+ 'to_date' => $enddate,
+ 'cdr_entity' => 'C', # can also be 'A', or blank for both
+ });
+ my @xdrs = @{$xdrs->{'xdr_list'}};
+ print scalar(@xdrs) . " xdrs retrieved\n" if $verbose;
+ $results->{$custnum} = \@xdrs;
+}
+
+call_api('Session','logout',$auth_info);
+
+$FS::UID::AutoCommit = 0;
+
+my $cdr_batch = new FS::cdr_batch({
+ 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
+ '_date' => $now,
+});
+my $error = $cdr_batch->insert;
+if ($error) {
+ dbh->rollback;
+ die "Error inserting batch: $error";
+}
+
+foreach my $custnum (keys %$results) {
+ foreach my $xdr (@{$results->{$custnum}}) {
+ my $cdr = FS::cdr->new ({
+ 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
+ 'acctid' => $xdr->{'i_xdr'}, #???
+ 'accountcode' => $xdr->{'account_id'}, #???
+ 'startdate' => $xdr->{'unix_connect_time'},
+ 'enddate' => $xdr->{'unix_disconnect_time'},
+
+### XXX NEED TO MAP FIELDS:
+#i_xdr int The unique ID of the xdr record
+#account_id int The unique ID of the account database record
+#CLI string Calling Line Identification
+#CLD string Called Line Identification
+#charged_amount float Amount charged
+#charged_quantity int Units charged
+#country string Country
+#subdivision string Country subdivision
+#description string Destination description
+#disconnect_cause string The code of disconnect cause
+#bill_status string Call bill status
+#disconnect_reason string Call disconnect reason
+#connect_time dateTime Call connect time
+#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch)
+#disconnect_time dateTime Call disconnect time
+#unix_disconnect_time int Call disconnect time i_xdr int The unique ID of the xdr record
+#account_id int The unique ID of the account database record
+#CLI string Calling Line Identification
+#CLD string Called Line Identification
+#charged_amount float Amount charged
+#charged_quantity int Units charged
+#country string Country
+#subdivision string Country subdivision
+#description string Destination description
+#disconnect_cause string The code of disconnect cause
+#bill_status string Call bill status
+#disconnect_reason string Call disconnect reason
+#connect_time dateTime Call connect time
+#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch)
+#disconnect_time dateTime Call disconnect time
+#unix_disconnect_time int Call disconnect time (expressed in Unix time format - seconds since epoch)
+#bill_time dateTime Call bill time
+#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
+#call_recording_url string Path to recorded .wav files
+#call_recording_server_url string URL to the recording server (expressed in Unix time format - seconds since epoch)
+#bill_time dateTime Call bill time
+#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
+#call_recording_url string Path to recorded .wav files
+#call_recording_server_url string URL to the recording server
+
+ });
+ $error = $cdr->insert;
+ if ($error) {
+ dbh->rollback;
+ die "Error inserting cdr: $error";
+ }
+ } #foreach $xdr
+} #foreach $custnum
+
+dbh->commit;
+
+exit;
+
+sub call_api {
+ my ($service,$method,$params) = @_;
+ my %auth_info = $auth_info ? ('auth_info' => encode_json($auth_info)) : ();
+ $params ||= {};
+ print "Calling $service/$method...\n" if $verbose;
+ my ( $page, $response, %reply_headers ) = https_post(
+ 'host' => $host,
+ 'port' => $port,
+ 'path' => '/rest/'.$service.'/'.$method.'/',
+ 'args' => [ %auth_info, 'params' => encode_json($params) ],
+ );
+ return decode_json($page) if $response eq '200 OK';
+ if ($response =~ /^500/) {
+ my $error = decode_json($page);
+ die "Server returned error during $service/$method: ".$error->{'faultstring'}
+ if $error->{'faultcode'};
+ }
+ die "Bad response from server during $service/$method: $response";
+}
+
+
-----------------------------------------------------------------------
Summary of changes:
FS/bin/freeside-cdr-portaone-import | 203 +++++++++++++++++++++++++++++++++++
1 file changed, 203 insertions(+)
create mode 100644 FS/bin/freeside-cdr-portaone-import
More information about the freeside-commits
mailing list