<?php
    /* License:
       Open source for private and commercial use.
       This source code is free to use and modify as long as this comment stays untouched on top.
       URL of original source code: http://scrape-google-finance.compunect.com/
       Author of original source code: http://www.compunect.com
       Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to anyone for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to the extent applicable law prohibits such limitation.
       Exceptions:
       Public redistributing modifications of this source code project is not allowed without written agreement.
       Using this work for private and commercial projects is allowed, redistributing it is not allowed without our written agreement.
       In simple words: You may power your project with this code or a customized version of it, but you may NOT redistribute the code. Also any legal consequences are your own problem.
     */

    // just comment out the echo or extend the function to your liking
    function verbose($text)
    {
        echo $text;
    }

    /*
     * Returns up to date data to use for this scraping.
     * includes: user agent (in future, rotating user agents), google codes, domains, etc.
     * This function will only work with a valid plan at us-proxies.com
     *
     * You can remove the API parts and hardcore the values if you do not wish to get a plan there and use a different service for IPs
     * Otherwise this snippet will allow your code to automatically react to some changes
     */
    function get_api_google_finance()
    {
        global $pwd;
        global $uid;
        global $PROXY;
        global $PLAN;
        global $portal;
        $fp = fsockopen("us-proxies.com", 80);
        if (!$fp)
        {
            echo "Unable to connect to google_cc API \n";

            return NULL; // connection not possible
        } else
        {
            $plan_size = $PLAN['total_ips'];
            fwrite($fp, "GET /g_api.php?api=1&uid=$uid&pwd=$pwd&cmd=google_finance&plan_size=$plan_size HTTP/1.0\r\nHost: us-proxies.com\r\nAccept: text/html, text/plain, text/*, */*;q=0.01\r\nAccept-Encoding: plain\r\nAccept-Language: en\r\n\r\n");
            stream_set_timeout($fp, 8);
            $res = "";
            $n = 0;
            while (!feof($fp))
            {
                if ($n++ > 4) break;
                $res .= fread($fp, 8192);
            }
            $info = stream_get_meta_data($fp);
            fclose($fp);

            if ($info['timed_out'])
            {
                echo 'API: Connection timed out! \n';

                return NULL; // api timeout
            } else
            {
                $data = extractBody($res);
                $obj = unserialize($data);
                if (isset($obj['error'])) echo $obj['error'] . "\n";
                if (isset($obj['info'])) echo $obj['info'] . "\n";

                return $obj['data'];

                if (strlen($data) < 4) return NULL; // invalid api response
            }
        }
    }

    function rmkdir($path, $mode = 0755)
    {
        if (file_exists($path)) return 1;

        return @mkdir($path, $mode);
    }

    /* Delay (sleep) based on the license size to allow optimal scraping
     *
     * Warning!
     * Do NOT change the delay to be shorter than the specified delay.
     * This function will create a delay based on your total IP addresses.
     *
     */
    function delay_time($reason = 'ip', $total_threads = 1)
    {
        global $PLAN;
        global $api_finance_data;


        if ($reason == 'ip')
        {
            $d = $total_threads * $api_finance_data['delay_rotate_us'];
            verbose("\twait.. \n");
        }
        if ($reason == 'request')
        {
            $d = $api_finance_data['delay_query_us'];
            verbose("\twait.. \n");
        }
        usleep($d);
    }


    /*
     * By default (no force) the function will load cached data within $max_hours hours otherwise reject the cache.
     * The time can be increased to reduce IP usage
     */
    function load_cache($keyword, $api_finance_data, $force_cache, $cache_type, $max_hours)
    {
        global $working_dir;

        if ($force_cache < 0) return NULL;

        $file = "$working_dir/$cache_type.$keyword.cache";
        $now = time();
        if (file_exists($file))
        {
            $ut = filemtime($file);
            $dif = $now - $ut;
            $hour = (int)($dif / (60 * 60));
            if ($force_cache || ($dif < (60 * 60 * $max_hours)))
            {
                $serdata = file_get_contents($file);
                $serp_data = unserialize($serdata);
                verbose("\tusing cache, file age: {$hour}h\n");

                return $serp_data;
            }

            return NULL;
        } else
        {
            return NULL;
        }

    }

    function store_cache($data, $keyword, $api_finance_data, $cache_type)
    {
        global $working_dir;

        $file = "$working_dir/$cache_type.$keyword.cache";
        $now = time();
        if (file_exists($file))
        {
            $ut = filemtime($file);
            $dif = $now - $ut;
            //if ($dif < (60 * 60 * 24)) echo "Warning: cache storage initated for $keyword which was already cached within the past 24 hours!\n";
        }
        $serdata = serialize($data);
        file_put_contents($file, $serdata, LOCK_EX);
        verbose("\tCache: stored file $file for $keyword.\n");
    }


    // check_ip_usage() must be called before first use of mark_ip_usage()
    function check_ip_usage()
    {
        global $PROXY;
        global $working_dir;
        global $ip_usage_data; // usage data object as array

        if (!isset($PROXY['ready'])) return 0; // proxy not ready/started
        if (!$PROXY['ready']) return 0; // proxy not ready/started

        if (!isset($ip_usage_data))
        {
            if (!file_exists($working_dir . "/ipdata.obj")) // usage data object as file
            {
                echo "Warning!\n" . "The ipdata.obj file was not found, if this is the first usage of the rank checker everything is alright.\n" . "Otherwise removal or failure to access the ip usage data will lead to damage of the IP quality.\n\n";
                sleep(2);
                $ip_usage_data = array();
            } else
            {
                $ser_data = file_get_contents($working_dir . "/ipdata.obj");
                $ip_usage_data = unserialize($ser_data);
            }
        }

        if (!isset($ip_usage_data[$PROXY['external_ip']]))
        {
            //verbose("IP $PROXY[external_ip] is ready for use \n");

            return 1; // the IP was not used yet
        }
        if (!isset($ip_usage_data[$PROXY['external_ip']]['requests'][20]['ut_google']))
        {
            //verbose("IP $PROXY[external_ip] is ready for use \n");

            return 1; // the IP has not been used 20+ times yet, return true
        }
        $ut_last = (int)$ip_usage_data[$PROXY['external_ip']]['ut_last-usage']; // last time this IP was used
        $req_total = (int)$ip_usage_data[$PROXY['external_ip']]['request-total']; // total number of requests made by this IP
        $req_20 = (int)$ip_usage_data[$PROXY['external_ip']]['requests'][20]['ut_google']; // the 20th request (if IP was used 20+ times) unixtime stamp

        $now = time();
        if (($now - $req_20) > (60 * 60))
        {
            //verbose("IP $PROXY[external_ip] is ready for use \n");

            return 1; // more than an hour passed since 20th usage of this IP
        } else
        {
            $cd_sec = (60 * 60) - ($now - $req_20);
            verbose("IP $PROXY[external_ip] needs $cd_sec seconds cooldown, not ready for use yet \n");

            return 0; // the IP is overused, it can not be used for scraping without being detected by the search engine yet
        }
    }

    /*
     * Updates and stores the ip usage data object
     * Marks an IP as used and re-sorts the access array
     */
    function mark_ip_usage()
    {
        global $PROXY;
        global $working_dir;
        global $ip_usage_data; // usage data object as array

        if (!isset($ip_usage_data)) die("ERROR: Incorrect usage. check_ip_usage() needs to be called once before mark_ip_usage()!\n");
        $now = time();

        $ip_usage_data[$PROXY['external_ip']]['ut_last-usage'] = $now; // last time this IP was used
        if (!isset($ip_usage_data[$PROXY['external_ip']]['request-total'])) $ip_usage_data[$PROXY['external_ip']]['request-total'] = 0;
        $ip_usage_data[$PROXY['external_ip']]['request-total']++; // total number of requests made by this IP
        // shift fifo queue
        for ($req = 19; $req >= 1; $req--)
        {
            if (isset($ip_usage_data[$PROXY['external_ip']]['requests'][$req]['ut_google']))
            {
                $ip_usage_data[$PROXY['external_ip']]['requests'][$req + 1]['ut_google'] = $ip_usage_data[$PROXY['external_ip']]['requests'][$req]['ut_google'];
            }
        }
        $ip_usage_data[$PROXY['external_ip']]['requests'][1]['ut_google'] = $now;

        $serdata = serialize($ip_usage_data);
        file_put_contents($working_dir . "/ipdata.obj", $serdata, LOCK_EX);

    }

    function new_curl_session($ch = NULL)
    {
        global $PROXY;
        global $api_finance_data;

        $default_agent = $api_finance_data['default_agent_chrome']; // rotating chrome useragent ?
        if ((!isset($PROXY['ready'])) || (!$PROXY['ready'])) return $ch; // proxy not ready

        if (isset($ch) && ($ch != NULL))
        {
            curl_close($ch);
        }
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_HEADER, 0);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        $curl_proxy = "$PROXY[address]:$PROXY[port]";
        curl_setopt($ch, CURLOPT_PROXY, $curl_proxy);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
        curl_setopt($ch, CURLOPT_TIMEOUT, 10);
        curl_setopt($ch, CURLOPT_USERAGENT, $default_agent);
        return $ch;
    }

    function getip()
    {
        global $PROXY;
        if (!$PROXY['ready']) return -1; // proxy not ready

        $curl_handle = curl_init();
        curl_setopt($curl_handle, CURLOPT_URL, 'http://ipcheck.ipnetic.com/remote_ip.php'); // returns the real IP
        curl_setopt($curl_handle, CURLOPT_CONNECTTIMEOUT, 10);
        curl_setopt($curl_handle, CURLOPT_TIMEOUT, 10);
        curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1);
        $curl_proxy = "$PROXY[address]:$PROXY[port]";
        curl_setopt($curl_handle, CURLOPT_PROXY, $curl_proxy);
        $tested_ip = curl_exec($curl_handle);

        if (preg_match("^([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}^", $tested_ip))
        {
            curl_close($curl_handle);

            return $tested_ip;
        } else
        {
            $info = curl_getinfo($curl_handle);
            curl_close($curl_handle);

            return 0; // possible error would be a wrong authentication IP or a firewall
        }
    }


    // return 1 if account is ready, otherwise 0
    function get_plan()
    {
        global $PLAN;

        $res = ip_service("plan"); // will fill $PLAN
        $ip = "";
        if ($res <= 0)
        {
            verbose("API error: Proxy API connection failed (Error $res). trying again later..\n\n");

            return 0;
        } else
        {
            ($PLAN['active'] == 1) ? $ready = "active" : $ready = "not active";
            verbose("\tAPI success: License is $ready.\n");
            if ($PLAN['active'] == 1) return 1;

            return 0;
        }

        return $PLAN;
    }

    /*
        * This is the API function to retrieve US IP addresses
        * This function handles the API calls "plan" and "rotate"
        *
        * Rotate: On success this function will define the global $PROXY variable, adding the elements ready,address,port,external_ip and return 1
        * On failure the return is 0 or smaller and the PROXY variable ready element is set to "0"
        * It is good practice to use the API response in $PROXY instead of hardcoding connection parameters
        *
        * Plan: On success this function will define the global $PLAN variable, adding the elements active, max_ips, total_ips, protocol, processes and return 1
        * It is good practice to make one call to "plan" upon starting your script to find out about the status and size of the plan
        */
    function extractBody($response_str)
    {
        $parts = preg_split('|(?:\r?\n){2}|m', $response_str, 2);
        if (isset($parts[1])) return $parts[1]; else  return '';
    }

    function ip_service($cmd, $x = "")
    {
        global $pwd;
        global $uid;
        global $PROXY;
        global $PLAN;

        $fp = fsockopen("us-proxies.com", 80);
        if (!$fp)
        {
            echo "Unable to connect to API \n";

            return -1; // connection not possible
        } else
        {
            if ($cmd == "plan")
            {
                fwrite($fp, "GET /api.php?api=1&uid=$uid&pwd=$pwd&cmd=plan&extended=1 HTTP/1.0\r\nHost: us-proxies.com\r\nAccept: text/html, text/plain, text/*, */*;q=0.01\r\nAccept-Encoding: plain\r\nAccept-Language: en\r\n\r\n");

                stream_set_timeout($fp, 8);
                $res = "";
                $n = 0;
                while (!feof($fp))
                {
                    if ($n++ > 4) break;
                    $res .= fread($fp, 8192);
                }
                $info = stream_get_meta_data($fp);
                fclose($fp);

                if ($info['timed_out'])
                {
                    echo 'API: Connection timed out! \n';
                    $PLAN['active'] = 0;

                    return -2; // api timeout
                } else
                {
                    if (strlen($res) > 1000) return -3; // invalid api response (check the API website for possible problems)
                    $data = extractBody($res);
                    $ar = explode(":", $data);
                    if (count($ar) < 4) return -100; // invalid api response
                    switch ($ar[0])
                    {
                        case "ERROR":
                            echo "API Error: $res \n";
                            $PLAN['active'] = 0;

                            return 0; // Error received
                            break;
                        case "PLAN":
                            $PLAN['max_ips'] = $ar[1]; // number of IPs licensed
                            $PLAN['total_ips'] = $ar[2]; // number of IPs assigned
                            $PLAN['protocol'] = $ar[3]; // current proxy protocol (http, socks, ..)
                            $PLAN['processes'] = $ar[4]; // number of available proxy processes
                            if ($PLAN['total_ips'] > 0) $PLAN['active'] = 1; else $PLAN['active'] = 0;

                            return 1;
                            break;
                        default:
                            echo "API Error: Received answer $ar[0], expected \"PLAN\"";
                            $PLAN['active'] = 0;

                            return -101; // unknown API response
                    }
                }

            } // cmd==plan


            if ($cmd == "rotate")
            {
                $PROXY['ready'] = 0;
                fwrite($fp, "GET /api.php?api=1&uid=$uid&pwd=$pwd&cmd=rotate&randomize=0&offset=0 HTTP/1.0\r\nHost: us-proxies.com\r\nAccept: text/html, text/plain, text/*, */*;q=0.01\r\nAccept-Encoding: plain\r\nAccept-Language: en\r\n\r\n");
                stream_set_timeout($fp, 8);
                $res = "";
                $n = 0;
                while (!feof($fp))
                {
                    if ($n++ > 4) break;
                    $res .= fread($fp, 8192);
                }
                $info = stream_get_meta_data($fp);
                fclose($fp);

                if ($info['timed_out'])
                {
                    echo 'API: Connection timed out! \n';

                    return -2; // api timeout
                } else
                {
                    if (strlen($res) > 1000) return -3; // invalid api response (check the API website for possible problems)
                    $data = extractBody($res);
                    $ar = explode(":", $data);
                    if (count($ar) < 4) return -100; // invalid api response
                    switch ($ar[0])
                    {
                        case "ERROR":
                            echo "API Error: $res \n";

                            return 0; // Error received
                            break;
                        case "ROTATE":
                            $PROXY['address'] = $ar[1];
                            $PROXY['port'] = $ar[2];
                            $PROXY['external_ip'] = $ar[3];
                            $PROXY['ready'] = 1;
                            usleep(230000); // additional time to avoid connecting during proxy bootup phase, removing/reducing this can cause random connection failures but will increase overall performance for very large plans
                            return 1;
                            break;
                        default:
                            echo "API Error: Received answer $ar[0], expected \"ROTATE\"";

                            return -101; // unknown API response
                    }
                }
            } // cmd==rotate
        }
    }

    // obtain a fresh IP through us-proxies.com API
    function rotate_proxy()
    {
        global $PROXY;
        global $ch;
        $max_errors = 3;
        $success = 0;
        while ($max_errors--)
        {
            $res = ip_service("rotate"); // will fill $PROXY
            $ip = "";
            if ($res <= 0)
            {
                verbose("API error: Proxy API connection failed (Error $res). trying again soon..\n\n");
                sleep(21); // retry after a while, maybe a routing failure
            } else
            {
                verbose("\tAPI success: Received new private IP\n"); // $PROXY[external_ip] on port $PROXY[port] // reduced message length a bit
                $success = 1;
                break;
            }
        }
        if ($success)
        {
            $ch = new_curl_session($ch);

            return 1;
        } else
        {
            return "API rotation failed. Check account, firewall and API credentials.\n";
        }
    }

    // gets company matches for one keyword
    function scrape_matches($keyword, $api_finance_data)
    {
        global $ch;

        $data = array('success' => 0, 'data' => array());
        $google_ip = $api_finance_data['domain'];
        $matchtype = $api_finance_data['matchtype'];

        $keyword_enc = urlencode($keyword);
        $url = "http://$google_ip/finance/match?matchtype=$matchtype&q=$keyword_enc";


        curl_setopt($ch, CURLOPT_URL, $url);
        $htmdata = curl_exec($ch);
        if (!$htmdata)
        {
            $error = curl_error($ch);
            $info = curl_getinfo($ch);
            echo "\tError scraping: $error [ $error ]\n";
            sleep(3);

            return $data;
        } else
        {
            if (strlen($htmdata) < 2)
            {
                sleep(3);
                echo "\tError scraping: empty result\n";

                return $data;
            }
        }
        if (($data_ar = json_decode($htmdata, true)) !== null)
        {

            if (isset($data_ar['matches']))
            {
                $matches=array();
                foreach ($data_ar['matches'] as $match)
                {
                    $tmp=array();
                    $tmp['short']=$match['t'];
                    $tmp['title']=$match['n'];
                    $tmp['market']=$match['e'];
                    $tmp['id']=$match['id'];
                    $matches[]=$tmp;
                }
                $data['success'] = 1;
                $data['data'] = $matches;
            }
        }

        return $data;
    }

    // wrapper to get a full set of keywords
    function get_companies(&$dataset, $keywords, $api_finance_data, $test_mode, $max_results = 0xffffff, $max_details = 0xffffff)
    {
        global $test_force_cache;
        global $test_force_cache_details;

        $rotate_now = 0; // set to 1 to force a rotation after launch, even if IP is not marked as overused
        $empty_counter = 0; // count empty replies
        $result = array('success' => 0);
        $start_time=time();
        $company_detail_count=0;

        $rcounter = 1; // used for rotation calls (do not modify!)
        $counter = 0; // used for regular reportings
        foreach ($keywords as $idx => $keys)
            foreach ($keys as $keyword)
            {
                verbose("Scraping matches for length $idx and search term '$keyword'\n");
                $cdata = load_cache($keyword, $api_finance_data, $test_force_cache, 'matches', 24*7*4*2); // default is two month expiration of cache data
                // check IP usage
                $ip_ready = check_ip_usage(); // test if ip has not been used within the critical time
                // obtain new IP if necessary
                if (!$cdata) // omit all of this if we have a cache
                {
                    if ((!$ip_ready || $rotate_now))
                    {
                        while (!$ip_ready || $rotate_now) // test if the IP is ready or overused
                        {
                            $ok = rotate_proxy(); // start/rotate to the IP that has not been started for the longest time, also tests if proxy connection is working
                            if ($ok != 1)
                            {
                                echo("Fatal error: proxy rotation failed:\n $ok\n");
                                $result['success'] = -1;

                                return $result;
                            }
                            $ip_ready = check_ip_usage(); // test if ip has not been used within the critical time
                            if (!$ip_ready)
                            {
                                echo("Fatal error: No fresh IPs left, wait a while and retry or obtain a larger plan. \n"); // proper error handling relies on exclusive use of the plan and rotation randomization == 0
                                $result['success'] = -2;

                                return $result;
                            } else
                            {
                                $rotate_now = 0;
                                delay_time('ip'); // proper delay
                                break; // ip rotated successfully
                            }
                        }
                    } else
                    {
                        delay_time('request');
                    }
                }


                if ($cdata)
                {
                    // we have the data already in cache
                    $result['success']++;
                    $dataset['matches'][$keyword] = $cdata['data'];
                } else
                {
                    // we have to make a live request
                    $scrape_result = scrape_matches($keyword, $api_finance_data);
                    if ($scrape_result['success'] == 1)
                    {
                        if (!($rcounter++ % 5)) $rotate_now = 1;
                        $result['success']++;
                        $result['errors'] = 0;
                        $dataset['matches'][$keyword] = $scrape_result['data'];
                        mark_ip_usage(); // store IP usage, this is very important to avoid detection and gray/blacklistings
                        $cdata['keyword'] = $keyword;
                        $cdata['type'] = 'match';
                        $cdata['result_count'] = count($scrape_result['data']);
                        $cdata['data'] = $scrape_result['data'];
                        store_cache($cdata, $keyword, $api_finance_data, 'matches'); // store results into local cache
                    } else
                    {
                        if (!($empty_counter++ % 5)) $rotate_now = 1;
                        $dataset['matches'][$keyword] = array();
                        if ($result['errors']++ > 20)
                        {
                            echo "More than 20 errors without results in between, hard abort\n";
                            $result['success'] = -3;
                            return $result;
                        }

                    }
                }




                if (($test_mode == 'matches_and_details') || ($test_mode == 'details_only') )
                {
                    // We scrape for all companies and detail pages at the same time
                    $matches_to_scrape=$dataset['matches'][$keyword];
                    if ($test_mode == 'details_only')
                    {
                        $match = each($dataset['matches'][$keyword]); $match=$match[1];
                        $matches_to_scrape=array($match);
                    }

                    foreach ($matches_to_scrape as $match)
                    {
                        if ($company_detail_count++ >= $max_details)
                        {
                            echo "Finished max detail count of $max_details\n";
                            return $result;
                        }
                        $identification= $match['market'].":".$match['short'];
                        verbose ("\tScraping stockinfo details for $identification ($match[title]) #$match[id]\n");
                        $cdata = load_cache($identification, $api_finance_data, $test_force_cache_details, 'stockinfo', 3); // 1 hour caching, if you want realtime data use 0 or
                        // check IP usage
                        $ip_ready = check_ip_usage(); // test if ip has not been used within the critical time
                        // obtain new IP if necessary
                        if (!$cdata) // omit all of this if we have a cache
                        {
                            if ((!$ip_ready || $rotate_now))
                            {
                                while (!$ip_ready || $rotate_now) // test if the IP is ready or overused
                                {
                                    $ok = rotate_proxy(); // start/rotate to the IP that has not been started for the longest time, also tests if proxy connection is working
                                    if ($ok != 1)
                                    {
                                        echo("Fatal error: proxy rotation failed:\n $ok\n");
                                        $result['success'] = -1;

                                        return $result;
                                    }
                                    $ip_ready = check_ip_usage(); // test if ip has not been used within the critical time
                                    if (!$ip_ready)
                                    {
                                        echo("Fatal error: No fresh IPs left, wait a while and retry or obtain a larger plan. \n"); // proper error handling relies on exclusive use of the plan and rotation randomization == 0
                                        $result['success'] = -2;

                                        return $result;
                                    } else
                                    {
                                        $rotate_now = 0;
                                        delay_time('ip'); // proper delay
                                        break; // ip rotated successfully
                                    }
                                }
                            } else
                            {
                                delay_time('request');
                            }
                        }


                        if ($cdata)
                        {
                            // we have the data already in cache
                            $result['success_detail']++;
                            $identification= $match['market'].":".$match['short'];
                            $dataset['stockinfo'][$identification] = $cdata['data'];
                        } else
                        {
                            // need to scrape

                            $scrape_result= get_company_details( $match['short'],$match['market'], $api_finance_data, 'm'); // minute mode
                            $identification= $match['market'].":".$match['short'];
                            if ($scrape_result['success'] == 0)
                            {

                                verbose("\t\tScraping realtime price failed, trying to the last price within a month (24h price resolution)\n");
                                $scrape_result= get_company_details( $match['short'],$match['market'], $api_finance_data, 'd'); // daily mode
                            }
                            if ($scrape_result['success'] == 1)
                            {
                                verbose("\tsuccessfully scraped stock prices for $identification\n");
                                $rotate_now = 1;
                                $result['success_detail']++;
                                $result['errors_details'] = 0;
                                mark_ip_usage(); // store IP usage, this is very important to avoid detection and gray/blacklistings
                                $cdata['keyword'] = $identification;
                                $cdata['type'] = 'stockinfo';
                                $cdata['data'] = $scrape_result['data'];
                                store_cache($cdata, $identification, $api_finance_data, 'stockinfo'); // store results into local cache
                                $dataset['stockinfo'][$identification]=$scrape_result['data'];
                            } elseif ($scrape_result['success'] == 0)
                            {
                                verbose("\tscraping exchange details for $identification failed due to no price data available\n");
                                $result['errors_details'] = 0;
                                mark_ip_usage(); // store IP usage, this is very important to avoid detection and gray/blacklistings
                                $cdata['keyword'] = $identification;
                                $cdata['type'] = 'stockinfo';
                                $cdata['data'] = $scrape_result['data'];
                                store_cache($cdata, $identification, $api_finance_data, 'stockinfo'); // store results into local cache
                                $dataset['stockinfo'][$identification]=$scrape_result['data'];
                            } else
                            {
                                verbose("\tscraping exchange details for $identification failed due to an error\n");
                                $cdata['keyword'] = $identification;
                                $cdata['type'] = 'stockinfo';
                                $cdata['error'] = 1;
                                $cdata['data'] = $scrape_result['data'];
                                store_cache($cdata, $identification, $api_finance_data, 'stockinfo'); // store error into local cache
                                $rotate_now = 1;
                                $dataset['stockinfo'][$identification] = array();
                                if ($result['errors_details']++ > 10)
                                {
                                    echo "More than 10 stockinfo errors without results in between, hard abort due to possible detection\n";
                                    $result['success'] = -30;
                                    return $result;
                                }

                            }
                        } // cache else
                        $identification= $match['market'].":".$match['short'];
                        $age=time()-$cdata['data']['timestamp_price'];verbose("\t\t$match[short] on exchange '{$cdata['data']['exchange']}': USD {$cdata['data']['price']}; Price age:$age\n");



                        echo "\n";
                    }
                }

                $num_results = count_results($dataset, 'matches');
                if ($num_results >= $max_results)
                {
                    echo "reached configured max results, ending..\n";
                    break;
                } else
                {
                    $spent = time()-$start_time;
                    $time_str="$spent seconds";
                    if ($spent > 3600)
                        $time_str=(int)($spent/60)." minutes";
                    if (!($counter++ % 10)) verbose( "\033[1m Time spent: $time_str, matches: $num_results\033[0m\n");
                }

            }

        return $result;
    }

    // get details, fill/.oad cache and dataset, return success or failure
    // on success a new IP rotation and proper delay is required to prevent getting blocked
    // rotation and cache support are included within this function to keep this function usable standalone
    // functions returns the data and success codes but also manages $dataset
    function get_company_details( $id,$ex, $api_finance_data, $test_detail_accuracy='m')
    {
        global $ch;

        $data = array('success' => 0, 'data' => array());
        $google_ip = "www.google.com";
        $i = $api_finance_data['get_price_now']['i'];
        $p = $api_finance_data['get_price_now']['p'];
        $f = $api_finance_data['get_price']['f'];

        if ($test_detail_accuracy == 'd')
        {
            // Daily accuracy mode, only useful for failed scrapes (closed exchanges, dead companies).
            $i = $api_finance_data['get_price_day']['i'];
            $p = $api_finance_data['get_price_day']['p'];
        }


        $url = "http://$google_ip/finance/getprices?q=$id&x=$ex&i=$i&p=$p&f=$f";

        curl_setopt($ch, CURLOPT_URL, $url);
        $htmdata = curl_exec($ch);
        if (!$htmdata)
        {
            $error = curl_error($ch);
            $info = curl_getinfo($ch);
            echo "\tError scraping: $error [ $error ]\n";
            sleep(3);

            return $data;
        } else
        {
            if (!strstr($htmdata,'EXCHANGE'))
            {
                sleep(3);
                echo "\tError scraping: invalid result\n";
                return $data;
            }
        }
        $htmdata=urldecode($htmdata);
        $regex ='/^EXCHANGE=([^\n$]*).*^INTERVAL=([\d]*).*^a([\d]+).*^([\d]{1,3}?),([\d\.]+)/sm';
        preg_match($regex, $htmdata, $results);
        if (isset($results[5]))
        {
            $data['success']=1;
            $data['data']['exchange']=$results[1]; // stock exchange
            $data['data']['price']=floatval($results[5]); //live stock closing price
            $data['data']['timestamp_scrape']=time();
            $data['data']['timestamp_price']=(int)$results[2]*(int)$results[4]+(int)$results[3]; // calculate the correct timestamp

        } else
        {
            $regex = '/^EXCHANGE=([^\n$]*).*/sm';
            preg_match($regex, $htmdata, $results);
            if (isset($results[1]))
            {
                $data['success']=0; // scrape successful but no price information available or no price information in selected accuracy mode found
                $data['data']['exchange']=$results[1]; // stock exchange
                $data['data']['price']='-1';
                $data['data']['timestamp_scrape']=time();
                $data['data']['timestamp_price']=0;
            } else
                $data['success']=-1; // hard error, response returned did not include the correct data
        }
        return $data;

    }


    // counts all results
    function count_results(&$dataset, $type='matches')
    {
        $num = 0;
        if ($type == 'matches')
        {
            foreach ($dataset['matches'] as $kw => $results) $num += count($results);
        }

        return $num;
    }


    function combine($len, $chars=NULL, $result='')
    {
        static $resultarray=array();
        static $max_len=null;
        if (is_null($max_len))$max_len = $len;
//        if (!$resultarray) $resultarray=new SplFixedArray(26^4); // php 5.3, optional memory reduction feature
        if (!$chars) $chars=range('A','Z');
        if ($len > 0)
        {
            if ($result != '') $resultarray[$max_len-$len][]=$result;
            foreach ($chars as $char)
            {
                combine($len - 1, $chars, $result.$char);
            }

        } else
            $resultarray[$max_len-$len][]=$result;
        if ($result == '')
            return $resultarray;

    }

    function display_results(&$dataset)
    {
        $mask = "| %-10.10s | %-10.10s | %-70.70s | %-10.10s | %-15.15s |\n";
        $separator=str_repeat("-", 70);

        echo "\nGoogle Finance Exchange Spider results (unsorted)\n";
        printf($mask, 'Identifier', 'Exchange', 'Company name', 'Price', 'Price time');
        printf($mask, $separator, $separator,$separator,$separator,$separator);

        foreach ($dataset['matches'] as $kw => $matches)
        {
            foreach ($matches as $match)
            {
                $identification= $match['market'].":".$match['short'];
                $identifier=$match['short'];
                $company=$match['title'];
                $exchange='UNKNOWN EXCHANGE';
                $price='unknown';
                $timestamp_price='unknown';
                if (isset($dataset['stockinfo'][$identification]['exchange'])) $exchange=$dataset['stockinfo'][$identification]['exchange'];
                if (isset($dataset['stockinfo'][$identification]['price'])) $price=$dataset['stockinfo'][$identification]['price'];
                if (isset($dataset['stockinfo'][$identification]['timestamp_price']))
                {
                    $timestamp_price=gmdate("H:i:s", $dataset['stockinfo'][$identification]['timestamp_price'])." GMT";

                }

                printf($mask, $identifier, $exchange, $company, $price, $timestamp_price);
            }
        }

    }


?>