"""
Shows different use cases of the library.
"""

from __future__ import print_function
from pyds import MassFunction
from itertools import product


print('=== creating mass functions ===')

m1 = MassFunction({'ab':0.6, 'bc':0.3, 'a':0.1, 'ad':0.0}) # using a dictionary
print('m_1 =', m1)

m2 = MassFunction([({'a', 'b', 'c'}, 0.2), ({'a', 'c'}, 0.5), ({'c'}, 0.3)]) # using a list of tuples
print('m_2 =', m2)

m3 = MassFunction()
m3['bc'] = 0.8
m3[{}] = 0.2
print('m_3 =', m3, ('(unnormalized mass function)'))

#print('\n=== belief, plausibility, and commonality ===')
#print('bel_1({a, b}) =', m1.bel({'a', 'b'}))
#print('pl_1({a, b}) =', m1.pl({'a', 'b'}))
#print('q_1({a, b}) =', m1.q({'a', 'b'}))
#print('bel_1 =', m1.bel()) # entire belief function
#print('bel_3 =', m3.bel())
#print('m_3 from bel_3 =', MassFunction.from_bel(m3.bel())) # construct a mass function from a belief function

#print('\n=== frame of discernment, focal sets, and core  ===')
#print('frame of discernment of m_1 =', m1.frame())
#print('focal sets of m_1 =', m1.focal())
#print('core of m_1 =', m1.core())
#print('combined core of m_1 and m_3 =', m1.core(m3))

#print('\n=== Dempster\'s combination rule, unnormalized conjunctive combination (exact and approximate) ===')
#print('Dempster\'s combination rule for m_1 and m_2 =', m1 & m2)
#print('Dempster\'s combination rule for m_1 and m_2 (Monte-Carlo, importance sampling) =', m1.combine_conjunctive(m2, sample_count=1000, importance_sampling=True))
#print('Dempster\'s combination rule for m_1, m_2, and m_3 =', m1.combine_conjunctive(m2, m3))
#print('unnormalized conjunctive combination of m_1 and m_2 =', m1.combine_conjunctive(m2, normalization=False))
#print('unnormalized conjunctive combination of m_1 and m_2 (Monte-Carlo) =', m1.combine_conjunctive(m2, normalization=False, sample_count=1000))
#print('unnormalized conjunctive combination of m_1, m_2, and m_3 =', m1.combine_conjunctive([m2, m3], normalization=False))

#print('\n=== normalized and unnormalized conditioning ===')
#print('normalized conditioning of m_1 with {a, b} =', m1.condition({'a', 'b'}))
#print('unnormalized conditioning of m_1 with {b, c} =', m1.condition({'b', 'c'}, normalization=False))

#print('\n=== disjunctive combination rule (exact and approximate) ===')
#print('disjunctive combination of m_1 and m_2 =', m1 | m2)
#print('disjunctive combination of m_1 and m_2 (Monte-Carlo) =', m1.combine_disjunctive(m2, sample_count=1000))
#print('disjunctive combination of m_1, m_2, and m_3 =', m1.combine_disjunctive([m2, m3]))

#print('\n=== weight of conflict ===')
#print('weight of conflict between m_1 and m_2 =', m1.conflict(m2))
#print('weight of conflict between m_1 and m_2 (Monte-Carlo) =', m1.conflict(m2, sample_count=1000))
#print('weight of conflict between m_1, m_2, and m_3 =', m1.conflict([m2, m3]))

#print('\n=== pignistic transformation ===')
#print('pignistic transformation of m_1 =', m1.pignistic())
#print('pignistic transformation of m_2 =', m2.pignistic())
#print('pignistic transformation of m_3 =', m3.pignistic())

print('\n=== local conflict uncertainty measure ===')
print('local conflict of m_1 =', m1.local_conflict())
print('entropy of the pignistic transformation of m_3 =', m3.pignistic().local_conflict())

#print('\n=== sampling ===')
#print('random samples drawn from m_1 =', m1.sample(5, quantization=False))
#print('sample frequencies of m_1 =', m1.sample(1000, quantization=False, as_dict=True))
#print('quantization of m_1 =', m1.sample(1000, as_dict=True))

#print('\n=== map: vacuous extension and projection ===')
#extended = m1.map(lambda h: product(h, {1, 2}))
#print('vacuous extension of m_1 to {1, 2} =', extended)
#projected = extended.map(lambda h: (t[0] for t in h))
#print('project m_1 back to its original frame =', projected)

#print('\n=== construct belief from data ===')
#hist = {'a':2, 'b':0, 'c':1}
#print('histogram:', hist)
#print('maximum likelihood:', MassFunction.from_samples(hist, 'bayesian', s=0))
#print('Laplace smoothing:', MassFunction.from_samples(hist, 'bayesian', s=1))
#print('IDM:', MassFunction.from_samples(hist, 'idm'))
#print('MaxBel:', MassFunction.from_samples(hist, 'maxbel'))
#print('MCD:', MassFunction.from_samples(hist, 'mcd'))

# -*- coding: utf-8 -*-

"""
This script implement algorithm from
THESIS "Automatic Detection of Click Fraud in Online Advertisements" AGARWAL
Calculate the belief of fraud
http://repositories.tdl.org/ttu-ir/bitstream/handle/2346/46429/AGARWAL-THESIS.pdf
Import library using pyds (https://github.com/reineking/pyds)
Require: Python3, numpy, scipy, redis (https://github.com/andymccurdy/redis-py)
Author: TrucLK
"""
from pyds import MassFunction
from itertools import product
import redis
import sys
import pprint
import time


################################################
class Config(object):
    """
    Redis for store click history
    """
    redis_host = '127.0.0.1'
    redis_port = 6379
    redis_db = 0

    # Time for redis key expire
    time_to_expire = 1800

    # Visitor length of checking
    visit_length = 1800

    # weight is an empirically derived value that signifies the strength of the evidence in supporting the user is fraud

    # caution changing it will change the maximum value of result.
    IDWeight = 0.5
    UAWeight = 0.4
    IPWeight = 0.4


class EcLogger(object):
    """
    This is object relate to log
    """

    def __init__(self):
        self.ro = redis.Redis(host=Config.redis_host, port=Config.redis_port, db=Config.redis_db)

    def record(self, hit):

        """
        Add click to history for checking
        """

        clickid = self.ro.incr('ec:clicknum')

        self.ro.zadd('click:ip:' + hit.ip, clickid, int(hit.time))
        self.ro.expire('click:ip:' + hit.ip, Config.time_to_expire)
        self.ro.zadd('click:cookie:' + hit.cookie, clickid, int(hit.time))
        self.ro.expire('click:cookie:' + hit.cookie, Config.time_to_expire)
        self.ro.zadd('click:config:' + hit.pubid + ':' + hit.config, clickid, int(hit.time))
        self.ro.expire('click:config:' + hit.pubid + ':' + hit.config, Config.time_to_expire)
        pass

    def getClickNumFromIp(self, hit):
        """
        Get click time from this IP address in this session
        """
        count = 0
        count = self.ro.zcount('click:ip:' + hit.ip, int(hit.time) - Config.visit_length, int(hit.time))
        if (count == 0):
            count = 1
        return count

    def getClickNumFromCookie(self, hit):
        """
        Get click time from this anonymous id in this session
        """
        count = self.ro.zcount('click:cookie:' + hit.cookie, int(hit.time) - Config.visit_length, int(hit.time))
        if (count == 0):
            count = 1
        return count

    def getClickNumFromConfig(self, hit):
        """
        Get cliock time from this user agent in this session
        """
        count = self.ro.zcount('click:config:' + hit.pubid + ':' + hit.config, int(hit.time) - Config.visit_length,
                               int(hit.time))
        if (count == 0):
            count = 1
        return count


class Hit(object):
    """
    It's a simple container.
    """

    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)
        super(Hit, self).__init__()


class ClickProcessingUnit(object):
    """
    Mass functions for Click Fraud Detection
    """

    def __init__(self, hit, ecLogger):
        self.hit = hit
        self.ecLogger = ecLogger


class ClickProcessingUnitIp(ClickProcessingUnit):
    """
    Evidence number of clicks on the ad by ip address
    Create mass function for ip address checking
    """

    def process(self):
        numberclick = self.ecLogger.getClickNumFromIp(self.hit)
        coefficient_value = Config.IPWeight

        a = coefficient_value * (1 - 1 / numberclick)
        b = 0
        ab = 1 - a

        massfunction = MassFunction({'a': a, 'b': b, 'ab': ab})
        return massfunction;


class ClickProcessingUnitCookie(ClickProcessingUnit):
    """
    Evidence number of clicks on the ad by user ID
    Create mass function for id cookie checking
    """

    def process(self):
        numberclick = self.ecLogger.getClickNumFromCookie(self.hit)
        coefficient_value = Config.IDWeight

        a = coefficient_value * (1 - 1 / numberclick)
        b = 0
        ab = 1 - a

        massfunction = MassFunction({'a': a, 'b': b, 'ab': ab})
        return massfunction;


class ClickProcessingUnitConfig(ClickProcessingUnit):
    """
    Evidence number of clicks on the ad by user agent
    Create mass function for user agent checking
    """

    def process(self):
        numberclick = self.ecLogger.getClickNumFromConfig(self.hit)
        coefficient_value = Config.UAWeight

        a = coefficient_value * (1 - 1 / numberclick)
        b = 0
        ab = 1 - a

        massfunction = MassFunction({'a': a, 'b': b, 'ab': ab})
        return massfunction


class ClickProcessing(object):
    """
    Main click processing function
    """

    def __init__(self, hit, ecLogger):
        self.hit = hit
        self.ecLogger = ecLogger

    def process(self):
        processingList = self.getListOfProcessing()
        m = None
        # Loop for list of processing class
        for processing in processingList:
            # Init first mass function
            if not m:
                m = processing.process()
            else:
                # Dempster's rule of combination create a new mass function
                m = m & processing.process()
        return m

    def getListOfProcessing(self):
        """
        Config list of processing
        """
        dict1 = []
        dict1.append(ClickProcessingUnitIp(self.hit, self.ecLogger))
        dict1.append(ClickProcessingUnitCookie(self.hit, self.ecLogger))
        dict1.append(ClickProcessingUnitConfig(self.hit, self.ecLogger))
        return dict1


if __name__ == '__main__':
    """
    Add click from command line argument
    """
    try:
        hit = hit = Hit(
            ip=sys.argv[1],
            time=sys.argv[2],
            config=sys.argv[3],
            cookie=sys.argv[4],
            pubid=sys.argv[5],
        )
        ecLogger = EcLogger()
        processing = ClickProcessing(hit, ecLogger)
        ecLogger.record(hit)
        m = processing.process()
        print(m.bel('a'))
        sys.exit(0)
    except Exception:
        print(0.01)
        sys.exit(0)

# -*- coding: utf-8 -*-

import os  
import time
import numpy as np  
import scipy.io as sio  
import sklearn

# 測試樣本對應的真實格點坐標
def test_real(path):
    real1=[]
    for i in range(grid_num):
        for j in range(grid_test_sample):
            real1.append(i+1)
    real=np.array(real1)
    return real


# 模型：一共有56個格點，每個格點上訓練數據100組，測試數據10組
def read_data(path):
    pathtrain = os.path.join(path,'train.mat')
    traindataset = sio.loadmat(pathtrain).get('data')   # 訓練數據

    pathtest = os.path.join(path,'test.mat')
    testdataset = sio.loadmat(pathtest).get('X_test')   # 測試數據
    trainlabelset1 = []
    for grid in range(grid_num):
        for sample in range(grid_train_sample):
            local_tmp = grid + 1
            trainlabelset1.append(local_tmp)
    trainlabelset = np.array(trainlabelset1)    # 訓練數據集對應坐標
    testlabelset = test_real(path)  # 測試數據集對應坐標
    return traindataset, trainlabelset, testdataset, testlabelset

  
# KNN Classifier  
# KNeighborsClassifier(n_neighbors=5, weights=』uniform』, algorithm=』auto』, leaf_size=30, p=2,
#                      metric=』minkowski』, metric_params=None, n_jobs=1, **kwargs)
def knn_classifier(train_x, train_y):  
    from sklearn.neighbors import KNeighborsClassifier  
    model = KNeighborsClassifier()  
    model.fit(train_x, train_y)  
    return model  
  
  
# Logistic Regression Classifier  
# LogisticRegression(penalty=』l2』, dual=False, tol=0.0001, C=1.0, fit_intercept=True,
#                   intercept_scaling=1, class_weight=None, random_state=None, solver=』liblinear』,
#                   max_iter=100, multi_class=』ovr』, verbose=0, warm_start=False, n_jobs=1)
def logistic_regression_classifier(train_x, train_y):  
    from sklearn.linear_model import LogisticRegression  
    model = LogisticRegression(penalty='l2')  
    model.fit(train_x, train_y)  
    return model  
  
  
# Random Forest Classifier  
def random_forest_classifier(train_x, train_y):  
    from sklearn.ensemble import RandomForestClassifier  
    model = RandomForestClassifier(n_estimators=8)  
    model.fit(train_x, train_y)  
    return model  
  
  
# Decision Tree Classifier  
def decision_tree_classifier(train_x, train_y):  
    from sklearn import tree  
    model = tree.DecisionTreeClassifier()  
    model.fit(train_x, train_y)  
    return model  
  
  
# GBDT(Gradient Boosting Decision Tree) Classifier  
def gradient_boosting_classifier(train_x, train_y):  
    from sklearn.ensemble import GradientBoostingClassifier  
    model = GradientBoostingClassifier(n_estimators=200)  
    model.fit(train_x, train_y)  
    return model  
  
  
# SVM Classifier  
def svm_classifier(train_x, train_y):  
    from sklearn.svm import SVC  
    model = SVC(kernel='rbf', probability=True)  
    model.fit(train_x, train_y)  
    return model  
  
# SVM Classifier using cross validation  
def svm_cross_validation(train_x, train_y):  
    from sklearn.grid_search import GridSearchCV  
    from sklearn.svm import SVC  
    model = SVC(kernel='rbf', probability=True)  
    param_grid = {'C': [1e-3, 1e-2, 1e-1, 1, 10, 100, 1000], 'gamma': [0.001, 0.0001]}  
    grid_search = GridSearchCV(model, param_grid, n_jobs = 1, verbose=1)  
    grid_search.fit(train_x, train_y)  
    best_parameters = grid_search.best_estimator_.get_params()  
    for para, val in best_parameters.items():  
        print para, val  
    model = SVC(kernel='rbf', C=best_parameters['C'], gamma=best_parameters['gamma'], probability=True)  
    model.fit(train_x, train_y)  
    return model  
    
def func_rmse(predict_label,real):
    pathlocal=os.path.join(path,'dist1.mat')
    local=sio.loadmat(pathlocal).get('dist')#56個格點坐標
    tmp = predict_label-np.ones(np.shape(predict_label))
    predict_2d = local[map(int,tmp)]
    tmp_real = real-np.ones(np.shape(predict_label))
    real_2d = local[map(int,tmp_real)]
    norm_sqrt = (np.linalg.norm(predict_2d-real_2d,axis=-1))**2
    rmse = np.sqrt(sum(norm_sqrt)/np.shape(predict_label))
    return rmse
    
def get_probability(data_chuang):
    """
    對窗數據來說，維度為：len_chuang*演算法個數
    計算每種演算法下格點序號出現次數及概率
    """
    mygrid = range(1,57)
    cishu_total=[]
    for sf in range(len_sf):
        cishu=[]
        for item in mygrid:
            cishu_sf = data_chuang[:,sf].tolist().count(item)
            cishu.append(cishu_sf)
        cishu = np.array(cishu)
        cishu_total.append(cishu)
    probio_total = [item/float(len_chuang) for item in cishu_total]
    probio_total = np.array(probio_total).transpose()
    return probio_total
    
    
if __name__ == '__main__':  
    path = os.path.abspath('.')
    thresh = 0.5  
    model_save_file = None  
    model_save = {}  
    grid_num = 56
    grid_train_sample = 100
    grid_test_sample = 10
      
    test_classifiers = ['KNN', 'LR', 'RF', 'DT', 'SVM']
    classifiers = {
        'KNN': knn_classifier,
        'LR': logistic_regression_classifier,
        'RF': random_forest_classifier,
        'DT': decision_tree_classifier,
        'SVM': svm_classifier,
        'SVMCV': svm_cross_validation,
        'GBDT': gradient_boosting_classifier
    }  
      
    print 'reading training and testing data...'

    train_x, train_y, test_x, test_y = read_data(path)
    # print "test_x: ", test_x
    # print "test_y: ", test_y
    num_train, num_feat = train_x.shape  
    num_test, num_feat = test_x.shape  
    is_binary_class = (len(np.unique(train_y)) == 56)

    print '******************** Data Info *********************'  
    print '#training data: %d, #testing_data: %d, dimension: %d' % (num_train, num_test, num_feat)  
    
    predict_total2 = []
    rmse_total = []
    for classifier in test_classifiers:  
        print '******************* %s ********************' % classifier  
        start_time = time.time()  
        model = classifiers[classifier](train_x, train_y)  
        print 'training took %fs!' % (time.time() - start_time)  
        predict = model.predict(test_x)  
        if model_save_file != None:  
            model_save[classifier] = model  
        if is_binary_class:
            tmp = func_rmse(predict, test_y)
            rmse_total.append(tmp)
        accuracy = sklearn.metrics.accuracy_score(test_y, predict)  
        print 'accuracy: %.2f%%' % (100 * accuracy)   
        predict_total2.append(predict)
    predict_total1 = np.array(predict_total2)
    predict_total = predict_total1.transpose()

    print "predict_total: ", predict_total
    
    """
    對已經被機器學習演算法進行預測之後的數據矩陣predict_total（維度為:測試樣本數*演算法數）
    滑窗對數據集進行處理，窗長度為len_chuang，每次下滑一格
    對窗長度內的數據進行概率計算並DS融合
    """
    len_chuang = 10     # 窗數據長度
    pre_ds = []
    pre_dsjq = []
    pre_max = []
    pre_test = []
    for start in range(0, 560 - len_chuang + 1):    # 滑動窗口
        data_chuang = predict_total[start:len_chuang+start, :]  # 取出窗長度的數據
        len_sf = len(data_chuang[0])    # 多演算法個數
        probio_total = get_probability(data_chuang)     # 窗數據矩陣中每種演算法對應所有格點出現概率
        
        """
        1.普通ds
         等同於求多種演算法概率平均之後，概率最大的格點為所預測格點
        """
        init_mean = np.mean(probio_total,1) # 每個格點下多種演算法提供的平均概率
        init_ds = init_mean
        # 這部分並沒有起到實際作用
        for i in range(len_sf-1):
            init_ds = np.multiply(init_ds,init_mean)
            k = sum(init_ds)
            bel = init_ds/float(k)
        pre_grid = np.argmax(bel)+1
        pre_ds.append(pre_grid)
        """
        2.加強ds
         求得多種演算法的源可信度，得到估計
        """
        m_between = np.zeros([len_sf,len_sf])
        for i in range(len_sf):
            for j in range(len_sf):
                m_between[i, j]=sum(np.multiply(probio_total[:,i],probio_total[:,j]))
                # m_between兩個證據的內積=兩個證據中，每個出現事件的概率乘積*（出現事件的交集/並集）
                # 對單個事件而不是集合事件而言，等同於對應事件的概率乘積之和
        d = np.zeros([len_sf,len_sf])
        sim = np.zeros([len_sf,len_sf])
        for i in range(len_sf):
            for j in range(len_sf):
                d[i,j]=np.sqrt(0.5*(m_between[i,i]+m_between[j,j]-2*m_between[i,j]))
                # d為兩個證據間的距離，距離越小表示兩個證據提出的意見越一致
                sim[i,j]=1-d[i,j]
                # sim為兩個證據之間的相似度，越大代表兩個證據之間的一致性越強
        sup = np.zeros(len_sf)
        for i in range(len_sf):
            sup[i]=sum(sim[i,:])-sim[i,i]
            # sup為對每個證據的支援度，為兩個證據之間的相似度之和減去該證據自己對自己的支援度
            # 證據對自己的支援度為1
        crd = np.zeros(len_sf)
        for i in range(len_sf):
            crd[i]=float(sup[i])/sum(sup)
            # crd為證據的可信度
            # 即為歸一化的支援度，其他證據對該證據支援度越高，則可信度越高
        A = np.zeros(grid_num)
        for i in range(grid_num):
            A[i] = sum(np.multiply(probio_total[i,:],crd))
            # 將可信度作為源權重，估計所有情況下數據出現的概率
        AA = A
        
        # 這部分並沒有起到實際作用
        # 對於所有元素均為0-1的概率值，進行元素對於相乘之後並不改變元素的大小排序
        for i in range(len_sf-1):
            init_ds = np.multiply(AA,A)
            # 分子為與某事件有交集的事件概率之乘積
            k = sum(init_ds)
            # 分母K=∑(所有有交集的事件的概率乘積)
            # 或者為1-∑(所有不相交的時間概率乘積)
            # 對全部都是單事件而不是集體事件而言，有交集的事件即為事件其本身
            # K表示了證據的衝突程度，衝突越大，越接近0，一致性越大，越接近1
            bel = init_ds/float(k)
        pre_grid = np.argmax(bel)+1
        pre_dsjq.append(pre_grid)
        # 下面兩行程式碼是為了驗證多次融合A矩陣並不能起到結果
        testgrid = np.where(A == np.max(A))[0][0]
        pre_test.append(testgrid+1)
        """
        3.使用窗數據中出現次數最多的作為正確結果
        """
        data_chuang_list = data_chuang.flatten().tolist()
        matrix_cishu = max(data_chuang_list,key=data_chuang_list.count)
        pre_max.append(matrix_cishu)
        
    truth_chuang = test_y[len_chuang-1:]
    rmse_ds = func_rmse(pre_ds, truth_chuang)
    rmse_max = func_rmse(pre_max, truth_chuang)
    rmse_dsjq = func_rmse(pre_dsjq, truth_chuang)
    rmse_test = func_rmse(pre_test, truth_chuang)

    print "pre_ds: ", pre_ds
    print "rmse_ds: ", rmse_ds
    accuracy = sklearn.metrics.accuracy_score(test_y[len_chuang-1:], pre_ds)
    print 'accuracy: %.2f%%' % (100 * accuracy)