# -*- coding: utf-8 -*-
"""
Created on Mon Sept 21 8:04:59 2020

@author: mly
"""
import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
import pandas as pd
from matplotlib import ticker

plt.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams["axes.unicode_minus"] = False

df = pd.read_csv('basicsanit_china2000to2017.csv')

y = df['rural_sanit']
y1 = df['urban_sanit']
y2 = df['peopl_sanit']
x = df['year']

plt.figure()
ax = plt.gca()
plt.grid(axis="y")
plt.title('农村居民与城市居民享受基本公共卫生服务的变化情况')
plt.ylabel('服务数值')
plt.xlabel('年份')
ax.plot(x, y, '-rp', lw = 1.5, label = 'rural_sanit')
ax.plot(x, y1, '-gp', lw = 1.5, label = 'rural_sanit')
ax.plot(x, y2, '-bp', lw = 1.5, label = 'peopl_sanit')
ax.legend(loc = 'upper right')

plt.show()

import requests
from bs4 import BeautifulSoup
import re


def getHTMLText(url, code="utf-8"):
    kv = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
    try:
        r = requests.get(url, headers=kv)
        r.raise_for_status()
        r.encoding = code
        return r.text
    except:
        return ""


def getStockList(lst, stockURL):
    html = getHTMLText(stockURL, "GB2312")
    soup = BeautifulSoup(html, 'html.parser')
    li = soup.find('section', attrs={'class': 'stockTable'})
    a = li.find_all('a')
    for i in a:
        try:
            href = i.attrs['href']
            lst.append(re.findall(r"[S][HZ]\d{6}", href)[0])
        except:
            continue


def getStockInfo(lst, stockURL, fpath):
    count = 0
    for stock in lst:
        url = stockURL + stock
        html = getHTMLText(url)
        try:
            if html == "":
                continue
            infoDict = {}
            soup = BeautifulSoup(html, 'html.parser')
            stockInfo = soup.find('section', attrs={'class': 'stock_price clearfix'})
            mc = soup.find('header', attrs={'class': 'stock_title'})
            name = mc.find('h1')
            infoDict.update({'股票名称': name.text})

            keyList = stockInfo.find_all('dt')
            valueList = stockInfo.find_all('dd')
            for i in range(len(keyList)):
                key = keyList[i].text
                val = valueList[i].text
                infoDict[key] = val

            with open(fpath, 'a', encoding='utf-8_sig') as f:
                f.write(str(infoDict) + '\n')
                count = count + 1
                print("\r当前进度: {:.2f}%".format(count * 100 / len(lst)), end="")
        except:
            count = count + 1
            print("\r当前进度: {:.2f}%".format(count * 100 / len(lst)), end="")
            continue


def main():
    stock_list_url = 'https://hq.gucheng.com/gpdmylb.html'
    stock_info_url = 'https://hq.gucheng.com/'
    output_file = 'BaiduStockInfo.csv'
    slist = []
    getStockList(slist, stock_list_url)
    getStockInfo(slist, stock_info_url, output_file)


main()

import matplotlib.pyplot as plt
import pandas as pd
import pymssql
import warnings
from pylab import *
import matplotlib.dates as mdates

warnings.filterwarnings('ignore')

connect = pymssql.connect('IP地址','用户名','密码','数据库名')
print("连接成功")
data = pd.read_sql("select TRADEDATE,cast(TCLOSE as int) from TQ_QT_SKDAILYPRICE where SECODE='2010000512'", con=connect)
#print(data.head()) #查看读取的结果
data.columns = ['day','close']
#print(data.head()) #查看读取的结果
data['day'] = pd.to_datetime(data['day'])    #轉換為日期，否則下面的日期設置不會生效

#matplotlib.pyplot方式
plt.rcParams['font.family'] = ['sans-serif']
plt.rcParams['font.sans-serif'] = ['SimHei']

fig = plt.figure(figsize=(20, 5))
ax = fig.add_subplot(1, 1, 1)

plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))  #設置x軸主刻度顯示格式（日期）
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=15))  #設置x軸主刻度間距

plt.xlabel('日期')
plt.ylabel('收盘价')
plt.title('2010000512收盘价折线图')
plt.plot(data['day'],data['close'])
plt.show()

fig = plt.figure(figsize=(20, 5))

plt.rcParams['font.family'] = ['sans-serif']
plt.rcParams['font.sans-serif'] = ['SimHei']

import matplotlib.dates as mdates
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))  

import matplotlib.dates as mdates
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=15))

'bmh','dark_background','fivethirtyeight','ggplot','grayscale','default'

#coding=utf-8
'''
Created on 2020.09.29
'''
import matplotlib.pyplot as plt
import pandas as pd

def dr_xtyp(_dat):
    #xtyp=['bmh','dark_background','fivethirtyeight','ggplot','grayscale','default']
    for xss in plt.style.available:
        plt.style.use(xss);print(xss)
        _dat['Open'].plot()
        _dat['Close'].plot()
        _dat['High'].plot()
        _dat['Low'].plot()
        fss="tmp\\stk001_"+xss+"_pd.png";plt.savefig(fss)
        plt.show()
    
# =======================
df = pd.read_csv('dat\\appl2014.csv',index_col=0,parse_dates=[0],encoding='gbk') 

d30=df[:30]
dr_xtyp(d30)

pip install scrapy

scrapy -h

scrapy startproject baidustocks

cd baidustocks
scrapy genspider stocks hq.gucheng.com/gpdmylb.html

# -*- coding: utf-8 -*-

import scrapy
import re
from scrapy.selector import Selector
 
 
class StocksSpider(scrapy.Spider):
    name = 'stocks'
    start_urls = ['https://hq.gucheng.com/gpdmylb.html']
 
    def parse(self, response):
        for href in response.css('a::attr(href)').extract():
            try:
                stock = re.search(r'S[HZ]\d{6}/', href)
                url = 'https://hq.gucheng.com/' + stock.group()
                yield scrapy.Request(url, callback=self.parse_stock)
            except:
                continue
 
    def parse_stock(self, response):
        infoDict = dict()
        stockInfo = response.css('.stock_top').extract()[0]
        stockprice = response.css('.s_price').extract()[0]
        stockname = response.css('.stock_title').extract()[0]
        stockname = Selector(text=stockname)
        stockprice = Selector(text=stockprice)
        stockInfo = Selector(text=stockInfo)
        infoDict['名字'] = re.search(r'>(.*?)</h1>', stockname.css('h1').extract()[0]).group(1)
        infoDict['编号'] = re.search(r'>(.*?)</h2>', stockname.css('h2').extract()[0]).group(1)
        infoDict['状态'] = re.search(r'>(.*?)</em>', stockname.css('em').extract()[0]).group(1)
        infoDict['时间'] = re.search(r'>(.*?)</time>', stockname.css('time').extract()[0]).group(1)
        price = stockprice.css('em').extract()
        infoDict['股价'] = re.search(r'>(.*?)</em>', price[0]).group(1)
        infoDict['涨跌额'] = re.search(r'>(.*?)</em>', price[1]).group(1)
        infoDict['涨跌幅'] = re.search(r'>(.*?)</em>', price[2]).group(1)
        keylist = stockInfo.css('dt').extract()
        valuelist = stockInfo.css('dd').extract()
        for i in range(len(keylist)):
            key = re.search(r'>(.*?)<', keylist[i], flags=re.S).group(1)
            key = str(key)
            key = key.replace('\n', '')
            try:
                val = re.search(r'>(.*?)<', valuelist[i], flags=re.S).group(1)
                val = str(val)
                val = val.replace('\n', '')
            except:
                val = '--'
            infoDict[key] = val
        yield infoDict

scrapy crawl stocks

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter


class BaidustocksPipeline:
    def process_item(self, item, spider):
        return item

class BaidustocksInfoPipeline(object):
    def open_spider(self, spider):
        self.f = open('BaiduStockInfo.txt', 'w')
 
    def close_spider(self, spider):
        self.f.close()
 
    def process_item(self, item, spider):
        try:
            line = str(dict(item)) + '\n'
            self.f.write(line)
        except:
            pass
        return item

scrapy crawl stocks

d = {"北京大学":"北京", "中国人民大学":"北京", "清华大学":"北京",
     "北京航空航天大学":"北京", "北京理工大学":"北京", "中国农业大学":"北京",
     "北京师范大学":"北京", "中央民族大学":"北京", "南开大学":"天津",
     "天津大学":"天津", "大连理工大学":"辽宁", "吉林大学":"吉林",
     "哈尔滨工业大学":"黑龙江", "复旦大学":"上海", "同济大学":"上海",
     "上海交通大学":"上海","华东师范大学":"上海", "南京大学":"江苏",
     "东南大学":"江苏", "浙江大学":"浙江", "中国科学技术大学":"安徽",
     "厦门大学":"福建", "山东大学":"山东", "中国海洋大学":"山东",
     "武汉大学":"湖北", "华中科技大学":"湖北", "中南大学":"湖南",
     "中山大学":"广东", "华南理工大学":"广东", "四川大学":"四川",
     "电子科技大学":"四川", "重庆大学":"重庆", "西安交通大学":"陕西",
     "西北工业大学":"陕西", "兰州大学":"甘肃", "国防科技大学":"湖南",
     "东北大学":"辽宁","郑州大学":"河南", "湖南大学":"湖南", "云南大学":"云南",
     "西北农林科技大学":"陕西", "新疆大学":"新疆"}
counts = {}
for place in d.values():
    counts[place] = counts.get(place, 0) + 1
items = list(counts.items())
items.sort(key=lambda x: x[1], reverse=True)
print(items[0][0]+'：'+str(items[0][1]))

import os

def isBigGrowth(L,rate):
    if float (L[2])>float(L[1])*(1+rate) and float(L[3])>float(L[2])*(1+rate) and float(L[4])>float(L[3])*(1+rate):
        return "快速"
    else:
        return "否"
        
with open(r"文件路径\smartPhone.txt") as f:
    line = f.read().strip()
    linestr = line.split("\n")  # 以换行符分隔
    del linestr[0]
print("手机公司    是否快速增长?")
for s in linestr:
    try:
        L = s.split('\t')
        print(L[0], end="    ")
        print(isBigGrowth(L, 0.3))
    except:
        print('运行失败')

import tushare as  ts
df = ts.get_money_supply()
df.head()
df.columns

import tushare as  ts
df = ts. get_loan_rate()
df.head()

import tushare as  ts
pro = ts.pro_api('用户tushare token')
df=pro.shibor()
df.head()

import  tushare  as  ts
df = ts.get_hist_data('600848', ktype = 'D')
df.head()

import tushare as  ts
pro = ts.pro_api()
df = pro.daily(ts_code = '600008.SH', start_date = '20000501', end_date = '20190808')
df.head()

import tushare as  ts
df = ts.get_realtime_quotes('300274')
df[['code','name','price','bid','ask','volume','amount','time']] 

import  tushare  as  ts
pro = ts.pro_api('用户tushare token')
df = pro.index_basic(market = 'SSE')
df.head()

import  tushare  as  ts
import  pandas as pd
import numpy as np
import  matplotlib.pyplot  as  plt

stock = ts.get_stock_basics()     # 下载股票基本面数据
stock.to_excel('stock.xlsx')     # 保存为电子表格
stock.shape                      # Out: (3678, 22)

df = pd.read_excel('stock.xlsx', dtype={'code': 'str'})   # code字符串类型
df.set_index('code', inplace=True)  # 将code设为索引列
df.loc['002522']                        # 显示某支股票基本面

len(df.industry.unique())   # 显示行业数

len(df.area.unique())  # 显示地区数（即股票的归属省份）

# 按地区统计上市公司数量，体现地区经济实力
df.groupby('area').size().sort_values(ascending=False)

year = df.timeToMarket.astype('str').str[:4]  # 转为字符串，提取前4位的年份
yearnum = df.groupby(year).size()    # 按年份统计，得到每年股票发行量
yearnum

plt.rcParams['font.sans-serif'] = ['SimHei'] # 指定中文黑体字体
# 下面的False修正坐标轴上负号'-'显示为方块的问题
plt.rcParams['axes.unicode_minus'] = False 
# 数据集中有几支股票没有发行年份（年份为0）, 作图时排除0年份
yearnum[yearnum.index!='0'].plot(fontsize=14, title='年IPO数量')

df.pe.mean()            # 简单的算术平均pe

df[df.pe > 0].pe.mean()     # 剔除亏损股票后计算pe均值

df['tvalue'] = 4 * df.esp * df.pe * df.totals   # 计算总市值，增加新列tvalue
np.sum(df.pe * df.tvalue) / df.tvalue.sum()   # 计算以市值为权重的加权pe

df['board'] = df.index.str[:2]     # 取code的前2个字符，新增board列
# 按板块类型统计pe均值，计数
df.groupby('board').pe.agg([('pe均值', 'mean'), ('股票数', 'count')])

import tushare as ts
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import ticker
from matplotlib.pylab import date2num
#from mpl_finance import candlestick_ochl #需要另外安装mpl_finance包
from mplfinance.original_flavor import candlestick_ochl #需要另外安装mpl_finance包
plt.rcParams['font.sans-serif'] = ['SimHei']   # 用来正常显示中文标签

#具有足够权限的用户是用pro接口获取数据
pro = ts.pro_api()
code = '600004.SH'
df = pro.daily(ts_code=code, start_date='20191201')
df.shape
#stock_daily = pro.daily(ts_code=code, start_date='20181201')
#stock_daily.to_excel('stock_daily.xlsx')   # 保存为电子表格

#不具有足够权限使用pro接口获取数据的用户执行下面代码，直接从xlxs文件中获取数据
df = pd.read_excel('stock_daily.xlsx', dtype={'code': 'str','trade_date': 'str'}) 
df.drop(df.columns[0], axis=1,  inplace=True)
df.shape

df2 = df.query('trade_date >= "20171001"').reset_index() #选取2017年10月1日后的数据
df2 = df2.sort_values(by='trade_date', ascending=True)  # 原始数据按照日期降序排列
df2['dates'] = np.arange(0, len(df2)) # len(df2)指记录数
fig, ax = plt.subplots(figsize=(20, 9))
fig.subplots_adjust(bottom=0.2) # 控制子图
###candlestick_ochl()函数的参数
# ax 绘图Axes的实例
# quotes  序列（时间，开盘价，收盘价，最高价，最低价） 时间是float类型，date必须转换为float
# width    图像中红绿矩形的宽度,代表天数
# colorup  收盘价格大于开盘价格时的颜色
# colordown   低于开盘价格时矩形的颜色
# alpha      矩形的颜色的透明度
candlestick_ochl(ax, quotes=df2[['dates', 'open', 'close', 'high', 'low']].values,
                 width=0.55, colorup='r', colordown='g', alpha=0.95)
date_tickers = df2['trade_date'].values  
def format_date(x, pos):
    if (x < 0) or (x > len(date_tickers)-1):
        return ''
    return date_tickers[int(x)]
ax.xaxis.set_major_formatter(ticker.FuncFormatter(format_date)) #按一定规则选取并在水平轴上显示时间刻度
plt.xticks(rotation=30)  # 设置日期刻度旋转的角度
ax.set_ylabel('交易价格')
plt.title(code)
plt.grid(True)  # 添加网格，可有可无，只是让图像好看一些
plt.xlabel('交易日期')
plt.show()