nakajo2011 / tvvbert

README

• 「Explainability of Transformer for Large Over-the-Top Media Viewing Logs」に用いたBERTのpre-trainingとfine-tuningの実装
• Attention Flow/Rolloutの計算とvisualizationの実装

How do I get set up?(requirement)

• pytorch
• huggingface/transformers
• huggingface/datasets

see 'requirements.txt' for more dependencies.

How to use

pre-training

python3 bert_pretraining.py pretraining_config.json

see 'bert_pretraining.py' for more details.

fine-tuning

python3 bert_cls.py cls_config.json

see 'bert_cls.py' for more details.

Visualizing

参考にしたリポジトリ： https://github.com/samiraabnar/attention_flow

How to use

python3 attention_visualization_v4.py

Sample datas for visualization

viewed.txt

{
  "L1913": "16278672008 16279503008 16283187001 16283190001 16283193001 16283196001 16283199001 16283202001 16283205001 16283208001 16283211001 16283214001 16283217001 16283220001 16283223001 16283226001 16283229001 16283232001 16283235001 16283238001 16283241001 16283244001 16283247001 16283250001 16283253001 16283256001 16283259001 16283262001 16283265001",
  "L7467": "16279062006 16279065006 16279068006 16279071006 16280796001 16280799001 16280802001 16280805001 16280808001 16280811001 16280814001 16280817001 16280820001 16280823001 16280826001 16280829001 16280832001 16280835001 16280838001 16280862001 16280865001 16280871001 16280874001",
  "L8906": "16280802007 16281669004 16281672004 16282548005 16282887006 16282890006 16282893004 16282896004 16282899004 16282902004 16282905004 16282908004 16282911004 16283046006 16283163001 16283166001 16283169001 16283172001 16283175001 16283178001 16283181001 16283184001 16283187001 16283190001 16283193001",
  "L11106": "16283184001 16283187001 16283190001 16283193001 16283196001 16283199001 16283202001 16283205001 16283208001 16283211001 16283214001 16283217001 16283220001 16283223001 16283226001 16283229001 16283232001 16283235001 16283238001 16283241001 16283244001 16283247001 16283250001 16283253001 16283256001 16283259001 16283262001",
  "L19724": "16280832001 16280835001 16280838001 16281489004 16281492004 16283046001 16283055001 16283058001 16283061001 16283064001 16283067001 16283091004 16283094004 16283097004 16283103001 16283106001 16283109001 16283112001 16283115001 16283124001 16283127001"
}

not_viewed.txt

{
  "L16": "16279890004 16279893004 16279896004 16279899004 16279902004 16279905004 16279908004 16279911004 16279914004 16279917004 16279920004 16279923004 16279926004 16279929004 16279932004 16279935004 16279938004 16279941004 16279944004 16279947004 16279950004 16279953004 16279956004 16279959004 16279962004 16279965004 16279968004 16279971004",
  "L17": "16277706004 16277709004 16277712004 16277757004 16277760004 16277763004 16277772004 16279362004 16279365004 16279368004 16279380004 16279383004 16279386004 16279389004 16279842004 16281576004 16281579004 16281582004 16281585004 16281618004 16281621004 16281624004 16281627004 16281630004 16281633004 16281636004 16281639004 16282434006",
  "L46": "16279161007 16280580007 16280583007 16280586007 16280589007 16280592007 16280595007 16280598007 16280601007 16280604007 16280607007 16280610007 16280649007 16280652007 16280655007 16280658007 16280661007 16280664007 16280685007 16280688007 16280691007 16280694007 16280697007 16280871007 16280874007 16280877007 16280880007 16280883007 16280886007",
  "L119": "16278246004 16278270004 16278273004 16278276004 16278279004 16279020008 16279023008 16279026008 16279029008 16279032008 16279035008 16279038008 16279041008 16279044008 16279047008 16279050008 16279053008 16279056008 16279059008 16279062008 16279065008 16279068008 16279071008 16279074008 16279077008 16279080008 16279083008 16281657006 16281702006",
  "L121": "16279020008 16279023008 16279026008 16279029008 16279032008 16279035008 16279038008 16279041008 16279044008 16279047008 16279050008 16279053008 16279056008 16279059008 16279062008 16279065008 16279068008 16279071008 16279074008 16279077008 16279080008 16279083008 16281657006 16281702006",
}

predict_missed

{
    "L275": "16278252004 16279932004 16279935004 16279938004 16279941004 16279944004 16279947004 16279950004 16279953004 16279956004 16279959004 16279962004 16279965004 16279968004 16279971004 16279974004 16279977004 16279980004 16280832001 16280835001 16280838001 16280850001 16280853001 16280856001 16280859001 16281168008 16281174008 16281180008 16281183008",
    "L337": "16278246004 16280760007 16280763007 16280766007 16280769007 16280772007 16280775007 16280778007 16280781007 16280784007 16280787007 16280790007 16280793007 16280796007 16280799007 16280850007 16280853007 16280856007 16280862007 16280865007 16280868007 16280871007 16280874007 16280877007 16280880007 16280883007 16280886007 16280889007 16283256006",
    "L379": "16277775006 16277841001 16277844001 16278147001 16278978004 16278981004 16279983005 16279986005 16282860004 16282863004 16282866004 16282869004 16282872004 16282875004 16282878004 16282881004 16282884004 16282887004 16282890004 16282893004 16282896004 16282899004 16283031004 16283034004 16283046008",
    "L386": "16279665004 16279668004 16279671004 16279692004 16279716004 16279719004 16279725004 16279734004 16279746004 16279749004 16279767004 16281372004 16281375004 16281378004 16281381004 16281384004 16281387004 16281390004 16281393004 16281396004 16281399004 16281402004 16281405004 16281408004 16281411004 16281414004 16281417004 16281420004 16281423004",
    "L531": "16279902004 16279905004 16279908004 16279911004 16279914004 16279917004 16279920004 16279923004 16279926004 16279929004 16279932004 16279935004 16279938004 16279941004 16279944004 16279947004 16279950004 16279953004 16279956004 16279959004 16279962004 16279965004 16279968004 16279971004 16279974004 16279977004",
}